@aphrody/shenron-dbxv2 0.0.0-stage → 1.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +20 -2
- package/package.json +77 -4
- package/src/cache.ts +84 -0
- package/src/client.ts +623 -0
- package/src/db.ts +67 -0
- package/src/declarations.ts +135 -0
- package/src/env.ts +111 -0
- package/src/errors.ts +141 -0
- package/src/guard.ts +115 -0
- package/src/index.ts +15 -0
- package/src/pixel-perfect.ts +139 -0
- package/src/plugin.ts +204 -0
- package/src/preload.ts +13 -0
- package/src/proxy.ts +243 -0
- package/src/refs.ts +126 -0
- package/src/runtime.ts +130 -0
- package/src/schemas.ts +60 -0
- package/src/texture.ts +83 -0
package/src/client.ts
ADDED
|
@@ -0,0 +1,623 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Typed client of the private on-demand DBXV2 server (`scripts/ops/dbxv2-serve.ts`).
|
|
3
|
+
*
|
|
4
|
+
* - Bearer token in the `Authorization` header only (never in a URL).
|
|
5
|
+
* - zod validation of every JSON response, typed errors, AbortSignal cancellation.
|
|
6
|
+
* - In-memory LRU with ETag revalidation, de-duplication of identical concurrent requests,
|
|
7
|
+
* bounded retries, ranged reads and a resumable chunked download.
|
|
8
|
+
*
|
|
9
|
+
* Returned byte arrays may be shared with the cache and between concurrent callers: treat them
|
|
10
|
+
* as read-only.
|
|
11
|
+
*/
|
|
12
|
+
import { LruCache, type LruOptions } from "./cache";
|
|
13
|
+
import { DBXV2_MIN_TOKEN_LENGTH, normalizeServerUrl } from "./env";
|
|
14
|
+
import {
|
|
15
|
+
Dbxv2AbortError,
|
|
16
|
+
Dbxv2ChangedError,
|
|
17
|
+
Dbxv2ConfigError,
|
|
18
|
+
Dbxv2Error,
|
|
19
|
+
Dbxv2NetworkError,
|
|
20
|
+
Dbxv2SchemaError,
|
|
21
|
+
Dbxv2TimeoutError,
|
|
22
|
+
errorForStatus,
|
|
23
|
+
} from "./errors";
|
|
24
|
+
import type { Dbxv2Partial } from "./errors";
|
|
25
|
+
import {
|
|
26
|
+
parseRef,
|
|
27
|
+
resourcePath,
|
|
28
|
+
resourceQuery,
|
|
29
|
+
type Dbxv2RefInput,
|
|
30
|
+
type Dbxv2ResourceKind,
|
|
31
|
+
type Dbxv2ResourceParams,
|
|
32
|
+
} from "./refs";
|
|
33
|
+
import {
|
|
34
|
+
Dbxv2CpkListSchema,
|
|
35
|
+
Dbxv2ErrorBodySchema,
|
|
36
|
+
Dbxv2FindResponseSchema,
|
|
37
|
+
Dbxv2HealthSchema,
|
|
38
|
+
Dbxv2ModelJsonSchema,
|
|
39
|
+
type Dbxv2CpkInfo,
|
|
40
|
+
type Dbxv2FindResponse,
|
|
41
|
+
type Dbxv2Health,
|
|
42
|
+
type Dbxv2ModelJson,
|
|
43
|
+
} from "./schemas";
|
|
44
|
+
import { describeTexture, type Dbxv2TextureDeclaration, type Dbxv2TextureInfo } from "./texture";
|
|
45
|
+
import type { z } from "zod";
|
|
46
|
+
|
|
47
|
+
export type Dbxv2Range =
|
|
48
|
+
| { readonly start: number; readonly end?: number }
|
|
49
|
+
| { readonly suffix: number };
|
|
50
|
+
|
|
51
|
+
export interface Dbxv2ContentRange {
|
|
52
|
+
readonly start: number;
|
|
53
|
+
readonly end: number;
|
|
54
|
+
readonly size: number;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export interface Dbxv2Binary {
|
|
58
|
+
readonly bytes: Uint8Array;
|
|
59
|
+
readonly status: 200 | 206;
|
|
60
|
+
readonly contentType: string;
|
|
61
|
+
readonly etag: string | null;
|
|
62
|
+
/** Present on a 206 answer. */
|
|
63
|
+
readonly contentRange: Dbxv2ContentRange | null;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export interface Dbxv2ClientOptions {
|
|
67
|
+
readonly baseUrl: string;
|
|
68
|
+
readonly token?: string;
|
|
69
|
+
/** Public gateways never receive a bearer token or browser credentials. */
|
|
70
|
+
readonly mode?: "private" | "public";
|
|
71
|
+
/** Replaces the global `fetch` (tests, custom transports). */
|
|
72
|
+
readonly fetch?: typeof fetch;
|
|
73
|
+
/** LRU settings, or `false` to disable the cache. */
|
|
74
|
+
readonly cache?: LruOptions | false;
|
|
75
|
+
/** Per-attempt timeout in milliseconds (default 60 000). */
|
|
76
|
+
readonly timeoutMs?: number;
|
|
77
|
+
/** Extra attempts after a network error, 502, 503 or 504 (default 2). */
|
|
78
|
+
readonly retries?: number;
|
|
79
|
+
/** Delay before the first retry, doubled each time (default 200). */
|
|
80
|
+
readonly retryDelayMs?: number;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export interface RequestOptions {
|
|
84
|
+
readonly signal?: AbortSignal;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export interface FindOptions extends RequestOptions {
|
|
88
|
+
/** `ext:emb`, a glob or a path fragment, as understood by the server. */
|
|
89
|
+
readonly q?: string;
|
|
90
|
+
readonly limit?: number;
|
|
91
|
+
readonly offset?: number;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export interface RawOptions extends RequestOptions {
|
|
95
|
+
readonly range?: Dbxv2Range;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export interface DownloadOptions extends RequestOptions {
|
|
99
|
+
/** Bytes requested per ranged read (default 4 MiB). */
|
|
100
|
+
readonly chunkSize?: number;
|
|
101
|
+
readonly onProgress?: (received: number, total: number) => void;
|
|
102
|
+
/** `error.partial` of a failed earlier download of the same entry. */
|
|
103
|
+
readonly resume?: Dbxv2Partial;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export interface Dbxv2Download {
|
|
107
|
+
readonly bytes: Uint8Array;
|
|
108
|
+
readonly etag: string | null;
|
|
109
|
+
readonly size: number;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
interface Fetched {
|
|
113
|
+
readonly status: number;
|
|
114
|
+
readonly contentType: string;
|
|
115
|
+
readonly etag: string | null;
|
|
116
|
+
readonly contentRange: Dbxv2ContentRange | null;
|
|
117
|
+
readonly bytes: Uint8Array;
|
|
118
|
+
readonly notModified: boolean;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
interface Shared {
|
|
122
|
+
promise: Promise<Fetched>;
|
|
123
|
+
controller: AbortController;
|
|
124
|
+
refs: number;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
const DEFAULT_CHUNK = 4 * 1024 * 1024;
|
|
128
|
+
const RETRYABLE = new Set([502, 503, 504]);
|
|
129
|
+
|
|
130
|
+
function formatRange(range: Dbxv2Range): string {
|
|
131
|
+
if ("suffix" in range) {
|
|
132
|
+
if (!Number.isInteger(range.suffix) || range.suffix <= 0)
|
|
133
|
+
throw new Dbxv2ConfigError("range suffix must be a positive integer");
|
|
134
|
+
return `bytes=-${range.suffix}`;
|
|
135
|
+
}
|
|
136
|
+
const { start, end } = range;
|
|
137
|
+
if (!Number.isInteger(start) || start < 0)
|
|
138
|
+
throw new Dbxv2ConfigError("range start must be a non-negative integer");
|
|
139
|
+
if (end !== undefined && (!Number.isInteger(end) || end < start))
|
|
140
|
+
throw new Dbxv2ConfigError("range end must be an integer >= start");
|
|
141
|
+
return end === undefined ? `bytes=${start}-` : `bytes=${start}-${end}`;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function parseContentRange(header: string | null): Dbxv2ContentRange | null {
|
|
145
|
+
const m = header ? /^bytes (\d+)-(\d+)\/(\d+)$/.exec(header.trim()) : null;
|
|
146
|
+
return m ? { start: Number(m[1]), end: Number(m[2]), size: Number(m[3]) } : null;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
const sleep = (ms: number, signal: AbortSignal): Promise<void> =>
|
|
150
|
+
new Promise((resolve, reject) => {
|
|
151
|
+
if (signal.aborted) return reject(new Dbxv2AbortError());
|
|
152
|
+
const timer = setTimeout(() => {
|
|
153
|
+
signal.removeEventListener("abort", onAbort);
|
|
154
|
+
resolve();
|
|
155
|
+
}, ms);
|
|
156
|
+
const onAbort = () => {
|
|
157
|
+
clearTimeout(timer);
|
|
158
|
+
reject(new Dbxv2AbortError());
|
|
159
|
+
};
|
|
160
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
export class Dbxv2Client {
|
|
164
|
+
readonly baseUrl: string;
|
|
165
|
+
readonly #token: string;
|
|
166
|
+
readonly #fetch: typeof fetch;
|
|
167
|
+
readonly #cache: LruCache<Fetched> | null;
|
|
168
|
+
readonly #timeoutMs: number;
|
|
169
|
+
readonly #retries: number;
|
|
170
|
+
readonly #retryDelayMs: number;
|
|
171
|
+
readonly #inflight = new Map<string, Shared>();
|
|
172
|
+
|
|
173
|
+
constructor(options: Dbxv2ClientOptions) {
|
|
174
|
+
const baseUrl = normalizeServerUrl(options.baseUrl);
|
|
175
|
+
if (!baseUrl)
|
|
176
|
+
throw new Dbxv2ConfigError(
|
|
177
|
+
"baseUrl must be an http(s) URL without credentials or wildcard host",
|
|
178
|
+
);
|
|
179
|
+
if (
|
|
180
|
+
options.mode === "public" &&
|
|
181
|
+
(options.token !== undefined || !baseUrl.startsWith("https://"))
|
|
182
|
+
)
|
|
183
|
+
throw new Dbxv2ConfigError("public mode requires HTTPS and forbids a token");
|
|
184
|
+
if (options.mode !== "public" && (options.token ?? "").length < DBXV2_MIN_TOKEN_LENGTH)
|
|
185
|
+
throw new Dbxv2ConfigError(`token must hold at least ${DBXV2_MIN_TOKEN_LENGTH} characters`);
|
|
186
|
+
this.baseUrl = baseUrl;
|
|
187
|
+
this.#token = options.mode === "public" ? "" : options.token!;
|
|
188
|
+
this.#fetch = options.fetch ?? fetch;
|
|
189
|
+
this.#cache = options.cache === false ? null : new LruCache<Fetched>(options.cache);
|
|
190
|
+
this.#timeoutMs = options.timeoutMs ?? 60_000;
|
|
191
|
+
this.#retries = Math.max(0, options.retries ?? 2);
|
|
192
|
+
this.#retryDelayMs = options.retryDelayMs ?? 200;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/** Never prints the token. */
|
|
196
|
+
toJSON(): { baseUrl: string } {
|
|
197
|
+
return { baseUrl: this.baseUrl };
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
clearCache(): void {
|
|
201
|
+
this.#cache?.clear();
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
get cacheStats(): { entries: number; bytes: number } {
|
|
205
|
+
return { entries: this.#cache?.size ?? 0, bytes: this.#cache?.bytes ?? 0 };
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// -- JSON endpoints -------------------------------------------------------------------------
|
|
209
|
+
|
|
210
|
+
async health(options: RequestOptions = {}): Promise<Dbxv2Health> {
|
|
211
|
+
const res = await this.#get("/health", {}, options.signal, { auth: false, cache: false });
|
|
212
|
+
return this.#json(res, Dbxv2HealthSchema, "/health");
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
async cpks(options: RequestOptions = {}): Promise<Dbxv2CpkInfo[]> {
|
|
216
|
+
const res = await this.#get("/cpk", {}, options.signal);
|
|
217
|
+
return this.#json(res, Dbxv2CpkListSchema, "/cpk");
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
async find(cpk: string, options: FindOptions = {}): Promise<Dbxv2FindResponse> {
|
|
221
|
+
const { cpk: name } = parseRef({ cpk, path: "x" });
|
|
222
|
+
const query = new URLSearchParams();
|
|
223
|
+
if (options.q !== undefined) query.set("q", options.q);
|
|
224
|
+
if (options.limit !== undefined) query.set("limit", String(options.limit));
|
|
225
|
+
if (options.offset !== undefined) query.set("offset", String(options.offset));
|
|
226
|
+
const res = await this.#get(`/cpk/${name}/find`, { query }, options.signal);
|
|
227
|
+
return this.#json(res, Dbxv2FindResponseSchema, "find");
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/** Iterates every page of a `find` (the server caps `limit` at 10 000). */
|
|
231
|
+
async *findAll(
|
|
232
|
+
cpk: string,
|
|
233
|
+
options: Omit<FindOptions, "offset"> & { readonly pageSize?: number } = {},
|
|
234
|
+
): AsyncGenerator<Dbxv2FindResponse["entries"][number]> {
|
|
235
|
+
const limit = Math.min(Math.max(options.pageSize ?? 1000, 1), 10_000);
|
|
236
|
+
let offset = 0;
|
|
237
|
+
for (;;) {
|
|
238
|
+
const page = await this.find(cpk, { ...options, limit, offset });
|
|
239
|
+
yield* page.entries;
|
|
240
|
+
offset += page.entries.length;
|
|
241
|
+
if (page.entries.length === 0 || offset >= page.total) return;
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
// -- Binary endpoints -----------------------------------------------------------------------
|
|
246
|
+
|
|
247
|
+
/** Decompressed bytes of an entry, optionally a byte range. */
|
|
248
|
+
async raw(ref: Dbxv2RefInput, options: RawOptions = {}): Promise<Dbxv2Binary> {
|
|
249
|
+
return this.#binary("raw", ref, {}, options, null);
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/** One EMB texture as PNG. */
|
|
253
|
+
async png(
|
|
254
|
+
ref: Dbxv2RefInput,
|
|
255
|
+
options: RequestOptions & { readonly index?: number } = {},
|
|
256
|
+
): Promise<Dbxv2Binary> {
|
|
257
|
+
return this.#binary("png", ref, { index: options.index }, options, "image/png");
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* One EMB texture: lossless PNG at native dimensions plus typed metadata (dimensions from the
|
|
262
|
+
* PNG header, declared colour space and alpha mode, SHA-256 of the bytes). Undecoded formats
|
|
263
|
+
* (GNF / PS4) are refused with `Dbxv2UnsupportedFormatError`, never approximated.
|
|
264
|
+
*/
|
|
265
|
+
async texture(
|
|
266
|
+
ref: Dbxv2RefInput,
|
|
267
|
+
options: RequestOptions & { readonly index?: number } & Dbxv2TextureDeclaration = {},
|
|
268
|
+
): Promise<{ readonly binary: Dbxv2Binary; readonly info: Dbxv2TextureInfo }> {
|
|
269
|
+
const binary = await this.png(ref, options);
|
|
270
|
+
return { binary, info: await describeTexture(binary.bytes, options) };
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/** One ACB sub-sound as WAV. */
|
|
274
|
+
async wav(
|
|
275
|
+
ref: Dbxv2RefInput,
|
|
276
|
+
options: RequestOptions & { readonly cue?: number } = {},
|
|
277
|
+
): Promise<Dbxv2Binary> {
|
|
278
|
+
return this.#binary("wav", ref, { cue: options.cue }, options, "audio/wav");
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/** A USM as MP4, WebM or MPEG-2 (see `contentType`). */
|
|
282
|
+
async video(ref: Dbxv2RefInput, options: RequestOptions = {}): Promise<Dbxv2Binary> {
|
|
283
|
+
return this.#binary("video", ref, {}, options, "video/");
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/** A USM segment (`t`, `d`) transcoded on demand to web fragmented MP4 (H.264, no audio), playable in every browser. */
|
|
287
|
+
async mp4(
|
|
288
|
+
ref: Dbxv2RefInput,
|
|
289
|
+
options: RequestOptions & { readonly t?: number; readonly d?: number } = {},
|
|
290
|
+
): Promise<Dbxv2Binary> {
|
|
291
|
+
return this.#binary("mp4", ref, { t: options.t, d: options.d }, options, "video/mp4");
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/** Assembled EMD metadata, validated. */
|
|
295
|
+
async modelJson(
|
|
296
|
+
ref: Dbxv2RefInput,
|
|
297
|
+
options: RequestOptions & { readonly emb?: string; readonly esk?: string } = {},
|
|
298
|
+
): Promise<Dbxv2ModelJson> {
|
|
299
|
+
const res = await this.#binary("model.json", ref, options, options, "application/json");
|
|
300
|
+
let value: unknown;
|
|
301
|
+
try {
|
|
302
|
+
value = JSON.parse(new TextDecoder().decode(res.bytes));
|
|
303
|
+
} catch (error) {
|
|
304
|
+
throw new Dbxv2SchemaError("model JSON is not valid JSON", [String(error)]);
|
|
305
|
+
}
|
|
306
|
+
return this.#validate(value, Dbxv2ModelJsonSchema, "model");
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/** Assembled EMD as binary glTF. */
|
|
310
|
+
async modelGlb(
|
|
311
|
+
ref: Dbxv2RefInput,
|
|
312
|
+
options: RequestOptions & { readonly emb?: string; readonly esk?: string } = {},
|
|
313
|
+
): Promise<Dbxv2Binary> {
|
|
314
|
+
return this.#binary("model.glb", ref, options, options, "model/gltf-binary");
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* Whole entry through successive ranged reads. A network failure keeps what was received in
|
|
319
|
+
* `error.partial`; pass it back as `resume` to continue. The ETag is sent as `If-Range`, so an
|
|
320
|
+
* entry changed meanwhile restarts (or fails with {@link Dbxv2ChangedError} when resuming).
|
|
321
|
+
*/
|
|
322
|
+
async download(ref: Dbxv2RefInput, options: DownloadOptions = {}): Promise<Dbxv2Download> {
|
|
323
|
+
const chunkSize = Math.max(1, Math.floor(options.chunkSize ?? DEFAULT_CHUNK));
|
|
324
|
+
const parts: Uint8Array[] = options.resume ? [options.resume.bytes] : [];
|
|
325
|
+
let received = options.resume?.bytes.byteLength ?? 0;
|
|
326
|
+
let etag: string | null = options.resume?.etag ?? null;
|
|
327
|
+
let size = options.resume?.size ?? -1;
|
|
328
|
+
let fromResume = options.resume !== undefined;
|
|
329
|
+
try {
|
|
330
|
+
for (;;) {
|
|
331
|
+
if (size >= 0 && received >= size) break;
|
|
332
|
+
const range: Dbxv2Range = { start: received, end: received + chunkSize - 1 };
|
|
333
|
+
const res = await this.#binary(
|
|
334
|
+
"raw",
|
|
335
|
+
ref,
|
|
336
|
+
{},
|
|
337
|
+
{ ...options, range },
|
|
338
|
+
null,
|
|
339
|
+
etag ? { ifRange: etag, cache: false } : { cache: false },
|
|
340
|
+
);
|
|
341
|
+
if (res.status === 200) {
|
|
342
|
+
// Whole entry: the range was ignored (full read) or the entry changed.
|
|
343
|
+
if (fromResume && etag !== null && res.etag !== etag)
|
|
344
|
+
throw new Dbxv2ChangedError("entry changed since the partial download");
|
|
345
|
+
parts.length = 0;
|
|
346
|
+
parts.push(res.bytes);
|
|
347
|
+
received = res.bytes.byteLength;
|
|
348
|
+
size = received;
|
|
349
|
+
etag = res.etag;
|
|
350
|
+
options.onProgress?.(received, size);
|
|
351
|
+
break;
|
|
352
|
+
}
|
|
353
|
+
const cr = res.contentRange;
|
|
354
|
+
if (!cr || cr.start !== received)
|
|
355
|
+
throw new Dbxv2SchemaError("server returned an unexpected Content-Range");
|
|
356
|
+
if (etag !== null && res.etag !== null && res.etag !== etag)
|
|
357
|
+
throw new Dbxv2ChangedError("entry changed during the download");
|
|
358
|
+
fromResume = false;
|
|
359
|
+
etag = res.etag ?? etag;
|
|
360
|
+
size = cr.size;
|
|
361
|
+
parts.push(res.bytes);
|
|
362
|
+
received += res.bytes.byteLength;
|
|
363
|
+
options.onProgress?.(received, size);
|
|
364
|
+
if (res.bytes.byteLength === 0) break;
|
|
365
|
+
}
|
|
366
|
+
} catch (error) {
|
|
367
|
+
if (error instanceof Dbxv2Error && etag !== null && size >= 0 && received > 0) {
|
|
368
|
+
error.partial = { bytes: concat(parts, received), etag, size };
|
|
369
|
+
}
|
|
370
|
+
throw error;
|
|
371
|
+
}
|
|
372
|
+
return { bytes: concat(parts, received), etag, size: Math.max(size, 0) };
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
// -- Internals ------------------------------------------------------------------------------
|
|
376
|
+
|
|
377
|
+
async #binary(
|
|
378
|
+
kind: Dbxv2ResourceKind,
|
|
379
|
+
ref: Dbxv2RefInput,
|
|
380
|
+
params: Dbxv2ResourceParams,
|
|
381
|
+
options: RawOptions,
|
|
382
|
+
expectedType: string | null,
|
|
383
|
+
extra: { readonly ifRange?: string; readonly cache?: boolean } = {},
|
|
384
|
+
): Promise<Dbxv2Binary> {
|
|
385
|
+
parseRef(ref);
|
|
386
|
+
const res = await this.#get(
|
|
387
|
+
resourcePath(kind, ref),
|
|
388
|
+
{ query: resourceQuery(kind, params), range: options.range, ifRange: extra.ifRange },
|
|
389
|
+
options.signal,
|
|
390
|
+
{ cache: extra.cache ?? true },
|
|
391
|
+
);
|
|
392
|
+
if (expectedType !== null && !res.contentType.toLowerCase().startsWith(expectedType))
|
|
393
|
+
throw new Dbxv2SchemaError(`unexpected content type "${res.contentType}" for ${kind}`);
|
|
394
|
+
return {
|
|
395
|
+
bytes: res.bytes,
|
|
396
|
+
status: res.status === 206 ? 206 : 200,
|
|
397
|
+
contentType: res.contentType,
|
|
398
|
+
etag: res.etag,
|
|
399
|
+
contentRange: res.contentRange,
|
|
400
|
+
};
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
#validate<S extends z.ZodType>(value: unknown, schema: S, what: string): z.infer<S> {
|
|
404
|
+
const parsed = schema.safeParse(value);
|
|
405
|
+
if (!parsed.success) {
|
|
406
|
+
throw new Dbxv2SchemaError(
|
|
407
|
+
`unexpected ${what} response`,
|
|
408
|
+
parsed.error.issues.map((i) => `${i.path.join(".") || "<root>"}: ${i.message}`),
|
|
409
|
+
);
|
|
410
|
+
}
|
|
411
|
+
return parsed.data;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
#json<S extends z.ZodType>(res: Fetched, schema: S, what: string): z.infer<S> {
|
|
415
|
+
let value: unknown;
|
|
416
|
+
try {
|
|
417
|
+
value = JSON.parse(new TextDecoder().decode(res.bytes));
|
|
418
|
+
} catch (error) {
|
|
419
|
+
throw new Dbxv2SchemaError(`${what} response is not valid JSON`, [String(error)]);
|
|
420
|
+
}
|
|
421
|
+
return this.#validate(value, schema, what);
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
async #get(
|
|
425
|
+
path: string,
|
|
426
|
+
init: {
|
|
427
|
+
readonly query?: URLSearchParams;
|
|
428
|
+
readonly range?: Dbxv2Range | undefined;
|
|
429
|
+
readonly ifRange?: string | undefined;
|
|
430
|
+
},
|
|
431
|
+
signal: AbortSignal | undefined,
|
|
432
|
+
flags: { readonly auth?: boolean; readonly cache?: boolean } = {},
|
|
433
|
+
): Promise<Fetched> {
|
|
434
|
+
const search = init.query && init.query.size > 0 ? `?${init.query}` : "";
|
|
435
|
+
const url = `${this.baseUrl}${path}${search}`;
|
|
436
|
+
const range = init.range ? formatRange(init.range) : null;
|
|
437
|
+
const cacheKey = `${url}|${range ?? ""}`;
|
|
438
|
+
const useCache = flags.cache !== false && this.#cache !== null;
|
|
439
|
+
|
|
440
|
+
let stale: Fetched | undefined;
|
|
441
|
+
if (useCache) {
|
|
442
|
+
const hit = this.#cache!.get(cacheKey);
|
|
443
|
+
if (hit?.fresh) return hit.value;
|
|
444
|
+
stale = hit?.value;
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
const headers: Record<string, string> = { Accept: "*/*" };
|
|
448
|
+
if (flags.auth !== false && this.#token) headers.Authorization = `Bearer ${this.#token}`;
|
|
449
|
+
if (range) headers.Range = range;
|
|
450
|
+
if (range && init.ifRange) headers["If-Range"] = init.ifRange;
|
|
451
|
+
if (stale?.etag) headers["If-None-Match"] = stale.etag;
|
|
452
|
+
|
|
453
|
+
const sharedKey = `${cacheKey}|${init.ifRange ?? ""}|${stale?.etag ?? ""}|${flags.auth === false ? 0 : 1}`;
|
|
454
|
+
const fetched = await this.#share(
|
|
455
|
+
sharedKey,
|
|
456
|
+
(s) => this.#fetchWithRetry(url, headers, s),
|
|
457
|
+
signal,
|
|
458
|
+
);
|
|
459
|
+
if (fetched.notModified && stale) {
|
|
460
|
+
this.#cache!.touch(cacheKey);
|
|
461
|
+
return stale;
|
|
462
|
+
}
|
|
463
|
+
if (useCache && (fetched.status === 200 || fetched.status === 206)) {
|
|
464
|
+
this.#cache!.set(cacheKey, fetched, fetched.bytes.byteLength);
|
|
465
|
+
}
|
|
466
|
+
return fetched;
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
/** Identical concurrent requests share one fetch, aborted only when every caller aborts. */
|
|
470
|
+
async #share(
|
|
471
|
+
key: string,
|
|
472
|
+
run: (signal: AbortSignal) => Promise<Fetched>,
|
|
473
|
+
callerSignal: AbortSignal | undefined,
|
|
474
|
+
): Promise<Fetched> {
|
|
475
|
+
if (callerSignal?.aborted) throw new Dbxv2AbortError();
|
|
476
|
+
let shared = this.#inflight.get(key);
|
|
477
|
+
if (!shared) {
|
|
478
|
+
const controller = new AbortController();
|
|
479
|
+
const promise = run(controller.signal);
|
|
480
|
+
shared = { promise, controller, refs: 0 };
|
|
481
|
+
this.#inflight.set(key, shared);
|
|
482
|
+
const created = shared;
|
|
483
|
+
const cleanup = () => {
|
|
484
|
+
if (this.#inflight.get(key) === created) this.#inflight.delete(key);
|
|
485
|
+
};
|
|
486
|
+
promise.then(cleanup, cleanup);
|
|
487
|
+
}
|
|
488
|
+
const entry = shared;
|
|
489
|
+
entry.refs++;
|
|
490
|
+
let detach = () => {};
|
|
491
|
+
const aborted = new Promise<never>((_, reject) => {
|
|
492
|
+
if (!callerSignal) return;
|
|
493
|
+
const onAbort = () => reject(new Dbxv2AbortError());
|
|
494
|
+
callerSignal.addEventListener("abort", onAbort, { once: true });
|
|
495
|
+
detach = () => callerSignal.removeEventListener("abort", onAbort);
|
|
496
|
+
});
|
|
497
|
+
aborted.catch(() => {});
|
|
498
|
+
try {
|
|
499
|
+
return await Promise.race([entry.promise, aborted]);
|
|
500
|
+
} catch (error) {
|
|
501
|
+
if (error instanceof Dbxv2AbortError && callerSignal?.aborted) {
|
|
502
|
+
entry.refs--;
|
|
503
|
+
if (entry.refs <= 0) {
|
|
504
|
+
entry.controller.abort();
|
|
505
|
+
if (this.#inflight.get(key) === entry) this.#inflight.delete(key);
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
throw error;
|
|
509
|
+
} finally {
|
|
510
|
+
detach();
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
async #fetchWithRetry(
|
|
515
|
+
url: string,
|
|
516
|
+
headers: Record<string, string>,
|
|
517
|
+
signal: AbortSignal,
|
|
518
|
+
): Promise<Fetched> {
|
|
519
|
+
let lastError: Dbxv2Error | null = null;
|
|
520
|
+
for (let attempt = 0; attempt <= this.#retries; attempt++) {
|
|
521
|
+
if (attempt > 0) await sleep(this.#retryDelayMs * 2 ** (attempt - 1), signal);
|
|
522
|
+
try {
|
|
523
|
+
return await this.#fetchOnce(url, headers, signal);
|
|
524
|
+
} catch (error) {
|
|
525
|
+
if (signal.aborted) throw new Dbxv2AbortError();
|
|
526
|
+
if (!(error instanceof Dbxv2Error)) throw error;
|
|
527
|
+
const retryable =
|
|
528
|
+
error.kind === "network" ||
|
|
529
|
+
error.kind === "timeout" ||
|
|
530
|
+
(error.status !== null && RETRYABLE.has(error.status));
|
|
531
|
+
if (!retryable || attempt === this.#retries) throw error;
|
|
532
|
+
lastError = error;
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
throw lastError ?? new Dbxv2NetworkError("request failed");
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
async #fetchOnce(
|
|
539
|
+
url: string,
|
|
540
|
+
headers: Record<string, string>,
|
|
541
|
+
signal: AbortSignal,
|
|
542
|
+
): Promise<Fetched> {
|
|
543
|
+
const attemptController = new AbortController();
|
|
544
|
+
let timedOut = false;
|
|
545
|
+
const timer = setTimeout(() => {
|
|
546
|
+
timedOut = true;
|
|
547
|
+
attemptController.abort();
|
|
548
|
+
}, this.#timeoutMs);
|
|
549
|
+
const onAbort = () => attemptController.abort();
|
|
550
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
551
|
+
try {
|
|
552
|
+
let response: Response;
|
|
553
|
+
try {
|
|
554
|
+
response = await this.#fetch(url, {
|
|
555
|
+
method: "GET",
|
|
556
|
+
headers,
|
|
557
|
+
signal: attemptController.signal,
|
|
558
|
+
redirect: "error",
|
|
559
|
+
credentials: "omit",
|
|
560
|
+
});
|
|
561
|
+
} catch (error) {
|
|
562
|
+
if (timedOut) throw new Dbxv2TimeoutError(`request timed out after ${this.#timeoutMs} ms`);
|
|
563
|
+
if (signal.aborted) throw new Dbxv2AbortError();
|
|
564
|
+
throw new Dbxv2NetworkError("network error talking to the DBXV2 server", error);
|
|
565
|
+
}
|
|
566
|
+
if (response.status === 304) {
|
|
567
|
+
return {
|
|
568
|
+
status: 304,
|
|
569
|
+
contentType: "",
|
|
570
|
+
etag: response.headers.get("etag"),
|
|
571
|
+
contentRange: null,
|
|
572
|
+
bytes: new Uint8Array(0),
|
|
573
|
+
notModified: true,
|
|
574
|
+
};
|
|
575
|
+
}
|
|
576
|
+
let bytes: Uint8Array;
|
|
577
|
+
try {
|
|
578
|
+
bytes = new Uint8Array(await response.arrayBuffer());
|
|
579
|
+
} catch (error) {
|
|
580
|
+
if (timedOut) throw new Dbxv2TimeoutError(`request timed out after ${this.#timeoutMs} ms`);
|
|
581
|
+
if (signal.aborted) throw new Dbxv2AbortError();
|
|
582
|
+
throw new Dbxv2NetworkError("connection lost while reading the response", error);
|
|
583
|
+
}
|
|
584
|
+
if (!response.ok) throw errorForStatus(response.status, errorDetail(bytes));
|
|
585
|
+
return {
|
|
586
|
+
status: response.status,
|
|
587
|
+
contentType: response.headers.get("content-type") ?? "",
|
|
588
|
+
etag: response.headers.get("etag"),
|
|
589
|
+
contentRange: parseContentRange(response.headers.get("content-range")),
|
|
590
|
+
bytes,
|
|
591
|
+
notModified: false,
|
|
592
|
+
};
|
|
593
|
+
} finally {
|
|
594
|
+
clearTimeout(timer);
|
|
595
|
+
signal.removeEventListener("abort", onAbort);
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
function errorDetail(bytes: Uint8Array): string {
|
|
601
|
+
try {
|
|
602
|
+
const body = Dbxv2ErrorBodySchema.safeParse(JSON.parse(new TextDecoder().decode(bytes)));
|
|
603
|
+
if (body.success) return body.data.error;
|
|
604
|
+
} catch {
|
|
605
|
+
// not JSON: keep the generic detail
|
|
606
|
+
}
|
|
607
|
+
return "request failed";
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
function concat(parts: readonly Uint8Array[], total: number): Uint8Array {
|
|
611
|
+
if (parts.length === 1 && parts[0]!.byteLength === total) return parts[0]!;
|
|
612
|
+
const out = new Uint8Array(total);
|
|
613
|
+
let at = 0;
|
|
614
|
+
for (const part of parts) {
|
|
615
|
+
out.set(part, at);
|
|
616
|
+
at += part.byteLength;
|
|
617
|
+
}
|
|
618
|
+
return out;
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
export function createDbxv2Client(options: Dbxv2ClientOptions): Dbxv2Client {
|
|
622
|
+
return new Dbxv2Client(options);
|
|
623
|
+
}
|
package/src/db.ts
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Typed access to the private DBXV2 database (hybrid search, files, characters, messages).
|
|
3
|
+
* The logic lives in `@aphrody/shenron-db/dbxv2`; this module only re-exports it and adds the
|
|
4
|
+
* environment gating. The database is a local private file (`DBXV2_DB_PATH`), never shipped.
|
|
5
|
+
*/
|
|
6
|
+
import {
|
|
7
|
+
openDbxv2Db,
|
|
8
|
+
type Dbxv2Db,
|
|
9
|
+
type Dbxv2File,
|
|
10
|
+
type OpenDbxv2Options,
|
|
11
|
+
} from "@aphrody/shenron-db/dbxv2";
|
|
12
|
+
import {
|
|
13
|
+
currentEnv,
|
|
14
|
+
dbxv2DbPath,
|
|
15
|
+
isPublicContext,
|
|
16
|
+
describeDisabledReason,
|
|
17
|
+
type Dbxv2EnvInput,
|
|
18
|
+
} from "./env";
|
|
19
|
+
import { Dbxv2DisabledError } from "./errors";
|
|
20
|
+
import type { Dbxv2Ref } from "./refs";
|
|
21
|
+
|
|
22
|
+
export {
|
|
23
|
+
DBXV2_LANGS,
|
|
24
|
+
decodeCms,
|
|
25
|
+
decodeMsg,
|
|
26
|
+
openDbxv2Db,
|
|
27
|
+
type CmsEntry,
|
|
28
|
+
type Dbxv2Db,
|
|
29
|
+
type Dbxv2File,
|
|
30
|
+
type Dbxv2FileQuery,
|
|
31
|
+
type Dbxv2Lang,
|
|
32
|
+
type Dbxv2Message,
|
|
33
|
+
type Dbxv2SearchOptions,
|
|
34
|
+
type Dbxv2SearchResult,
|
|
35
|
+
type MsgEntry,
|
|
36
|
+
type OpenDbxv2Options,
|
|
37
|
+
} from "@aphrody/shenron-db/dbxv2";
|
|
38
|
+
|
|
39
|
+
/** Opens the database at `DBXV2_DB_PATH` (read-only by default); throws when disabled. */
|
|
40
|
+
export function openDbxv2DbFromEnv(
|
|
41
|
+
env: Dbxv2EnvInput = currentEnv(),
|
|
42
|
+
options: OpenDbxv2Options = {},
|
|
43
|
+
): Dbxv2Db {
|
|
44
|
+
const blocked = isPublicContext(env);
|
|
45
|
+
if (blocked) throw new Dbxv2DisabledError(describeDisabledReason(blocked));
|
|
46
|
+
const path = dbxv2DbPath(env);
|
|
47
|
+
if (!path) throw new Dbxv2DisabledError("DBXV2_DB_PATH is not set");
|
|
48
|
+
return openDbxv2Db(path, { readOnly: true, ...options });
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Reference usable with the server client and the proxy URLs. */
|
|
52
|
+
export function refOfFile(file: Pick<Dbxv2File, "cpk" | "path">): Dbxv2Ref {
|
|
53
|
+
return { cpk: file.cpk, path: file.path };
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Every file reference of the database matching `query`, paged internally. */
|
|
57
|
+
export function* iterateDbxv2Refs(
|
|
58
|
+
db: Dbxv2Db,
|
|
59
|
+
query: { readonly format?: string; readonly kind?: string; readonly cpk?: string } = {},
|
|
60
|
+
): Generator<Dbxv2Ref> {
|
|
61
|
+
const limit = 500;
|
|
62
|
+
for (let offset = 0; ; offset += limit) {
|
|
63
|
+
const page = db.files({ ...query, limit, offset });
|
|
64
|
+
for (const file of page) yield refOfFile(file);
|
|
65
|
+
if (page.length < limit) return;
|
|
66
|
+
}
|
|
67
|
+
}
|