@kecan0406/aninames 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +90 -2
- package/lib/client/chunks.d.ts +26 -0
- package/lib/client/chunks.js +121 -0
- package/lib/client/engine.d.ts +20 -0
- package/lib/client/engine.js +244 -0
- package/lib/client/format.d.ts +74 -0
- package/lib/client/format.js +212 -0
- package/lib/client/index.d.ts +3 -0
- package/lib/client/index.js +2 -0
- package/lib/client/names.d.ts +37 -0
- package/lib/client/names.js +202 -0
- package/lib/client/order.d.ts +1 -0
- package/lib/client/order.js +13 -0
- package/lib/client/source.d.ts +14 -0
- package/lib/client/source.js +180 -0
- package/lib/match.d.ts +9 -0
- package/lib/match.js +58 -0
- package/lib/model.d.ts +34 -0
- package/lib/model.js +5 -0
- package/lib/text.d.ts +24 -0
- package/lib/text.js +611 -0
- package/package.json +34 -4
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
import { LAYERS } from "../model.js";
|
|
2
|
+
import { flat } from "../text.js";
|
|
3
|
+
import { compare } from "./order.js";
|
|
4
|
+
export const FORMAT = 1;
|
|
5
|
+
export const KEY_TABLES = ['flat', 'latin', 'sound', 'part', 'initial'];
|
|
6
|
+
export const NAME_KINDS = [
|
|
7
|
+
'main',
|
|
8
|
+
'official',
|
|
9
|
+
'synonym',
|
|
10
|
+
'short',
|
|
11
|
+
'reading',
|
|
12
|
+
'redirect',
|
|
13
|
+
'section',
|
|
14
|
+
'generated',
|
|
15
|
+
];
|
|
16
|
+
export const TRUSTED = ['wikidata', 'aod', 'anidb', 'vndb'];
|
|
17
|
+
export const CHARACTER = 1;
|
|
18
|
+
export const MAIN = 2;
|
|
19
|
+
export const ADULT = 4;
|
|
20
|
+
export function present(layer) {
|
|
21
|
+
return 8 << LAYERS.indexOf(layer);
|
|
22
|
+
}
|
|
23
|
+
const HASH = /^[0-9a-f]{12}$/;
|
|
24
|
+
export const HASH_LENGTH = 12;
|
|
25
|
+
export function chunkPath(layer, table, hash) {
|
|
26
|
+
return `${layer}/${table}/${hash}.json.gz`;
|
|
27
|
+
}
|
|
28
|
+
const PATH = new RegExp(`^(${LAYERS.join('|')})/(${[...KEY_TABLES, 'entity'].join('|')})/([0-9a-f]{12})\\.json\\.gz$`);
|
|
29
|
+
export function parsePath(path) {
|
|
30
|
+
const match = PATH.exec(path);
|
|
31
|
+
return match ? { layer: match[1], table: match[2], hash: match[3] } : undefined;
|
|
32
|
+
}
|
|
33
|
+
export function shared(a, b) {
|
|
34
|
+
const length = Math.min(a.length, b.length);
|
|
35
|
+
let at = 0;
|
|
36
|
+
while (at < length && a.charCodeAt(at) === b.charCodeAt(at))
|
|
37
|
+
at++;
|
|
38
|
+
const last = a.charCodeAt(at - 1);
|
|
39
|
+
return at > 0 && last >= 0xd800 && last <= 0xdbff ? at - 1 : at;
|
|
40
|
+
}
|
|
41
|
+
export function pack(keys, hashes, eids) {
|
|
42
|
+
const share = [];
|
|
43
|
+
const rest = [];
|
|
44
|
+
let previous = '';
|
|
45
|
+
for (const key of keys) {
|
|
46
|
+
const n = shared(previous, key);
|
|
47
|
+
share.push(n);
|
|
48
|
+
rest.push(key.slice(n));
|
|
49
|
+
previous = key;
|
|
50
|
+
}
|
|
51
|
+
const out = { share, rest, files: hashes.join('') };
|
|
52
|
+
if (eids)
|
|
53
|
+
out.eids = eids.map((eid, at) => eid - (at > 0 ? (eids[at - 1] ?? 0) : 0));
|
|
54
|
+
return out;
|
|
55
|
+
}
|
|
56
|
+
const isObject = (value) => typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
57
|
+
const isCount = (value) => Number.isSafeInteger(value) && value >= 0;
|
|
58
|
+
function bad(why) {
|
|
59
|
+
return new Error(`aninames: cannot read the manifest: ${why}`);
|
|
60
|
+
}
|
|
61
|
+
function unpack(value, where, numbered) {
|
|
62
|
+
if (!isObject(value))
|
|
63
|
+
throw bad(`${where} is missing`);
|
|
64
|
+
const { share, rest, files, eids } = value;
|
|
65
|
+
if (!Array.isArray(share) || !Array.isArray(rest) || typeof files !== 'string' || share.length !== rest.length)
|
|
66
|
+
throw bad(`${where} is malformed`);
|
|
67
|
+
if (files.length !== share.length * HASH_LENGTH)
|
|
68
|
+
throw bad(`${where} names ${files.length} hash characters`);
|
|
69
|
+
if (numbered && (!Array.isArray(eids) || eids.length !== share.length))
|
|
70
|
+
throw bad(`${where} has no entity numbers`);
|
|
71
|
+
const out = { bounds: [], files: [], eids: [] };
|
|
72
|
+
let previous = '';
|
|
73
|
+
let eid = 0;
|
|
74
|
+
for (let at = 0; at < share.length; at++) {
|
|
75
|
+
const n = share[at];
|
|
76
|
+
const tail = rest[at];
|
|
77
|
+
if (!isCount(n) || n > previous.length || typeof tail !== 'string')
|
|
78
|
+
throw bad(`${where} span ${at} is malformed`);
|
|
79
|
+
const key = previous.slice(0, n) + tail;
|
|
80
|
+
if (at > 0 && compare(previous, key) > 0)
|
|
81
|
+
throw bad(`${where} is out of order at span ${at}`);
|
|
82
|
+
const hash = files.slice(at * HASH_LENGTH, (at + 1) * HASH_LENGTH);
|
|
83
|
+
if (!HASH.test(hash))
|
|
84
|
+
throw bad(`${where} span ${at} has no file name`);
|
|
85
|
+
if (numbered) {
|
|
86
|
+
const delta = eids[at];
|
|
87
|
+
if (!isCount(delta) || (at > 0 && delta === 0))
|
|
88
|
+
throw bad(`${where} span ${at} has a bad entity number`);
|
|
89
|
+
eid += delta;
|
|
90
|
+
out.eids.push(eid);
|
|
91
|
+
}
|
|
92
|
+
out.bounds.push(key);
|
|
93
|
+
out.files.push(hash);
|
|
94
|
+
previous = key;
|
|
95
|
+
}
|
|
96
|
+
return out;
|
|
97
|
+
}
|
|
98
|
+
export function readManifest(text) {
|
|
99
|
+
let value;
|
|
100
|
+
try {
|
|
101
|
+
value = JSON.parse(text);
|
|
102
|
+
}
|
|
103
|
+
catch {
|
|
104
|
+
throw bad('it is not JSON');
|
|
105
|
+
}
|
|
106
|
+
if (!isObject(value))
|
|
107
|
+
throw bad('it is not an object');
|
|
108
|
+
if (value.aninames !== FORMAT) {
|
|
109
|
+
throw new Error(`aninames: the manifest has format ${String(value.aninames)}, this library reads format ${FORMAT}; update @kecan0406/aninames`);
|
|
110
|
+
}
|
|
111
|
+
const built = typeof value.built === 'string' ? value.built : '';
|
|
112
|
+
if (!isObject(value.layers))
|
|
113
|
+
throw bad('it lists no layers');
|
|
114
|
+
const layers = new Map();
|
|
115
|
+
for (const layer of LAYERS) {
|
|
116
|
+
const tables = value.layers[layer];
|
|
117
|
+
if (tables === undefined)
|
|
118
|
+
continue;
|
|
119
|
+
if (!isObject(tables))
|
|
120
|
+
throw bad(`layer ${layer} is malformed`);
|
|
121
|
+
const keys = {};
|
|
122
|
+
for (const table of KEY_TABLES) {
|
|
123
|
+
const { bounds, files } = unpack(tables[table], `${layer}/${table}`, false);
|
|
124
|
+
keys[table] = { bounds, files };
|
|
125
|
+
}
|
|
126
|
+
layers.set(layer, { keys, entity: unpack(tables.entity, `${layer}/entity`, true) });
|
|
127
|
+
}
|
|
128
|
+
if (!layers.has('core'))
|
|
129
|
+
throw bad('it has no core layer');
|
|
130
|
+
return { built, layers };
|
|
131
|
+
}
|
|
132
|
+
export function manifestPaths(manifest) {
|
|
133
|
+
const paths = new Set();
|
|
134
|
+
for (const [layer, spans] of manifest.layers) {
|
|
135
|
+
for (const table of KEY_TABLES)
|
|
136
|
+
for (const hash of spans.keys[table].files)
|
|
137
|
+
paths.add(chunkPath(layer, table, hash));
|
|
138
|
+
for (const hash of spans.entity.files)
|
|
139
|
+
paths.add(chunkPath(layer, 'entity', hash));
|
|
140
|
+
}
|
|
141
|
+
return paths;
|
|
142
|
+
}
|
|
143
|
+
const isInt = (value) => Number.isSafeInteger(value);
|
|
144
|
+
function keyRow(value) {
|
|
145
|
+
if (!Array.isArray(value) || value.length < 7)
|
|
146
|
+
return undefined;
|
|
147
|
+
const [key, eid, weight, flags, rank, standing, extra] = value;
|
|
148
|
+
if (typeof key !== 'string' || !isCount(eid))
|
|
149
|
+
return undefined;
|
|
150
|
+
if (!isInt(weight) || !isInt(flags) || !isInt(rank) || !isInt(standing) || !isInt(extra))
|
|
151
|
+
return undefined;
|
|
152
|
+
return [key, eid, weight, flags, rank, standing, extra];
|
|
153
|
+
}
|
|
154
|
+
function nameOf(value) {
|
|
155
|
+
if (!Array.isArray(value) || value.length < 4)
|
|
156
|
+
return undefined;
|
|
157
|
+
const [text, lang, kind, trust, qual] = value;
|
|
158
|
+
if (typeof text !== 'string' || (lang !== null && typeof lang !== 'string'))
|
|
159
|
+
return undefined;
|
|
160
|
+
const named = isCount(kind) ? NAME_KINDS[kind] : undefined;
|
|
161
|
+
if (!named || !isCount(trust))
|
|
162
|
+
return undefined;
|
|
163
|
+
if (qual !== undefined && qual !== null && typeof qual !== 'string')
|
|
164
|
+
return undefined;
|
|
165
|
+
return { text, lang, kind: named, trust, qual: qual ?? null, flat: flat(text) };
|
|
166
|
+
}
|
|
167
|
+
function list(value, item) {
|
|
168
|
+
if (!Array.isArray(value))
|
|
169
|
+
return undefined;
|
|
170
|
+
const out = [];
|
|
171
|
+
for (const entry of value) {
|
|
172
|
+
const read = item(entry);
|
|
173
|
+
if (read === undefined)
|
|
174
|
+
return undefined;
|
|
175
|
+
out.push(read);
|
|
176
|
+
}
|
|
177
|
+
return out;
|
|
178
|
+
}
|
|
179
|
+
const tagOf = (value) => Array.isArray(value) && typeof value[0] === 'string' && (value[1] === null || typeof value[1] === 'number')
|
|
180
|
+
? [value[0], value[1]]
|
|
181
|
+
: undefined;
|
|
182
|
+
const workOf = (value) => Array.isArray(value) && isCount(value[0]) && typeof value[1] === 'string' ? [value[0], value[1]] : undefined;
|
|
183
|
+
const eidOf = (value) => (isCount(value) ? value : undefined);
|
|
184
|
+
function entityRow(value) {
|
|
185
|
+
if (!Array.isArray(value) || value.length < 10)
|
|
186
|
+
return undefined;
|
|
187
|
+
const [eid, id, rawNames, rawTags, rawWorks, rawReach, rank, standing, extra, adult] = value;
|
|
188
|
+
if (!isCount(eid) || typeof id !== 'string')
|
|
189
|
+
return undefined;
|
|
190
|
+
const kind = id.startsWith('w/') ? 'work' : id.startsWith('c/') ? 'character' : undefined;
|
|
191
|
+
if (!kind || !isInt(rank) || !isInt(standing) || !isInt(extra))
|
|
192
|
+
return undefined;
|
|
193
|
+
const names = list(rawNames, nameOf);
|
|
194
|
+
const danbooru = list(rawTags, tagOf);
|
|
195
|
+
const works = list(rawWorks, workOf);
|
|
196
|
+
const reach = list(rawReach, eidOf);
|
|
197
|
+
if (!names || !danbooru || !works || !reach)
|
|
198
|
+
return undefined;
|
|
199
|
+
return { eid, id, kind, names, danbooru, works, reach, rank, standing, extra, adult: adult === 1 };
|
|
200
|
+
}
|
|
201
|
+
export function readRows(table, value) {
|
|
202
|
+
if (!Array.isArray(value))
|
|
203
|
+
return undefined;
|
|
204
|
+
const read = table === 'entity' ? entityRow : keyRow;
|
|
205
|
+
const rows = [];
|
|
206
|
+
for (const row of value) {
|
|
207
|
+
const decoded = read(row);
|
|
208
|
+
if (decoded)
|
|
209
|
+
rows.push(decoded);
|
|
210
|
+
}
|
|
211
|
+
return rows;
|
|
212
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { type Kind, type Layer } from '../model.ts';
|
|
2
|
+
import { type Rows } from './chunks.ts';
|
|
3
|
+
import { type Source } from './source.ts';
|
|
4
|
+
export interface Found {
|
|
5
|
+
id: string;
|
|
6
|
+
kind: Kind;
|
|
7
|
+
label: string;
|
|
8
|
+
names: {
|
|
9
|
+
text: string;
|
|
10
|
+
lang: string | null;
|
|
11
|
+
}[];
|
|
12
|
+
danbooru: string[];
|
|
13
|
+
works: string[];
|
|
14
|
+
score: number;
|
|
15
|
+
adult: boolean;
|
|
16
|
+
}
|
|
17
|
+
export interface Trace {
|
|
18
|
+
rounds: number;
|
|
19
|
+
labels: number;
|
|
20
|
+
files: number;
|
|
21
|
+
bytes: number;
|
|
22
|
+
}
|
|
23
|
+
export declare function trace(names: Names): Readonly<Trace>;
|
|
24
|
+
export declare function decode(path: string, bytes: Uint8Array): Rows;
|
|
25
|
+
export declare class Names {
|
|
26
|
+
#private;
|
|
27
|
+
constructor(source: Source, options?: {
|
|
28
|
+
layers?: Layer[];
|
|
29
|
+
});
|
|
30
|
+
search(query: string, options?: {
|
|
31
|
+
kind?: Kind;
|
|
32
|
+
limit?: number;
|
|
33
|
+
adult?: boolean;
|
|
34
|
+
signal?: AbortSignal;
|
|
35
|
+
}): Promise<Found[]>;
|
|
36
|
+
lookup(ids: string[], signal?: AbortSignal): Promise<Found[]>;
|
|
37
|
+
}
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import { gunzipSync } from 'node:zlib';
|
|
2
|
+
import { LAYERS } from "../model.js";
|
|
3
|
+
import { View } from "./chunks.js";
|
|
4
|
+
import { search } from "./engine.js";
|
|
5
|
+
import { parsePath, readManifest, readRows } from "./format.js";
|
|
6
|
+
import { Missing } from "./source.js";
|
|
7
|
+
const KEEP = 256;
|
|
8
|
+
const PARALLEL = 16;
|
|
9
|
+
const TIMEOUT = 30_000;
|
|
10
|
+
const LABEL_LANGS = ['ja', 'en', 'ja-Latn', 'ko'];
|
|
11
|
+
const SHOWN = new Set(['main', 'official', 'synonym', 'short']);
|
|
12
|
+
const traces = new WeakMap();
|
|
13
|
+
export function trace(names) {
|
|
14
|
+
return traces.get(names) ?? { rounds: 0, labels: 0, files: 0, bytes: 0 };
|
|
15
|
+
}
|
|
16
|
+
function race(promise, signal) {
|
|
17
|
+
if (!signal)
|
|
18
|
+
return promise;
|
|
19
|
+
signal.throwIfAborted();
|
|
20
|
+
return new Promise((resolve, reject) => {
|
|
21
|
+
const stop = () => reject(signal.reason);
|
|
22
|
+
signal.addEventListener('abort', stop, { once: true });
|
|
23
|
+
promise.then(resolve, reject).finally(() => signal.removeEventListener('abort', stop));
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
export function decode(path, bytes) {
|
|
27
|
+
const where = parsePath(path);
|
|
28
|
+
let rows;
|
|
29
|
+
try {
|
|
30
|
+
const text = bytes[0] === 0x1f && bytes[1] === 0x8b ? gunzipSync(bytes) : bytes;
|
|
31
|
+
rows = where && readRows(where.table, JSON.parse(new TextDecoder().decode(text)));
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
rows = undefined;
|
|
35
|
+
}
|
|
36
|
+
if (!rows)
|
|
37
|
+
throw new Error(`aninames: ${path} is not a chunk file`);
|
|
38
|
+
return rows;
|
|
39
|
+
}
|
|
40
|
+
function label(id, rows) {
|
|
41
|
+
const names = rows.flatMap((row) => row.names);
|
|
42
|
+
for (const lang of LABEL_LANGS) {
|
|
43
|
+
for (const trust of [0, 1, 2, 3, undefined]) {
|
|
44
|
+
const found = names.find((n) => n.lang === lang && (n.kind === 'official' || n.kind === 'main') && (trust === undefined || n.trust === trust));
|
|
45
|
+
if (found)
|
|
46
|
+
return found.text;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return names.find((n) => n.kind === 'main')?.text ?? names[0]?.text ?? id;
|
|
50
|
+
}
|
|
51
|
+
function card(rows, score, adult) {
|
|
52
|
+
const head = rows[0];
|
|
53
|
+
const seen = new Set();
|
|
54
|
+
const names = [];
|
|
55
|
+
for (const name of rows.flatMap((row) => row.names)) {
|
|
56
|
+
const key = `${name.lang ?? ''}\u0000${name.text}`;
|
|
57
|
+
if (!SHOWN.has(name.kind) || seen.has(key))
|
|
58
|
+
continue;
|
|
59
|
+
seen.add(key);
|
|
60
|
+
names.push({ text: name.text, lang: name.lang });
|
|
61
|
+
}
|
|
62
|
+
const posts = new Map();
|
|
63
|
+
for (const [tag, count] of rows.flatMap((row) => row.danbooru)) {
|
|
64
|
+
if (!posts.has(tag) || (count ?? -1) > (posts.get(tag) ?? -1))
|
|
65
|
+
posts.set(tag, count);
|
|
66
|
+
}
|
|
67
|
+
const danbooru = [...posts]
|
|
68
|
+
.map(([tag, count], at) => ({ tag, count, at }))
|
|
69
|
+
.sort((a, b) => (b.count ?? -1) - (a.count ?? -1) || a.at - b.at)
|
|
70
|
+
.map((entry) => entry.tag);
|
|
71
|
+
const works = head.kind === 'character' ? [...new Set(rows.flatMap((row) => row.works.map(([, id]) => id)))] : [];
|
|
72
|
+
return { id: head.id, kind: head.kind, label: label(head.id, rows), names, danbooru, works, score, adult };
|
|
73
|
+
}
|
|
74
|
+
export class Names {
|
|
75
|
+
#source;
|
|
76
|
+
#layers;
|
|
77
|
+
#state;
|
|
78
|
+
#cache = new Map();
|
|
79
|
+
#loading = new Map();
|
|
80
|
+
#queue = [];
|
|
81
|
+
#active = 0;
|
|
82
|
+
#runs = 0;
|
|
83
|
+
constructor(source, options = {}) {
|
|
84
|
+
this.#source = source;
|
|
85
|
+
this.#layers = options.layers ?? LAYERS;
|
|
86
|
+
traces.set(this, { rounds: 0, labels: 0, files: 0, bytes: 0 });
|
|
87
|
+
}
|
|
88
|
+
search(query, options = {}) {
|
|
89
|
+
const { signal, ...rest } = options;
|
|
90
|
+
return this.#run(signal, (view) => {
|
|
91
|
+
const hits = search(view, query, rest);
|
|
92
|
+
if (!hits)
|
|
93
|
+
return undefined;
|
|
94
|
+
view.labels = true;
|
|
95
|
+
const out = hits.flatMap((hit) => {
|
|
96
|
+
const rows = view.entity(hit.eid, hit.mask);
|
|
97
|
+
return rows.length > 0 ? [card(rows, hit.score, hit.adult)] : [];
|
|
98
|
+
});
|
|
99
|
+
return view.wanted.size > 0 ? undefined : out;
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
lookup(ids, signal) {
|
|
103
|
+
const wanted = [...new Set(ids.filter((id) => typeof id === 'string'))];
|
|
104
|
+
return this.#run(signal, (view) => {
|
|
105
|
+
view.labels = true;
|
|
106
|
+
const out = wanted.flatMap((id) => {
|
|
107
|
+
const rows = view.byId(id);
|
|
108
|
+
return rows.length > 0 ? [card(rows, 0, rows[0].adult)] : [];
|
|
109
|
+
});
|
|
110
|
+
return view.wanted.size > 0 ? undefined : out;
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
#open() {
|
|
114
|
+
if (this.#state)
|
|
115
|
+
return this.#state;
|
|
116
|
+
const state = this.#source
|
|
117
|
+
.manifest(AbortSignal.timeout(TIMEOUT))
|
|
118
|
+
.then((text) => ({ text, manifest: readManifest(text) }));
|
|
119
|
+
this.#state = state;
|
|
120
|
+
state.catch(() => {
|
|
121
|
+
if (this.#state === state)
|
|
122
|
+
this.#state = undefined;
|
|
123
|
+
});
|
|
124
|
+
return state;
|
|
125
|
+
}
|
|
126
|
+
#view(manifest) {
|
|
127
|
+
const layers = LAYERS.filter((layer) => (layer === 'core' || this.#layers.includes(layer)) && manifest.layers.has(layer));
|
|
128
|
+
return new View(manifest, layers, (path) => {
|
|
129
|
+
const rows = this.#cache.get(path);
|
|
130
|
+
if (rows) {
|
|
131
|
+
this.#cache.delete(path);
|
|
132
|
+
this.#cache.set(path, rows);
|
|
133
|
+
}
|
|
134
|
+
return rows;
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
async #run(signal, compute) {
|
|
138
|
+
signal?.throwIfAborted();
|
|
139
|
+
const counts = traces.get(this);
|
|
140
|
+
this.#runs++;
|
|
141
|
+
try {
|
|
142
|
+
for (let retry = true;; retry = false) {
|
|
143
|
+
const state = await race(this.#open(), signal);
|
|
144
|
+
try {
|
|
145
|
+
for (;;) {
|
|
146
|
+
const view = this.#view(state.manifest);
|
|
147
|
+
const value = compute(view);
|
|
148
|
+
if (view.wanted.size === 0)
|
|
149
|
+
return value;
|
|
150
|
+
if (view.labels)
|
|
151
|
+
counts.labels++;
|
|
152
|
+
else
|
|
153
|
+
counts.rounds++;
|
|
154
|
+
await race(this.#load([...view.wanted]), signal);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
catch (error) {
|
|
158
|
+
if (!retry || !(error instanceof Missing))
|
|
159
|
+
throw error;
|
|
160
|
+
const text = await race(this.#source.manifest(AbortSignal.timeout(TIMEOUT)), signal);
|
|
161
|
+
if (text === state.text)
|
|
162
|
+
throw error;
|
|
163
|
+
this.#state = Promise.resolve({ text, manifest: readManifest(text) });
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
finally {
|
|
168
|
+
this.#runs--;
|
|
169
|
+
if (this.#runs === 0)
|
|
170
|
+
for (const path of [...this.#cache.keys()].slice(0, -KEEP))
|
|
171
|
+
this.#cache.delete(path);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
#load(paths) {
|
|
175
|
+
return Promise.all(paths.map((path) => {
|
|
176
|
+
if (this.#cache.has(path))
|
|
177
|
+
return undefined;
|
|
178
|
+
let loading = this.#loading.get(path);
|
|
179
|
+
if (!loading) {
|
|
180
|
+
loading = this.#fetch(path).finally(() => this.#loading.delete(path));
|
|
181
|
+
this.#loading.set(path, loading);
|
|
182
|
+
}
|
|
183
|
+
return loading;
|
|
184
|
+
}));
|
|
185
|
+
}
|
|
186
|
+
async #fetch(path) {
|
|
187
|
+
while (this.#active >= PARALLEL)
|
|
188
|
+
await new Promise((go) => this.#queue.push(go));
|
|
189
|
+
this.#active++;
|
|
190
|
+
try {
|
|
191
|
+
const bytes = await this.#source.file(path, AbortSignal.timeout(TIMEOUT));
|
|
192
|
+
this.#cache.set(path, decode(path, bytes));
|
|
193
|
+
const counts = traces.get(this);
|
|
194
|
+
counts.files++;
|
|
195
|
+
counts.bytes += bytes.byteLength;
|
|
196
|
+
}
|
|
197
|
+
finally {
|
|
198
|
+
this.#active--;
|
|
199
|
+
this.#queue.shift()?.();
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function compare(a: string, b: string): number;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
function unit(code) {
|
|
2
|
+
return code >= 0xe000 ? code - 0x800 : code >= 0xd800 ? code + 0x2000 : code;
|
|
3
|
+
}
|
|
4
|
+
export function compare(a, b) {
|
|
5
|
+
const length = Math.min(a.length, b.length);
|
|
6
|
+
for (let i = 0; i < length; i++) {
|
|
7
|
+
const x = a.charCodeAt(i);
|
|
8
|
+
const y = b.charCodeAt(i);
|
|
9
|
+
if (x !== y)
|
|
10
|
+
return unit(x) - unit(y);
|
|
11
|
+
}
|
|
12
|
+
return a.length - b.length;
|
|
13
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export interface Source {
|
|
2
|
+
manifest(signal?: AbortSignal): Promise<string>;
|
|
3
|
+
file(path: string, signal?: AbortSignal): Promise<Uint8Array>;
|
|
4
|
+
}
|
|
5
|
+
export declare class Missing extends Error {
|
|
6
|
+
}
|
|
7
|
+
export declare function httpSource(base: string, options?: {
|
|
8
|
+
fetch?: typeof fetch;
|
|
9
|
+
headers?: Record<string, string>;
|
|
10
|
+
}): Source;
|
|
11
|
+
export declare function fileSource(dir: string): Source;
|
|
12
|
+
export declare function cachedSource(source: Source, dir: string, options?: {
|
|
13
|
+
maxAge?: number;
|
|
14
|
+
}): Source;
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
2
|
+
import { mkdir, readdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { dirname, join } from 'node:path';
|
|
4
|
+
import { USER_AGENT } from "../model.js";
|
|
5
|
+
import { KEY_TABLES, manifestPaths, parsePath, readManifest } from "./format.js";
|
|
6
|
+
export class Missing extends Error {
|
|
7
|
+
}
|
|
8
|
+
const revalidators = new WeakMap();
|
|
9
|
+
function checked(path) {
|
|
10
|
+
if (!parsePath(path))
|
|
11
|
+
throw new Error(`aninames: ${JSON.stringify(path)} is not a chunk file name`);
|
|
12
|
+
return path;
|
|
13
|
+
}
|
|
14
|
+
export function httpSource(base, options = {}) {
|
|
15
|
+
const root = base.endsWith('/') ? base : `${base}/`;
|
|
16
|
+
const request = async (path, signal, extra = {}) => {
|
|
17
|
+
const url = new URL(path, root);
|
|
18
|
+
const response = await (options.fetch ?? fetch)(url, {
|
|
19
|
+
headers: { 'user-agent': USER_AGENT, ...options.headers, ...extra },
|
|
20
|
+
...(signal ? { signal } : {}),
|
|
21
|
+
});
|
|
22
|
+
if (response.status === 404)
|
|
23
|
+
throw new Missing(`aninames: ${url} is not there (404)`);
|
|
24
|
+
if (!response.ok && response.status !== 304)
|
|
25
|
+
throw new Error(`aninames: ${url} answered ${response.status}`);
|
|
26
|
+
return response;
|
|
27
|
+
};
|
|
28
|
+
const source = {
|
|
29
|
+
manifest: async (signal) => (await request('manifest.json', signal)).text(),
|
|
30
|
+
file: async (path, signal) => new Uint8Array(await (await request(checked(path), signal)).arrayBuffer()),
|
|
31
|
+
};
|
|
32
|
+
revalidators.set(source, async ({ etag, modified }, signal) => {
|
|
33
|
+
const response = await request('manifest.json', signal, {
|
|
34
|
+
...(etag ? { 'if-none-match': etag } : {}),
|
|
35
|
+
...(modified ? { 'if-modified-since': modified } : {}),
|
|
36
|
+
});
|
|
37
|
+
if (response.status === 304)
|
|
38
|
+
return undefined;
|
|
39
|
+
return {
|
|
40
|
+
text: await response.text(),
|
|
41
|
+
etag: response.headers.get('etag'),
|
|
42
|
+
modified: response.headers.get('last-modified'),
|
|
43
|
+
};
|
|
44
|
+
});
|
|
45
|
+
return source;
|
|
46
|
+
}
|
|
47
|
+
const missing = (error) => error?.code === 'ENOENT';
|
|
48
|
+
export function fileSource(dir) {
|
|
49
|
+
return {
|
|
50
|
+
async manifest(signal) {
|
|
51
|
+
try {
|
|
52
|
+
return await readFile(join(dir, 'manifest.json'), { encoding: 'utf8', ...(signal ? { signal } : {}) });
|
|
53
|
+
}
|
|
54
|
+
catch (error) {
|
|
55
|
+
if (missing(error))
|
|
56
|
+
throw new Error(`aninames: ${join(dir, 'manifest.json')} is missing; run mise run chunk`);
|
|
57
|
+
throw error;
|
|
58
|
+
}
|
|
59
|
+
},
|
|
60
|
+
async file(path, signal) {
|
|
61
|
+
try {
|
|
62
|
+
return new Uint8Array(await readFile(join(dir, checked(path)), signal ? { signal } : {}));
|
|
63
|
+
}
|
|
64
|
+
catch (error) {
|
|
65
|
+
if (missing(error))
|
|
66
|
+
throw new Missing(`aninames: ${join(dir, path)} is missing`);
|
|
67
|
+
throw error;
|
|
68
|
+
}
|
|
69
|
+
},
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
function cachedOf(value) {
|
|
73
|
+
if (typeof value !== 'object' || value === null)
|
|
74
|
+
return undefined;
|
|
75
|
+
const { text, etag, modified, checked } = value;
|
|
76
|
+
if (typeof text !== 'string' || typeof checked !== 'number')
|
|
77
|
+
return undefined;
|
|
78
|
+
return {
|
|
79
|
+
text,
|
|
80
|
+
checked,
|
|
81
|
+
etag: typeof etag === 'string' ? etag : null,
|
|
82
|
+
modified: typeof modified === 'string' ? modified : null,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
async function put(path, data) {
|
|
86
|
+
const part = `${path}.${randomUUID()}.part`;
|
|
87
|
+
try {
|
|
88
|
+
await mkdir(dirname(path), { recursive: true });
|
|
89
|
+
await writeFile(part, data);
|
|
90
|
+
await rename(part, path);
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
await rm(part, { force: true });
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
async function prune(dir, text) {
|
|
97
|
+
const named = manifestPaths(readManifest(text));
|
|
98
|
+
for (const layer of await readdir(dir).catch(() => [])) {
|
|
99
|
+
for (const table of [...KEY_TABLES, 'entity']) {
|
|
100
|
+
const folder = join(dir, layer, table);
|
|
101
|
+
for (const file of await readdir(folder).catch(() => [])) {
|
|
102
|
+
const path = `${layer}/${table}/${file}`;
|
|
103
|
+
if (parsePath(path) && !named.has(path))
|
|
104
|
+
await rm(join(folder, file), { force: true });
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
export function cachedSource(source, dir, options = {}) {
|
|
110
|
+
const maxAge = options.maxAge ?? 86_400_000;
|
|
111
|
+
const state = join(dir, 'manifest.cache.json');
|
|
112
|
+
let stale = false;
|
|
113
|
+
const read = async () => {
|
|
114
|
+
try {
|
|
115
|
+
return cachedOf(JSON.parse(await readFile(state, 'utf8')));
|
|
116
|
+
}
|
|
117
|
+
catch {
|
|
118
|
+
return undefined;
|
|
119
|
+
}
|
|
120
|
+
};
|
|
121
|
+
const fetchFresh = async (cached, signal) => {
|
|
122
|
+
const revalidate = revalidators.get(source);
|
|
123
|
+
if (revalidate)
|
|
124
|
+
return revalidate(cached ?? { etag: null, modified: null }, signal);
|
|
125
|
+
return { text: await source.manifest(signal), etag: null, modified: null };
|
|
126
|
+
};
|
|
127
|
+
return {
|
|
128
|
+
async manifest(signal) {
|
|
129
|
+
const cached = await read();
|
|
130
|
+
if (cached && !stale && Date.now() - cached.checked < maxAge)
|
|
131
|
+
return cached.text;
|
|
132
|
+
let fresh;
|
|
133
|
+
try {
|
|
134
|
+
fresh = await fetchFresh(cached, signal);
|
|
135
|
+
}
|
|
136
|
+
catch (error) {
|
|
137
|
+
if (cached && !signal?.aborted)
|
|
138
|
+
return cached.text;
|
|
139
|
+
throw error;
|
|
140
|
+
}
|
|
141
|
+
stale = false;
|
|
142
|
+
if (!fresh) {
|
|
143
|
+
if (!cached)
|
|
144
|
+
throw new Error('aninames: the server answered 304 to a manifest request without validators');
|
|
145
|
+
await put(state, JSON.stringify({ ...cached, checked: Date.now() }));
|
|
146
|
+
return cached.text;
|
|
147
|
+
}
|
|
148
|
+
readManifest(fresh.text);
|
|
149
|
+
await put(state, JSON.stringify({ ...fresh, checked: Date.now() }));
|
|
150
|
+
if (fresh.text !== cached?.text)
|
|
151
|
+
await prune(dir, fresh.text);
|
|
152
|
+
return fresh.text;
|
|
153
|
+
},
|
|
154
|
+
async file(path, signal) {
|
|
155
|
+
const target = join(dir, checked(path));
|
|
156
|
+
try {
|
|
157
|
+
return new Uint8Array(await readFile(target));
|
|
158
|
+
}
|
|
159
|
+
catch (error) {
|
|
160
|
+
if (!missing(error))
|
|
161
|
+
throw error;
|
|
162
|
+
}
|
|
163
|
+
let bytes;
|
|
164
|
+
try {
|
|
165
|
+
bytes = await source.file(path, signal);
|
|
166
|
+
}
|
|
167
|
+
catch (error) {
|
|
168
|
+
if (error instanceof Missing)
|
|
169
|
+
stale = true;
|
|
170
|
+
throw error;
|
|
171
|
+
}
|
|
172
|
+
const hash = createHash('sha256').update(bytes).digest('hex');
|
|
173
|
+
if (!hash.startsWith(parsePath(path)?.hash ?? '-')) {
|
|
174
|
+
throw new Error(`aninames: ${path} arrived with other contents (sha256 ${hash.slice(0, 12)})`);
|
|
175
|
+
}
|
|
176
|
+
await put(target, bytes);
|
|
177
|
+
return bytes;
|
|
178
|
+
},
|
|
179
|
+
};
|
|
180
|
+
}
|
package/lib/match.d.ts
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { NameKind } from './model.ts';
|
|
2
|
+
export declare const TOP = "\uDBFF\uDFFF";
|
|
3
|
+
export declare function weightOf(row: {
|
|
4
|
+
kind: NameKind;
|
|
5
|
+
variant: number;
|
|
6
|
+
support: number;
|
|
7
|
+
}): number;
|
|
8
|
+
export declare function typing(q: string): [string, string][];
|
|
9
|
+
export declare function swapLayout(query: string): string | undefined;
|