@kecan0406/aninames 0.1.0 → 0.2.0

package/README.md

@@ -56,7 +56,7 @@ Built sizes: `core.sqlite` 610 MB, `kowiki.sqlite` 25 MB, `unlicensed.sqlite` 83
 | `<layer>/<table>/<hash>.json.gz` | about 32 KB of JSON rows sorted by key (16 KB for `entity`), named by the first 48 bits of the SHA-256 of the file |
 | `NOTICE.md`, `index.html` | each layer's sources and terms |
 
-The key tables are `flat` (folded names), `latin` (romaji keys), `sound` (sound keys across kana, Hangul and romaji), `part` (name tokens) and `initial` (Korean initials); their rows are `[key, entity number, name weight, flags, popularity, standing, bonus]`. `entity` rows, keyed by entity number (which follows the entity ids' order), carry the id, names, Danbooru tags, works and family of one entity in that layer. Layers stay in separate files; the only values a file carries from other layers are derived numbers and flags (popularity, family standing, adult, which layers know the entity).
+The key tables are `flat` (folded names), `latin` (romaji keys), `sound` (sound keys across kana, Hangul and romaji), `part` (name tokens) and `initial` (Korean initials); their rows are `[key, entity number, name weight, flags, popularity, standing, bonus]`. `tag` holds the same rows for Danbooru tags, keyed by the tag folded like a name without its qualifier and weighted by the tag's posts. `entity` rows, keyed by entity number (which follows the entity ids' order), carry the id, names, Danbooru tags, works, family and the ids other databases give one entity in that layer (TMDB's `tmdb:tv/…` and `tmdb:movie/…` among them, read from Wikidata). Layers stay in separate files; the only values a file carries from other layers are derived numbers and flags (popularity, family standing, adult, which layers know the entity).
 
 On the 10-04 build: 19,167 files, 149 MB gzip'd, a manifest of 236 KB gzip'd. `mise run compare` searches the probe sample (4,370 queries) both ways: the files find the entity first 87% of the time, SQLite 86% (FTS substring search is not in the files; typo search reads the rows around the query's position instead). A cold query needs a median of 2 rounds (p90 3), 31 files (p90 78) and 287 KB (p90 584 KB).
 
@@ -71,7 +71,7 @@ const [first] = await names.search('봇치', { limit: 10, signal })
 const works = await names.lookup(first?.works ?? [])
 ```
 
-`search` returns `Found` cards (`id`, `kind`, `label`, `names`, `danbooru` tags with the most posts first, `works`, `score`, `adult`); `search(query, { kind, limit, adult: false })` narrows it, and `lookup(ids)` returns the cards of given ids in one round. `cachedSource` keeps the files it fetched forever (their names are hashes of their contents), revalidates the manifest with its ETag or Last-Modified once a day (`maxAge`), answers from the cache when offline, refetches the manifest when the site has moved on to a new build, and deletes files a new manifest no longer names. `fileSource(dir)` reads a local `dist/site`.
+`search` returns `Found` cards (`id`, `kind`, `label`, `names` with their `lang` and `kind`, `danbooru` tags with the most posts first, `works`, `refs`, `score`, `adult`); `search(query, { kind, limit, adult: false })` narrows it, `lookup(ids)` returns the cards of given ids in one round, and `tagged(tag, { kind, adult })` the cards of the entities that carry a Danbooru tag, most posts first, whatever its case, punctuation or qualifier (`soul-eater`, `soul_eater` and `soul_eater_(series)` find the same works); `prefix: true` also finds the tags that start with it. `cachedSource` keeps the files it fetched forever (their names are hashes of their contents), revalidates the manifest with its ETag or Last-Modified once a day (`maxAge`), answers from the cache when offline, refetches the manifest when the site has moved on to a new build, and deletes files a new manifest no longer names. `fileSource(dir)` reads a local `dist/site`.
 
 ## Development
 

package/lib/client/chunks.d.ts

@@ -19,6 +19,7 @@ export declare class View {
     constructor(manifest: Manifest, layers: Layer[], loaded: Loaded);
     private rows;
     keys(table: KeyTable, range: Range): KeyRow[];
+    tagged(key: string, prefix: boolean): KeyRow[];
     private keysIn;
     around(table: KeyTable, key: string, radius: number): KeyRow[];
     entity(eid: number, mask?: number): EntityRow[];

package/lib/client/chunks.js

@@ -1,4 +1,4 @@
-import { chunkPath, present } from "./format.js";
+import { chunkPath, present, } from "./format.js";
 import { compare } from "./order.js";
 function last(count, below) {
     let lo = 0;
@@ -37,10 +37,13 @@ export class View {
         return rows;
     }
     keys(table, range) {
-        return this.layers.flatMap((layer) => this.keysIn(layer, table, range));
+        return this.layers.flatMap((layer) => this.keysIn(layer, table, this.manifest.layers.get(layer)?.keys[table], range));
     }
-    keysIn(layer, table, range) {
-        const spans = this.manifest.layers.get(layer)?.keys[table];
+    tagged(key, prefix) {
+        const range = { low: key, high: prefix ? `${key}\u{10FFFF}` : key };
+        return this.layers.flatMap((layer) => this.keysIn(layer, 'tag', this.manifest.layers.get(layer)?.tag, range));
+    }
+    keysIn(layer, table, spans, range) {
         if (!spans)
             return [];
         const { low, high, open = false, limit = Number.POSITIVE_INFINITY, spans: most = Number.POSITIVE_INFINITY } = range;

package/lib/client/format.d.ts

@@ -2,7 +2,8 @@ import { type Kind, type Layer, type NameKind } from '../model.ts';
 export declare const FORMAT = 1;
 export declare const KEY_TABLES: readonly ["flat", "latin", "sound", "part", "initial"];
 export type KeyTable = (typeof KEY_TABLES)[number];
-export type Table = KeyTable | 'entity';
+export declare const TABLES: readonly ["flat", "latin", "sound", "part", "initial", "tag", "entity"];
+export type Table = (typeof TABLES)[number];
 export declare const NAME_KINDS: readonly NameKind[];
 export declare const TRUSTED: readonly string[];
 export declare const CHARACTER = 1;
@@ -33,6 +34,7 @@ export interface EntitySpans extends Spans {
 }
 export interface LayerSpans {
     keys: Record<KeyTable, Spans>;
+    tag: Spans | undefined;
     entity: EntitySpans;
 }
 export interface Manifest {
@@ -70,5 +72,6 @@ export interface EntityRow {
     standing: number;
     extra: number;
     adult: boolean;
+    refs: string[];
 }
 export declare function readRows(table: Table, value: unknown): KeyRow[] | EntityRow[] | undefined;

package/lib/client/format.js

@@ -3,6 +3,7 @@ import { flat } from "../text.js";
 import { compare } from "./order.js";
 export const FORMAT = 1;
 export const KEY_TABLES = ['flat', 'latin', 'sound', 'part', 'initial'];
+export const TABLES = [...KEY_TABLES, 'tag', 'entity'];
 export const NAME_KINDS = [
     'main',
     'official',
@@ -25,7 +26,7 @@ export const HASH_LENGTH = 12;
 export function chunkPath(layer, table, hash) {
     return `${layer}/${table}/${hash}.json.gz`;
 }
-const PATH = new RegExp(`^(${LAYERS.join('|')})/(${[...KEY_TABLES, 'entity'].join('|')})/([0-9a-f]{12})\\.json\\.gz$`);
+const PATH = new RegExp(`^(${LAYERS.join('|')})/(${TABLES.join('|')})/([0-9a-f]{12})\\.json\\.gz$`);
 export function parsePath(path) {
     const match = PATH.exec(path);
     return match ? { layer: match[1], table: match[2], hash: match[3] } : undefined;
@@ -95,6 +96,10 @@ function unpack(value, where, numbered) {
     }
     return out;
 }
+function spansOf(value, where) {
+    const { bounds, files } = unpack(value, where, false);
+    return { bounds, files };
+}
 export function readManifest(text) {
     let value;
     try {
@@ -119,11 +124,10 @@ export function readManifest(text) {
         if (!isObject(tables))
             throw bad(`layer ${layer} is malformed`);
         const keys = {};
-        for (const table of KEY_TABLES) {
-            const { bounds, files } = unpack(tables[table], `${layer}/${table}`, false);
-            keys[table] = { bounds, files };
-        }
-        layers.set(layer, { keys, entity: unpack(tables.entity, `${layer}/entity`, true) });
+        for (const table of KEY_TABLES)
+            keys[table] = spansOf(tables[table], `${layer}/${table}`);
+        const tag = tables.tag === undefined ? undefined : spansOf(tables.tag, `${layer}/tag`);
+        layers.set(layer, { keys, tag, entity: unpack(tables.entity, `${layer}/entity`, true) });
     }
     if (!layers.has('core'))
         throw bad('it has no core layer');
@@ -135,6 +139,8 @@ export function manifestPaths(manifest) {
         for (const table of KEY_TABLES)
             for (const hash of spans.keys[table].files)
                 paths.add(chunkPath(layer, table, hash));
+        for (const hash of spans.tag?.files ?? [])
+            paths.add(chunkPath(layer, 'tag', hash));
         for (const hash of spans.entity.files)
             paths.add(chunkPath(layer, 'entity', hash));
     }
@@ -181,10 +187,11 @@ const tagOf = (value) => Array.isArray(value) && typeof value[0] === 'string' &&
     : undefined;
 const workOf = (value) => Array.isArray(value) && isCount(value[0]) && typeof value[1] === 'string' ? [value[0], value[1]] : undefined;
 const eidOf = (value) => (isCount(value) ? value : undefined);
+const refOf = (value) => typeof value === 'string' && value.indexOf(':') > 0 ? value : undefined;
 function entityRow(value) {
     if (!Array.isArray(value) || value.length < 10)
         return undefined;
-    const [eid, id, rawNames, rawTags, rawWorks, rawReach, rank, standing, extra, adult] = value;
+    const [eid, id, rawNames, rawTags, rawWorks, rawReach, rank, standing, extra, adult, rawRefs = []] = value;
     if (!isCount(eid) || typeof id !== 'string')
         return undefined;
     const kind = id.startsWith('w/') ? 'work' : id.startsWith('c/') ? 'character' : undefined;
@@ -194,9 +201,10 @@ function entityRow(value) {
     const danbooru = list(rawTags, tagOf);
     const works = list(rawWorks, workOf);
     const reach = list(rawReach, eidOf);
-    if (!names || !danbooru || !works || !reach)
+    const refs = list(rawRefs, refOf);
+    if (!names || !danbooru || !works || !reach || !refs)
         return undefined;
-    return { eid, id, kind, names, danbooru, works, reach, rank, standing, extra, adult: adult === 1 };
+    return { eid, id, kind, names, danbooru, works, reach, rank, standing, extra, adult: adult === 1, refs };
 }
 export function readRows(table, value) {
     if (!Array.isArray(value))

package/lib/client/names.d.ts

@@ -1,4 +1,4 @@
-import { type Kind, type Layer } from '../model.ts';
+import { type Kind, type Layer, type NameKind } from '../model.ts';
 import { type Rows } from './chunks.ts';
 import { type Source } from './source.ts';
 export interface Found {
@@ -8,9 +8,11 @@ export interface Found {
     names: {
         text: string;
         lang: string | null;
+        kind: NameKind;
     }[];
     danbooru: string[];
     works: string[];
+    refs: string[];
     score: number;
     adult: boolean;
 }
@@ -34,4 +36,10 @@ export declare class Names {
         signal?: AbortSignal;
     }): Promise<Found[]>;
     lookup(ids: string[], signal?: AbortSignal): Promise<Found[]>;
+    tagged(tag: string, options?: {
+        kind?: Kind;
+        adult?: boolean;
+        prefix?: boolean;
+        signal?: AbortSignal;
+    }): Promise<Found[]>;
 }

package/lib/client/names.js

@@ -1,8 +1,9 @@
 import { gunzipSync } from 'node:zlib';
 import { LAYERS } from "../model.js";
+import { flat, qualifiers } from "../text.js";
 import { View } from "./chunks.js";
 import { search } from "./engine.js";
-import { parsePath, readManifest, readRows } from "./format.js";
+import { ADULT, CHARACTER, MAIN, NAME_KINDS, parsePath, readManifest, readRows, } from "./format.js";
 import { Missing } from "./source.js";
 const KEEP = 256;
 const PARALLEL = 16;
@@ -50,14 +51,21 @@ function label(id, rows) {
 }
 function card(rows, score, adult) {
     const head = rows[0];
-    const seen = new Set();
+    const seen = new Map();
     const names = [];
     for (const name of rows.flatMap((row) => row.names)) {
+        if (!SHOWN.has(name.kind))
+            continue;
         const key = `${name.lang ?? ''}\u0000${name.text}`;
-        if (!SHOWN.has(name.kind) || seen.has(key))
+        const known = seen.get(key);
+        if (known) {
+            if (NAME_KINDS.indexOf(name.kind) < NAME_KINDS.indexOf(known.kind))
+                known.kind = name.kind;
             continue;
-        seen.add(key);
-        names.push({ text: name.text, lang: name.lang });
+        }
+        const shown = { text: name.text, lang: name.lang, kind: name.kind };
+        seen.set(key, shown);
+        names.push(shown);
     }
     const posts = new Map();
     for (const [tag, count] of rows.flatMap((row) => row.danbooru)) {
@@ -69,7 +77,8 @@ function card(rows, score, adult) {
         .sort((a, b) => (b.count ?? -1) - (a.count ?? -1) || a.at - b.at)
         .map((entry) => entry.tag);
     const works = head.kind === 'character' ? [...new Set(rows.flatMap((row) => row.works.map(([, id]) => id)))] : [];
-    return { id: head.id, kind: head.kind, label: label(head.id, rows), names, danbooru, works, score, adult };
+    const refs = [...new Set(rows.flatMap((row) => row.refs))].sort();
+    return { id: head.id, kind: head.kind, label: label(head.id, rows), names, danbooru, works, refs, score, adult };
 }
 export class Names {
     #source;
@@ -110,6 +119,34 @@ export class Names {
             return view.wanted.size > 0 ? undefined : out;
         });
     }
+    tagged(tag, options = {}) {
+        const { kind, adult = true, prefix = false, signal } = options;
+        const key = flat(qualifiers(tag).base);
+        return this.#run(signal, (view) => {
+            if (!key)
+                return [];
+            const best = new Map();
+            for (const [, eid, posts, flags, , standing] of view.tagged(key, prefix)) {
+                if (kind && (flags & CHARACTER ? 'character' : 'work') !== kind)
+                    continue;
+                if (!adult && flags & ADULT)
+                    continue;
+                const known = best.get(eid);
+                const mask = flags & ~(CHARACTER | MAIN | ADULT);
+                if (!known || posts > known.posts)
+                    best.set(eid, { posts, mask: mask | (known?.mask ?? 0), flags, standing });
+                else
+                    known.mask |= mask;
+            }
+            const order = [...best].sort(([a, x], [b, y]) => y.posts - x.posts || y.standing - x.standing || a - b);
+            view.labels = true;
+            const out = order.flatMap(([eid, hit]) => {
+                const rows = view.entity(eid, hit.mask);
+                return rows.length > 0 ? [card(rows, 0, (hit.flags & ADULT) !== 0)] : [];
+            });
+            return view.wanted.size > 0 ? undefined : out;
+        });
+    }
     #open() {
         if (this.#state)
             return this.#state;

package/lib/client/source.js

@@ -2,7 +2,7 @@ import { createHash, randomUUID } from 'node:crypto';
 import { mkdir, readdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
 import { USER_AGENT } from "../model.js";
-import { KEY_TABLES, manifestPaths, parsePath, readManifest } from "./format.js";
+import { manifestPaths, parsePath, readManifest, TABLES } from "./format.js";
 export class Missing extends Error {
 }
 const revalidators = new WeakMap();
@@ -96,7 +96,7 @@ async function put(path, data) {
 async function prune(dir, text) {
     const named = manifestPaths(readManifest(text));
     for (const layer of await readdir(dir).catch(() => [])) {
-        for (const table of [...KEY_TABLES, 'entity']) {
+        for (const table of TABLES) {
             const folder = join(dir, layer, table);
             for (const file of await readdir(folder).catch(() => [])) {
                 const path = `${layer}/${table}/${file}`;

package/lib/model.d.ts

@@ -3,7 +3,7 @@ export type Layer = 'core' | 'kowiki' | 'unlicensed';
 export declare const LAYERS: readonly Layer[];
 export declare const USER_AGENT = "aninames/0.1 (+https://github.com/kecan0406/aninames)";
 export type NameKind = 'main' | 'official' | 'synonym' | 'short' | 'reading' | 'redirect' | 'section' | 'generated';
-export type Namespace = 'wikidata' | 'mal' | 'anilist' | 'anidb' | 'kitsu' | 'ann' | 'animeplanet' | 'anisearch' | 'livechart' | 'simkl' | 'animecountdown' | 'vndb' | 'bangumi' | 'danbooru' | 'kowiki' | 'anissia';
+export type Namespace = 'wikidata' | 'mal' | 'anilist' | 'anidb' | 'kitsu' | 'ann' | 'animeplanet' | 'anisearch' | 'livechart' | 'simkl' | 'animecountdown' | 'tmdb' | 'vndb' | 'bangumi' | 'danbooru' | 'kowiki' | 'anissia';
 export type Ref = `${Namespace}:${string}`;
 export interface Name {
     text: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kecan0406/aninames",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "type": "module",
   "description": "Multilingual alias index for anime titles and characters",
   "license": "MIT",