@kecan0406/aninames 0.2.0 → 0.3.1

package/README.md

@@ -56,7 +56,7 @@ Built sizes: `core.sqlite` 610 MB, `kowiki.sqlite` 25 MB, `unlicensed.sqlite` 83
 | `<layer>/<table>/<hash>.json.gz` | about 32 KB of JSON rows sorted by key (16 KB for `entity`), named by the first 48 bits of the SHA-256 of the file |
 | `NOTICE.md`, `index.html` | each layer's sources and terms |
 
-The key tables are `flat` (folded names), `latin` (romaji keys), `sound` (sound keys across kana, Hangul and romaji), `part` (name tokens) and `initial` (Korean initials); their rows are `[key, entity number, name weight, flags, popularity, standing, bonus]`. `tag` holds the same rows for Danbooru tags, keyed by the tag folded like a name without its qualifier and weighted by the tag's posts. `entity` rows, keyed by entity number (which follows the entity ids' order), carry the id, names, Danbooru tags, works, family and the ids other databases give one entity in that layer (TMDB's `tmdb:tv/…` and `tmdb:movie/…` among them, read from Wikidata). Layers stay in separate files; the only values a file carries from other layers are derived numbers and flags (popularity, family standing, adult, which layers know the entity).
+The key tables are `flat` (folded names), `latin` (romaji keys), `sound` (sound keys across kana, Hangul and romaji), `part` (name tokens) and `initial` (Korean initials); their rows are `[key, entity number, name weight, flags, popularity, standing, bonus]`. `tag` holds the same rows for Danbooru tags, keyed by the tag folded like a name without its qualifier and weighted by the tag's posts. `home` holds them for the copyright tag each character comes from, weighted by the character's own posts; a `core` row whose character the `unlicensed` layer gives another home carries flag 64, which a client that reads `unlicensed` skips. `entity` rows, keyed by entity number (which follows the entity ids' order), carry the id, names, Danbooru tags, works, family, the ids other databases give one entity in that layer (TMDB's `tmdb:tv/…` and `tmdb:movie/…`, IGDB's `igdb:…` and Steam's `steam:…` among them, read from Wikidata), a character's home tag and a work's countries of origin (Wikidata P495, as ISO 3166-1 alpha-2 codes such as `JP`). Layers stay in separate files; the only values a file carries from other layers are derived numbers and flags (popularity, family standing, adult, which layers know the entity).
 
 On the 10-04 build: 19,167 files, 149 MB gzip'd, a manifest of 236 KB gzip'd. `mise run compare` searches the probe sample (4,370 queries) both ways: the files find the entity first 87% of the time, SQLite 86% (FTS substring search is not in the files; typo search reads the rows around the query's position instead). A cold query needs a median of 2 rounds (p90 3), 31 files (p90 78) and 287 KB (p90 584 KB).
 
@@ -71,7 +71,7 @@ const [first] = await names.search('봇치', { limit: 10, signal })
 const works = await names.lookup(first?.works ?? [])
 ```
 
-`search` returns `Found` cards (`id`, `kind`, `label`, `names` with their `lang` and `kind`, `danbooru` tags with the most posts first, `works`, `refs`, `score`, `adult`); `search(query, { kind, limit, adult: false })` narrows it, `lookup(ids)` returns the cards of given ids in one round, and `tagged(tag, { kind, adult })` the cards of the entities that carry a Danbooru tag, most posts first, whatever its case, punctuation or qualifier (`soul-eater`, `soul_eater` and `soul_eater_(series)` find the same works); `prefix: true` also finds the tags that start with it. `cachedSource` keeps the files it fetched forever (their names are hashes of their contents), revalidates the manifest with its ETag or Last-Modified once a day (`maxAge`), answers from the cache when offline, refetches the manifest when the site has moved on to a new build, and deletes files a new manifest no longer names. `fileSource(dir)` reads a local `dist/site`.
+`search` returns `Found` cards (`id`, `kind`, `label`, `names` with their `lang` and `kind`, `danbooru` tags with the most posts first, `works`, `refs`, `home`, the copyright tag a character comes from, `origin`, the countries a work comes from, `score`, `adult`); `search(query, { kind, limit, adult: false })` narrows it, `lookup(ids)` returns the cards of given ids in one round, and `tagged(tag, { kind, adult })` the cards of the entities that carry a Danbooru tag, most posts first, whatever its case, punctuation or qualifier (`soul-eater`, `soul_eater` and `soul_eater_(series)` find the same works); `prefix: true` also finds the tags that start with it. `cast(tag, { adult, prefix, limit })` returns the characters whose home is that tag, most posts first. `cachedSource` keeps the files it fetched forever (their names are hashes of their contents), revalidates the manifest with its ETag or Last-Modified once a day (`maxAge`), answers from the cache when offline, refetches the manifest when the site has moved on to a new build, and deletes files a new manifest no longer names. `fileSource(dir)` reads a local `dist/site`.
 
 ## Development
 

package/lib/client/chunks.d.ts

@@ -19,7 +19,7 @@ export declare class View {
     constructor(manifest: Manifest, layers: Layer[], loaded: Loaded);
     private rows;
     keys(table: KeyTable, range: Range): KeyRow[];
-    tagged(key: string, prefix: boolean): KeyRow[];
+    tagged(key: string, prefix: boolean, table?: 'tag' | 'home'): KeyRow[];
     private keysIn;
     around(table: KeyTable, key: string, radius: number): KeyRow[];
     entity(eid: number, mask?: number): EntityRow[];

package/lib/client/chunks.js

@@ -39,9 +39,9 @@ export class View {
     keys(table, range) {
         return this.layers.flatMap((layer) => this.keysIn(layer, table, this.manifest.layers.get(layer)?.keys[table], range));
     }
-    tagged(key, prefix) {
+    tagged(key, prefix, table = 'tag') {
         const range = { low: key, high: prefix ? `${key}\u{10FFFF}` : key };
-        return this.layers.flatMap((layer) => this.keysIn(layer, 'tag', this.manifest.layers.get(layer)?.tag, range));
+        return this.layers.flatMap((layer) => this.keysIn(layer, table, this.manifest.layers.get(layer)?.[table], range));
     }
     keysIn(layer, table, spans, range) {
         if (!spans)

package/lib/client/format.d.ts

@@ -2,13 +2,14 @@ import { type Kind, type Layer, type NameKind } from '../model.ts';
 export declare const FORMAT = 1;
 export declare const KEY_TABLES: readonly ["flat", "latin", "sound", "part", "initial"];
 export type KeyTable = (typeof KEY_TABLES)[number];
-export declare const TABLES: readonly ["flat", "latin", "sound", "part", "initial", "tag", "entity"];
+export declare const TABLES: readonly ["flat", "latin", "sound", "part", "initial", "tag", "home", "entity"];
 export type Table = (typeof TABLES)[number];
 export declare const NAME_KINDS: readonly NameKind[];
 export declare const TRUSTED: readonly string[];
 export declare const CHARACTER = 1;
 export declare const MAIN = 2;
 export declare const ADULT = 4;
+export declare const SHADOWED = 64;
 export declare function present(layer: Layer): number;
 export declare const HASH_LENGTH = 12;
 export declare function chunkPath(layer: Layer, table: Table, hash: string): string;
@@ -35,6 +36,7 @@ export interface EntitySpans extends Spans {
 export interface LayerSpans {
     keys: Record<KeyTable, Spans>;
     tag: Spans | undefined;
+    home: Spans | undefined;
     entity: EntitySpans;
 }
 export interface Manifest {
@@ -73,5 +75,7 @@ export interface EntityRow {
     extra: number;
     adult: boolean;
     refs: string[];
+    home: string | null;
+    origin: string[];
 }
 export declare function readRows(table: Table, value: unknown): KeyRow[] | EntityRow[] | undefined;

package/lib/client/format.js

@@ -3,7 +3,7 @@ import { flat } from "../text.js";
 import { compare } from "./order.js";
 export const FORMAT = 1;
 export const KEY_TABLES = ['flat', 'latin', 'sound', 'part', 'initial'];
-export const TABLES = [...KEY_TABLES, 'tag', 'entity'];
+export const TABLES = [...KEY_TABLES, 'tag', 'home', 'entity'];
 export const NAME_KINDS = [
     'main',
     'official',
@@ -18,6 +18,7 @@ export const TRUSTED = ['wikidata', 'aod', 'anidb', 'vndb'];
 export const CHARACTER = 1;
 export const MAIN = 2;
 export const ADULT = 4;
+export const SHADOWED = 64;
 export function present(layer) {
     return 8 << LAYERS.indexOf(layer);
 }
@@ -127,7 +128,8 @@ export function readManifest(text) {
         for (const table of KEY_TABLES)
             keys[table] = spansOf(tables[table], `${layer}/${table}`);
         const tag = tables.tag === undefined ? undefined : spansOf(tables.tag, `${layer}/tag`);
-        layers.set(layer, { keys, tag, entity: unpack(tables.entity, `${layer}/entity`, true) });
+        const home = tables.home === undefined ? undefined : spansOf(tables.home, `${layer}/home`);
+        layers.set(layer, { keys, tag, home, entity: unpack(tables.entity, `${layer}/entity`, true) });
     }
     if (!layers.has('core'))
         throw bad('it has no core layer');
@@ -141,6 +143,8 @@ export function manifestPaths(manifest) {
                 paths.add(chunkPath(layer, table, hash));
         for (const hash of spans.tag?.files ?? [])
             paths.add(chunkPath(layer, 'tag', hash));
+        for (const hash of spans.home?.files ?? [])
+            paths.add(chunkPath(layer, 'home', hash));
         for (const hash of spans.entity.files)
             paths.add(chunkPath(layer, 'entity', hash));
     }
@@ -188,10 +192,11 @@ const tagOf = (value) => Array.isArray(value) && typeof value[0] === 'string' &&
 const workOf = (value) => Array.isArray(value) && isCount(value[0]) && typeof value[1] === 'string' ? [value[0], value[1]] : undefined;
 const eidOf = (value) => (isCount(value) ? value : undefined);
 const refOf = (value) => typeof value === 'string' && value.indexOf(':') > 0 ? value : undefined;
+const countryOf = (value) => typeof value === 'string' && /^[A-Z]{2}$/.test(value) ? value : undefined;
 function entityRow(value) {
     if (!Array.isArray(value) || value.length < 10)
         return undefined;
-    const [eid, id, rawNames, rawTags, rawWorks, rawReach, rank, standing, extra, adult, rawRefs = []] = value;
+    const [eid, id, rawNames, rawTags, rawWorks, rawReach, rank, standing, extra, adult, rawRefs = [], home = null, rawOrigin = [],] = value;
     if (!isCount(eid) || typeof id !== 'string')
         return undefined;
     const kind = id.startsWith('w/') ? 'work' : id.startsWith('c/') ? 'character' : undefined;
@@ -202,9 +207,12 @@ function entityRow(value) {
     const works = list(rawWorks, workOf);
     const reach = list(rawReach, eidOf);
     const refs = list(rawRefs, refOf);
-    if (!names || !danbooru || !works || !reach || !refs)
+    const origin = list(rawOrigin, countryOf);
+    if (!names || !danbooru || !works || !reach || !refs || !origin)
         return undefined;
-    return { eid, id, kind, names, danbooru, works, reach, rank, standing, extra, adult: adult === 1, refs };
+    if (home !== null && typeof home !== 'string')
+        return undefined;
+    return { eid, id, kind, names, danbooru, works, reach, rank, standing, extra, adult: adult === 1, refs, home, origin };
 }
 export function readRows(table, value) {
     if (!Array.isArray(value))

package/lib/client/names.d.ts

@@ -13,6 +13,8 @@ export interface Found {
     danbooru: string[];
     works: string[];
     refs: string[];
+    home: string | null;
+    origin: string[];
     score: number;
     adult: boolean;
 }
@@ -42,4 +44,10 @@ export declare class Names {
         prefix?: boolean;
         signal?: AbortSignal;
     }): Promise<Found[]>;
+    cast(tag: string, options?: {
+        adult?: boolean;
+        prefix?: boolean;
+        limit?: number;
+        signal?: AbortSignal;
+    }): Promise<Found[]>;
 }

package/lib/client/names.js

@@ -3,7 +3,7 @@ import { LAYERS } from "../model.js";
 import { flat, qualifiers } from "../text.js";
 import { View } from "./chunks.js";
 import { search } from "./engine.js";
-import { ADULT, CHARACTER, MAIN, NAME_KINDS, parsePath, readManifest, readRows, } from "./format.js";
+import { ADULT, CHARACTER, MAIN, NAME_KINDS, parsePath, readManifest, readRows, SHADOWED, } from "./format.js";
 import { Missing } from "./source.js";
 const KEEP = 256;
 const PARALLEL = 16;
@@ -78,7 +78,21 @@ function card(rows, score, adult) {
         .map((entry) => entry.tag);
     const works = head.kind === 'character' ? [...new Set(rows.flatMap((row) => row.works.map(([, id]) => id)))] : [];
     const refs = [...new Set(rows.flatMap((row) => row.refs))].sort();
-    return { id: head.id, kind: head.kind, label: label(head.id, rows), names, danbooru, works, refs, score, adult };
+    const home = rows.findLast((row) => row.home !== null)?.home ?? null;
+    const origin = [...new Set(rows.flatMap((row) => row.origin))];
+    return {
+        id: head.id,
+        kind: head.kind,
+        label: label(head.id, rows),
+        names,
+        danbooru,
+        works,
+        refs,
+        home,
+        origin,
+        score,
+        adult,
+    };
 }
 export class Names {
     #source;
@@ -120,25 +134,36 @@ export class Names {
         });
     }
     tagged(tag, options = {}) {
-        const { kind, adult = true, prefix = false, signal } = options;
+        return this.#tagged('tag', tag, options);
+    }
+    cast(tag, options = {}) {
+        return this.#tagged('home', tag, options);
+    }
+    #tagged(table, tag, options) {
+        const { kind, adult = true, prefix = false, limit = Number.POSITIVE_INFINITY, signal } = options;
         const key = flat(qualifiers(tag).base);
         return this.#run(signal, (view) => {
             if (!key)
                 return [];
+            const covered = view.layers.includes('unlicensed');
             const best = new Map();
-            for (const [, eid, posts, flags, , standing] of view.tagged(key, prefix)) {
+            for (const [, eid, posts, flags, , standing] of view.tagged(key, prefix, table)) {
                 if (kind && (flags & CHARACTER ? 'character' : 'work') !== kind)
                     continue;
                 if (!adult && flags & ADULT)
                     continue;
+                if (covered && flags & SHADOWED)
+                    continue;
                 const known = best.get(eid);
-                const mask = flags & ~(CHARACTER | MAIN | ADULT);
+                const mask = flags & ~(CHARACTER | MAIN | ADULT | SHADOWED);
                 if (!known || posts > known.posts)
                     best.set(eid, { posts, mask: mask | (known?.mask ?? 0), flags, standing });
                 else
                     known.mask |= mask;
             }
-            const order = [...best].sort(([a, x], [b, y]) => y.posts - x.posts || y.standing - x.standing || a - b);
+            const order = [...best]
+                .sort(([a, x], [b, y]) => y.posts - x.posts || y.standing - x.standing || a - b)
+                .slice(0, limit);
             view.labels = true;
             const out = order.flatMap(([eid, hit]) => {
                 const rows = view.entity(eid, hit.mask);

package/lib/model.d.ts

@@ -3,7 +3,7 @@ export type Layer = 'core' | 'kowiki' | 'unlicensed';
 export declare const LAYERS: readonly Layer[];
 export declare const USER_AGENT = "aninames/0.1 (+https://github.com/kecan0406/aninames)";
 export type NameKind = 'main' | 'official' | 'synonym' | 'short' | 'reading' | 'redirect' | 'section' | 'generated';
-export type Namespace = 'wikidata' | 'mal' | 'anilist' | 'anidb' | 'kitsu' | 'ann' | 'animeplanet' | 'anisearch' | 'livechart' | 'simkl' | 'animecountdown' | 'tmdb' | 'vndb' | 'bangumi' | 'danbooru' | 'kowiki' | 'anissia';
+export type Namespace = 'wikidata' | 'mal' | 'anilist' | 'anidb' | 'kitsu' | 'ann' | 'animeplanet' | 'anisearch' | 'livechart' | 'simkl' | 'animecountdown' | 'tmdb' | 'igdb' | 'steam' | 'vndb' | 'bangumi' | 'danbooru' | 'kowiki' | 'anissia';
 export type Ref = `${Namespace}:${string}`;
 export interface Name {
     text: string;
@@ -14,15 +14,19 @@ export interface Entry {
     id: Ref;
     kind?: Kind;
     attach?: boolean;
+    family?: boolean;
     refs?: Ref[];
     names: Name[];
     works?: Ref[];
+    series?: Ref[];
     related?: Ref[];
+    near?: Ref[];
     link?: string[];
     rank?: number;
     adult?: boolean;
     type?: string;
     year?: number;
+    origin?: string[];
 }
 export interface Source {
     name: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kecan0406/aninames",
-  "version": "0.2.0",
+  "version": "0.3.1",
   "type": "module",
   "description": "Multilingual alias index for anime titles and characters",
   "license": "MIT",