@team-antioch/sc2-doodad-finder 0.1.2 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/asker.d.ts CHANGED
@@ -1,89 +1 @@
1
- import type { NarrowOptions, Narrowing } from "./finder.ts";
2
- import type { Narrowed, Tag } from "./narrow.ts";
3
- import { type Ranker, type RankerName, type Ranking } from "./ranker.ts";
4
- import type { SpendCap } from "./spend-cap.ts";
5
- export type AskMode = "type" | "enter" | "refresh";
6
- /** What an ask runs with, besides its text: the page's state when it was made. */
7
- export type AskContext = {
8
- readonly tags?: readonly Tag[] | undefined;
9
- /** The object the ask is a part of, when there is one: context for the ranker. */
10
- readonly object?: string | undefined;
11
- /** A map's dependencies: their doodads are marked and kept first. */
12
- readonly dependencies?: readonly string[] | undefined;
13
- };
14
- export type AskRequest = AskContext & {
15
- readonly text: string;
16
- /** `type` for a keystroke, `enter` to rank now, `refresh` to narrow again now (the tags changed). */
17
- readonly mode: AskMode;
18
- /** The page state's version: an ask is its text and version. */
19
- readonly version: number;
20
- /** How long typing pauses before the ranker runs, in ms: the page's setting. */
21
- readonly idleMs?: number | undefined;
22
- };
23
- export type RankedCandidate = Narrowed & {
24
- readonly level: number;
25
- };
26
- export type AskEvent = {
27
- readonly type: "narrowed";
28
- readonly text: string;
29
- readonly version: number;
30
- readonly candidates: readonly Narrowed[];
31
- readonly ms: number;
32
- } | {
33
- readonly type: "ranked";
34
- readonly text: string;
35
- readonly version: number;
36
- readonly ranker: RankerName;
37
- /** The tags it ran with. */
38
- readonly tags: readonly Tag[];
39
- readonly candidates: readonly RankedCandidate[];
40
- /** From the ranker being asked for to its order: the narrowing, the call and the order. */
41
- readonly ms: number;
42
- readonly cost: number;
43
- } | {
44
- readonly type: "failed";
45
- readonly text: string;
46
- readonly version: number;
47
- /** The ranker the ask was for; none when no ranker is on. */
48
- readonly ranker?: RankerName;
49
- readonly error: string;
50
- /** Whether the narrowing's order is shown: false when the narrowing itself failed. */
51
- readonly narrowed: boolean;
52
- } | {
53
- readonly type: "capped";
54
- readonly text: string;
55
- readonly version: number;
56
- readonly reason: string;
57
- };
58
- export type AskTiming = {
59
- readonly debounceMs: number;
60
- readonly idleMs: number;
61
- readonly budgetMs: number;
62
- };
63
- /** The spec's timings: narrow after 250 ms, rank after 1.5 s idle, wait 300 ms for the ranker. */
64
- export declare const ASK_TIMING: AskTiming;
65
- /** An ask's ranking: the ranker's answer and the narrowed candidates in its order. */
66
- export type AskRanking = {
67
- readonly ranking: Ranking;
68
- readonly tags: readonly Tag[];
69
- readonly candidates: readonly RankedCandidate[];
70
- readonly ms: number;
71
- };
72
- export type AskerOptions = {
73
- readonly narrow: (text: string, options: NarrowOptions) => Promise<Narrowing>;
74
- /** The ranker; with none, asks are narrowed only. */
75
- readonly ranker?: Ranker | undefined;
76
- /** The session's spend cap: past it, no more ranker calls. Each ranking's cost is recorded in it. */
77
- readonly budget?: SpendCap;
78
- readonly emit: (event: AskEvent) => void;
79
- readonly timing?: Partial<AskTiming>;
80
- /** Called with each ranker's answer. */
81
- readonly onRanking?: (ranking: Ranking) => void;
82
- };
83
- export type Asker = {
84
- ask(request: AskRequest): void;
85
- /** The ranking for an ask, for ticking a card from it: the one it has, else one made now. */
86
- ranked(request: AskRequest): Promise<AskRanking>;
87
- close(): void;
88
- };
89
- export declare function createAsker(options: AskerOptions): Asker;
1
+ export * from "@team-antioch/sc2-finder-kit/asker.js";
package/dist/asker.js CHANGED
@@ -1,214 +1,2 @@
1
- // The Doodad Finder process's asks (docs/adr/0001), behind the page's ask endpoint. The page sends
2
- // each change to the ask as it is typed; the process decides when to narrow and when to rank:
3
- //
4
- // typing the narrowing runs 250 ms after typing pauses, and its cards are shown
5
- // idle 1.5 s after typing pauses (a setting), the ranker runs
6
- // Enter the narrowing runs at once if it hasn't, then the ranker
7
- // ranking the page waits up to 300 ms for the ranker; past that it is shown the narrowing's
8
- // order, and the ranked order replaces it once, when the ranker answers
9
- //
10
- // A newer ask cancels the older one: its timers stop, its ranker call is aborted, and nothing it
11
- // finds afterwards is sent. Asks are known by their text and the state version they were made in
12
- // (its tags and the map's dependencies), and by the tags it runs with, so a change of tags is a new ask too. Once the session's spend cap is
13
- // reached, an ask that would rank shows the narrowing's order and a `capped` event says why.
14
- import { orderByFit, RankerError } from "./ranker.js";
15
- /** The spec's timings: narrow after 250 ms, rank after 1.5 s idle, wait 300 ms for the ranker. */
16
- export const ASK_TIMING = { debounceMs: 250, idleMs: 1500, budgetMs: 300 };
17
- /** An ask's key: its version, its tags and its text. The version moves when the page's tags do, but a ticked card's ask can be ranked with the tags it was ticked with. */
18
- const keyOf = ({ text, version, tags }) => `${version}\n${JSON.stringify((tags ?? []).map(({ facet, value, pinned }) => [facet, value, pinned === true]))}\n${text.trim()}`;
19
- /** How many past asks' rankings are kept: going back to an earlier ask costs no call. */
20
- const KEPT_RANKINGS = 50;
21
- const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
22
- export function createAsker(options) {
23
- const timing = { ...ASK_TIMING, ...options.timing };
24
- const kept = new Map();
25
- let current;
26
- const { budget } = options;
27
- const stale = (ask) => current !== ask;
28
- const narrowNow = (ask) => {
29
- if (ask.narrowing)
30
- return ask.narrowing;
31
- const { text, tags, dependencies } = ask.request;
32
- const narrowing = options.narrow(text, { ...(tags ? { tags } : {}), dependencies });
33
- ask.narrowing = narrowing;
34
- // A failed narrowing isn't kept either: the next Enter or idle pause on this ask narrows again.
35
- narrowing.catch(() => {
36
- if (ask.narrowing === narrowing)
37
- delete ask.narrowing;
38
- });
39
- return narrowing;
40
- };
41
- const showNarrowing = async (ask) => {
42
- const narrowing = await narrowNow(ask);
43
- if (stale(ask) || ask.shown)
44
- return;
45
- ask.shown = "narrowed";
46
- options.emit({ type: "narrowed", text: ask.request.text, version: ask.request.version, candidates: narrowing.candidates, ms: narrowing.timings.total });
47
- };
48
- const rankingOf = (ask, ranker) => {
49
- if (ask.ranking)
50
- return ask.ranking;
51
- const known = kept.get(ask.key);
52
- if (known?.ranking.ranker === ranker.name)
53
- return (ask.ranking = Promise.resolve(known));
54
- const started = performance.now();
55
- ask.ranking = (async () => {
56
- const { candidates: narrowed } = await narrowNow(ask);
57
- const { text, object, tags } = ask.request;
58
- let ranking;
59
- try {
60
- ranking = narrowed.length ? await ranker.rank({ ask: text, object, tags, candidates: narrowed.map(({ card }) => card) }, { signal: ask.controller.signal }) : undefined;
61
- }
62
- catch (error) {
63
- // A call answered (and paid for) but not read still counts against the cap.
64
- if (error instanceof RankerError && error.cost > 0)
65
- budget?.record(error.cost);
66
- throw error;
67
- }
68
- if (ranking) {
69
- budget?.record(ranking.cost);
70
- options.onRanking?.(ranking);
71
- }
72
- const candidates = ranking ? orderByFit(narrowed, ranking.grades) : [];
73
- const result = {
74
- ranking: ranking ?? { ranker: ranker.name, model: "", grades: new Map(), timings: { request: 0, total: 0 }, usage: { input: 0, output: 0 }, cost: 0 },
75
- tags: tags ?? [],
76
- candidates,
77
- ms: performance.now() - started,
78
- };
79
- kept.delete(ask.key);
80
- kept.set(ask.key, result);
81
- if (kept.size > KEPT_RANKINGS)
82
- kept.delete(kept.keys().next().value);
83
- return result;
84
- })();
85
- // A ranking no one waits for (it was cancelled) mustn't be an unhandled rejection.
86
- ask.ranking.catch(() => { });
87
- return ask.ranking;
88
- };
89
- /** Whether the ask's ranking is known, so no call is needed for it. */
90
- const known = (ask, ranker) => ask.ranking !== undefined || kept.get(ask.key)?.ranking.ranker === ranker.name;
91
- const rankNow = async (ask) => {
92
- const { ranker } = options;
93
- // With no ranker, or none allowed, a failed narrowing is the ask's only failure: said, not dropped.
94
- const narrowOrSay = () => showNarrowing(ask).then(() => true, (error) => {
95
- if (!stale(ask))
96
- options.emit({ type: "failed", text: ask.request.text, version: ask.request.version, error: error.message, narrowed: false });
97
- return false;
98
- });
99
- if (!ranker)
100
- return void (await narrowOrSay());
101
- if (ask.rankStarted)
102
- return;
103
- ask.rankStarted = true;
104
- if (budget && !budget.allows() && !known(ask, ranker)) {
105
- if (!(await narrowOrSay())) {
106
- // Not capped yet as far as this ask goes: the next Enter narrows again.
107
- ask.rankStarted = false;
108
- return;
109
- }
110
- if (!stale(ask))
111
- options.emit({ type: "capped", text: ask.request.text, version: ask.request.version, reason: budget.reason });
112
- return;
113
- }
114
- const ranking = rankingOf(ask, ranker);
115
- const late = await Promise.race([ranking.then(() => false, () => false), sleep(timing.budgetMs).then(() => true)]);
116
- try {
117
- if (late)
118
- await showNarrowing(ask);
119
- const { candidates, ranking: answer, ms, tags } = await ranking;
120
- if (stale(ask))
121
- return;
122
- ask.shown = "ranked";
123
- options.emit({ type: "ranked", text: ask.request.text, version: ask.request.version, ranker: ranker.name, tags, candidates, ms, cost: answer.cost });
124
- }
125
- catch (error) {
126
- if (stale(ask))
127
- return;
128
- // A failed ranking isn't kept: the next Enter or idle pause on this ask calls the ranker again.
129
- delete ask.ranking;
130
- ask.rankStarted = false;
131
- // The narrowing's order is shown when the narrowing worked; when it failed, it isn't run again now.
132
- const narrowed = ask.narrowing ? await showNarrowing(ask).then(() => true, () => false) : false;
133
- if (stale(ask))
134
- return;
135
- options.emit({ type: "failed", text: ask.request.text, version: ask.request.version, ranker: ranker.name, error: error.message, narrowed });
136
- }
137
- };
138
- const later = (ask, timer, ms, run) => {
139
- clearTimeout(ask[timer]);
140
- ask[timer] = setTimeout(() => void run(ask).catch(() => { }), ms);
141
- };
142
- const stopTimers = (ask) => {
143
- clearTimeout(ask.debounce);
144
- clearTimeout(ask.idle);
145
- };
146
- const cancel = (ask) => {
147
- if (!ask)
148
- return;
149
- stopTimers(ask);
150
- ask.controller.abort(new Error("A newer ask replaced this one"));
151
- };
152
- return {
153
- ask(request) {
154
- const key = keyOf(request);
155
- let ask = current;
156
- if (ask?.key !== key) {
157
- cancel(current);
158
- current = undefined;
159
- if (!request.text.trim())
160
- return;
161
- ask = { key, request, controller: new AbortController() };
162
- current = ask;
163
- if (request.mode === "type")
164
- later(ask, "debounce", timing.debounceMs, showNarrowing);
165
- }
166
- const idle = request.idleMs ?? timing.idleMs;
167
- if (request.mode === "type") {
168
- // The same ask typed again (a trailing space): the idle wait starts over.
169
- if (!ask.rankStarted)
170
- later(ask, "idle", idle, rankNow);
171
- }
172
- else if (request.mode === "enter") {
173
- stopTimers(ask);
174
- void rankNow(ask).catch(() => { });
175
- }
176
- else {
177
- stopTimers(ask);
178
- void showNarrowing(ask).catch(() => { });
179
- if (!ask.rankStarted)
180
- later(ask, "idle", idle, rankNow);
181
- }
182
- },
183
- async ranked(request) {
184
- const key = keyOf(request);
185
- const known = kept.get(key);
186
- if (known)
187
- return known;
188
- const { ranker } = options;
189
- if (!ranker)
190
- throw new Error("No ranker is on");
191
- // A ranking already under way for the current ask was paid for before the cap was reached.
192
- if (budget && !budget.allows() && !(current?.key === key && current.ranking))
193
- throw new Error(`No ranking: ${budget.reason}`);
194
- if (current?.key === key) {
195
- const ask = current;
196
- void rankNow(ask).catch(() => { });
197
- try {
198
- return await rankingOf(ask, ranker);
199
- }
200
- catch (error) {
201
- // A newer ask cancelled it: the tick still needs it, so it is ranked on its own below.
202
- if (!ask.controller.signal.aborted)
203
- throw error;
204
- }
205
- }
206
- // An ask no longer current: ranked on its own, its results not sent.
207
- return rankingOf({ key, request, controller: new AbortController() }, ranker);
208
- },
209
- close() {
210
- cancel(current);
211
- current = undefined;
212
- },
213
- };
214
- }
1
+ // Moved to @team-antioch/sc2-finder-kit; kept here so deep imports of this package still resolve.
2
+ export * from "@team-antioch/sc2-finder-kit/asker.js";
@@ -1,4 +1 @@
1
- import { type CachePlatform } from "@team-antioch/sc2-asset-catalog/catalog/catalog-dirs.js";
2
- export type { CachePlatform } from "@team-antioch/sc2-asset-catalog/catalog/catalog-dirs.js";
3
- /** Where the model's weights are kept: $SC2_ASSET_CATALOG_MODEL_CACHE (or, for sc2-adjutant, $SC2_ADJUTANT_MODEL_CACHE), else the cache folder's models. */
4
- export declare function modelCacheDir(on?: CachePlatform): string;
1
+ export * from "@team-antioch/sc2-finder-kit/cache-dir.js";
package/dist/cache-dir.js CHANGED
@@ -1,17 +1,2 @@
1
- // The Finder keeps its files in the catalog's cache folder (catalogCacheDir, catalog-dirs.ts in
2
- // @team-antioch/sc2-asset-catalog), on each platform where that platform keeps caches: %LOCALAPPDATA%
3
- // on Windows, ~/Library/Caches on macOS, $XDG_CACHE_HOME (else ~/.cache) elsewhere, or
4
- // SC2_ASSET_CATALOG_CACHE on any of them. It holds the extracted catalog fields, the embeddings
5
- // model's weights, the card vectors, Jev's payload logs and the page's settings: all of it a copy
6
- // of something else, safe to delete.
7
- import path from "node:path";
8
- import { catalogCacheDir } from "@team-antioch/sc2-asset-catalog/catalog/catalog-dirs.js";
9
- /** Where the model's weights are kept: $SC2_ASSET_CATALOG_MODEL_CACHE (or, for sc2-adjutant, $SC2_ADJUTANT_MODEL_CACHE), else the cache folder's models. */
10
- export function modelCacheDir(on = {}) {
11
- const env = on.env ?? process.env;
12
- const moved = env.SC2_ASSET_CATALOG_MODEL_CACHE || env.SC2_ADJUTANT_MODEL_CACHE;
13
- if (moved)
14
- return moved;
15
- const dir = catalogCacheDir(on);
16
- return (on.platform ?? process.platform) === "win32" ? path.win32.join(dir, "models") : path.posix.join(dir, "models");
17
- }
1
+ // Moved to @team-antioch/sc2-finder-kit; kept here so deep imports of this package still resolve.
2
+ export * from "@team-antioch/sc2-finder-kit/cache-dir.js";
package/dist/cards.d.ts CHANGED
@@ -1,16 +1,9 @@
1
+ import type { FinderCard } from "@team-antioch/sc2-finder-kit/card.js";
1
2
  import { type DoodadEntry, type DoodadSearchSources } from "@team-antioch/sc2-asset-catalog/catalog/doodad-search.js";
2
- export type CandidateCard = {
3
- readonly id: string;
4
- readonly name: string;
3
+ export type CandidateCard = FinderCard & {
5
4
  readonly source: DoodadEntry["source"];
6
- /** The card's text: what is embedded and what the ranker reads. */
7
- readonly text: string;
8
5
  /** The entry the card was written from: its facets, for tags, and its look, for the page. */
9
6
  readonly entry: DoodadEntry;
10
- /** The map dependency it needs, absent when the default Void (Mod) brings it in. */
11
- readonly dependency?: string;
12
- /** For a map's dependencies: whether its dependencies already bring it in. */
13
- readonly provided?: boolean;
14
7
  /** A mod doodad its index has no description of for its current model. */
15
8
  readonly pending?: true;
16
9
  };
@@ -1,6 +1 @@
1
- /**
2
- * Whether the module at `url` is the one the process was started with (`argv1`, process.argv[1]),
3
- * so a CLI module runs only then and is safe to import. Never throws: an argv[1] that isn't a file
4
- * (a REPL, `node -e`, a test runner's own) is no module's.
5
- */
6
- export declare function isMainModule(url: string, argv1?: string | undefined): boolean;
1
+ export { isMainModule } from "@team-antioch/sc2-finder-kit/is-main.js";
@@ -1,17 +1,2 @@
1
- import { realpathSync } from "node:fs";
2
- import { pathToFileURL } from "node:url";
3
- /**
4
- * Whether the module at `url` is the one the process was started with (`argv1`, process.argv[1]),
5
- * so a CLI module runs only then and is safe to import. Never throws: an argv[1] that isn't a file
6
- * (a REPL, `node -e`, a test runner's own) is no module's.
7
- */
8
- export function isMainModule(url, argv1 = process.argv[1]) {
9
- if (!argv1)
10
- return false;
11
- try {
12
- return url === pathToFileURL(realpathSync(argv1)).href;
13
- }
14
- catch {
15
- return false;
16
- }
17
- }
1
+ // Moved to the kit (docs/adr/0002); kept so the old import path still resolves.
2
+ export { isMainModule } from "@team-antioch/sc2-finder-kit/is-main.js";
@@ -1,29 +1,11 @@
1
- import { type ConfigPlatform } from "./key-file.ts";
2
- import type { PayloadLog, Ranker } from "./ranker.ts";
1
+ import { type OpenedRanker as KitOpenedRanker } from "@team-antioch/sc2-finder-kit/credentials.js";
2
+ import type { PayloadLog } from "@team-antioch/sc2-finder-kit/ranker.js";
3
+ import type { CandidateCard } from "./cards.ts";
4
+ export { type KeySource, jevApiKey, jevKey, keySourceText } from "@team-antioch/sc2-finder-kit/credentials.js";
3
5
  export declare const NO_JEV_KEY: string;
4
- /** Where a key came from: what /status and the CLI may say, never the key. */
5
- export type KeySource = "environment" | "key file";
6
- /** Jev's key and where it came from: TYPESAFE_API_KEY, else the key file, else none. */
7
- export declare function jevKey(on?: ConfigPlatform): {
8
- readonly key: string;
9
- readonly source: KeySource;
10
- } | undefined;
11
- /** Jev's key alone: TYPESAFE_API_KEY, else the key file, else none. */
12
- export declare const jevApiKey: (on?: ConfigPlatform) => string | undefined;
13
- /** The ranker when it has its credential, and where that came from, else why it is off. */
14
- export type OpenedRanker = {
15
- readonly ranker: Ranker;
16
- readonly keySource: KeySource;
17
- readonly off?: undefined;
18
- } | {
19
- readonly ranker?: undefined;
20
- readonly keySource?: undefined;
21
- readonly off: string;
22
- };
23
- /** Where a key came from, in words. */
24
- export declare const keySourceText: (source: KeySource) => string;
6
+ export type OpenedRanker = KitOpenedRanker<CandidateCard>;
25
7
  /** Jev with its key, else off: with no key, or (Jev being optional) a key file that can't be read. */
26
- export declare function openRanker({ env, log }?: {
8
+ export declare const openRanker: ({ env, log }?: {
27
9
  readonly env?: Readonly<Record<string, string | undefined>>;
28
10
  readonly log?: PayloadLog;
29
- }): OpenedRanker;
11
+ }) => OpenedRanker;
@@ -1,32 +1,11 @@
1
- // The ranker's credential (docs/adr/0001), held by the Finder's process alone: Jev's key from
2
- // TYPESAFE_API_KEY in the environment, else the key file `sc2-doodad-finder login` saved
3
- // (key-file.ts), on every platform. The variable wins. With no key, Jev is off, with a message
1
+ // The ranker's credential (docs/adr/0001), held by the Finder's process alone: the kit's, for this
2
+ // finder: Jev's key from TYPESAFE_API_KEY in the environment, else the key file
3
+ // `sc2-doodad-finder login` saved, on every platform. With no key, Jev is off, with a message
4
4
  // saying how to give it one, and asks are narrowed only.
5
- import { keyFile, readKeyFile } from "./key-file.js";
6
- import { jevRanker } from "./jev-ranker.js";
7
- import { npxHint } from "@team-antioch/sc2-asset-catalog/util/npx.js";
8
- export const NO_JEV_KEY = `no TypeSafe key: run \`${npxHint("@team-antioch/sc2-doodad-finder login")}\`, or set TYPESAFE_API_KEY, to rank with Jev; until then the narrowing's order is shown`;
9
- /** Jev's key and where it came from: TYPESAFE_API_KEY, else the key file, else none. */
10
- export function jevKey(on = {}) {
11
- const fromEnv = (on.env ?? process.env).TYPESAFE_API_KEY?.trim();
12
- if (fromEnv)
13
- return { key: fromEnv, source: "environment" };
14
- const fromFile = readKeyFile(on);
15
- return fromFile ? { key: fromFile, source: "key file" } : undefined;
16
- }
17
- /** Jev's key alone: TYPESAFE_API_KEY, else the key file, else none. */
18
- export const jevApiKey = (on = {}) => jevKey(on)?.key;
19
- /** Where a key came from, in words. */
20
- export const keySourceText = (source) => (source === "environment" ? "the key in TYPESAFE_API_KEY" : "the saved key");
5
+ import { noJevKey, openRanker as kitOpenRanker } from "@team-antioch/sc2-finder-kit/credentials.js";
6
+ import { JEV_WORDING } from "./wording.js";
7
+ export { jevApiKey, jevKey, keySourceText } from "@team-antioch/sc2-finder-kit/credentials.js";
8
+ const COMMAND = "@team-antioch/sc2-doodad-finder";
9
+ export const NO_JEV_KEY = noJevKey(COMMAND);
21
10
  /** Jev with its key, else off: with no key, or (Jev being optional) a key file that can't be read. */
22
- export function openRanker({ env, log } = {}) {
23
- const on = env ? { env } : {};
24
- let found;
25
- try {
26
- found = jevKey(on);
27
- }
28
- catch (error) {
29
- return { off: `can't read the key file ${keyFile(on)} (${error.message}): run \`${npxHint("@team-antioch/sc2-doodad-finder login")}\` again, or set TYPESAFE_API_KEY` };
30
- }
31
- return found ? { ranker: jevRanker({ apiKey: found.key, ...(log ? { log } : {}) }), keySource: found.source } : { off: NO_JEV_KEY };
32
- }
11
+ export const openRanker = ({ env, log } = {}) => kitOpenRanker({ ...(env ? { env } : {}), ...(log ? { log } : {}), wording: JEV_WORDING, command: COMMAND });
@@ -1,39 +1 @@
1
- export { modelCacheDir } from "./cache-dir.ts";
2
- /** Embeds texts as unit vectors: an ask as a query, a card as a passage. */
3
- export type Embedder = {
4
- /** The model's name: vectors from two models aren't comparable, so the cache keeps them apart. */
5
- readonly model: string;
6
- embed(texts: readonly string[], as: "query" | "passage"): Promise<Float32Array[]>;
7
- };
8
- export declare const FINDER_MODEL = "Xenova/gte-small";
9
- /**
10
- * How each model the test command can compare pools its tokens into one vector, and what it puts
11
- * before a query: bge pools by its first token and asks for an instruction before a short query
12
- * searched against passages; gte averages its tokens and takes none.
13
- */
14
- export declare const MODELS: Readonly<Record<string, {
15
- readonly pooling: "cls" | "mean";
16
- readonly queryPrefix: string;
17
- }>>;
18
- /**
19
- * On Windows, ONNX Runtime's native files need the Microsoft Visual C++ runtime (vcruntime140.dll,
20
- * msvcp140.dll), which a fresh Windows, and a fresh Node, don't have; loading them then fails with
21
- * only "The specified module could not be found". The error that says what to install, for that
22
- * failure; else none.
23
- */
24
- export declare function missingRuntime(error: unknown, { platform, arch, hasRuntime }?: {
25
- readonly platform?: NodeJS.Platform;
26
- readonly arch?: string;
27
- readonly hasRuntime?: () => boolean;
28
- }): Error | undefined;
29
- /** The embeddings model can't load on this machine until something is installed: no use waiting on the rest of a start. */
30
- export declare class MissingRuntimeError extends Error {
31
- name: string;
32
- }
33
- /** Why the model didn't load, in words: the connection when a download failed, else the cache. */
34
- export declare function modelLoadError(model: string, error: unknown, cacheDir: string): Error;
35
- /** The local model, loaded once (downloading its weights the first time). */
36
- export declare function localEmbedder(options?: {
37
- readonly model?: string;
38
- readonly cacheDir?: string;
39
- }): Promise<Embedder>;
1
+ export * from "@team-antioch/sc2-finder-kit/embedder.js";
package/dist/embedder.js CHANGED
@@ -1,94 +1,2 @@
1
- // The Doodad Finder's embeddings model (docs/adr/0001): it runs on this machine, with no network call
2
- // once its weights are downloaded. The model is Alibaba's gte-small (384 dimensions, about 34 MB
3
- // quantized to 8 bits), run by transformers.js on ONNX Runtime for Node. On the test asks
4
- // it found as many candidates as bge-small and bge-base, and more of the top-ranked than
5
- // bge-small, at the fastest narrowing (the test command's --model compares them). Its weights
6
- // download to a machine cache the first time it runs, never into the repo. Tests use a fake behind
7
- // the same interface, so they need no model.
8
- import { existsSync } from "node:fs";
9
- import path from "node:path";
10
- import { modelCacheDir } from "./cache-dir.js";
11
- export { modelCacheDir } from "./cache-dir.js";
12
- export const FINDER_MODEL = "Xenova/gte-small";
13
- /**
14
- * How each model the test command can compare pools its tokens into one vector, and what it puts
15
- * before a query: bge pools by its first token and asks for an instruction before a short query
16
- * searched against passages; gte averages its tokens and takes none.
17
- */
18
- export const MODELS = {
19
- "Xenova/bge-small-en-v1.5": { pooling: "cls", queryPrefix: "Represent this sentence for searching relevant passages: " },
20
- "Xenova/bge-base-en-v1.5": { pooling: "cls", queryPrefix: "Represent this sentence for searching relevant passages: " },
21
- "Xenova/gte-small": { pooling: "mean", queryPrefix: "" },
22
- "Xenova/gte-base": { pooling: "mean", queryPrefix: "" },
23
- };
24
- /**
25
- * On Windows, ONNX Runtime's native files need the Microsoft Visual C++ runtime (vcruntime140.dll,
26
- * msvcp140.dll), which a fresh Windows, and a fresh Node, don't have; loading them then fails with
27
- * only "The specified module could not be found". The error that says what to install, for that
28
- * failure; else none.
29
- */
30
- export function missingRuntime(error, { platform = process.platform, arch = process.arch, hasRuntime = runtimeInstalled } = {}) {
31
- const message = error instanceof Error ? error.message : String(error);
32
- if (platform !== "win32" || !/specified module could not be found/i.test(message) || !/onnxruntime/i.test(message))
33
- return undefined;
34
- // The same words come for any DLL missing (onnxruntime.dll quarantined, a partial npx cache):
35
- // the runtime is only blamed when its own DLLs aren't there.
36
- if (hasRuntime())
37
- return undefined;
38
- const redist = arch === "arm64" ? "arm64" : "x64";
39
- return new MissingRuntimeError(`The embeddings model can't load: ONNX Runtime needs the Microsoft Visual C++ Redistributable (${redist}, to match this Node), which isn't installed or is too old. Install it from https://aka.ms/vs/17/release/vc_redist.${redist}.exe (or \`winget install Microsoft.VCRedist.2015+.${redist}\`), then start the Finder again. (ONNX Runtime said: ${message.split(/\r?\n/)[0]})`, { cause: error });
40
- }
41
- /** The embeddings model can't load on this machine until something is installed: no use waiting on the rest of a start. */
42
- export class MissingRuntimeError extends Error {
43
- name = "MissingRuntimeError";
44
- }
45
- /** Whether the Visual C++ runtime's DLLs are in Windows's System32. */
46
- function runtimeInstalled() {
47
- const system32 = path.join(process.env.SystemRoot ?? "C:\\Windows", "System32");
48
- // vcruntime140_1.dll came with the 2019 runtime: a 2015 or 2017 one has the other two without it.
49
- return ["vcruntime140.dll", "vcruntime140_1.dll", "msvcp140.dll"].every((dll) => existsSync(path.join(system32, dll)));
50
- }
51
- /** A network failure's words and codes, as fetch and Node give them. */
52
- const NETWORK = /fetch failed|network|ENOTFOUND|EAI_AGAIN|ECONNRESET|ECONNREFUSED|ETIMEDOUT|getaddrinfo|socket hang up/i;
53
- /** Why the model didn't load, in words: the connection when a download failed, else the cache. */
54
- export function modelLoadError(model, error, cacheDir) {
55
- const cause = error instanceof Error ? error : new Error(String(error));
56
- const said = `${cause.message} ${cause.cause?.code ?? ""}`;
57
- const message = cause.message.replace(/[.\s]+$/, "");
58
- return new Error(NETWORK.test(said)
59
- ? `The embeddings model (${model}) couldn't download: ${message}. Check the connection and start again: it downloads once.`
60
- : `The embeddings model (${model}) couldn't load: ${message}. If it keeps failing, delete its cache folder, ${cacheDir}, and start again to download it afresh.`, { cause: error });
61
- }
62
- /** At most this many texts per call to the model, so a cold start's thousands of cards don't make one huge batch. */
63
- const BATCH = 32;
64
- /** The local model, loaded once (downloading its weights the first time). */
65
- export async function localEmbedder(options = {}) {
66
- const model = options.model ?? FINDER_MODEL;
67
- const settings = MODELS[model];
68
- if (!settings)
69
- throw new Error(`No settings for the embeddings model ${model}: it takes one of ${Object.keys(MODELS).join(", ")}`);
70
- let extract;
71
- try {
72
- const { env, pipeline } = await import("@huggingface/transformers");
73
- env.cacheDir = options.cacheDir ?? modelCacheDir();
74
- extract = await pipeline("feature-extraction", model, { dtype: "q8" });
75
- }
76
- catch (error) {
77
- throw missingRuntime(error) ?? modelLoadError(model, error, options.cacheDir ?? modelCacheDir());
78
- }
79
- return {
80
- model,
81
- async embed(texts, as) {
82
- const vectors = [];
83
- for (let start = 0; start < texts.length; start += BATCH) {
84
- const batch = texts.slice(start, start + BATCH).map((text) => (as === "query" ? settings.queryPrefix + text : text));
85
- const output = await extract(batch, { pooling: settings.pooling, normalize: true });
86
- const [rows, dimensions] = output.dims;
87
- const data = output.data;
88
- for (let row = 0; row < rows; row++)
89
- vectors.push(data.slice(row * dimensions, (row + 1) * dimensions));
90
- }
91
- return vectors;
92
- },
93
- };
94
- }
1
+ // Moved to @team-antioch/sc2-finder-kit; kept here so deep imports of this package still resolve.
2
+ export * from "@team-antioch/sc2-finder-kit/embedder.js";
@@ -1,25 +1 @@
1
- import type { CandidateCard } from "./cards.ts";
2
- import type { Embedder } from "./embedder.ts";
3
- export type Neighbour = {
4
- readonly id: string;
5
- readonly similarity: number;
6
- };
7
- export declare class EmbeddingIndex {
8
- #private;
9
- private constructor();
10
- /** The index over the cache file's vectors for the embedder's model; none until the first update. */
11
- static open(embedder: Embedder, file: string): Promise<EmbeddingIndex>;
12
- /**
13
- * Takes these cards as the whole set: embeds the ones whose text it has no vector for, forgets the
14
- * rest, and saves the cache when it changed. Gives these cards' own nearest, which later updates
15
- * leave as it is, so a caller can swap it in with the cards it was made for.
16
- */
17
- update(cards: readonly CandidateCard[]): Promise<{
18
- embedded: number;
19
- nearest: Nearest;
20
- }>;
21
- /** The last update's cards most like a unit vector (see Nearest). */
22
- nearest(query: Float32Array, k: number, keep?: (id: string) => boolean): Neighbour[];
23
- }
24
- /** The k cards most like a unit vector, most alike first, among those `keep` allows. */
25
- export type Nearest = (query: Float32Array, k: number, keep?: (id: string) => boolean) => Neighbour[];
1
+ export * from "@team-antioch/sc2-finder-kit/embedding-index.js";