@team-antioch/sc2-finder-kit 0.0.0-stage → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +36 -2
  3. package/dist/asker.d.ts +91 -0
  4. package/dist/asker.js +214 -0
  5. package/dist/cache-dir.d.ts +4 -0
  6. package/dist/cache-dir.js +17 -0
  7. package/dist/card.d.ts +35 -0
  8. package/dist/card.js +4 -0
  9. package/dist/credentials.d.ts +34 -0
  10. package/dist/credentials.js +33 -0
  11. package/dist/embedder.d.ts +39 -0
  12. package/dist/embedder.js +94 -0
  13. package/dist/embedding-index.d.ts +26 -0
  14. package/dist/embedding-index.js +129 -0
  15. package/dist/finder.d.ts +87 -0
  16. package/dist/finder.js +162 -0
  17. package/dist/gateway-translator.d.ts +21 -0
  18. package/dist/gateway-translator.js +62 -0
  19. package/dist/is-main.d.ts +6 -0
  20. package/dist/is-main.js +17 -0
  21. package/dist/jev-language-detector.d.ts +15 -0
  22. package/dist/jev-language-detector.js +44 -0
  23. package/dist/jev-ranker.d.ts +45 -0
  24. package/dist/jev-ranker.js +93 -0
  25. package/dist/key-file.d.ts +13 -0
  26. package/dist/key-file.js +62 -0
  27. package/dist/language-detector.d.ts +20 -0
  28. package/dist/language-detector.js +38 -0
  29. package/dist/load-together.d.ts +26 -0
  30. package/dist/load-together.js +39 -0
  31. package/dist/login.d.ts +34 -0
  32. package/dist/login.js +137 -0
  33. package/dist/narrow.d.ts +57 -0
  34. package/dist/narrow.js +74 -0
  35. package/dist/payload-log.d.ts +5 -0
  36. package/dist/payload-log.js +12 -0
  37. package/dist/ranker.d.ts +80 -0
  38. package/dist/ranker.js +40 -0
  39. package/dist/server.d.ts +178 -0
  40. package/dist/server.js +565 -0
  41. package/dist/settings.d.ts +7 -0
  42. package/dist/settings.js +20 -0
  43. package/dist/spend-cap.d.ts +14 -0
  44. package/dist/spend-cap.js +22 -0
  45. package/dist/test-asks.d.ts +60 -0
  46. package/dist/test-asks.js +107 -0
  47. package/dist/translator.d.ts +48 -0
  48. package/dist/translator.js +51 -0
  49. package/dist/tray.d.ts +22 -0
  50. package/dist/tray.js +12 -0
  51. package/package.json +45 -4
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 TeamAntioch
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -1,3 +1,37 @@
1
- # Temporary Holding Version
1
+ # @team-antioch/sc2-finder-kit
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ What the StarCraft II asset finders share. The [Doodad Finder](../sc2-doodad-finder) is built on it, and so can another finder be: the Music Finder is next.
4
+
5
+ A finder in this repo is a small local web page that answers an ask in plain words with cards. Asking narrows in well under a second from text alone (a keyword search and a local embeddings search, merged), then Jev puts the narrowed cards in order. The kit is the half of that which doesn't depend on what the cards are.
6
+
7
+ ## What is here
8
+
9
+ | Module | What it does |
10
+ | --- | --- |
11
+ | `card.js` | `FinderCard`, the least the kit asks of a card: an id, a name, the text that is embedded and ranked, and optionally the dependency it needs and whether a map provides it. Also the tag types. |
12
+ | `finder.js` | `openFinder`: opens a finder's sources, embeds its cards (only the ones the cache hasn't seen), narrows each ask, and writes the cards again when a watched file changes. |
13
+ | `narrow.js` | The merge of the keyword results and the nearest cards, with the tags' raise and filter. |
14
+ | `embedder.js`, `embedding-index.js` | The local embeddings model (gte-small) and the vector cache. |
15
+ | `ranker.js`, `jev-ranker.js` | The ranker slot, the order by fit, and the Jev ranker, which takes the finder's wording. |
16
+ | `credentials.js`, `key-file.js`, `login.js` | Jev's key from `TYPESAFE_API_KEY` or the key file, and the `login` and `logout` commands. One key serves every finder. |
17
+ | `spend-cap.js`, `payload-log.js`, `settings.js`, `cache-dir.js` | The session's spend cap, the log of what each ask sent, the page's remembered settings, and where the model's weights go. |
18
+ | `asker.js` | When an ask narrows and when it ranks, and cancelling the older one. |
19
+ | `server.js` | `startFinderServer`: the process on 127.0.0.1 that serves the page and answers it, with its host and origin checks, the tray, the tray's objects and actions and the tick log. |
20
+ | `tray.js`, `test-asks.js` | The tray, and the test-ask cases and the tick log's lines, the test command's speed targets and its warm start. |
21
+ | `load-together.js`, `is-main.js` | The start-up race between a finder's extraction and the model, and the check that a CLI module is the one running. |
22
+
23
+ ## What a finder passes in
24
+
25
+ - To `openFinder`: its sources' loader, how a source makes its cards, its keyword search, what a tag matches and what a dependency provides.
26
+ - To `startFinderServer`: a `FinderKind`, which is its name for `/status`, the noun its refusals use, its page, each card's markup, its tags, its copy text, an optional route of its own and a close hook.
27
+ - To `jevRanker` and `openRanker`: its `JevWording`, the words Jev judges by.
28
+
29
+ [ADR 0002](../../docs/adr/0002-the-finders-share-a-kit.md) has the reasoning. The tests build a toy finder from these, with no doodads or tracks in it.
30
+
31
+ ## Using it
32
+
33
+ Every module imports on its own, as `@team-antioch/sc2-finder-kit/<module>.js`. The kit keeps its files in the asset catalog's cache folder, so it depends on `@team-antioch/sc2-asset-catalog` for where that is. It has no command of its own: a finder's package brings the CLI.
34
+
35
+ This project is not affiliated with or endorsed by Blizzard Entertainment. StarCraft II and its assets are Blizzard's.
36
+
37
+ MIT licence.
@@ -0,0 +1,91 @@
1
+ import type { NarrowOptions, Narrowing } from "./finder.ts";
2
+ import type { Narrowed, Tag } from "./narrow.ts";
3
+ import { type Ranker, type RankerName, type Ranking } from "./ranker.ts";
4
+ import type { SpendCap } from "./spend-cap.ts";
5
+ export type AskMode = "type" | "enter" | "refresh";
6
+ /** What an ask runs with, besides its text: the page's state when it was made. */
7
+ export type AskContext = {
8
+ readonly tags?: readonly Tag[] | undefined;
9
+ /** The object the ask is a part of, when there is one: context for the ranker. */
10
+ readonly object?: string | undefined;
11
+ /** A map's dependencies: the cards they provide are marked and kept first. */
12
+ readonly dependencies?: readonly string[] | undefined;
13
+ };
14
+ export type AskRequest = AskContext & {
15
+ readonly text: string;
16
+ /** `type` for a keystroke, `enter` to rank now, `refresh` to narrow again now (the tags changed). */
17
+ readonly mode: AskMode;
18
+ /** The page state's version: an ask is its text and version. */
19
+ readonly version: number;
20
+ /** How long typing pauses before the ranker runs, in ms: the page's setting. */
21
+ readonly idleMs?: number | undefined;
22
+ };
23
+ export type RankedCandidate = Narrowed & {
24
+ readonly level: number;
25
+ };
26
+ export type AskEvent = {
27
+ readonly type: "narrowed";
28
+ readonly text: string;
29
+ readonly version: number;
30
+ readonly candidates: readonly Narrowed[];
31
+ readonly ms: number;
32
+ /** How many candidates the rank budget dropped from the narrowing: 0 when none, so a finder can say the list was trimmed. */
33
+ readonly cut: number;
34
+ } | {
35
+ readonly type: "ranked";
36
+ readonly text: string;
37
+ readonly version: number;
38
+ readonly ranker: RankerName;
39
+ /** The tags it ran with. */
40
+ readonly tags: readonly Tag[];
41
+ readonly candidates: readonly RankedCandidate[];
42
+ /** From the ranker being asked for to its order: the narrowing, the call and the order. */
43
+ readonly ms: number;
44
+ readonly cost: number;
45
+ } | {
46
+ readonly type: "failed";
47
+ readonly text: string;
48
+ readonly version: number;
49
+ /** The ranker the ask was for; none when no ranker is on. */
50
+ readonly ranker?: RankerName;
51
+ readonly error: string;
52
+ /** Whether the narrowing's order is shown: false when the narrowing itself failed. */
53
+ readonly narrowed: boolean;
54
+ } | {
55
+ readonly type: "capped";
56
+ readonly text: string;
57
+ readonly version: number;
58
+ readonly reason: string;
59
+ };
60
+ export type AskTiming = {
61
+ readonly debounceMs: number;
62
+ readonly idleMs: number;
63
+ readonly budgetMs: number;
64
+ };
65
+ /** The spec's timings: narrow after 250 ms, rank after 1.5 s idle, wait 300 ms for the ranker. */
66
+ export declare const ASK_TIMING: AskTiming;
67
+ /** An ask's ranking: the ranker's answer and the narrowed candidates in its order. */
68
+ export type AskRanking = {
69
+ readonly ranking: Ranking;
70
+ readonly tags: readonly Tag[];
71
+ readonly candidates: readonly RankedCandidate[];
72
+ readonly ms: number;
73
+ };
74
+ export type AskerOptions = {
75
+ readonly narrow: (text: string, options: NarrowOptions) => Promise<Narrowing>;
76
+ /** The ranker; with none, asks are narrowed only. */
77
+ readonly ranker?: Ranker | undefined;
78
+ /** The session's spend cap: past it, no more ranker calls. Each ranking's cost is recorded in it. */
79
+ readonly budget?: SpendCap;
80
+ readonly emit: (event: AskEvent) => void;
81
+ readonly timing?: Partial<AskTiming>;
82
+ /** Called with each ranker's answer. */
83
+ readonly onRanking?: (ranking: Ranking) => void;
84
+ };
85
+ export type Asker = {
86
+ ask(request: AskRequest): void;
87
+ /** The ranking for an ask, for ticking a card from it: the one it has, else one made now. */
88
+ ranked(request: AskRequest): Promise<AskRanking>;
89
+ close(): void;
90
+ };
91
+ export declare function createAsker(options: AskerOptions): Asker;
package/dist/asker.js ADDED
@@ -0,0 +1,214 @@
1
+ // A finder process's asks (docs/adr/0001), behind the page's ask endpoint. The page sends
2
+ // each change to the ask as it is typed; the process decides when to narrow and when to rank:
3
+ //
4
+ // typing the narrowing runs 250 ms after typing pauses, and its cards are shown
5
+ // idle 1.5 s after typing pauses (a setting), the ranker runs
6
+ // Enter the narrowing runs at once if it hasn't, then the ranker
7
+ // ranking the page waits up to 300 ms for the ranker; past that it is shown the narrowing's
8
+ // order, and the ranked order replaces it once, when the ranker answers
9
+ //
10
+ // A newer ask cancels the older one: its timers stop, its ranker call is aborted, and nothing it
11
+ // finds afterwards is sent. Asks are known by their text and the state version they were made in
12
+ // (its tags and the map's dependencies), and by the tags it runs with, so a change of tags is a new ask too. Once the session's spend cap is
13
+ // reached, an ask that would rank shows the narrowing's order and a `capped` event says why.
14
+ import { orderByFit, RankerError } from "./ranker.js";
15
+ /** The spec's timings: narrow after 250 ms, rank after 1.5 s idle, wait 300 ms for the ranker. */
16
+ export const ASK_TIMING = { debounceMs: 250, idleMs: 1500, budgetMs: 300 };
17
+ /** An ask's key: its version, its tags and its text. The version moves when the page's tags do, but a ticked card's ask can be ranked with the tags it was ticked with. */
18
+ const keyOf = ({ text, version, tags }) => `${version}\n${JSON.stringify((tags ?? []).map(({ facet, value, pinned }) => [facet, value, pinned === true]))}\n${text.trim()}`;
19
+ /** How many past asks' rankings are kept: going back to an earlier ask costs no call. */
20
+ const KEPT_RANKINGS = 50;
21
+ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
22
+ export function createAsker(options) {
23
+ const timing = { ...ASK_TIMING, ...options.timing };
24
+ const kept = new Map();
25
+ let current;
26
+ const { budget } = options;
27
+ const stale = (ask) => current !== ask;
28
+ const narrowNow = (ask) => {
29
+ if (ask.narrowing)
30
+ return ask.narrowing;
31
+ const { text, tags, dependencies } = ask.request;
32
+ const narrowing = options.narrow(text, { ...(tags ? { tags } : {}), dependencies });
33
+ ask.narrowing = narrowing;
34
+ // A failed narrowing isn't kept either: the next Enter or idle pause on this ask narrows again.
35
+ narrowing.catch(() => {
36
+ if (ask.narrowing === narrowing)
37
+ delete ask.narrowing;
38
+ });
39
+ return narrowing;
40
+ };
41
+ const showNarrowing = async (ask) => {
42
+ const narrowing = await narrowNow(ask);
43
+ if (stale(ask) || ask.shown)
44
+ return;
45
+ ask.shown = "narrowed";
46
+ options.emit({ type: "narrowed", text: ask.request.text, version: ask.request.version, candidates: narrowing.candidates, ms: narrowing.timings.total, cut: narrowing.cut });
47
+ };
48
+ const rankingOf = (ask, ranker) => {
49
+ if (ask.ranking)
50
+ return ask.ranking;
51
+ const known = kept.get(ask.key);
52
+ if (known?.ranking.ranker === ranker.name)
53
+ return (ask.ranking = Promise.resolve(known));
54
+ const started = performance.now();
55
+ ask.ranking = (async () => {
56
+ const { candidates: narrowed } = await narrowNow(ask);
57
+ const { text, object, tags } = ask.request;
58
+ let ranking;
59
+ try {
60
+ ranking = narrowed.length ? await ranker.rank({ ask: text, object, tags, candidates: narrowed.map(({ card }) => card) }, { signal: ask.controller.signal }) : undefined;
61
+ }
62
+ catch (error) {
63
+ // A call answered (and paid for) but not read still counts against the cap.
64
+ if (error instanceof RankerError && error.cost > 0)
65
+ budget?.record(error.cost);
66
+ throw error;
67
+ }
68
+ if (ranking) {
69
+ budget?.record(ranking.cost);
70
+ options.onRanking?.(ranking);
71
+ }
72
+ const candidates = ranking ? orderByFit(narrowed, ranking.grades) : [];
73
+ const result = {
74
+ ranking: ranking ?? { ranker: ranker.name, model: "", grades: new Map(), timings: { request: 0, total: 0 }, usage: { input: 0, output: 0 }, cost: 0 },
75
+ tags: tags ?? [],
76
+ candidates,
77
+ ms: performance.now() - started,
78
+ };
79
+ kept.delete(ask.key);
80
+ kept.set(ask.key, result);
81
+ if (kept.size > KEPT_RANKINGS)
82
+ kept.delete(kept.keys().next().value);
83
+ return result;
84
+ })();
85
+ // A ranking no one waits for (it was cancelled) mustn't be an unhandled rejection.
86
+ ask.ranking.catch(() => { });
87
+ return ask.ranking;
88
+ };
89
+ /** Whether the ask's ranking is known, so no call is needed for it. */
90
+ const known = (ask, ranker) => ask.ranking !== undefined || kept.get(ask.key)?.ranking.ranker === ranker.name;
91
+ const rankNow = async (ask) => {
92
+ const { ranker } = options;
93
+ // With no ranker, or none allowed, a failed narrowing is the ask's only failure: said, not dropped.
94
+ const narrowOrSay = () => showNarrowing(ask).then(() => true, (error) => {
95
+ if (!stale(ask))
96
+ options.emit({ type: "failed", text: ask.request.text, version: ask.request.version, error: error.message, narrowed: false });
97
+ return false;
98
+ });
99
+ if (!ranker)
100
+ return void (await narrowOrSay());
101
+ if (ask.rankStarted)
102
+ return;
103
+ ask.rankStarted = true;
104
+ if (budget && !budget.allows() && !known(ask, ranker)) {
105
+ if (!(await narrowOrSay())) {
106
+ // Not capped yet as far as this ask goes: the next Enter narrows again.
107
+ ask.rankStarted = false;
108
+ return;
109
+ }
110
+ if (!stale(ask))
111
+ options.emit({ type: "capped", text: ask.request.text, version: ask.request.version, reason: budget.reason });
112
+ return;
113
+ }
114
+ const ranking = rankingOf(ask, ranker);
115
+ const late = await Promise.race([ranking.then(() => false, () => false), sleep(timing.budgetMs).then(() => true)]);
116
+ try {
117
+ if (late)
118
+ await showNarrowing(ask);
119
+ const { candidates, ranking: answer, ms, tags } = await ranking;
120
+ if (stale(ask))
121
+ return;
122
+ ask.shown = "ranked";
123
+ options.emit({ type: "ranked", text: ask.request.text, version: ask.request.version, ranker: ranker.name, tags, candidates, ms, cost: answer.cost });
124
+ }
125
+ catch (error) {
126
+ if (stale(ask))
127
+ return;
128
+ // A failed ranking isn't kept: the next Enter or idle pause on this ask calls the ranker again.
129
+ delete ask.ranking;
130
+ ask.rankStarted = false;
131
+ // The narrowing's order is shown when the narrowing worked; when it failed, it isn't run again now.
132
+ const narrowed = ask.narrowing ? await showNarrowing(ask).then(() => true, () => false) : false;
133
+ if (stale(ask))
134
+ return;
135
+ options.emit({ type: "failed", text: ask.request.text, version: ask.request.version, ranker: ranker.name, error: error.message, narrowed });
136
+ }
137
+ };
138
+ const later = (ask, timer, ms, run) => {
139
+ clearTimeout(ask[timer]);
140
+ ask[timer] = setTimeout(() => void run(ask).catch(() => { }), ms);
141
+ };
142
+ const stopTimers = (ask) => {
143
+ clearTimeout(ask.debounce);
144
+ clearTimeout(ask.idle);
145
+ };
146
+ const cancel = (ask) => {
147
+ if (!ask)
148
+ return;
149
+ stopTimers(ask);
150
+ ask.controller.abort(new Error("A newer ask replaced this one"));
151
+ };
152
+ return {
153
+ ask(request) {
154
+ const key = keyOf(request);
155
+ let ask = current;
156
+ if (ask?.key !== key) {
157
+ cancel(current);
158
+ current = undefined;
159
+ if (!request.text.trim())
160
+ return;
161
+ ask = { key, request, controller: new AbortController() };
162
+ current = ask;
163
+ if (request.mode === "type")
164
+ later(ask, "debounce", timing.debounceMs, showNarrowing);
165
+ }
166
+ const idle = request.idleMs ?? timing.idleMs;
167
+ if (request.mode === "type") {
168
+ // The same ask typed again (a trailing space): the idle wait starts over.
169
+ if (!ask.rankStarted)
170
+ later(ask, "idle", idle, rankNow);
171
+ }
172
+ else if (request.mode === "enter") {
173
+ stopTimers(ask);
174
+ void rankNow(ask).catch(() => { });
175
+ }
176
+ else {
177
+ stopTimers(ask);
178
+ void showNarrowing(ask).catch(() => { });
179
+ if (!ask.rankStarted)
180
+ later(ask, "idle", idle, rankNow);
181
+ }
182
+ },
183
+ async ranked(request) {
184
+ const key = keyOf(request);
185
+ const known = kept.get(key);
186
+ if (known)
187
+ return known;
188
+ const { ranker } = options;
189
+ if (!ranker)
190
+ throw new Error("No ranker is on");
191
+ // A ranking already under way for the current ask was paid for before the cap was reached.
192
+ if (budget && !budget.allows() && !(current?.key === key && current.ranking))
193
+ throw new Error(`No ranking: ${budget.reason}`);
194
+ if (current?.key === key) {
195
+ const ask = current;
196
+ void rankNow(ask).catch(() => { });
197
+ try {
198
+ return await rankingOf(ask, ranker);
199
+ }
200
+ catch (error) {
201
+ // A newer ask cancelled it: the tick still needs it, so it is ranked on its own below.
202
+ if (!ask.controller.signal.aborted)
203
+ throw error;
204
+ }
205
+ }
206
+ // An ask no longer current: ranked on its own, its results not sent.
207
+ return rankingOf({ key, request, controller: new AbortController() }, ranker);
208
+ },
209
+ close() {
210
+ cancel(current);
211
+ current = undefined;
212
+ },
213
+ };
214
+ }
@@ -0,0 +1,4 @@
1
+ import { type CachePlatform } from "@team-antioch/sc2-asset-catalog/catalog/catalog-dirs.js";
2
+ export type { CachePlatform } from "@team-antioch/sc2-asset-catalog/catalog/catalog-dirs.js";
3
+ /** Where the model's weights are kept: $SC2_ASSET_CATALOG_MODEL_CACHE (or, for sc2-adjutant, $SC2_ADJUTANT_MODEL_CACHE), else the cache folder's models. */
4
+ export declare function modelCacheDir(on?: CachePlatform): string;
@@ -0,0 +1,17 @@
1
+ // The Finder keeps its files in the catalog's cache folder (catalogCacheDir, catalog-dirs.ts in
2
+ // @team-antioch/sc2-asset-catalog), on each platform where that platform keeps caches: %LOCALAPPDATA%
3
+ // on Windows, ~/Library/Caches on macOS, $XDG_CACHE_HOME (else ~/.cache) elsewhere, or
4
+ // SC2_ASSET_CATALOG_CACHE on any of them. It holds the extracted catalog fields, the embeddings
5
+ // model's weights, the card vectors, Jev's payload logs and the page's settings: all of it a copy
6
+ // of something else, safe to delete.
7
+ import path from "node:path";
8
+ import { catalogCacheDir } from "@team-antioch/sc2-asset-catalog/catalog/catalog-dirs.js";
9
+ /** Where the model's weights are kept: $SC2_ASSET_CATALOG_MODEL_CACHE (or, for sc2-adjutant, $SC2_ADJUTANT_MODEL_CACHE), else the cache folder's models. */
10
+ export function modelCacheDir(on = {}) {
11
+ const env = on.env ?? process.env;
12
+ const moved = env.SC2_ASSET_CATALOG_MODEL_CACHE || env.SC2_ADJUTANT_MODEL_CACHE;
13
+ if (moved)
14
+ return moved;
15
+ const dir = catalogCacheDir(on);
16
+ return (on.platform ?? process.platform) === "win32" ? path.win32.join(dir, "models") : path.posix.join(dir, "models");
17
+ }
package/dist/card.d.ts ADDED
@@ -0,0 +1,35 @@
1
+ /** A candidate as the kit sees it: an id, a name and the few lines of text that are embedded and ranked. */
2
+ export type FinderCard = {
3
+ readonly id: string;
4
+ readonly name: string;
5
+ /** The card's text: what the ranker reads and the keyword search may use; the embeddings model embeds it too unless `embedText` is set. */
6
+ readonly text: string;
7
+ /** What the embeddings model embeds when that differs from `text`; absent, it embeds `text` (see `embeddedText`). */
8
+ readonly embedText?: string;
9
+ /** The map dependency it needs, absent when the default brings it in. */
10
+ readonly dependency?: string;
11
+ /** For a map's dependencies: whether its dependencies already bring it in. */
12
+ readonly provided?: boolean;
13
+ };
14
+ /** What the embeddings model reads of a card: its `embedText`, else its `text`. The one place that is decided. */
15
+ export declare const embeddedText: (card: FinderCard) => string;
16
+ /** A facet tag, such as `story: industry`. A pinned tag filters; the rest only raise. */
17
+ export type Tag = {
18
+ readonly facet: string;
19
+ readonly value: string;
20
+ readonly pinned?: boolean | undefined;
21
+ };
22
+ /** A tag the mapper has chosen: pinned (a filter) or dropped (hidden). */
23
+ export type TagChoice = {
24
+ readonly facet: string;
25
+ readonly value: string;
26
+ };
27
+ export type TagChoices = {
28
+ readonly pinned?: readonly TagChoice[];
29
+ readonly dropped?: readonly TagChoice[];
30
+ };
31
+ /** A shown tag: pinned or not, and how many ticked cards have it. */
32
+ export type ObjectTag = Tag & {
33
+ readonly pinned: boolean;
34
+ readonly count: number;
35
+ };
package/dist/card.js ADDED
@@ -0,0 +1,4 @@
1
+ // What the finder kit asks of a card, whatever it is a card of: a doodad, a track. A finder's own
2
+ // card type extends this and adds what its page shows and its tags are made from.
3
+ /** What the embeddings model reads of a card: its `embedText`, else its `text`. The one place that is decided. */
4
+ export const embeddedText = (card) => card.embedText ?? card.text;
@@ -0,0 +1,34 @@
1
+ import { type ConfigPlatform } from "./key-file.ts";
2
+ import type { FinderCard } from "./card.ts";
3
+ import { type JevWording } from "./jev-ranker.ts";
4
+ import type { PayloadLog, Ranker } from "./ranker.ts";
5
+ /** Why Jev is off with no key, naming the package whose `login` gives one. */
6
+ export declare const noJevKey: (command: string) => string;
7
+ /** Where a key came from: what /status and the CLI may say, never the key. */
8
+ export type KeySource = "environment" | "key file";
9
+ /** Jev's key and where it came from: TYPESAFE_API_KEY, else the key file, else none. */
10
+ export declare function jevKey(on?: ConfigPlatform): {
11
+ readonly key: string;
12
+ readonly source: KeySource;
13
+ } | undefined;
14
+ /** Jev's key alone: TYPESAFE_API_KEY, else the key file, else none. */
15
+ export declare const jevApiKey: (on?: ConfigPlatform) => string | undefined;
16
+ /** The ranker when it has its credential, and where that came from, else why it is off. */
17
+ export type OpenedRanker<C extends FinderCard = FinderCard> = {
18
+ readonly ranker: Ranker<C>;
19
+ readonly keySource: KeySource;
20
+ readonly off?: undefined;
21
+ } | {
22
+ readonly ranker?: undefined;
23
+ readonly keySource?: undefined;
24
+ readonly off: string;
25
+ };
26
+ /** Where a key came from, in words. */
27
+ export declare const keySourceText: (source: KeySource) => string;
28
+ /** Jev with its key, else off: with no key, or (Jev being optional) a key file that can't be read. */
29
+ export declare function openRanker<C extends FinderCard = FinderCard>({ env, log, wording, command }: {
30
+ readonly env?: Readonly<Record<string, string | undefined>>;
31
+ readonly log?: PayloadLog;
32
+ readonly wording: JevWording;
33
+ readonly command: string;
34
+ }): OpenedRanker<C>;
@@ -0,0 +1,33 @@
1
+ // The ranker's credential (docs/adr/0001), held by the Finder's process alone: Jev's key from
2
+ // TYPESAFE_API_KEY in the environment, else the key file a finder's `login` saved
3
+ // (key-file.ts), on every platform. The variable wins. With no key, Jev is off, with a message
4
+ // saying how to give it one, and asks are narrowed only.
5
+ import { keyFile, readKeyFile } from "./key-file.js";
6
+ import { jevRanker } from "./jev-ranker.js";
7
+ import { npxHint } from "@team-antioch/sc2-asset-catalog/util/npx.js";
8
+ /** Why Jev is off with no key, naming the package whose `login` gives one. */
9
+ export const noJevKey = (command) => `no TypeSafe key: run \`${npxHint(`${command} login`)}\`, or set TYPESAFE_API_KEY, to rank with Jev; until then the narrowing's order is shown`;
10
+ /** Jev's key and where it came from: TYPESAFE_API_KEY, else the key file, else none. */
11
+ export function jevKey(on = {}) {
12
+ const fromEnv = (on.env ?? process.env).TYPESAFE_API_KEY?.trim();
13
+ if (fromEnv)
14
+ return { key: fromEnv, source: "environment" };
15
+ const fromFile = readKeyFile(on);
16
+ return fromFile ? { key: fromFile, source: "key file" } : undefined;
17
+ }
18
+ /** Jev's key alone: TYPESAFE_API_KEY, else the key file, else none. */
19
+ export const jevApiKey = (on = {}) => jevKey(on)?.key;
20
+ /** Where a key came from, in words. */
21
+ export const keySourceText = (source) => (source === "environment" ? "the key in TYPESAFE_API_KEY" : "the saved key");
22
+ /** Jev with its key, else off: with no key, or (Jev being optional) a key file that can't be read. */
23
+ export function openRanker({ env, log, wording, command }) {
24
+ const on = env ? { env } : {};
25
+ let found;
26
+ try {
27
+ found = jevKey(on);
28
+ }
29
+ catch (error) {
30
+ return { off: `can't read the key file ${keyFile(on)} (${error.message}): run \`${npxHint(`${command} login`)}\` again, or set TYPESAFE_API_KEY` };
31
+ }
32
+ return found ? { ranker: jevRanker({ apiKey: found.key, wording, ...(log ? { log } : {}) }), keySource: found.source } : { off: noJevKey(command) };
33
+ }
@@ -0,0 +1,39 @@
1
+ export { modelCacheDir } from "./cache-dir.ts";
2
+ /** Embeds texts as unit vectors: an ask as a query, a card as a passage. */
3
+ export type Embedder = {
4
+ /** The model's name: vectors from two models aren't comparable, so the cache keeps them apart. */
5
+ readonly model: string;
6
+ embed(texts: readonly string[], as: "query" | "passage"): Promise<Float32Array[]>;
7
+ };
8
+ export declare const FINDER_MODEL = "Xenova/gte-small";
9
+ /**
10
+ * How each model the test command can compare pools its tokens into one vector, and what it puts
11
+ * before a query: bge pools by its first token and asks for an instruction before a short query
12
+ * searched against passages; gte averages its tokens and takes none.
13
+ */
14
+ export declare const MODELS: Readonly<Record<string, {
15
+ readonly pooling: "cls" | "mean";
16
+ readonly queryPrefix: string;
17
+ }>>;
18
+ /**
19
+ * On Windows, ONNX Runtime's native files need the Microsoft Visual C++ runtime (vcruntime140.dll,
20
+ * msvcp140.dll), which a fresh Windows, and a fresh Node, don't have; loading them then fails with
21
+ * only "The specified module could not be found". The error that says what to install, for that
22
+ * failure; else none.
23
+ */
24
+ export declare function missingRuntime(error: unknown, { platform, arch, hasRuntime }?: {
25
+ readonly platform?: NodeJS.Platform;
26
+ readonly arch?: string;
27
+ readonly hasRuntime?: () => boolean;
28
+ }): Error | undefined;
29
+ /** The embeddings model can't load on this machine until something is installed: no use waiting on the rest of a start. */
30
+ export declare class MissingRuntimeError extends Error {
31
+ name: string;
32
+ }
33
+ /** Why the model didn't load, in words: the connection when a download failed, else the cache. */
34
+ export declare function modelLoadError(model: string, error: unknown, cacheDir: string): Error;
35
+ /** The local model, loaded once (downloading its weights the first time). */
36
+ export declare function localEmbedder(options?: {
37
+ readonly model?: string;
38
+ readonly cacheDir?: string;
39
+ }): Promise<Embedder>;
@@ -0,0 +1,94 @@
1
+ // The finder kit's embeddings model (docs/adr/0001): it runs on this machine, with no network call
2
+ // once its weights are downloaded. The model is Alibaba's gte-small (384 dimensions, about 34 MB
3
+ // quantized to 8 bits), run by transformers.js on ONNX Runtime for Node. On the test asks
4
+ // it found as many candidates as bge-small and bge-base, and more of the top-ranked than
5
+ // bge-small, at the fastest narrowing (the test command's --model compares them). Its weights
6
+ // download to a machine cache the first time it runs, never into the repo. Tests use a fake behind
7
+ // the same interface, so they need no model.
8
+ import { existsSync } from "node:fs";
9
+ import path from "node:path";
10
+ import { modelCacheDir } from "./cache-dir.js";
11
+ export { modelCacheDir } from "./cache-dir.js";
12
+ export const FINDER_MODEL = "Xenova/gte-small";
13
+ /**
14
+ * How each model the test command can compare pools its tokens into one vector, and what it puts
15
+ * before a query: bge pools by its first token and asks for an instruction before a short query
16
+ * searched against passages; gte averages its tokens and takes none.
17
+ */
18
+ export const MODELS = {
19
+ "Xenova/bge-small-en-v1.5": { pooling: "cls", queryPrefix: "Represent this sentence for searching relevant passages: " },
20
+ "Xenova/bge-base-en-v1.5": { pooling: "cls", queryPrefix: "Represent this sentence for searching relevant passages: " },
21
+ "Xenova/gte-small": { pooling: "mean", queryPrefix: "" },
22
+ "Xenova/gte-base": { pooling: "mean", queryPrefix: "" },
23
+ };
24
+ /**
25
+ * On Windows, ONNX Runtime's native files need the Microsoft Visual C++ runtime (vcruntime140.dll,
26
+ * msvcp140.dll), which a fresh Windows, and a fresh Node, don't have; loading them then fails with
27
+ * only "The specified module could not be found". The error that says what to install, for that
28
+ * failure; else none.
29
+ */
30
+ export function missingRuntime(error, { platform = process.platform, arch = process.arch, hasRuntime = runtimeInstalled } = {}) {
31
+ const message = error instanceof Error ? error.message : String(error);
32
+ if (platform !== "win32" || !/specified module could not be found/i.test(message) || !/onnxruntime/i.test(message))
33
+ return undefined;
34
+ // The same words come for any DLL missing (onnxruntime.dll quarantined, a partial npx cache):
35
+ // the runtime is only blamed when its own DLLs aren't there.
36
+ if (hasRuntime())
37
+ return undefined;
38
+ const redist = arch === "arm64" ? "arm64" : "x64";
39
+ return new MissingRuntimeError(`The embeddings model can't load: ONNX Runtime needs the Microsoft Visual C++ Redistributable (${redist}, to match this Node), which isn't installed or is too old. Install it from https://aka.ms/vs/17/release/vc_redist.${redist}.exe (or \`winget install Microsoft.VCRedist.2015+.${redist}\`), then start the Finder again. (ONNX Runtime said: ${message.split(/\r?\n/)[0]})`, { cause: error });
40
+ }
41
+ /** The embeddings model can't load on this machine until something is installed: no use waiting on the rest of a start. */
42
+ export class MissingRuntimeError extends Error {
43
+ name = "MissingRuntimeError";
44
+ }
45
+ /** Whether the Visual C++ runtime's DLLs are in Windows's System32. */
46
+ function runtimeInstalled() {
47
+ const system32 = path.join(process.env.SystemRoot ?? "C:\\Windows", "System32");
48
+ // vcruntime140_1.dll came with the 2019 runtime: a 2015 or 2017 one has the other two without it.
49
+ return ["vcruntime140.dll", "vcruntime140_1.dll", "msvcp140.dll"].every((dll) => existsSync(path.join(system32, dll)));
50
+ }
51
+ /** A network failure's words and codes, as fetch and Node give them. */
52
+ const NETWORK = /fetch failed|network|ENOTFOUND|EAI_AGAIN|ECONNRESET|ECONNREFUSED|ETIMEDOUT|getaddrinfo|socket hang up/i;
53
+ /** Why the model didn't load, in words: the connection when a download failed, else the cache. */
54
+ export function modelLoadError(model, error, cacheDir) {
55
+ const cause = error instanceof Error ? error : new Error(String(error));
56
+ const said = `${cause.message} ${cause.cause?.code ?? ""}`;
57
+ const message = cause.message.replace(/[.\s]+$/, "");
58
+ return new Error(NETWORK.test(said)
59
+ ? `The embeddings model (${model}) couldn't download: ${message}. Check the connection and start again: it downloads once.`
60
+ : `The embeddings model (${model}) couldn't load: ${message}. If it keeps failing, delete its cache folder, ${cacheDir}, and start again to download it afresh.`, { cause: error });
61
+ }
62
+ /** At most this many texts per call to the model, so a cold start's thousands of cards don't make one huge batch. */
63
+ const BATCH = 32;
64
+ /** The local model, loaded once (downloading its weights the first time). */
65
+ export async function localEmbedder(options = {}) {
66
+ const model = options.model ?? FINDER_MODEL;
67
+ const settings = MODELS[model];
68
+ if (!settings)
69
+ throw new Error(`No settings for the embeddings model ${model}: it takes one of ${Object.keys(MODELS).join(", ")}`);
70
+ let extract;
71
+ try {
72
+ const { env, pipeline } = await import("@huggingface/transformers");
73
+ env.cacheDir = options.cacheDir ?? modelCacheDir();
74
+ extract = await pipeline("feature-extraction", model, { dtype: "q8" });
75
+ }
76
+ catch (error) {
77
+ throw missingRuntime(error) ?? modelLoadError(model, error, options.cacheDir ?? modelCacheDir());
78
+ }
79
+ return {
80
+ model,
81
+ async embed(texts, as) {
82
+ const vectors = [];
83
+ for (let start = 0; start < texts.length; start += BATCH) {
84
+ const batch = texts.slice(start, start + BATCH).map((text) => (as === "query" ? settings.queryPrefix + text : text));
85
+ const output = await extract(batch, { pooling: settings.pooling, normalize: true });
86
+ const [rows, dimensions] = output.dims;
87
+ const data = output.data;
88
+ for (let row = 0; row < rows; row++)
89
+ vectors.push(data.slice(row * dimensions, (row + 1) * dimensions));
90
+ }
91
+ return vectors;
92
+ },
93
+ };
94
+ }