@team-antioch/sc2-doodad-finder 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/bin/sc2-doodad-finder.js +4 -2
- package/dist/asker.d.ts +4 -1
- package/dist/asker.js +14 -5
- package/dist/cli/doodad-finder.js +1 -2
- package/dist/credentials.d.ts +1 -1
- package/dist/credentials.js +3 -2
- package/dist/embedder.d.ts +17 -0
- package/dist/embedder.js +49 -3
- package/dist/embedding-index.js +2 -1
- package/dist/jev-ranker.js +13 -2
- package/dist/login.js +3 -2
- package/dist/page.js +1 -1
- package/dist/server.d.ts +7 -0
- package/dist/server.js +23 -2
- package/dist/start.d.ts +3 -3
- package/dist/start.js +41 -4
- package/dist/stock-sources.js +2 -1
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -6,6 +6,8 @@ Ask for StarCraft II doodads in plain words and see ranked cards at once: a loca
|
|
|
6
6
|
npx @team-antioch/sc2-doodad-finder --open
|
|
7
7
|
```
|
|
8
8
|
|
|
9
|
+
On Windows, type `npx.cmd` in place of `npx` (PowerShell's default policy refuses `npx`), and install the [Microsoft Visual C++ Redistributable](https://learn.microsoft.com/cpp/windows/latest-supported-vc-redist) first if the machine doesn't have it: the embeddings model needs it. Ctrl-C stops the Finder; answer Y if Windows then asks "Terminate batch job (Y/N)?".
|
|
10
|
+
|
|
9
11
|
The first start extracts the catalog's fields from SC2GameData into your cache folder and downloads the embeddings model. `npx @team-antioch/sc2-doodad-finder extract` extracts again. `login` asks for your TypeSafe key and saves it in your config folder, and `logout` deletes it. Options: `--port <n>`, `--budget <dollars>` (Jev's spend cap per session, $1.00 by default), `--tick-log <folder>`, `--open`.
|
|
10
12
|
|
|
11
13
|
Getting a TypeSafe key, where it is kept, what each ask sends, the cost and the cap, the cache folders and the platforms: see the [repo's README](https://github.com/TeamAntioch/sc2-asset-catalog#readme).
|
package/bin/sc2-doodad-finder.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { main } from "../dist/cli/doodad-finder.js";
|
|
3
3
|
|
|
4
|
+
// A failed start exits once its message is written: a download still under way (the model's, the
|
|
5
|
+
// catalog's) would otherwise hold the terminal until it finished. Lines already sent to stdout
|
|
6
|
+
// are written first.
|
|
4
7
|
main().catch((error) => {
|
|
5
|
-
|
|
6
|
-
process.exitCode = 1;
|
|
8
|
+
process.stdout.write("", () => process.stderr.write(`${error instanceof Error ? error.message : error}\n`, () => process.exit(1)));
|
|
7
9
|
});
|
package/dist/asker.d.ts
CHANGED
|
@@ -44,8 +44,11 @@ export type AskEvent = {
|
|
|
44
44
|
readonly type: "failed";
|
|
45
45
|
readonly text: string;
|
|
46
46
|
readonly version: number;
|
|
47
|
-
|
|
47
|
+
/** The ranker the ask was for; none when no ranker is on. */
|
|
48
|
+
readonly ranker?: RankerName;
|
|
48
49
|
readonly error: string;
|
|
50
|
+
/** Whether the narrowing's order is shown: false when the narrowing itself failed. */
|
|
51
|
+
readonly narrowed: boolean;
|
|
49
52
|
} | {
|
|
50
53
|
readonly type: "capped";
|
|
51
54
|
readonly text: string;
|
package/dist/asker.js
CHANGED
|
@@ -90,13 +90,23 @@ export function createAsker(options) {
|
|
|
90
90
|
const known = (ask, ranker) => ask.ranking !== undefined || kept.get(ask.key)?.ranking.ranker === ranker.name;
|
|
91
91
|
const rankNow = async (ask) => {
|
|
92
92
|
const { ranker } = options;
|
|
93
|
+
// With no ranker, or none allowed, a failed narrowing is the ask's only failure: said, not dropped.
|
|
94
|
+
const narrowOrSay = () => showNarrowing(ask).then(() => true, (error) => {
|
|
95
|
+
if (!stale(ask))
|
|
96
|
+
options.emit({ type: "failed", text: ask.request.text, version: ask.request.version, error: error.message, narrowed: false });
|
|
97
|
+
return false;
|
|
98
|
+
});
|
|
93
99
|
if (!ranker)
|
|
94
|
-
return
|
|
100
|
+
return void (await narrowOrSay());
|
|
95
101
|
if (ask.rankStarted)
|
|
96
102
|
return;
|
|
97
103
|
ask.rankStarted = true;
|
|
98
104
|
if (budget && !budget.allows() && !known(ask, ranker)) {
|
|
99
|
-
await
|
|
105
|
+
if (!(await narrowOrSay())) {
|
|
106
|
+
// Not capped yet as far as this ask goes: the next Enter narrows again.
|
|
107
|
+
ask.rankStarted = false;
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
100
110
|
if (!stale(ask))
|
|
101
111
|
options.emit({ type: "capped", text: ask.request.text, version: ask.request.version, reason: budget.reason });
|
|
102
112
|
return;
|
|
@@ -119,11 +129,10 @@ export function createAsker(options) {
|
|
|
119
129
|
delete ask.ranking;
|
|
120
130
|
ask.rankStarted = false;
|
|
121
131
|
// The narrowing's order is shown when the narrowing worked; when it failed, it isn't run again now.
|
|
122
|
-
|
|
123
|
-
await showNarrowing(ask).catch(() => { });
|
|
132
|
+
const narrowed = ask.narrowing ? await showNarrowing(ask).then(() => true, () => false) : false;
|
|
124
133
|
if (stale(ask))
|
|
125
134
|
return;
|
|
126
|
-
options.emit({ type: "failed", text: ask.request.text, version: ask.request.version, ranker: ranker.name, error: error.message });
|
|
135
|
+
options.emit({ type: "failed", text: ask.request.text, version: ask.request.version, ranker: ranker.name, error: error.message, narrowed });
|
|
127
136
|
}
|
|
128
137
|
};
|
|
129
138
|
const later = (ask, timer, ms, run) => {
|
|
@@ -89,7 +89,6 @@ export async function main(argv = process.argv.slice(2)) {
|
|
|
89
89
|
// Run as a file in a checkout (node src/cli/doodad-finder.ts); the package's bin calls main.
|
|
90
90
|
if (isMainModule(import.meta.url)) {
|
|
91
91
|
main().catch((error) => {
|
|
92
|
-
|
|
93
|
-
process.exitCode = 1;
|
|
92
|
+
process.stdout.write("", () => process.stderr.write(`${error instanceof Error ? error.message : error}\n`, () => process.exit(1)));
|
|
94
93
|
});
|
|
95
94
|
}
|
package/dist/credentials.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type ConfigPlatform } from "./key-file.ts";
|
|
2
2
|
import type { PayloadLog, Ranker } from "./ranker.ts";
|
|
3
|
-
export declare const NO_JEV_KEY
|
|
3
|
+
export declare const NO_JEV_KEY: string;
|
|
4
4
|
/** Where a key came from: what /status and the CLI may say, never the key. */
|
|
5
5
|
export type KeySource = "environment" | "key file";
|
|
6
6
|
/** Jev's key and where it came from: TYPESAFE_API_KEY, else the key file, else none. */
|
package/dist/credentials.js
CHANGED
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
// saying how to give it one, and asks are narrowed only.
|
|
5
5
|
import { keyFile, readKeyFile } from "./key-file.js";
|
|
6
6
|
import { jevRanker } from "./jev-ranker.js";
|
|
7
|
-
|
|
7
|
+
import { npxHint } from "@team-antioch/sc2-asset-catalog/util/npx.js";
|
|
8
|
+
export const NO_JEV_KEY = `no TypeSafe key: run \`${npxHint("@team-antioch/sc2-doodad-finder login")}\`, or set TYPESAFE_API_KEY, to rank with Jev; until then the narrowing's order is shown`;
|
|
8
9
|
/** Jev's key and where it came from: TYPESAFE_API_KEY, else the key file, else none. */
|
|
9
10
|
export function jevKey(on = {}) {
|
|
10
11
|
const fromEnv = (on.env ?? process.env).TYPESAFE_API_KEY?.trim();
|
|
@@ -25,7 +26,7 @@ export function openRanker({ env, log } = {}) {
|
|
|
25
26
|
found = jevKey(on);
|
|
26
27
|
}
|
|
27
28
|
catch (error) {
|
|
28
|
-
return { off: `can't read the key file ${keyFile(on)} (${error.message}): run
|
|
29
|
+
return { off: `can't read the key file ${keyFile(on)} (${error.message}): run \`${npxHint("@team-antioch/sc2-doodad-finder login")}\` again, or set TYPESAFE_API_KEY` };
|
|
29
30
|
}
|
|
30
31
|
return found ? { ranker: jevRanker({ apiKey: found.key, ...(log ? { log } : {}) }), keySource: found.source } : { off: NO_JEV_KEY };
|
|
31
32
|
}
|
package/dist/embedder.d.ts
CHANGED
|
@@ -15,6 +15,23 @@ export declare const MODELS: Readonly<Record<string, {
|
|
|
15
15
|
readonly pooling: "cls" | "mean";
|
|
16
16
|
readonly queryPrefix: string;
|
|
17
17
|
}>>;
|
|
18
|
+
/**
|
|
19
|
+
* On Windows, ONNX Runtime's native files need the Microsoft Visual C++ runtime (vcruntime140.dll,
|
|
20
|
+
* msvcp140.dll), which a fresh Windows, and a fresh Node, don't have; loading them then fails with
|
|
21
|
+
* only "The specified module could not be found". The error that says what to install, for that
|
|
22
|
+
* failure; else none.
|
|
23
|
+
*/
|
|
24
|
+
export declare function missingRuntime(error: unknown, { platform, arch, hasRuntime }?: {
|
|
25
|
+
readonly platform?: NodeJS.Platform;
|
|
26
|
+
readonly arch?: string;
|
|
27
|
+
readonly hasRuntime?: () => boolean;
|
|
28
|
+
}): Error | undefined;
|
|
29
|
+
/** The embeddings model can't load on this machine until something is installed: no use waiting on the rest of a start. */
|
|
30
|
+
export declare class MissingRuntimeError extends Error {
|
|
31
|
+
name: string;
|
|
32
|
+
}
|
|
33
|
+
/** Why the model didn't load, in words: the connection when a download failed, else the cache. */
|
|
34
|
+
export declare function modelLoadError(model: string, error: unknown, cacheDir: string): Error;
|
|
18
35
|
/** The local model, loaded once (downloading its weights the first time). */
|
|
19
36
|
export declare function localEmbedder(options?: {
|
|
20
37
|
readonly model?: string;
|
package/dist/embedder.js
CHANGED
|
@@ -5,6 +5,8 @@
|
|
|
5
5
|
// bge-small, at the fastest narrowing (the test command's --model compares them). Its weights
|
|
6
6
|
// download to a machine cache the first time it runs, never into the repo. Tests use a fake behind
|
|
7
7
|
// the same interface, so they need no model.
|
|
8
|
+
import { existsSync } from "node:fs";
|
|
9
|
+
import path from "node:path";
|
|
8
10
|
import { modelCacheDir } from "./cache-dir.js";
|
|
9
11
|
export { modelCacheDir } from "./cache-dir.js";
|
|
10
12
|
export const FINDER_MODEL = "Xenova/gte-small";
|
|
@@ -19,6 +21,44 @@ export const MODELS = {
|
|
|
19
21
|
"Xenova/gte-small": { pooling: "mean", queryPrefix: "" },
|
|
20
22
|
"Xenova/gte-base": { pooling: "mean", queryPrefix: "" },
|
|
21
23
|
};
|
|
24
|
+
/**
|
|
25
|
+
* On Windows, ONNX Runtime's native files need the Microsoft Visual C++ runtime (vcruntime140.dll,
|
|
26
|
+
* msvcp140.dll), which a fresh Windows, and a fresh Node, don't have; loading them then fails with
|
|
27
|
+
* only "The specified module could not be found". The error that says what to install, for that
|
|
28
|
+
* failure; else none.
|
|
29
|
+
*/
|
|
30
|
+
export function missingRuntime(error, { platform = process.platform, arch = process.arch, hasRuntime = runtimeInstalled } = {}) {
|
|
31
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
32
|
+
if (platform !== "win32" || !/specified module could not be found/i.test(message) || !/onnxruntime/i.test(message))
|
|
33
|
+
return undefined;
|
|
34
|
+
// The same words come for any DLL missing (onnxruntime.dll quarantined, a partial npx cache):
|
|
35
|
+
// the runtime is only blamed when its own DLLs aren't there.
|
|
36
|
+
if (hasRuntime())
|
|
37
|
+
return undefined;
|
|
38
|
+
const redist = arch === "arm64" ? "arm64" : "x64";
|
|
39
|
+
return new MissingRuntimeError(`The embeddings model can't load: ONNX Runtime needs the Microsoft Visual C++ Redistributable (${redist}, to match this Node), which isn't installed or is too old. Install it from https://aka.ms/vs/17/release/vc_redist.${redist}.exe (or \`winget install Microsoft.VCRedist.2015+.${redist}\`), then start the Finder again. (ONNX Runtime said: ${message.split(/\r?\n/)[0]})`, { cause: error });
|
|
40
|
+
}
|
|
41
|
+
/** The embeddings model can't load on this machine until something is installed: no use waiting on the rest of a start. */
|
|
42
|
+
export class MissingRuntimeError extends Error {
|
|
43
|
+
name = "MissingRuntimeError";
|
|
44
|
+
}
|
|
45
|
+
/** Whether the Visual C++ runtime's DLLs are in Windows's System32. */
|
|
46
|
+
function runtimeInstalled() {
|
|
47
|
+
const system32 = path.join(process.env.SystemRoot ?? "C:\\Windows", "System32");
|
|
48
|
+
// vcruntime140_1.dll came with the 2019 runtime: a 2015 or 2017 one has the other two without it.
|
|
49
|
+
return ["vcruntime140.dll", "vcruntime140_1.dll", "msvcp140.dll"].every((dll) => existsSync(path.join(system32, dll)));
|
|
50
|
+
}
|
|
51
|
+
/** A network failure's words and codes, as fetch and Node give them. */
|
|
52
|
+
const NETWORK = /fetch failed|network|ENOTFOUND|EAI_AGAIN|ECONNRESET|ECONNREFUSED|ETIMEDOUT|getaddrinfo|socket hang up/i;
|
|
53
|
+
/** Why the model didn't load, in words: the connection when a download failed, else the cache. */
|
|
54
|
+
export function modelLoadError(model, error, cacheDir) {
|
|
55
|
+
const cause = error instanceof Error ? error : new Error(String(error));
|
|
56
|
+
const said = `${cause.message} ${cause.cause?.code ?? ""}`;
|
|
57
|
+
const message = cause.message.replace(/[.\s]+$/, "");
|
|
58
|
+
return new Error(NETWORK.test(said)
|
|
59
|
+
? `The embeddings model (${model}) couldn't download: ${message}. Check the connection and start again: it downloads once.`
|
|
60
|
+
: `The embeddings model (${model}) couldn't load: ${message}. If it keeps failing, delete its cache folder, ${cacheDir}, and start again to download it afresh.`, { cause: error });
|
|
61
|
+
}
|
|
22
62
|
/** At most this many texts per call to the model, so a cold start's thousands of cards don't make one huge batch. */
|
|
23
63
|
const BATCH = 32;
|
|
24
64
|
/** The local model, loaded once (downloading its weights the first time). */
|
|
@@ -27,9 +67,15 @@ export async function localEmbedder(options = {}) {
|
|
|
27
67
|
const settings = MODELS[model];
|
|
28
68
|
if (!settings)
|
|
29
69
|
throw new Error(`No settings for the embeddings model ${model}: it takes one of ${Object.keys(MODELS).join(", ")}`);
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
70
|
+
let extract;
|
|
71
|
+
try {
|
|
72
|
+
const { env, pipeline } = await import("@huggingface/transformers");
|
|
73
|
+
env.cacheDir = options.cacheDir ?? modelCacheDir();
|
|
74
|
+
extract = await pipeline("feature-extraction", model, { dtype: "q8" });
|
|
75
|
+
}
|
|
76
|
+
catch (error) {
|
|
77
|
+
throw missingRuntime(error) ?? modelLoadError(model, error, options.cacheDir ?? modelCacheDir());
|
|
78
|
+
}
|
|
33
79
|
return {
|
|
34
80
|
model,
|
|
35
81
|
async embed(texts, as) {
|
package/dist/embedding-index.js
CHANGED
|
@@ -91,7 +91,8 @@ export class EmbeddingIndex {
|
|
|
91
91
|
this.#nearest = nearest;
|
|
92
92
|
if (missing.size || forgot)
|
|
93
93
|
await writeCache(this.#file, this.#embedder.model, vectors);
|
|
94
|
-
|
|
94
|
+
// Counted in cards, not texts: cards that share a text share its one vector.
|
|
95
|
+
return { embedded: hashes.filter((hash) => missing.has(hash)).length, nearest };
|
|
95
96
|
}
|
|
96
97
|
/** The last update's cards most like a unit vector (see Nearest). */
|
|
97
98
|
nearest(query, k, keep) {
|
package/dist/jev-ranker.js
CHANGED
|
@@ -32,8 +32,19 @@ export function jevRanker(options) {
|
|
|
32
32
|
result = await client.systemOne(payload, signal ? { signal } : {});
|
|
33
33
|
}
|
|
34
34
|
catch (error) {
|
|
35
|
-
|
|
36
|
-
|
|
35
|
+
// The key never goes further than the client: not into the log, the terminal or the page.
|
|
36
|
+
const hide = (text) => (options.apiKey ? text.split(options.apiKey).join("[the key]") : text);
|
|
37
|
+
options.log?.({ ranker: "jev", at, ms: performance.now() - sent, request: payload, error: hide(String(error)) });
|
|
38
|
+
// The SDK's error is kept as a copy with the key hidden: its name, message and status.
|
|
39
|
+
// Its network code too (ENOTFOUND, say), from its own cause, so an outage still reads as one.
|
|
40
|
+
const failure = error;
|
|
41
|
+
const code = failure.cause?.code;
|
|
42
|
+
const cause = Object.assign(new Error(hide(failure.message)), {
|
|
43
|
+
name: failure.name,
|
|
44
|
+
...(failure.status !== undefined ? { status: failure.status } : {}),
|
|
45
|
+
...(typeof code === "string" ? { code } : {}),
|
|
46
|
+
});
|
|
47
|
+
throw new RankerError("jev", `the request failed: ${cause.message}`, { cause });
|
|
37
48
|
}
|
|
38
49
|
const answered = performance.now();
|
|
39
50
|
options.log?.({ ranker: "jev", at, ms: answered - sent, request: payload, response: result });
|
package/dist/login.js
CHANGED
|
@@ -8,6 +8,7 @@ import { createInterface } from "node:readline";
|
|
|
8
8
|
import { StringDecoder } from "node:string_decoder";
|
|
9
9
|
import { AuthenticationError, PermissionDeniedError, TypeSafeClient } from "@typesafe-ai/sdk";
|
|
10
10
|
import { keyFile, removeKeyFile, saveKeyFile } from "./key-file.js";
|
|
11
|
+
import { npxHint } from "@team-antioch/sc2-asset-catalog/util/npx.js";
|
|
11
12
|
/** Checks a key by listing the account's models (GET /v1/models): no Jev request, no tokens. */
|
|
12
13
|
export function typesafeKeyCheck({ fetch } = {}) {
|
|
13
14
|
return async (key) => {
|
|
@@ -99,7 +100,7 @@ export async function login(options = {}) {
|
|
|
99
100
|
}
|
|
100
101
|
else {
|
|
101
102
|
if (output.isTTY)
|
|
102
|
-
say(
|
|
103
|
+
say(`The input isn't a terminal, so the key will show as you paste it. Pipe it in instead: \`echo <key> | ${npxHint("@team-antioch/sc2-doodad-finder login")}\`. Or paste it and press Enter:`);
|
|
103
104
|
read = await readLine(input);
|
|
104
105
|
}
|
|
105
106
|
if (read === CANCELLED)
|
|
@@ -120,7 +121,7 @@ export async function login(options = {}) {
|
|
|
120
121
|
await saveKeyFile(key, options);
|
|
121
122
|
const file = keyFile(options);
|
|
122
123
|
say(checked.ok ? `TypeSafe accepted the key. Saved it to ${file}.` : hide(`Saved the key to ${file}, but couldn't check it: ${checked.reason}`));
|
|
123
|
-
say(
|
|
124
|
+
say(`The Finder ranks with Jev from its next start. \`${npxHint("@team-antioch/sc2-doodad-finder logout")}\` deletes the key.`);
|
|
124
125
|
if (env.TYPESAFE_API_KEY?.trim())
|
|
125
126
|
say("TYPESAFE_API_KEY is set here too, and while it is set it wins over the saved key.");
|
|
126
127
|
return 0;
|
package/dist/page.js
CHANGED
|
@@ -137,7 +137,7 @@ events.addEventListener("ranked", (event) => {
|
|
|
137
137
|
});
|
|
138
138
|
events.addEventListener("failed", (event) => {
|
|
139
139
|
const data = JSON.parse(event.data);
|
|
140
|
-
if (data.text.trim() === currentAsk()) showError(new Error(
|
|
140
|
+
if (data.text.trim() === currentAsk()) showError(new Error(data.message));
|
|
141
141
|
});
|
|
142
142
|
events.addEventListener("capped", (event) => {
|
|
143
143
|
const data = JSON.parse(event.data);
|
package/dist/server.d.ts
CHANGED
|
@@ -128,6 +128,8 @@ export type FinderServerOptions = {
|
|
|
128
128
|
readonly port?: number;
|
|
129
129
|
/** More for /status to say. */
|
|
130
130
|
readonly status?: Readonly<Record<string, unknown>>;
|
|
131
|
+
/** Lines worth showing the user as the Finder runs, such as a failed ranking. */
|
|
132
|
+
readonly log?: (line: string) => void;
|
|
131
133
|
};
|
|
132
134
|
export type FinderServer = {
|
|
133
135
|
readonly url: string;
|
|
@@ -135,6 +137,11 @@ export type FinderServer = {
|
|
|
135
137
|
readonly spend: number;
|
|
136
138
|
close(): Promise<void>;
|
|
137
139
|
};
|
|
140
|
+
/** A failed ask in words, the same on the page and in the log. */
|
|
141
|
+
export declare const failureText: ({ error, narrowed }: {
|
|
142
|
+
readonly error: string;
|
|
143
|
+
readonly narrowed: boolean;
|
|
144
|
+
}) => string;
|
|
138
145
|
/** A tick log's line: the ask, its tags and the ids in the tray ticked from it, in the order ticked. */
|
|
139
146
|
export type TickLogLine = {
|
|
140
147
|
readonly ask: string;
|
package/dist/server.js
CHANGED
|
@@ -39,6 +39,13 @@ import { objectTags } from "./tags.js";
|
|
|
39
39
|
import { itemCopyText, ticked, trayCopyText } from "./tray.js";
|
|
40
40
|
/** The port the page is at unless another is asked for. */
|
|
41
41
|
export const FINDER_PORT = 4634;
|
|
42
|
+
/** How often a failure that lasts is said again in the log. */
|
|
43
|
+
const FAILURE_REPEAT_MS = 60_000;
|
|
44
|
+
/** A failed ask in words, the same on the page and in the log. */
|
|
45
|
+
export const failureText = ({ error, narrowed }) => {
|
|
46
|
+
const said = error.replace(/[.\s]+$/, "");
|
|
47
|
+
return narrowed ? `The ranker failed: ${said}. The narrowing's order is shown.` : `The search failed: ${said}.`;
|
|
48
|
+
};
|
|
42
49
|
/** How many asks' grades are kept for ticking from them. */
|
|
43
50
|
const KEPT_GRADES = 50;
|
|
44
51
|
class HttpError extends Error {
|
|
@@ -129,12 +136,26 @@ export async function startFinderServer(options) {
|
|
|
129
136
|
...(level !== undefined ? { level: Math.round(level * 100) / 100, levelName: levelName(level) } : {}),
|
|
130
137
|
}));
|
|
131
138
|
};
|
|
139
|
+
/** The last failure the log said, and when, so a lasting one isn't said on every ask. */
|
|
140
|
+
let lastFailure;
|
|
132
141
|
const emit = (event) => {
|
|
133
|
-
if (event.type === "failed")
|
|
134
|
-
|
|
142
|
+
if (event.type === "failed") {
|
|
143
|
+
const line = failureText(event);
|
|
144
|
+
// A failure that lasts (a revoked key, an outage) comes again on every Enter and idle pause:
|
|
145
|
+
// said once a minute while it fails the same way, long numbers aside (a timeout's
|
|
146
|
+
// milliseconds; a status code still tells two failures apart). A ranking that works ends it.
|
|
147
|
+
const same = line.replace(/\d{4,}/g, "#");
|
|
148
|
+
const now = Date.now();
|
|
149
|
+
if (same !== lastFailure?.same || now - lastFailure.at >= FAILURE_REPEAT_MS) {
|
|
150
|
+
options.log?.(line);
|
|
151
|
+
lastFailure = { same, at: now };
|
|
152
|
+
}
|
|
153
|
+
return send("failed", { ...event, message: line });
|
|
154
|
+
}
|
|
135
155
|
if (event.type === "capped")
|
|
136
156
|
return send("capped", event);
|
|
137
157
|
if (event.type === "ranked") {
|
|
158
|
+
lastFailure = undefined;
|
|
138
159
|
grades.delete(event.text.trim());
|
|
139
160
|
grades.set(event.text.trim(), { tags: event.tags, levels: new Map(event.candidates.map(({ id, level }) => [id, level])) });
|
|
140
161
|
if (grades.size > KEPT_GRADES)
|
package/dist/start.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type ListGitTree } from "@team-antioch/sc2-asset-catalog/catalog/extract-catalog.js";
|
|
2
2
|
import { type KeySource } from "./credentials.ts";
|
|
3
3
|
import { type Embedder } from "./embedder.ts";
|
|
4
4
|
import type { Ranker } from "./ranker.ts";
|
|
@@ -30,8 +30,8 @@ export type StartFinderOptions = {
|
|
|
30
30
|
readonly cacheDir?: string;
|
|
31
31
|
/** Where the page's settings are kept (in the cache folder by default). */
|
|
32
32
|
readonly settingsFile?: string;
|
|
33
|
-
/** For tests: the embeddings model in place of the local one. */
|
|
34
|
-
readonly embedder?: Embedder;
|
|
33
|
+
/** For tests: the embeddings model in place of the local one, or what loads it. */
|
|
34
|
+
readonly embedder?: Embedder | (() => Promise<Embedder>);
|
|
35
35
|
/** For tests: the fetch a first start extracts the catalog with, and what lists the gallery when GitHub's API refuses. */
|
|
36
36
|
readonly fetch?: typeof globalThis.fetch;
|
|
37
37
|
readonly listGitTree?: ListGitTree;
|
package/dist/start.js
CHANGED
|
@@ -8,8 +8,9 @@
|
|
|
8
8
|
// here; nothing is discovered or loaded by name.
|
|
9
9
|
import path from "node:path";
|
|
10
10
|
import { catalogCacheDir } from "@team-antioch/sc2-asset-catalog/catalog/catalog-dirs.js";
|
|
11
|
+
import { gitTreeFiles } from "@team-antioch/sc2-asset-catalog/catalog/extract-catalog.js";
|
|
11
12
|
import { openRanker } from "./credentials.js";
|
|
12
|
-
import { localEmbedder } from "./embedder.js";
|
|
13
|
+
import { localEmbedder, MissingRuntimeError } from "./embedder.js";
|
|
13
14
|
import { openFinder, vectorCacheFile } from "./finder.js";
|
|
14
15
|
import { payloadLog, payloadLogFile } from "./payload-log.js";
|
|
15
16
|
import { FINDER_PORT, startFinderServer } from "./server.js";
|
|
@@ -21,12 +22,10 @@ export { ensureCatalog, stockSources } from "./stock-sources.js";
|
|
|
21
22
|
export async function startFinder(options = {}) {
|
|
22
23
|
const log = options.log ?? (() => { });
|
|
23
24
|
const cacheDir = options.cacheDir ?? catalogCacheDir();
|
|
24
|
-
|
|
25
|
-
await ensureCatalog(cacheDir, { log, ...(options.fetch ? { fetch: options.fetch } : {}), ...(options.listGitTree ? { listGitTree: options.listGitTree } : {}) });
|
|
25
|
+
const embedder = await loadTogether(cacheDir, options, log);
|
|
26
26
|
const sources = options.sources ?? stockSources(cacheDir);
|
|
27
27
|
const logFile = payloadLogFile("finder-", cacheDir);
|
|
28
28
|
const opened = options.ranker === null ? { off: "ranking is turned off" } : options.ranker ? { ranker: options.ranker } : openRanker({ ...(options.env ? { env: options.env } : {}), log: payloadLog(logFile) });
|
|
29
|
-
const embedder = options.embedder ?? (await localEmbedder());
|
|
30
29
|
const finder = await openFinder({
|
|
31
30
|
load: sources.load,
|
|
32
31
|
embedder,
|
|
@@ -53,6 +52,7 @@ export async function startFinder(options = {}) {
|
|
|
53
52
|
saveSettings: (settings) => writeFinderSettings(settingsFile, settings),
|
|
54
53
|
port: options.port ?? FINDER_PORT,
|
|
55
54
|
...(options.status ? { status: options.status } : {}),
|
|
55
|
+
log,
|
|
56
56
|
});
|
|
57
57
|
}
|
|
58
58
|
catch (error) {
|
|
@@ -75,3 +75,40 @@ export async function startFinder(options = {}) {
|
|
|
75
75
|
},
|
|
76
76
|
};
|
|
77
77
|
}
|
|
78
|
+
/**
|
|
79
|
+
* The catalog (extracted on a first start) and the model, loaded together: on a first start both
|
|
80
|
+
* download. Whichever fails first is said, and a machine that can't load the model (on Windows,
|
|
81
|
+
* without the Visual C++ runtime) is told so at once, the extraction stopped and quiet. Any other
|
|
82
|
+
* model failure waits on the catalog, whose own failure (offline, say) is the one worth saying.
|
|
83
|
+
*/
|
|
84
|
+
async function loadTogether(cacheDir, options, log) {
|
|
85
|
+
const stop = new AbortController();
|
|
86
|
+
const fetch = options.fetch ?? globalThis.fetch;
|
|
87
|
+
const stoppable = (input, init) => fetch(input, { ...init, signal: stop.signal });
|
|
88
|
+
const quiet = (line) => {
|
|
89
|
+
if (!stop.signal.aborted)
|
|
90
|
+
log(line);
|
|
91
|
+
};
|
|
92
|
+
// A stopped API listing must not fall back to a git clone that nothing could stop.
|
|
93
|
+
const listGitTree = (url, folder) => (stop.signal.aborted ? Promise.reject(new Error("stopped")) : (options.listGitTree ?? gitTreeFiles)(url, folder));
|
|
94
|
+
const catalog = options.sources ? Promise.resolve() : ensureCatalog(cacheDir, { log: quiet, fetch: stoppable, listGitTree });
|
|
95
|
+
const model = typeof options.embedder === "function" ? options.embedder() : options.embedder ? Promise.resolve(options.embedder) : localEmbedder();
|
|
96
|
+
const first = await Promise.race([
|
|
97
|
+
model.then((embedder) => ({ embedder }), (error) => ({ modelError: error })),
|
|
98
|
+
// The catalog's success doesn't end the race: only its failure does.
|
|
99
|
+
catalog.then(() => new Promise(() => { }), (error) => ({ catalogError: error })),
|
|
100
|
+
]);
|
|
101
|
+
if ("catalogError" in first)
|
|
102
|
+
throw first.catalogError;
|
|
103
|
+
if ("modelError" in first) {
|
|
104
|
+
if (first.modelError instanceof MissingRuntimeError) {
|
|
105
|
+
stop.abort();
|
|
106
|
+
catalog.catch(() => { });
|
|
107
|
+
throw first.modelError;
|
|
108
|
+
}
|
|
109
|
+
await catalog;
|
|
110
|
+
throw first.modelError;
|
|
111
|
+
}
|
|
112
|
+
await catalog;
|
|
113
|
+
return first.embedder;
|
|
114
|
+
}
|
package/dist/stock-sources.js
CHANGED
|
@@ -7,6 +7,7 @@ import { extractedDir } from "@team-antioch/sc2-asset-catalog/catalog/catalog-di
|
|
|
7
7
|
import { SHEET_FOLDERS } from "@team-antioch/sc2-asset-catalog/catalog/catalog-sheets.js";
|
|
8
8
|
import { readStockDoodadSources, stockDoodadSourceFiles } from "@team-antioch/sc2-asset-catalog/catalog/doodad-search.js";
|
|
9
9
|
import { extractCatalog, hasExtractedCatalog } from "@team-antioch/sc2-asset-catalog/catalog/extract-catalog.js";
|
|
10
|
+
import { npxHint } from "@team-antioch/sc2-asset-catalog/util/npx.js";
|
|
10
11
|
/**
|
|
11
12
|
* The stock catalog in a cache folder. Its sheets folder is watched too (made, if it isn't there, so
|
|
12
13
|
* it can be), so sheets rendered or removed while the Finder runs show on its cards.
|
|
@@ -29,6 +30,6 @@ export async function ensureCatalog(cacheDir, { log = () => { }, ...fetching } =
|
|
|
29
30
|
await extractCatalog({ extractedDir: dir, log, ...fetching });
|
|
30
31
|
}
|
|
31
32
|
catch (error) {
|
|
32
|
-
throw new Error(`Couldn't extract the catalog: ${error.message}. Check the connection, then run
|
|
33
|
+
throw new Error(`Couldn't extract the catalog: ${error.message}. Check the connection, then run ${npxHint("@team-antioch/sc2-doodad-finder extract")} (or sc2-doodad-finder extract, where it is installed)`, { cause: error });
|
|
33
34
|
}
|
|
34
35
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@team-antioch/sc2-doodad-finder",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.2",
|
|
4
4
|
"description": "Ask for StarCraft II doodads in plain words and see ranked cards at once: a local page that narrows with a keyword search and a local embeddings model, then ranks with Jev.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"starcraft",
|
|
@@ -51,7 +51,7 @@
|
|
|
51
51
|
},
|
|
52
52
|
"dependencies": {
|
|
53
53
|
"@huggingface/transformers": "^4.3.2",
|
|
54
|
-
"@team-antioch/sc2-asset-catalog": "0.1.
|
|
54
|
+
"@team-antioch/sc2-asset-catalog": "0.1.2",
|
|
55
55
|
"@typesafe-ai/sdk": "^0.7.0",
|
|
56
56
|
"zod": "^4.0.0"
|
|
57
57
|
}
|