dsh-llm-verifier 0.6.2 → 0.6.4

package/lib/types/pricing.js

@@ -0,0 +1,279 @@
+/**
+ * Automatic USD pricing for one judge route.
+ *
+ * Why this module exists: the harness ships no price to plugins. `@deepseek-ai/dsh-llm`
+ * exposes usage tokens and `resolveModel` metadata, never money, and `llm-pi-ai`
+ * deliberately zeroes pi-ai's own `cost` metadata ("The harness never reads pi-ai's cost
+ * metadata … no consumer reports spend"). Before this module the operator had to type two
+ * USD-per-million rates by hand and every statistics row stayed $0 until they did.
+ *
+ * Four sources, in order, each best-effort and independently degradable:
+ * 1. the operator's explicit rates (`estimated*UsdPerMillion`) — they always win;
+ * 2. the installed pi-ai provider catalog on disk (`@earendil-works/pi-ai/dist/providers/data`),
+ *    the same models.dev snapshot the host itself carries, cached in-process;
+ * 3. models.dev over HTTPS, cached for the same TTL, consulted only when 2 misses;
+ * 4. `none` — zero, which the statistics UI treats as "not priced" rather than as "free".
+ *
+ * A route may be a reseller whose provider id is in neither table (the bundled catalog does
+ * not ship `command-code`, and models.dev has no such provider). The optional
+ * `priceProviderOverride` makes that explicit: it names the provider whose list price the
+ * operator chose to follow. Nothing is ever guessed across providers — the same model id is
+ * listed at 0.10/0.40 by one vendor and 0.65/1.45 by another, so a silent cross-provider
+ * guess would put a fabricated number in a cost column.
+ *
+ * Every failure (package not installed, unreadable file, network error, unknown shape)
+ * degrades to the next source and finally to `none`: this module never throws, so automatic
+ * pricing can never block or fail a verification.
+ *
+ * @module dsh-llm-verifier/pricing
+ */
+import { existsSync, readFileSync, readdirSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+/** Rates for a route nothing could price. */
+export const UNPRICED = { input: 0, output: 0, cachedInput: 0, source: 'none' };
+/**
+ * USD cost of one usage figure under a rate table.
+ *
+ * Cached prompt tokens are billed at their own rate instead of the full input rate. The old
+ * formula added them to `inputTokens` and multiplied by the input price, which over-reported
+ * every cache-heavy call by the cache-read discount — on a real acceptance run
+ * (2,374 uncached + 134,912 cached input tokens) that is the difference between ~$0.0004 and
+ * an order of magnitude more.
+ * @param usage - token counts of the call.
+ * @param prices - rates in effect for the route that produced them.
+ * @returns The estimated cost in USD.
+ */
+export function costUsd(usage, prices) {
+    return (usage.inputTokens * prices.input + usage.cachedInputTokens * prices.cachedInput + usage.outputTokens * prices.output) / 1_000_000;
+}
+/** Identity of one route inside a {@link PriceTable}. */
+export function priceKey(provider, model) {
+    return provider + '\u0000' + model;
+}
+function object(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value) ? value : undefined;
+}
+function rate(value) {
+    return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : undefined;
+}
+/** Reads one `{input, output, cacheRead|cache_read}` record; `undefined` when nothing usable is there. */
+export function priceEntry(value) {
+    const row = object(value);
+    if (row === undefined)
+        return undefined;
+    const input = rate(row.input);
+    const output = rate(row.output);
+    if (input === undefined || output === undefined)
+        return undefined;
+    return { input, output, cachedInput: rate(row.cacheRead) ?? rate(row.cache_read) ?? input };
+}
+/**
+ * Merge one provider document into a table.
+ *
+ * Accepts both shapes this module reads: the pi-ai catalog (`{ api: { modelId: { cost } } }`,
+ * `cost` nested) and models.dev (`{ models: { modelId: { cost } } }`, model keys at the top of
+ * the provider object). Unrecognized entries are skipped, never fatal.
+ * @param provider - provider id the models belong to.
+ * @param document - parsed provider document.
+ * @param into - table to merge into.
+ * @returns How many models were added.
+ */
+export function mergeProviderDocument(provider, document, into) {
+    const root = object(document);
+    if (root === undefined)
+        return 0;
+    const models = {};
+    // pi-ai: one key per wire protocol, each holding { modelId: model }.
+    for (const [key, value] of Object.entries(root)) {
+        if (key === 'models')
+            continue;
+        const block = object(value);
+        if (block === undefined)
+            continue;
+        for (const [id, model] of Object.entries(block))
+            models[id] = model;
+    }
+    // models.dev: a single { models: { modelId: model } } block.
+    for (const [id, model] of Object.entries(object(root.models) ?? {}))
+        models[id] = model;
+    let added = 0;
+    for (const [id, model] of Object.entries(models)) {
+        const row = object(model);
+        if (row === undefined)
+            continue;
+        const entry = priceEntry(row.cost);
+        if (entry === undefined)
+            continue;
+        into.set(priceKey(provider, id), entry);
+        added += 1;
+    }
+    return added;
+}
+/** Parses one models.dev snapshot (`{ provider: { models } }`) into a table. */
+export function parseModelsDevDocument(document, into) {
+    let added = 0;
+    for (const [provider, value] of Object.entries(object(document) ?? {}))
+        added += mergeProviderDocument(provider, value, into);
+    return added;
+}
+/** The catalog inside one package root, if the adapter is installed there. */
+function catalogUnder(root) {
+    const candidate = join(root, 'node_modules', '@earendil-works', 'pi-ai', 'dist', 'providers', 'data');
+    return existsSync(candidate) ? candidate : undefined;
+}
+/**
+ * Locate the installed pi-ai catalog directory.
+ *
+ * The package's `exports` map publishes no subpath, so a module resolver cannot even name its
+ * files and this walks the filesystem instead: upward from this module (the plugin normally
+ * sits in the same `profiles/<name>/node_modules` tree as the adapter), then the host profile
+ * directories for a plugin loaded straight from a checkout. `DSH_VERIFIER_PI_AI_DATA`
+ * overrides both for a deployment whose layout differs.
+ *
+ * Returns `undefined` when the package is not installed — the caller then simply has one
+ * source fewer, which is the documented degradation and not an error.
+ * @param from - directory to start the upward walk from; defaults to this module's directory.
+ * @param home - DSH home whose `profiles` are searched last; defaults to `$DSH_HOME`.
+ * @returns Absolute `dist/providers/data` directory, or `undefined`.
+ */
+export function resolveCatalogDataDir(from = dirname(fileURLToPath(import.meta.url)), home = process.env.DSH_HOME?.trim()) {
+    const override = process.env.DSH_VERIFIER_PI_AI_DATA?.trim();
+    if (override)
+        return existsSync(override) ? override : undefined;
+    let dir = from;
+    for (let depth = 0; depth < 12; depth += 1) {
+        const found = catalogUnder(dir);
+        if (found !== undefined)
+            return found;
+        const parent = dirname(dir);
+        if (parent === dir)
+            break;
+        dir = parent;
+    }
+    if (!home)
+        return undefined;
+    const profiles = join(home, 'profiles');
+    const direct = catalogUnder(profiles);
+    if (direct !== undefined)
+        return direct;
+    try {
+        for (const entry of readdirSync(profiles, { withFileTypes: true })) {
+            if (!entry.isDirectory())
+                continue;
+            const found = catalogUnder(join(profiles, entry.name));
+            if (found !== undefined)
+                return found;
+        }
+    }
+    catch { /* no profiles directory: nothing left to find */ }
+    return undefined;
+}
+/** Reads every `*.json` provider document in a catalog directory into a table. */
+export function loadCatalogTable(dataDir) {
+    const table = new Map();
+    let files;
+    try {
+        files = readdirSync(dataDir).filter(name => name.endsWith('.json'));
+    }
+    catch {
+        return table;
+    }
+    for (const file of files) {
+        try {
+            mergeProviderDocument(file.slice(0, -'.json'.length), JSON.parse(readFileSync(join(dataDir, file), 'utf8')), table);
+        }
+        catch { /* one unreadable provider must not lose the other 39 */ }
+    }
+    return table;
+}
+/** Default lifetime of a loaded price table, matching the capability memory's TTL. */
+export const PRICE_TTL_MS = 24 * 60 * 60 * 1000;
+/** Public models.dev snapshot; the upstream of the pi-ai catalog the host ships. */
+export const MODELS_DEV_URL = 'https://models.dev/api.json';
+/**
+ * Resolves a route's rates once per TTL, caching both tables in-process.
+ *
+ * The two tables are independent: a broken catalog read still leaves the online lookup, and
+ * a network failure still leaves the catalog. Both are single-flighted so concurrent topics
+ * cannot each fetch the 4.6 MB models.dev snapshot.
+ */
+export class PriceResolver {
+    deps;
+    catalog;
+    online;
+    onlineFlight;
+    constructor(deps = {}) {
+        this.deps = deps;
+    }
+    /** The installed catalog table, loaded at most once per TTL. */
+    catalogTable() {
+        const now = (this.deps.now ?? Date.now)();
+        const ttl = this.deps.ttlMs ?? PRICE_TTL_MS;
+        if (this.catalog !== undefined && now - this.catalog.at <= ttl)
+            return this.catalog.table;
+        const dir = (this.deps.catalogDir ?? (() => resolveCatalogDataDir()))();
+        const table = dir === undefined ? new Map() : loadCatalogTable(dir);
+        this.catalog = { at: now, table };
+        return table;
+    }
+    /** The models.dev table, fetched at most once per TTL. Never rejects. */
+    async onlineTable() {
+        const now = (this.deps.now ?? Date.now)();
+        const ttl = this.deps.ttlMs ?? PRICE_TTL_MS;
+        if (this.online !== undefined && now - this.online.at <= ttl)
+            return this.online.table;
+        this.onlineFlight ??= (async () => {
+            const table = new Map();
+            try {
+                const fetchImpl = this.deps.fetch ?? fetch;
+                const response = await fetchImpl(this.deps.url ?? MODELS_DEV_URL, { signal: AbortSignal.timeout(this.deps.timeoutMs ?? 20_000) });
+                if (response.ok)
+                    parseModelsDevDocument(JSON.parse(await response.text()), table);
+            }
+            catch { /* offline, blocked, or a changed shape: no price beats a fabricated one */ }
+            this.online = { at: (this.deps.now ?? Date.now)(), table };
+            this.onlineFlight = undefined;
+            return table;
+        })();
+        return this.onlineFlight;
+    }
+    fromTable(table, providers, model) {
+        for (const provider of providers) {
+            const entry = table.get(priceKey(provider, model));
+            if (entry !== undefined)
+                return entry;
+        }
+        return undefined;
+    }
+    /**
+     * Rates in effect for one route.
+     * @param provider - route's provider id.
+     * @param model - route's model id.
+     * @param options - manual rates plus which automatic sources may run.
+     * @returns The rates, always, with the source that answered.
+     */
+    async resolve(provider, model, options) {
+        const manual = options.manual;
+        if (manual.input > 0 || manual.output > 0) {
+            // A typed rate is a fact about the deployment. Cache reads keep the operator's own
+            // cached rate when they gave one; otherwise they fall back to the input rate, which is
+            // exactly the pre-existing estimate rather than silently inventing a discount.
+            return { input: manual.input, output: manual.output, cachedInput: manual.cachedInput > 0 ? manual.cachedInput : manual.input, source: 'manual' };
+        }
+        const override = options.overrideProvider?.trim();
+        const candidates = override && override !== provider ? [override, provider] : [provider];
+        if (options.fromCatalog !== false) {
+            const entry = this.fromTable(this.catalogTable(), candidates, model);
+            if (entry !== undefined)
+                return { ...entry, source: 'catalog' };
+        }
+        if (options.online === true) {
+            const entry = this.fromTable(await this.onlineTable(), candidates, model);
+            if (entry !== undefined)
+                return { ...entry, source: 'online' };
+        }
+        return UNPRICED;
+    }
+}
+//# sourceMappingURL=pricing.js.map

package/lib/types/pricing.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pricing.js","sourceRoot":"","sources":["../../src/pricing.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AAC/D,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AA4BxC,6CAA6C;AAC7C,MAAM,CAAC,MAAM,QAAQ,GAAgB,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAA;AAY5F;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,OAAO,CAAC,KAAkB,EAAE,MAAmC;IAC7E,OAAO,CAAC,KAAK,CAAC,WAAW,GAAG,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC,iBAAiB,GAAG,MAAM,CAAC,WAAW,GAAG,KAAK,CAAC,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,SAAS,CAAA;AAC3I,CAAC;AAED,yDAAyD;AACzD,MAAM,UAAU,QAAQ,CAAC,QAAgB,EAAE,KAAa;IACtD,OAAO,QAAQ,GAAG,QAAQ,GAAG,KAAK,CAAA;AACpC,CAAC;AAED,SAAS,MAAM,CAAC,KAAc;IAC5B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAgC,CAAC,CAAC,CAAC,SAAS,CAAA;AAC5H,CAAC;AAED,SAAS,IAAI,CAAC,KAAc;IAC1B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAA;AAC9F,CAAC;AAED,0GAA0G;AAC1G,MAAM,UAAU,UAAU,CAAC,KAAc;IACvC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA;IACzB,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,SAAS,CAAA;IACvC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;IAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IAC/B,IAAI,KAAK,KAAK,SAAS,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,SAAS,CAAA;IACjE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,KAAK,EAAE,CAAA;AAC7F,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,qBAAqB,CAAC,QAAgB,EAAE,QAAiB,EAAE,IAAgB;IACzF,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAA;IAC7B,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,CAAC,CAAA;IAChC,MAAM,MAAM,GAA4B,EAAE,CAAA;IAC1C,qEAAqE;IACrE,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAChD,IAAI,GAAG,KAAK,QAAQ;YAAE,SAAQ;QAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA;QAC3B,IAAI,KAAK,KAAK,SAAS;YAAE,SAAQ;QACjC,KAAK,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,MAAM,CAAC,EAAE,CAAC,GAAG,KAAK,CAAA;IACrE,CAAC;IACD,6DAA6D;IAC7D,KAAK,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAAE,MAAM,CAAC,EAAE,CAAC,GAAG,KAAK,CAAA;IACvF,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,KAAK,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACjD,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA;QACzB,IAAI,GAAG,KAAK,SAAS;YAAE,SAAQ;QAC/B,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAClC,IAAI,KAAK,KAAK,SAAS;YAAE,SAAQ;QACjC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,KAAK,CAAC,CAAA;QACvC,KAAK,IAAI,CAAC,CAAA;IACZ,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,sBAAsB,CAAC,QAAiB,EAAE,IAAgB;IACxE,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QAAE,KAAK,IAAI,qBAAqB,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,CAAA;IAC7H,OAAO,KAAK,CAAA;AACd,CAAC;AAED,8EAA8E;AAC9E,SAAS,YAAY,CAAC,IAAY;IAChC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,EAAE,cAAc,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,CAAC,CAAA;IACrG,OAAO,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAA;AACtD,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,qBAAqB,CAAC,OAAe,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAA2B,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,EAAE;IACnJ,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,IAAI,EAAE,CAAA;IAC5D,IAAI,QAAQ;QAAE,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAA;IAChE,IAAI,GAAG,GAAG,IAAI,CAAA;IACd,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,EAAE,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC3C,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAA;QAC/B,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO,KAAK,CAAA;QACrC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;QAC3B,IAAI,MAAM,KAAK,GAAG;YAAE,MAAK;QACzB,GAAG,GAAG,MAAM,CAAA;IACd,CAAC;IACD,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAA;IAC3B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,CAAA;IACvC,MAAM,MAAM,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAA;IACrC,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,MAAM,CAAA;IACvC,IAAI,CAAC;QACH,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YACnE,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE;gBAAE,SAAQ;YAClC,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;YACtD,IAAI,KAAK,KAAK,SAAS;gBAAE,OAAO,KAAK,CAAA;QACvC,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,iDAAiD,CAAC,CAAC;IAC7D,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,kFAAkF;AAClF,MAAM,UAAU,gBAAgB,CAAC,OAAe;IAC9C,MAAM,KAAK,GAAe,IAAI,GAAG,EAAE,CAAA;IACnC,IAAI,KAAe,CAAA;IACnB,IAAI,CAAC;QACH,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAA;IACrE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAA;IACd,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,CAAA;QACrH,CAAC;QAAC,MAAM,CAAC,CAAC,wDAAwD,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,sFAAsF;AACtF,MAAM,CAAC,MAAM,YAAY,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AAE/C,oFAAoF;AACpF,MAAM,CAAC,MAAM,cAAc,GAAG,6BAA6B,CAAA;AAuB3D;;;;;;GAMG;AACH,MAAM,OAAO,aAAa;IAKK;IAJrB,OAAO,CAA+C;IACtD,MAAM,CAA+C;IACrD,YAAY,CAAiC;IAErD,YAA6B,OAA0B,EAAE;QAA5B,SAAI,GAAJ,IAAI,CAAwB;IAAG,CAAC;IAE7D,gEAAgE;IAChE,YAAY;QACV,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAA;QACzC,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,YAAY,CAAA;QAC3C,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,GAAG;YAAE,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAA;QACzF,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,CAAC,qBAAqB,EAAE,CAAC,CAAC,EAAE,CAAA;QACvE,MAAM,KAAK,GAAG,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAA;QACnE,IAAI,CAAC,OAAO,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAA;QACjC,OAAO,KAAK,CAAA;IACd,CAAC;IAED,yEAAyE;IACzE,KAAK,CAAC,WAAW;QACf,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAA;QACzC,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,YAAY,CAAA;QAC3C,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,GAAG;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAA;QACtF,IAAI,CAAC,YAAY,KAAK,CAAC,KAAK,IAAI,EAAE;YAChC,MAAM,KAAK,GAAe,IAAI,GAAG,EAAE,CAAA;YACnC,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,KAAK,CAAA;gBAC1C,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,cAAc,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC,EAAE,CAAC,CAAA;gBACjI,IAAI,QAAQ,CAAC,EAAE;oBAAE,sBAAsB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,CAAC,CAAA;YACnF,CAAC;YAAC,MAAM,CAAC,CAAC,2EAA2E,CAAC,CAAC;YACvF,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,CAAA;YAC1D,IAAI,CAAC,YAAY,GAAG,SAAS,CAAA;YAC7B,OAAO,KAAK,CAAA;QACd,CAAC,CAAC,EAAE,CAAA;QACJ,OAAO,IAAI,CAAC,YAAY,CAAA;IAC1B,CAAC;IAEO,SAAS,CAAC,KAAiB,EAAE,SAA4B,EAAE,KAAa;QAC9E,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAA;YAClD,IAAI,KAAK,KAAK,SAAS;gBAAE,OAAO,KAAK,CAAA;QACvC,CAAC;QACD,OAAO,SAAS,CAAA;IAClB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,OAAO,CAAC,QAAgB,EAAE,KAAa,EAAE,OAA6B;QAC1E,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAA;QAC7B,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1C,mFAAmF;YACnF,uFAAuF;YACvF,+EAA+E;YAC/E,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;QAClJ,CAAC;QACD,MAAM,QAAQ,GAAG,OAAO,CAAC,gBAAgB,EAAE,IAAI,EAAE,CAAA;QACjD,MAAM,UAAU,GAAG,QAAQ,IAAI,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAA;QACxF,IAAI,OAAO,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAClC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,UAAU,EAAE,KAAK,CAAC,CAAA;YACpE,IAAI,KAAK,KAAK,SAAS;gBAAE,OAAO,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAA;QACjE,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;YAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,IAAI,CAAC,WAAW,EAAE,EAAE,UAAU,EAAE,KAAK,CAAC,CAAA;YACzE,IAAI,KAAK,KAAK,SAAS;gBAAE,OAAO,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;QAChE,CAAC;QACD,OAAO,QAAQ,CAAA;IACjB,CAAC;CACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-llm-verifier",
-  "version": "0.6.2",
+  "version": "0.6.4",
   "description": "Configurable DSH-native LLM verifier with a Web settings page",
   "repository": {
     "type": "git",

package/scripts/sync-installed-profiles.mjs

@@ -6,7 +6,18 @@
  * they were hard-linked at install time and pnpm considers the dependency unchanged, so it will not
  * re-link them (verified: a plain install, --force and a re-add all leave the old inode in place).
  * Only removing the installed directory and re-running `pnpm install` re-creates the links. That is
- * what this does, for every profile whose package.json depends on this package.
+ * what this does, for every profile whose package.json owns a copy of this package.
+ *
+ * A profile that only lists this package in `dsh.profile.bundles` (its dependencies stay empty)
+ * owns no copy: the harness resolves it through the shared `$DSH_HOME/profiles/node_modules`
+ * fallback, which links the bundle's real directory, so there is nothing to refresh there.
+ *
+ * Refreshing is destructive (pnpm re-links a `file:` directory dependency only once its entry is
+ * gone), so the current copy is moved aside first and restored whenever the install fails: a stale
+ * copy is far better than a missing plugin. Failures print the tail of pnpm's own log instead of a
+ * bare "Command failed", and a `file:` pin left behind by a moved repository is repaired in place,
+ * because that state makes every install variant fail with ENOENT before anything needs to be
+ * removed (verified: `--force`, `--fix-lockfile`, `pnpm add` and a plain install all reuse the pin).
  *
  * With no matching profile (CI, a fresh clone, another machine) it is a no-op and exits 0, so it can
  * sit at the end of the build without affecting the release gate.
@@ -14,9 +25,9 @@
  * Usage: node scripts/sync-installed-profiles.mjs [--check] [--home <DSH_HOME>]
  */
 import { execFileSync } from 'node:child_process'
-import { existsSync, readdirSync, readFileSync, rmSync, statSync, unlinkSync, writeFileSync } from 'node:fs'
-import { join, resolve } from 'node:path'
-import { homedir } from 'node:os'
+import { closeSync, existsSync, openSync, readdirSync, readFileSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync } from 'node:fs'
+import { basename, join, relative, resolve, sep } from 'node:path'
+import { homedir, tmpdir } from 'node:os'
 
 const repo = resolve(import.meta.dirname, '..')
 const argv = process.argv.slice(2)
@@ -24,27 +35,186 @@ const check = argv.includes('--check')
 const flag = (name) => { const i = argv.indexOf(name); return i >= 0 ? argv[i + 1] : undefined }
 const home = resolve(flag('--home') ?? process.env.DSH_HOME ?? join(homedir(), '.dsh'))
 const profilesDir = join(home, 'profiles')
+const sharedDir = join(profilesDir, 'node_modules')
 const name = JSON.parse(readFileSync(join(repo, 'package.json'), 'utf8')).name
 
-/** Profiles that declare this package as a dependency or as a bundle. */
-function installedProfiles() {
+/** Profiles that declare this package as a dependency or as a bundle, with the declared specifier. */
+function declaredProfiles() {
   const found = []
   let entries
   try { entries = readdirSync(profilesDir, { withFileTypes: true }) } catch { return found }
   for (const entry of entries) {
     if (!entry.isDirectory() || entry.name === 'node_modules') continue
-    const manifest = join(profilesDir, entry.name, 'package.json')
+    const dir = join(profilesDir, entry.name)
+    const manifest = join(dir, 'package.json')
     if (!existsSync(manifest)) continue
     try {
       const pkg = JSON.parse(readFileSync(manifest, 'utf8'))
-      const asDependency = pkg.dependencies?.[name] !== undefined
+      const dependency = pkg.dependencies?.[name]
       const asBundle = Array.isArray(pkg.dsh?.profile?.bundles) && pkg.dsh.profile.bundles.includes(name)
-      if (asDependency || asBundle) found.push(join(profilesDir, entry.name))
+      if (dependency !== undefined || asBundle) found.push({ dir, dependency })
     } catch { /* an unreadable manifest is not fatal */ }
   }
   return found
 }
 
+/** Absolute directory a `file:`/`link:` dependency points at, or undefined for other specifiers. */
+function localDependencyTarget(dir, specifier) {
+  if (typeof specifier !== 'string') return undefined
+  const match = /^(?:file|link):(.*)$/i.exec(specifier.trim())
+  const target = match === null ? '' : match[1].trim()
+  return target === '' ? undefined : resolve(dir, target)
+}
+
+/** Drop one layer of surrounding quotes from a YAML scalar. */
+function unquote(value) {
+  const trimmed = value.trim()
+  const quoted = /^(['"])([\s\S]*)\1$/.exec(trimmed)
+  return quoted === null ? trimmed : quoted[2]
+}
+
+/**
+ * The `file:` directory pnpm-lock.yaml pinned for this package, or undefined when it pins none.
+ *
+ * pnpm resolves a `file:` directory dependency from this pin, not from package.json, which is why a
+ * moved repository keeps installing from the old location.
+ */
+function lockedPin(dir) {
+  const lockfile = join(dir, 'pnpm-lock.yaml')
+  if (!existsSync(lockfile)) return undefined
+  const lines = readFileSync(lockfile, 'utf8').split(/\r?\n/)
+  const key = new RegExp('^  ' + name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '@(?:file|link):(.+):$')
+  for (let i = 0; i < lines.length; i += 1) {
+    const match = key.exec(lines[i])
+    if (match === null) continue
+    const resolution = /^\s+resolution:\s*\{\s*directory:\s*([^,}]+?)\s*[,}]/.exec(lines[i + 1] ?? '')
+    const pinned = unquote(resolution === null ? match[1] : resolution[1])
+    return { lockfile, pinned, dir: resolve(dir, pinned) }
+  }
+  return undefined
+}
+
+/** A pin that is already broken, with the path the manifest declares instead, or undefined. */
+function stalePin(dir, declared) {
+  if (declared === undefined) return undefined
+  const pin = lockedPin(dir)
+  if (pin === undefined || existsSync(pin.dir)) return undefined
+  const replacement = relative(dir, declared).split(sep).join('/')
+  return replacement === pin.pinned ? undefined : { ...pin, replacement }
+}
+
+/**
+ * Rewrite a stale pin to the directory the manifest declares.
+ *
+ * Only the pin is rewritten: re-resolving the whole lockfile would silently move every other
+ * dependency of the profile to a newer version.
+ */
+function repairStalePin(dir, declared) {
+  const stale = stalePin(dir, declared)
+  if (stale === undefined) return
+  const content = readFileSync(stale.lockfile, 'utf8')
+  const repaired = content.split(stale.pinned).join(stale.replacement)
+  if (repaired === content) return
+  writeFileSync(stale.lockfile, repaired)
+  console.log('sync-installed-profiles: repaired the stale ' + basename(stale.lockfile) + ' pin in ' + dir + ' (' + stale.pinned + ' -> ' + stale.replacement + ')')
+}
+
+/**
+ * The specifier this profile should declare instead, when its `file:` dependency points at a
+ * repository that is no longer there.
+ *
+ * The declared path is absolute because pnpm materializes a directory dependency that way; nothing
+ * in package.json can express "this package, wherever it happens to live". The rewrite is therefore
+ * done here, by the build of the package that is being moved, which is the only party that knows
+ * both locations at once.
+ */
+function repointableSpecifier(profile) {
+  if (typeof profile.dependency !== 'string') return undefined
+  const match = /^\s*(file|link):/i.exec(profile.dependency)
+  if (match === null) return undefined
+  const declared = localDependencyTarget(profile.dir, profile.dependency)
+  if (declared === undefined || existsSync(declared)) return undefined
+  return { protocol: match[1].toLowerCase(), specifier: match[1].toLowerCase() + ':' + repo.split(sep).join('/'), declared }
+}
+
+/**
+ * Point a profile whose dependency still names an earlier checkout at this repository.
+ *
+ * Both places that carry the path are rewritten: package.json, and the lockfile line that mirrors
+ * its specifier verbatim (pnpm keeps resolving a `file:` directory from the lockfile, so fixing
+ * only the manifest would leave the install failing exactly as before).
+ */
+function repointProfile(profile, target) {
+  const manifest = join(profile.dir, 'package.json')
+  const text = readFileSync(manifest, 'utf8')
+  const repointed = text.split(profile.dependency).join(target.specifier)
+  if (repointed === text) return undefined
+  writeFileSync(manifest, repointed)
+  const lockfile = join(profile.dir, 'pnpm-lock.yaml')
+  if (existsSync(lockfile)) {
+    const before = readFileSync(lockfile, 'utf8')
+    const after = before.split(profile.dependency.slice(profile.dependency.indexOf(':') + 1))
+      .join(target.specifier.slice(target.specifier.indexOf(':') + 1))
+    if (after !== before) writeFileSync(lockfile, after)
+  }
+  console.log('sync-installed-profiles: repointed ' + manifest + ' at ' + target.specifier)
+  return target.specifier
+}
+
+/** Last lines of pnpm's own log, so a failed install reports a cause instead of "Command failed". */
+function installLogTail(log) {
+  try {
+    const lines = readFileSync(log, 'utf8').trim().split(/\r?\n/).filter(line => line.trim() !== '')
+    return lines.length === 0 ? undefined : lines.slice(-8).join(' / ')
+  } catch { return undefined }
+}
+
+/**
+ * Run `pnpm install` in one profile.
+ *
+ * pnpm's output goes to a log file rather than a pipe: the harness sandbox refuses piped stdio, and
+ * the log is what turns a bare execFileSync error into an actionable message.
+ */
+function installProfile(dir) {
+  const log = join(tmpdir(), 'dsh-sync-installed-profiles-' + process.pid + '.log')
+  let fd
+  try { fd = openSync(log, 'w') } catch { fd = undefined }
+  const stdio = fd === undefined ? 'ignore' : ['ignore', fd, fd]
+  try {
+    if (process.platform === 'win32') execFileSync('cmd', ['/c', 'pnpm install'], { cwd: dir, stdio })
+    else execFileSync('pnpm', ['install'], { cwd: dir, stdio })
+    return { ok: true }
+  } catch (error) {
+    return { ok: false, detail: installLogTail(log) ?? (error instanceof Error ? error.message : String(error)) }
+  } finally {
+    if (fd !== undefined) closeSync(fd)
+    try { rmSync(log, { force: true }) } catch { /* a leftover log is harmless */ }
+  }
+}
+
+/** Move the installed copy aside, reinstall it, and restore the previous copy when that fails. */
+function refreshInstalled(dir) {
+  const installed = join(dir, 'node_modules', name)
+  const backup = installed + '.sync-backup'
+  rmSync(backup, { recursive: true, force: true })
+  renameSync(installed, backup)
+  const result = installProfile(dir)
+  if (!result.ok) {
+    rmSync(installed, { recursive: true, force: true })
+    renameSync(backup, installed)
+    return result
+  }
+  // pnpm exits 0 after pruning a copy the profile no longer declares; keeping the old copy is right
+  // there too, because the plugin disappearing from the profile is exactly what this must prevent.
+  if (!existsSync(installed)) {
+    renameSync(backup, installed)
+    return { ok: false, detail: 'pnpm install completed without creating the package directory (is it still a dependency of that profile?)' }
+  }
+  try { rmSync(backup, { recursive: true, force: true }) } catch { /* a stale .sync-backup is inert */ }
+  breakHardlinks(installed)
+  return result
+}
+
 /**
  * Break hard links in a directory or file by rewriting any file that has nlink > 1.
  *
@@ -78,7 +248,7 @@ function ensureRepoHasNoHardlinks() {
   }
 }
 
-const profiles = installedProfiles()
+const profiles = declaredProfiles()
 if (profiles.length === 0) {
   ensureRepoHasNoHardlinks()
   console.log('sync-installed-profiles: no profile installs ' + name + ' (nothing to refresh)')
@@ -86,28 +256,63 @@ if (profiles.length === 0) {
 }
 
 let refreshed = 0
+let installedProfiles = 0
 for (const profile of profiles) {
-  const installed = join(profile, 'node_modules', name)
+  const installed = join(profile.dir, 'node_modules', name)
   if (!existsSync(installed)) {
-    console.log('sync-installed-profiles: ' + profile + ' declares ' + name + ' but has no installed copy; run pnpm install there')
+    const shared = join(sharedDir, name)
+    // A bundle-only profile is served by the harness' shared fallback (a link to the bundle's real
+    // directory), so it owns no copy to keep in sync; pointing it at `pnpm install` would be wrong
+    // advice, since its dependencies are deliberately empty.
+    if (profile.dependency === undefined && existsSync(shared)) {
+      console.log('sync-installed-profiles: ' + profile.dir + ' declares ' + name + ' as a bundle only; it resolves through ' + shared + ' (nothing to refresh)')
+    } else {
+      console.log('sync-installed-profiles: ' + profile.dir + ' declares ' + name + ' but has no installed copy; run pnpm install there')
+    }
+    continue
+  }
+  installedProfiles += 1
+  // A profile that still names an earlier checkout is repointed before anything is removed, so a
+  // moved repository refreshes instead of leaving the profile on a stale copy forever.
+  let declared = localDependencyTarget(profile.dir, profile.dependency)
+  const repoint = repointableSpecifier(profile)
+  if (repoint !== undefined) {
+    const applied = check ? repoint.specifier : repointProfile(profile, repoint)
+    if (applied === undefined) {
+      console.log('! sync-installed-profiles: ' + profile.dir + ' depends on ' + repoint.declared + ', which does not exist and could not be rewritten; kept the installed copy untouched')
+      continue
+    }
+    if (check) console.log('sync-installed-profiles: would repoint ' + join(profile.dir, 'package.json') + ' at ' + applied)
+    declared = repo
+  }
+  if (declared !== undefined && !existsSync(declared)) {
+    console.log('! sync-installed-profiles: ' + profile.dir + ' depends on ' + declared + ', which does not exist; kept the installed copy untouched')
     continue
   }
   if (check) {
+    const stale = stalePin(profile.dir, declared)
+    if (stale !== undefined) console.log('sync-installed-profiles: would repair the stale ' + basename(stale.lockfile) + ' pin in ' + profile.dir + ' (' + stale.pinned + ' -> ' + stale.replacement + ')')
     console.log('sync-installed-profiles: would refresh ' + installed)
     continue
   }
   try {
-    // Removing the entry is what makes pnpm re-create it from the freshly built files.
-    rmSync(installed, { recursive: true, force: true })
-    execFileSync('cmd', ['/c', 'pnpm install'], { cwd: profile, stdio: 'ignore' })
-    breakHardlinks(installed)
+    repairStalePin(profile.dir, declared)
+    const result = refreshInstalled(profile.dir)
+    if (!result.ok) {
+      console.log('! sync-installed-profiles: could not refresh ' + installed + ' — ' + result.detail + ' (kept the previous copy)')
+      continue
+    }
     refreshed += 1
-    console.log('sync-installed-profiles: refreshed ' + installed)
+    const entry = join(installed, 'lib', 'index.js')
+    console.log('sync-installed-profiles: refreshed ' + installed + (existsSync(entry) ? '' : ' (warning: ' + entry + ' is missing)'))
   } catch (error) {
-    // The build itself succeeded; a failed refresh is reported loudly but does not fail it.
-    console.log('! sync-installed-profiles: could not refresh ' + installed + ' — ' + (error instanceof Error ? error.message : String(error)))
+    // The install never ran, or the copy could not be moved aside: put it back and report.
+    const backup = installed + '.sync-backup'
+    try { if (!existsSync(installed) && existsSync(backup)) renameSync(backup, installed) } catch { /* nothing else can be done here */ }
+    console.log('! sync-installed-profiles: could not refresh ' + installed + ' — ' + (error instanceof Error ? error.message : String(error)) + ' (kept the previous copy)')
   }
 }
 ensureRepoHasNoHardlinks()
-console.log('sync-installed-profiles: ' + refreshed + ' of ' + profiles.length + ' profile(s) refreshed')
-
+// The denominator counts profiles that really own a copy: bundle-only profiles have nothing to
+// refresh, so folding them in would make a healthy run read as "1 of 2 refreshed".
+console.log('sync-installed-profiles: ' + refreshed + ' of ' + installedProfiles + ' installed profile copy/copies refreshed')

package/src/client-fields.test.ts

@@ -187,6 +187,8 @@ describe('settings validation', () => {
     expect(codesFor({ ...base, cacheDir: 'nested/verifier' }, 'cacheDir')).toEqual([])
     expect(codesFor({ ...base, provider: '' }, 'provider')).toEqual(['required'])
     expect(codesFor({ ...base, model: '  ' }, 'model')).toEqual(['required'])
+    expect(codesFor({ ...base, priceProviderOverride: '' }, 'priceProviderOverride')).toEqual([])
+    expect(codesFor({ ...base, priceProviderOverride: 'deepseek-official' }, 'priceProviderOverride')).toEqual([])
   })
 
   it('never blocks a missing custom rubric file: the resolver falls back and reports why', () => {