@danceiny/gotry 0.0.1-rc.14 → 0.0.1-rc.16

package/dist/scripts/nightly-evidence-tests.js

@@ -0,0 +1,186 @@
+import assert from 'node:assert/strict';
+import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { buildNightlyRunRecord, loadPriceTable, loadPromptSet, priceRunCost, runNightlyEvidence } from './nightly-evidence.js';
+import { parseNightlyRun } from './product-metrics.js';
+let passed = 0;
+function ok(label) {
+    passed += 1;
+    console.log(`  ok ${label}`);
+}
+const priceTable = loadPriceTable('data/llm-price-table.json');
+const promptSet = loadPromptSet('data/m3-nightly-prompts.json');
+assert.equal(priceTable.schema_version, 'gotry_llm_price_table_v2');
+assert.equal(promptSet.schema_version, 'gotry_m3_nightly_prompt_set_v1');
+assert.ok(promptSet.turns.length >= 1);
+assert.ok(priceTable.prices['deepseek-v4-flash'], '裸键 deepseek-v4-flash 必须可查');
+assert.ok(priceTable.prices['MiniMax-M3'], '裸键 MiniMax-M3 必须可查(provider 为单一时透明)');
+assert.ok(priceTable.prices['minimax:MiniMax-M3'], 'qualified 键 minimax:MiniMax-M3 必须可查');
+assert.ok(priceTable.aliases['deepseek-chat'], 'deepseek-chat 必须在 aliases 表');
+assert.equal(priceTable.aliases['deepseek-chat'], 'deepseek-v4-flash');
+ok('v2 schema 载入 + 裸键 + qualified 键 + alias 双查找');
+for (const [name, entry] of Object.entries(priceTable.prices)){
+    assert.ok(entry.input_cache_hit_usd_per_1m_peak <= entry.input_cache_miss_usd_per_1m_peak, `${name} hit<=miss`);
+    assert.ok(entry.output_usd_per_1m_offpeak <= entry.output_usd_per_1m_peak, `${name} offpeak<=peak`);
+}
+ok('价目保守上界不变量(hit<=miss, offpeak<=peak)');
+const baseUsage = {
+    calls: 3,
+    inputTokens: 1_000_000,
+    outputTokens: 1_000_000,
+    inputCacheHitTokens: 0,
+    inputCacheMissTokens: 1_000_000,
+    responsesMissingUsage: 0
+};
+const cost = priceRunCost('deepseek-chat', baseUsage, priceTable);
+assert.equal(cost, 1.76);
+ok('peak miss 换算 0.44+1.32=1.76(经 alias)');
+const hitUsage = {
+    ...baseUsage,
+    inputCacheHitTokens: 1_000_000,
+    inputCacheMissTokens: 0
+};
+assert.equal(priceRunCost('deepseek-chat', hitUsage, priceTable), 1.334);
+ok('peak hit 换算 0.014+1.32=1.334');
+const m3cost = priceRunCost('MiniMax-M3', baseUsage, priceTable);
+assert.equal(m3cost, 3.00);
+ok('MiniMax-M3 peak miss 换算 0.60+2.40=3.00');
+const m3hit = priceRunCost('MiniMax-M3', hitUsage, priceTable);
+assert.equal(m3hit, 2.52);
+ok('MiniMax-M3 peak hit 换算 0.12+2.40=2.52');
+const m3prov = priceTable.prices['minimax:MiniMax-M3'];
+assert.equal(m3prov.input_cache_miss_usd_per_1m_peak, m3prov.input_cache_miss_usd_per_1m_offpeak, 'MiniMax-M3 flat 纪律:peak==offpeak miss');
+assert.equal(m3prov.input_cache_hit_usd_per_1m_peak, m3prov.input_cache_hit_usd_per_1m_offpeak, 'MiniMax-M3 flat 纪律:peak==offpeak hit');
+assert.equal(m3prov.output_usd_per_1m_peak, m3prov.output_usd_per_1m_offpeak, 'MiniMax-M3 flat 纪律:peak==offpeak output');
+ok('MiniMax flat_no_offpeak 纪律同价守门');
+assert.throws(()=>priceRunCost('unknown-model-xyz', baseUsage, priceTable), /no entry for model/);
+ok('未知模型名 → fail-closed');
+const dangling = {
+    ...priceTable,
+    aliases: {
+        'deepseek-chat': 'not-a-model'
+    }
+};
+assert.throws(()=>priceRunCost('deepseek-chat', baseUsage, dangling), /has no entry/);
+assert.throws(()=>priceRunCost('deepseek-chat', {
+        ...baseUsage,
+        responsesMissingUsage: 1
+    }, priceTable), /cost unprovable/);
+const tmpSchemaDir = mkdtempSync(join(tmpdir(), 'nightly-bad-schema-'));
+const badSchemaPath = join(tmpSchemaDir, 'bad.json');
+try {
+    writeFileSync(badSchemaPath, JSON.stringify({
+        schema_version: 'gotry_llm_price_table_v99',
+        prices: {}
+    }));
+    assert.throws(()=>loadPriceTable(badSchemaPath), /schema_version must be/);
+    ok('非法 schema_version 拒绝');
+} finally{
+    rmSync(tmpSchemaDir, {
+        recursive: true,
+        force: true
+    });
+}
+const tmpStrategyDir = mkdtempSync(join(tmpdir(), 'nightly-bad-strategy-'));
+const badStrategyPath = join(tmpStrategyDir, 'bad.json');
+try {
+    writeFileSync(badStrategyPath, JSON.stringify({
+        schema_version: 'gotry_llm_price_table_v2',
+        providers: {
+            bad: {
+                family: 'x',
+                price_strategy: 'mystery_strategy',
+                aliases: {},
+                models: {
+                    'm': {
+                        input_cache_miss_usd_per_1m_peak: 0.1,
+                        input_cache_hit_usd_per_1m_peak: 0.01,
+                        output_usd_per_1m_peak: 0.5,
+                        input_cache_miss_usd_per_1m_offpeak: 0.1,
+                        input_cache_hit_usd_per_1m_offpeak: 0.01,
+                        output_usd_per_1m_offpeak: 0.5
+                    }
+                }
+            }
+        }
+    }));
+    assert.throws(()=>loadPriceTable(badStrategyPath), /price_strategy must be/);
+    ok('v2 非法 price_strategy 拒绝');
+} finally{
+    rmSync(tmpStrategyDir, {
+        recursive: true,
+        force: true
+    });
+}
+ok('未知模型/悬空alias/usage缺失 fail-closed');
+const executeAt = '2026-08-29T12:00:00.000Z';
+const recordInput = {
+    schema_version: 'gotry_m3_nightly_run_v1',
+    executed_at: executeAt,
+    real_llm: true,
+    prompt_set_sha256: 'a'.repeat(64),
+    output_sha256: 'b'.repeat(64),
+    cost_usd: 0.25
+};
+const rec1 = buildNightlyRunRecord(recordInput);
+const rec2 = buildNightlyRunRecord({
+    ...recordInput
+});
+assert.equal(rec1.run_key, rec2.run_key);
+assert.match(rec1.run_key, /^hmac-sha256:[0-9a-f]{64}$/);
+parseNightlyRun(JSON.parse(JSON.stringify(rec1)), 0);
+const rec3 = buildNightlyRunRecord({
+    ...recordInput,
+    cost_usd: 0.26
+});
+assert.notEqual(rec3.run_key, rec1.run_key);
+ok('run_key 确定性 + 已过消费方 parseNightlyRun');
+const prevKey = process.env['LLM_API_KEY'];
+const prevDeepseekKey = process.env['DEEPSEEK_API_KEY'];
+const waitParent = mkdtempSync(join(tmpdir(), 'nightly-wait-'));
+const waitRoot = join(waitParent, 'evidence', 'm3');
+try {
+    delete process.env['LLM_API_KEY'];
+    delete process.env['DEEPSEEK_API_KEY'];
+    const waiting = await runNightlyEvidence({
+        evidenceRoot: waitRoot,
+        dryRun: false,
+        clock: ()=>new Date(executeAt)
+    });
+    assert.equal(waiting.state, 'waiting_external_evidence');
+    assert.equal(waiting.record, null);
+    assert.ok(waiting.reason?.includes('backoff'));
+    assert.equal(existsSync(waitRoot), false, 'waiting 不得创建任何证据目录');
+} finally{
+    if (prevKey !== undefined) process.env['LLM_API_KEY'] = prevKey;
+    if (prevDeepseekKey !== undefined) process.env['DEEPSEEK_API_KEY'] = prevDeepseekKey;
+    rmSync(waitParent, {
+        recursive: true,
+        force: true
+    });
+}
+ok('无凭证 → waiting_external_evidence 且零写入');
+const dryParent = mkdtempSync(join(tmpdir(), 'nightly-dry-'));
+const dryRoot = join(dryParent, 'evidence', 'm3');
+try {
+    const dry = await runNightlyEvidence({
+        evidenceRoot: dryRoot,
+        dryRun: true,
+        clock: ()=>new Date(executeAt)
+    });
+    assert.equal(dry.state, 'dry_run');
+    assert.equal(dry.record?.real_llm, false);
+    parseNightlyRun(JSON.parse(JSON.stringify(dry.record)), 0);
+    assert.equal(existsSync(dryRoot), false, 'dry-run 不得写任何证据文件');
+} finally{
+    rmSync(dryParent, {
+        recursive: true,
+        force: true
+    });
+}
+ok('dry-run mock 全链演练 + real_llm=false + 零落盘');
+console.log(`nightly-evidence tests: ${passed} 组断言全绿(offline,真跑不在 CI)`);
+
+
+//# sourceURL=/Users/bytedance/work/gotry-publish/ts/scripts/nightly-evidence-tests.ts

package/dist/scripts/nightly-evidence.js

@@ -0,0 +1,289 @@
+import { createHash } from 'node:crypto';
+import { appendFileSync, mkdirSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { createMockLlm } from '../src/mock-llm.js';
+import { createOpenAICompatLlm } from '../src/dsh-llm.js';
+import { newState, runTurn } from '../src/loop.js';
+import { solveUnified } from '../src/unified.js';
+import { realtimeSolvePort } from '../src/realtime-pricing.js';
+import { parseNightlyRun } from './product-metrics.js';
+const SHA256 = /^[0-9a-f]{64}$/;
+const DEFAULT_BUDGET_USD = 1;
+function canonical(value) {
+    if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`;
+    if (typeof value === 'object' && value !== null) {
+        const record = value;
+        return `{${Object.keys(record).sort().map((key)=>`${JSON.stringify(key)}:${canonical(record[key])}`).join(',')}}`;
+    }
+    return JSON.stringify(value);
+}
+function sha256Of(value) {
+    return createHash('sha256').update(canonical(value)).digest('hex');
+}
+function round6(value) {
+    return Number(value.toFixed(6));
+}
+export function loadPromptSet(path) {
+    const raw = JSON.parse(readFileSync(path, 'utf8'));
+    if (raw['schema_version'] !== 'gotry_m3_nightly_prompt_set_v1') {
+        throw new Error(`prompt set schema_version must be gotry_m3_nightly_prompt_set_v1, got: ${String(raw['schema_version'])}`);
+    }
+    const turns = raw['turns'];
+    if (!Array.isArray(turns) || turns.length === 0 || turns.some((t)=>typeof t !== 'string' || t.length === 0)) {
+        throw new Error('prompt set turns must be a non-empty array of non-empty strings');
+    }
+    return {
+        schema_version: 'gotry_m3_nightly_prompt_set_v1',
+        description: raw['description'],
+        turns: turns
+    };
+}
+function assertPriceEntry(label, value) {
+    const entry = typeof value === 'object' && value !== null ? value : null;
+    if (!entry) throw new Error(`${label} must be an object`);
+    for (const field of [
+        'input_cache_miss_usd_per_1m_peak',
+        'input_cache_hit_usd_per_1m_peak',
+        'output_usd_per_1m_peak'
+    ]){
+        const num = entry[field];
+        if (typeof num !== 'number' || !Number.isFinite(num) || num < 0) {
+            throw new Error(`${label}.${field} must be a non-negative finite number`);
+        }
+    }
+    return value;
+}
+export function loadPriceTable(path) {
+    const raw = JSON.parse(readFileSync(path, 'utf8'));
+    const schemaVersion = raw['schema_version'];
+    if (schemaVersion === 'gotry_llm_price_table_v2') {
+        return loadPriceTableV2(raw);
+    }
+    if (schemaVersion === 'gotry_llm_price_table_v1') {
+        return loadPriceTableV1(raw);
+    }
+    throw new Error(`price table schema_version must be gotry_llm_price_table_v1 or _v2, got: ${String(schemaVersion)}`);
+}
+function loadPriceTableV1(raw) {
+    const aliases = typeof raw['aliases'] === 'object' && raw['aliases'] !== null ? raw['aliases'] : {};
+    for (const [from, to] of Object.entries(aliases)){
+        if (typeof to !== 'string') throw new Error(`price table alias ${from} must map to a model name`);
+    }
+    const prices = {};
+    for (const [name, entry] of Object.entries(typeof raw['prices'] === 'object' && raw['prices'] !== null ? raw['prices'] : {})){
+        prices[name] = assertPriceEntry(`price table entry ${name}`, entry);
+    }
+    return {
+        schema_version: 'gotry_llm_price_table_v1',
+        aliases,
+        prices
+    };
+}
+function loadPriceTableV2(raw) {
+    const providersObj = typeof raw['providers'] === 'object' && raw['providers'] !== null ? raw['providers'] : null;
+    if (!providersObj) throw new Error('price table v2 requires providers object');
+    const aliases = {};
+    const prices = {};
+    for (const [providerId, rawProvider] of Object.entries(providersObj)){
+        const provider = assertProviderEntry(`price table provider ${providerId}`, rawProvider);
+        for (const [aliasName, target] of Object.entries(provider.aliases)){
+            if (typeof target !== 'string') throw new Error(`provider ${providerId} alias ${aliasName} must map to a model name`);
+            if (!provider.models[target]) {
+                throw new Error(`provider ${providerId} alias ${aliasName} -> ${target} has no entry; update data/llm-price-table.json via PR`);
+            }
+            aliases[aliasName] = target;
+        }
+        for (const [modelName, entry] of Object.entries(provider.models)){
+            const qualified = `${providerId}:${modelName}`;
+            const validated = assertPriceEntry(`price table provider ${providerId} model ${modelName}`, entry);
+            if (prices[qualified]) throw new Error(`duplicate price table key ${qualified}`);
+            prices[qualified] = validated;
+            if (prices[modelName]) throw new Error(`duplicate price table key ${modelName} (across providers, disambiguate via aliases or qualified lookup)`);
+            prices[modelName] = validated;
+        }
+    }
+    return {
+        schema_version: 'gotry_llm_price_table_v2',
+        aliases,
+        prices
+    };
+}
+function assertProviderEntry(label, value) {
+    const entry = typeof value === 'object' && value !== null ? value : null;
+    if (!entry) throw new Error(`${label} must be an object`);
+    const family = entry['family'];
+    if (typeof family !== 'string' || family.length === 0) throw new Error(`${label}.family must be a non-empty string`);
+    const strategy = entry['price_strategy'];
+    if (strategy !== 'tiered_peak_offpeak' && strategy !== 'flat_no_offpeak') {
+        throw new Error(`${label}.price_strategy must be 'tiered_peak_offpeak' or 'flat_no_offpeak', got: ${String(strategy)}`);
+    }
+    const aliases = typeof entry['aliases'] === 'object' && entry['aliases'] !== null ? entry['aliases'] : {};
+    const models = typeof entry['models'] === 'object' && entry['models'] !== null ? entry['models'] : {};
+    return {
+        family,
+        price_strategy: strategy,
+        api_base: typeof entry['api_base'] === 'string' ? entry['api_base'] : undefined,
+        source_url: typeof entry['source_url'] === 'string' ? entry['source_url'] : undefined,
+        aliases,
+        models
+    };
+}
+export function priceRunCost(model, usage, table) {
+    if (usage.responsesMissingUsage > 0) {
+        throw new Error(`usage provider omitted usage on ${usage.responsesMissingUsage} response(s) — cost unprovable, refusing to price (fail-closed)`);
+    }
+    const alias = table.aliases[model];
+    if (alias && !table.prices[alias]) throw new Error(`price table alias ${model} -> ${alias} has no entry; update data/llm-price-table.json via PR`);
+    const entry = table.prices[alias ?? model];
+    if (!entry) throw new Error(`price table has no entry for model ${model}; update data/llm-price-table.json via PR (fail-closed, no guessed price)`);
+    const cost = (usage.inputCacheMissTokens * entry.input_cache_miss_usd_per_1m_peak + usage.inputCacheHitTokens * entry.input_cache_hit_usd_per_1m_peak + usage.outputTokens * entry.output_usd_per_1m_peak) / 1_000_000;
+    return round6(cost);
+}
+export function buildNightlyRunRecord(input) {
+    const { schema_version, executed_at, real_llm, prompt_set_sha256, output_sha256, cost_usd } = input;
+    const core = {
+        schema_version,
+        executed_at,
+        real_llm,
+        prompt_set_sha256,
+        output_sha256,
+        cost_usd
+    };
+    if (!SHA256.test(prompt_set_sha256) || !SHA256.test(output_sha256)) {
+        throw new Error('nightly record hashes must be lowercase SHA-256 digests');
+    }
+    const record = {
+        ...core,
+        run_key: `hmac-sha256:${createHash('sha256').update(canonical(core)).digest('hex')}`
+    };
+    parseNightlyRun(JSON.parse(JSON.stringify(record)), 0);
+    return record;
+}
+function activeModel() {
+    return process.env['LLM_MODEL'] ?? process.env['DEEPSEEK_MODEL'] ?? 'MiniMax-M2';
+}
+function arg(name) {
+    const index = process.argv.indexOf(name);
+    return index >= 0 ? process.argv[index + 1] : undefined;
+}
+export async function runNightlyEvidence(options) {
+    const clock = options.clock ?? (()=>new Date());
+    const llmKey = process.env['LLM_API_KEY'] ?? process.env['DEEPSEEK_API_KEY'];
+    if (!options.dryRun && !llmKey) {
+        return {
+            state: 'waiting_external_evidence',
+            record: null,
+            reason: 'no real LLM credential (LLM_API_KEY/DEEPSEEK_API_KEY); waiting & backoff & no-spend per issue #22 停机纪律'
+        };
+    }
+    const promptSet = loadPromptSet('data/m3-nightly-prompts.json');
+    const priceTable = loadPriceTable('data/llm-price-table.json');
+    const packPath = join('..', 'data', 'flights_2026.json');
+    const solvePort = realtimeSolvePort(solveUnified);
+    const port = options.dryRun ? createMockLlm(packPath) : createOpenAICompatLlm(packPath);
+    const state = newState();
+    const history = [];
+    for (const turn of promptSet.turns){
+        const { reply } = await runTurn(state, turn, port, [
+            ...history
+        ], solvePort);
+        history.push({
+            role: 'user',
+            text: turn
+        }, {
+            role: 'assistant',
+            text: reply
+        });
+    }
+    const promptSetSha = sha256Of(promptSet);
+    const turnsTranscript = promptSet.turns.map((turn, index)=>({
+            turn_index: index,
+            user: turn,
+            reply: history[index * 2 + 1]?.text ?? ''
+        }));
+    const outputSha = sha256Of(turnsTranscript);
+    let record;
+    if (options.dryRun) {
+        record = buildNightlyRunRecord({
+            schema_version: 'gotry_m3_nightly_run_v1',
+            executed_at: clock().toISOString(),
+            real_llm: false,
+            prompt_set_sha256: promptSetSha,
+            output_sha256: outputSha,
+            cost_usd: 0
+        });
+        return {
+            state: 'dry_run',
+            record,
+            reason: 'dry-run exercises the pipeline against mock LLM; no evidence written'
+        };
+    }
+    const usage = port.usage;
+    if (!usage) throw new Error('real LLM port did not expose usage tracker (dsh-llm regression) — refusing to write cost evidence');
+    if (usage.calls !== promptSet.turns.length) {
+        throw new Error(`usage calls ${usage.calls} != expected turns ${promptSet.turns.length} — fail-closed, no evidence written`);
+    }
+    const costUsd = priceRunCost(activeModel(), usage, priceTable);
+    record = buildNightlyRunRecord({
+        schema_version: 'gotry_m3_nightly_run_v1',
+        executed_at: clock().toISOString(),
+        real_llm: true,
+        prompt_set_sha256: promptSetSha,
+        output_sha256: outputSha,
+        cost_usd: costUsd
+    });
+    mkdirSync(options.evidenceRoot, {
+        recursive: true
+    });
+    appendFileSync(join(options.evidenceRoot, 'cohort.jsonl'), `${JSON.stringify(record)}\n`, 'utf8');
+    const budget = Number(process.env['GOTRY_NIGHTLY_BUDGET_USD'] ?? DEFAULT_BUDGET_USD);
+    const overBudget = Number.isFinite(budget) && budget > 0 ? costUsd > budget : false;
+    return {
+        state: 'evidence_written',
+        record,
+        cost_over_budget: overBudget,
+        reason: overBudget ? `cost_usd ${costUsd} exceeds GOTRY_NIGHTLY_BUDGET_USD ${budget} — backoff suggested` : undefined
+    };
+}
+function output(payload, asJson) {
+    if (asJson) console.log(JSON.stringify(payload));
+    else {
+        console.log(`nightly-evidence state: ${payload.state}${payload.error ? ` — ${payload.error}` : ''}`);
+        if (payload.record) console.log(JSON.stringify(payload.record, null, 2));
+        if (payload.reason) console.log(payload.reason);
+    }
+}
+async function main() {
+    if (!process.argv.includes('--no-env-file')) {
+        try {
+            for (const line of readFileSync(join('..', '.env'), 'utf-8').split('\n')){
+                const m = line.match(/^([A-Z_]+)=(.*)$/);
+                if (m && !process.env[m[1]]) process.env[m[1]] = m[2].trim();
+            }
+        } catch  {}
+    }
+    const asJson = (arg('--format') ?? 'markdown') === 'json';
+    const evidenceRoot = arg('--evidence-root') ?? 'gotry-state/evidence/m3';
+    const dryRun = process.argv.includes('--dry-run');
+    try {
+        const result = await runNightlyEvidence({
+            evidenceRoot,
+            dryRun
+        });
+        output(result, asJson);
+        if (result.cost_over_budget) process.exitCode = 3;
+    } catch (error) {
+        output({
+            state: 'error',
+            record: null,
+            error: error.message
+        }, asJson);
+        process.exitCode = 1;
+    }
+}
+if (process.argv[1]?.endsWith('nightly-evidence.ts')) {
+    await main();
+}
+
+
+//# sourceURL=/Users/bytedance/work/gotry-publish/ts/scripts/nightly-evidence.ts

package/dist/scripts/nudge-digest.js

@@ -80,4 +80,4 @@ if (channel === 'file') {
 }
 
 
-//# sourceURL=/Users/bytedance/work/gotry/ts/scripts/nudge-digest.ts
+//# sourceURL=/Users/bytedance/work/gotry-publish/ts/scripts/nudge-digest.ts