@danceiny/gotry 0.0.1-rc.15 → 0.0.1-rc.16

package/dist/scripts/price-drift-tests.js

@@ -0,0 +1,441 @@
+import assert from 'node:assert/strict';
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { runPriceDriftWatch, parseOpenaiHtml } from './price-drift-watch.js';
+let passed = 0;
+function ok(label) {
+    passed += 1;
+    console.log(`  ok ${label}`);
+}
+function tmpRoot() {
+    return mkdtempSync(join(tmpdir(), 'drift-'));
+}
+function makeBaseline(rows, asOf = '2026-08-30T00:00:00.000Z') {
+    return JSON.stringify({
+        schema_version: 'gotry_llm_price_baseline_v1',
+        as_of: asOf,
+        source_url: 'mock://test',
+        rows
+    });
+}
+function makeMockProvider(id, baselinePath, _fetchReturn) {
+    const parser = (html)=>{
+        const rows = [];
+        for (const seg of html.split(';')){
+            const [model, nums] = seg.split(':');
+            if (!model || !nums) continue;
+            const [miss, hit, out] = nums.split(',').map(Number);
+            if (!Number.isFinite(miss) || !Number.isFinite(out)) continue;
+            rows.push({
+                model: model.trim(),
+                input_cache_miss_usd_per_1m: miss,
+                input_cache_hit_usd_per_1m: Number.isFinite(hit) ? hit : null,
+                output_usd_per_1m: out
+            });
+        }
+        return rows;
+    };
+    return [
+        {
+            id,
+            source_url: `mock://${id}`,
+            baseline_path: baselinePath,
+            parse: parser
+        }
+    ];
+}
+function fetcherFromMap(map) {
+    return async (url)=>{
+        const v = map[url];
+        if (v instanceof Error) throw v;
+        if (typeof v === 'string') return v;
+        throw new Error(`no mock for ${url}`);
+    };
+}
+async function testOfflineNoBaseline() {
+    const root = tmpRoot();
+    try {
+        const report = await runPriceDriftWatch({
+            mode: 'offline',
+            observedAt: ()=>new Date('2026-08-30T12:00:00Z'),
+            providers: makeMockProvider('mockprov', join(root, 'no-such-file.json'), '')
+        });
+        assert.equal(report.schema_version, 'gotry_llm_price_drift_v1');
+        assert.equal(report.providers['mockprov'].status, 'skipped_no_baseline');
+        ok('offline 无 baseline → skipped_no_baseline');
+    } finally{
+        rmSync(root, {
+            recursive: true,
+            force: true
+        });
+    }
+}
+async function testOfflineWithBaseline() {
+    const root = tmpRoot();
+    try {
+        const bp = join(root, 'b.json');
+        writeFileSync(bp, makeBaseline([
+            {
+                model: 'm1',
+                input_cache_miss_usd_per_1m: 1.0,
+                input_cache_hit_usd_per_1m: 0.1,
+                output_usd_per_1m: 3.0
+            }
+        ]));
+        const report = await runPriceDriftWatch({
+            mode: 'offline',
+            observedAt: ()=>new Date('2026-08-30T12:00:00Z'),
+            providers: makeMockProvider('mockprov', bp, '')
+        });
+        assert.equal(report.providers['mockprov'].status, 'unchanged');
+        assert.match(report.providers['mockprov'].reason ?? '', /--fetch/, 'offline reason 应提示 --fetch');
+        ok('offline 有 baseline → unchanged + reason 提示 --fetch');
+    } finally{
+        rmSync(root, {
+            recursive: true,
+            force: true
+        });
+    }
+}
+async function testFetchFailure() {
+    const root = tmpRoot();
+    try {
+        const bp = join(root, 'b.json');
+        writeFileSync(bp, makeBaseline([
+            {
+                model: 'm1',
+                input_cache_miss_usd_per_1m: 1.0,
+                input_cache_hit_usd_per_1m: 0.1,
+                output_usd_per_1m: 3.0
+            }
+        ]));
+        const report = await runPriceDriftWatch({
+            mode: 'fetch',
+            observedAt: ()=>new Date('2026-08-30T12:00:00Z'),
+            providers: makeMockProvider('mockprov', bp, ''),
+            fetcher: fetcherFromMap({
+                'mock://mockprov': new Error('ECONNREFUSED')
+            })
+        });
+        assert.equal(report.providers['mockprov'].status, 'skipped_no_network');
+        assert.match(report.providers['mockprov'].reason ?? '', /ECONNREFUSED/);
+        ok('fetch 失败 → skipped_no_network');
+    } finally{
+        rmSync(root, {
+            recursive: true,
+            force: true
+        });
+    }
+}
+async function testFetchFirstTimeWritesBaseline() {
+    const root = tmpRoot();
+    try {
+        const bp = join(root, 'b.json');
+        const report = await runPriceDriftWatch({
+            mode: 'fetch',
+            observedAt: ()=>new Date('2026-08-30T12:00:00Z'),
+            providers: makeMockProvider('mockprov', bp, ''),
+            fetcher: fetcherFromMap({
+                'mock://mockprov': 'm1:1.0,0.1,3.0;m2:2.0,0.2,5.0'
+            })
+        });
+        assert.equal(report.providers['mockprov'].status, 'fetched_no_table_entry');
+        const { existsSync, readFileSync } = await import('node:fs');
+        assert.ok(existsSync(bp), 'baseline fixture 必须被写');
+        const written = JSON.parse(readFileSync(bp, 'utf8'));
+        assert.equal(written['schema_version'], 'gotry_llm_price_baseline_v1');
+        assert.ok(Array.isArray(written['rows']));
+        assert.equal(written['rows'].length, 2);
+        ok('fetch 首次 → fetched_no_table_entry + 写 fixture');
+    } finally{
+        rmSync(root, {
+            recursive: true,
+            force: true
+        });
+    }
+}
+async function testFetchUnchanged() {
+    const root = tmpRoot();
+    try {
+        const bp = join(root, 'b.json');
+        writeFileSync(bp, makeBaseline([
+            {
+                model: 'm1',
+                input_cache_miss_usd_per_1m: 1.0,
+                input_cache_hit_usd_per_1m: 0.1,
+                output_usd_per_1m: 3.0
+            }
+        ]));
+        const report = await runPriceDriftWatch({
+            mode: 'fetch',
+            observedAt: ()=>new Date('2026-08-30T12:00:00Z'),
+            providers: makeMockProvider('mockprov', bp, ''),
+            fetcher: fetcherFromMap({
+                'mock://mockprov': 'm1:1.0,0.1,3.0'
+            })
+        });
+        assert.equal(report.providers['mockprov'].status, 'unchanged');
+        ok('fetch 模式:baseline + current 同 → unchanged');
+    } finally{
+        rmSync(root, {
+            recursive: true,
+            force: true
+        });
+    }
+}
+async function testFetchDrift() {
+    const root = tmpRoot();
+    try {
+        const bp = join(root, 'b.json');
+        writeFileSync(bp, makeBaseline([
+            {
+                model: 'm-keep',
+                input_cache_miss_usd_per_1m: 1.0,
+                input_cache_hit_usd_per_1m: 0.1,
+                output_usd_per_1m: 3.0
+            },
+            {
+                model: 'm-up',
+                input_cache_miss_usd_per_1m: 1.0,
+                input_cache_hit_usd_per_1m: 0.1,
+                output_usd_per_1m: 3.0
+            },
+            {
+                model: 'm-down',
+                input_cache_miss_usd_per_1m: 1.0,
+                input_cache_hit_usd_per_1m: 0.1,
+                output_usd_per_1m: 3.0
+            },
+            {
+                model: 'm-removed',
+                input_cache_miss_usd_per_1m: 1.0,
+                input_cache_hit_usd_per_1m: 0.1,
+                output_usd_per_1m: 3.0
+            }
+        ]));
+        const report = await runPriceDriftWatch({
+            mode: 'fetch',
+            observedAt: ()=>new Date('2026-08-30T12:00:00Z'),
+            providers: makeMockProvider('mockprov', bp, ''),
+            fetcher: fetcherFromMap({
+                'mock://mockprov': [
+                    'm-keep:1.0,0.1,3.0',
+                    'm-up:1.5,0.1,3.0',
+                    'm-down:0.8,0.1,3.0',
+                    'm-new:2.0,0.2,4.0'
+                ].join(';')
+            })
+        });
+        const r = report.providers['mockprov'];
+        assert.equal(r.status, 'drift');
+        const drift = r.drift ?? [];
+        const fields = drift.map((d)=>`${d.model}|${d.field}|${d.direction}`).sort();
+        assert.deepEqual(fields, [
+            'm-down|input_cache_miss_usd_per_1m|down',
+            'm-new|input_cache_miss_usd_per_1m|new',
+            'm-removed|input_cache_miss_usd_per_1m|removed',
+            'm-up|input_cache_miss_usd_per_1m|up'
+        ], 'drift 字段方向必须逐项正确');
+        ok('fetch drift:up/down/new/removed 四向全检');
+    } finally{
+        rmSync(root, {
+            recursive: true,
+            force: true
+        });
+    }
+}
+async function testPrParagraph() {
+    const root = tmpRoot();
+    try {
+        const bp = join(root, 'b.json');
+        writeFileSync(bp, makeBaseline([
+            {
+                model: 'deepseek-v4-flash',
+                input_cache_miss_usd_per_1m: 1.32,
+                input_cache_hit_usd_per_1m: 0.014,
+                output_usd_per_1m: 3.0
+            }
+        ]));
+        const report = await runPriceDriftWatch({
+            mode: 'fetch',
+            observedAt: ()=>new Date('2026-08-30T12:00:00Z'),
+            providers: makeMockProvider('mockprov', bp, ''),
+            fetcher: fetcherFromMap({
+                'mock://mockprov': 'deepseek-v4-flash:1.32,0.014,3.0'
+            })
+        });
+        assert.equal(report.providers['mockprov'].status, 'unchanged');
+        assert.match(report.suggested_pr_paragraph, /价格漂移监测/);
+        assert.match(report.suggested_pr_paragraph, /mockprov/);
+        assert.match(report.suggested_pr_paragraph, /人工 PR/);
+        ok('PR 段落含监测标题 + provider 变动 + 纪律注脚');
+    } finally{
+        rmSync(root, {
+            recursive: true,
+            force: true
+        });
+    }
+}
+async function testBadBaseline() {
+    const root = tmpRoot();
+    try {
+        const bp = join(root, 'b.json');
+        writeFileSync(bp, JSON.stringify({
+            schema_version: 'gotry_llm_price_baseline_v99',
+            rows: []
+        }));
+        const report = await runPriceDriftWatch({
+            mode: 'offline',
+            observedAt: ()=>new Date('2026-08-30T12:00:00Z'),
+            providers: makeMockProvider('mockprov', bp, '')
+        });
+        assert.equal(report.providers['mockprov'].status, 'skipped_no_baseline', '坏 baseline schema_version 视为不存在,offline 标 SKIP');
+        ok('坏 baseline schema_version → offline 视为不存在(SKIP)');
+    } finally{
+        rmSync(root, {
+            recursive: true,
+            force: true
+        });
+    }
+}
+await testOfflineNoBaseline();
+await testOfflineWithBaseline();
+await testFetchFailure();
+await testFetchFirstTimeWritesBaseline();
+await testFetchUnchanged();
+await testFetchDrift();
+await testPrParagraph();
+await testBadBaseline();
+async function testSnapshotNextDataParser() {
+    const root = tmpRoot();
+    try {
+        const bp = join(root, 'b.json');
+        const sp = join(root, 'snapshot.html');
+        writeFileSync(bp, makeBaseline([
+            {
+                model: 'gpt-4o',
+                input_cache_miss_usd_per_1m: 2.5,
+                input_cache_hit_usd_per_1m: 1.25,
+                output_usd_per_1m: 10.0
+            }
+        ]));
+        const html = `<html><script id="__NEXT_DATA__" type="application/json">${JSON.stringify({
+            props: {
+                pageProps: {
+                    models: [
+                        {
+                            name: 'gpt-4o',
+                            input_cost_per_token: 0.0000025,
+                            output_cost_per_token: 0.00001,
+                            cached_input_cost_per_token: 0.00000125
+                        }
+                    ]
+                }
+            }
+        })}</script></html>`;
+        writeFileSync(sp, html);
+        const report = await runPriceDriftWatch({
+            mode: 'offline',
+            observedAt: ()=>new Date('2026-08-30T12:00:00Z'),
+            providers: [
+                {
+                    id: 'mockprov',
+                    source_url: 'mock://mockprov',
+                    baseline_path: bp,
+                    snapshot_path: sp,
+                    parse: parseOpenaiHtml
+                }
+            ]
+        });
+        assert.equal(report.providers['mockprov'].status, 'unchanged', 'snapshot 与 baseline 同价 → unchanged(offline)');
+        ok('snapshot 路径:__NEXT_DATA__ 内嵌 JSON dump 解析成功 + 与 baseline 同 → unchanged');
+    } finally{
+        rmSync(root, {
+            recursive: true,
+            force: true
+        });
+    }
+}
+async function testSnapshotFallbackDrift() {
+    const root = tmpRoot();
+    try {
+        const bp = join(root, 'b.json');
+        const sp = join(root, 'snapshot.html');
+        writeFileSync(bp, makeBaseline([
+            {
+                model: 'gpt-4o',
+                input_cache_miss_usd_per_1m: 2.5,
+                input_cache_hit_usd_per_1m: 1.25,
+                output_usd_per_1m: 10.0
+            }
+        ]));
+        const html = '<html><body><p>gpt-4o … $5.00 / 1M input … $20.00 / 1M output</p></body></html>';
+        writeFileSync(sp, html);
+        const report = await runPriceDriftWatch({
+            mode: 'offline',
+            observedAt: ()=>new Date('2026-08-30T12:00:00Z'),
+            providers: [
+                {
+                    id: 'mockprov',
+                    source_url: 'mock://mockprov',
+                    baseline_path: bp,
+                    snapshot_path: sp,
+                    parse: parseOpenaiHtml
+                }
+            ]
+        });
+        const r = report.providers['mockprov'];
+        assert.equal(r.status, 'drift', 'snapshot fallback 解析 + baseline 不一致 → drift');
+        const drift = r.drift ?? [];
+        assert.ok(drift.some((d)=>d.field === 'input_cache_miss_usd_per_1m' && d.from === 2.5 && d.to === 5.0 && d.direction === 'up'), 'miss 字段 up drift');
+        assert.ok(drift.some((d)=>d.field === 'output_usd_per_1m' && d.from === 10.0 && d.to === 20.0 && d.direction === 'up'), 'output 字段 up drift');
+        ok('snapshot fallback:段落文字解析 + 与 baseline 不同 → drift(含 up/down/new/removed)');
+    } finally{
+        rmSync(root, {
+            recursive: true,
+            force: true
+        });
+    }
+}
+async function testSnapshotMissing() {
+    const root = tmpRoot();
+    try {
+        const bp = join(root, 'b.json');
+        writeFileSync(bp, makeBaseline([
+            {
+                model: 'gpt-4o',
+                input_cache_miss_usd_per_1m: 2.5,
+                input_cache_hit_usd_per_1m: 1.25,
+                output_usd_per_1m: 10.0
+            }
+        ]));
+        const report = await runPriceDriftWatch({
+            mode: 'offline',
+            observedAt: ()=>new Date('2026-08-30T12:00:00Z'),
+            providers: [
+                {
+                    id: 'mockprov',
+                    source_url: 'mock://mockprov',
+                    baseline_path: bp,
+                    snapshot_path: join(root, 'no-such-snapshot.html'),
+                    parse: parseOpenaiHtml
+                }
+            ]
+        });
+        assert.equal(report.providers['mockprov'].status, 'skipped_no_snapshot', 'snapshot 缺失 → SKIP');
+        ok('snapshot 缺失 → skipped_no_snapshot(offline)');
+    } finally{
+        rmSync(root, {
+            recursive: true,
+            force: true
+        });
+    }
+}
+await testSnapshotNextDataParser();
+await testSnapshotFallbackDrift();
+await testSnapshotMissing();
+console.log(`price-drift-watch tests: ${passed} 组断言全绿(offline,fetch 模式手动/CI 心跳)`);
+
+
+//# sourceURL=/Users/bytedance/work/gotry-publish/ts/scripts/price-drift-tests.ts