@danceiny/gotry 0.0.1-rc.9 → 0.2.0-rc.28

package/dist/scripts/sf-live-benchmark.js

@@ -0,0 +1,367 @@
+/**
+ * 真实 sf-01..08 双源 e2e 跑批(issue #21 验收清单,goal 2 sf-live-benchmark)
+ * ——pluggable golden 版:
+ *
+ * 之前版本绑 flyaiSearch 失败(Trial limit reached);本版改 pluggable:
+ *   - 默认 comparator = 手工 golden(data/sf-golden-manifest.json 时间/价格带,零网络零 vendor)
+ *   - 可选 --golden=flyai / --golden=static 显式切换；未知 vendor 在网络调用前 fail-closed
+ *   - static = OpenFlights 固定 route/carrier 快照 + manual 时间/价格带；估算字段与回退原因显式落证据
+ *   - 字段评分改软命中:硬字段(query_id/from/to/currency/source/verdict)必中,软字段
+ *     (transport_number/departure_at/arrival_at/price)落在窗口内算 correct
+ *
+ * 与 ts/scripts/session-benchmark.ts 的关系:session-benchmark.ts 是离线 fixture 自测
+ * (13/18 字段黄金 + 双源合同纯函数断言);sf-live-benchmark.ts 是真网络真扩展端到端。
+ *
+ * evidence 落盘:--evidence-root PATH/sf-XX/<ISO ts>.json;默认 ~/.gotry/evidence/session。
+ * 自动化测试必须显式传入隔离目录;--help 查看用法(issue #503)。
+ *
+ * 挑战红线(RFC §3.5,issue #411):任一 query 的 session verdict=challenged
+ * (或双源 state=challenge_stop/guard_violation)即停止本批后续查询——不重试、
+ * 不绕过,session 与 comparator 都不再调用;批次以部分结果落盘,stop_reason 标注
+ * challenge_stop/guard_violation,并给出 attempted/not_attempted 清单。sf-summary
+ * 侧缺条批次一律 fail_closed,不得标为完整或有效校准。
+ *
+ * 退出码:0 = 正常结束(跑满 8 条,或按红线提前停止;stop_reason 区分;
+ * 即便部分 query miss 也是 0);1 = 参数/加载异常。人类评审 evidence 决定 issue 是否可关
+ *
+ * 测试节律:同 query 间隔固定 35_000ms(§3.4 ≥30s)。离线 harness 通过子进程
+ * timer preload 加速，不向生产暴露节律逃生开关。
+ */
+import { mkdirSync, writeFileSync, readFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { parseSfLiveOptions, SF_LIVE_HELP } from './sf-live-options.js';
+import { validateSfManifest } from './sf-manifest.js';
+import { evaluateDoubleSource, SESSION_FIELD_ACCURACY_THRESHOLD, } from '../capabilities/session/benchmark.js';
+import { sessionFlightSearch } from '../capabilities/session-search.js';
+import { flyaiSearch } from '../capabilities/flyai.js';
+import { loadStaticFlightSnapshot, resolveStaticGolden, } from '../capabilities/session/static-flight-golden.js';
+import { scoreSessionAgainstGoldenBand, } from '../capabilities/session/golden-score.js';
+/** Read the source manifest before validating the frozen eight-case slice. */
+function loadFlightQueries() {
+    return JSON.parse(readFileSync(join(import.meta.dirname, '..', 'data', 'session-golden-20.json'), 'utf8'));
+}
+/** 从 sf-golden-manifest.json 拿手工 golden(默认 official 来源) */
+function loadManualGolden() {
+    return JSON.parse(readFileSync(join(import.meta.dirname, '..', 'data', 'sf-golden-manifest.json'), 'utf8'));
+}
+/** 手工 golden 的单条 → SessionComparableRecord */
+function manualToRecord(m, sample) {
+    return {
+        query_id: m.query_id,
+        route_segments: [{
+                from: m.from,
+                to: m.to,
+                departure_at: `${m.date}T${sample.depTime}:00+08:00`,
+                arrival_at: `${m.date}T${sample.arrTime}:00+08:00`,
+                transport_number: sample.flightNo,
+            }],
+        journey_type: 'direct',
+        currency: 'CNY',
+        price: sample.price,
+        source: 'manual-golden',
+        fetched_at: new Date().toISOString(),
+        verdict: 'hit',
+        latency_ms: 0,
+        read_guard_blocked: 0,
+    };
+}
+/** FlyAI vendor: 真打飞猪官方(可能 trial limit,落 verdict=error) */
+async function flyaiOfficial(q) {
+    const started = Date.now();
+    try {
+        const r = await flyaiSearch({ kind: 'flight', origin: q.from, destination: q.to, depDate: q.date, timeoutMs: 30_000 });
+        const latencyMs = Date.now() - started;
+        if (r.ok && r.options && r.options.length > 0) {
+            const sorted = r.options.slice().sort((a, b) => Date.parse(a.depDateTime) - Date.parse(b.depDateTime));
+            const opt = sorted[0];
+            const rec = {
+                query_id: q.id,
+                route_segments: [{
+                        from: q.from,
+                        to: q.to,
+                        departure_at: opt.depDateTime,
+                        arrival_at: opt.arrDateTime,
+                        transport_number: opt.no,
+                    }],
+                journey_type: 'direct',
+                currency: 'CNY',
+                price: opt.price,
+                source: 'flyai',
+                fetched_at: new Date().toISOString(),
+                verdict: 'hit',
+                latency_ms: latencyMs,
+                read_guard_blocked: 0,
+            };
+            return { record: rec, latencyMs };
+        }
+        return { record: { ...emptyOfficialRecord(q, 'flyai'), latency_ms: latencyMs, verdict: 'error' }, latencyMs, error: r.error ?? 'no options' };
+    }
+    catch (e) {
+        return { record: emptyOfficialRecord(q, 'flyai'), latencyMs: Date.now() - started, error: e instanceof Error ? e.message : String(e) };
+    }
+}
+/** 手工 golden vendor: 不打网络,从 manifest 取首个 known_flight 的时刻窗口算中位时间 + 价格带中位 */
+function manualOfficial(q, manifest) {
+    const started = Date.now();
+    const m = manifest.matches.find((x) => x.query_id === q.id);
+    if (!m)
+        return { record: emptyOfficialRecord(q, 'manual-golden'), latencyMs: Date.now() - started };
+    // 中位时间(早 + 晚 ÷ 2);中位价格(带中位)
+    const depMedian = hhmmMid(m.window_dep_local.earliest, m.window_dep_local.latest);
+    const arrMedian = hhmmMid(m.window_arr_local.earliest, m.window_arr_local.latest);
+    const priceMedian = Math.round((m.price_cny.min + m.price_cny.max) / 2);
+    return {
+        record: manualToRecord(m, { depTime: depMedian, arrTime: arrMedian, flightNo: m.known_flights[0] ?? '', price: priceMedian }),
+        latencyMs: Date.now() - started,
+    };
+}
+function hhmmMid(a, b) {
+    const aMin = parseInt(a.split(':')[0], 10) * 60 + parseInt(a.split(':')[1], 10);
+    const bMin = parseInt(b.split(':')[0], 10) * 60 + parseInt(b.split(':')[1], 10);
+    const mid = Math.round((aMin + bMin) / 2);
+    return `${Math.floor(mid / 60).toString().padStart(2, '0')}:${(mid % 60).toString().padStart(2, '0')}`;
+}
+function emptyOfficialRecord(q, source) {
+    return {
+        query_id: q.id,
+        route_segments: [],
+        journey_type: 'direct',
+        currency: '',
+        price: 0,
+        source,
+        fetched_at: new Date().toISOString(),
+        verdict: 'error',
+        latency_ms: 0,
+        read_guard_blocked: 0,
+    };
+}
+/** 生产节律(§3.4 同站点 ≥30s)。测试只可在隔离子进程 mock timer。 */
+const DEFAULT_INTER_QUERY_DELAY_MS = 35_000;
+async function runOne(q, requestedSource, manifest, staticContext) {
+    const startedAt = new Date().toISOString();
+    let official = null;
+    let officialLatencyMs = null;
+    let effectiveSource = requestedSource === 'manual' ? 'manual-golden' : requestedSource;
+    let fallbackReason = null;
+    let estimatedFields = [];
+    let officialProvenance = null;
+    if (requestedSource === 'manual') {
+        const m = manualOfficial(q, manifest);
+        official = m.record;
+        officialLatencyMs = m.latencyMs;
+    }
+    else if (requestedSource === 'flyai') {
+        const r = await flyaiOfficial(q);
+        official = r.record;
+        officialLatencyMs = r.latencyMs;
+    }
+    else {
+        const started = Date.now();
+        const manual = manualOfficial(q, manifest);
+        const resolution = resolveStaticGolden({
+            query: q,
+            manualRecord: manual.record ?? emptyOfficialRecord(q, 'manual-golden'),
+            snapshot: staticContext.snapshot,
+            snapshotError: staticContext.error,
+            warn: (message) => console.error(message),
+        });
+        official = resolution.record;
+        officialLatencyMs = Date.now() - started;
+        effectiveSource = resolution.effective_source;
+        fallbackReason = resolution.fallback_reason ?? null;
+        estimatedFields = resolution.estimated_fields;
+        officialProvenance = resolution.provenance;
+    }
+    // session = sessionFlightSearch(节律闸 ≥30s 由 caller sleep 35s 处理)
+    const sessStarted = Date.now();
+    const sessRes = await sessionFlightSearch({ from: q.from, to: q.to, date: q.date, timeoutMs: 25_000 });
+    const sessionLatencyMs = Date.now() - sessStarted;
+    const sessionVerdict = sessRes.verdict;
+    const sessionError = sessRes.error;
+    let session = null;
+    if (sessRes.verdict === 'hit' && sessRes.options && sessRes.options.length > 0) {
+        const sorted = sessRes.options.slice().sort((a, b) => Date.parse(a.depDateTime) - Date.parse(b.depDateTime));
+        const opt = sorted[0];
+        session = {
+            query_id: q.id,
+            route_segments: [{
+                    from: q.from,
+                    to: q.to,
+                    departure_at: opt.depDateTime,
+                    arrival_at: opt.arrDateTime,
+                    transport_number: opt.flightNo,
+                }],
+            journey_type: 'direct',
+            currency: 'CNY',
+            price: opt.price,
+            source: 'ctrip-flight',
+            fetched_at: new Date().toISOString(),
+            verdict: 'hit',
+            latency_ms: sessionLatencyMs,
+            read_guard_blocked: 0,
+        };
+    }
+    else {
+        // challenged 原样保留(issue #411):SessionVerdict 与 SessionEvidenceVerdict 同构,
+        // 改写为 error 会让 evaluateDoubleSource 丢失 challenge_stop 语义变成 source_unavailable
+        session = {
+            query_id: q.id,
+            route_segments: [],
+            journey_type: 'direct',
+            currency: '',
+            price: 0,
+            source: 'ctrip-flight',
+            fetched_at: new Date().toISOString(),
+            verdict: sessRes.verdict,
+            latency_ms: sessionLatencyMs,
+            read_guard_blocked: 0,
+        };
+    }
+    const doubleSource = evaluateDoubleSource({ official: official ?? undefined, session: session ?? undefined });
+    // 字段准确率始终对照同一 manual band；provider 只改变 official 来源与 provenance。
+    let softResult = null;
+    const m = manifest.matches.find((x) => x.query_id === q.id);
+    if (session?.verdict === 'hit' && m) {
+        softResult = scoreSessionAgainstGoldenBand(session, m, effectiveSource);
+    }
+    return {
+        query_id: q.id,
+        expected: { from: q.from, to: q.to, date: q.date, kind: q.kind },
+        requested_source: requestedSource,
+        effective_source: effectiveSource,
+        fallback_reason: fallbackReason,
+        comparator_evidence_scope: requestedSource === 'flyai' ? 'live_provider' : 'deterministic_only',
+        estimated_fields: estimatedFields,
+        official_provenance: officialProvenance,
+        official,
+        official_source: effectiveSource,
+        session,
+        doubleSource,
+        softScore: softResult,
+        officialLatencyMs,
+        sessionLatencyMs,
+        sessionVerdict: String(sessionVerdict),
+        sessionTransportShape: sessRes.transportShape ?? 'not_observed',
+        sessionError,
+        started_at: startedAt,
+    };
+}
+async function main() {
+    const args = process.argv.slice(2);
+    const { evidenceRoot, source: requestedSource, help } = parseSfLiveOptions(args);
+    if (help) {
+        console.log(SF_LIVE_HELP);
+        return;
+    }
+    const manifest = loadManualGolden();
+    const queries = validateSfManifest(loadFlightQueries(), manifest);
+    // Fail before supplier calls if the destination cannot be created (e.g. a file).
+    mkdirSync(evidenceRoot, { recursive: true });
+    const staticContext = {};
+    if (requestedSource === 'static') {
+        try {
+            staticContext.snapshot = loadStaticFlightSnapshot();
+        }
+        catch (error) {
+            staticContext.error = error instanceof Error ? error.message : String(error);
+        }
+    }
+    const sourceDescription = requestedSource === 'manual'
+        ? '手工 golden (sf-golden-manifest.json,零网络零 vendor)'
+        : requestedSource === 'flyai'
+            ? 'flyai (FlyAI 官方)'
+            : 'static (OpenFlights route/carrier + manual time/price band,估算字段显式标注)';
+    console.log(`[sf-live-benchmark] requested official source = ${sourceDescription}`);
+    console.log(`[sf-live-benchmark] evidence root = ${evidenceRoot}`);
+    console.log(`[sf-live-benchmark] 阈值:字段准确率 ≥${SESSION_FIELD_ACCURACY_THRESHOLD * 100}%(软命中)`);
+    const runStartedAt = new Date().toISOString();
+    const runStartedMs = Date.now();
+    const records = [];
+    const attemptedQueryIds = [];
+    let stopReason = 'completed';
+    for (const q of queries) {
+        if (records.length > 0)
+            await new Promise((r) => setTimeout(r, DEFAULT_INTER_QUERY_DELAY_MS));
+        console.log(`\n[sf-live-benchmark] ${q.id} ${q.from}→${q.to} ${q.date}`);
+        const rec = await runOne(q, requestedSource, manifest, staticContext);
+        records.push(rec);
+        attemptedQueryIds.push(q.id);
+        const evPath = join(evidenceRoot, q.id, `${runStartedAt.replace(/[:.]/g, '-')}.json`);
+        mkdirSync(dirname(evPath), { recursive: true });
+        writeFileSync(evPath, JSON.stringify(rec, null, 2));
+        console.log(`  official(requested=${rec.requested_source},effective=${rec.effective_source}): verdict=${rec.official?.verdict ?? '-'} latency=${rec.officialLatencyMs ?? 0}ms${rec.fallback_reason ? ` fallback=${rec.fallback_reason}` : ''}`);
+        console.log(`  session(ctrip): verdict=${rec.sessionVerdict} latency=${rec.sessionLatencyMs ?? 0}ms${rec.sessionError ? ` err=${rec.sessionError.slice(0, 80)}` : ''}`);
+        console.log(`  doubleSource: state=${rec.doubleSource.state} quota=${rec.doubleSource.quota_disposition} price_delta=${rec.doubleSource.price_delta ?? '-'} mismatches=${rec.doubleSource.mismatches.length}`);
+        if (rec.softScore) {
+            console.log(`  soft score: ${(rec.softScore.accuracy * 100).toFixed(1)}% (${rec.softScore.correct}/${rec.softScore.total}) ${rec.softScore.pass ? '✅' : '❌'} missing=${JSON.stringify(rec.softScore.missing)} incorrect=${JSON.stringify(rec.softScore.incorrect)}`);
+        }
+        // RFC §3.5 红线:风控/验证码或读守卫违例即停止本批后续查询——不重试、不绕过;
+        // session 与 comparator 都不再被调用,余下 query 记入 not_attempted(issue #411)
+        if (rec.sessionVerdict === 'challenged' || rec.doubleSource.state === 'challenge_stop') {
+            stopReason = 'challenge_stop';
+            console.log(`  [sf-live-benchmark] 挑战触发(verdict=${rec.sessionVerdict},state=${rec.doubleSource.state}):停止本批后续查询(RFC §3.5;不重试不绕过)`);
+            break;
+        }
+        if (rec.doubleSource.state === 'guard_violation') {
+            stopReason = 'guard_violation';
+            console.log(`  [sf-live-benchmark] 读守卫违例(state=guard_violation):停止本批后续查询(RFC §3.5;不重试不绕过)`);
+            break;
+        }
+    }
+    const attemptedSet = new Set(attemptedQueryIds);
+    const notAttemptedQueryIds = queries.filter((q) => !attemptedSet.has(q.id)).map((q) => q.id);
+    const accuracy_pass = records.filter((r) => r.softScore?.pass === true).length;
+    const comparable = records.filter((r) => r.doubleSource.state === 'comparable').length;
+    const liveProviderComparable = records.filter((r) => r.comparator_evidence_scope === 'live_provider' && r.doubleSource.state === 'comparable').length;
+    const hit = records.filter((r) => r.sessionVerdict === 'hit').length;
+    const challenge = records.filter((r) => r.sessionVerdict === 'challenged' || r.doubleSource.state === 'challenge_stop' || r.doubleSource.state === 'guard_violation').length;
+    const no_data = records.filter((r) => r.official?.verdict !== 'hit' || (r.sessionVerdict !== 'hit' && r.sessionVerdict !== 'challenged')).length;
+    const live_under_15s = records.filter((r) => (r.sessionLatencyMs !== null && r.sessionVerdict === 'hit' && r.sessionLatencyMs < 15_000)).length;
+    const effectiveSources = Array.from(new Set(records.map((record) => record.effective_source)));
+    const fallbackCount = records.filter((record) => record.fallback_reason !== null).length;
+    const summary = {
+        started_at: runStartedAt,
+        duration_ms: Date.now() - runStartedMs,
+        threshold: SESSION_FIELD_ACCURACY_THRESHOLD,
+        total: records.length,
+        accuracy_pass,
+        comparable,
+        live_provider_comparable: liveProviderComparable,
+        hit,
+        challenge,
+        no_data,
+        live_under_15s,
+        requested_source: requestedSource,
+        effective_sources: effectiveSources,
+        fallback_count: fallbackCount,
+        golden_source: requestedSource === 'manual' ? 'manual-golden' : requestedSource,
+        comparator_evidence_scope: requestedSource === 'flyai' ? 'live_provider' : 'deterministic_only',
+        batch_complete: stopReason === 'completed' && records.length === queries.length,
+        stop_reason: stopReason,
+        attempted_query_ids: attemptedQueryIds,
+        not_attempted_query_ids: notAttemptedQueryIds,
+        records,
+    };
+    const summaryPath = join(evidenceRoot, 'sf-summary', `${runStartedAt.replace(/[:.]/g, '-')}.json`);
+    mkdirSync(dirname(summaryPath), { recursive: true });
+    writeFileSync(summaryPath, JSON.stringify(summary, null, 2));
+    console.log(`\n──── sf-live-benchmark 汇总 ────`);
+    console.log(`golden requested = ${summary.requested_source}`);
+    console.log(`comparator evidence scope = ${summary.comparator_evidence_scope}${summary.comparator_evidence_scope === 'deterministic_only' ? ' (not live provider inventory evidence)' : ''}`);
+    console.log(`golden effective = ${summary.effective_sources.join(',')};fallback=${summary.fallback_count}`);
+    console.log(`跑批 query 数: ${records.length}`);
+    console.log(`stop reason: ${summary.stop_reason}${summary.stop_reason !== 'completed' ? `;attempted=[${summary.attempted_query_ids.join(',')}] not_attempted=[${summary.not_attempted_query_ids.join(',')}]` : ''}`);
+    console.log(`verdict=hit: ${hit}/${records.length}`);
+    console.log(`双源合同=comparable: ${comparable}/${records.length}`);
+    console.log(`live-provider comparable: ${liveProviderComparable}/${records.length}`);
+    console.log(`字段准确率 ≥${SESSION_FIELD_ACCURACY_THRESHOLD * 100}% (软命中): ${accuracy_pass}/${records.filter((r) => r.softScore !== null).length}`);
+    console.log(`live <15s (hit): ${live_under_15s}/${hit || 0}`);
+    console.log(`challenge/guard 触发: ${challenge}`);
+    console.log(`总耗时: ${summary.duration_ms}ms`);
+    process.exit(0);
+}
+main().catch((e) => {
+    console.error(`[sf-live-benchmark] 异常: ${e instanceof Error ? e.message : String(e)}`);
+    console.error(e instanceof Error ? e.stack : '');
+    process.exit(1);
+});