@danceiny/gotry 0.0.1-rc.14 → 0.0.1-rc.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -19
- package/README.zh-CN.md +21 -19
- package/bin/gotry-bootstrap.js +39 -0
- package/dist/capabilities/artifacts.js +217 -0
- package/dist/scripts/booking-saga-tests.js +187 -0
- package/dist/scripts/bootstrap-tests.js +12 -2
- package/dist/scripts/nightly-evidence-tests.js +123 -0
- package/dist/scripts/nightly-evidence.js +233 -0
- package/dist/scripts/smoke.js +83 -0
- package/dist/src/booking-saga.js +153 -0
- package/dist/src/dsh-llm.js +30 -6
- package/dist/src/index.js +133 -0
- package/package.json +4 -2
- package/ts/capabilities/artifacts.ts +235 -0
- package/ts/src/booking-saga.ts +133 -0
- package/ts/src/dsh-llm.ts +40 -3
- package/ts/src/index.ts +128 -20
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
import { mkdtempSync, rmSync } from 'node:fs';
|
|
2
|
+
import { tmpdir } from 'node:os';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { BOOKING_SAGA_SCHEMA, SAGA_EDGES, SAGA_REJECTIONS, SAGA_STATUSES, SAGA_TRIGGER_EVENT, resolveSagaTrigger, sagaLegalPaths, sagaTraceViolations } from '../src/booking-saga.js';
|
|
5
|
+
import { ensureLedger } from '../src/state-ledger.js';
|
|
6
|
+
let pass = 0;
|
|
7
|
+
let fail = 0;
|
|
8
|
+
function assert(cond, msg) {
|
|
9
|
+
if (cond) {
|
|
10
|
+
pass++;
|
|
11
|
+
console.log(` ok - ${msg}`);
|
|
12
|
+
} else {
|
|
13
|
+
fail++;
|
|
14
|
+
console.error(` FAIL - ${msg}`);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
function parseReceipt(payload) {
|
|
18
|
+
try {
|
|
19
|
+
const doc = JSON.parse(payload);
|
|
20
|
+
return typeof doc.receipt === 'string' && doc.receipt !== '' ? doc.receipt : undefined;
|
|
21
|
+
} catch {
|
|
22
|
+
return undefined;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
const TRIGGERS = [
|
|
26
|
+
'propose',
|
|
27
|
+
'confirm',
|
|
28
|
+
'compensate'
|
|
29
|
+
];
|
|
30
|
+
const ORIGINS = [
|
|
31
|
+
'none',
|
|
32
|
+
...SAGA_STATUSES
|
|
33
|
+
];
|
|
34
|
+
const COMBOS = ORIGINS.flatMap((from)=>TRIGGERS.map((trigger)=>`${from}:${trigger}`));
|
|
35
|
+
const edgeKeys = new Set(SAGA_EDGES.map((e)=>`${e.from}:${e.trigger}`));
|
|
36
|
+
const rejectKeys = new Set(SAGA_REJECTIONS.map((e)=>`${e.from}:${e.trigger}`));
|
|
37
|
+
assert(BOOKING_SAGA_SCHEMA === 'booking_saga_fsm.v1' && SAGA_STATUSES.join(',') === 'pending,confirmed,compensated', 'schema 词唯一;状态字母表与 pending_writes CHECK 逐字一致');
|
|
38
|
+
assert(COMBOS.length === 12 && new Set(COMBOS).size === 12 && edgeKeys.size === SAGA_EDGES.length && rejectKeys.size === SAGA_REJECTIONS.length, '12 个 (origin,trigger) 组合;边表与拒绝表内部无重复');
|
|
39
|
+
assert(COMBOS.every((k)=>edgeKeys.has(k) !== rejectKeys.has(k)), '每格恰为一条合法边或一条结构化拒绝(全函数,无空洞无第三态)');
|
|
40
|
+
assert(SAGA_EDGES.every((e)=>SAGA_STATUSES.includes(e.to) && e.event === SAGA_TRIGGER_EVENT[e.trigger] && e.guard.length > 0), '合法边:终点在字母表内、事件 kind 与触发器一一对应、守卫语义齐全');
|
|
41
|
+
const allConsistent = ORIGINS.every((from)=>TRIGGERS.every((trigger)=>{
|
|
42
|
+
const v = resolveSagaTrigger(from, trigger);
|
|
43
|
+
const edge = SAGA_EDGES.find((e)=>e.from === from && e.trigger === trigger);
|
|
44
|
+
if (edge) return v.ok && v.to === edge.to && v.event === edge.event;
|
|
45
|
+
return !v.ok && v.reason === SAGA_REJECTIONS.find((x)=>x.from === from && x.trigger === trigger).reason;
|
|
46
|
+
}));
|
|
47
|
+
assert(allConsistent, 'resolveSagaTrigger 十二格与边表/拒绝表全一致(解析器无平行第二逻辑)');
|
|
48
|
+
{
|
|
49
|
+
const vProp = resolveSagaTrigger('none', 'propose');
|
|
50
|
+
const vConf = resolveSagaTrigger('pending', 'confirm');
|
|
51
|
+
const vComp = resolveSagaTrigger('confirmed', 'compensate');
|
|
52
|
+
assert(vProp.ok && vProp.to === 'pending' && vConf.ok && vConf.to === 'confirmed' && vComp.ok && vComp.to === 'compensated', '主路径 ∅→pending→confirmed→compensated 连通(L2 登记只不执行 / L3 确认携 receipt / saga 补偿)');
|
|
53
|
+
const vDouble = resolveSagaTrigger('confirmed', 'confirm');
|
|
54
|
+
assert(!vDouble.ok && vDouble.reason === 'already-confirmed', '二次确认结构化拒绝(already-confirmed)');
|
|
55
|
+
const vIdem = resolveSagaTrigger('compensated', 'propose');
|
|
56
|
+
const vRevive = resolveSagaTrigger('compensated', 'confirm');
|
|
57
|
+
const vReComp = resolveSagaTrigger('compensated', 'compensate');
|
|
58
|
+
assert(!vIdem.ok && vIdem.reason === 'idem-exists' && !vRevive.ok && vRevive.reason === 'absorbed-compensated' && !vReComp.ok && vReComp.reason === 'already-compensated', 'compensated 吸收态:propose=幂等存在,confirm/compensate 拒绝(无复活路径)');
|
|
59
|
+
assert(sagaLegalPaths().length === 4, '合法路径恰四条(主路径及其真前缀)');
|
|
60
|
+
const legalClean = sagaLegalPaths().every((path)=>{
|
|
61
|
+
const events = path.map((trigger, i)=>({
|
|
62
|
+
seq: i + 1,
|
|
63
|
+
kind: SAGA_TRIGGER_EVENT[trigger],
|
|
64
|
+
receipt: trigger === 'confirm' ? 'PNR-X' : undefined
|
|
65
|
+
}));
|
|
66
|
+
return sagaTraceViolations(events).length === 0;
|
|
67
|
+
});
|
|
68
|
+
assert(legalClean, '四条合法路径的审计链均零违例');
|
|
69
|
+
const flagOf = (events)=>sagaTraceViolations(events)[0];
|
|
70
|
+
assert(flagOf([
|
|
71
|
+
{
|
|
72
|
+
kind: 'write.confirmed',
|
|
73
|
+
receipt: 'PNR-A'
|
|
74
|
+
}
|
|
75
|
+
]) !== undefined, '缺 pending 起点的 confirm 被审计链抓出');
|
|
76
|
+
assert(flagOf([
|
|
77
|
+
{
|
|
78
|
+
kind: 'write.pending'
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
kind: 'write.confirmed',
|
|
82
|
+
receipt: 'PNR-A'
|
|
83
|
+
},
|
|
84
|
+
{
|
|
85
|
+
kind: 'write.confirmed',
|
|
86
|
+
receipt: 'PNR-B'
|
|
87
|
+
}
|
|
88
|
+
]) !== undefined, '双确认被审计链抓出(每键至多确认一次)');
|
|
89
|
+
assert(flagOf([
|
|
90
|
+
{
|
|
91
|
+
kind: 'write.pending'
|
|
92
|
+
},
|
|
93
|
+
{
|
|
94
|
+
kind: 'write.compensated'
|
|
95
|
+
},
|
|
96
|
+
{
|
|
97
|
+
kind: 'write.confirmed',
|
|
98
|
+
receipt: 'X'
|
|
99
|
+
}
|
|
100
|
+
]) !== undefined, 'compensated 之后不得再有 saga 事件(吸收态)');
|
|
101
|
+
assert((flagOf([
|
|
102
|
+
{
|
|
103
|
+
kind: 'write.pending'
|
|
104
|
+
},
|
|
105
|
+
{
|
|
106
|
+
kind: 'write.confirmed'
|
|
107
|
+
}
|
|
108
|
+
]) ?? '').includes('receipt'), 'write.confirmed 空 receipt 被抓(L3 必携外部回执)');
|
|
109
|
+
}const root = mkdtempSync(join(tmpdir(), 'gotry-booking-saga-'));
|
|
110
|
+
try {
|
|
111
|
+
const ledger = ensureLedger(root);
|
|
112
|
+
const sagaEventsOf = (l, key)=>{
|
|
113
|
+
const rows = l.db.prepare(`SELECT seq, kind, payload FROM events WHERE tenant_id = ? AND subject_id = ? AND kind IN ('write.pending','write.confirmed','write.compensated') ORDER BY seq`).all(l.tenant, key);
|
|
114
|
+
return rows.map((r)=>({
|
|
115
|
+
seq: r.seq,
|
|
116
|
+
kind: r.kind,
|
|
117
|
+
receipt: parseReceipt(r.payload)
|
|
118
|
+
}));
|
|
119
|
+
};
|
|
120
|
+
const K1 = 'booking:K1';
|
|
121
|
+
const proposed = ledger.requestPendingWrite({
|
|
122
|
+
idemKey: K1,
|
|
123
|
+
seam: 'flight-order-confirm',
|
|
124
|
+
payload: {
|
|
125
|
+
flight: 'MU123'
|
|
126
|
+
}
|
|
127
|
+
});
|
|
128
|
+
assert(proposed.created === true && proposed.status === 'pending' && resolveSagaTrigger('none', 'propose').to === 'pending', '物理提议 ∅→pending 与状态机同归宿(L2 只登记不执行)');
|
|
129
|
+
const dup = ledger.requestPendingWrite({
|
|
130
|
+
idemKey: K1,
|
|
131
|
+
seam: 'flight-order-confirm',
|
|
132
|
+
payload: {
|
|
133
|
+
flight: 'MU123'
|
|
134
|
+
}
|
|
135
|
+
});
|
|
136
|
+
assert(dup.created === false && dup.status === 'pending' && resolveSagaTrigger('pending', 'propose').reason === 'idem-exists', '同幂等键重复提议:账本 no-op = 状态机 idem-exists');
|
|
137
|
+
const confirmed = ledger.confirmPendingWrite(K1, 'PNR-A');
|
|
138
|
+
assert(confirmed.ok === true && confirmed.status === 'confirmed' && resolveSagaTrigger('pending', 'confirm').to === 'confirmed', 'L3 具名确认 pending→confirmed(携 receipt)');
|
|
139
|
+
const reConfirmed = ledger.confirmPendingWrite(K1, 'PNR-SHOULD-IGNORE');
|
|
140
|
+
assert(reConfirmed.ok === false && (ledger.listPendingWrites().find((w)=>w.idem_key === K1)?.receipt ?? null) === 'PNR-A' && resolveSagaTrigger('confirmed', 'confirm').reason === 'already-confirmed', '二次确认被账本守卫拒绝且 receipt 不可变(仍为 PNR-A)');
|
|
141
|
+
const compensated = ledger.compensatePendingWrite(K1, '用户改签退款');
|
|
142
|
+
assert(compensated.ok === true && ledger.listPendingWrites().find((w)=>w.idem_key === K1)?.status === 'compensated', '已确认副作用可 saga 补偿(pending|confirmed → compensated)');
|
|
143
|
+
const afterConfirm = ledger.confirmPendingWrite(K1, 'PNR-POST');
|
|
144
|
+
const afterCompensate = ledger.compensatePendingWrite(K1, '重复补偿');
|
|
145
|
+
const afterPropose = ledger.requestPendingWrite({
|
|
146
|
+
idemKey: K1,
|
|
147
|
+
seam: 'flight-order-confirm',
|
|
148
|
+
payload: {}
|
|
149
|
+
});
|
|
150
|
+
assert(afterConfirm.ok === false && afterCompensate.ok === false && afterPropose.created === false, '吸收态:compensated 后 confirm/compensate/propose 全拒,零复活');
|
|
151
|
+
const audit = sagaEventsOf(ledger, K1);
|
|
152
|
+
assert(audit.map((e)=>e.kind).join(',') === 'write.pending,write.confirmed,write.compensated', '审计事件链与边表路径同构(被拒调用零事件)');
|
|
153
|
+
assert(sagaTraceViolations(audit).length === 0, '真实账本审计链过词汇层校验零违例');
|
|
154
|
+
const K3 = 'booking:K3';
|
|
155
|
+
ledger.requestPendingWrite({
|
|
156
|
+
idemKey: K3,
|
|
157
|
+
seam: 'hotel-order-confirm',
|
|
158
|
+
payload: {}
|
|
159
|
+
});
|
|
160
|
+
const emptyReceipt = ledger.confirmPendingWrite(K3, '');
|
|
161
|
+
assert(emptyReceipt.ok === true && sagaTraceViolations(sagaEventsOf(ledger, K3)).some((v)=>v.includes('receipt')), '已知边界:账本物理层暂不挡空 receipt,词汇层审计链抓出(物理 CHECK 随 M5 拍板入 schema)');
|
|
162
|
+
const other = ensureLedger(root, 'tenant-b');
|
|
163
|
+
const crossProp = other.requestPendingWrite({
|
|
164
|
+
idemKey: K1,
|
|
165
|
+
seam: 'flight-order-confirm',
|
|
166
|
+
payload: {}
|
|
167
|
+
});
|
|
168
|
+
assert(crossProp.created === true, '同 idem_key 跨租户互不可见:tenant-b 可建自己的 K1(tenant_id 一等字段,ADR-16)');
|
|
169
|
+
const LOCAL_ONLY = 'booking:local-only';
|
|
170
|
+
ledger.requestPendingWrite({
|
|
171
|
+
idemKey: LOCAL_ONLY,
|
|
172
|
+
seam: 'flight-order-confirm',
|
|
173
|
+
payload: {}
|
|
174
|
+
});
|
|
175
|
+
const crossMissing = other.confirmPendingWrite(LOCAL_ONLY, 'PNR-CROSS');
|
|
176
|
+
assert(crossMissing.ok === false && crossMissing.status === 'missing', 'tenant-b 看不到 local 的 saga 键(拒绝 missing)');
|
|
177
|
+
} finally{
|
|
178
|
+
rmSync(root, {
|
|
179
|
+
recursive: true,
|
|
180
|
+
force: true
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
console.log(`\nBOOKING SAGA TESTS: ${pass} ok, ${fail} fail`);
|
|
184
|
+
if (fail > 0) process.exit(1);
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
//# sourceURL=/Users/bytedance/work/gotry/ts/scripts/booking-saga-tests.ts
|
|
@@ -35,7 +35,8 @@ assert.equal(c1.code, 0, `--check-only 应 exit 0,实际 ${c1.code}\n${c1.out}`)
|
|
|
35
35
|
assert.ok(c1.out.includes('hbcli'), '报告应含 hbcli 节');
|
|
36
36
|
assert.ok(c1.out.includes('agent-reach'), '报告应含 agent-reach 节');
|
|
37
37
|
assert.ok(c1.out.includes('flyai'), '报告应含 flyai(无需安装)节');
|
|
38
|
-
|
|
38
|
+
assert.ok(c1.out.includes('dsh-better-sidebar'), '报告应含 dsh-better-sidebar 节(issue #25 产物查看面)');
|
|
39
|
+
console.log('1. --check-only 探测报告 exit 0(hbcli/agent-reach/flyai/dsh-better-sidebar 四节齐)OK');
|
|
39
40
|
const c2 = runBootstrap([
|
|
40
41
|
'--auto'
|
|
41
42
|
], {
|
|
@@ -50,7 +51,16 @@ const c3 = runBootstrap([], {
|
|
|
50
51
|
assert.equal(c3.code, 0, '显式模式跳过态应 exit 0');
|
|
51
52
|
assert.ok(c3.out.includes('跳过'), '应输出跳过说明');
|
|
52
53
|
console.log('3. 显式模式 + GOTRY_SETUP_SKIP=1 → exit 0 OK');
|
|
53
|
-
|
|
54
|
+
const c4 = runBootstrap([
|
|
55
|
+
'--check-only'
|
|
56
|
+
], {
|
|
57
|
+
GOTRY_SETUP_SIDEBAR: '0'
|
|
58
|
+
});
|
|
59
|
+
assert.equal(c4.code, 0, '单项跳过态应 exit 0');
|
|
60
|
+
assert.ok(c4.out.includes('GOTRY_SETUP_SIDEBAR=0 跳过'), '应输出侧栏单项跳过说明');
|
|
61
|
+
assert.ok(c4.out.includes('hbcli'), '其余节不受单项开关影响');
|
|
62
|
+
console.log('4. GOTRY_SETUP_SIDEBAR=0 单项跳过(其余节照常)OK');
|
|
63
|
+
console.log('BOOTSTRAP TESTS: 4/4 OK(探测报告/跳过开关/单项开关/postinstall 非致命)');
|
|
54
64
|
|
|
55
65
|
|
|
56
66
|
//# sourceURL=/Users/bytedance/work/gotry/ts/scripts/bootstrap-tests.ts
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import { existsSync, mkdtempSync, rmSync } from 'node:fs';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { tmpdir } from 'node:os';
|
|
5
|
+
import { buildNightlyRunRecord, loadPriceTable, loadPromptSet, priceRunCost, runNightlyEvidence } from './nightly-evidence.js';
|
|
6
|
+
import { parseNightlyRun } from './product-metrics.js';
|
|
7
|
+
let passed = 0;
|
|
8
|
+
function ok(label) {
|
|
9
|
+
passed += 1;
|
|
10
|
+
console.log(` ok ${label}`);
|
|
11
|
+
}
|
|
12
|
+
const priceTable = loadPriceTable('data/llm-price-table.json');
|
|
13
|
+
const promptSet = loadPromptSet('data/m3-nightly-prompts.json');
|
|
14
|
+
assert.equal(priceTable.schema_version, 'gotry_llm_price_table_v1');
|
|
15
|
+
assert.equal(promptSet.schema_version, 'gotry_m3_nightly_prompt_set_v1');
|
|
16
|
+
assert.ok(promptSet.turns.length >= 1);
|
|
17
|
+
ok('封存数据文件 schema 与轮次');
|
|
18
|
+
for (const [name, entry] of Object.entries(priceTable.prices)){
|
|
19
|
+
assert.ok(entry.input_cache_hit_usd_per_1m_peak <= entry.input_cache_miss_usd_per_1m_peak, `${name} hit<=miss`);
|
|
20
|
+
assert.ok(entry.output_usd_per_1m_offpeak <= entry.output_usd_per_1m_peak, `${name} offpeak<=peak`);
|
|
21
|
+
}
|
|
22
|
+
ok('价目保守上界不变量(hit<=miss, offpeak<=peak)');
|
|
23
|
+
const baseUsage = {
|
|
24
|
+
calls: 3,
|
|
25
|
+
inputTokens: 1_000_000,
|
|
26
|
+
outputTokens: 1_000_000,
|
|
27
|
+
inputCacheHitTokens: 0,
|
|
28
|
+
inputCacheMissTokens: 1_000_000,
|
|
29
|
+
responsesMissingUsage: 0
|
|
30
|
+
};
|
|
31
|
+
const cost = priceRunCost('deepseek-chat', baseUsage, priceTable);
|
|
32
|
+
assert.equal(cost, 1.76);
|
|
33
|
+
ok('peak miss 换算 0.44+1.32=1.76(经 alias)');
|
|
34
|
+
const hitUsage = {
|
|
35
|
+
...baseUsage,
|
|
36
|
+
inputCacheHitTokens: 1_000_000,
|
|
37
|
+
inputCacheMissTokens: 0
|
|
38
|
+
};
|
|
39
|
+
assert.equal(priceRunCost('deepseek-chat', hitUsage, priceTable), 1.334);
|
|
40
|
+
ok('peak hit 换算 0.014+1.32=1.334');
|
|
41
|
+
assert.throws(()=>priceRunCost('unknown-model-xyz', baseUsage, priceTable), /no entry for model/);
|
|
42
|
+
const dangling = {
|
|
43
|
+
...priceTable,
|
|
44
|
+
aliases: {
|
|
45
|
+
'deepseek-chat': 'not-a-model'
|
|
46
|
+
}
|
|
47
|
+
};
|
|
48
|
+
assert.throws(()=>priceRunCost('deepseek-chat', baseUsage, dangling), /has no entry/);
|
|
49
|
+
assert.throws(()=>priceRunCost('deepseek-chat', {
|
|
50
|
+
...baseUsage,
|
|
51
|
+
responsesMissingUsage: 1
|
|
52
|
+
}, priceTable), /cost unprovable/);
|
|
53
|
+
ok('未知模型/悬空alias/usage缺失 fail-closed');
|
|
54
|
+
const executeAt = '2026-08-29T12:00:00.000Z';
|
|
55
|
+
const recordInput = {
|
|
56
|
+
schema_version: 'gotry_m3_nightly_run_v1',
|
|
57
|
+
executed_at: executeAt,
|
|
58
|
+
real_llm: true,
|
|
59
|
+
prompt_set_sha256: 'a'.repeat(64),
|
|
60
|
+
output_sha256: 'b'.repeat(64),
|
|
61
|
+
cost_usd: 0.25
|
|
62
|
+
};
|
|
63
|
+
const rec1 = buildNightlyRunRecord(recordInput);
|
|
64
|
+
const rec2 = buildNightlyRunRecord({
|
|
65
|
+
...recordInput
|
|
66
|
+
});
|
|
67
|
+
assert.equal(rec1.run_key, rec2.run_key);
|
|
68
|
+
assert.match(rec1.run_key, /^hmac-sha256:[0-9a-f]{64}$/);
|
|
69
|
+
parseNightlyRun(JSON.parse(JSON.stringify(rec1)), 0);
|
|
70
|
+
const rec3 = buildNightlyRunRecord({
|
|
71
|
+
...recordInput,
|
|
72
|
+
cost_usd: 0.26
|
|
73
|
+
});
|
|
74
|
+
assert.notEqual(rec3.run_key, rec1.run_key);
|
|
75
|
+
ok('run_key 确定性 + 已过消费方 parseNightlyRun');
|
|
76
|
+
const prevKey = process.env['LLM_API_KEY'];
|
|
77
|
+
const prevDeepseekKey = process.env['DEEPSEEK_API_KEY'];
|
|
78
|
+
const waitParent = mkdtempSync(join(tmpdir(), 'nightly-wait-'));
|
|
79
|
+
const waitRoot = join(waitParent, 'evidence', 'm3');
|
|
80
|
+
try {
|
|
81
|
+
delete process.env['LLM_API_KEY'];
|
|
82
|
+
delete process.env['DEEPSEEK_API_KEY'];
|
|
83
|
+
const waiting = await runNightlyEvidence({
|
|
84
|
+
evidenceRoot: waitRoot,
|
|
85
|
+
dryRun: false,
|
|
86
|
+
clock: ()=>new Date(executeAt)
|
|
87
|
+
});
|
|
88
|
+
assert.equal(waiting.state, 'waiting_external_evidence');
|
|
89
|
+
assert.equal(waiting.record, null);
|
|
90
|
+
assert.ok(waiting.reason?.includes('backoff'));
|
|
91
|
+
assert.equal(existsSync(waitRoot), false, 'waiting 不得创建任何证据目录');
|
|
92
|
+
} finally{
|
|
93
|
+
if (prevKey !== undefined) process.env['LLM_API_KEY'] = prevKey;
|
|
94
|
+
if (prevDeepseekKey !== undefined) process.env['DEEPSEEK_API_KEY'] = prevDeepseekKey;
|
|
95
|
+
rmSync(waitParent, {
|
|
96
|
+
recursive: true,
|
|
97
|
+
force: true
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
ok('无凭证 → waiting_external_evidence 且零写入');
|
|
101
|
+
const dryParent = mkdtempSync(join(tmpdir(), 'nightly-dry-'));
|
|
102
|
+
const dryRoot = join(dryParent, 'evidence', 'm3');
|
|
103
|
+
try {
|
|
104
|
+
const dry = await runNightlyEvidence({
|
|
105
|
+
evidenceRoot: dryRoot,
|
|
106
|
+
dryRun: true,
|
|
107
|
+
clock: ()=>new Date(executeAt)
|
|
108
|
+
});
|
|
109
|
+
assert.equal(dry.state, 'dry_run');
|
|
110
|
+
assert.equal(dry.record?.real_llm, false);
|
|
111
|
+
parseNightlyRun(JSON.parse(JSON.stringify(dry.record)), 0);
|
|
112
|
+
assert.equal(existsSync(dryRoot), false, 'dry-run 不得写任何证据文件');
|
|
113
|
+
} finally{
|
|
114
|
+
rmSync(dryParent, {
|
|
115
|
+
recursive: true,
|
|
116
|
+
force: true
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
ok('dry-run mock 全链演练 + real_llm=false + 零落盘');
|
|
120
|
+
console.log(`nightly-evidence tests: ${passed} 组断言全绿(offline,真跑不在 CI)`);
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
//# sourceURL=/Users/bytedance/work/gotry/ts/scripts/nightly-evidence-tests.ts
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { appendFileSync, mkdirSync, readFileSync } from 'node:fs';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { createMockLlm } from '../src/mock-llm.js';
|
|
5
|
+
import { createOpenAICompatLlm } from '../src/dsh-llm.js';
|
|
6
|
+
import { newState, runTurn } from '../src/loop.js';
|
|
7
|
+
import { solveUnified } from '../src/unified.js';
|
|
8
|
+
import { realtimeSolvePort } from '../src/realtime-pricing.js';
|
|
9
|
+
import { parseNightlyRun } from './product-metrics.js';
|
|
10
|
+
const SHA256 = /^[0-9a-f]{64}$/;
|
|
11
|
+
const DEFAULT_BUDGET_USD = 1;
|
|
12
|
+
function canonical(value) {
|
|
13
|
+
if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`;
|
|
14
|
+
if (typeof value === 'object' && value !== null) {
|
|
15
|
+
const record = value;
|
|
16
|
+
return `{${Object.keys(record).sort().map((key)=>`${JSON.stringify(key)}:${canonical(record[key])}`).join(',')}}`;
|
|
17
|
+
}
|
|
18
|
+
return JSON.stringify(value);
|
|
19
|
+
}
|
|
20
|
+
function sha256Of(value) {
|
|
21
|
+
return createHash('sha256').update(canonical(value)).digest('hex');
|
|
22
|
+
}
|
|
23
|
+
function round6(value) {
|
|
24
|
+
return Number(value.toFixed(6));
|
|
25
|
+
}
|
|
26
|
+
export function loadPromptSet(path) {
|
|
27
|
+
const raw = JSON.parse(readFileSync(path, 'utf8'));
|
|
28
|
+
if (raw['schema_version'] !== 'gotry_m3_nightly_prompt_set_v1') {
|
|
29
|
+
throw new Error(`prompt set schema_version must be gotry_m3_nightly_prompt_set_v1, got: ${String(raw['schema_version'])}`);
|
|
30
|
+
}
|
|
31
|
+
const turns = raw['turns'];
|
|
32
|
+
if (!Array.isArray(turns) || turns.length === 0 || turns.some((t)=>typeof t !== 'string' || t.length === 0)) {
|
|
33
|
+
throw new Error('prompt set turns must be a non-empty array of non-empty strings');
|
|
34
|
+
}
|
|
35
|
+
return {
|
|
36
|
+
schema_version: 'gotry_m3_nightly_prompt_set_v1',
|
|
37
|
+
description: raw['description'],
|
|
38
|
+
turns: turns
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
function assertPriceEntry(label, value) {
|
|
42
|
+
const entry = typeof value === 'object' && value !== null ? value : null;
|
|
43
|
+
if (!entry) throw new Error(`${label} must be an object`);
|
|
44
|
+
for (const field of [
|
|
45
|
+
'input_cache_miss_usd_per_1m_peak',
|
|
46
|
+
'input_cache_hit_usd_per_1m_peak',
|
|
47
|
+
'output_usd_per_1m_peak'
|
|
48
|
+
]){
|
|
49
|
+
const num = entry[field];
|
|
50
|
+
if (typeof num !== 'number' || !Number.isFinite(num) || num < 0) {
|
|
51
|
+
throw new Error(`${label}.${field} must be a non-negative finite number`);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return value;
|
|
55
|
+
}
|
|
56
|
+
export function loadPriceTable(path) {
|
|
57
|
+
const raw = JSON.parse(readFileSync(path, 'utf8'));
|
|
58
|
+
if (raw['schema_version'] !== 'gotry_llm_price_table_v1') {
|
|
59
|
+
throw new Error(`price table schema_version must be gotry_llm_price_table_v1, got: ${String(raw['schema_version'])}`);
|
|
60
|
+
}
|
|
61
|
+
const aliases = typeof raw['aliases'] === 'object' && raw['aliases'] !== null ? raw['aliases'] : {};
|
|
62
|
+
for (const [from, to] of Object.entries(aliases)){
|
|
63
|
+
if (typeof to !== 'string') throw new Error(`price table alias ${from} must map to a model name`);
|
|
64
|
+
}
|
|
65
|
+
const prices = {};
|
|
66
|
+
for (const [name, entry] of Object.entries(typeof raw['prices'] === 'object' && raw['prices'] !== null ? raw['prices'] : {})){
|
|
67
|
+
prices[name] = assertPriceEntry(`price table entry ${name}`, entry);
|
|
68
|
+
}
|
|
69
|
+
return {
|
|
70
|
+
schema_version: 'gotry_llm_price_table_v1',
|
|
71
|
+
aliases,
|
|
72
|
+
prices
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
export function priceRunCost(model, usage, table) {
|
|
76
|
+
if (usage.responsesMissingUsage > 0) {
|
|
77
|
+
throw new Error(`usage provider omitted usage on ${usage.responsesMissingUsage} response(s) — cost unprovable, refusing to price (fail-closed)`);
|
|
78
|
+
}
|
|
79
|
+
const alias = table.aliases[model];
|
|
80
|
+
if (alias && !table.prices[alias]) throw new Error(`price table alias ${model} -> ${alias} has no entry; update data/llm-price-table.json via PR`);
|
|
81
|
+
const entry = table.prices[alias ?? model];
|
|
82
|
+
if (!entry) throw new Error(`price table has no entry for model ${model}; update data/llm-price-table.json via PR (fail-closed, no guessed price)`);
|
|
83
|
+
const cost = (usage.inputCacheMissTokens * entry.input_cache_miss_usd_per_1m_peak + usage.inputCacheHitTokens * entry.input_cache_hit_usd_per_1m_peak + usage.outputTokens * entry.output_usd_per_1m_peak) / 1_000_000;
|
|
84
|
+
return round6(cost);
|
|
85
|
+
}
|
|
86
|
+
export function buildNightlyRunRecord(input) {
|
|
87
|
+
const { schema_version, executed_at, real_llm, prompt_set_sha256, output_sha256, cost_usd } = input;
|
|
88
|
+
const core = {
|
|
89
|
+
schema_version,
|
|
90
|
+
executed_at,
|
|
91
|
+
real_llm,
|
|
92
|
+
prompt_set_sha256,
|
|
93
|
+
output_sha256,
|
|
94
|
+
cost_usd
|
|
95
|
+
};
|
|
96
|
+
if (!SHA256.test(prompt_set_sha256) || !SHA256.test(output_sha256)) {
|
|
97
|
+
throw new Error('nightly record hashes must be lowercase SHA-256 digests');
|
|
98
|
+
}
|
|
99
|
+
const record = {
|
|
100
|
+
...core,
|
|
101
|
+
run_key: `hmac-sha256:${createHash('sha256').update(canonical(core)).digest('hex')}`
|
|
102
|
+
};
|
|
103
|
+
parseNightlyRun(JSON.parse(JSON.stringify(record)), 0);
|
|
104
|
+
return record;
|
|
105
|
+
}
|
|
106
|
+
function activeModel() {
|
|
107
|
+
return process.env['LLM_MODEL'] ?? process.env['DEEPSEEK_MODEL'] ?? 'MiniMax-M2';
|
|
108
|
+
}
|
|
109
|
+
function arg(name) {
|
|
110
|
+
const index = process.argv.indexOf(name);
|
|
111
|
+
return index >= 0 ? process.argv[index + 1] : undefined;
|
|
112
|
+
}
|
|
113
|
+
export async function runNightlyEvidence(options) {
|
|
114
|
+
const clock = options.clock ?? (()=>new Date());
|
|
115
|
+
const llmKey = process.env['LLM_API_KEY'] ?? process.env['DEEPSEEK_API_KEY'];
|
|
116
|
+
if (!options.dryRun && !llmKey) {
|
|
117
|
+
return {
|
|
118
|
+
state: 'waiting_external_evidence',
|
|
119
|
+
record: null,
|
|
120
|
+
reason: 'no real LLM credential (LLM_API_KEY/DEEPSEEK_API_KEY); waiting & backoff & no-spend per issue #22 停机纪律'
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
const promptSet = loadPromptSet('data/m3-nightly-prompts.json');
|
|
124
|
+
const priceTable = loadPriceTable('data/llm-price-table.json');
|
|
125
|
+
const packPath = join('..', 'data', 'flights_2026.json');
|
|
126
|
+
const solvePort = realtimeSolvePort(solveUnified);
|
|
127
|
+
const port = options.dryRun ? createMockLlm(packPath) : createOpenAICompatLlm(packPath);
|
|
128
|
+
const state = newState();
|
|
129
|
+
const history = [];
|
|
130
|
+
for (const turn of promptSet.turns){
|
|
131
|
+
const { reply } = await runTurn(state, turn, port, [
|
|
132
|
+
...history
|
|
133
|
+
], solvePort);
|
|
134
|
+
history.push({
|
|
135
|
+
role: 'user',
|
|
136
|
+
text: turn
|
|
137
|
+
}, {
|
|
138
|
+
role: 'assistant',
|
|
139
|
+
text: reply
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
const promptSetSha = sha256Of(promptSet);
|
|
143
|
+
const turnsTranscript = promptSet.turns.map((turn, index)=>({
|
|
144
|
+
turn_index: index,
|
|
145
|
+
user: turn,
|
|
146
|
+
reply: history[index * 2 + 1]?.text ?? ''
|
|
147
|
+
}));
|
|
148
|
+
const outputSha = sha256Of(turnsTranscript);
|
|
149
|
+
let record;
|
|
150
|
+
if (options.dryRun) {
|
|
151
|
+
record = buildNightlyRunRecord({
|
|
152
|
+
schema_version: 'gotry_m3_nightly_run_v1',
|
|
153
|
+
executed_at: clock().toISOString(),
|
|
154
|
+
real_llm: false,
|
|
155
|
+
prompt_set_sha256: promptSetSha,
|
|
156
|
+
output_sha256: outputSha,
|
|
157
|
+
cost_usd: 0
|
|
158
|
+
});
|
|
159
|
+
return {
|
|
160
|
+
state: 'dry_run',
|
|
161
|
+
record,
|
|
162
|
+
reason: 'dry-run exercises the pipeline against mock LLM; no evidence written'
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
const usage = port.usage;
|
|
166
|
+
if (!usage) throw new Error('real LLM port did not expose usage tracker (dsh-llm regression) — refusing to write cost evidence');
|
|
167
|
+
if (usage.calls !== promptSet.turns.length) {
|
|
168
|
+
throw new Error(`usage calls ${usage.calls} != expected turns ${promptSet.turns.length} — fail-closed, no evidence written`);
|
|
169
|
+
}
|
|
170
|
+
const costUsd = priceRunCost(activeModel(), usage, priceTable);
|
|
171
|
+
record = buildNightlyRunRecord({
|
|
172
|
+
schema_version: 'gotry_m3_nightly_run_v1',
|
|
173
|
+
executed_at: clock().toISOString(),
|
|
174
|
+
real_llm: true,
|
|
175
|
+
prompt_set_sha256: promptSetSha,
|
|
176
|
+
output_sha256: outputSha,
|
|
177
|
+
cost_usd: costUsd
|
|
178
|
+
});
|
|
179
|
+
mkdirSync(options.evidenceRoot, {
|
|
180
|
+
recursive: true
|
|
181
|
+
});
|
|
182
|
+
appendFileSync(join(options.evidenceRoot, 'cohort.jsonl'), `${JSON.stringify(record)}\n`, 'utf8');
|
|
183
|
+
const budget = Number(process.env['GOTRY_NIGHTLY_BUDGET_USD'] ?? DEFAULT_BUDGET_USD);
|
|
184
|
+
const overBudget = Number.isFinite(budget) && budget > 0 ? costUsd > budget : false;
|
|
185
|
+
return {
|
|
186
|
+
state: 'evidence_written',
|
|
187
|
+
record,
|
|
188
|
+
cost_over_budget: overBudget,
|
|
189
|
+
reason: overBudget ? `cost_usd ${costUsd} exceeds GOTRY_NIGHTLY_BUDGET_USD ${budget} — backoff suggested` : undefined
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
function output(payload, asJson) {
|
|
193
|
+
if (asJson) console.log(JSON.stringify(payload));
|
|
194
|
+
else {
|
|
195
|
+
console.log(`nightly-evidence state: ${payload.state}${payload.error ? ` — ${payload.error}` : ''}`);
|
|
196
|
+
if (payload.record) console.log(JSON.stringify(payload.record, null, 2));
|
|
197
|
+
if (payload.reason) console.log(payload.reason);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
async function main() {
|
|
201
|
+
if (!process.argv.includes('--no-env-file')) {
|
|
202
|
+
try {
|
|
203
|
+
for (const line of readFileSync(join('..', '.env'), 'utf-8').split('\n')){
|
|
204
|
+
const m = line.match(/^([A-Z_]+)=(.*)$/);
|
|
205
|
+
if (m && !process.env[m[1]]) process.env[m[1]] = m[2].trim();
|
|
206
|
+
}
|
|
207
|
+
} catch {}
|
|
208
|
+
}
|
|
209
|
+
const asJson = (arg('--format') ?? 'markdown') === 'json';
|
|
210
|
+
const evidenceRoot = arg('--evidence-root') ?? 'gotry-state/evidence/m3';
|
|
211
|
+
const dryRun = process.argv.includes('--dry-run');
|
|
212
|
+
try {
|
|
213
|
+
const result = await runNightlyEvidence({
|
|
214
|
+
evidenceRoot,
|
|
215
|
+
dryRun
|
|
216
|
+
});
|
|
217
|
+
output(result, asJson);
|
|
218
|
+
if (result.cost_over_budget) process.exitCode = 3;
|
|
219
|
+
} catch (error) {
|
|
220
|
+
output({
|
|
221
|
+
state: 'error',
|
|
222
|
+
record: null,
|
|
223
|
+
error: error.message
|
|
224
|
+
}, asJson);
|
|
225
|
+
process.exitCode = 1;
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
if (process.argv[1]?.endsWith('nightly-evidence.ts')) {
|
|
229
|
+
await main();
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
//# sourceURL=/Users/bytedance/work/gotry/ts/scripts/nightly-evidence.ts
|
package/dist/scripts/smoke.js
CHANGED
|
@@ -410,6 +410,89 @@ async function main() {
|
|
|
410
410
|
}
|
|
411
411
|
console.log('login tool: registered; 登录=外部网站+用户自己的浏览器,gotry 只读票据名(0 值过手)语义钉死');
|
|
412
412
|
}
|
|
413
|
+
{
|
|
414
|
+
const { writeFileSync } = await import('node:fs');
|
|
415
|
+
const { listArtifacts, readArtifact } = await import('../capabilities/artifacts.js');
|
|
416
|
+
const { ensureLedger } = await import('../src/state-ledger.js');
|
|
417
|
+
const cwdDir = mkdtempSync(join(tmpdir(), 'gotry-artifacts-cwd-'));
|
|
418
|
+
const twelve = Array.from({
|
|
419
|
+
length: 12
|
|
420
|
+
}, (_, i)=>`第 ${i + 1} 行`).join('\n');
|
|
421
|
+
writeFileSync(join(cwdDir, 'trip-2027-probe.md'), `# 行程·产物探针\n${twelve}\n`);
|
|
422
|
+
const ledger = ensureLedger(smokeRoot);
|
|
423
|
+
ledger.createWorkflowRun({
|
|
424
|
+
id: 'art-probe-1',
|
|
425
|
+
goal: '行程·产物探针(账本)',
|
|
426
|
+
ticket: {
|
|
427
|
+
objective: 'probe'
|
|
428
|
+
},
|
|
429
|
+
state: {}
|
|
430
|
+
});
|
|
431
|
+
ledger.settleWorkflowRun('art-probe-1', '# 交付·账本权威\nD1 大理\nD2 洱海');
|
|
432
|
+
const listTool = byName('gotry_artifacts_list');
|
|
433
|
+
const ledList = await listTool.execute({
|
|
434
|
+
query: {}
|
|
435
|
+
}, null);
|
|
436
|
+
const run = ledList.artifacts?.find((a)=>a.id === 'art-probe-1');
|
|
437
|
+
if (!ledList.ok || !run || run.source !== 'async-run' || run.status !== 'settled') {
|
|
438
|
+
throw new Error(`FAIL: artifacts list 应发现账本已交付工单,实际:${JSON.stringify(ledList).slice(0, 200)}`);
|
|
439
|
+
}
|
|
440
|
+
const direct = await listArtifacts({
|
|
441
|
+
stateRoot: smokeRoot,
|
|
442
|
+
cwd: cwdDir
|
|
443
|
+
});
|
|
444
|
+
if (!direct.artifacts.some((a)=>a.source === 'cwd-file' && a.id === 'trip-2027-probe.md')) {
|
|
445
|
+
throw new Error(`FAIL: artifacts list 应发现 cwd 顶层 md,实际:${JSON.stringify(direct.artifacts.map((a)=>a.id))}`);
|
|
446
|
+
}
|
|
447
|
+
const readTool = byName('gotry_artifacts_read');
|
|
448
|
+
const r1 = await readTool.execute({
|
|
449
|
+
query: {
|
|
450
|
+
path: 'art-probe-1'
|
|
451
|
+
}
|
|
452
|
+
}, null);
|
|
453
|
+
if (!r1.ok || r1.lines?.[0]?.number !== 1 || !r1.lines?.[0]?.text.includes('交付·账本权威') || r1.lang !== 'markdown') {
|
|
454
|
+
throw new Error(`FAIL: 裸工单 id 应从账本读出行号视图,实际:${JSON.stringify(r1).slice(0, 200)}`);
|
|
455
|
+
}
|
|
456
|
+
const view = readTool.presentResult?.({
|
|
457
|
+
query: {
|
|
458
|
+
path: 'art-probe-1'
|
|
459
|
+
}
|
|
460
|
+
}, r1);
|
|
461
|
+
if (view?.card !== 'read' || view.path !== r1.path || view.offset !== 1 || view.lines?.length !== r1.lines?.length || view.totalLines !== r1.totalLines || view.lang !== 'markdown') {
|
|
462
|
+
throw new Error(`FAIL: read 卡字段不齐,实际:${JSON.stringify(view).slice(0, 200)}`);
|
|
463
|
+
}
|
|
464
|
+
const r2 = await readArtifact({
|
|
465
|
+
stateRoot: smokeRoot,
|
|
466
|
+
cwd: cwdDir,
|
|
467
|
+
path: 'trip-2027-probe.md',
|
|
468
|
+
offset: 10,
|
|
469
|
+
limit: 5
|
|
470
|
+
});
|
|
471
|
+
if (!r2.ok || r2.lines[0].number !== 10 || r2.windowed !== false) {
|
|
472
|
+
throw new Error(`FAIL: 行窗口 12 行文件 offset=10 应从行号 10 起,实际:${JSON.stringify(r2.ok ? {
|
|
473
|
+
o: r2.offset,
|
|
474
|
+
first: r2.lines[0]?.number
|
|
475
|
+
} : r2)}`);
|
|
476
|
+
}
|
|
477
|
+
if (r2.lines.length !== 5) throw new Error(`FAIL: 14 行文件 offset=10/limit=5 窗口应恰 5 行(10-14),实际 ${r2.lines.length}`);
|
|
478
|
+
const badPath = await readArtifact({
|
|
479
|
+
stateRoot: smokeRoot,
|
|
480
|
+
cwd: cwdDir,
|
|
481
|
+
path: '../../../../../etc/passwd'
|
|
482
|
+
});
|
|
483
|
+
if (badPath.ok) throw new Error('FAIL: 越界路径必须被拒');
|
|
484
|
+
const badExt = await readArtifact({
|
|
485
|
+
stateRoot: smokeRoot,
|
|
486
|
+
cwd: cwdDir,
|
|
487
|
+
path: 'gotry-state/gotry-state.db'
|
|
488
|
+
});
|
|
489
|
+
if (badExt.ok) throw new Error('FAIL: 白名单外扩展名(.db)必须被拒');
|
|
490
|
+
rmSync(cwdDir, {
|
|
491
|
+
recursive: true,
|
|
492
|
+
force: true
|
|
493
|
+
});
|
|
494
|
+
console.log(`artifacts: ledger run + cwd md discovered; read window(${r2.ok ? r2.lines.length : '?'} lines @10) + read card; path/ext guardrails hold`);
|
|
495
|
+
}
|
|
413
496
|
rmSync(smokeRoot, {
|
|
414
497
|
recursive: true,
|
|
415
498
|
force: true
|