@letrquan/book 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2504 -0
- package/LICENSE +123 -0
- package/README.md +1383 -0
- package/dist/app-MOXXWEOY.js +18945 -0
- package/dist/app-MOXXWEOY.js.map +1 -0
- package/dist/chunk-24AXE6SP.js +12 -0
- package/dist/chunk-24AXE6SP.js.map +1 -0
- package/dist/chunk-3U5IBM24.js +213 -0
- package/dist/chunk-3U5IBM24.js.map +1 -0
- package/dist/chunk-574C6MQR.js +743 -0
- package/dist/chunk-574C6MQR.js.map +1 -0
- package/dist/chunk-5GDZ22YP.js +449 -0
- package/dist/chunk-5GDZ22YP.js.map +1 -0
- package/dist/chunk-5KLVH3PY.js +167 -0
- package/dist/chunk-5KLVH3PY.js.map +1 -0
- package/dist/chunk-5RDRAO4B.js +92 -0
- package/dist/chunk-5RDRAO4B.js.map +1 -0
- package/dist/chunk-7N4J557C.js +10 -0
- package/dist/chunk-7N4J557C.js.map +1 -0
- package/dist/chunk-A7JLBK2Y.js +3506 -0
- package/dist/chunk-A7JLBK2Y.js.map +1 -0
- package/dist/chunk-ANSJPYGI.js +84 -0
- package/dist/chunk-ANSJPYGI.js.map +1 -0
- package/dist/chunk-BTZZUCLI.js +264 -0
- package/dist/chunk-BTZZUCLI.js.map +1 -0
- package/dist/chunk-C7D4NJ2D.js +217 -0
- package/dist/chunk-C7D4NJ2D.js.map +1 -0
- package/dist/chunk-CAGOB2N4.js +208 -0
- package/dist/chunk-CAGOB2N4.js.map +1 -0
- package/dist/chunk-DGMQEI6Y.js +98 -0
- package/dist/chunk-DGMQEI6Y.js.map +1 -0
- package/dist/chunk-E75IPST5.js +289 -0
- package/dist/chunk-E75IPST5.js.map +1 -0
- package/dist/chunk-EYGD6X2T.js +180 -0
- package/dist/chunk-EYGD6X2T.js.map +1 -0
- package/dist/chunk-GTVB4HEM.js +765 -0
- package/dist/chunk-GTVB4HEM.js.map +1 -0
- package/dist/chunk-J7MIZ5CA.js +1907 -0
- package/dist/chunk-J7MIZ5CA.js.map +1 -0
- package/dist/chunk-KWHQ2XXL.js +626 -0
- package/dist/chunk-KWHQ2XXL.js.map +1 -0
- package/dist/chunk-L6LK2QIQ.js +657 -0
- package/dist/chunk-L6LK2QIQ.js.map +1 -0
- package/dist/chunk-MAQXCUR4.js +1007 -0
- package/dist/chunk-MAQXCUR4.js.map +1 -0
- package/dist/chunk-S4XL7HOM.js +402 -0
- package/dist/chunk-S4XL7HOM.js.map +1 -0
- package/dist/chunk-URLQAXTR.js +60 -0
- package/dist/chunk-URLQAXTR.js.map +1 -0
- package/dist/chunk-V4BOYR52.js +3504 -0
- package/dist/chunk-V4BOYR52.js.map +1 -0
- package/dist/chunk-VZQXX3WW.js +210 -0
- package/dist/chunk-VZQXX3WW.js.map +1 -0
- package/dist/chunk-WKBO5R4O.js +34 -0
- package/dist/chunk-WKBO5R4O.js.map +1 -0
- package/dist/chunk-XIFFTATU.js +15753 -0
- package/dist/chunk-XIFFTATU.js.map +1 -0
- package/dist/command-approvals-6Y57VCWZ.js +28 -0
- package/dist/command-approvals-6Y57VCWZ.js.map +1 -0
- package/dist/context-ZC5IFEAX.js +25 -0
- package/dist/context-ZC5IFEAX.js.map +1 -0
- package/dist/hook-approvals-CTUFKEJT.js +22 -0
- package/dist/hook-approvals-CTUFKEJT.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +2582 -0
- package/dist/index.js.map +1 -0
- package/dist/interactive-assets-CCAAWVKH.js +31 -0
- package/dist/interactive-assets-CCAAWVKH.js.map +1 -0
- package/dist/job-runner.d.ts +2 -0
- package/dist/job-runner.js +235 -0
- package/dist/job-runner.js.map +1 -0
- package/dist/loader-4IDS2RIE.js +15 -0
- package/dist/loader-4IDS2RIE.js.map +1 -0
- package/dist/mcp-approvals-WNPH65BK.js +18 -0
- package/dist/mcp-approvals-WNPH65BK.js.map +1 -0
- package/dist/mcp-config-YIJIES2C.js +16 -0
- package/dist/mcp-config-YIJIES2C.js.map +1 -0
- package/dist/model-window-store-YFVNXCIP.js +27 -0
- package/dist/model-window-store-YFVNXCIP.js.map +1 -0
- package/dist/openai-compatible-RHZUBHV6.js +10 -0
- package/dist/openai-compatible-RHZUBHV6.js.map +1 -0
- package/dist/permission-approvals-76PVREFD.js +14 -0
- package/dist/permission-approvals-76PVREFD.js.map +1 -0
- package/dist/permissions-CHCKTFQQ.js +28 -0
- package/dist/permissions-CHCKTFQQ.js.map +1 -0
- package/dist/sandbox-CNVAQDA7.js +23 -0
- package/dist/sandbox-CNVAQDA7.js.map +1 -0
- package/dist/scrollback-LUKLIBVI.js +136 -0
- package/dist/scrollback-LUKLIBVI.js.map +1 -0
- package/dist/sdk.d.ts +4574 -0
- package/dist/sdk.js +665 -0
- package/dist/sdk.js.map +1 -0
- package/dist/settings-loader-M2QP5TX2.js +21 -0
- package/dist/settings-loader-M2QP5TX2.js.map +1 -0
- package/dist/settings-removed-JWWPT4WA.js +19 -0
- package/dist/settings-removed-JWWPT4WA.js.map +1 -0
- package/dist/shell-selection-V6QAYXPX.js +17 -0
- package/dist/shell-selection-V6QAYXPX.js.map +1 -0
- package/package.json +143 -0
- package/patches/ink+6.8.0.patch +13 -0
- package/scripts/apply-ink-patch.mjs +29 -0
- package/scripts/ink-patch.mjs +59 -0
- package/scripts/verify-ink-patch.mjs +11 -0
package/dist/sdk.js
ADDED
|
@@ -0,0 +1,665 @@
|
|
|
1
|
+
import {
|
|
2
|
+
collectWithheldProjectNotices,
|
|
3
|
+
connectMcpServers,
|
|
4
|
+
disconnectMcpServers,
|
|
5
|
+
runHeadless
|
|
6
|
+
} from "./chunk-J7MIZ5CA.js";
|
|
7
|
+
import {
|
|
8
|
+
SessionStore,
|
|
9
|
+
createAgentSessionSnapshot,
|
|
10
|
+
reduceAgentSessionSnapshot,
|
|
11
|
+
resolveSessionBootstrap
|
|
12
|
+
} from "./chunk-V4BOYR52.js";
|
|
13
|
+
import {
|
|
14
|
+
AgentManager,
|
|
15
|
+
SessionRuntime,
|
|
16
|
+
SkillRegistry,
|
|
17
|
+
SkillRegistryError,
|
|
18
|
+
createDefaultRegistry,
|
|
19
|
+
createRegistry,
|
|
20
|
+
createToolSurface,
|
|
21
|
+
freezeAgentConfig,
|
|
22
|
+
loadConfig
|
|
23
|
+
} from "./chunk-XIFFTATU.js";
|
|
24
|
+
import {
|
|
25
|
+
buildSkillReport,
|
|
26
|
+
resolvePermissionMode
|
|
27
|
+
} from "./chunk-A7JLBK2Y.js";
|
|
28
|
+
import {
|
|
29
|
+
applySkillOverrides,
|
|
30
|
+
buildSkillListing,
|
|
31
|
+
discoverSkills,
|
|
32
|
+
explicitSkillMentions,
|
|
33
|
+
generateSkillListing,
|
|
34
|
+
loadSkillBody,
|
|
35
|
+
skillRoots
|
|
36
|
+
} from "./chunk-KWHQ2XXL.js";
|
|
37
|
+
import "./chunk-VZQXX3WW.js";
|
|
38
|
+
import "./chunk-E75IPST5.js";
|
|
39
|
+
import "./chunk-5KLVH3PY.js";
|
|
40
|
+
import {
|
|
41
|
+
resolveMcpServerList
|
|
42
|
+
} from "./chunk-3U5IBM24.js";
|
|
43
|
+
import {
|
|
44
|
+
mcpServersToRecord,
|
|
45
|
+
partitionMcpServersByApproval
|
|
46
|
+
} from "./chunk-URLQAXTR.js";
|
|
47
|
+
import "./chunk-MAQXCUR4.js";
|
|
48
|
+
import {
|
|
49
|
+
toolFailure,
|
|
50
|
+
toolSuccess
|
|
51
|
+
} from "./chunk-5GDZ22YP.js";
|
|
52
|
+
import "./chunk-EYGD6X2T.js";
|
|
53
|
+
import {
|
|
54
|
+
DEFAULT_LOCAL_DATA_RETENTION_DAYS,
|
|
55
|
+
cleanupDebugLogs,
|
|
56
|
+
getDebugLogPath
|
|
57
|
+
} from "./chunk-GTVB4HEM.js";
|
|
58
|
+
import "./chunk-S4XL7HOM.js";
|
|
59
|
+
import "./chunk-7N4J557C.js";
|
|
60
|
+
import "./chunk-BTZZUCLI.js";
|
|
61
|
+
import "./chunk-WKBO5R4O.js";
|
|
62
|
+
import "./chunk-ANSJPYGI.js";
|
|
63
|
+
import "./chunk-DGMQEI6Y.js";
|
|
64
|
+
import "./chunk-5RDRAO4B.js";
|
|
65
|
+
import "./chunk-CAGOB2N4.js";
|
|
66
|
+
import "./chunk-574C6MQR.js";
|
|
67
|
+
import {
|
|
68
|
+
resolveBookHome
|
|
69
|
+
} from "./chunk-24AXE6SP.js";
|
|
70
|
+
|
|
71
|
+
// src/sdk.ts
|
|
72
|
+
import { join } from "path";
|
|
73
|
+
|
|
74
|
+
// src/skill-evaluation.ts
|
|
75
|
+
import { createHash } from "crypto";
|
|
76
|
+
import { mkdirSync, writeFileSync } from "fs";
|
|
77
|
+
import { dirname } from "path";
|
|
78
|
+
var SKILL_EVALUATION_CATEGORIES = [
|
|
79
|
+
"direct",
|
|
80
|
+
"indirect",
|
|
81
|
+
"negative",
|
|
82
|
+
"ambiguous",
|
|
83
|
+
"conflicting",
|
|
84
|
+
"disabled",
|
|
85
|
+
"invalid",
|
|
86
|
+
"missing-body",
|
|
87
|
+
"missing-resource"
|
|
88
|
+
];
|
|
89
|
+
var DEFAULT_SKILL_EVALUATION_THRESHOLDS = {
|
|
90
|
+
minimumPrecision: 0.95,
|
|
91
|
+
minimumRecall: 0.8,
|
|
92
|
+
maximumFalseActivations: 0,
|
|
93
|
+
maximumUnnecessaryPermissionPrompts: 0,
|
|
94
|
+
maximumSkillToolFailures: 0
|
|
95
|
+
};
|
|
96
|
+
var DEFAULT_SKILL_EVALUATION_FIXTURES = [
|
|
97
|
+
{
|
|
98
|
+
id: "direct-review",
|
|
99
|
+
category: "direct",
|
|
100
|
+
prompt: "$review inspect the current change",
|
|
101
|
+
expectedSkills: ["review"]
|
|
102
|
+
},
|
|
103
|
+
{
|
|
104
|
+
id: "indirect-review",
|
|
105
|
+
category: "indirect",
|
|
106
|
+
prompt: "Inspect this diff for correctness regressions and missing tests",
|
|
107
|
+
expectedSkills: ["review"]
|
|
108
|
+
},
|
|
109
|
+
{
|
|
110
|
+
id: "negative-general-question",
|
|
111
|
+
category: "negative",
|
|
112
|
+
prompt: "Explain what this function returns",
|
|
113
|
+
expectedSkills: []
|
|
114
|
+
},
|
|
115
|
+
{
|
|
116
|
+
id: "ambiguous-review-word",
|
|
117
|
+
category: "ambiguous",
|
|
118
|
+
prompt: "Review the available deployment choices with me",
|
|
119
|
+
expectedSkills: []
|
|
120
|
+
},
|
|
121
|
+
{
|
|
122
|
+
id: "conflicting-tool-ceilings",
|
|
123
|
+
category: "conflicting",
|
|
124
|
+
prompt: "$read-review $write-review inspect the change",
|
|
125
|
+
expectedSkills: ["read-review"],
|
|
126
|
+
expectedBlockedCodes: ["skill_tool_intersection_empty"]
|
|
127
|
+
},
|
|
128
|
+
{
|
|
129
|
+
id: "disabled-skill",
|
|
130
|
+
category: "disabled",
|
|
131
|
+
prompt: "$disabled-skill run the workflow",
|
|
132
|
+
expectedSkills: [],
|
|
133
|
+
expectedBlockedCodes: ["skill_disabled"]
|
|
134
|
+
},
|
|
135
|
+
{
|
|
136
|
+
id: "invalid-skill",
|
|
137
|
+
category: "invalid",
|
|
138
|
+
prompt: "$invalid-skill run the workflow",
|
|
139
|
+
expectedSkills: [],
|
|
140
|
+
expectedBlockedCodes: ["skill_invalid"]
|
|
141
|
+
},
|
|
142
|
+
{
|
|
143
|
+
id: "missing-body",
|
|
144
|
+
category: "missing-body",
|
|
145
|
+
prompt: "$missing-body run the workflow",
|
|
146
|
+
expectedSkills: [],
|
|
147
|
+
expectedBlockedCodes: ["skill_load_failed"]
|
|
148
|
+
},
|
|
149
|
+
{
|
|
150
|
+
id: "missing-resource",
|
|
151
|
+
category: "missing-resource",
|
|
152
|
+
prompt: "$missing-resource load references/checklist.md",
|
|
153
|
+
expectedSkills: ["missing-resource"],
|
|
154
|
+
expectedBlockedCodes: ["skill_resource_unreadable"]
|
|
155
|
+
}
|
|
156
|
+
];
|
|
157
|
+
function numberDetail(event, key) {
|
|
158
|
+
const value = event.details?.[key];
|
|
159
|
+
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
|
160
|
+
}
|
|
161
|
+
function median(values) {
|
|
162
|
+
if (values.length === 0) return 0;
|
|
163
|
+
const sorted = [...values].sort((left, right) => left - right);
|
|
164
|
+
return sorted[Math.floor(sorted.length / 2)] ?? 0;
|
|
165
|
+
}
|
|
166
|
+
function percentile(values, fraction) {
|
|
167
|
+
if (values.length === 0) return 0;
|
|
168
|
+
const sorted = [...values].sort((left, right) => left - right);
|
|
169
|
+
const index = Math.min(sorted.length - 1, Math.ceil(sorted.length * fraction) - 1);
|
|
170
|
+
return sorted[index] ?? 0;
|
|
171
|
+
}
|
|
172
|
+
function observeSkillEvaluation(fixture, events, outcome = {}) {
|
|
173
|
+
const applied = events.filter(
|
|
174
|
+
(event) => event.type === "skill_activation_applied" && event.skill
|
|
175
|
+
);
|
|
176
|
+
const activatedSkills = [...new Set(applied.map((event) => event.skill))];
|
|
177
|
+
const expected = new Set(fixture.expectedSkills);
|
|
178
|
+
const falseActivations = applied.filter((event) => event.skill && !expected.has(event.skill));
|
|
179
|
+
const consentEvents = events.filter((event) => event.type === "skill_consent_requested");
|
|
180
|
+
const blockedCodes = events.filter(
|
|
181
|
+
(event) => event.type === "skill_activation_blocked" || event.type === "skill_resource_blocked"
|
|
182
|
+
).map((event) => event.details?.code).filter((code) => typeof code === "string");
|
|
183
|
+
const expectedBlockedCodes = fixture.expectedBlockedCodes ?? [];
|
|
184
|
+
const promptChars = outcome.promptChars ?? 0;
|
|
185
|
+
const bodyBytes = applied.reduce(
|
|
186
|
+
(total, event) => total + numberDetail(event, "bodyByteSize"),
|
|
187
|
+
0
|
|
188
|
+
);
|
|
189
|
+
return {
|
|
190
|
+
...fixture,
|
|
191
|
+
expectedSkills: [...fixture.expectedSkills],
|
|
192
|
+
activatedSkills,
|
|
193
|
+
expectedBlockedCodes: [...expectedBlockedCodes],
|
|
194
|
+
blockedCodes,
|
|
195
|
+
promptChars,
|
|
196
|
+
promptTokens: outcome.promptTokens ?? Math.ceil(promptChars / 4),
|
|
197
|
+
bodyBytes,
|
|
198
|
+
bodyTokens: outcome.bodyTokens ?? Math.ceil(bodyBytes / 4),
|
|
199
|
+
falseActivationBytes: falseActivations.reduce(
|
|
200
|
+
(total, event) => total + numberDetail(event, "bodyByteSize"),
|
|
201
|
+
0
|
|
202
|
+
),
|
|
203
|
+
activationLatencyMs: applied.map((event) => numberDetail(event, "activationLatencyMs")),
|
|
204
|
+
consentRequests: consentEvents.length,
|
|
205
|
+
unnecessaryPermissionPrompts: consentEvents.filter(
|
|
206
|
+
(event) => !event.skill || !expected.has(event.skill)
|
|
207
|
+
).length,
|
|
208
|
+
taskCompleted: outcome.taskCompleted,
|
|
209
|
+
userCorrections: outcome.userCorrections ?? 0,
|
|
210
|
+
skillToolFailures: outcome.skillToolFailures ?? 0,
|
|
211
|
+
blockingMismatch: expectedBlockedCodes.some((code) => !blockedCodes.includes(code)) || blockedCodes.some((code) => !expectedBlockedCodes.includes(code)),
|
|
212
|
+
exposure: outcome.exposure,
|
|
213
|
+
activationTurns: outcome.activationTurns
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
function evaluateSkillActivation(observations, thresholds = DEFAULT_SKILL_EVALUATION_THRESHOLDS) {
|
|
217
|
+
const categoryCounts = Object.fromEntries(
|
|
218
|
+
SKILL_EVALUATION_CATEGORIES.map((category) => [
|
|
219
|
+
category,
|
|
220
|
+
observations.filter((observation) => observation.category === category).length
|
|
221
|
+
])
|
|
222
|
+
);
|
|
223
|
+
let truePositives = 0;
|
|
224
|
+
let falsePositives = 0;
|
|
225
|
+
let falseNegatives = 0;
|
|
226
|
+
for (const observation of observations) {
|
|
227
|
+
const expected = new Set(observation.expectedSkills);
|
|
228
|
+
const activated = new Set(observation.activatedSkills);
|
|
229
|
+
truePositives += [...activated].filter((skill) => expected.has(skill)).length;
|
|
230
|
+
falsePositives += [...activated].filter((skill) => !expected.has(skill)).length;
|
|
231
|
+
falseNegatives += [...expected].filter((skill) => !activated.has(skill)).length;
|
|
232
|
+
}
|
|
233
|
+
const precision = truePositives + falsePositives === 0 ? 1 : truePositives / (truePositives + falsePositives);
|
|
234
|
+
const recall = truePositives + falseNegatives === 0 ? 1 : truePositives / (truePositives + falseNegatives);
|
|
235
|
+
const completed = observations.filter(
|
|
236
|
+
(observation) => observation.taskCompleted !== void 0
|
|
237
|
+
);
|
|
238
|
+
const exposure = Object.fromEntries(
|
|
239
|
+
["eager", "deferred"].map((mode) => {
|
|
240
|
+
const samples = observations.filter((observation) => observation.exposure === mode);
|
|
241
|
+
return [
|
|
242
|
+
mode,
|
|
243
|
+
{
|
|
244
|
+
samples: samples.length,
|
|
245
|
+
activationRate: samples.length === 0 ? 0 : samples.filter((observation) => observation.activatedSkills.length > 0).length / samples.length,
|
|
246
|
+
medianActivationTurns: median(
|
|
247
|
+
samples.map((observation) => observation.activationTurns ?? 0)
|
|
248
|
+
)
|
|
249
|
+
}
|
|
250
|
+
];
|
|
251
|
+
})
|
|
252
|
+
);
|
|
253
|
+
const reasons = [];
|
|
254
|
+
const missingCategories = SKILL_EVALUATION_CATEGORIES.filter(
|
|
255
|
+
(category) => categoryCounts[category] === 0
|
|
256
|
+
);
|
|
257
|
+
if (missingCategories.length)
|
|
258
|
+
reasons.push(`Missing fixture categories: ${missingCategories.join(", ")}.`);
|
|
259
|
+
if (precision < thresholds.minimumPrecision) {
|
|
260
|
+
reasons.push(
|
|
261
|
+
`Activation precision ${precision.toFixed(3)} is below ${thresholds.minimumPrecision}.`
|
|
262
|
+
);
|
|
263
|
+
}
|
|
264
|
+
if (recall < thresholds.minimumRecall) {
|
|
265
|
+
reasons.push(`Activation recall ${recall.toFixed(3)} is below ${thresholds.minimumRecall}.`);
|
|
266
|
+
}
|
|
267
|
+
if (falsePositives > thresholds.maximumFalseActivations) {
|
|
268
|
+
reasons.push(
|
|
269
|
+
`False activations ${falsePositives} exceed ${thresholds.maximumFalseActivations}.`
|
|
270
|
+
);
|
|
271
|
+
}
|
|
272
|
+
const unnecessaryPermissionPrompts = observations.reduce(
|
|
273
|
+
(total, observation) => total + observation.unnecessaryPermissionPrompts,
|
|
274
|
+
0
|
|
275
|
+
);
|
|
276
|
+
if (unnecessaryPermissionPrompts > thresholds.maximumUnnecessaryPermissionPrompts) {
|
|
277
|
+
reasons.push(
|
|
278
|
+
`Unnecessary permission prompts ${unnecessaryPermissionPrompts} exceed ${thresholds.maximumUnnecessaryPermissionPrompts}.`
|
|
279
|
+
);
|
|
280
|
+
}
|
|
281
|
+
const skillToolFailures = observations.reduce(
|
|
282
|
+
(total, observation) => total + observation.skillToolFailures,
|
|
283
|
+
0
|
|
284
|
+
);
|
|
285
|
+
if (skillToolFailures > thresholds.maximumSkillToolFailures) {
|
|
286
|
+
reasons.push(
|
|
287
|
+
`Skill-caused tool failures ${skillToolFailures} exceed ${thresholds.maximumSkillToolFailures}.`
|
|
288
|
+
);
|
|
289
|
+
}
|
|
290
|
+
const blockingMismatches = observations.filter(
|
|
291
|
+
(observation) => observation.blockingMismatch
|
|
292
|
+
).length;
|
|
293
|
+
if (blockingMismatches > 0) {
|
|
294
|
+
reasons.push(`${blockingMismatches} fixture(s) had unexpected or missing block outcomes.`);
|
|
295
|
+
}
|
|
296
|
+
const latencies = observations.flatMap((observation) => observation.activationLatencyMs);
|
|
297
|
+
return {
|
|
298
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
299
|
+
fixtureCount: observations.length,
|
|
300
|
+
categoryCounts,
|
|
301
|
+
truePositives,
|
|
302
|
+
falsePositives,
|
|
303
|
+
falseNegatives,
|
|
304
|
+
precision,
|
|
305
|
+
recall,
|
|
306
|
+
falseActivationBytes: observations.reduce(
|
|
307
|
+
(total, observation) => total + observation.falseActivationBytes,
|
|
308
|
+
0
|
|
309
|
+
),
|
|
310
|
+
promptChars: {
|
|
311
|
+
total: observations.reduce((total, observation) => total + observation.promptChars, 0),
|
|
312
|
+
median: median(observations.map((observation) => observation.promptChars))
|
|
313
|
+
},
|
|
314
|
+
promptTokens: {
|
|
315
|
+
total: observations.reduce((total, observation) => total + observation.promptTokens, 0),
|
|
316
|
+
median: median(observations.map((observation) => observation.promptTokens))
|
|
317
|
+
},
|
|
318
|
+
bodyBytes: {
|
|
319
|
+
total: observations.reduce((total, observation) => total + observation.bodyBytes, 0),
|
|
320
|
+
median: median(observations.map((observation) => observation.bodyBytes))
|
|
321
|
+
},
|
|
322
|
+
bodyTokens: {
|
|
323
|
+
total: observations.reduce((total, observation) => total + observation.bodyTokens, 0),
|
|
324
|
+
median: median(observations.map((observation) => observation.bodyTokens))
|
|
325
|
+
},
|
|
326
|
+
activationLatencyMs: { median: median(latencies), p95: percentile(latencies, 0.95) },
|
|
327
|
+
consentRequests: observations.reduce(
|
|
328
|
+
(total, observation) => total + observation.consentRequests,
|
|
329
|
+
0
|
|
330
|
+
),
|
|
331
|
+
unnecessaryPermissionPrompts,
|
|
332
|
+
taskCompletionRate: completed.length === 0 ? void 0 : completed.filter((observation) => observation.taskCompleted).length / completed.length,
|
|
333
|
+
userCorrections: observations.reduce(
|
|
334
|
+
(total, observation) => total + observation.userCorrections,
|
|
335
|
+
0
|
|
336
|
+
),
|
|
337
|
+
skillToolFailures,
|
|
338
|
+
blockingMismatches,
|
|
339
|
+
exposure,
|
|
340
|
+
thresholds: { ...thresholds },
|
|
341
|
+
rolloutReady: reasons.length === 0,
|
|
342
|
+
reasons,
|
|
343
|
+
observations: observations.map((observation) => ({
|
|
344
|
+
...observation,
|
|
345
|
+
expectedSkills: [...observation.expectedSkills],
|
|
346
|
+
expectedBlockedCodes: observation.expectedBlockedCodes ? [...observation.expectedBlockedCodes] : void 0,
|
|
347
|
+
activatedSkills: [...observation.activatedSkills],
|
|
348
|
+
blockedCodes: [...observation.blockedCodes],
|
|
349
|
+
activationLatencyMs: [...observation.activationLatencyMs]
|
|
350
|
+
}))
|
|
351
|
+
};
|
|
352
|
+
}
|
|
353
|
+
async function runSkillActivationEvaluation(fixtures, execute, thresholds = DEFAULT_SKILL_EVALUATION_THRESHOLDS) {
|
|
354
|
+
const observations = [];
|
|
355
|
+
for (const fixture of fixtures) {
|
|
356
|
+
const result = await execute(fixture);
|
|
357
|
+
observations.push(
|
|
358
|
+
observeSkillEvaluation(
|
|
359
|
+
{
|
|
360
|
+
id: fixture.id,
|
|
361
|
+
category: fixture.category,
|
|
362
|
+
promptHash: createHash("sha256").update(fixture.prompt).digest("hex"),
|
|
363
|
+
expectedSkills: [...fixture.expectedSkills],
|
|
364
|
+
expectedBlockedCodes: fixture.expectedBlockedCodes ? [...fixture.expectedBlockedCodes] : void 0
|
|
365
|
+
},
|
|
366
|
+
result.events,
|
|
367
|
+
result.outcome
|
|
368
|
+
)
|
|
369
|
+
);
|
|
370
|
+
}
|
|
371
|
+
return evaluateSkillActivation(observations, thresholds);
|
|
372
|
+
}
|
|
373
|
+
function renderSkillEvaluationReport(report) {
|
|
374
|
+
const categories = SKILL_EVALUATION_CATEGORIES.map(
|
|
375
|
+
(category) => `- ${category}: ${report.categoryCounts[category]}`
|
|
376
|
+
).join("\n");
|
|
377
|
+
return [
|
|
378
|
+
"# Skill Activation Evaluation",
|
|
379
|
+
"",
|
|
380
|
+
`Generated: ${report.generatedAt}`,
|
|
381
|
+
`Rollout ready: ${report.rolloutReady ? "yes" : "no"}`,
|
|
382
|
+
`Precision: ${report.precision.toFixed(3)}`,
|
|
383
|
+
`Recall: ${report.recall.toFixed(3)}`,
|
|
384
|
+
`False activations: ${report.falsePositives}`,
|
|
385
|
+
`False-activation body bytes: ${report.falseActivationBytes}`,
|
|
386
|
+
`Median prompt tokens: ${report.promptTokens.median}`,
|
|
387
|
+
`Median activated body tokens: ${report.bodyTokens.median}`,
|
|
388
|
+
`Median activation latency: ${report.activationLatencyMs.median} ms`,
|
|
389
|
+
`Unnecessary permission prompts: ${report.unnecessaryPermissionPrompts}`,
|
|
390
|
+
`Skill-caused tool failures: ${report.skillToolFailures}`,
|
|
391
|
+
`Blocking mismatches: ${report.blockingMismatches}`,
|
|
392
|
+
...report.evaluation ? [
|
|
393
|
+
`Evidence kind: ${report.evaluation.evidenceKind}`,
|
|
394
|
+
`Provider run eligibility: ${report.evaluation.providerRunEligibility}`,
|
|
395
|
+
`Evaluation date: ${report.evaluation.controls.evaluationDate}`,
|
|
396
|
+
`Evaluation seed: ${report.evaluation.controls.randomSeed}`,
|
|
397
|
+
`Runtime revision: ${report.evaluation.controls.runtimeRevision}`,
|
|
398
|
+
`Fixture revision: ${report.evaluation.controls.fixtureRevision} (${report.evaluation.controls.fixtureRevisionStatus})`
|
|
399
|
+
] : [],
|
|
400
|
+
"",
|
|
401
|
+
"## Fixture Coverage",
|
|
402
|
+
"",
|
|
403
|
+
categories,
|
|
404
|
+
"",
|
|
405
|
+
"## Gate",
|
|
406
|
+
"",
|
|
407
|
+
...report.reasons.length ? report.reasons.map((reason) => `- ${reason}`) : ["- All thresholds passed."],
|
|
408
|
+
""
|
|
409
|
+
].join("\n");
|
|
410
|
+
}
|
|
411
|
+
function writeSkillEvaluationReport(report, jsonPath, markdownPath) {
|
|
412
|
+
mkdirSync(dirname(jsonPath), { recursive: true });
|
|
413
|
+
writeFileSync(jsonPath, `${JSON.stringify(report, null, 2)}
|
|
414
|
+
`, "utf8");
|
|
415
|
+
if (markdownPath) {
|
|
416
|
+
mkdirSync(dirname(markdownPath), { recursive: true });
|
|
417
|
+
writeFileSync(markdownPath, renderSkillEvaluationReport(report), "utf8");
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
// src/agents/evaluation.ts
|
|
422
|
+
import { createHash as createHash2 } from "crypto";
|
|
423
|
+
import { mkdirSync as mkdirSync2, writeFileSync as writeFileSync2 } from "fs";
|
|
424
|
+
import { dirname as dirname2 } from "path";
|
|
425
|
+
async function runPairedEvaluation(fixtures, execute, outputPath) {
|
|
426
|
+
const metrics = [];
|
|
427
|
+
for (const fixture of fixtures) {
|
|
428
|
+
for (const mode of ["off", "adaptive"]) {
|
|
429
|
+
const result = await execute(fixture, mode);
|
|
430
|
+
metrics.push({
|
|
431
|
+
fixtureId: fixture.id,
|
|
432
|
+
promptHash: createHash2("sha256").update(fixture.prompt).digest("hex"),
|
|
433
|
+
cohort: fixture.cohort,
|
|
434
|
+
mode,
|
|
435
|
+
...result
|
|
436
|
+
});
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
mkdirSync2(dirname2(outputPath), { recursive: true });
|
|
440
|
+
writeFileSync2(
|
|
441
|
+
outputPath,
|
|
442
|
+
`${metrics.map((metric) => JSON.stringify(metric)).join("\n")}
|
|
443
|
+
`,
|
|
444
|
+
"utf8"
|
|
445
|
+
);
|
|
446
|
+
return metrics;
|
|
447
|
+
}
|
|
448
|
+
function evaluateSuccess(metrics) {
|
|
449
|
+
const reasons = [];
|
|
450
|
+
const byMode = (cohorts, mode) => metrics.filter((metric) => cohorts.includes(metric.cohort) && metric.mode === mode);
|
|
451
|
+
const rate = (items) => items.length === 0 ? 0 : items.filter((item) => item.passed).length / items.length;
|
|
452
|
+
const median2 = (values) => {
|
|
453
|
+
if (values.length === 0) return 0;
|
|
454
|
+
const sorted = [...values].sort((left, right) => left - right);
|
|
455
|
+
return sorted[Math.floor(sorted.length / 2)];
|
|
456
|
+
};
|
|
457
|
+
const treatment = byMode(["decomposable", "ambiguous"], "adaptive");
|
|
458
|
+
const control = byMode(["decomposable", "ambiguous"], "off");
|
|
459
|
+
const passGain = rate(treatment) - rate(control);
|
|
460
|
+
const treatmentWall = median2(treatment.map((item) => item.wallTimeMs));
|
|
461
|
+
const controlWall = median2(control.map((item) => item.wallTimeMs));
|
|
462
|
+
if (passGain < 0.05 && !(passGain >= 0 && treatmentWall <= controlWall * 0.8)) {
|
|
463
|
+
reasons.push("Decomposable/ambiguous cohort missed the pass-rate or wall-time target.");
|
|
464
|
+
}
|
|
465
|
+
const sequentialTreatment = byMode(["sequential"], "adaptive");
|
|
466
|
+
const sequentialControl = byMode(["sequential"], "off");
|
|
467
|
+
if (rate(sequentialTreatment) < rate(sequentialControl) - 0.02) {
|
|
468
|
+
reasons.push("Sequential cohort regressed by more than two pass-rate points.");
|
|
469
|
+
}
|
|
470
|
+
const adaptiveTokens = median2(
|
|
471
|
+
metrics.filter((item) => item.mode === "adaptive").map((item) => item.totalTokens ?? 0)
|
|
472
|
+
);
|
|
473
|
+
const controlTokens = median2(
|
|
474
|
+
metrics.filter((item) => item.mode === "off").map((item) => item.totalTokens ?? 0)
|
|
475
|
+
);
|
|
476
|
+
if (controlTokens > 0 && adaptiveTokens >= controlTokens * 3) {
|
|
477
|
+
reasons.push("Median adaptive token overhead is not below 3x.");
|
|
478
|
+
}
|
|
479
|
+
if (metrics.some((item) => (item.unvalidatedApplications ?? 0) > 0)) {
|
|
480
|
+
reasons.push("At least one unvalidated patch was applied.");
|
|
481
|
+
}
|
|
482
|
+
return { successful: reasons.length === 0, reasons };
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
// src/sdk.ts
|
|
486
|
+
var AsyncEventQueue = class {
|
|
487
|
+
values = [];
|
|
488
|
+
waiters = [];
|
|
489
|
+
closed = false;
|
|
490
|
+
push(value) {
|
|
491
|
+
if (this.closed) return;
|
|
492
|
+
const waiter = this.waiters.shift();
|
|
493
|
+
if (waiter) waiter(value);
|
|
494
|
+
else this.values.push(value);
|
|
495
|
+
}
|
|
496
|
+
close() {
|
|
497
|
+
if (this.closed) return;
|
|
498
|
+
this.closed = true;
|
|
499
|
+
for (const waiter of this.waiters.splice(0)) waiter(void 0);
|
|
500
|
+
}
|
|
501
|
+
next() {
|
|
502
|
+
const value = this.values.shift();
|
|
503
|
+
if (value !== void 0) return Promise.resolve(value);
|
|
504
|
+
if (this.closed) return Promise.resolve(void 0);
|
|
505
|
+
return new Promise((resolve) => this.waiters.push(resolve));
|
|
506
|
+
}
|
|
507
|
+
};
|
|
508
|
+
async function* query(prompt, options = {}) {
|
|
509
|
+
const workspace = options.workspace || process.cwd();
|
|
510
|
+
const loadedConfig = loadConfig(workspace, {
|
|
511
|
+
settingsOverridePath: options.settingsPath,
|
|
512
|
+
noSettings: options.noSettings,
|
|
513
|
+
runMigrations: !options.noSettings
|
|
514
|
+
});
|
|
515
|
+
const config = freezeAgentConfig({
|
|
516
|
+
...loadedConfig,
|
|
517
|
+
...options.model ? { model: options.model } : {},
|
|
518
|
+
...options.maxTurns ? { maxTurns: options.maxTurns } : {},
|
|
519
|
+
...options.agents ? {
|
|
520
|
+
settings: {
|
|
521
|
+
...loadedConfig.settings,
|
|
522
|
+
agents: { ...loadedConfig.settings.agents, mode: options.agents }
|
|
523
|
+
}
|
|
524
|
+
} : {}
|
|
525
|
+
});
|
|
526
|
+
const controller = new AbortController();
|
|
527
|
+
const abortFromCaller = () => controller.abort(options.signal?.reason);
|
|
528
|
+
if (options.signal?.aborted) abortFromCaller();
|
|
529
|
+
else options.signal?.addEventListener("abort", abortFromCaller, { once: true });
|
|
530
|
+
const persistSession = options.persistSession !== false;
|
|
531
|
+
const sessionStore = persistSession ? options.sessionStore ?? new SessionStore(join(resolveBookHome(), "sessions")) : void 0;
|
|
532
|
+
const bootstrap = resolveSessionBootstrap(sessionStore, {
|
|
533
|
+
cwd: config.workspace,
|
|
534
|
+
sessionId: options.sessionId
|
|
535
|
+
});
|
|
536
|
+
if (persistSession && !options.sessionStore && sessionStore) {
|
|
537
|
+
cleanupDebugLogs(DEFAULT_LOCAL_DATA_RETENTION_DAYS, getDebugLogPath());
|
|
538
|
+
sessionStore.cleanup(DEFAULT_LOCAL_DATA_RETENTION_DAYS, /* @__PURE__ */ new Set([bootstrap.sessionId]));
|
|
539
|
+
}
|
|
540
|
+
const queue = new AsyncEventQueue();
|
|
541
|
+
let connections = [];
|
|
542
|
+
queue.push({ type: "system", model: config.model, cwd: config.workspace });
|
|
543
|
+
queue.push({ type: "session", sessionId: bootstrap.sessionId });
|
|
544
|
+
const running = (async () => {
|
|
545
|
+
try {
|
|
546
|
+
for (const notice of collectWithheldProjectNotices({
|
|
547
|
+
workspace: config.workspace,
|
|
548
|
+
settings: config.settings,
|
|
549
|
+
settingsEnabled: !options.noSettings
|
|
550
|
+
})) {
|
|
551
|
+
console.warn(notice);
|
|
552
|
+
}
|
|
553
|
+
const declaredMcpServers = resolveMcpServerList(config.workspace);
|
|
554
|
+
const mcpPartition = partitionMcpServersByApproval(declaredMcpServers, config.settings);
|
|
555
|
+
for (const server of mcpPartition.pending) {
|
|
556
|
+
console.warn(
|
|
557
|
+
`\u26A0 Skipping MCP server "${server.name}" (${server.path}): project-declared servers require one-time approval. Approve it in an interactive session first.`
|
|
558
|
+
);
|
|
559
|
+
}
|
|
560
|
+
const mcp = await connectMcpServers(config.workspace, {
|
|
561
|
+
signal: controller.signal,
|
|
562
|
+
servers: mcpServersToRecord(mcpPartition.allowed),
|
|
563
|
+
onElicit: options.onElicit
|
|
564
|
+
});
|
|
565
|
+
connections = mcp.connections;
|
|
566
|
+
const registry = createDefaultRegistry({ agents: config.settings.agents.mode !== "off" });
|
|
567
|
+
if (mcp.tools.length > 0) registry.registerAll(mcp.tools);
|
|
568
|
+
const result = await runHeadless(config, registry, {
|
|
569
|
+
prompt,
|
|
570
|
+
inputFormat: "text",
|
|
571
|
+
outputFormat: "text",
|
|
572
|
+
history: bootstrap.history,
|
|
573
|
+
transcript: bootstrap.transcript,
|
|
574
|
+
compactBoundaries: bootstrap.compactBoundaries,
|
|
575
|
+
mode: resolvePermissionMode(config.settings, options.permissionMode),
|
|
576
|
+
maxTurns: options.maxTurns,
|
|
577
|
+
persistSession,
|
|
578
|
+
sessionStore,
|
|
579
|
+
sessionId: bootstrap.sessionId,
|
|
580
|
+
sessionCreated: bootstrap.created,
|
|
581
|
+
runSource: "sdk",
|
|
582
|
+
signal: controller.signal,
|
|
583
|
+
onUserQuestionRequired: options.onUserQuestionRequired,
|
|
584
|
+
forwardSubagentText: options.forwardSubagentText,
|
|
585
|
+
stdout: { write: () => true },
|
|
586
|
+
onAgentEvent: (event) => {
|
|
587
|
+
if (event.type !== "system" && event.type !== "session" && event.type !== "result" && event.type !== "done" && (event.type !== "agent_text_delta" || options.forwardSubagentText === true)) {
|
|
588
|
+
queue.push(event);
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
});
|
|
592
|
+
queue.push({
|
|
593
|
+
type: "result",
|
|
594
|
+
messages: result.messages,
|
|
595
|
+
usage: result.usage,
|
|
596
|
+
sessionId: bootstrap.sessionId,
|
|
597
|
+
outcome: result.outcome,
|
|
598
|
+
runs: result.runs,
|
|
599
|
+
// `stdout` above is a discard sink, so this is the only way an SDK
|
|
600
|
+
// caller sees a slash command the host performed on its behalf.
|
|
601
|
+
commandResults: result.commandResults
|
|
602
|
+
});
|
|
603
|
+
} catch (error) {
|
|
604
|
+
if (!controller.signal.aborted) {
|
|
605
|
+
queue.push({
|
|
606
|
+
type: "error",
|
|
607
|
+
error: error instanceof Error ? error.message : String(error)
|
|
608
|
+
});
|
|
609
|
+
}
|
|
610
|
+
} finally {
|
|
611
|
+
await disconnectMcpServers(connections);
|
|
612
|
+
queue.push({ type: "done" });
|
|
613
|
+
queue.close();
|
|
614
|
+
}
|
|
615
|
+
})();
|
|
616
|
+
try {
|
|
617
|
+
while (true) {
|
|
618
|
+
const event = await queue.next();
|
|
619
|
+
if (!event) break;
|
|
620
|
+
yield event;
|
|
621
|
+
}
|
|
622
|
+
} finally {
|
|
623
|
+
controller.abort(new Error("SDK event consumer stopped"));
|
|
624
|
+
options.signal?.removeEventListener("abort", abortFromCaller);
|
|
625
|
+
await running;
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
function createAgentManager(config) {
|
|
629
|
+
return new AgentManager(config, createDefaultRegistry({ agents: true }).getDefinitions());
|
|
630
|
+
}
|
|
631
|
+
export {
|
|
632
|
+
AgentManager,
|
|
633
|
+
DEFAULT_SKILL_EVALUATION_FIXTURES,
|
|
634
|
+
DEFAULT_SKILL_EVALUATION_THRESHOLDS,
|
|
635
|
+
SKILL_EVALUATION_CATEGORIES,
|
|
636
|
+
SessionRuntime,
|
|
637
|
+
SkillRegistry,
|
|
638
|
+
SkillRegistryError,
|
|
639
|
+
applySkillOverrides,
|
|
640
|
+
buildSkillListing,
|
|
641
|
+
buildSkillReport,
|
|
642
|
+
createAgentManager,
|
|
643
|
+
createAgentSessionSnapshot,
|
|
644
|
+
createDefaultRegistry,
|
|
645
|
+
createRegistry,
|
|
646
|
+
createToolSurface,
|
|
647
|
+
discoverSkills,
|
|
648
|
+
evaluateSkillActivation,
|
|
649
|
+
evaluateSuccess,
|
|
650
|
+
explicitSkillMentions,
|
|
651
|
+
generateSkillListing,
|
|
652
|
+
loadConfig,
|
|
653
|
+
loadSkillBody,
|
|
654
|
+
observeSkillEvaluation,
|
|
655
|
+
query,
|
|
656
|
+
reduceAgentSessionSnapshot,
|
|
657
|
+
renderSkillEvaluationReport,
|
|
658
|
+
runPairedEvaluation,
|
|
659
|
+
runSkillActivationEvaluation,
|
|
660
|
+
skillRoots,
|
|
661
|
+
toolFailure,
|
|
662
|
+
toolSuccess,
|
|
663
|
+
writeSkillEvaluationReport
|
|
664
|
+
};
|
|
665
|
+
//# sourceMappingURL=sdk.js.map
|