jev-agent-tools 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -1
- package/CONTRIBUTING.md +3 -0
- package/README.md +22 -14
- package/SECURITY.md +17 -1
- package/dist/adapters/ask-files.js +11 -2
- package/dist/adapters/ask-proof.js +63 -7
- package/dist/adapters/command.js +82 -29
- package/dist/adapters/docs.js +30 -10
- package/dist/adapters/evidence-context.js +119 -0
- package/dist/adapters/files.js +141 -16
- package/dist/adapters/find.js +34 -6
- package/dist/adapters/git-base.js +7 -1
- package/dist/adapters/git.js +51 -7
- package/dist/adapters/locate-file.js +47 -9
- package/dist/adapters/private-storage.js +14 -6
- package/dist/adapters/risk-callers.js +3 -0
- package/dist/adapters/shell.js +23 -7
- package/dist/adapters/test-inventory.js +10 -2
- package/dist/configuration.js +17 -7
- package/dist/constants.js +26 -5
- package/dist/core/ask-references.js +193 -109
- package/dist/core/asks.js +78 -7
- package/dist/core/locate.js +8 -8
- package/dist/core/output.js +17 -0
- package/dist/core/result-report.js +302 -0
- package/dist/core/secret-path.js +34 -0
- package/dist/core/state.js +8 -1
- package/dist/core/units.js +1 -1
- package/dist/jev/client.js +34 -12
- package/dist/mcp/protocol.js +50 -27
- package/dist/mcp/tools.js +20 -7
- package/dist/render.js +72 -0
- package/dist/report-schema.js +1356 -0
- package/dist/result-types.js +1 -0
- package/dist/texts/ask-files.js +3 -1
- package/dist/texts/ask.js +3 -1
- package/dist/texts/check-diff.js +7 -4
- package/dist/texts/find.js +7 -2
- package/dist/texts/guide.js +3 -16
- package/dist/texts/instructions.js +72 -0
- package/dist/texts/locate.js +7 -2
- package/dist/texts/select-tests.js +3 -1
- package/dist/tools/ask-files.js +248 -15
- package/dist/tools/ask.js +523 -62
- package/dist/tools/check-diff.js +222 -30
- package/dist/tools/docs-check.js +122 -13
- package/dist/tools/find.js +320 -27
- package/dist/tools/locate.js +317 -18
- package/dist/tools/review-report.js +230 -0
- package/dist/tools/select-tests.js +273 -19
- package/dist/tools/spec-check.js +119 -22
- package/docs/adr/0001-strict-typescript-pure-core-offline-tests.md +3 -3
- package/docs/agent-instructions.md +59 -30
- package/docs/design.md +13 -1
- package/docs/mcp.md +8 -6
- package/docs/tools/jev_ask.md +8 -5
- package/docs/tools/jev_ask_files.md +2 -1
- package/docs/tools/jev_check_diff.md +4 -1
- package/docs/tools/jev_find_files.md +2 -1
- package/docs/tools/jev_locate_in_file.md +5 -0
- package/docs/tools/jev_select_tests.md +4 -1
- package/package.json +1 -1
- package/rules/jev-ask.md +22 -1
- package/server.json +2 -2
- package/src/adapters/ask-files.ts +11 -3
- package/src/adapters/ask-proof.ts +69 -11
- package/src/adapters/command.ts +96 -33
- package/src/adapters/docs.ts +33 -14
- package/src/adapters/evidence-context.ts +169 -0
- package/src/adapters/files.ts +146 -16
- package/src/adapters/find.ts +37 -7
- package/src/adapters/git-base.ts +7 -1
- package/src/adapters/git.ts +61 -8
- package/src/adapters/locate-file.ts +51 -9
- package/src/adapters/private-storage.ts +17 -5
- package/src/adapters/risk-callers.ts +3 -0
- package/src/adapters/shell.ts +23 -7
- package/src/adapters/test-inventory.ts +12 -4
- package/src/configuration.ts +16 -2
- package/src/constants.ts +26 -5
- package/src/core/ask-references.ts +262 -146
- package/src/core/asks.ts +79 -7
- package/src/core/import-boundaries.ts +8 -3
- package/src/core/locate.ts +8 -5
- package/src/core/output.ts +34 -0
- package/src/core/result-report.ts +410 -0
- package/src/core/secret-path.ts +37 -0
- package/src/core/state.ts +8 -1
- package/src/core/units.ts +3 -2
- package/src/index.ts +3 -0
- package/src/jev/client.ts +54 -16
- package/src/jev/types.ts +18 -3
- package/src/mcp/protocol.ts +91 -41
- package/src/mcp/tools.ts +26 -13
- package/src/render.ts +109 -0
- package/src/report-schema.ts +1380 -0
- package/src/result-types.ts +234 -0
- package/src/result.ts +4 -1
- package/src/runtime.ts +6 -0
- package/src/texts/ask-files.ts +4 -1
- package/src/texts/ask.ts +8 -1
- package/src/texts/check-diff.ts +7 -4
- package/src/texts/find.ts +8 -2
- package/src/texts/guide.ts +8 -16
- package/src/texts/instructions.ts +98 -0
- package/src/texts/locate.ts +8 -2
- package/src/texts/run-end.ts +2 -2
- package/src/texts/select-tests.ts +4 -1
- package/src/tools/ask-files.ts +309 -14
- package/src/tools/ask.ts +700 -77
- package/src/tools/check-diff.ts +331 -28
- package/src/tools/docs-check.ts +241 -39
- package/src/tools/find.ts +386 -29
- package/src/tools/locate.ts +384 -19
- package/src/tools/review-report.ts +308 -0
- package/src/tools/select-tests.ts +479 -21
- package/src/tools/spec-check.ts +193 -19
package/dist/tools/ask.js
CHANGED
|
@@ -6,6 +6,7 @@ import { collectAskRepository, repositoryPath } from "../adapters/ask-proof.js";
|
|
|
6
6
|
import { collectProofSyntax } from "../adapters/ask-syntax.js";
|
|
7
7
|
import { canonicalPath } from "../adapters/canonical-path.js";
|
|
8
8
|
import { captureCommand } from "../adapters/command.js";
|
|
9
|
+
import { resolveEvidenceContext, withEvidenceContext, } from "../adapters/evidence-context.js";
|
|
9
10
|
import { collectFiles } from "../adapters/files.js";
|
|
10
11
|
import { shareGitInventory } from "../adapters/git-inventory.js";
|
|
11
12
|
import { hostUsage } from "../adapters/usage.js";
|
|
@@ -17,17 +18,20 @@ import { outputChunks, selectOutput } from "../core/command-output.js";
|
|
|
17
18
|
import { createImportGraphBuilder, } from "../core/imports.js";
|
|
18
19
|
import { checkIntegrity } from "../core/integrity.js";
|
|
19
20
|
import { buildEnvelope } from "../core/output.js";
|
|
21
|
+
import { answerItemFromInput, buildResultReport, contextFromEvidence, known, notApplicable, unknown, } from "../core/result-report.js";
|
|
20
22
|
import { assembleState } from "../core/state.js";
|
|
21
23
|
import { discoverTests } from "../core/test-discovery.js";
|
|
22
24
|
import { describeAsk } from "../describe.js";
|
|
23
|
-
import {
|
|
25
|
+
import { renderResultReport } from "../render.js";
|
|
24
26
|
import { isRecord } from "../result.js";
|
|
25
27
|
import { COMMAND_LIMIT_NOTICE } from "../texts/ask.js";
|
|
26
28
|
import { NOT_CONFIGURED } from "../texts/configuration.js";
|
|
29
|
+
import { ASK_GUIDELINE } from "../texts/instructions.js";
|
|
27
30
|
import { asksParameter } from "./ask-schema.js";
|
|
28
31
|
export const askParameters = Type.Object({
|
|
32
|
+
root: Type.Optional(Type.String()),
|
|
29
33
|
state: Type.Optional(Type.String({ maxLength: ASK_NOTE_MAX_CHARS })),
|
|
30
|
-
paths: Type.Optional(Type.Array(Type.String({ minLength: 1 })
|
|
34
|
+
paths: Type.Optional(Type.Array(Type.String({ minLength: 1 }))),
|
|
31
35
|
base: Type.Optional(Type.String({ minLength: 1 })),
|
|
32
36
|
command: Type.Optional(Type.String({ minLength: 1 })),
|
|
33
37
|
timeout_s: Type.Optional(Type.Number({
|
|
@@ -57,14 +61,49 @@ export function createAskTool(dependencies) {
|
|
|
57
61
|
}
|
|
58
62
|
: {
|
|
59
63
|
promptSnippet: "Typed answers about one situation built from a note, files and a command's output",
|
|
60
|
-
promptGuidelines: [
|
|
61
|
-
"jev_ask: before you conclude that a failure is a bug in the code, a wrong test or the environment, or that a plan matches the docs, pass the files to jev_ask and weigh its answer against your own reading",
|
|
62
|
-
],
|
|
64
|
+
promptGuidelines: [ASK_GUIDELINE],
|
|
63
65
|
}),
|
|
64
66
|
async execute(_id, args, signal, _update, ctx) {
|
|
65
67
|
const client = dependencies.client;
|
|
66
|
-
const
|
|
68
|
+
const evidence = await resolveEvidenceContext(ctx.cwd, args.root, {
|
|
69
|
+
exec: execute,
|
|
70
|
+
signal,
|
|
71
|
+
origin: dependencies.evidenceOrigin,
|
|
72
|
+
});
|
|
73
|
+
const evidenceContext = evidence.context;
|
|
74
|
+
const cwd = evidence.ok ? evidence.cwd : evidenceContext.authority.path;
|
|
67
75
|
const started = performance.now();
|
|
76
|
+
const evidenceFacts = {
|
|
77
|
+
omissions: [],
|
|
78
|
+
blockedGroups: [],
|
|
79
|
+
inventory: [],
|
|
80
|
+
};
|
|
81
|
+
evidenceContext.requestedBase = args.base;
|
|
82
|
+
const asks = compileAsks(args.asks, { surface: "ask" });
|
|
83
|
+
let commandContext = args.command
|
|
84
|
+
? {
|
|
85
|
+
execution: "not_started",
|
|
86
|
+
cwd: evidence.ok
|
|
87
|
+
? known(cwd)
|
|
88
|
+
: unknown("effective root not established"),
|
|
89
|
+
exitCode: unknown("command not started"),
|
|
90
|
+
timedOut: unknown("command not started"),
|
|
91
|
+
}
|
|
92
|
+
: {
|
|
93
|
+
execution: "not_requested",
|
|
94
|
+
cwd: notApplicable("no command"),
|
|
95
|
+
exitCode: notApplicable("no command"),
|
|
96
|
+
timedOut: notApplicable("no command"),
|
|
97
|
+
};
|
|
98
|
+
let refusalCause = "invalid_arguments";
|
|
99
|
+
let inventoryCollected = false;
|
|
100
|
+
const auxiliary = {
|
|
101
|
+
controls: { fresh: 0, cache: 0, notJudged: 0, static: 0 },
|
|
102
|
+
passages: { fresh: 0, cache: 0, notJudged: 0 },
|
|
103
|
+
};
|
|
104
|
+
const attributionAnswers = {};
|
|
105
|
+
const attributionMissing = new Set();
|
|
106
|
+
const findResults = [];
|
|
68
107
|
const exec = shareGitInventory(execute);
|
|
69
108
|
const finish = (result, input) => {
|
|
70
109
|
const envelope = buildEnvelope({
|
|
@@ -79,15 +118,246 @@ export function createAskTool(dependencies) {
|
|
|
79
118
|
},
|
|
80
119
|
});
|
|
81
120
|
runtime.session.record(envelope);
|
|
121
|
+
const diagnostics = [];
|
|
122
|
+
const actions = [];
|
|
123
|
+
const globalMissing = new Map();
|
|
124
|
+
const diagnose = (cause, fact, scope, target, origin, effect = "blocking", next) => {
|
|
125
|
+
const id = `diagnostic:${diagnostics.length}`;
|
|
126
|
+
const actionId = `action:${actions.length}`;
|
|
127
|
+
const code = cause === "invalid_root" || cause === "invalid_base"
|
|
128
|
+
? "correct_context"
|
|
129
|
+
: cause === "forbidden_path"
|
|
130
|
+
? "correct_reference"
|
|
131
|
+
: cause === "missing_required" || cause === "empty_required"
|
|
132
|
+
? "provide_evidence"
|
|
133
|
+
: cause === "not_configured"
|
|
134
|
+
? "configure_client"
|
|
135
|
+
: "inspect_native";
|
|
136
|
+
actions.push({
|
|
137
|
+
id: actionId,
|
|
138
|
+
code,
|
|
139
|
+
target: target === undefined
|
|
140
|
+
? unknown("target not established")
|
|
141
|
+
: known(target),
|
|
142
|
+
scope,
|
|
143
|
+
condition: "Only if materially new evidence or corrected context is available",
|
|
144
|
+
instruction: next ??
|
|
145
|
+
(code === "provide_evidence"
|
|
146
|
+
? `Provide the actually missing evidence${target ? `: ${target}` : ""}; do not repeat an already captured command.`
|
|
147
|
+
: "Inspect the supplied evidence natively; correct the named cause before any useful new Jev call."),
|
|
148
|
+
repeatUnchanged: false,
|
|
149
|
+
});
|
|
150
|
+
diagnostics.push({
|
|
151
|
+
id,
|
|
152
|
+
cause,
|
|
153
|
+
fact,
|
|
154
|
+
target: target === undefined
|
|
155
|
+
? unknown("target not established")
|
|
156
|
+
: known(target),
|
|
157
|
+
origin,
|
|
158
|
+
scope,
|
|
159
|
+
effect,
|
|
160
|
+
material: true,
|
|
161
|
+
omittedMembers: [],
|
|
162
|
+
memberCount: known(0),
|
|
163
|
+
actionIds: [actionId],
|
|
164
|
+
});
|
|
165
|
+
return { diagnosticIds: [id], actionIds: [actionId] };
|
|
166
|
+
};
|
|
167
|
+
const items = asks.ok
|
|
168
|
+
? asks.readings.map((reading, index) => {
|
|
169
|
+
const groupId = `group:${asks.groups.findIndex((group) => group.includes(reading.id))}`;
|
|
170
|
+
const target = asks.asks[reading.askIndex]?.about ?? "state";
|
|
171
|
+
const pieces = [
|
|
172
|
+
{
|
|
173
|
+
target,
|
|
174
|
+
canonicalPath: unknown("selector does not establish a canonical file identity"),
|
|
175
|
+
aliases: [],
|
|
176
|
+
side: target === "output" ? "output" : "state",
|
|
177
|
+
revision: notApplicable("non-file evidence"),
|
|
178
|
+
},
|
|
179
|
+
];
|
|
180
|
+
for (const [alias, path] of Object.entries(evidenceFacts.references?.aliases ?? {})) {
|
|
181
|
+
const before = evidenceFacts.references?.additions.some((addition) => addition.reference === alias &&
|
|
182
|
+
addition.side === "files_before") || target.includes("files_before");
|
|
183
|
+
pieces.push({
|
|
184
|
+
target: alias,
|
|
185
|
+
canonicalPath: evidenceContext.effectiveRoot
|
|
186
|
+
? known(resolve(evidenceContext.effectiveRoot.path, path))
|
|
187
|
+
: unknown("effective root unavailable"),
|
|
188
|
+
aliases: [alias],
|
|
189
|
+
side: before ? "before" : "current",
|
|
190
|
+
revision: before && evidenceContext.resolvedBase
|
|
191
|
+
? known(evidenceContext.resolvedBase)
|
|
192
|
+
: unknown("working tree evidence"),
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
const common = {
|
|
196
|
+
id: reading.id,
|
|
197
|
+
kind: reading.kind === "verify"
|
|
198
|
+
? "claim"
|
|
199
|
+
: "question",
|
|
200
|
+
label: reading.label,
|
|
201
|
+
groupId,
|
|
202
|
+
evidence: pieces,
|
|
203
|
+
diagnosticIds: [],
|
|
204
|
+
actionIds: [],
|
|
205
|
+
};
|
|
206
|
+
const originalAnswer = input.answers?.[index];
|
|
207
|
+
const attributionControls = attributionAnswers[reading.id];
|
|
208
|
+
const answer = originalAnswer && attributionMissing.has(reading.id)
|
|
209
|
+
? {
|
|
210
|
+
...originalAnswer,
|
|
211
|
+
unjudged: true,
|
|
212
|
+
reason: "Required attribution control is incomplete",
|
|
213
|
+
}
|
|
214
|
+
: originalAnswer && attributionControls
|
|
215
|
+
? {
|
|
216
|
+
...originalAnswer,
|
|
217
|
+
reportControls: [
|
|
218
|
+
...(originalAnswer.reportControls ?? []),
|
|
219
|
+
...attributionControls,
|
|
220
|
+
],
|
|
221
|
+
}
|
|
222
|
+
: originalAnswer;
|
|
223
|
+
if (answer &&
|
|
224
|
+
!answer.unjudged &&
|
|
225
|
+
(answer.source || answer.answer?.source))
|
|
226
|
+
return answerItemFromInput(common, input.controls?.length
|
|
227
|
+
? {
|
|
228
|
+
...answer,
|
|
229
|
+
band: "unsure",
|
|
230
|
+
reason: input.controls
|
|
231
|
+
.map((control) => control.fact)
|
|
232
|
+
.join("; "),
|
|
233
|
+
}
|
|
234
|
+
: answer);
|
|
235
|
+
const blocked = evidenceFacts.blockedGroups.find((group) => group.ids.includes(reading.id));
|
|
236
|
+
const omission = evidenceFacts.omissions.find((item) => item.required &&
|
|
237
|
+
(item.origin === "note" || item.scope === reading.id));
|
|
238
|
+
const groupFailure = result.ok
|
|
239
|
+
? (asks.groups.find((group) => group.includes(reading.id)) ??
|
|
240
|
+
[])
|
|
241
|
+
.map((id) => result.answers[id])
|
|
242
|
+
.find((answer) => answer?.type === "unjudged" && answer.cause)
|
|
243
|
+
: undefined;
|
|
244
|
+
const cause = blocked
|
|
245
|
+
? (blocked.cause ?? "missing_required")
|
|
246
|
+
: groupFailure?.type === "unjudged" && groupFailure.cause
|
|
247
|
+
? groupFailure.cause
|
|
248
|
+
: input.budget
|
|
249
|
+
? input.budget.kind === "session"
|
|
250
|
+
? "session_budget"
|
|
251
|
+
: "call_budget"
|
|
252
|
+
: (result.failureCause ??
|
|
253
|
+
(input.refusal ? refusalCause : "control_failure"));
|
|
254
|
+
const globalKey = omission?.origin === "note" ? omission.reference : undefined;
|
|
255
|
+
const existingLinks = globalKey
|
|
256
|
+
? globalMissing.get(globalKey)
|
|
257
|
+
: undefined;
|
|
258
|
+
const links = existingLinks ??
|
|
259
|
+
diagnose(cause, blocked?.reason ??
|
|
260
|
+
answer?.reason ??
|
|
261
|
+
(result.ok
|
|
262
|
+
? "Required judgment group incomplete or response provenance not established"
|
|
263
|
+
: result.error), omission?.origin === "note"
|
|
264
|
+
? { kind: "call" }
|
|
265
|
+
: { kind: "group", groupIds: [groupId] }, omission?.reference, omission?.origin === "note"
|
|
266
|
+
? "note"
|
|
267
|
+
: blocked
|
|
268
|
+
? "ask"
|
|
269
|
+
: input.budget
|
|
270
|
+
? "budget"
|
|
271
|
+
: "provider");
|
|
272
|
+
if (globalKey)
|
|
273
|
+
globalMissing.set(globalKey, links);
|
|
274
|
+
return {
|
|
275
|
+
...common,
|
|
276
|
+
...links,
|
|
277
|
+
treatment: "not_judged",
|
|
278
|
+
source: "none",
|
|
279
|
+
};
|
|
280
|
+
})
|
|
281
|
+
: [];
|
|
282
|
+
if (!items.length)
|
|
283
|
+
diagnose(input.refusal ? refusalCause : "collection_empty", input.refusal ?? "No requested results collected", { kind: "call" }, undefined, "input");
|
|
284
|
+
for (const control of input.controls ?? [])
|
|
285
|
+
diagnose("control_failure", control.fact, { kind: "call" }, undefined, "control", "reservation", control.next);
|
|
286
|
+
for (const limitation of input.limitations ?? [])
|
|
287
|
+
diagnose("evidence_limit", limitation.fact, { kind: "call" }, limitation.path, "closure", "reservation", limitation.next);
|
|
288
|
+
const report = buildResultReport({
|
|
289
|
+
tool: "jev_ask",
|
|
290
|
+
context: contextFromEvidence(evidenceContext, {
|
|
291
|
+
command: commandContext,
|
|
292
|
+
inventories: inventoryCollected
|
|
293
|
+
? [
|
|
294
|
+
{
|
|
295
|
+
id: "repository",
|
|
296
|
+
kind: "repository",
|
|
297
|
+
rules: ["Git tracked and admitted repository evidence"],
|
|
298
|
+
restrictions: args.paths ?? [],
|
|
299
|
+
discovered: known(evidenceFacts.inventory.length),
|
|
300
|
+
considered: known(evidenceFacts.inventory.length),
|
|
301
|
+
scopeRestricted: false,
|
|
302
|
+
criteria: [],
|
|
303
|
+
},
|
|
304
|
+
]
|
|
305
|
+
: [],
|
|
306
|
+
}),
|
|
307
|
+
items,
|
|
308
|
+
diagnostics,
|
|
309
|
+
actions,
|
|
310
|
+
metrics: {
|
|
311
|
+
calls: result.calls ?? 0,
|
|
312
|
+
questions: result.questions ?? 0,
|
|
313
|
+
cacheHits: result.cacheHits ?? 0,
|
|
314
|
+
cacheRequests: result.cacheRequests ?? 0,
|
|
315
|
+
costUsd: result.usage?.costUsd,
|
|
316
|
+
elapsedMs: performance.now() - started,
|
|
317
|
+
},
|
|
318
|
+
auxiliary,
|
|
319
|
+
total: known(items.length),
|
|
320
|
+
refused: !!input.refusal &&
|
|
321
|
+
refusalCause !== "not_configured" &&
|
|
322
|
+
!result.failureCause,
|
|
323
|
+
});
|
|
82
324
|
runtime.guide.deliver(ctx);
|
|
83
325
|
return {
|
|
84
|
-
content: [
|
|
85
|
-
|
|
326
|
+
content: [
|
|
327
|
+
{
|
|
328
|
+
type: "text",
|
|
329
|
+
text: renderResultReport(report, { details: envelope }),
|
|
330
|
+
},
|
|
331
|
+
],
|
|
332
|
+
details: {
|
|
333
|
+
...result,
|
|
334
|
+
result: report,
|
|
335
|
+
evidenceContext,
|
|
336
|
+
evidenceFacts,
|
|
337
|
+
},
|
|
86
338
|
};
|
|
87
339
|
};
|
|
88
|
-
const textResult = (text
|
|
340
|
+
const textResult = (text, cause = "invalid_arguments") => {
|
|
341
|
+
refusalCause = cause;
|
|
342
|
+
const technical = findResults.reduce((totals, judgment) => ({
|
|
343
|
+
calls: totals.calls + (judgment.calls ?? 0),
|
|
344
|
+
questions: totals.questions + (judgment.questions ?? 0),
|
|
345
|
+
cacheHits: totals.cacheHits + (judgment.cacheHits ?? 0),
|
|
346
|
+
cacheRequests: totals.cacheRequests + (judgment.cacheRequests ?? 0),
|
|
347
|
+
}), { calls: 0, questions: 0, cacheHits: 0, cacheRequests: 0 });
|
|
348
|
+
return finish({ ok: false, error: text, ...technical }, { refusal: text });
|
|
349
|
+
};
|
|
350
|
+
if (!evidence.ok)
|
|
351
|
+
return textResult(evidence.error, evidence.cause ?? "invalid_root");
|
|
352
|
+
if (!asks.ok)
|
|
353
|
+
return textResult(asks.error);
|
|
354
|
+
const preliminary = resolveAskReferences(asks, args.state, {}, []);
|
|
355
|
+
if (preliminary.invalid.length) {
|
|
356
|
+
evidenceFacts.references = preliminary;
|
|
357
|
+
return textResult(preliminary.invalid.map((item) => item.reason).join("; "), "forbidden_path");
|
|
358
|
+
}
|
|
89
359
|
if (!client)
|
|
90
|
-
return textResult(NOT_CONFIGURED);
|
|
360
|
+
return textResult(NOT_CONFIGURED, "not_configured");
|
|
91
361
|
if (args.command && !allowCommand)
|
|
92
362
|
return textResult("command disabled by JEV_TOOLS_ALLOW_COMMAND=0");
|
|
93
363
|
if (args.timeout_s !== undefined &&
|
|
@@ -97,48 +367,80 @@ export function createAskTool(dependencies) {
|
|
|
97
367
|
return textResult(`timeout_s must be between 1 and ${ASK_TIMEOUT_MAX_S}`);
|
|
98
368
|
if (args.state === undefined && !args.paths?.length && !args.command)
|
|
99
369
|
return textResult("Provide state, command or at least one path.");
|
|
100
|
-
const asks = compileAsks(args.asks, { surface: "ask" });
|
|
101
|
-
if (!asks.ok)
|
|
102
|
-
return textResult(asks.error);
|
|
103
370
|
const files = await collectFiles(cwd, args.paths ?? [], signal, {
|
|
104
371
|
exec,
|
|
105
372
|
allowEmpty: true,
|
|
106
373
|
skipInvalidUtf8: true,
|
|
107
374
|
});
|
|
108
375
|
if (!files.ok)
|
|
109
|
-
return textResult(files.error);
|
|
376
|
+
return textResult(files.error, files.cause ?? "file_unavailable");
|
|
110
377
|
if (!Object.keys(files.files).length &&
|
|
111
378
|
args.state === undefined &&
|
|
112
379
|
!args.command &&
|
|
113
380
|
files.skipped.length)
|
|
114
|
-
return textResult(`No readable evidence: ${files.skipped.join("; ")}
|
|
381
|
+
return textResult(`No readable evidence: ${files.skipped.join("; ")}`, "collection_empty");
|
|
115
382
|
const proofLimitations = files.skipped.map((fact) => ({
|
|
116
383
|
fact,
|
|
117
384
|
next: "Provide this file as UTF-8 text if the judgment needs it.",
|
|
118
385
|
}));
|
|
119
386
|
let repository = await collectAskRepository(exec, cwd, args.base, signal);
|
|
387
|
+
inventoryCollected = repository.ok;
|
|
388
|
+
if (repository.ok) {
|
|
389
|
+
evidenceFacts.inventory = [...repository.known];
|
|
390
|
+
evidenceContext.resolvedBase = repository.baseSha;
|
|
391
|
+
}
|
|
120
392
|
if (!repository.ok && args.base)
|
|
121
|
-
return textResult(repository.error);
|
|
393
|
+
return textResult(repository.error, "invalid_base");
|
|
122
394
|
if (!repository.ok)
|
|
123
395
|
proofLimitations.push({
|
|
124
396
|
fact: `closure unavailable: ${repository.error}`,
|
|
125
397
|
next: "pass dependencies explicitly in paths",
|
|
126
398
|
});
|
|
399
|
+
if (args.command)
|
|
400
|
+
commandContext = {
|
|
401
|
+
...commandContext,
|
|
402
|
+
execution: "started",
|
|
403
|
+
cwd: known(repository.ok ? repository.root : cwd),
|
|
404
|
+
};
|
|
127
405
|
const command = args.command
|
|
128
|
-
? await captureCommand(exec, repository.ok ? repository.root : cwd, args.command, args.timeout_s, signal)
|
|
406
|
+
? await captureCommand(exec, repository.ok ? repository.root : cwd, args.command, args.timeout_s, signal, dependencies.apiKey ?? process.env.JEV_TOOLS_API_KEY)
|
|
129
407
|
: undefined;
|
|
130
|
-
if (command
|
|
131
|
-
|
|
408
|
+
if (command?.ok)
|
|
409
|
+
commandContext = {
|
|
410
|
+
execution: "finished",
|
|
411
|
+
cwd: known(repository.ok ? repository.root : cwd),
|
|
412
|
+
exitCode: known(command.output.exit_code),
|
|
413
|
+
timedOut: known(command.output.timed_out),
|
|
414
|
+
};
|
|
415
|
+
if (command && !command.ok) {
|
|
416
|
+
commandContext = {
|
|
417
|
+
// Reported execution states are not_requested, not_started, started
|
|
418
|
+
// and finished. A spawn whose completion was not observed (adapter
|
|
419
|
+
// "unknown") did start, so it reports "started" with unknown exit
|
|
420
|
+
// code and timeout.
|
|
421
|
+
execution: command.commandExecution === "unknown"
|
|
422
|
+
? "started"
|
|
423
|
+
: command.commandExecution,
|
|
424
|
+
cwd: known(repository.ok ? repository.root : cwd),
|
|
425
|
+
exitCode: command.commandExitCode !== undefined
|
|
426
|
+
? known(command.commandExitCode)
|
|
427
|
+
: unknown("command completion not observed"),
|
|
428
|
+
timedOut: command.commandTimedOut !== undefined
|
|
429
|
+
? known(command.commandTimedOut)
|
|
430
|
+
: unknown("command completion not observed"),
|
|
431
|
+
};
|
|
432
|
+
return textResult(command.error, command.cause ?? "file_unavailable");
|
|
433
|
+
}
|
|
132
434
|
if (command?.ok) {
|
|
133
435
|
repository = await collectAskRepository(exec, cwd, args.base, signal);
|
|
134
436
|
if (!repository.ok && args.base)
|
|
135
|
-
return textResult(repository.error);
|
|
437
|
+
return textResult(repository.error, "invalid_base");
|
|
136
438
|
}
|
|
137
439
|
const initialState = assembleState(args.state, files.files);
|
|
138
440
|
if (!initialState.ok)
|
|
139
441
|
return textResult(initialState.error);
|
|
140
442
|
if (command?.ok && Object.hasOwn(initialState.state, "output"))
|
|
141
|
-
return textResult("state contains reserved key output when command is supplied; rename it in your note");
|
|
443
|
+
return textResult("state contains reserved key output when command is supplied; rename it in your note", "reserved_key");
|
|
142
444
|
let outputNotIdentifiable = false;
|
|
143
445
|
if (command?.ok && repository.ok) {
|
|
144
446
|
for (const target of command.targets) {
|
|
@@ -211,11 +513,42 @@ export function createAskTool(dependencies) {
|
|
|
211
513
|
(!command.targets.length || !repository.ok) &&
|
|
212
514
|
!discoverTests(Object.entries(files.files).map(([path, text]) => ({ path, text }))).entries.length)
|
|
213
515
|
outputNotIdentifiable = true;
|
|
214
|
-
const references = resolveAskReferences(asks, args.state,
|
|
516
|
+
const references = resolveAskReferences(asks, args.state, repository.ok
|
|
517
|
+
? {
|
|
518
|
+
files: Object.fromEntries(Object.entries(files.files).map(([path, text]) => [
|
|
519
|
+
repositoryPath(repository.root, cwd, path),
|
|
520
|
+
text,
|
|
521
|
+
])),
|
|
522
|
+
beforeInventory: args.base ? [...repository.beforeKnown] : [],
|
|
523
|
+
}
|
|
524
|
+
: files.files, repository?.ok ? [...repository.known] : []);
|
|
525
|
+
evidenceFacts.references = references;
|
|
526
|
+
evidenceFacts.inventory = repository.ok ? [...repository.known] : [];
|
|
527
|
+
if (references.invalid.length)
|
|
528
|
+
return textResult(references.invalid.map((item) => item.reason).join("; "), "forbidden_path");
|
|
215
529
|
const unresolved = [...references.unresolved];
|
|
530
|
+
const historical = {};
|
|
531
|
+
if (repository.ok && args.base) {
|
|
532
|
+
for (const reference of references.additions.filter((reference) => reference.side === "files_before" && reference.required)) {
|
|
533
|
+
const before = await repository.readBefore(reference.path);
|
|
534
|
+
if (before.ok)
|
|
535
|
+
historical[reference.path] = before.text;
|
|
536
|
+
else {
|
|
537
|
+
if (before.cause === "forbidden_path")
|
|
538
|
+
return textResult(before.error, before.cause);
|
|
539
|
+
unresolved.push({
|
|
540
|
+
...reference,
|
|
541
|
+
reason: before.error,
|
|
542
|
+
...(before.cause ? { cause: before.cause } : {}),
|
|
543
|
+
});
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
}
|
|
216
547
|
if (repository?.ok) {
|
|
217
548
|
const root = repository.root;
|
|
218
|
-
const additions = await Promise.all(references.additions
|
|
549
|
+
const additions = await Promise.all(references.additions
|
|
550
|
+
.filter((addition) => addition.side !== "files_before")
|
|
551
|
+
.map(async (addition) => ({
|
|
219
552
|
addition,
|
|
220
553
|
added: await collectFiles(root, [addition.path], signal, {
|
|
221
554
|
allowEmpty: true,
|
|
@@ -225,16 +558,18 @@ export function createAskTool(dependencies) {
|
|
|
225
558
|
})));
|
|
226
559
|
for (const { addition, added } of additions) {
|
|
227
560
|
if (added.ok) {
|
|
228
|
-
|
|
561
|
+
const local = repositoryPath(cwd, root, addition.path);
|
|
562
|
+
files.files[local] = added.files[addition.path] ?? "";
|
|
229
563
|
files.identities.push(...added.identities.map((identity) => ({
|
|
230
564
|
...identity,
|
|
231
|
-
requestedPath:
|
|
565
|
+
requestedPath: local,
|
|
232
566
|
})));
|
|
233
567
|
}
|
|
234
568
|
else
|
|
235
569
|
unresolved.push({
|
|
236
|
-
|
|
570
|
+
...addition,
|
|
237
571
|
reason: added.error,
|
|
572
|
+
...(added.cause ? { cause: added.cause } : {}),
|
|
238
573
|
});
|
|
239
574
|
}
|
|
240
575
|
}
|
|
@@ -242,36 +577,48 @@ export function createAskTool(dependencies) {
|
|
|
242
577
|
if (!state.ok)
|
|
243
578
|
return textResult(state.error);
|
|
244
579
|
if (command?.ok && Object.hasOwn(state.state, "output"))
|
|
245
|
-
return textResult("state contains reserved key output when command is supplied; rename it in your note");
|
|
580
|
+
return textResult("state contains reserved key output when command is supplied; rename it in your note", "reserved_key");
|
|
246
581
|
if (command?.ok)
|
|
247
582
|
state.state.output = { ...command.output };
|
|
248
583
|
if (repository?.ok) {
|
|
584
|
+
if (evidenceContext.effectiveRoot)
|
|
585
|
+
evidenceContext.effectiveRoot.path = repository.root;
|
|
586
|
+
evidenceContext.requestedBase = args.base;
|
|
587
|
+
evidenceContext.resolvedBase = repository.baseSha;
|
|
249
588
|
const canonical = Object.fromEntries(Object.entries(files.files).map(([path, text]) => [
|
|
250
589
|
repositoryPath(repository.root, cwd, path),
|
|
251
590
|
text,
|
|
252
591
|
]));
|
|
253
|
-
|
|
254
|
-
|
|
592
|
+
state.state.files = canonical;
|
|
593
|
+
state.state.evidence = {
|
|
594
|
+
context: evidenceContext,
|
|
595
|
+
aliases: Object.fromEntries(Object.entries(references.aliases)
|
|
596
|
+
.map(([alias, path]) => [
|
|
597
|
+
alias,
|
|
598
|
+
repository.known.has(path) || repository.beforeKnown.has(path)
|
|
599
|
+
? path
|
|
600
|
+
: repositoryPath(repository.root, cwd, path),
|
|
601
|
+
])
|
|
602
|
+
.sort(([a], [b]) => a.localeCompare(b))),
|
|
603
|
+
};
|
|
604
|
+
// Canonical repository paths are the sole serialized content keys.
|
|
255
605
|
const sources = Object.entries(canonical).map(([path, text]) => ({ path, text }));
|
|
256
|
-
const before = {};
|
|
606
|
+
const before = { ...historical };
|
|
257
607
|
if (args.base) {
|
|
258
608
|
for (const path of Object.keys(canonical)) {
|
|
259
609
|
const old = await repository.readBefore(path);
|
|
260
610
|
if (!old.ok)
|
|
261
|
-
return textResult(old.error);
|
|
611
|
+
return textResult(old.error, old.cause ?? "file_unavailable");
|
|
262
612
|
before[path] = old.text;
|
|
263
613
|
}
|
|
264
614
|
state.state.base_sha = repository.baseSha ?? "";
|
|
265
|
-
state.state.files_before =
|
|
266
|
-
|
|
267
|
-
return [path, before[canonicalPath] ?? null];
|
|
268
|
-
}));
|
|
269
|
-
const stateSize = JSON.stringify(state.state).length;
|
|
615
|
+
state.state.files_before = before;
|
|
616
|
+
const stateSize = JSON.stringify(command?.ok ? { ...state.state, output: undefined } : state.state).length;
|
|
270
617
|
if (stateSize > STATE_MAX_CHARS) {
|
|
271
618
|
const partSizes = Object.entries(state.state)
|
|
272
619
|
.map(([part, value]) => `${part} ${JSON.stringify(value).length}`)
|
|
273
620
|
.join(", ");
|
|
274
|
-
return textResult(`serialized state including files_before exceeds ${STATE_MAX_CHARS} chars (${stateSize}); serialized parts: ${partSizes}; split the situation into smaller states
|
|
621
|
+
return textResult(`serialized state including files_before exceeds ${STATE_MAX_CHARS} chars (${stateSize}); serialized parts: ${partSizes}; split the situation into smaller states.`, "evidence_too_large");
|
|
275
622
|
}
|
|
276
623
|
}
|
|
277
624
|
const configuration = [];
|
|
@@ -360,14 +707,19 @@ export function createAskTool(dependencies) {
|
|
|
360
707
|
sources,
|
|
361
708
|
...(args.base ? { before } : {}),
|
|
362
709
|
};
|
|
710
|
+
const closureState = { ...state.state };
|
|
711
|
+
if (command?.ok)
|
|
712
|
+
delete closureState.output;
|
|
363
713
|
const closure = buildAskClosure({
|
|
364
714
|
graph: { edges, limits },
|
|
365
715
|
syntax,
|
|
366
716
|
paths: Object.keys(canonical),
|
|
367
717
|
intentions: JSON.stringify(asks.asks),
|
|
368
|
-
state:
|
|
718
|
+
state: closureState,
|
|
369
719
|
});
|
|
370
720
|
state.state = closure.state;
|
|
721
|
+
if (command?.ok)
|
|
722
|
+
state.state.output = { ...command.output };
|
|
371
723
|
proofLimitations.push(...closure.limitations.filter((limit) => !limit.fact.startsWith("Import closure only,")));
|
|
372
724
|
proofLimitations.push({
|
|
373
725
|
fact: `closure: +${closure.added.length}${closure.added.length ? ` (${closure.added.map((piece) => `${piece.path} ${piece.name}, ${piece.chars} chars`).join("; ")})` : ""}; imports only, depth 1; files with no import link are not checked`,
|
|
@@ -379,14 +731,27 @@ export function createAskTool(dependencies) {
|
|
|
379
731
|
fact: "closure: +0; imports only, depth 1; repository inventory unavailable",
|
|
380
732
|
next: "pass dependencies explicitly in paths",
|
|
381
733
|
});
|
|
734
|
+
if (!repository.ok)
|
|
735
|
+
state.state.evidence = {
|
|
736
|
+
context: evidenceContext,
|
|
737
|
+
aliases: references.aliases,
|
|
738
|
+
};
|
|
739
|
+
// Command output has its own passage-selection budget below. Only immutable
|
|
740
|
+
// file/note evidence can refuse here, before selection has had a chance.
|
|
741
|
+
const serializedSize = JSON.stringify(command?.ok ? { ...state.state, output: undefined } : state.state).length;
|
|
742
|
+
if (serializedSize > STATE_MAX_CHARS)
|
|
743
|
+
return textResult(`serialized state exceeds ${STATE_MAX_CHARS} chars (${serializedSize}); files and versions are never truncated`, "evidence_too_large");
|
|
744
|
+
evidenceFacts.omissions = unresolved;
|
|
382
745
|
const blocked = blockedAskReadings(asks, state.state, unresolved, args.state);
|
|
383
746
|
const inserted = isRecord(state.state.files) ? state.state.files : {};
|
|
384
747
|
const identities = files.identities.map((file) => {
|
|
385
|
-
const
|
|
386
|
-
|
|
748
|
+
const insertedPath = repository.ok
|
|
749
|
+
? repositoryPath(repository.root, cwd, file.requestedPath)
|
|
750
|
+
: file.requestedPath;
|
|
751
|
+
const content = inserted[insertedPath];
|
|
387
752
|
return {
|
|
388
753
|
...file,
|
|
389
|
-
insertedPath:
|
|
754
|
+
insertedPath: resolve(repository.ok ? repository.root : cwd, insertedPath),
|
|
390
755
|
content: typeof content === "string" ? content : "",
|
|
391
756
|
insertedSha256: createHash("sha256")
|
|
392
757
|
.update(typeof content === "string" ? content : "")
|
|
@@ -396,15 +761,32 @@ export function createAskTool(dependencies) {
|
|
|
396
761
|
const blockedMembers = new Set(asks.groups
|
|
397
762
|
.filter((group) => group.some((id) => blocked.has(id)))
|
|
398
763
|
.flat());
|
|
764
|
+
evidenceFacts.blockedGroups = asks.groups
|
|
765
|
+
.filter((group) => group.some((id) => blocked.has(id)))
|
|
766
|
+
.map((ids) => {
|
|
767
|
+
const reason = ids
|
|
768
|
+
.map((id) => blocked.get(id))
|
|
769
|
+
.find((reason) => reason !== undefined) ??
|
|
770
|
+
"Required evidence unavailable";
|
|
771
|
+
// A secret refusal stays named as such instead of missing_required.
|
|
772
|
+
const secret = unresolved.some((item) => item.reason === reason && item.cause === "secret_pattern");
|
|
773
|
+
return {
|
|
774
|
+
ids,
|
|
775
|
+
reason,
|
|
776
|
+
...(secret ? { cause: "secret_pattern" } : {}),
|
|
777
|
+
};
|
|
778
|
+
});
|
|
399
779
|
const questions = Object.fromEntries(Object.entries(asks.questions).filter(([id]) => !blockedMembers.has(id)));
|
|
400
780
|
const integrity = checkIntegrity(state.state, asks.asks, identities.filter((identity) => identity.content.trim().length > 0));
|
|
401
781
|
let sent = 0;
|
|
402
782
|
let refusal;
|
|
403
|
-
const findResults = [];
|
|
404
783
|
const options = {
|
|
405
784
|
signal,
|
|
406
785
|
cache: !args.command,
|
|
407
786
|
...runtime.session.requestGate(),
|
|
787
|
+
admissionCause: () => refusal?.kind === "session"
|
|
788
|
+
? "session_budget"
|
|
789
|
+
: "call_budget",
|
|
408
790
|
beforeRequest: (questionCount) => {
|
|
409
791
|
if (args.max_calls !== undefined && sent >= args.max_calls) {
|
|
410
792
|
refusal = {
|
|
@@ -434,9 +816,12 @@ export function createAskTool(dependencies) {
|
|
|
434
816
|
if (command.compressedChars >
|
|
435
817
|
OUTPUT_CHUNK_CHARS * OUTPUT_FIND_MAX_CALLS ||
|
|
436
818
|
chunks.length > OUTPUT_FIND_MAX_CALLS)
|
|
437
|
-
return textResult(`output requires more than ${OUTPUT_FIND_MAX_CALLS} find calls: ${command.originalBytes} bytes before, ${command.compressedChars} chars after compression; narrow command (remove verbosity or filter)
|
|
438
|
-
const
|
|
439
|
-
|
|
819
|
+
return textResult(`output requires more than ${OUTPUT_FIND_MAX_CALLS} find calls: ${command.originalBytes} bytes before, ${command.compressedChars} chars after compression; narrow command (remove verbosity or filter)`, "evidence_too_large");
|
|
820
|
+
const chunkStates = chunks.map((chunk) => withEvidenceContext({ output: chunk.text }, evidenceContext));
|
|
821
|
+
if (chunkStates.some((chunkState) => JSON.stringify(chunkState).length > STATE_MAX_CHARS))
|
|
822
|
+
return textResult(`serialized output chunk including evidence context exceeds ${STATE_MAX_CHARS} chars; narrow command`, "evidence_too_large");
|
|
823
|
+
const found = await Promise.all(chunkStates.map(async (chunkState) => {
|
|
824
|
+
const judgment = await client.judge(chunkState, {
|
|
440
825
|
find: {
|
|
441
826
|
type: "bool",
|
|
442
827
|
instructions: "Does this excerpt of a command's output show something failing (a failed test or assertion, a compiler error, a crash, an error message) with its details?",
|
|
@@ -458,7 +843,7 @@ export function createAskTool(dependencies) {
|
|
|
458
843
|
}).length;
|
|
459
844
|
const budget = STATE_MAX_CHARS - emptySize - 16;
|
|
460
845
|
if (budget < 100)
|
|
461
|
-
return textResult("No space for command output; reduce paths or state.");
|
|
846
|
+
return textResult("No space for command output; reduce paths or state.", "evidence_too_large");
|
|
462
847
|
const stderrSize = JSON.stringify(output.stderr).length;
|
|
463
848
|
const stdoutSize = JSON.stringify(output.stdout).length;
|
|
464
849
|
const stdoutBudget = !output.stderr
|
|
@@ -496,7 +881,7 @@ export function createAskTool(dependencies) {
|
|
|
496
881
|
integrity.controls.push(...checked.controls);
|
|
497
882
|
integrity.notes.push(...checked.notes);
|
|
498
883
|
if (JSON.stringify(state.state).length > STATE_MAX_CHARS)
|
|
499
|
-
return textResult("serialized command output exceeds state budget; narrow command or reduce paths");
|
|
884
|
+
return textResult("serialized command output exceeds state budget; narrow command or reduce paths", "evidence_too_large");
|
|
500
885
|
if (outputNotIdentifiable)
|
|
501
886
|
integrity.controls.push({
|
|
502
887
|
fact: "not identifiable: the output shows an assertion failure and the failing test is absent; a wrong test and a bug look alike",
|
|
@@ -542,11 +927,12 @@ export function createAskTool(dependencies) {
|
|
|
542
927
|
questions: (first.questions ?? 0) + (reversed.questions ?? 0),
|
|
543
928
|
cacheHits: (first.cacheHits ?? 0) + (reversed.cacheHits ?? 0),
|
|
544
929
|
cacheRequests: (first.cacheRequests ?? 0) + (reversed.cacheRequests ?? 0),
|
|
545
|
-
usage:
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
930
|
+
usage: first.usage && reversed.usage
|
|
931
|
+
? {
|
|
932
|
+
inputTokens: first.usage.inputTokens + reversed.usage.inputTokens,
|
|
933
|
+
costUsd: first.usage.costUsd + reversed.usage.costUsd,
|
|
934
|
+
}
|
|
935
|
+
: undefined,
|
|
550
936
|
};
|
|
551
937
|
}
|
|
552
938
|
}
|
|
@@ -555,11 +941,14 @@ export function createAskTool(dependencies) {
|
|
|
555
941
|
...result,
|
|
556
942
|
calls: (result.calls ?? 0) + (found.calls ?? 0),
|
|
557
943
|
questions: (result.questions ?? 0) + (found.questions ?? 0),
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
944
|
+
cacheHits: (result.cacheHits ?? 0) + (found.cacheHits ?? 0),
|
|
945
|
+
cacheRequests: (result.cacheRequests ?? 0) + (found.cacheRequests ?? 0),
|
|
946
|
+
usage: result.usage && found.usage
|
|
947
|
+
? {
|
|
948
|
+
inputTokens: result.usage.inputTokens + found.usage.inputTokens,
|
|
949
|
+
costUsd: result.usage.costUsd + found.usage.costUsd,
|
|
950
|
+
}
|
|
951
|
+
: undefined,
|
|
563
952
|
};
|
|
564
953
|
const attribution = [];
|
|
565
954
|
if (result.ok) {
|
|
@@ -572,6 +961,8 @@ export function createAskTool(dependencies) {
|
|
|
572
961
|
fact: `attribution ${reading.label}: no designated candidate`,
|
|
573
962
|
next: "provide decisive candidate evidence before acting",
|
|
574
963
|
});
|
|
964
|
+
attributionMissing.add(reading.id);
|
|
965
|
+
auxiliary.controls.notJudged++;
|
|
575
966
|
continue;
|
|
576
967
|
}
|
|
577
968
|
if (!Object.hasOwn(inserted, selected)) {
|
|
@@ -579,6 +970,8 @@ export function createAskTool(dependencies) {
|
|
|
579
970
|
fact: `attribution ${reading.label}: designated piece ${selected} is absent`,
|
|
580
971
|
next: `add ${selected} to paths before acting`,
|
|
581
972
|
});
|
|
973
|
+
attributionMissing.add(reading.id);
|
|
974
|
+
auxiliary.controls.notJudged++;
|
|
582
975
|
continue;
|
|
583
976
|
}
|
|
584
977
|
const selectedPath = repository?.ok
|
|
@@ -609,6 +1002,38 @@ export function createAskTool(dependencies) {
|
|
|
609
1002
|
continue;
|
|
610
1003
|
const control = await client.judge(ablated, { [reading.id]: question }, options);
|
|
611
1004
|
const answer = control.ok ? control.answers[reading.id] : undefined;
|
|
1005
|
+
if (answer && answer.type !== "unjudged" && answer.source)
|
|
1006
|
+
attributionAnswers[reading.id] = [
|
|
1007
|
+
{
|
|
1008
|
+
id: `${reading.id}:attribution`,
|
|
1009
|
+
kind: "attribution",
|
|
1010
|
+
source: answer.source,
|
|
1011
|
+
outcome: answer.type === "choice"
|
|
1012
|
+
? answer.choice === selected
|
|
1013
|
+
? "does not rest on it"
|
|
1014
|
+
: "attributed"
|
|
1015
|
+
: "invalid attribution response",
|
|
1016
|
+
rawValues: answer.type === "bool"
|
|
1017
|
+
? [
|
|
1018
|
+
{
|
|
1019
|
+
label: "attribution",
|
|
1020
|
+
value: answer.p >= 0.5,
|
|
1021
|
+
probability: known(answer.p),
|
|
1022
|
+
},
|
|
1023
|
+
]
|
|
1024
|
+
: Object.entries(answer.probabilities).map(([label, value]) => ({
|
|
1025
|
+
label,
|
|
1026
|
+
value: label,
|
|
1027
|
+
probability: known(value),
|
|
1028
|
+
})),
|
|
1029
|
+
},
|
|
1030
|
+
];
|
|
1031
|
+
if (answer?.type !== "choice")
|
|
1032
|
+
attributionMissing.add(reading.id);
|
|
1033
|
+
if (answer && answer.type !== "unjudged" && answer.source)
|
|
1034
|
+
auxiliary.controls[answer.source]++;
|
|
1035
|
+
else
|
|
1036
|
+
auxiliary.controls.notJudged++;
|
|
612
1037
|
attribution.push(answer?.type === "choice"
|
|
613
1038
|
? {
|
|
614
1039
|
fact: `attribution ${reading.label}: ${answer.choice === selected ? "does not rest on it" : "attributed"} (${selected})`,
|
|
@@ -624,11 +1049,12 @@ export function createAskTool(dependencies) {
|
|
|
624
1049
|
questions: (result.questions ?? 0) + (control.questions ?? 0),
|
|
625
1050
|
cacheHits: (result.cacheHits ?? 0) + (control.cacheHits ?? 0),
|
|
626
1051
|
cacheRequests: (result.cacheRequests ?? 0) + (control.cacheRequests ?? 0),
|
|
627
|
-
usage:
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
1052
|
+
usage: result.usage && control.usage
|
|
1053
|
+
? {
|
|
1054
|
+
inputTokens: result.usage.inputTokens + control.usage.inputTokens,
|
|
1055
|
+
costUsd: result.usage.costUsd + control.usage.costUsd,
|
|
1056
|
+
}
|
|
1057
|
+
: undefined,
|
|
632
1058
|
};
|
|
633
1059
|
}
|
|
634
1060
|
}
|
|
@@ -641,6 +1067,33 @@ export function createAskTool(dependencies) {
|
|
|
641
1067
|
].some((id) => !result.answers[id] || result.answers[id]?.type === "unjudged") ||
|
|
642
1068
|
(reversed?.ok && reversed.answers[reading.id]?.type === "unjudged"))
|
|
643
1069
|
.map((reading) => reading.label);
|
|
1070
|
+
for (const reading of asks.readings) {
|
|
1071
|
+
for (const id of [
|
|
1072
|
+
reading.twin,
|
|
1073
|
+
...Object.values(reading.controls ?? {}),
|
|
1074
|
+
].filter((id) => !!id)) {
|
|
1075
|
+
const answer = result.ok ? result.answers[id] : undefined;
|
|
1076
|
+
if (answer && answer.type !== "unjudged" && answer.source)
|
|
1077
|
+
auxiliary.controls[answer.source]++;
|
|
1078
|
+
else
|
|
1079
|
+
auxiliary.controls.notJudged++;
|
|
1080
|
+
}
|
|
1081
|
+
if (reversed &&
|
|
1082
|
+
Object.hasOwn(reversed.ok ? reversed.answers : {}, reading.id)) {
|
|
1083
|
+
const answer = reversed.ok ? reversed.answers[reading.id] : undefined;
|
|
1084
|
+
if (answer && answer.type !== "unjudged" && answer.source)
|
|
1085
|
+
auxiliary.controls[answer.source]++;
|
|
1086
|
+
else
|
|
1087
|
+
auxiliary.controls.notJudged++;
|
|
1088
|
+
}
|
|
1089
|
+
}
|
|
1090
|
+
for (const found of findResults) {
|
|
1091
|
+
const answer = found.ok ? found.answers.find : undefined;
|
|
1092
|
+
if (answer && answer.type !== "unjudged" && answer.source)
|
|
1093
|
+
auxiliary.passages[answer.source]++;
|
|
1094
|
+
else
|
|
1095
|
+
auxiliary.passages.notJudged++;
|
|
1096
|
+
}
|
|
644
1097
|
const output = finish(result, {
|
|
645
1098
|
...(result.ok
|
|
646
1099
|
? {
|
|
@@ -666,6 +1119,14 @@ export function createAskTool(dependencies) {
|
|
|
666
1119
|
},
|
|
667
1120
|
]
|
|
668
1121
|
: []),
|
|
1122
|
+
...(command?.ok && command.redactions
|
|
1123
|
+
? [
|
|
1124
|
+
{
|
|
1125
|
+
fact: `output: ${command.redactions} occurrence${command.redactions === 1 ? "" : "s"} of the configured Jev API key replaced with [redacted]`,
|
|
1126
|
+
next: "remove the key from the command's output; it is never sent to Jev",
|
|
1127
|
+
},
|
|
1128
|
+
]
|
|
1129
|
+
: []),
|
|
669
1130
|
...(command?.ok && command.shapeLimitExceeded
|
|
670
1131
|
? [
|
|
671
1132
|
{
|