@inflexa-ai/harness 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/synthesize-run.d.ts +3 -1
- package/dist/app/synthesize-run.js +14 -3
- package/dist/contracts/chat-parts.d.ts +0 -2
- package/dist/contracts/schemas/chat-parts.d.ts +3 -5
- package/dist/contracts/schemas/chat-parts.js +0 -1
- package/dist/execution/run-synthesis.d.ts +7 -1
- package/dist/execution/run-synthesis.js +181 -39
- package/dist/lib/zod-issues.d.ts +80 -0
- package/dist/lib/zod-issues.js +227 -0
- package/dist/loop/run-agent.js +18 -6
- package/dist/prompts/conversation.d.ts +1 -1
- package/dist/prompts/conversation.js +12 -5
- package/dist/prompts/synthesis-agent.d.ts +1 -1
- package/dist/prompts/synthesis-agent.js +39 -8
- package/dist/schemas/run-synthesis.d.ts +8 -2
- package/dist/schemas/run-synthesis.js +6 -5
- package/dist/state/index.d.ts +2 -2
- package/dist/state/index.js +2 -2
- package/dist/state/init.js +6 -0
- package/dist/state/runs.d.ts +7 -1
- package/dist/state/runs.js +21 -4
- package/dist/state/schema.d.ts +14 -0
- package/dist/state/schema.js +9 -0
- package/dist/tools/iterate-report.js +2 -0
- package/dist/tools/research/generate-plan.js +4 -2
- package/dist/tools/research/inspect-run.d.ts +4 -2
- package/dist/tools/research/inspect-run.js +12 -4
- package/dist/workflows/execute-analysis.d.ts +3 -0
- package/dist/workflows/execute-analysis.js +24 -1
- package/package.json +1 -1
|
@@ -22,6 +22,7 @@ import type { ChatProvider, EmbeddingProvider } from "../providers/types.js";
|
|
|
22
22
|
import type { BioToolKeys } from "../tools/bio/keys.js";
|
|
23
23
|
import type { RunFinding } from "../workflows/execute-analysis.js";
|
|
24
24
|
import type { ResolveWorkspaceRoot } from "../workspace/paths.js";
|
|
25
|
+
import type { SynthesisStatus } from "../state/index.js";
|
|
25
26
|
import type { Logger } from "../lib/logger.js";
|
|
26
27
|
export interface SynthesizeRunDeps {
|
|
27
28
|
/** Operational logging seam; omitted falls back to no-op. */
|
|
@@ -40,7 +41,6 @@ export interface SynthesizeRunDeps {
|
|
|
40
41
|
/** Optional fields for a progress update, mirroring the wire part. */
|
|
41
42
|
export interface SynthesisProgressExtra {
|
|
42
43
|
readonly delegationCount?: number;
|
|
43
|
-
readonly validationAttempts?: number;
|
|
44
44
|
readonly reason?: string;
|
|
45
45
|
readonly error?: string;
|
|
46
46
|
}
|
|
@@ -56,5 +56,7 @@ export interface SynthesizeRunParams {
|
|
|
56
56
|
}
|
|
57
57
|
export interface SynthesizeRunResult {
|
|
58
58
|
readonly findings: readonly RunFinding[];
|
|
59
|
+
readonly synthesisStatus: Exclude<SynthesisStatus, "failed">;
|
|
60
|
+
readonly synthesisReason: string | null;
|
|
59
61
|
}
|
|
60
62
|
export declare function synthesizeRun(deps: SynthesizeRunDeps, params: SynthesizeRunParams): Promise<SynthesizeRunResult>;
|
|
@@ -33,10 +33,12 @@ export async function synthesizeRun(deps, params) {
|
|
|
33
33
|
completedSteps,
|
|
34
34
|
});
|
|
35
35
|
if (summaries.length === 0) {
|
|
36
|
+
const logger = (deps.logger ?? createNoopLogger()).named("synthesize-run");
|
|
37
|
+
logger.warn("synthesis skipped — no step summaries on disk", { runId, reason: "no-summaries" });
|
|
36
38
|
await onProgress("skipped", "No step summaries available for synthesis", {
|
|
37
39
|
reason: "no-summaries",
|
|
38
40
|
});
|
|
39
|
-
return { findings: [] };
|
|
41
|
+
return { findings: [], synthesisStatus: "skipped_no_summaries", synthesisReason: "no-summaries" };
|
|
40
42
|
}
|
|
41
43
|
const planNarrative = await loadPlanNarrative(pool, runId, analysisId);
|
|
42
44
|
const synthesizerSession = forSubAgent(session, "run-synthesizer");
|
|
@@ -51,8 +53,15 @@ export async function synthesizeRun(deps, params) {
|
|
|
51
53
|
emit,
|
|
52
54
|
});
|
|
53
55
|
if (result.kind === "skipped") {
|
|
54
|
-
|
|
55
|
-
|
|
56
|
+
const logger = (deps.logger ?? createNoopLogger()).named("synthesize-run");
|
|
57
|
+
logger.warn("synthesis skipped — synthesizer reported a blocker", {
|
|
58
|
+
runId,
|
|
59
|
+
reason: result.reason,
|
|
60
|
+
});
|
|
61
|
+
await onProgress("skipped", "Synthesis skipped — run did not produce synthesizable content", {
|
|
62
|
+
reason: result.reason,
|
|
63
|
+
});
|
|
64
|
+
return { findings: [], synthesisStatus: "skipped_blocker", synthesisReason: result.reason };
|
|
56
65
|
}
|
|
57
66
|
const synthesis = result.synthesis;
|
|
58
67
|
await onProgress("indexing", "Indexing synthesis for search");
|
|
@@ -84,6 +93,8 @@ export async function synthesizeRun(deps, params) {
|
|
|
84
93
|
title: f.title,
|
|
85
94
|
confidence: f.confidence,
|
|
86
95
|
})),
|
|
96
|
+
synthesisStatus: "produced",
|
|
97
|
+
synthesisReason: null,
|
|
87
98
|
};
|
|
88
99
|
}
|
|
89
100
|
catch (err) {
|
|
@@ -262,8 +262,6 @@ export interface SynthesisProgressPart {
|
|
|
262
262
|
activity: string;
|
|
263
263
|
/** Number of literature-reviewer delegations started so far. */
|
|
264
264
|
delegationCount?: number;
|
|
265
|
-
/** Number of times the synthesizer's submit_synthesis call was rejected. */
|
|
266
|
-
validationAttempts?: number;
|
|
267
265
|
/** Agent-provided reason — set when phase === "skipped". */
|
|
268
266
|
reason?: string;
|
|
269
267
|
/** Error message — set when phase === "failed". */
|
|
@@ -298,11 +298,11 @@ export declare const SynthesizedFindingSchema: z.ZodObject<{
|
|
|
298
298
|
low: "low";
|
|
299
299
|
}>;
|
|
300
300
|
noveltyStatus: z.ZodEnum<{
|
|
301
|
+
expected: "expected";
|
|
301
302
|
novel: "novel";
|
|
302
303
|
confirmed: "confirmed";
|
|
303
304
|
partially_confirmed: "partially_confirmed";
|
|
304
305
|
contradicted: "contradicted";
|
|
305
|
-
expected: "expected";
|
|
306
306
|
}>;
|
|
307
307
|
literatureInterpretation: z.ZodString;
|
|
308
308
|
}, z.core.$strip>;
|
|
@@ -330,11 +330,11 @@ export declare const RunSynthesisPartSchema: z.ZodObject<{
|
|
|
330
330
|
low: "low";
|
|
331
331
|
}>;
|
|
332
332
|
noveltyStatus: z.ZodEnum<{
|
|
333
|
+
expected: "expected";
|
|
333
334
|
novel: "novel";
|
|
334
335
|
confirmed: "confirmed";
|
|
335
336
|
partially_confirmed: "partially_confirmed";
|
|
336
337
|
contradicted: "contradicted";
|
|
337
|
-
expected: "expected";
|
|
338
338
|
}>;
|
|
339
339
|
literatureInterpretation: z.ZodString;
|
|
340
340
|
}, z.core.$strip>>;
|
|
@@ -383,7 +383,6 @@ export declare const SynthesisProgressPartSchema: z.ZodObject<{
|
|
|
383
383
|
}>;
|
|
384
384
|
activity: z.ZodString;
|
|
385
385
|
delegationCount: z.ZodOptional<z.ZodNumber>;
|
|
386
|
-
validationAttempts: z.ZodOptional<z.ZodNumber>;
|
|
387
386
|
reason: z.ZodOptional<z.ZodString>;
|
|
388
387
|
error: z.ZodOptional<z.ZodString>;
|
|
389
388
|
}, z.core.$strip>;
|
|
@@ -589,11 +588,11 @@ export declare const CortexChatPartSchema: z.ZodDiscriminatedUnion<[z.ZodObject<
|
|
|
589
588
|
low: "low";
|
|
590
589
|
}>;
|
|
591
590
|
noveltyStatus: z.ZodEnum<{
|
|
591
|
+
expected: "expected";
|
|
592
592
|
novel: "novel";
|
|
593
593
|
confirmed: "confirmed";
|
|
594
594
|
partially_confirmed: "partially_confirmed";
|
|
595
595
|
contradicted: "contradicted";
|
|
596
|
-
expected: "expected";
|
|
597
596
|
}>;
|
|
598
597
|
literatureInterpretation: z.ZodString;
|
|
599
598
|
}, z.core.$strip>>;
|
|
@@ -629,7 +628,6 @@ export declare const CortexChatPartSchema: z.ZodDiscriminatedUnion<[z.ZodObject<
|
|
|
629
628
|
}>;
|
|
630
629
|
activity: z.ZodString;
|
|
631
630
|
delegationCount: z.ZodOptional<z.ZodNumber>;
|
|
632
|
-
validationAttempts: z.ZodOptional<z.ZodNumber>;
|
|
633
631
|
reason: z.ZodOptional<z.ZodString>;
|
|
634
632
|
error: z.ZodOptional<z.ZodString>;
|
|
635
633
|
}, z.core.$strip>, z.ZodObject<{
|
|
@@ -243,7 +243,6 @@ export const SynthesisProgressPartSchema = z.object({
|
|
|
243
243
|
phase: SynthesisPhaseSchema,
|
|
244
244
|
activity: z.string(),
|
|
245
245
|
delegationCount: z.number().int().nonnegative().optional(),
|
|
246
|
-
validationAttempts: z.number().int().nonnegative().optional(),
|
|
247
246
|
reason: z.string().optional(),
|
|
248
247
|
error: z.string().optional(),
|
|
249
248
|
});
|
|
@@ -1,9 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Run synthesizer — a single-shot agent loop driven through `runAgent`:
|
|
3
3
|
*
|
|
4
|
+
* - `validate_synthesis` — non-terminal: dry-run schema + semantic validation
|
|
5
|
+
* of a candidate in any shape. Never records.
|
|
4
6
|
* - `submit_synthesis` — terminal: schema + semantic validation, captures
|
|
5
|
-
* the outcome. Re-callable
|
|
7
|
+
* the outcome. Re-callable; a later valid submission
|
|
8
|
+
* supersedes an earlier outcome (last-valid-wins).
|
|
6
9
|
* - `report_blocker` — terminal: the run produced nothing synthesizable.
|
|
10
|
+
* Only records when no outcome exists yet, so a
|
|
11
|
+
* blocker can never clobber a real synthesis.
|
|
7
12
|
* - `literature_reviewer` — research sub-agent exposed as a tool. The tool
|
|
8
13
|
* re-uses `createLiteratureReviewerTool`; the
|
|
9
14
|
* sub-agent loop's brief is the only context handed
|
|
@@ -43,6 +48,7 @@ export declare function __buildInnerToolsForTest(args: {
|
|
|
43
48
|
knownStepIds: ReadonlySet<string>;
|
|
44
49
|
runId: string;
|
|
45
50
|
}): {
|
|
51
|
+
validate: Tool;
|
|
46
52
|
submit: Tool;
|
|
47
53
|
blocker: Tool;
|
|
48
54
|
holder: OutcomeHolder;
|
|
@@ -1,9 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Run synthesizer — a single-shot agent loop driven through `runAgent`:
|
|
3
3
|
*
|
|
4
|
+
* - `validate_synthesis` — non-terminal: dry-run schema + semantic validation
|
|
5
|
+
* of a candidate in any shape. Never records.
|
|
4
6
|
* - `submit_synthesis` — terminal: schema + semantic validation, captures
|
|
5
|
-
* the outcome. Re-callable
|
|
7
|
+
* the outcome. Re-callable; a later valid submission
|
|
8
|
+
* supersedes an earlier outcome (last-valid-wins).
|
|
6
9
|
* - `report_blocker` — terminal: the run produced nothing synthesizable.
|
|
10
|
+
* Only records when no outcome exists yet, so a
|
|
11
|
+
* blocker can never clobber a real synthesis.
|
|
7
12
|
* - `literature_reviewer` — research sub-agent exposed as a tool. The tool
|
|
8
13
|
* re-uses `createLiteratureReviewerTool`; the
|
|
9
14
|
* sub-agent loop's brief is the only context handed
|
|
@@ -23,7 +28,8 @@ import { join } from "node:path";
|
|
|
23
28
|
import { ok } from "neverthrow";
|
|
24
29
|
import { z } from "zod";
|
|
25
30
|
import { writeFileWithinRoot } from "../lib/fs-helpers.js";
|
|
26
|
-
import {
|
|
31
|
+
import { hintForZodIssue } from "../lib/zod-issues.js";
|
|
32
|
+
import { BiologicalThemeSchema, KeyReferenceSchema, RunSynthesisSchema, SynthesizedFindingSchema, } from "../schemas/run-synthesis.js";
|
|
27
33
|
import { synthesisAgentPrompt } from "../prompts/synthesis-agent.js";
|
|
28
34
|
import { composeSystemPrompt } from "../agents/system-prompt.js";
|
|
29
35
|
import { runDir } from "../workspace/paths.js";
|
|
@@ -40,23 +46,37 @@ const AGENT_ID = "run-synthesizer";
|
|
|
40
46
|
*/
|
|
41
47
|
const SYNTHESIZER_MAX_ITERATIONS = 25;
|
|
42
48
|
// ── Validation ──────────────────────────────────────────────────────
|
|
43
|
-
function zodIssuesToValidationIssues(error, rootPath = "synthesis") {
|
|
49
|
+
function zodIssuesToValidationIssues(error, input, rootPath = "synthesis") {
|
|
44
50
|
return error.issues.map((i) => ({
|
|
45
51
|
path: [rootPath, ...i.path.map((p) => String(p))].join("."),
|
|
46
52
|
code: "schema",
|
|
47
53
|
message: i.message,
|
|
54
|
+
hint: hintForZodIssue(i, input),
|
|
48
55
|
}));
|
|
49
56
|
}
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
57
|
+
const PMID_PATTERN = /^\d+$/;
|
|
58
|
+
// ── Semantic checks ─────────────────────────────────────────────────
|
|
59
|
+
//
|
|
60
|
+
// Each check takes only the fields it reads, never the whole synthesis. That is
|
|
61
|
+
// what lets a candidate which failed whole-payload schema validation still be
|
|
62
|
+
// checked: whichever fields parsed on their own get their checks run, and only
|
|
63
|
+
// the checks whose inputs are genuinely missing are deferred. `semanticCheck`
|
|
64
|
+
// composes them for the schema-valid path — it is the one definition of "all
|
|
65
|
+
// semantic checks, in report order".
|
|
66
|
+
function checkRunId(runId, ctx) {
|
|
67
|
+
if (runId === ctx.runId)
|
|
68
|
+
return [];
|
|
69
|
+
return [
|
|
70
|
+
{
|
|
54
71
|
path: "synthesis.runId",
|
|
55
72
|
code: "semantic",
|
|
56
73
|
message: `runId must be "${ctx.runId}"`,
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
|
|
74
|
+
},
|
|
75
|
+
];
|
|
76
|
+
}
|
|
77
|
+
function checkFindingStepIds(findings, ctx) {
|
|
78
|
+
const issues = [];
|
|
79
|
+
for (const [i, f] of findings.entries()) {
|
|
60
80
|
if (!ctx.knownStepIds.has(f.stepId)) {
|
|
61
81
|
issues.push({
|
|
62
82
|
path: `synthesis.findings[${i}].stepId`,
|
|
@@ -66,8 +86,12 @@ function semanticCheck(synthesis, ctx) {
|
|
|
66
86
|
});
|
|
67
87
|
}
|
|
68
88
|
}
|
|
69
|
-
|
|
70
|
-
|
|
89
|
+
return issues;
|
|
90
|
+
}
|
|
91
|
+
function checkThemeReferences(themes, findings) {
|
|
92
|
+
const issues = [];
|
|
93
|
+
const findingKeys = new Set(findings.map((f) => `${f.stepId}::${f.title}`));
|
|
94
|
+
for (const [ti, t] of themes.entries()) {
|
|
71
95
|
for (const [fi, ref] of t.findings.entries()) {
|
|
72
96
|
if (!findingKeys.has(`${ref.stepId}::${ref.title}`)) {
|
|
73
97
|
issues.push({
|
|
@@ -78,8 +102,12 @@ function semanticCheck(synthesis, ctx) {
|
|
|
78
102
|
}
|
|
79
103
|
}
|
|
80
104
|
}
|
|
81
|
-
|
|
82
|
-
|
|
105
|
+
return issues;
|
|
106
|
+
}
|
|
107
|
+
function checkKeyReferencesCited(keyReferences, findings) {
|
|
108
|
+
const issues = [];
|
|
109
|
+
const citedPmids = new Set(findings.flatMap((f) => f.references.map((r) => r.pmid)));
|
|
110
|
+
for (const [i, r] of keyReferences.entries()) {
|
|
83
111
|
if (!citedPmids.has(r.pmid)) {
|
|
84
112
|
issues.push({
|
|
85
113
|
path: `synthesis.keyReferences[${i}].pmid`,
|
|
@@ -89,10 +117,13 @@ function semanticCheck(synthesis, ctx) {
|
|
|
89
117
|
});
|
|
90
118
|
}
|
|
91
119
|
}
|
|
92
|
-
|
|
93
|
-
|
|
120
|
+
return issues;
|
|
121
|
+
}
|
|
122
|
+
function checkFindingPmidFormat(findings) {
|
|
123
|
+
const issues = [];
|
|
124
|
+
for (const [fi, f] of findings.entries()) {
|
|
94
125
|
for (const [ri, r] of f.references.entries()) {
|
|
95
|
-
if (!
|
|
126
|
+
if (!PMID_PATTERN.test(r.pmid)) {
|
|
96
127
|
issues.push({
|
|
97
128
|
path: `synthesis.findings[${fi}].references[${ri}].pmid`,
|
|
98
129
|
code: "semantic",
|
|
@@ -101,8 +132,12 @@ function semanticCheck(synthesis, ctx) {
|
|
|
101
132
|
}
|
|
102
133
|
}
|
|
103
134
|
}
|
|
104
|
-
|
|
105
|
-
|
|
135
|
+
return issues;
|
|
136
|
+
}
|
|
137
|
+
function checkKeyReferencePmidFormat(keyReferences) {
|
|
138
|
+
const issues = [];
|
|
139
|
+
for (const [i, r] of keyReferences.entries()) {
|
|
140
|
+
if (!PMID_PATTERN.test(r.pmid)) {
|
|
106
141
|
issues.push({
|
|
107
142
|
path: `synthesis.keyReferences[${i}].pmid`,
|
|
108
143
|
code: "semantic",
|
|
@@ -112,10 +147,91 @@ function semanticCheck(synthesis, ctx) {
|
|
|
112
147
|
}
|
|
113
148
|
return issues;
|
|
114
149
|
}
|
|
150
|
+
function semanticCheck(synthesis, ctx) {
|
|
151
|
+
return [
|
|
152
|
+
...checkRunId(synthesis.runId, ctx),
|
|
153
|
+
...checkFindingStepIds(synthesis.findings, ctx),
|
|
154
|
+
...checkThemeReferences(synthesis.themes, synthesis.findings),
|
|
155
|
+
...checkKeyReferencesCited(synthesis.keyReferences, synthesis.findings),
|
|
156
|
+
...checkFindingPmidFormat(synthesis.findings),
|
|
157
|
+
...checkKeyReferencePmidFormat(synthesis.keyReferences),
|
|
158
|
+
];
|
|
159
|
+
}
|
|
160
|
+
function salvageInputs(candidate) {
|
|
161
|
+
const record = typeof candidate === "object" && candidate !== null ? candidate : {};
|
|
162
|
+
const runId = z.string().safeParse(record.runId);
|
|
163
|
+
const findings = z.array(SynthesizedFindingSchema).safeParse(record.findings);
|
|
164
|
+
const themes = z.array(BiologicalThemeSchema).safeParse(record.themes);
|
|
165
|
+
const keyReferences = z.array(KeyReferenceSchema).safeParse(record.keyReferences);
|
|
166
|
+
return {
|
|
167
|
+
runId: runId.success ? runId.data : undefined,
|
|
168
|
+
findings: findings.success ? findings.data : undefined,
|
|
169
|
+
themes: themes.success ? themes.data : undefined,
|
|
170
|
+
keyReferences: keyReferences.success ? keyReferences.data : undefined,
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* A short issue list must never read as "everything else is fine". When the
|
|
175
|
+
* payload failed schema validation and some checks could not run, the deferred
|
|
176
|
+
* ones are named back to the model as an issue of their own.
|
|
177
|
+
*/
|
|
178
|
+
function deferredChecksNotice(deferred) {
|
|
179
|
+
return {
|
|
180
|
+
path: "synthesis",
|
|
181
|
+
code: "semantic",
|
|
182
|
+
message: `This issue list is INCOMPLETE: ${deferred.length} semantic check(s) could not run because the fields ` +
|
|
183
|
+
`they read did not survive schema validation — ${deferred.join("; ")}.`,
|
|
184
|
+
hint: "Fix the schema issues above and call validate_synthesis again; the deferred checks will run then and may report further problems.",
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Semantic checks over a candidate that failed `RunSynthesisSchema`. Every check
|
|
189
|
+
* whose own inputs parsed is run; the rest are reported as deferred.
|
|
190
|
+
*/
|
|
191
|
+
function salvagedSemanticCheck(candidate, ctx) {
|
|
192
|
+
const { runId, findings, themes, keyReferences } = salvageInputs(candidate);
|
|
193
|
+
const issues = [];
|
|
194
|
+
const deferred = [];
|
|
195
|
+
if (runId !== undefined)
|
|
196
|
+
issues.push(...checkRunId(runId, ctx));
|
|
197
|
+
else
|
|
198
|
+
deferred.push("runId matches the run");
|
|
199
|
+
if (findings !== undefined)
|
|
200
|
+
issues.push(...checkFindingStepIds(findings, ctx));
|
|
201
|
+
else
|
|
202
|
+
deferred.push("finding stepIds are present in the input summaries");
|
|
203
|
+
if (themes !== undefined && findings !== undefined)
|
|
204
|
+
issues.push(...checkThemeReferences(themes, findings));
|
|
205
|
+
else
|
|
206
|
+
deferred.push("theme references resolve to entries in findings[]");
|
|
207
|
+
if (keyReferences !== undefined && findings !== undefined)
|
|
208
|
+
issues.push(...checkKeyReferencesCited(keyReferences, findings));
|
|
209
|
+
else
|
|
210
|
+
deferred.push("keyReferences PMIDs are cited by some finding");
|
|
211
|
+
if (findings !== undefined)
|
|
212
|
+
issues.push(...checkFindingPmidFormat(findings));
|
|
213
|
+
else
|
|
214
|
+
deferred.push("finding reference PMIDs are numeric");
|
|
215
|
+
if (keyReferences !== undefined)
|
|
216
|
+
issues.push(...checkKeyReferencePmidFormat(keyReferences));
|
|
217
|
+
else
|
|
218
|
+
deferred.push("keyReferences PMIDs are numeric");
|
|
219
|
+
if (deferred.length > 0)
|
|
220
|
+
issues.push(deferredChecksNotice(deferred));
|
|
221
|
+
return issues;
|
|
222
|
+
}
|
|
223
|
+
/**
|
|
224
|
+
* Validate a candidate in one pass. A schema failure no longer hides the
|
|
225
|
+
* semantic problems behind it: structure issues come first, then every semantic
|
|
226
|
+
* check whose inputs survived, then an explicit note of what was deferred.
|
|
227
|
+
*/
|
|
115
228
|
function fullyValidate(candidate, ctx) {
|
|
116
229
|
const parsed = RunSynthesisSchema.safeParse(candidate);
|
|
117
230
|
if (!parsed.success) {
|
|
118
|
-
return {
|
|
231
|
+
return {
|
|
232
|
+
valid: false,
|
|
233
|
+
issues: [...zodIssuesToValidationIssues(parsed.error, candidate), ...salvagedSemanticCheck(candidate, ctx)],
|
|
234
|
+
};
|
|
119
235
|
}
|
|
120
236
|
const semanticIssues = semanticCheck(parsed.data, ctx);
|
|
121
237
|
if (semanticIssues.length > 0) {
|
|
@@ -124,6 +240,36 @@ function fullyValidate(candidate, ctx) {
|
|
|
124
240
|
return { valid: true, synthesis: parsed.data };
|
|
125
241
|
}
|
|
126
242
|
// ── Inner tools ─────────────────────────────────────────────────────
|
|
243
|
+
function buildValidateTool(ctx) {
|
|
244
|
+
return defineTool({
|
|
245
|
+
id: "validate_synthesis",
|
|
246
|
+
description: "Dry-run a candidate synthesis and get back everything that is wrong " +
|
|
247
|
+
"with it. Takes the synthesis in ANY shape: a malformed, partial, or " +
|
|
248
|
+
"wrong-typed candidate is REPORTED, not rejected. Returns " +
|
|
249
|
+
"{valid, issues[]} covering both schema problems (missing / " +
|
|
250
|
+
"wrong-typed fields) and the semantic checks a schema cannot express " +
|
|
251
|
+
"(unknown stepId refs, theme→finding refs, keyReferences PMIDs not " +
|
|
252
|
+
"cited by any finding, non-numeric PMIDs) in a SINGLE pass — a " +
|
|
253
|
+
"schema problem does not hide the semantic problems behind it. If a " +
|
|
254
|
+
"field is too malformed for a semantic check to read, that check is " +
|
|
255
|
+
"reported as deferred rather than silently skipped. The authoritative " +
|
|
256
|
+
"field-by-field synthesis schema is the arg schema of " +
|
|
257
|
+
"submit_synthesis — this tool deliberately does not restate it. " +
|
|
258
|
+
"Non-terminal: call as often as needed to iterate toward a clean " +
|
|
259
|
+
"synthesis, then submit_synthesis.",
|
|
260
|
+
// Deliberately permissive: a structurally-invalid candidate must reach
|
|
261
|
+
// `execute` so the model gets a structured {valid:false, issues} result —
|
|
262
|
+
// including semantic issues — instead of a bare Zod rejection at the loop's
|
|
263
|
+
// input boundary. `execute` re-parses against RunSynthesisSchema itself.
|
|
264
|
+
inputSchema: z.object({
|
|
265
|
+
synthesis: z.unknown().describe("The candidate synthesis, in any shape. Field-by-field schema: see submit_synthesis."),
|
|
266
|
+
}),
|
|
267
|
+
execute: async (input) => {
|
|
268
|
+
const result = fullyValidate(input.synthesis, ctx);
|
|
269
|
+
return ok(result.valid ? { valid: true, issues: [] } : { valid: false, issues: result.issues });
|
|
270
|
+
},
|
|
271
|
+
});
|
|
272
|
+
}
|
|
127
273
|
function buildSubmitTool(holder, ctx) {
|
|
128
274
|
return defineTool({
|
|
129
275
|
id: "submit_synthesis",
|
|
@@ -133,23 +279,16 @@ function buildSubmitTool(holder, ctx) {
|
|
|
133
279
|
"success returns {accepted: true} — STOP after this. On rejection " +
|
|
134
280
|
"returns {accepted: false, issues} — fix the specific fields at each " +
|
|
135
281
|
"issue path and call again, or switch to report_blocker if the " +
|
|
136
|
-
"synthesis cannot be made valid."
|
|
282
|
+
"synthesis cannot be made valid. A rejected submission leaves any " +
|
|
283
|
+
"already-accepted synthesis untouched; a later accepted submission " +
|
|
284
|
+
"supersedes an earlier one.",
|
|
137
285
|
inputSchema: z.object({ synthesis: RunSynthesisSchema }),
|
|
138
286
|
execute: async (input) => {
|
|
139
|
-
if (holder.outcome !== null) {
|
|
140
|
-
return ok({
|
|
141
|
-
accepted: false,
|
|
142
|
-
issues: [
|
|
143
|
-
{
|
|
144
|
-
path: "synthesis",
|
|
145
|
-
code: "semantic",
|
|
146
|
-
message: "A terminal outcome has already been recorded; submit_synthesis " + "can only be called once per invocation.",
|
|
147
|
-
},
|
|
148
|
-
],
|
|
149
|
-
});
|
|
150
|
-
}
|
|
151
287
|
const result = fullyValidate(input.synthesis, ctx);
|
|
152
288
|
if (!result.valid) {
|
|
289
|
+
// A rejection is not terminal — submission is last-valid-wins, so this
|
|
290
|
+
// leaves any already-recorded outcome intact and the model may fix the
|
|
291
|
+
// cited paths and submit again.
|
|
153
292
|
return ok({ accepted: false, issues: result.issues });
|
|
154
293
|
}
|
|
155
294
|
holder.outcome = { kind: "submitted", synthesis: result.synthesis };
|
|
@@ -163,11 +302,12 @@ function buildBlockerTool(holder) {
|
|
|
163
302
|
if (holder.outcome === null)
|
|
164
303
|
holder.outcome = outcome;
|
|
165
304
|
},
|
|
166
|
-
blockedWhen: "Ends run synthesis with no synthesis written. Use it when the
|
|
167
|
-
"produced no synthesizable content
|
|
168
|
-
"contradictory to the point of incoherence
|
|
169
|
-
"
|
|
170
|
-
"submit."
|
|
305
|
+
blockedWhen: "Ends run synthesis with no synthesis written. Use it ONLY when the " +
|
|
306
|
+
"run produced no synthesizable content — every step summary is empty, " +
|
|
307
|
+
"or the summaries are contradictory to the point of incoherence. A run " +
|
|
308
|
+
"with non-empty summaries but no individually notable findings is NOT a " +
|
|
309
|
+
"blocker: submit a synthesis with an empty findings[] instead. Never " +
|
|
310
|
+
"use it as an escape from a synthesis you could fix and submit.",
|
|
171
311
|
});
|
|
172
312
|
}
|
|
173
313
|
/**
|
|
@@ -181,6 +321,7 @@ export function __buildInnerToolsForTest(args) {
|
|
|
181
321
|
runId: args.runId,
|
|
182
322
|
};
|
|
183
323
|
return {
|
|
324
|
+
validate: buildValidateTool(ctx),
|
|
184
325
|
submit: buildSubmitTool(holder, ctx),
|
|
185
326
|
blocker: buildBlockerTool(holder),
|
|
186
327
|
holder,
|
|
@@ -203,9 +344,10 @@ export async function generateRunSynthesis(input) {
|
|
|
203
344
|
model: input.model,
|
|
204
345
|
bioKeys: input.bioKeys,
|
|
205
346
|
});
|
|
347
|
+
const validateTool = buildValidateTool(innerCtx);
|
|
206
348
|
const submitTool = buildSubmitTool(holder, innerCtx);
|
|
207
349
|
const blockerTool = buildBlockerTool(holder);
|
|
208
|
-
const tools = [submitTool, blockerTool, reviewer];
|
|
350
|
+
const tools = [validateTool, submitTool, blockerTool, reviewer];
|
|
209
351
|
const agent = {
|
|
210
352
|
id: AGENT_ID,
|
|
211
353
|
systemPrompt: composeSystemPrompt(synthesisAgentPrompt),
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Diagnosis and repair of model-authored arguments that failed Zod validation.
|
|
3
|
+
*
|
|
4
|
+
* A Zod issue message is accurate but inert: "Invalid input: expected object,
|
|
5
|
+
* received string" tells a model what the schema wanted, not what it did wrong
|
|
6
|
+
* or how to change the next call. Two failure modes put a whole, well-formed
|
|
7
|
+
* JSON value behind that message:
|
|
8
|
+
*
|
|
9
|
+
* - **Double encoding** — the model serialized a nested object or array with
|
|
10
|
+
* `JSON.stringify` and passed the resulting string as the argument value.
|
|
11
|
+
* - **Wrapper artifacts** — the JSON value arrived intact but with function-call
|
|
12
|
+
* markup or a markdown fence stuck to it (`{…}</parameter>\n</invoke>`,
|
|
13
|
+
* ` ```json {…}``` `). The payload is complete; only the wrapper makes it a
|
|
14
|
+
* string.
|
|
15
|
+
*
|
|
16
|
+
* This module owns both halves of the response for the four model-facing
|
|
17
|
+
* validation sites (`loop/run-agent.ts`'s tool-input boundary,
|
|
18
|
+
* `execution/run-synthesis.ts`, `tools/research/generate-plan.ts`,
|
|
19
|
+
* `tools/iterate-report.ts`):
|
|
20
|
+
*
|
|
21
|
+
* - `hintForZodIssue` — a **diagnostic only**. It parses nothing into the
|
|
22
|
+
* caller's data; it distinguishes the two modes so the message the model
|
|
23
|
+
* reads matches the mistake it actually made. When neither mode explains the
|
|
24
|
+
* string it stays silent, because speculative advice to a model that is
|
|
25
|
+
* already misdiagnosing is worse than no advice at all.
|
|
26
|
+
* - `repairToolInput` — an **issue-guided** repair: only a path where the
|
|
27
|
+
* schema wanted an `object` or `array` and found a string is even considered,
|
|
28
|
+
* so a tool that legitimately declares a `z.string()` field is never touched.
|
|
29
|
+
* It makes validation *reachable*; it never substitutes for it. The caller
|
|
30
|
+
* re-runs the full schema over the repaired value, and the tool's own
|
|
31
|
+
* semantic checks still run unchanged.
|
|
32
|
+
*
|
|
33
|
+
* Neither is used by `tools/lib/api-utils.ts`'s `summarizeZodIssues`, whose
|
|
34
|
+
* audience is a developer reading a log about a third-party API response shape —
|
|
35
|
+
* "pass the object directly" is meaningless advice for a payload no one here
|
|
36
|
+
* constructed.
|
|
37
|
+
*
|
|
38
|
+
* Detection walks `issue.path` into the raw input rather than reading the
|
|
39
|
+
* received type off the issue. Zod 4's `invalid_type` issue exposes `expected`
|
|
40
|
+
* but carries no `received` field — the received type appears only inside the
|
|
41
|
+
* human-readable `message`, which is version- and locale-dependent and must
|
|
42
|
+
* not be pattern-matched.
|
|
43
|
+
*/
|
|
44
|
+
import type { z } from "zod";
|
|
45
|
+
/**
|
|
46
|
+
* A hint for a single Zod issue, or `undefined` when nothing actionable can be
|
|
47
|
+
* said. `input` is the raw value that failed validation — the same value handed
|
|
48
|
+
* to `safeParse` — because the issue alone does not carry the offending value.
|
|
49
|
+
*
|
|
50
|
+
* Fires only for an `invalid_type` issue wanting an `object` or an `array` whose
|
|
51
|
+
* path resolves to a string, and only when that string demonstrably *is* the
|
|
52
|
+
* expected value:
|
|
53
|
+
*
|
|
54
|
+
* 1. it parses in full as JSON of the expected type → double encoding, and
|
|
55
|
+
* 2. it parses only once a wrapper artifact is stripped → the value is intact
|
|
56
|
+
* but surrounded by content that does not belong.
|
|
57
|
+
*
|
|
58
|
+
* Anything else — a plain string, a truncated fragment, a stray
|
|
59
|
+
* `<parameter name=…>` opener — yields `undefined`.
|
|
60
|
+
*
|
|
61
|
+
* A path that does not resolve — a mismatched shape, a pruned branch — yields
|
|
62
|
+
* `undefined` rather than throwing.
|
|
63
|
+
*/
|
|
64
|
+
export declare function hintForZodIssue(issue: z.core.$ZodIssue, input: unknown): string | undefined;
|
|
65
|
+
/**
|
|
66
|
+
* Repair model-authored tool input whose JSON values arrived as strings behind a
|
|
67
|
+
* wrapper artifact, or double-encoded. Returns a **new** input value with the
|
|
68
|
+
* repairs applied, or `undefined` when nothing was repaired.
|
|
69
|
+
*
|
|
70
|
+
* The repair is guided entirely by `error`: a path is considered only when the
|
|
71
|
+
* schema reported an `invalid_type` there wanting an `object` or `array` and the
|
|
72
|
+
* value sitting at that path is a string. The parsed result is accepted only if
|
|
73
|
+
* its own type matches what the schema expected, so a string field that happens
|
|
74
|
+
* to hold JSON-looking text is never rewritten.
|
|
75
|
+
*
|
|
76
|
+
* The input object graph is never mutated: the repaired copy shares every
|
|
77
|
+
* untouched branch and clones only along the paths it patches. Repaired content
|
|
78
|
+
* is not re-examined — one pass, no recursion — and no input causes a throw.
|
|
79
|
+
*/
|
|
80
|
+
export declare function repairToolInput(input: unknown, error: z.ZodError): unknown | undefined;
|