imprint-mcp 0.6.2 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/prompts/audit-agent.md +6 -5
- package/prompts/build-planning.md +2 -2
- package/prompts/compile-agent.md +6 -4
- package/prompts/live-verifier-agent.md +24 -11
- package/src/cli.ts +38 -3
- package/src/imprint/agent.ts +7 -2
- package/src/imprint/audit.ts +212 -20
- package/src/imprint/auth-compile-tools.ts +64 -13
- package/src/imprint/compile-agent-types.ts +25 -2
- package/src/imprint/compile-agent.ts +197 -40
- package/src/imprint/compile-tools.ts +164 -22
- package/src/imprint/live-verifier.ts +167 -6
- package/src/imprint/llm.ts +4 -0
- package/src/imprint/mcp-compile-server.ts +188 -83
- package/src/imprint/mcp-server.ts +16 -1
- package/src/imprint/probe-backends.ts +60 -25
- package/src/imprint/teach-state.ts +52 -2
- package/src/imprint/teach.ts +94 -69
|
@@ -32,6 +32,7 @@ import {
|
|
|
32
32
|
type probeResolvedTool,
|
|
33
33
|
rebindBackendsCacheToWorkflow,
|
|
34
34
|
} from './probe-backends.ts';
|
|
35
|
+
import { ensureImprintRuntimeLink } from './runtime-link.ts';
|
|
35
36
|
import { loadCredentialStore } from './runtime.ts';
|
|
36
37
|
import { isSensitiveKey } from './sensitive-keys.ts';
|
|
37
38
|
import { buildZodValidator } from './tool-loader.ts';
|
|
@@ -43,9 +44,18 @@ const SYSTEM_PROMPT_PATH = pathJoin(REPO_ROOT, 'prompts', 'live-verifier-agent.m
|
|
|
43
44
|
const MCP_SERVER_NAME = 'imprint-live-verifier';
|
|
44
45
|
export const LIVE_VERIFICATION_EVIDENCE_FILE = '.live-verification-evidence.json';
|
|
45
46
|
const LIVE_VERIFIER_LOG_FILE = '.live-verifier-log.jsonl';
|
|
46
|
-
|
|
47
|
+
// Cheap probes normally finish well inside the original two-minute window and
|
|
48
|
+
// skip browser transports. The larger ceiling is for the fallback path: a cold
|
|
49
|
+
// CDP request must finish before fetch-bootstrap gets its own browser attempt.
|
|
50
|
+
// Comprehensive preparation is sequential on purpose: concurrent browser
|
|
51
|
+
// transports can trip shared anti-bot/session defenses. Reserve enough time
|
|
52
|
+
// for the normal fetch/stealth/playbook candidates plus CDP and fetch-bootstrap
|
|
53
|
+
// fallbacks; the former 240s cap could kill CDP after a 150s playbook attempt.
|
|
54
|
+
const BACKEND_PREPARATION_BUDGET_MS = 8 * 60_000;
|
|
47
55
|
const MAX_CREDITED_BACKEND_PREPARATIONS = 2;
|
|
48
56
|
const BACKEND_PROBE_SHUTDOWN_GRACE_MS = 5_000;
|
|
57
|
+
const MAX_PROMPT_EVIDENCE_RECORDS = 48;
|
|
58
|
+
const MAX_PROMPT_EVIDENCE_RECORD_CHARS = 8_000;
|
|
49
59
|
let evidenceTempSequence = 0;
|
|
50
60
|
|
|
51
61
|
const BaselineSchema = z
|
|
@@ -158,6 +168,17 @@ export const LiveVerificationReportSchema = z
|
|
|
158
168
|
|
|
159
169
|
type LiveVerificationReport = z.infer<typeof LiveVerificationReportSchema>;
|
|
160
170
|
|
|
171
|
+
export function isInfrastructureOnlyInconclusiveReport(report: LiveVerificationReport): boolean {
|
|
172
|
+
return (
|
|
173
|
+
report.status === 'inconclusive' &&
|
|
174
|
+
report.baseline.verdict === 'infrastructure' &&
|
|
175
|
+
report.issues.length === 0 &&
|
|
176
|
+
report.parameters.every(
|
|
177
|
+
(parameter) => parameter.verdict === 'works' || parameter.verdict === 'untestable',
|
|
178
|
+
)
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
|
|
161
182
|
interface LiveIntegrationSuiteResult {
|
|
162
183
|
exitCode: number;
|
|
163
184
|
timedOut: boolean;
|
|
@@ -251,6 +272,11 @@ export async function runLiveIntegrationSuite(opts: {
|
|
|
251
272
|
reason?: string;
|
|
252
273
|
sessionLabel?: string;
|
|
253
274
|
}): Promise<LiveIntegrationSuiteResult> {
|
|
275
|
+
// The suite is launched directly from the generated tool directory, bypassing
|
|
276
|
+
// normal tool discovery. Repair a stale global/worktree runtime link here so
|
|
277
|
+
// imports such as `imprint/compile-verification` resolve against this running
|
|
278
|
+
// Imprint installation.
|
|
279
|
+
ensureImprintRuntimeLink(dirname(dirname(opts.toolDir)));
|
|
254
280
|
const integrationPath = pathJoin(opts.toolDir, 'integration.test.ts');
|
|
255
281
|
if (!existsSync(integrationPath)) throw new Error('integration.test.ts is missing');
|
|
256
282
|
const workflowPath = pathJoin(opts.toolDir, 'workflow.json');
|
|
@@ -461,7 +487,11 @@ export function readPersistedLiveVerificationEvidence(
|
|
|
461
487
|
|
|
462
488
|
export function hasSuiteReceiptForSession(path: string, sessionLabel: string): boolean {
|
|
463
489
|
return readPersistedLiveVerificationEvidence(path).some(
|
|
464
|
-
(item) =>
|
|
490
|
+
(item) =>
|
|
491
|
+
item.kind === 'suite' &&
|
|
492
|
+
item.verifierSession === sessionLabel &&
|
|
493
|
+
item.status !== 'running' &&
|
|
494
|
+
typeof item.finishedAt === 'string',
|
|
465
495
|
);
|
|
466
496
|
}
|
|
467
497
|
|
|
@@ -628,6 +658,11 @@ async function runBackendProbeSubprocess(opts: {
|
|
|
628
658
|
attempt?: number;
|
|
629
659
|
label: string;
|
|
630
660
|
}): Promise<Awaited<ReturnType<typeof probeResolvedTool>>> {
|
|
661
|
+
const previousCacheContents = existsSync(opts.outPath)
|
|
662
|
+
? readFileSync(opts.outPath, 'utf8')
|
|
663
|
+
: undefined;
|
|
664
|
+
const tool = resolveWorkflowTool(opts.workflowPath);
|
|
665
|
+
const assetRoot = pathJoin(tool.dir, '..', '..');
|
|
631
666
|
const child = spawn(
|
|
632
667
|
process.execPath,
|
|
633
668
|
[
|
|
@@ -674,6 +709,30 @@ async function runBackendProbeSubprocess(opts: {
|
|
|
674
709
|
unregisterParentCleanup();
|
|
675
710
|
}
|
|
676
711
|
if (outcome.timedOut) {
|
|
712
|
+
// probeResolvedTool checkpoints every successful candidate. If a later
|
|
713
|
+
// optional rung consumed the aggregate deadline, use the new valid cache
|
|
714
|
+
// instead of discarding an already-proven backend. Do not reuse an unchanged
|
|
715
|
+
// pre-probe cache: forceReprobe may have been requested because it failed.
|
|
716
|
+
const currentCacheContents = existsSync(opts.outPath)
|
|
717
|
+
? readFileSync(opts.outPath, 'utf8')
|
|
718
|
+
: undefined;
|
|
719
|
+
const cacheStatus = loadBackendsCacheStatus(tool.site, assetRoot, tool.dir, {
|
|
720
|
+
warn: false,
|
|
721
|
+
toolName: tool.workflow.toolName,
|
|
722
|
+
});
|
|
723
|
+
if (
|
|
724
|
+
currentCacheContents !== undefined &&
|
|
725
|
+
currentCacheContents !== previousCacheContents &&
|
|
726
|
+
cacheStatus.status === 'ok'
|
|
727
|
+
) {
|
|
728
|
+
appendLiveVerifierLog(opts.logPath, {
|
|
729
|
+
type: 'backend.probe.partial-cache-recovered',
|
|
730
|
+
attempt: opts.attempt,
|
|
731
|
+
label: opts.label,
|
|
732
|
+
preferredOrder: cacheStatus.cache.preferredOrder,
|
|
733
|
+
});
|
|
734
|
+
return { cache: cacheStatus.cache, outPath: opts.outPath };
|
|
735
|
+
}
|
|
677
736
|
throw new Error(
|
|
678
737
|
`backend probe exceeded its separate ${Math.round(BACKEND_PREPARATION_BUDGET_MS / 1_000)}s budget and was terminated`,
|
|
679
738
|
);
|
|
@@ -684,8 +743,6 @@ async function runBackendProbeSubprocess(opts: {
|
|
|
684
743
|
);
|
|
685
744
|
}
|
|
686
745
|
|
|
687
|
-
const tool = resolveWorkflowTool(opts.workflowPath);
|
|
688
|
-
const assetRoot = pathJoin(tool.dir, '..', '..');
|
|
689
746
|
const cacheStatus = loadBackendsCacheStatus(tool.site, assetRoot, tool.dir, {
|
|
690
747
|
warn: false,
|
|
691
748
|
toolName: tool.workflow.toolName,
|
|
@@ -701,6 +758,10 @@ interface LiveSemanticVerificationResult {
|
|
|
701
758
|
provider: ProviderName;
|
|
702
759
|
model: string;
|
|
703
760
|
attempts: number;
|
|
761
|
+
/** True only when an independent verifier returned a schema-valid report.
|
|
762
|
+
* Provider startup/input failures synthesize an inconclusive report for
|
|
763
|
+
* compiler feedback but do not consume a semantic-review cycle. */
|
|
764
|
+
completedReview: boolean;
|
|
704
765
|
}
|
|
705
766
|
|
|
706
767
|
export async function runLiveSemanticVerification(opts: {
|
|
@@ -828,6 +889,7 @@ export async function runLiveSemanticVerification(opts: {
|
|
|
828
889
|
provider: opts.provider,
|
|
829
890
|
model,
|
|
830
891
|
attempts: attempt,
|
|
892
|
+
completedReview: true,
|
|
831
893
|
};
|
|
832
894
|
} catch (error) {
|
|
833
895
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -890,7 +952,13 @@ export async function runLiveSemanticVerification(opts: {
|
|
|
890
952
|
status: 'inconclusive',
|
|
891
953
|
attempts: 2,
|
|
892
954
|
});
|
|
893
|
-
return {
|
|
955
|
+
return {
|
|
956
|
+
report: inconclusive,
|
|
957
|
+
provider: opts.provider,
|
|
958
|
+
model,
|
|
959
|
+
attempts: 2,
|
|
960
|
+
completedReview: false,
|
|
961
|
+
};
|
|
894
962
|
}
|
|
895
963
|
|
|
896
964
|
function mergePersistedRecords(
|
|
@@ -955,13 +1023,106 @@ function buildInitialMessage(
|
|
|
955
1023
|
timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone,
|
|
956
1024
|
},
|
|
957
1025
|
artifactContext: buildVerifierArtifactContext(toolDir, workflow.toolName),
|
|
958
|
-
baselineEvidence: evidence,
|
|
1026
|
+
baselineEvidence: compactVerifierEvidenceContext(evidence),
|
|
959
1027
|
},
|
|
960
1028
|
null,
|
|
961
1029
|
2,
|
|
962
1030
|
)}`;
|
|
963
1031
|
}
|
|
964
1032
|
|
|
1033
|
+
/** Keep the verifier prompt bounded without weakening the durable evidence
|
|
1034
|
+
* artifact. Full, sanitized receipts remain on disk; the model gets recent
|
|
1035
|
+
* records with representative payload previews and navigation metadata. */
|
|
1036
|
+
export function compactVerifierEvidenceContext(
|
|
1037
|
+
evidence: Array<LiveIntegrationEvidence | PersistedLiveVerificationRecord>,
|
|
1038
|
+
): unknown[] {
|
|
1039
|
+
const omittedRecords = Math.max(0, evidence.length - MAX_PROMPT_EVIDENCE_RECORDS);
|
|
1040
|
+
const selected = evidence.slice(-MAX_PROMPT_EVIDENCE_RECORDS);
|
|
1041
|
+
const compacted = selected.map((record) => compactVerifierEvidenceRecord(record));
|
|
1042
|
+
return omittedRecords > 0
|
|
1043
|
+
? [
|
|
1044
|
+
{
|
|
1045
|
+
kind: 'prompt-compaction',
|
|
1046
|
+
omittedRecords,
|
|
1047
|
+
note: `Full receipts remain in ${LIVE_VERIFICATION_EVIDENCE_FILE}.`,
|
|
1048
|
+
},
|
|
1049
|
+
...compacted,
|
|
1050
|
+
]
|
|
1051
|
+
: compacted;
|
|
1052
|
+
}
|
|
1053
|
+
|
|
1054
|
+
function compactVerifierEvidenceRecord(
|
|
1055
|
+
record: LiveIntegrationEvidence | PersistedLiveVerificationRecord,
|
|
1056
|
+
): unknown {
|
|
1057
|
+
const compacted = compactVerifierValue(record, 0);
|
|
1058
|
+
const serialized = JSON.stringify(compacted);
|
|
1059
|
+
if (serialized.length <= MAX_PROMPT_EVIDENCE_RECORD_CHARS) return compacted;
|
|
1060
|
+
|
|
1061
|
+
const navigationKeys = [
|
|
1062
|
+
'schemaVersion',
|
|
1063
|
+
'kind',
|
|
1064
|
+
'label',
|
|
1065
|
+
'caseName',
|
|
1066
|
+
'toolName',
|
|
1067
|
+
'status',
|
|
1068
|
+
'verifierSession',
|
|
1069
|
+
'startedAt',
|
|
1070
|
+
'finishedAt',
|
|
1071
|
+
'requestedParams',
|
|
1072
|
+
'effectiveParams',
|
|
1073
|
+
'preferredBackend',
|
|
1074
|
+
'usedBackend',
|
|
1075
|
+
'attempts',
|
|
1076
|
+
'durationMs',
|
|
1077
|
+
'exitCode',
|
|
1078
|
+
'timedOut',
|
|
1079
|
+
'passedTests',
|
|
1080
|
+
'failedTests',
|
|
1081
|
+
'completedCallLabels',
|
|
1082
|
+
'reason',
|
|
1083
|
+
'error',
|
|
1084
|
+
] as const;
|
|
1085
|
+
const source = record as Record<string, unknown>;
|
|
1086
|
+
const navigation: Record<string, unknown> = {};
|
|
1087
|
+
for (const key of navigationKeys) {
|
|
1088
|
+
if (source[key] !== undefined) navigation[key] = compactVerifierValue(source[key], 0);
|
|
1089
|
+
}
|
|
1090
|
+
return {
|
|
1091
|
+
...navigation,
|
|
1092
|
+
payloadPreview: clipVerifierString(serialized, MAX_PROMPT_EVIDENCE_RECORD_CHARS),
|
|
1093
|
+
promptCompacted: true,
|
|
1094
|
+
note: `Full receipt remains in ${LIVE_VERIFICATION_EVIDENCE_FILE}.`,
|
|
1095
|
+
};
|
|
1096
|
+
}
|
|
1097
|
+
|
|
1098
|
+
function compactVerifierValue(value: unknown, depth: number): unknown {
|
|
1099
|
+
if (typeof value === 'string') return clipVerifierString(value, 2_000);
|
|
1100
|
+
if (value === null || typeof value !== 'object') return value;
|
|
1101
|
+
if (depth >= 6) return clipVerifierString(JSON.stringify(value), 2_000);
|
|
1102
|
+
if (Array.isArray(value)) {
|
|
1103
|
+
if (value.length <= 4) return value.map((item) => compactVerifierValue(item, depth + 1));
|
|
1104
|
+
return [
|
|
1105
|
+
...value.slice(0, 3).map((item) => compactVerifierValue(item, depth + 1)),
|
|
1106
|
+
{ promptCompactedItems: value.length - 4 },
|
|
1107
|
+
compactVerifierValue(value.at(-1), depth + 1),
|
|
1108
|
+
];
|
|
1109
|
+
}
|
|
1110
|
+
return Object.fromEntries(
|
|
1111
|
+
Object.entries(value as Record<string, unknown>).map(([key, item]) => [
|
|
1112
|
+
key,
|
|
1113
|
+
compactVerifierValue(item, depth + 1),
|
|
1114
|
+
]),
|
|
1115
|
+
);
|
|
1116
|
+
}
|
|
1117
|
+
|
|
1118
|
+
function clipVerifierString(value: string, maxChars: number): string {
|
|
1119
|
+
const redacted = redactFreeformText(value).redacted;
|
|
1120
|
+
if (redacted.length <= maxChars) return redacted;
|
|
1121
|
+
const headChars = Math.ceil(maxChars * 0.6);
|
|
1122
|
+
const tailChars = Math.floor(maxChars * 0.4);
|
|
1123
|
+
return `${redacted.slice(0, headChars)}\n...[prompt preview truncated ${redacted.length - maxChars} chars]...\n${redacted.slice(-tailChars)}`;
|
|
1124
|
+
}
|
|
1125
|
+
|
|
965
1126
|
function creditBackendPreparation(
|
|
966
1127
|
budget: { creditedCount: number; creditMs: number },
|
|
967
1128
|
durationMs: number,
|
package/src/imprint/llm.ts
CHANGED
|
@@ -1254,6 +1254,10 @@ export function preferredVerificationModel(provider: ProviderName): string {
|
|
|
1254
1254
|
throw new Error(`Provider ${provider} does not support live semantic verification`);
|
|
1255
1255
|
}
|
|
1256
1256
|
|
|
1257
|
+
/** The semantic reviewer intentionally uses a provider independent from the
|
|
1258
|
+
* compiler. Claude/Sonnet compilation therefore still receives a Terra review. */
|
|
1259
|
+
export const DEFAULT_VERIFICATION_PROVIDER: ProviderName = 'codex-cli';
|
|
1260
|
+
|
|
1257
1261
|
interface ModelOption {
|
|
1258
1262
|
model: string;
|
|
1259
1263
|
isDefault: boolean;
|
|
@@ -35,6 +35,10 @@ import {
|
|
|
35
35
|
type SharedModuleManifestEntry,
|
|
36
36
|
resolvePlanSliceFromFile,
|
|
37
37
|
} from './build-plan.ts';
|
|
38
|
+
import {
|
|
39
|
+
advanceIncompleteSemanticVerificationRuns,
|
|
40
|
+
advanceSemanticVerificationCycle,
|
|
41
|
+
} from './compile-agent-types.ts';
|
|
38
42
|
import {
|
|
39
43
|
applyLiveVerification,
|
|
40
44
|
applyParamVerification,
|
|
@@ -42,10 +46,12 @@ import {
|
|
|
42
46
|
externalVerification,
|
|
43
47
|
} from './compile-tools.ts';
|
|
44
48
|
import {
|
|
49
|
+
isInfrastructureOnlyInconclusiveReport,
|
|
45
50
|
mergeSemanticParamVerification,
|
|
46
51
|
runLiveSemanticVerification,
|
|
47
52
|
semanticVerificationFailures,
|
|
48
53
|
} from './live-verifier.ts';
|
|
54
|
+
import { DEFAULT_VERIFICATION_PROVIDER } from './llm.ts';
|
|
49
55
|
import type { ProviderName } from './llm.ts';
|
|
50
56
|
import { loadJsonFile } from './load-json.ts';
|
|
51
57
|
import { createLog } from './log.ts';
|
|
@@ -210,22 +216,45 @@ export function compileArtifactFingerprint(toolDir: string): string {
|
|
|
210
216
|
}
|
|
211
217
|
|
|
212
218
|
export function canAcceptInconclusiveDecision(opts: {
|
|
219
|
+
semanticVerificationCycles: number;
|
|
220
|
+
maxVerificationCycles: number;
|
|
221
|
+
completedReview: boolean;
|
|
222
|
+
infrastructureOnly: boolean;
|
|
213
223
|
pendingFingerprint?: string;
|
|
214
224
|
currentFingerprint: string;
|
|
215
225
|
acceptInconclusive?: boolean;
|
|
216
226
|
inconclusiveReason?: string;
|
|
217
227
|
}): boolean {
|
|
218
228
|
return (
|
|
229
|
+
opts.completedReview &&
|
|
230
|
+
opts.infrastructureOnly &&
|
|
231
|
+
opts.semanticVerificationCycles >= opts.maxVerificationCycles &&
|
|
219
232
|
opts.acceptInconclusive === true &&
|
|
220
233
|
Boolean(opts.inconclusiveReason?.trim()) &&
|
|
221
234
|
opts.pendingFingerprint === opts.currentFingerprint
|
|
222
235
|
);
|
|
223
236
|
}
|
|
224
237
|
|
|
238
|
+
export function inconclusiveDecisionAtSemanticCap(opts: {
|
|
239
|
+
semanticVerificationCycles: number;
|
|
240
|
+
maxVerificationCycles: number;
|
|
241
|
+
pendingFingerprint?: string;
|
|
242
|
+
currentFingerprint: string;
|
|
243
|
+
}): 'continue-verification' | 'await-compiler-decision' | 'fail-artifact-changed' {
|
|
244
|
+
if (!opts.pendingFingerprint || opts.semanticVerificationCycles < opts.maxVerificationCycles) {
|
|
245
|
+
return 'continue-verification';
|
|
246
|
+
}
|
|
247
|
+
return opts.pendingFingerprint === opts.currentFingerprint
|
|
248
|
+
? 'await-compiler-decision'
|
|
249
|
+
: 'fail-artifact-changed';
|
|
250
|
+
}
|
|
251
|
+
|
|
225
252
|
export async function runCompileMcpServer(opts: RunCompileMcpServerOptions): Promise<void> {
|
|
226
253
|
const isAuthMode = !!opts.authToolPlan;
|
|
227
254
|
const maxVerificationCycles = opts.maxVerificationCycles ?? 5;
|
|
255
|
+
const maxIncompleteVerifierRuns = 2;
|
|
228
256
|
let pendingInconclusive: PendingInconclusiveDecision | undefined;
|
|
257
|
+
let incompleteVerifierRuns = 0;
|
|
229
258
|
|
|
230
259
|
// Load + auto-redact the session, exactly as compile-agent.ts does.
|
|
231
260
|
let session: Session = loadJsonFile(
|
|
@@ -412,7 +441,7 @@ export async function runCompileMcpServer(opts: RunCompileMcpServerOptions): Pro
|
|
|
412
441
|
]
|
|
413
442
|
: [];
|
|
414
443
|
|
|
415
|
-
let
|
|
444
|
+
let semanticVerificationCycles = 0;
|
|
416
445
|
/** One pending checkpoint per segment — refuse a second so the orchestrator
|
|
417
446
|
* acts on exactly one request. Cleared implicitly: each segment is a fresh
|
|
418
447
|
* process with the sentinel removed before spawn. */
|
|
@@ -487,15 +516,15 @@ export async function runCompileMcpServer(opts: RunCompileMcpServerOptions): Pro
|
|
|
487
516
|
};
|
|
488
517
|
}
|
|
489
518
|
|
|
490
|
-
|
|
491
|
-
log(`auth verification failed (cycle ${
|
|
519
|
+
semanticVerificationCycles++;
|
|
520
|
+
log(`auth verification failed (cycle ${semanticVerificationCycles})`);
|
|
492
521
|
|
|
493
522
|
return {
|
|
494
523
|
isError: true,
|
|
495
524
|
content: [
|
|
496
525
|
{
|
|
497
526
|
type: 'text',
|
|
498
|
-
text: `You called done but verification failed (cycle ${
|
|
527
|
+
text: `You called done but verification failed (cycle ${semanticVerificationCycles}):
|
|
499
528
|
|
|
500
529
|
${failures.map((f) => `- ${f}`).join('\n')}
|
|
501
530
|
|
|
@@ -507,6 +536,8 @@ Fix the issues in workflow.json, re-test with run_verification, and call done ag
|
|
|
507
536
|
|
|
508
537
|
const finishVerification = beginCompileVerification(opts.toolDir);
|
|
509
538
|
try {
|
|
539
|
+
let semanticReviewCompleted = false;
|
|
540
|
+
let semanticReviewAttempted = false;
|
|
510
541
|
const { failures, warnings, paramVerification, integrationEvidence } =
|
|
511
542
|
await externalVerification(opts.toolDir, session, opts.sessionPath, {
|
|
512
543
|
expectedToolName: opts.candidate?.toolName,
|
|
@@ -536,6 +567,12 @@ Fix the issues in workflow.json, re-test with run_verification, and call done ag
|
|
|
536
567
|
if (
|
|
537
568
|
pendingInconclusive &&
|
|
538
569
|
canAcceptInconclusiveDecision({
|
|
570
|
+
semanticVerificationCycles,
|
|
571
|
+
maxVerificationCycles,
|
|
572
|
+
completedReview: pendingInconclusive.report.completedReview,
|
|
573
|
+
infrastructureOnly: isInfrastructureOnlyInconclusiveReport(
|
|
574
|
+
pendingInconclusive.report.report,
|
|
575
|
+
),
|
|
539
576
|
pendingFingerprint: pendingInconclusive.artifactFingerprint,
|
|
540
577
|
currentFingerprint,
|
|
541
578
|
acceptInconclusive: doneArgs.accept_inconclusive,
|
|
@@ -601,89 +638,154 @@ Fix the issues in workflow.json, re-test with run_verification, and call done ag
|
|
|
601
638
|
};
|
|
602
639
|
}
|
|
603
640
|
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
'compile artifacts changed after inconclusive review; running verification again',
|
|
610
|
-
);
|
|
611
|
-
pendingInconclusive = undefined;
|
|
612
|
-
}
|
|
613
|
-
const semantic = await runLiveSemanticVerification({
|
|
614
|
-
provider: opts.provider,
|
|
615
|
-
toolDir: opts.toolDir,
|
|
616
|
-
evidence: integrationEvidence,
|
|
641
|
+
const capDecision = inconclusiveDecisionAtSemanticCap({
|
|
642
|
+
semanticVerificationCycles,
|
|
643
|
+
maxVerificationCycles,
|
|
644
|
+
pendingFingerprint: pendingInconclusive?.artifactFingerprint,
|
|
645
|
+
currentFingerprint,
|
|
617
646
|
});
|
|
618
|
-
if (
|
|
619
|
-
pendingInconclusive = {
|
|
620
|
-
artifactFingerprint: compileArtifactFingerprint(opts.toolDir),
|
|
621
|
-
report: semantic,
|
|
622
|
-
};
|
|
623
|
-
} else {
|
|
647
|
+
if (capDecision === 'fail-artifact-changed') {
|
|
624
648
|
pendingInconclusive = undefined;
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
failures.push(...semanticFailures);
|
|
628
|
-
if (semantic.report.status === 'approved_with_gaps') {
|
|
629
|
-
warnings.push(
|
|
630
|
-
`independent semantic verification approved with gaps: ${semantic.report.gaps.join('; ')}`,
|
|
649
|
+
failures.push(
|
|
650
|
+
'compile artifacts changed after the final allowed inconclusive semantic review; refusing to run an additional review beyond the semantic cycle cap',
|
|
631
651
|
);
|
|
632
|
-
}
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
paramVerification,
|
|
636
|
-
semantic.report,
|
|
652
|
+
} else if (capDecision === 'await-compiler-decision') {
|
|
653
|
+
failures.push(
|
|
654
|
+
'semantic review limit reached with an unchanged infrastructure-only inconclusive result; explicitly accept the unverified artifact with reasoning or revise it',
|
|
637
655
|
);
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
656
|
+
} else {
|
|
657
|
+
if (
|
|
658
|
+
pendingInconclusive &&
|
|
659
|
+
pendingInconclusive.artifactFingerprint !== currentFingerprint
|
|
660
|
+
) {
|
|
661
|
+
log(
|
|
662
|
+
'compile artifacts changed after inconclusive review; running verification again',
|
|
663
|
+
);
|
|
664
|
+
pendingInconclusive = undefined;
|
|
647
665
|
}
|
|
648
|
-
|
|
649
|
-
const
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
666
|
+
semanticReviewAttempted = true;
|
|
667
|
+
const semantic = await runLiveSemanticVerification({
|
|
668
|
+
provider: DEFAULT_VERIFICATION_PROVIDER,
|
|
669
|
+
toolDir: opts.toolDir,
|
|
670
|
+
evidence: integrationEvidence,
|
|
671
|
+
});
|
|
672
|
+
semanticReviewCompleted = semantic.completedReview;
|
|
673
|
+
if (semantic.report.status === 'inconclusive') {
|
|
674
|
+
pendingInconclusive = {
|
|
675
|
+
artifactFingerprint: compileArtifactFingerprint(opts.toolDir),
|
|
676
|
+
report: semantic,
|
|
677
|
+
};
|
|
678
|
+
} else {
|
|
679
|
+
pendingInconclusive = undefined;
|
|
680
|
+
}
|
|
681
|
+
const semanticFailures = semanticVerificationFailures(semantic.report);
|
|
682
|
+
failures.push(...semanticFailures);
|
|
683
|
+
if (semantic.report.status === 'approved_with_gaps') {
|
|
684
|
+
warnings.push(
|
|
685
|
+
`independent semantic verification approved with gaps: ${semantic.report.gaps.join('; ')}`,
|
|
686
|
+
);
|
|
687
|
+
}
|
|
688
|
+
if (semanticFailures.length === 0) {
|
|
689
|
+
const semanticParams = mergeSemanticParamVerification(
|
|
690
|
+
paramVerification,
|
|
691
|
+
semantic.report,
|
|
692
|
+
);
|
|
693
|
+
// Only the independent semantic agent can mint liveVerified=true.
|
|
694
|
+
applyLiveVerification(opts.toolDir, undefined);
|
|
695
|
+
const paramWarnings = applyParamVerification(opts.toolDir, semanticParams);
|
|
696
|
+
// liveVerified/parameter annotations are metadata-only, but they
|
|
697
|
+
// change the hash-strict workflow cache key. Keep the verifier's
|
|
698
|
+
// already-proven backend preference current without probing again.
|
|
699
|
+
rebindExistingBackendsCacheToWorkflow(opts.toolDir);
|
|
700
|
+
if (paramWarnings.length > 0) {
|
|
701
|
+
log(`parameter verification:\n${paramWarnings.join('\n')}`);
|
|
702
|
+
}
|
|
703
|
+
const allWarnings = [...warnings, ...paramWarnings];
|
|
704
|
+
const sentinel = pathJoin(opts.toolDir, DONE_SENTINEL);
|
|
705
|
+
writeFileSync(
|
|
706
|
+
sentinel,
|
|
707
|
+
JSON.stringify(
|
|
708
|
+
{
|
|
709
|
+
summary,
|
|
710
|
+
verification: 'passed',
|
|
711
|
+
semanticVerification: {
|
|
712
|
+
status: semantic.report.status,
|
|
713
|
+
provider: semantic.provider,
|
|
714
|
+
model: semantic.model,
|
|
715
|
+
attempts: semantic.attempts,
|
|
716
|
+
evidenceArtifact: semantic.report.evidenceArtifact,
|
|
717
|
+
logArtifact: semantic.report.logArtifact,
|
|
718
|
+
},
|
|
719
|
+
warnings: allWarnings,
|
|
720
|
+
timestamp: Date.now(),
|
|
663
721
|
},
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
)
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
],
|
|
680
|
-
};
|
|
722
|
+
null,
|
|
723
|
+
2,
|
|
724
|
+
),
|
|
725
|
+
'utf8',
|
|
726
|
+
);
|
|
727
|
+
log(`verification passed; wrote ${sentinel}`);
|
|
728
|
+
return {
|
|
729
|
+
content: [
|
|
730
|
+
{
|
|
731
|
+
type: 'text',
|
|
732
|
+
text: `DONE_VERIFIED — deterministic checks and independent live semantic verification passed (${semantic.report.status}). Evidence: ${semantic.report.evidenceArtifact ?? '.live-verification-evidence.json'}. Verifier log: ${semantic.report.logArtifact ?? '.live-verifier-log.jsonl'}. The orchestrator will exit shortly. Do not call any more tools.`,
|
|
733
|
+
},
|
|
734
|
+
],
|
|
735
|
+
};
|
|
736
|
+
}
|
|
681
737
|
}
|
|
682
738
|
}
|
|
683
739
|
}
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
740
|
+
semanticVerificationCycles = advanceSemanticVerificationCycle(
|
|
741
|
+
semanticVerificationCycles,
|
|
742
|
+
semanticReviewCompleted,
|
|
743
|
+
);
|
|
744
|
+
incompleteVerifierRuns = advanceIncompleteSemanticVerificationRuns(
|
|
745
|
+
incompleteVerifierRuns,
|
|
746
|
+
semanticReviewAttempted,
|
|
747
|
+
semanticReviewCompleted,
|
|
748
|
+
);
|
|
749
|
+
if (incompleteVerifierRuns >= maxIncompleteVerifierRuns) {
|
|
750
|
+
const sentinel = pathJoin(opts.toolDir, DONE_SENTINEL);
|
|
751
|
+
writeFileSync(
|
|
752
|
+
sentinel,
|
|
753
|
+
JSON.stringify(
|
|
754
|
+
{
|
|
755
|
+
summary,
|
|
756
|
+
verification: 'failed',
|
|
757
|
+
incompleteVerifierRuns,
|
|
758
|
+
failures,
|
|
759
|
+
warnings,
|
|
760
|
+
timestamp: Date.now(),
|
|
761
|
+
},
|
|
762
|
+
null,
|
|
763
|
+
2,
|
|
764
|
+
),
|
|
765
|
+
'utf8',
|
|
766
|
+
);
|
|
767
|
+
return {
|
|
768
|
+
isError: true,
|
|
769
|
+
content: [
|
|
770
|
+
{
|
|
771
|
+
type: 'text',
|
|
772
|
+
text: `Independent semantic verifier failed to complete after ${maxIncompleteVerifierRuns} bounded runs. Final failures:\n${failures.map((failure) => `- ${failure}`).join('\n')}`,
|
|
773
|
+
},
|
|
774
|
+
],
|
|
775
|
+
};
|
|
776
|
+
}
|
|
777
|
+
const failurePhase = semanticReviewCompleted
|
|
778
|
+
? `semantic verification failed (cycle ${semanticVerificationCycles}/${maxVerificationCycles})`
|
|
779
|
+
: semanticReviewAttempted
|
|
780
|
+
? 'semantic verifier did not complete a review (semantic cycle unchanged)'
|
|
781
|
+
: 'deterministic verification failed (semantic cycle unchanged)';
|
|
782
|
+
log(failurePhase);
|
|
783
|
+
const awaitingFinalInconclusiveDecision =
|
|
784
|
+
semanticVerificationCycles >= maxVerificationCycles && pendingInconclusive !== undefined;
|
|
785
|
+
if (
|
|
786
|
+
semanticVerificationCycles >= maxVerificationCycles &&
|
|
787
|
+
!awaitingFinalInconclusiveDecision
|
|
788
|
+
) {
|
|
687
789
|
const sentinel = pathJoin(opts.toolDir, DONE_SENTINEL);
|
|
688
790
|
writeFileSync(
|
|
689
791
|
sentinel,
|
|
@@ -691,7 +793,7 @@ Fix the issues in workflow.json, re-test with run_verification, and call done ag
|
|
|
691
793
|
{
|
|
692
794
|
summary,
|
|
693
795
|
verification: 'failed',
|
|
694
|
-
cycles:
|
|
796
|
+
cycles: semanticVerificationCycles,
|
|
695
797
|
failures,
|
|
696
798
|
warnings,
|
|
697
799
|
timestamp: Date.now(),
|
|
@@ -706,20 +808,23 @@ Fix the issues in workflow.json, re-test with run_verification, and call done ag
|
|
|
706
808
|
content: [
|
|
707
809
|
{
|
|
708
810
|
type: 'text',
|
|
709
|
-
text: `
|
|
811
|
+
text: `Semantic verification failed after ${maxVerificationCycles} cycles. Giving up. Final failures:\n${failures.map((f) => `- ${f}`).join('\n')}`,
|
|
710
812
|
},
|
|
711
813
|
],
|
|
712
814
|
};
|
|
713
815
|
}
|
|
714
816
|
|
|
715
|
-
const continuationMessage = `You called done but
|
|
817
|
+
const continuationMessage = `You called done but ${failurePhase}:
|
|
716
818
|
|
|
717
819
|
${failures.map((f) => `- ${f}`).join('\n')}
|
|
718
820
|
|
|
719
821
|
Resume your work. Read the files you wrote (workflow.json, parser.ts, parser.test.ts), fix the issues, re-run tests, and call done again when fixed.`;
|
|
720
|
-
const compilerDecisionGuidance = `\n\nUse the verifier's semantic evidence, not a requirement for 100% of the original candidate inputs. If the core intent works but a secondary parameter cannot be supported from the recording and live evidence without guessing, remove that parameter and add workflow.limitations with omittedParameters and a specific reason.
|
|
822
|
+
const compilerDecisionGuidance = `\n\nUse the verifier's semantic evidence, not a requirement for 100% of the original candidate inputs. If the core intent works but a secondary parameter cannot be supported from the recording and live evidence without guessing, remove that parameter and add workflow.limitations with omittedParameters and a specific reason. Treat emitsTokens as downstream compatibility targets, not automatic definitions of producer success: when useful core records lack an optional consumer token, preserve the records and supported input classes, document the missing output and affected consumers, and leave those consumers limited/unverified. Narrow an input class only when its core results are unusable or misleading. Remove claims that directly depend on an omitted feature, but do not erase valid producer results solely to make a downstream chain total. Do not use a limitation to hide a broken core tool, and do not keep a known broken or ignored public input.`;
|
|
721
823
|
const inconclusiveDecisionGuidance = pendingInconclusive
|
|
722
|
-
?
|
|
824
|
+
? semanticVerificationCycles >= maxVerificationCycles &&
|
|
825
|
+
pendingInconclusive.report.completedReview
|
|
826
|
+
? '\n\nThe final allowed live semantic review was infrastructure-only inconclusive. Review its evidence and log. If they expose a tool defect, do not attempt to change the artifact after the review cap. If they expose no tool-level defect and the unchanged deterministic artifact should ship explicitly unverified, call done again without changing compile artifacts, set accept_inconclusive=true, and provide your concrete reasoning in inconclusive_reason. This compiler decision is persisted and does not mint liveVerified.'
|
|
827
|
+
: '\n\nThe live verifier was infrastructure-only inconclusive. Review its evidence and log. Fix any exposed tool defect, or call done normally to request another independent review. Explicit unverified acceptance is available only after the semantic review limit is reached.'
|
|
723
828
|
: '';
|
|
724
829
|
return {
|
|
725
830
|
isError: true,
|