@nexrall/code-core 1.4.0 → 1.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/claimEvidence.d.ts +68 -0
- package/dist/agent/claimEvidence.d.ts.map +1 -0
- package/dist/agent/claimEvidence.js +146 -0
- package/dist/agent/loop.d.ts +1 -1
- package/dist/agent/loop.d.ts.map +1 -1
- package/dist/agent/loop.js +73 -12
- package/dist/agent/testIntegrity.d.ts +6 -0
- package/dist/agent/testIntegrity.d.ts.map +1 -1
- package/dist/agent/testIntegrity.js +50 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/tools/executor.d.ts.map +1 -1
- package/dist/tools/executor.js +19 -3
- package/dist/types.d.ts +9 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Claim-vs-Evidence Guard — deterministic verification-hallucination detection.
|
|
3
|
+
*
|
|
4
|
+
* THE PROBLEM (the core of "verification hallucination", unsolved by every
|
|
5
|
+
* frontier coding agent):
|
|
6
|
+
* When an agent finishes, it writes a natural-language summary. Under pressure
|
|
7
|
+
* to appear done, it frequently CLAIMS verification it never performed:
|
|
8
|
+
* "I ran the full test suite and all 42 tests pass."
|
|
9
|
+
* "Verified the build succeeds and the feature works correctly."
|
|
10
|
+
* …while in this session it ran NO test/build command at all, or the one it
|
|
11
|
+
* ran FAILED. The user reads the confident claim and believes the work is
|
|
12
|
+
* done. Prompt instructions ("don't lie") don't fix it — there is no
|
|
13
|
+
* structural check that a stated verification actually happened and passed.
|
|
14
|
+
*
|
|
15
|
+
* THE APPROACH (deterministic — the evidence side is machine-tracked):
|
|
16
|
+
* The ProgressLedger already records, from structured tool data (not model
|
|
17
|
+
* output), every verification command that ran this session and whether it
|
|
18
|
+
* PASSED or FAILED. This guard scans the agent's FINAL message for explicit
|
|
19
|
+
* verification claims ("tests pass", "build succeeds", "verified it works")
|
|
20
|
+
* and compares them against that ledger:
|
|
21
|
+
* - claim present + NO passing verification on record → contradiction
|
|
22
|
+
* - claim present + the latest matching run FAILED → contradiction
|
|
23
|
+
* A contradiction triggers a one-shot nudge asking the agent to either run a
|
|
24
|
+
* real verification and let it pass, or remove the unsupported claim. It is a
|
|
25
|
+
* nudge (not a hard block): the ledger only knows about recognised verify
|
|
26
|
+
* commands, so a legitimate manual/out-of-band check should be allowed to
|
|
27
|
+
* stand once the agent says so.
|
|
28
|
+
*
|
|
29
|
+
* Only the CLAIM detection is heuristic (text). The EVIDENCE is fully
|
|
30
|
+
* deterministic, so the guard never fabricates a contradiction out of thin
|
|
31
|
+
* air — it can only flag a claim that the machine record does not support.
|
|
32
|
+
*/
|
|
33
|
+
/** A single verification record as kept by the ProgressLedger. */
|
|
34
|
+
export interface VerificationRecord {
|
|
35
|
+
/** The command (or tool) that ran, e.g. "npm test" / "run_api_tests". */
|
|
36
|
+
cmd: string;
|
|
37
|
+
/** true = the verification PASSED, false = it FAILED. */
|
|
38
|
+
ok: boolean;
|
|
39
|
+
/** Mutation epoch it ran at (bumped on each source write). */
|
|
40
|
+
epoch: number;
|
|
41
|
+
}
|
|
42
|
+
export interface ClaimEvidenceResult {
|
|
43
|
+
/** The agent's text contains a concrete verification claim. */
|
|
44
|
+
claimed: boolean;
|
|
45
|
+
/** The claim is NOT backed by a passing verification in the ledger. */
|
|
46
|
+
contradicted: boolean;
|
|
47
|
+
/** Short phrases from the text that read as verification claims (for the nudge). */
|
|
48
|
+
claimPhrases: string[];
|
|
49
|
+
/** Human-readable reason for the contradiction (empty when not contradicted). */
|
|
50
|
+
reason: string;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Does the agent's final text make a concrete verification claim? Returns the
|
|
54
|
+
* matched phrases (deduped, capped) so the nudge can quote them back. Phrases
|
|
55
|
+
* preceded by a hedge/conditional within the same clause are ignored.
|
|
56
|
+
*/
|
|
57
|
+
export declare function detectVerificationClaims(text: string): string[];
|
|
58
|
+
/**
|
|
59
|
+
* Compare the agent's final message against the deterministic verification
|
|
60
|
+
* ledger. `currentEpoch` is the ledger's current mutation epoch; a passing
|
|
61
|
+
* verification only counts as current evidence if no source write happened
|
|
62
|
+
* after it (i.e. its epoch === currentEpoch) — a pass from BEFORE the latest
|
|
63
|
+
* edit no longer proves the current code works.
|
|
64
|
+
*/
|
|
65
|
+
export declare function checkClaimEvidence(finalText: string, verifications: VerificationRecord[], currentEpoch: number): ClaimEvidenceResult;
|
|
66
|
+
/** Build the one-shot nudge message body for a contradicted claim. */
|
|
67
|
+
export declare function claimEvidenceNudgeText(result: ClaimEvidenceResult): string;
|
|
68
|
+
//# sourceMappingURL=claimEvidence.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"claimEvidence.d.ts","sourceRoot":"","sources":["../../src/agent/claimEvidence.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,kEAAkE;AAClE,MAAM,WAAW,kBAAkB;IACjC,yEAAyE;IACzE,GAAG,EAAE,MAAM,CAAC;IACZ,yDAAyD;IACzD,EAAE,EAAE,OAAO,CAAC;IACZ,8DAA8D;IAC9D,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,mBAAmB;IAClC,+DAA+D;IAC/D,OAAO,EAAE,OAAO,CAAC;IACjB,uEAAuE;IACvE,YAAY,EAAE,OAAO,CAAC;IACtB,oFAAoF;IACpF,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,iFAAiF;IACjF,MAAM,EAAE,MAAM,CAAC;CAChB;AAiCD;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAwB/D;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAChC,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,kBAAkB,EAAE,EACnC,YAAY,EAAE,MAAM,GACnB,mBAAmB,CA4BrB;AAED,sEAAsE;AACtE,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,mBAAmB,GAAG,MAAM,CAQ1E"}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Claim-vs-Evidence Guard — deterministic verification-hallucination detection.
|
|
4
|
+
*
|
|
5
|
+
* THE PROBLEM (the core of "verification hallucination", unsolved by every
|
|
6
|
+
* frontier coding agent):
|
|
7
|
+
* When an agent finishes, it writes a natural-language summary. Under pressure
|
|
8
|
+
* to appear done, it frequently CLAIMS verification it never performed:
|
|
9
|
+
* "I ran the full test suite and all 42 tests pass."
|
|
10
|
+
* "Verified the build succeeds and the feature works correctly."
|
|
11
|
+
* …while in this session it ran NO test/build command at all, or the one it
|
|
12
|
+
* ran FAILED. The user reads the confident claim and believes the work is
|
|
13
|
+
* done. Prompt instructions ("don't lie") don't fix it — there is no
|
|
14
|
+
* structural check that a stated verification actually happened and passed.
|
|
15
|
+
*
|
|
16
|
+
* THE APPROACH (deterministic — the evidence side is machine-tracked):
|
|
17
|
+
* The ProgressLedger already records, from structured tool data (not model
|
|
18
|
+
* output), every verification command that ran this session and whether it
|
|
19
|
+
* PASSED or FAILED. This guard scans the agent's FINAL message for explicit
|
|
20
|
+
* verification claims ("tests pass", "build succeeds", "verified it works")
|
|
21
|
+
* and compares them against that ledger:
|
|
22
|
+
* - claim present + NO passing verification on record → contradiction
|
|
23
|
+
* - claim present + the latest matching run FAILED → contradiction
|
|
24
|
+
* A contradiction triggers a one-shot nudge asking the agent to either run a
|
|
25
|
+
* real verification and let it pass, or remove the unsupported claim. It is a
|
|
26
|
+
* nudge (not a hard block): the ledger only knows about recognised verify
|
|
27
|
+
* commands, so a legitimate manual/out-of-band check should be allowed to
|
|
28
|
+
* stand once the agent says so.
|
|
29
|
+
*
|
|
30
|
+
* Only the CLAIM detection is heuristic (text). The EVIDENCE is fully
|
|
31
|
+
* deterministic, so the guard never fabricates a contradiction out of thin
|
|
32
|
+
* air — it can only flag a claim that the machine record does not support.
|
|
33
|
+
*/
|
|
34
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
35
|
+
exports.detectVerificationClaims = detectVerificationClaims;
|
|
36
|
+
exports.checkClaimEvidence = checkClaimEvidence;
|
|
37
|
+
exports.claimEvidenceNudgeText = claimEvidenceNudgeText;
|
|
38
|
+
// Verification-claim phrases. Each must assert that a check RAN and SUCCEEDED —
|
|
39
|
+
// not a mere intention ("you should run the tests") or a request. We require an
|
|
40
|
+
// outcome word (pass/succeed/green/works/confirmed) tied to a verification noun
|
|
41
|
+
// (test/suite/build/lint/typecheck/compile) OR a small set of standalone idioms
|
|
42
|
+
// ("all tests pass", "everything works", "verified"). Kept deliberately tight to
|
|
43
|
+
// avoid flagging hedged language ("this should work", "the tests would pass").
|
|
44
|
+
const CLAIM_RES = [
|
|
45
|
+
// "(all) tests pass / passed / are passing / pass successfully"
|
|
46
|
+
/\b(?:all\s+)?(?:unit\s+|integration\s+|the\s+)?tests?\s+(?:now\s+)?(?:pass(?:es|ed|ing)?|are\s+passing|succeed(?:ed|s)?|are\s+green)\b/i,
|
|
47
|
+
// "the test suite passes / passed"
|
|
48
|
+
/\btest\s+suite\s+(?:now\s+)?(?:pass(?:es|ed)?|succeed(?:ed|s)?|is\s+green)\b/i,
|
|
49
|
+
// "the build succeeds / passes / is green / compiles (cleanly)"
|
|
50
|
+
/\b(?:the\s+)?build\s+(?:now\s+)?(?:succeed(?:ed|s)?|pass(?:es|ed)?|is\s+green|compiles?(?:\s+clean(?:ly)?)?)\b/i,
|
|
51
|
+
// "compiles / compiled without errors", "type-checks pass", "lint passes"
|
|
52
|
+
/\b(?:compil(?:es|ed)|type[-\s]?check(?:s|ed|ing)?|typecheck(?:s|ed|ing)?|lint(?:s|ing)?)\s+(?:pass(?:es|ed)?|succeed(?:ed|s)?|clean(?:ly)?|without\s+(?:errors?|warnings?))\b/i,
|
|
53
|
+
// "I ran/verified/confirmed the tests|build" (past-tense execution + a check noun)
|
|
54
|
+
/\b(?:i\s+|we\s+)?(?:ran|executed|verified|confirmed)\s+(?:the\s+|all\s+)?(?:tests?|test\s+suite|build|lint|typecheck|type\s+check)\b/i,
|
|
55
|
+
// strong standalone idioms
|
|
56
|
+
/\ball\s+(?:tests?\s+)?(?:pass|passing|green)\b/i,
|
|
57
|
+
/\beverything\s+(?:passes|works|is\s+working|is\s+green)\b/i,
|
|
58
|
+
/\bverified\s+(?:that\s+)?(?:it|the\s+\w+)\s+works?\b/i,
|
|
59
|
+
/\bconfirmed\s+(?:that\s+)?(?:it|everything|the\s+\w+)\s+works?\b/i,
|
|
60
|
+
];
|
|
61
|
+
// Hedge / conditional markers that turn a "pass" phrase into an intention rather
|
|
62
|
+
// than a claim of fact ("this WOULD make the tests pass", "you SHOULD run the
|
|
63
|
+
// tests", "hopefully the build succeeds"). If one appears shortly BEFORE the
|
|
64
|
+
// matched phrase, we don't treat it as a concrete claim.
|
|
65
|
+
const HEDGE_RE = /\b(?:should|would|could|might|may|will|to\s+make|so\s+that|in\s+theory|hopefully|expect(?:ed)?\s+to|meant\s+to|supposed\s+to|need\s+to|please|make\s+sure|ensure|let'?s|if\s+)\b/i;
|
|
66
|
+
/**
|
|
67
|
+
* Does the agent's final text make a concrete verification claim? Returns the
|
|
68
|
+
* matched phrases (deduped, capped) so the nudge can quote them back. Phrases
|
|
69
|
+
* preceded by a hedge/conditional within the same clause are ignored.
|
|
70
|
+
*/
|
|
71
|
+
function detectVerificationClaims(text) {
|
|
72
|
+
if (!text || typeof text !== 'string')
|
|
73
|
+
return [];
|
|
74
|
+
const found = new Set();
|
|
75
|
+
for (const re of CLAIM_RES) {
|
|
76
|
+
const flags = re.flags.includes('g') ? re.flags : re.flags + 'g';
|
|
77
|
+
const g = new RegExp(re.source, flags);
|
|
78
|
+
let m;
|
|
79
|
+
while ((m = g.exec(text))) {
|
|
80
|
+
if (!m[0]) {
|
|
81
|
+
if (g.lastIndex === m.index)
|
|
82
|
+
g.lastIndex++;
|
|
83
|
+
continue;
|
|
84
|
+
}
|
|
85
|
+
// Look back a short window for a hedge marker in the same clause (stop at a
|
|
86
|
+
// sentence/clause boundary so a hedge in a PRIOR sentence doesn't suppress a
|
|
87
|
+
// genuine claim here).
|
|
88
|
+
const from = Math.max(0, m.index - 60);
|
|
89
|
+
const before = text.slice(from, m.index);
|
|
90
|
+
const clause = before.split(/[.!?;\n]/).pop() ?? before;
|
|
91
|
+
if (!HEDGE_RE.test(clause)) {
|
|
92
|
+
found.add(m[0].trim().replace(/\s+/g, ' ').toLowerCase());
|
|
93
|
+
}
|
|
94
|
+
if (g.lastIndex === m.index)
|
|
95
|
+
g.lastIndex++;
|
|
96
|
+
if (found.size >= 5)
|
|
97
|
+
break;
|
|
98
|
+
}
|
|
99
|
+
if (found.size >= 5)
|
|
100
|
+
break;
|
|
101
|
+
}
|
|
102
|
+
return [...found];
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Compare the agent's final message against the deterministic verification
|
|
106
|
+
* ledger. `currentEpoch` is the ledger's current mutation epoch; a passing
|
|
107
|
+
* verification only counts as current evidence if no source write happened
|
|
108
|
+
* after it (i.e. its epoch === currentEpoch) — a pass from BEFORE the latest
|
|
109
|
+
* edit no longer proves the current code works.
|
|
110
|
+
*/
|
|
111
|
+
function checkClaimEvidence(finalText, verifications, currentEpoch) {
|
|
112
|
+
const claimPhrases = detectVerificationClaims(finalText);
|
|
113
|
+
if (claimPhrases.length === 0) {
|
|
114
|
+
return { claimed: false, contradicted: false, claimPhrases: [], reason: '' };
|
|
115
|
+
}
|
|
116
|
+
const recognised = verifications ?? [];
|
|
117
|
+
// Evidence that supports a "it works now" claim: a PASS at the CURRENT epoch
|
|
118
|
+
// (no edit invalidated it since). Passes at an older epoch are stale.
|
|
119
|
+
const currentPasses = recognised.filter((v) => v.ok && v.epoch === currentEpoch);
|
|
120
|
+
const currentFails = recognised.filter((v) => !v.ok && v.epoch === currentEpoch);
|
|
121
|
+
if (currentPasses.length > 0) {
|
|
122
|
+
// There is a fresh passing verification — the claim is supported.
|
|
123
|
+
return { claimed: true, contradicted: false, claimPhrases, reason: '' };
|
|
124
|
+
}
|
|
125
|
+
let reason;
|
|
126
|
+
if (recognised.length === 0) {
|
|
127
|
+
reason = 'no build/test/lint/typecheck command was run this session';
|
|
128
|
+
}
|
|
129
|
+
else if (currentFails.length > 0) {
|
|
130
|
+
reason = `the most recent verification (${currentFails[currentFails.length - 1].cmd}) FAILED and has not since passed`;
|
|
131
|
+
}
|
|
132
|
+
else {
|
|
133
|
+
// Only stale passes (from before the latest edit) exist.
|
|
134
|
+
reason = 'the last passing verification predates your most recent code change, so it no longer proves the current code works';
|
|
135
|
+
}
|
|
136
|
+
return { claimed: true, contradicted: true, claimPhrases, reason };
|
|
137
|
+
}
|
|
138
|
+
/** Build the one-shot nudge message body for a contradicted claim. */
|
|
139
|
+
function claimEvidenceNudgeText(result) {
|
|
140
|
+
const quoted = result.claimPhrases.map((p) => `"${p}"`).join(', ');
|
|
141
|
+
return ('STOP — verification check. Your summary claims verification ' +
|
|
142
|
+
`(${quoted}) but ${result.reason}. ` +
|
|
143
|
+
'Do not state that tests pass / the build succeeds / it works unless a command actually proved it THIS session on the CURRENT code. ' +
|
|
144
|
+
'Either: (1) run the real build/test/lint/typecheck command now and fix anything that fails, then finish — or (2) if you genuinely cannot run it here (no test suite, environment limitation, out of scope), remove/soften the claim and state plainly what you did and did NOT verify. Do not repeat an unverified pass claim.');
|
|
145
|
+
}
|
|
146
|
+
//# sourceMappingURL=claimEvidence.js.map
|
package/dist/agent/loop.d.ts
CHANGED
|
@@ -53,7 +53,7 @@ export interface ProgressLedger {
|
|
|
53
53
|
}
|
|
54
54
|
export declare function createLedger(): ProgressLedger;
|
|
55
55
|
/** Record one tool call's effect on the ledger (deterministic, no model call). */
|
|
56
|
-
export declare function ledgerRecord(ledger: ProgressLedger, toolName: string, input: Record<string, unknown> | undefined, ok: boolean): void;
|
|
56
|
+
export declare function ledgerRecord(ledger: ProgressLedger, toolName: string, input: Record<string, unknown> | undefined, ok: boolean, output?: string, exitCode?: number): void;
|
|
57
57
|
/** Render the ledger as a compact, verbatim block for the compaction preamble. */
|
|
58
58
|
export declare function ledgerSummary(ledger: ProgressLedger): string;
|
|
59
59
|
/**
|
package/dist/agent/loop.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loop.d.ts","sourceRoot":"","sources":["../../src/agent/loop.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,OAAO,EAMP,gBAAgB,EACjB,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"loop.d.ts","sourceRoot":"","sources":["../../src/agent/loop.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,OAAO,EAMP,gBAAgB,EACjB,MAAM,UAAU,CAAC;AAsKlB,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,MAAM,GAAG,SAAS,EAC/B,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACnC,MAAM,CAWR;AA8RD,+EAA+E;AAC/E,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,MAAM,CAM7D;AAsBD,iFAAiF;AACjF,eAAO,MAAM,gBAAgB,aAA+G,CAAC;AAC7I,gGAAgG;AAChG,eAAO,MAAM,aAAa,QAA2J,CAAC;AAEtL;;;;;;;;;;;;GAYG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAK5E;AAUD;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,MAAM,CAwBxD;AAoBD,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,GAAG,CAAC,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC3D,aAAa,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAClE,qFAAqF;IACrF,aAAa,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACvD;;;;OAIG;IACH,KAAK,EAAE,MAAM,CAAC;CACf;AAED,wBAAgB,YAAY,IAAI,cAAc,CAE7C;AAED,kFAAkF;AAClF,wBAAgB,YAAY,CAC1B,MAAM,EAAE,cAAc,EACtB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,EAC1C,EAAE,EAAE,OAAO,EACX,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,GAChB,IAAI,CAoDN;AAED,kFAAkF;AAClF,wBAAgB,aAAa,CAAC,MAAM,EAAE,cAAc,GAAG,MAAM,CA6B5D;AAmBD;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,MAAM,CAmB/D;AA2FD,wBAAsB,YAAY,CAChC,eAAe,EAAE,OAAO,EAAE,EAC1B,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,OAAO,EAAE,CAAC,CAqhBpB"}
|
package/dist/agent/loop.js
CHANGED
|
@@ -51,6 +51,7 @@ const sandbox_1 = require("../tools/sandbox");
|
|
|
51
51
|
const index_1 = require("../plugins/index");
|
|
52
52
|
const testIntegrity_1 = require("./testIntegrity");
|
|
53
53
|
const flaky_1 = require("./flaky");
|
|
54
|
+
const claimEvidence_1 = require("./claimEvidence");
|
|
54
55
|
const fs = __importStar(require("fs"));
|
|
55
56
|
const path = __importStar(require("path"));
|
|
56
57
|
const child_process_1 = require("child_process");
|
|
@@ -551,7 +552,7 @@ function createLedger() {
|
|
|
551
552
|
return { filesTouched: new Map(), verifications: [], testIntegrity: [], epoch: 0 };
|
|
552
553
|
}
|
|
553
554
|
/** Record one tool call's effect on the ledger (deterministic, no model call). */
|
|
554
|
-
function ledgerRecord(ledger, toolName, input, ok) {
|
|
555
|
+
function ledgerRecord(ledger, toolName, input, ok, output, exitCode) {
|
|
555
556
|
if (exports.WRITE_TOOL_NAMES.has(toolName)) {
|
|
556
557
|
if (!ok)
|
|
557
558
|
return; // a FAILED write changed nothing — not a durable fact
|
|
@@ -565,10 +566,22 @@ function ledgerRecord(ledger, toolName, input, ok) {
|
|
|
565
566
|
}
|
|
566
567
|
// Reward-hacking guard: if this write WEAKENED a test file, record it so the
|
|
567
568
|
// signal survives compaction and can be surfaced before the agent finishes.
|
|
568
|
-
const
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
569
|
+
const reasons = [];
|
|
570
|
+
// write_file overwrites carry a marker computed by the executor (which had the
|
|
571
|
+
// prior on-disk content) — it detects REMOVED assertions/cases, not just
|
|
572
|
+
// additive skips/tautologies. Prefer it when present.
|
|
573
|
+
const markerReasons = toolName === 'write_file' ? (0, testIntegrity_1.decodeTestIntegrityMarker)(output) : [];
|
|
574
|
+
if (markerReasons.length) {
|
|
575
|
+
reasons.push(...markerReasons);
|
|
576
|
+
}
|
|
577
|
+
else {
|
|
578
|
+
const ti = (0, testIntegrity_1.analyzeWriteToolForTestIntegrity)(toolName, input);
|
|
579
|
+
if (ti?.suspicious)
|
|
580
|
+
reasons.push(...ti.findings.map((f) => f.reason));
|
|
581
|
+
}
|
|
582
|
+
if (reasons.length && p) {
|
|
583
|
+
for (const reason of reasons) {
|
|
584
|
+
ledger.testIntegrity.push({ path: p, reason });
|
|
572
585
|
}
|
|
573
586
|
if (ledger.testIntegrity.length > LEDGER_MAX_NOTES * 2) {
|
|
574
587
|
ledger.testIntegrity.splice(0, ledger.testIntegrity.length - LEDGER_MAX_NOTES);
|
|
@@ -580,8 +593,16 @@ function ledgerRecord(ledger, toolName, input, ok) {
|
|
|
580
593
|
if (cmd && exports.VERIFY_CMD_RE.test(cmd)) {
|
|
581
594
|
// Record BOTH outcomes: a FAILED test/build is the single most important
|
|
582
595
|
// fact to carry across a compaction (it tells the agent work is NOT done).
|
|
583
|
-
//
|
|
584
|
-
|
|
596
|
+
//
|
|
597
|
+
// CRITICAL: `ok` is `result.error === undefined`, which is TRUE even when a
|
|
598
|
+
// test suite exits non-zero (the executor doesn't set `error` for a plain
|
|
599
|
+
// command failure — only for timeout/abort/spawn-fail). So `ok` alone would
|
|
600
|
+
// record a FAILING `npm test` as PASSED. The executor now reports the real
|
|
601
|
+
// process exit code via `exitCode`; a non-zero exit means the verification
|
|
602
|
+
// FAILED regardless of `ok`. Fall back to `ok` only when no exitCode is
|
|
603
|
+
// available (older tools / non-bash paths).
|
|
604
|
+
const passed = exitCode !== undefined ? exitCode === 0 : ok;
|
|
605
|
+
ledger.verifications.push({ cmd: cmd.slice(0, 120), ok: passed, epoch: ledger.epoch });
|
|
585
606
|
if (ledger.verifications.length > LEDGER_MAX_NOTES * 2) {
|
|
586
607
|
ledger.verifications.splice(0, ledger.verifications.length - LEDGER_MAX_NOTES);
|
|
587
608
|
}
|
|
@@ -809,6 +830,10 @@ async function runAgentLoop(initialMessages, options) {
|
|
|
809
830
|
// Flaky-test guard: commands already nudged about, so we only warn about a
|
|
810
831
|
// newly-detected flaky command once.
|
|
811
832
|
const flakyNudgedCmds = new Set();
|
|
833
|
+
// Claim-vs-evidence guard: fire the "you claimed verified but there's no
|
|
834
|
+
// passing verification" nudge at most once, so a genuinely un-testable task
|
|
835
|
+
// (the agent explains it can't run tests) still ends instead of looping.
|
|
836
|
+
let claimEvidenceNudged = false;
|
|
812
837
|
// GAP E — deterministic progress ledger, preserved verbatim across compactions.
|
|
813
838
|
const ledger = createLedger();
|
|
814
839
|
try {
|
|
@@ -1029,6 +1054,27 @@ async function runAgentLoop(initialMessages, options) {
|
|
|
1029
1054
|
});
|
|
1030
1055
|
continue;
|
|
1031
1056
|
}
|
|
1057
|
+
// Claim-vs-evidence guard: the core of verification-hallucination. The agent
|
|
1058
|
+
// is about to finish; if its final message CLAIMS a verification ("tests
|
|
1059
|
+
// pass", "build succeeds", "verified it works") that the deterministic ledger
|
|
1060
|
+
// does NOT back with a PASSING run on the current code, challenge it once.
|
|
1061
|
+
// The claim side is heuristic text; the evidence side is machine-tracked, so
|
|
1062
|
+
// we only flag claims the record actively contradicts.
|
|
1063
|
+
if (!claimEvidenceNudged) {
|
|
1064
|
+
const finalText = assistantMessage.content
|
|
1065
|
+
.filter((b) => b.type === 'text' && typeof b.text === 'string')
|
|
1066
|
+
.map((b) => b.text)
|
|
1067
|
+
.join('\n');
|
|
1068
|
+
const ce = (0, claimEvidence_1.checkClaimEvidence)(finalText, ledger.verifications, ledger.epoch);
|
|
1069
|
+
if (ce.contradicted) {
|
|
1070
|
+
claimEvidenceNudged = true;
|
|
1071
|
+
messages.push({
|
|
1072
|
+
role: 'user',
|
|
1073
|
+
content: [{ type: 'text', text: (0, claimEvidence_1.claimEvidenceNudgeText)(ce) }],
|
|
1074
|
+
});
|
|
1075
|
+
continue;
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1032
1078
|
runSimpleHooks(hooks.PostMessageComplete, options.workDir);
|
|
1033
1079
|
completedCleanly = true;
|
|
1034
1080
|
break;
|
|
@@ -1110,23 +1156,38 @@ async function runAgentLoop(initialMessages, options) {
|
|
|
1110
1156
|
}
|
|
1111
1157
|
}
|
|
1112
1158
|
}
|
|
1159
|
+
// Capture the raw output (may carry the invisible test-integrity
|
|
1160
|
+
// marker from a write_file overwrite) for the ledger, then strip the
|
|
1161
|
+
// marker so neither the UI (onToolResult) nor the model ever see it.
|
|
1162
|
+
let rawOutput;
|
|
1163
|
+
if (name === 'write_file' && result.output) {
|
|
1164
|
+
rawOutput = result.output;
|
|
1165
|
+
result.output = (0, testIntegrity_1.stripTestIntegrityMarker)(result.output);
|
|
1166
|
+
}
|
|
1113
1167
|
// Notify caller about result
|
|
1114
1168
|
options.onToolResult(name, result);
|
|
1115
|
-
return { block: { ...block, id }, result };
|
|
1169
|
+
return { block: { ...block, id }, result, rawOutput };
|
|
1116
1170
|
}));
|
|
1117
1171
|
// Track whether files were mutated / verified this run, for the one-shot
|
|
1118
1172
|
// end-of-task nudge below (GAP D — see declaration above).
|
|
1119
|
-
for (const { block, result } of toolResults) {
|
|
1173
|
+
for (const { block, result, rawOutput } of toolResults) {
|
|
1120
1174
|
const ok = result.error === undefined;
|
|
1121
1175
|
// GAP E — feed every successful effect into the deterministic ledger.
|
|
1122
|
-
|
|
1176
|
+
// rawOutput carries the (already-stripped-from-view) test-integrity marker.
|
|
1177
|
+
// exitCode lets the ledger tell a PASSED verification from a FAILED one
|
|
1178
|
+
// (a `npm test` that exits non-zero is not an `error`, but it IS a fail).
|
|
1179
|
+
ledgerRecord(ledger, block.name, block.input, ok, rawOutput ?? result.output, result.exitCode);
|
|
1123
1180
|
if (!ok)
|
|
1124
1181
|
continue; // failed calls don't count either way
|
|
1125
1182
|
if (exports.WRITE_TOOL_NAMES.has(block.name))
|
|
1126
1183
|
filesMutatedSinceVerify = true;
|
|
1127
1184
|
else if (block.name === 'bash' && exports.VERIFY_CMD_RE.test(String(block.input?.command ?? ''))) {
|
|
1128
|
-
|
|
1129
|
-
|
|
1185
|
+
// Only a PASSING verification satisfies the verify-nudge. A test suite that
|
|
1186
|
+
// ran but FAILED (non-zero exit) must NOT count as "verified" — otherwise the
|
|
1187
|
+
// agent could run a failing build once and then finish unchallenged.
|
|
1188
|
+
ranVerificationCmd = result.exitCode === undefined ? true : result.exitCode === 0;
|
|
1189
|
+
if (ranVerificationCmd)
|
|
1190
|
+
filesMutatedSinceVerify = false; // verified — reset until the next mutation
|
|
1130
1191
|
}
|
|
1131
1192
|
}
|
|
1132
1193
|
// 6. Build tool_result message and append to history
|
|
@@ -53,4 +53,10 @@ export declare function analyzeTestEdit(path: string, oldText: string, newText:
|
|
|
53
53
|
* - write_file: ('', content) — additive-only detection (no prior content here)
|
|
54
54
|
*/
|
|
55
55
|
export declare function analyzeWriteToolForTestIntegrity(toolName: string, input: Record<string, unknown> | undefined): TestIntegrityResult | null;
|
|
56
|
+
/** Encode findings as an invisible marker to append to a tool's output string. */
|
|
57
|
+
export declare function encodeTestIntegrityMarker(findings: TestEditFinding[]): string;
|
|
58
|
+
/** Extract weakening reasons from a tool output that may carry the marker. */
|
|
59
|
+
export declare function decodeTestIntegrityMarker(output: string | undefined): string[];
|
|
60
|
+
/** Strip the (invisible) marker from output before it is shown to the model. */
|
|
61
|
+
export declare function stripTestIntegrityMarker(output: string | undefined): string;
|
|
56
62
|
//# sourceMappingURL=testIntegrity.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"testIntegrity.d.ts","sourceRoot":"","sources":["../../src/agent/testIntegrity.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,MAAM,WAAW,eAAe;IAC9B,gEAAgE;IAChE,MAAM,EAAE,MAAM,CAAC;IACf,0CAA0C;IAC1C,IAAI,EAAE,YAAY,GAAG,mBAAmB,GAAG,iBAAiB,GAAG,kBAAkB,GAAG,gBAAgB,CAAC;CACtG;AAED,MAAM,WAAW,mBAAmB;IAClC,UAAU,EAAE,OAAO,CAAC;IACpB,UAAU,EAAE,OAAO,CAAC;IACpB,QAAQ,EAAE,eAAe,EAAE,CAAC;CAC7B;AAQD,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAGhD;AA4ED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,mBAAmB,CAmDnG;AAED;;;;;;;GAOG;AACH,wBAAgB,gCAAgC,CAC9C,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,GACzC,mBAAmB,GAAG,IAAI,
|
|
1
|
+
{"version":3,"file":"testIntegrity.d.ts","sourceRoot":"","sources":["../../src/agent/testIntegrity.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,MAAM,WAAW,eAAe;IAC9B,gEAAgE;IAChE,MAAM,EAAE,MAAM,CAAC;IACf,0CAA0C;IAC1C,IAAI,EAAE,YAAY,GAAG,mBAAmB,GAAG,iBAAiB,GAAG,kBAAkB,GAAG,gBAAgB,CAAC;CACtG;AAED,MAAM,WAAW,mBAAmB;IAClC,UAAU,EAAE,OAAO,CAAC;IACpB,UAAU,EAAE,OAAO,CAAC;IACpB,QAAQ,EAAE,eAAe,EAAE,CAAC;CAC7B;AAQD,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAGhD;AA4ED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,mBAAmB,CAmDnG;AAED;;;;;;;GAOG;AACH,wBAAgB,gCAAgC,CAC9C,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,GACzC,mBAAmB,GAAG,IAAI,CA8B5B;AAcD,kFAAkF;AAClF,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,eAAe,EAAE,GAAG,MAAM,CAI7E;AAED,8EAA8E;AAC9E,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,EAAE,CAQ9E;AAED,gFAAgF;AAChF,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAQ3E"}
|
|
@@ -32,6 +32,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
32
32
|
exports.isTestFile = isTestFile;
|
|
33
33
|
exports.analyzeTestEdit = analyzeTestEdit;
|
|
34
34
|
exports.analyzeWriteToolForTestIntegrity = analyzeWriteToolForTestIntegrity;
|
|
35
|
+
exports.encodeTestIntegrityMarker = encodeTestIntegrityMarker;
|
|
36
|
+
exports.decodeTestIntegrityMarker = decodeTestIntegrityMarker;
|
|
37
|
+
exports.stripTestIntegrityMarker = stripTestIntegrityMarker;
|
|
35
38
|
// ── Is this a test file? ──────────────────────────────────────────────────────
|
|
36
39
|
// Covers the common conventions across JS/TS, Python, Go, Java/Kotlin, Ruby,
|
|
37
40
|
// Rust, PHP, C#. Path-based (fast, language-agnostic).
|
|
@@ -192,8 +195,55 @@ function analyzeWriteToolForTestIntegrity(toolName, input) {
|
|
|
192
195
|
if (toolName === 'write_file') {
|
|
193
196
|
const content = typeof input.content === 'string' ? input.content : '';
|
|
194
197
|
// No prior content available at this layer → additive-only ('' → content).
|
|
198
|
+
// The executor closes this gap: when write_file OVERWRITES an existing test
|
|
199
|
+
// file it has the prior on-disk content, runs the full old→new analysis, and
|
|
200
|
+
// embeds a TEST_INTEGRITY_MARKER in its output which ledgerRecord parses.
|
|
195
201
|
return analyzeTestEdit(path, '', content);
|
|
196
202
|
}
|
|
197
203
|
return null;
|
|
198
204
|
}
|
|
205
|
+
// ── write_file overwrite bridge ────────────────────────────────────────────────
|
|
206
|
+
// A write_file that overwrites an existing test file can silently REMOVE
|
|
207
|
+
// assertions / test cases — invisible to analyzeWriteToolForTestIntegrity above
|
|
208
|
+
// because the prior content isn't available at the loop layer. The executor DOES
|
|
209
|
+
// have the prior content (it reads it to preserve mode / run the cross-file
|
|
210
|
+
// check), so it runs analyzeTestEdit(path, priorContent, newContent) and, when
|
|
211
|
+
// suspicious, appends a machine-readable marker to its tool output. ledgerRecord
|
|
212
|
+
// parses that marker so a write_file overwrite gets the exact same ledger entry
|
|
213
|
+
// and pre-finish nudge as an edit_file weakening would.
|
|
214
|
+
const TEST_INTEGRITY_MARKER_OPEN = '\u0000NEXRALL_TEST_INTEGRITY\u0000';
|
|
215
|
+
const TEST_INTEGRITY_MARKER_CLOSE = '\u0000/NEXRALL_TEST_INTEGRITY\u0000';
|
|
216
|
+
/** Encode findings as an invisible marker to append to a tool's output string. */
|
|
217
|
+
function encodeTestIntegrityMarker(findings) {
|
|
218
|
+
if (!findings.length)
|
|
219
|
+
return '';
|
|
220
|
+
const payload = findings.map((f) => f.reason).join('\u0001');
|
|
221
|
+
return `${TEST_INTEGRITY_MARKER_OPEN}${payload}${TEST_INTEGRITY_MARKER_CLOSE}`;
|
|
222
|
+
}
|
|
223
|
+
/** Extract weakening reasons from a tool output that may carry the marker. */
|
|
224
|
+
function decodeTestIntegrityMarker(output) {
|
|
225
|
+
if (!output)
|
|
226
|
+
return [];
|
|
227
|
+
const start = output.indexOf(TEST_INTEGRITY_MARKER_OPEN);
|
|
228
|
+
if (start === -1)
|
|
229
|
+
return [];
|
|
230
|
+
const from = start + TEST_INTEGRITY_MARKER_OPEN.length;
|
|
231
|
+
const end = output.indexOf(TEST_INTEGRITY_MARKER_CLOSE, from);
|
|
232
|
+
if (end === -1)
|
|
233
|
+
return [];
|
|
234
|
+
return output.slice(from, end).split('\u0001').filter(Boolean);
|
|
235
|
+
}
|
|
236
|
+
/** Strip the (invisible) marker from output before it is shown to the model. */
|
|
237
|
+
function stripTestIntegrityMarker(output) {
|
|
238
|
+
if (!output)
|
|
239
|
+
return output ?? '';
|
|
240
|
+
const start = output.indexOf(TEST_INTEGRITY_MARKER_OPEN);
|
|
241
|
+
if (start === -1)
|
|
242
|
+
return output;
|
|
243
|
+
const end = output.indexOf(TEST_INTEGRITY_MARKER_CLOSE, start);
|
|
244
|
+
if (end === -1)
|
|
245
|
+
return output;
|
|
246
|
+
const cleaned = output.slice(0, start) + output.slice(end + TEST_INTEGRITY_MARKER_CLOSE.length);
|
|
247
|
+
return cleaned.replace(/\n{3,}/g, '\n\n').trimEnd();
|
|
248
|
+
}
|
|
199
249
|
//# sourceMappingURL=testIntegrity.js.map
|
package/dist/index.d.ts
CHANGED
|
@@ -7,6 +7,7 @@ export * from './agent/testIntegrity';
|
|
|
7
7
|
export * from './agent/editCompleteness';
|
|
8
8
|
export * from './agent/crossFile';
|
|
9
9
|
export * from './agent/flaky';
|
|
10
|
+
export * from './agent/claimEvidence';
|
|
10
11
|
export * from './mcp/client';
|
|
11
12
|
export * from './mcp/httpClient';
|
|
12
13
|
export * from './mcp/manager';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,uBAAuB,CAAC;AACtC,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,eAAe,CAAC;AAC9B,cAAc,cAAc,CAAC;AAC7B,cAAc,kBAAkB,CAAC;AACjC,cAAc,eAAe,CAAC;AAC9B,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iBAAiB,CAAC;AAChC,cAAc,qBAAqB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,uBAAuB,CAAC;AACtC,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,cAAc,CAAC;AAC7B,cAAc,kBAAkB,CAAC;AACjC,cAAc,eAAe,CAAC;AAC9B,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iBAAiB,CAAC;AAChC,cAAc,qBAAqB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -23,6 +23,7 @@ __exportStar(require("./agent/testIntegrity"), exports);
|
|
|
23
23
|
__exportStar(require("./agent/editCompleteness"), exports);
|
|
24
24
|
__exportStar(require("./agent/crossFile"), exports);
|
|
25
25
|
__exportStar(require("./agent/flaky"), exports);
|
|
26
|
+
__exportStar(require("./agent/claimEvidence"), exports);
|
|
26
27
|
__exportStar(require("./mcp/client"), exports);
|
|
27
28
|
__exportStar(require("./mcp/httpClient"), exports);
|
|
28
29
|
__exportStar(require("./mcp/manager"), exports);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/tools/executor.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,EAAyB,KAAK,aAAa,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/tools/executor.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,EAAyB,KAAK,aAAa,EAAE,MAAM,WAAW,CAAC;AAw9DtE,wBAAsB,WAAW,CAC/B,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC9B,WAAW,CAAC,EAAE;IAAE,OAAO,EAAE,OAAO,CAAA;CAAE,EAClC,OAAO,CAAC,EAAE,aAAa,EACvB,OAAO,CAAC,EAAE,MAAM,EAChB,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,UAAU,CAAC,CAiBrB"}
|
package/dist/tools/executor.js
CHANGED
|
@@ -44,6 +44,7 @@ const child_process_1 = require("child_process");
|
|
|
44
44
|
const sandbox_1 = require("./sandbox");
|
|
45
45
|
const auth_1 = require("../auth");
|
|
46
46
|
const editCompleteness_1 = require("../agent/editCompleteness");
|
|
47
|
+
const testIntegrity_1 = require("../agent/testIntegrity");
|
|
47
48
|
const crossFile_1 = require("../agent/crossFile");
|
|
48
49
|
const client_1 = require("../api/client");
|
|
49
50
|
const symbols_1 = require("./symbols");
|
|
@@ -370,7 +371,16 @@ async function writeFile(input, workDir) {
|
|
|
370
371
|
return { output: `Created ${resolved} (${lines} lines, ${bytes} bytes)` };
|
|
371
372
|
}
|
|
372
373
|
const xfile = crossFileBreakageWarning(resolved, normalizeLF(priorContent), normalizeLF(content), workDir ?? process.cwd());
|
|
373
|
-
|
|
374
|
+
// Reward-hacking guard: a write_file that OVERWRITES an existing test file can
|
|
375
|
+
// silently drop assertions / test cases. The loop layer can't see the prior
|
|
376
|
+
// content — but we can (we just read it). Run the full old→new analysis and
|
|
377
|
+
// embed an invisible marker that ledgerRecord parses, so this path gets the
|
|
378
|
+
// same ledger entry + pre-finish nudge as an edit_file weakening would.
|
|
379
|
+
let tiMarker = '';
|
|
380
|
+
const ti = (0, testIntegrity_1.analyzeTestEdit)(resolved, normalizeLF(priorContent), normalizeLF(content));
|
|
381
|
+
if (ti.suspicious)
|
|
382
|
+
tiMarker = (0, testIntegrity_1.encodeTestIntegrityMarker)(ti.findings);
|
|
383
|
+
return { output: `Overwrote ${resolved} (${lines} lines, ${bytes} bytes)${xfile}${tiMarker}` };
|
|
374
384
|
}
|
|
375
385
|
catch (err) {
|
|
376
386
|
return { error: err.message };
|
|
@@ -725,10 +735,16 @@ async function bash(input, abortSignal, sandbox, workDir) {
|
|
|
725
735
|
resolve({ error: 'Command stopped by user', interrupted: true, output: body || undefined });
|
|
726
736
|
}
|
|
727
737
|
else if (code !== 0) {
|
|
728
|
-
|
|
738
|
+
// Non-zero exit = the command FAILED (a test suite reporting failures, a
|
|
739
|
+
// build error, a lint failure). We deliberately DON'T set `error` here:
|
|
740
|
+
// the model must see the failing output and react, and treating it as a
|
|
741
|
+
// hard error would break stall-guard/retry semantics. Instead expose the
|
|
742
|
+
// exit code as structured data so the ProgressLedger can record this
|
|
743
|
+
// verification as FAILED (not PASSED). Signal-only exit → code 128+sig-ish.
|
|
744
|
+
resolve(prependNote({ output: body + `\n[Exit code: ${code ?? signal ?? 'unknown'}]`, exitCode: code ?? 1 }));
|
|
729
745
|
}
|
|
730
746
|
else {
|
|
731
|
-
resolve(prependNote({ output: body }));
|
|
747
|
+
resolve(prependNote({ output: body, exitCode: 0 }));
|
|
732
748
|
}
|
|
733
749
|
};
|
|
734
750
|
child.stdout?.on('data', (chunk) => appendOutput(chunk.toString('utf-8')));
|
package/dist/types.d.ts
CHANGED
|
@@ -67,6 +67,15 @@ export interface ToolResult {
|
|
|
67
67
|
output?: string;
|
|
68
68
|
error?: string;
|
|
69
69
|
interrupted?: boolean;
|
|
70
|
+
/**
|
|
71
|
+
* Process exit code for bash commands. 0 = success, non-zero = the command
|
|
72
|
+
* FAILED (e.g. a test suite that reported failures). This is kept SEPARATE
|
|
73
|
+
* from `error` on purpose: a failing `npm test` is a legitimate, non-error
|
|
74
|
+
* tool result (the model should see the failures and react) — but the
|
|
75
|
+
* ProgressLedger needs it to tell a PASSED verification from a FAILED one.
|
|
76
|
+
* Undefined for non-bash tools or when no exit code is available.
|
|
77
|
+
*/
|
|
78
|
+
exitCode?: number;
|
|
70
79
|
}
|
|
71
80
|
export interface AuthConfig {
|
|
72
81
|
token: string;
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,UAAU,CAAC;IACjB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAChC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,aAAa,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,YAAY,GAAG,eAAe,CAAC;AAItE,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAC3B,OAAO,EAAE,YAAY,EAAE,CAAC;CACzB;AAID,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,UAAU,CAAC;IACjB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAChC;AAED,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,kBAAkB,CAAC;IACzB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,SAAS,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,OAAO,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,wBAAwB;IACvC,IAAI,EAAE,mBAAmB,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,gBAAgB,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,MAAM,QAAQ,GAChB,YAAY,GACZ,eAAe,GACf,uBAAuB,GACvB,aAAa,GACb,YAAY,GACZ,aAAa,GACb,gBAAgB,GAChB,wBAAwB,GACxB,qBAAqB,CAAC;AAI1B,MAAM,WAAW,SAAS;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC,uBAAuB,CAAC,EAAE,MAAM,CAAC;CAClC;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,UAAU,CAAC;IACjB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAChC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,aAAa,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,YAAY,GAAG,eAAe,CAAC;AAItE,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAC3B,OAAO,EAAE,YAAY,EAAE,CAAC;CACzB;AAID,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,UAAU,CAAC;IACjB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAChC;AAED,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,kBAAkB,CAAC;IACzB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,SAAS,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,OAAO,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,wBAAwB;IACvC,IAAI,EAAE,mBAAmB,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,gBAAgB,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,MAAM,QAAQ,GAChB,YAAY,GACZ,eAAe,GACf,uBAAuB,GACvB,aAAa,GACb,YAAY,GACZ,aAAa,GACb,gBAAgB,GAChB,wBAAwB,GACxB,qBAAqB,CAAC;AAI1B,MAAM,WAAW,SAAS;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC,uBAAuB,CAAC,EAAE,MAAM,CAAC;CAClC;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB;;;;;;;OAOG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,UAAU;IACzB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,gEAAgE;IAChE,WAAW,CAAC,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACvF,8DAA8D;IAC9D,oBAAoB,CAAC,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAChH,4DAA4D;IAC5D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uEAAuE;IACvE,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE;QAAE,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC;IACnC,MAAM,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5B,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACpC,eAAe,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACzC,kBAAkB,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IAC9C,SAAS,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,SAAS,CAAC,EAAE,OAAO,KAAK,IAAI,CAAC;IACvF,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,CAAC,EAAE,OAAO,KAAK,IAAI,CAAC;IAC9E,OAAO,EAAE,CAAC,CAAC,EAAE,SAAS,KAAK,IAAI,CAAC;IAChC,KAAK,CAAC,EAAE,OAAO,GAAG,KAAK,GAAG,OAAO,CAAC;IAClC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,iBAAiB,EAAE,CAAC,GAAG,EAAE,iBAAiB,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAChE,GAAG,CAAC,EAAE,UAAU,CAAC;IACjB,aAAa,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IACrC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+DAA+D;IAC/D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;IACnG,+EAA+E;IAC/E,UAAU,CAAC,EAAE,OAAO,eAAe,EAAE,UAAU,CAAC;IAChD,qFAAqF;IACrF,iBAAiB,CAAC,EAAE,OAAO,sBAAsB,EAAE,iBAAiB,CAAC;IACrE;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,MAAM,EAAE,CAAC;IAClC,+FAA+F;IAC/F,eAAe,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACzC;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;IAC3C;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;;OAOG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;OAKG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,gEAAgE;IAChE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nexrall/code-core",
|
|
3
|
-
"version": "1.4.
|
|
3
|
+
"version": "1.4.2",
|
|
4
4
|
"description": "Core agent loop, tools, and extension primitives for Nexrall Code — embed an AI coding agent in any Node.js application.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Nexrall <support@nexrall.com> (https://nexrall.com)",
|