@harness-engineering/intelligence 0.2.7 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +263 -1
- package/dist/index.d.ts +263 -1
- package/dist/index.js +419 -27
- package/dist/index.mjs +410 -25
- package/package.json +4 -1
package/dist/index.js
CHANGED
|
@@ -35,16 +35,21 @@ __export(index_exports, {
|
|
|
35
35
|
ExecutionOutcomeConnector: () => ExecutionOutcomeConnector,
|
|
36
36
|
GraphValidator: () => GraphValidator,
|
|
37
37
|
IntelligencePipeline: () => IntelligencePipeline,
|
|
38
|
+
OUTCOME_EVAL_SYSTEM_PROMPT: () => OUTCOME_EVAL_SYSTEM_PROMPT,
|
|
38
39
|
OpenAICompatibleAnalysisProvider: () => OpenAICompatibleAnalysisProvider,
|
|
40
|
+
OutcomeEvaluator: () => OutcomeEvaluator,
|
|
39
41
|
PeslSimulator: () => PeslSimulator,
|
|
40
42
|
buildSpecializationProfile: () => buildSpecializationProfile,
|
|
43
|
+
buildUserPrompt: () => buildUserPrompt2,
|
|
41
44
|
computeExpertiseLevel: () => computeExpertiseLevel,
|
|
42
45
|
computeHistoricalComplexity: () => computeHistoricalComplexity,
|
|
43
46
|
computePersonaEffectiveness: () => computePersonaEffectiveness,
|
|
44
47
|
computeSemanticComplexity: () => computeSemanticComplexity,
|
|
45
48
|
computeSpecialization: () => computeSpecialization,
|
|
46
49
|
computeStructuralComplexity: () => computeStructuralComplexity,
|
|
50
|
+
createCanaryAdapter: () => createCanaryAdapter,
|
|
47
51
|
decayWeight: () => decayWeight,
|
|
52
|
+
deriveAuthority: () => deriveAuthority,
|
|
48
53
|
detectBlindSpots: () => detectBlindSpots,
|
|
49
54
|
enrich: () => enrich,
|
|
50
55
|
githubToRawWorkItem: () => githubToRawWorkItem,
|
|
@@ -54,6 +59,7 @@ __export(index_exports, {
|
|
|
54
59
|
manualToRawWorkItem: () => manualToRawWorkItem,
|
|
55
60
|
recommendPersona: () => recommendPersona,
|
|
56
61
|
refreshProfiles: () => refreshProfiles,
|
|
62
|
+
resolveSection: () => resolveSection,
|
|
57
63
|
runGraphOnlyChecks: () => runGraphOnlyChecks,
|
|
58
64
|
runLlmSimulation: () => runLlmSimulation,
|
|
59
65
|
saveProfiles: () => saveProfiles,
|
|
@@ -61,6 +67,7 @@ __export(index_exports, {
|
|
|
61
67
|
scoreToConcernSignals: () => scoreToConcernSignals,
|
|
62
68
|
temporalSuccessRate: () => temporalSuccessRate,
|
|
63
69
|
toRawWorkItem: () => toRawWorkItem,
|
|
70
|
+
verdictSchema: () => verdictSchema,
|
|
64
71
|
weightedRecommendPersona: () => weightedRecommendPersona
|
|
65
72
|
});
|
|
66
73
|
module.exports = __toCommonJS(index_exports);
|
|
@@ -173,6 +180,105 @@ function manualToRawWorkItem(input) {
|
|
|
173
180
|
};
|
|
174
181
|
}
|
|
175
182
|
|
|
183
|
+
// src/adapters/canary.ts
|
|
184
|
+
var import_zod = require("zod");
|
|
185
|
+
var import_node_child_process = require("child_process");
|
|
186
|
+
var frameworkRecommendationSchema = import_zod.z.object({
|
|
187
|
+
status: import_zod.z.string(),
|
|
188
|
+
test_type: import_zod.z.string(),
|
|
189
|
+
framework: import_zod.z.string(),
|
|
190
|
+
file_extension: import_zod.z.string(),
|
|
191
|
+
reasoning: import_zod.z.array(import_zod.z.string()),
|
|
192
|
+
alternatives: import_zod.z.array(import_zod.z.string())
|
|
193
|
+
});
|
|
194
|
+
var canaryFindingSchema = import_zod.z.object({
|
|
195
|
+
file: import_zod.z.string(),
|
|
196
|
+
line: import_zod.z.number(),
|
|
197
|
+
rule: import_zod.z.string(),
|
|
198
|
+
severity: import_zod.z.string(),
|
|
199
|
+
message: import_zod.z.string(),
|
|
200
|
+
suggestion: import_zod.z.string()
|
|
201
|
+
});
|
|
202
|
+
var canaryFindingsSchema = import_zod.z.array(canaryFindingSchema);
|
|
203
|
+
var EXEC_TIMEOUT_MS = 3e4;
|
|
204
|
+
var EXEC_MAX_BUFFER = 16 * 1024 * 1024;
|
|
205
|
+
var defaultExec = (cmd, args) => new Promise((resolve, reject) => {
|
|
206
|
+
(0, import_node_child_process.execFile)(
|
|
207
|
+
cmd,
|
|
208
|
+
args,
|
|
209
|
+
{ encoding: "utf8", timeout: EXEC_TIMEOUT_MS, maxBuffer: EXEC_MAX_BUFFER },
|
|
210
|
+
(err, stdout) => {
|
|
211
|
+
if (err) {
|
|
212
|
+
reject(err);
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
resolve({ stdout });
|
|
216
|
+
}
|
|
217
|
+
);
|
|
218
|
+
});
|
|
219
|
+
function canaryInvocation(subArgs) {
|
|
220
|
+
return ["canary", subArgs];
|
|
221
|
+
}
|
|
222
|
+
function parseVersion(stdout) {
|
|
223
|
+
return stdout.match(/\d+\.\d+\.\d+/)?.[0];
|
|
224
|
+
}
|
|
225
|
+
function safeJson(stdout) {
|
|
226
|
+
try {
|
|
227
|
+
return JSON.parse(stdout);
|
|
228
|
+
} catch {
|
|
229
|
+
return void 0;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
function degradedRecommendation() {
|
|
233
|
+
return {
|
|
234
|
+
status: "degraded",
|
|
235
|
+
test_type: "",
|
|
236
|
+
framework: "",
|
|
237
|
+
file_extension: "",
|
|
238
|
+
reasoning: [],
|
|
239
|
+
alternatives: []
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
function createCanaryAdapter(exec = defaultExec) {
|
|
243
|
+
async function execCanary(subArgs) {
|
|
244
|
+
const [cmd, args] = canaryInvocation(subArgs);
|
|
245
|
+
try {
|
|
246
|
+
const { stdout } = await exec(cmd, args);
|
|
247
|
+
return { ok: true, stdout };
|
|
248
|
+
} catch (err) {
|
|
249
|
+
const e = err;
|
|
250
|
+
if (e.code === "ENOENT") return { ok: false, reason: "not-installed" };
|
|
251
|
+
if (e.code === 1 && /canary binary not found/i.test(e.stderr ?? "")) {
|
|
252
|
+
return { ok: false, reason: "binary-missing" };
|
|
253
|
+
}
|
|
254
|
+
return { ok: false, reason: "exec-failed" };
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
let cachedProbe;
|
|
258
|
+
const probe = () => cachedProbe ??= (async () => {
|
|
259
|
+
const res = await execCanary(["version"]);
|
|
260
|
+
if (!res.ok) return { status: "degraded", reason: res.reason };
|
|
261
|
+
if (res.stdout.trim() === "") return { status: "degraded", reason: "bad-output" };
|
|
262
|
+
const version = parseVersion(res.stdout);
|
|
263
|
+
return version ? { status: "available", version } : { status: "available" };
|
|
264
|
+
})();
|
|
265
|
+
const recommendFramework = async (prompt) => {
|
|
266
|
+
const res = await execCanary(["recommend", prompt, "--json"]);
|
|
267
|
+
if (!res.ok) return degradedRecommendation();
|
|
268
|
+
const parsed = frameworkRecommendationSchema.safeParse(safeJson(res.stdout));
|
|
269
|
+
return parsed.success ? parsed.data : degradedRecommendation();
|
|
270
|
+
};
|
|
271
|
+
const reviewTest = async (path2, framework) => {
|
|
272
|
+
const args = ["review-test", path2, "--json"];
|
|
273
|
+
if (framework) args.push("--framework", framework);
|
|
274
|
+
const res = await execCanary(args);
|
|
275
|
+
if (!res.ok) return [];
|
|
276
|
+
const parsed = canaryFindingsSchema.safeParse(safeJson(res.stdout));
|
|
277
|
+
return parsed.success ? parsed.data : [];
|
|
278
|
+
};
|
|
279
|
+
return { probe, recommendFramework, reviewTest };
|
|
280
|
+
}
|
|
281
|
+
|
|
176
282
|
// src/analysis-provider/anthropic.ts
|
|
177
283
|
var import_sdk = __toESM(require("@anthropic-ai/sdk"));
|
|
178
284
|
|
|
@@ -372,7 +478,7 @@ ${this.promptSuffix}` : request.prompt
|
|
|
372
478
|
};
|
|
373
479
|
|
|
374
480
|
// src/analysis-provider/claude-cli.ts
|
|
375
|
-
var
|
|
481
|
+
var import_node_child_process2 = require("child_process");
|
|
376
482
|
var DEFAULT_TIMEOUT_MS2 = 18e4;
|
|
377
483
|
var ClaudeCliAnalysisProvider = class {
|
|
378
484
|
command;
|
|
@@ -418,7 +524,7 @@ ${request.prompt}` : request.prompt;
|
|
|
418
524
|
}
|
|
419
525
|
runClaude(args) {
|
|
420
526
|
return new Promise((resolve, reject) => {
|
|
421
|
-
const child = (0,
|
|
527
|
+
const child = (0, import_node_child_process2.spawn)(this.command, args, {
|
|
422
528
|
env: process.env,
|
|
423
529
|
timeout: this.timeoutMs
|
|
424
530
|
});
|
|
@@ -464,7 +570,7 @@ ${request.prompt}` : request.prompt;
|
|
|
464
570
|
};
|
|
465
571
|
|
|
466
572
|
// src/sel/prompts.ts
|
|
467
|
-
var
|
|
573
|
+
var import_zod2 = require("zod");
|
|
468
574
|
var SEL_SYSTEM_PROMPT = `You are a spec enrichment agent. Your job is to analyze a work item (feature request, bug report, task, etc.) and produce structured output that captures the full engineering intent.
|
|
469
575
|
|
|
470
576
|
Analyze the provided work item and extract:
|
|
@@ -528,22 +634,22 @@ ${metaEntries.map(([k, v]) => `- **${k}:** ${String(v)}`).join("\n")}`
|
|
|
528
634
|
}
|
|
529
635
|
return parts.join("\n");
|
|
530
636
|
}
|
|
531
|
-
var selResponseSchema =
|
|
532
|
-
intent:
|
|
533
|
-
summary:
|
|
534
|
-
affectedSystems:
|
|
535
|
-
functionalRequirements:
|
|
536
|
-
nonFunctionalRequirements:
|
|
537
|
-
apiChanges:
|
|
538
|
-
dbChanges:
|
|
539
|
-
integrationPoints:
|
|
540
|
-
assumptions:
|
|
541
|
-
unknowns:
|
|
542
|
-
ambiguities:
|
|
543
|
-
riskSignals:
|
|
544
|
-
initialComplexityHints:
|
|
545
|
-
textualComplexity:
|
|
546
|
-
structuralComplexity:
|
|
637
|
+
var selResponseSchema = import_zod2.z.object({
|
|
638
|
+
intent: import_zod2.z.string(),
|
|
639
|
+
summary: import_zod2.z.string(),
|
|
640
|
+
affectedSystems: import_zod2.z.array(import_zod2.z.object({ name: import_zod2.z.string() })),
|
|
641
|
+
functionalRequirements: import_zod2.z.array(import_zod2.z.string()),
|
|
642
|
+
nonFunctionalRequirements: import_zod2.z.array(import_zod2.z.string()),
|
|
643
|
+
apiChanges: import_zod2.z.array(import_zod2.z.string()),
|
|
644
|
+
dbChanges: import_zod2.z.array(import_zod2.z.string()),
|
|
645
|
+
integrationPoints: import_zod2.z.array(import_zod2.z.string()),
|
|
646
|
+
assumptions: import_zod2.z.array(import_zod2.z.string()),
|
|
647
|
+
unknowns: import_zod2.z.array(import_zod2.z.string()),
|
|
648
|
+
ambiguities: import_zod2.z.array(import_zod2.z.string()),
|
|
649
|
+
riskSignals: import_zod2.z.array(import_zod2.z.string()),
|
|
650
|
+
initialComplexityHints: import_zod2.z.object({
|
|
651
|
+
textualComplexity: import_zod2.z.number().min(0).max(1),
|
|
652
|
+
structuralComplexity: import_zod2.z.number().min(0).max(1)
|
|
547
653
|
})
|
|
548
654
|
});
|
|
549
655
|
|
|
@@ -917,7 +1023,7 @@ function runGraphOnlyChecks(spec, score2, store) {
|
|
|
917
1023
|
}
|
|
918
1024
|
|
|
919
1025
|
// src/pesl/prompts.ts
|
|
920
|
-
var
|
|
1026
|
+
var import_zod3 = require("zod");
|
|
921
1027
|
var PESL_SYSTEM_PROMPT = `You are a pre-execution simulation agent. Your job is to analyze an enriched specification and its complexity assessment, then simulate what would happen if an autonomous coding agent attempted to implement this change.
|
|
922
1028
|
|
|
923
1029
|
Your simulation should:
|
|
@@ -929,13 +1035,13 @@ Your simulation should:
|
|
|
929
1035
|
Be realistic and specific. Reference actual system names from the spec. Err on the side of flagging potential issues -- it is better to over-predict failures than to miss them.
|
|
930
1036
|
|
|
931
1037
|
Return your analysis using the structured_output tool.`;
|
|
932
|
-
var peslResponseSchema =
|
|
933
|
-
simulatedPlan:
|
|
934
|
-
predictedFailures:
|
|
935
|
-
riskHotspots:
|
|
936
|
-
missingSteps:
|
|
937
|
-
testGaps:
|
|
938
|
-
recommendedChanges:
|
|
1038
|
+
var peslResponseSchema = import_zod3.z.object({
|
|
1039
|
+
simulatedPlan: import_zod3.z.array(import_zod3.z.string()).describe("Ordered implementation steps the agent would take"),
|
|
1040
|
+
predictedFailures: import_zod3.z.array(import_zod3.z.string()).describe("Likely failure modes during implementation"),
|
|
1041
|
+
riskHotspots: import_zod3.z.array(import_zod3.z.string()).describe("Files or modules that are high-risk change points"),
|
|
1042
|
+
missingSteps: import_zod3.z.array(import_zod3.z.string()).describe("Steps the agent might miss or overlook"),
|
|
1043
|
+
testGaps: import_zod3.z.array(import_zod3.z.string()).describe("Tests that should exist but likely do not"),
|
|
1044
|
+
recommendedChanges: import_zod3.z.array(import_zod3.z.string()).describe("Adjustments to improve success likelihood")
|
|
939
1045
|
});
|
|
940
1046
|
function appendSection(parts, heading, items) {
|
|
941
1047
|
if (items.length === 0) return;
|
|
@@ -1073,6 +1179,30 @@ var PeslSimulator = class {
|
|
|
1073
1179
|
};
|
|
1074
1180
|
|
|
1075
1181
|
// src/outcome/connector.ts
|
|
1182
|
+
var RESERVED_METADATA_KEYS = /* @__PURE__ */ new Set([
|
|
1183
|
+
"id",
|
|
1184
|
+
"identifier",
|
|
1185
|
+
"type",
|
|
1186
|
+
"name",
|
|
1187
|
+
"result",
|
|
1188
|
+
"retryCount",
|
|
1189
|
+
"failureReasons",
|
|
1190
|
+
"durationMs",
|
|
1191
|
+
"linkedSpecId",
|
|
1192
|
+
"timestamp",
|
|
1193
|
+
"issueId",
|
|
1194
|
+
"agentPersona",
|
|
1195
|
+
"taskType",
|
|
1196
|
+
"affectedSystemNodeIds",
|
|
1197
|
+
"edges"
|
|
1198
|
+
]);
|
|
1199
|
+
function stripReservedKeys(metadata) {
|
|
1200
|
+
const safe = {};
|
|
1201
|
+
for (const [key, value] of Object.entries(metadata)) {
|
|
1202
|
+
if (!RESERVED_METADATA_KEYS.has(key)) safe[key] = value;
|
|
1203
|
+
}
|
|
1204
|
+
return safe;
|
|
1205
|
+
}
|
|
1076
1206
|
var ExecutionOutcomeConnector = class {
|
|
1077
1207
|
constructor(store) {
|
|
1078
1208
|
this.store = store;
|
|
@@ -1085,6 +1215,11 @@ var ExecutionOutcomeConnector = class {
|
|
|
1085
1215
|
type: "execution_outcome",
|
|
1086
1216
|
name: `${outcome.result}: ${outcome.identifier}`,
|
|
1087
1217
|
metadata: {
|
|
1218
|
+
// Caller-supplied extras are STRIPPED of all reserved/core keys before
|
|
1219
|
+
// merge, so they can never shadow a core field — even conditionally-
|
|
1220
|
+
// written ones like agentPersona/taskType. Only genuinely additive
|
|
1221
|
+
// keys (e.g. verdict/confidence/source) survive.
|
|
1222
|
+
...stripReservedKeys(outcome.metadata ?? {}),
|
|
1088
1223
|
issueId: outcome.issueId,
|
|
1089
1224
|
identifier: outcome.identifier,
|
|
1090
1225
|
result: outcome.result,
|
|
@@ -1112,6 +1247,256 @@ var ExecutionOutcomeConnector = class {
|
|
|
1112
1247
|
}
|
|
1113
1248
|
};
|
|
1114
1249
|
|
|
1250
|
+
// src/outcome-eval/authority.ts
|
|
1251
|
+
function deriveAuthority(verdict, confidence) {
|
|
1252
|
+
return verdict === "NOT_SATISFIED" && confidence === "high" ? "blocking" : "advisory";
|
|
1253
|
+
}
|
|
1254
|
+
|
|
1255
|
+
// src/outcome-eval/prompts.ts
|
|
1256
|
+
var import_zod4 = require("zod");
|
|
1257
|
+
var verdictSchema = import_zod4.z.object({
|
|
1258
|
+
verdict: import_zod4.z.enum(["SATISFIED", "NOT_SATISFIED", "INCONCLUSIVE"]).describe("Whether the change satisfies the judged spec section"),
|
|
1259
|
+
confidence: import_zod4.z.enum(["low", "medium", "high"]).describe("Confidence in the verdict; high requires a named criterion"),
|
|
1260
|
+
rationale: import_zod4.z.string().describe("Cites specific met / unmet criteria"),
|
|
1261
|
+
unmetCriteria: import_zod4.z.array(import_zod4.z.string()).describe("Unmet criteria; empty when SATISFIED")
|
|
1262
|
+
}).strict();
|
|
1263
|
+
var OUTCOME_EVAL_SYSTEM_PROMPT = `You are a post-execution outcome judge. Given a spec acceptance section, a unified diff, and test output, decide whether the change SATISFIED, NOT_SATISFIED, or is INCONCLUSIVE against that section.
|
|
1264
|
+
|
|
1265
|
+
Confidence calibration (be conservative \u2014 false alarms are costly):
|
|
1266
|
+
- Default to "medium" confidence.
|
|
1267
|
+
- Use "high" ONLY when you can name a SPECIFIC criterion from the section that the diff and test output clearly met or clearly failed to meet, and quote or paraphrase it in the rationale.
|
|
1268
|
+
- Use "low" when the diff or test output is ambiguous, partial, or insufficient to judge.
|
|
1269
|
+
- When the change only PARTIALLY meets the criteria, do not exceed "medium" confidence.
|
|
1270
|
+
- Bias toward advisory caution: if unsure between two confidence levels, choose the lower one.
|
|
1271
|
+
|
|
1272
|
+
Rules:
|
|
1273
|
+
- The rationale MUST cite specific met or unmet criteria from the section.
|
|
1274
|
+
- "unmetCriteria" lists the section criteria the change failed to meet; it is empty when the verdict is SATISFIED.
|
|
1275
|
+
- Do NOT emit an "authority" field. Authority is computed downstream in TypeScript from (verdict, confidence) and must never come from you.
|
|
1276
|
+
|
|
1277
|
+
Return your judgment using the structured_output tool.`;
|
|
1278
|
+
var PROMPT_FIELD_MAX_CHARS = 12e3;
|
|
1279
|
+
var FENCE = "````";
|
|
1280
|
+
function clampField(text) {
|
|
1281
|
+
const trimmed = text.trim();
|
|
1282
|
+
if (trimmed.length <= PROMPT_FIELD_MAX_CHARS) return trimmed;
|
|
1283
|
+
const dropped = trimmed.length - PROMPT_FIELD_MAX_CHARS;
|
|
1284
|
+
return `${trimmed.slice(0, PROMPT_FIELD_MAX_CHARS)}
|
|
1285
|
+
\u2026 [truncated ${dropped} chars]`;
|
|
1286
|
+
}
|
|
1287
|
+
function buildUserPrompt2(section, diff, testOutput) {
|
|
1288
|
+
return [
|
|
1289
|
+
"## Spec Acceptance Criteria (judge against this section)",
|
|
1290
|
+
section.trim() || "(empty \u2014 treat as inconclusive)",
|
|
1291
|
+
"",
|
|
1292
|
+
"## Change Diff",
|
|
1293
|
+
`${FENCE}diff`,
|
|
1294
|
+
clampField(diff) || "(empty diff)",
|
|
1295
|
+
FENCE,
|
|
1296
|
+
"",
|
|
1297
|
+
"## Test Output",
|
|
1298
|
+
FENCE,
|
|
1299
|
+
clampField(testOutput) || "(no test output captured)",
|
|
1300
|
+
FENCE,
|
|
1301
|
+
"",
|
|
1302
|
+
"## Instructions",
|
|
1303
|
+
"Judge whether the diff satisfies the acceptance criteria above. Calibrate confidence conservatively per your system instructions. Cite specific criteria in the rationale."
|
|
1304
|
+
].join("\n");
|
|
1305
|
+
}
|
|
1306
|
+
|
|
1307
|
+
// src/outcome-eval/section-resolver.ts
|
|
1308
|
+
var CHAIN = [
|
|
1309
|
+
{ tag: "success-criteria", matches: (h) => h === "success criteria" },
|
|
1310
|
+
{ tag: "user-visible-behavior", matches: (h) => h === "user visible behavior" },
|
|
1311
|
+
{ tag: "overview", matches: (h) => h === "overview" }
|
|
1312
|
+
];
|
|
1313
|
+
var HEADING_RE = /^(#{1,6})\s+(.*\S)\s*$/;
|
|
1314
|
+
var FENCE_RE = /^\s*(```|~~~)/;
|
|
1315
|
+
var normalizeHeading = (text) => text.toLowerCase().replace(/-/g, " ").replace(/\s+/g, " ").trim();
|
|
1316
|
+
var parseHeading = (line, index) => {
|
|
1317
|
+
const m = HEADING_RE.exec(line);
|
|
1318
|
+
if (!m || m[1] === void 0 || m[2] === void 0) return null;
|
|
1319
|
+
const normalized = normalizeHeading(m[2]);
|
|
1320
|
+
const entry = CHAIN.find((c) => c.matches(normalized));
|
|
1321
|
+
return { index, level: m[1].length, tag: entry ? entry.tag : null };
|
|
1322
|
+
};
|
|
1323
|
+
function resolveSection(markdown) {
|
|
1324
|
+
const lines = markdown.split(/\r?\n/);
|
|
1325
|
+
const headings = [];
|
|
1326
|
+
let inFence = false;
|
|
1327
|
+
lines.forEach((line, index) => {
|
|
1328
|
+
if (FENCE_RE.test(line)) {
|
|
1329
|
+
inFence = !inFence;
|
|
1330
|
+
return;
|
|
1331
|
+
}
|
|
1332
|
+
if (inFence) return;
|
|
1333
|
+
const heading = parseHeading(line, index);
|
|
1334
|
+
if (heading) headings.push(heading);
|
|
1335
|
+
});
|
|
1336
|
+
for (const { tag } of CHAIN) {
|
|
1337
|
+
const start = headings.find((h) => h.tag === tag);
|
|
1338
|
+
if (!start) continue;
|
|
1339
|
+
const next = headings.find((h) => h.index > start.index && h.level <= start.level);
|
|
1340
|
+
const endExclusive = next ? next.index : lines.length;
|
|
1341
|
+
const body = lines.slice(start.index + 1, endExclusive).join("\n").trim();
|
|
1342
|
+
return { judgedAgainst: tag, body };
|
|
1343
|
+
}
|
|
1344
|
+
return null;
|
|
1345
|
+
}
|
|
1346
|
+
|
|
1347
|
+
// src/outcome-eval/evaluator.ts
|
|
1348
|
+
var import_promises = require("fs/promises");
|
|
1349
|
+
var import_node_crypto2 = require("crypto");
|
|
1350
|
+
var OutcomeEvaluator = class {
|
|
1351
|
+
provider;
|
|
1352
|
+
store;
|
|
1353
|
+
options;
|
|
1354
|
+
constructor(provider, store, options = {}) {
|
|
1355
|
+
this.provider = provider;
|
|
1356
|
+
this.store = store;
|
|
1357
|
+
this.options = options;
|
|
1358
|
+
}
|
|
1359
|
+
async evaluate(input) {
|
|
1360
|
+
let resolved;
|
|
1361
|
+
try {
|
|
1362
|
+
resolved = await this.resolveJudgmentSection(input);
|
|
1363
|
+
} catch {
|
|
1364
|
+
return this.finish(this.degradedVerdict("overview"), input);
|
|
1365
|
+
}
|
|
1366
|
+
if (resolved === null) {
|
|
1367
|
+
const verdict2 = this.buildVerdict(
|
|
1368
|
+
"INCONCLUSIVE",
|
|
1369
|
+
"low",
|
|
1370
|
+
"No judgable spec section found.",
|
|
1371
|
+
"overview",
|
|
1372
|
+
[]
|
|
1373
|
+
);
|
|
1374
|
+
return this.finish(verdict2, input);
|
|
1375
|
+
}
|
|
1376
|
+
const verdict = await this.judge(resolved, input);
|
|
1377
|
+
return this.finish(verdict, input);
|
|
1378
|
+
}
|
|
1379
|
+
/**
|
|
1380
|
+
* Run the provider call and strict re-parse. ANY failure here — provider
|
|
1381
|
+
* rejection (rate limit/network), or a strict-parse rejection of a malformed
|
|
1382
|
+
* or authority-injected payload — degrades safely to INCONCLUSIVE/low/
|
|
1383
|
+
* advisory rather than throwing. This reconciles Criterion 3 (never block on
|
|
1384
|
+
* noise) with Criterion 4: an injected `authority` key is discarded by the
|
|
1385
|
+
* .strict() re-parse, and the degraded verdict's authority is DERIVED from
|
|
1386
|
+
* INCONCLUSIVE/low = advisory — so the LLM gains nothing by injecting it.
|
|
1387
|
+
*/
|
|
1388
|
+
async judge(resolved, input) {
|
|
1389
|
+
try {
|
|
1390
|
+
const response = await this.provider.analyze({
|
|
1391
|
+
prompt: buildUserPrompt2(resolved.body, input.diff, input.testOutput),
|
|
1392
|
+
systemPrompt: OUTCOME_EVAL_SYSTEM_PROMPT,
|
|
1393
|
+
responseSchema: verdictSchema,
|
|
1394
|
+
...this.options.model !== void 0 && { model: this.options.model }
|
|
1395
|
+
});
|
|
1396
|
+
const llm = verdictSchema.parse(response.result);
|
|
1397
|
+
return this.buildVerdict(
|
|
1398
|
+
llm.verdict,
|
|
1399
|
+
llm.confidence,
|
|
1400
|
+
llm.rationale,
|
|
1401
|
+
resolved.judgedAgainst,
|
|
1402
|
+
llm.unmetCriteria
|
|
1403
|
+
);
|
|
1404
|
+
} catch {
|
|
1405
|
+
return this.degradedVerdict(resolved.judgedAgainst);
|
|
1406
|
+
}
|
|
1407
|
+
}
|
|
1408
|
+
/**
|
|
1409
|
+
* Build the safe-degradation verdict. INCONCLUSIVE/low yields advisory
|
|
1410
|
+
* authority via deriveAuthority — never blocking. The rationale names only a
|
|
1411
|
+
* coarse reason category, never a stack trace or secret.
|
|
1412
|
+
*/
|
|
1413
|
+
degradedVerdict(judgedAgainst) {
|
|
1414
|
+
return this.buildVerdict(
|
|
1415
|
+
"INCONCLUSIVE",
|
|
1416
|
+
"low",
|
|
1417
|
+
"Evaluation could not be completed; defaulting to an inconclusive, advisory verdict.",
|
|
1418
|
+
judgedAgainst,
|
|
1419
|
+
[]
|
|
1420
|
+
);
|
|
1421
|
+
}
|
|
1422
|
+
/** Persist (Phase 4 seam) then return the verdict. */
|
|
1423
|
+
async finish(verdict, input) {
|
|
1424
|
+
await this.persistOutcome(verdict, input);
|
|
1425
|
+
return verdict;
|
|
1426
|
+
}
|
|
1427
|
+
async resolveJudgmentSection(input) {
|
|
1428
|
+
if (input.specSection !== void 0) {
|
|
1429
|
+
return input.specSection.trim() === "" ? null : { judgedAgainst: "success-criteria", body: input.specSection };
|
|
1430
|
+
}
|
|
1431
|
+
const markdown = await (0, import_promises.readFile)(input.specPath, "utf8");
|
|
1432
|
+
return resolveSection(markdown);
|
|
1433
|
+
}
|
|
1434
|
+
buildVerdict(verdict, confidence, rationale, judgedAgainst, unmetCriteria) {
|
|
1435
|
+
return {
|
|
1436
|
+
verdict,
|
|
1437
|
+
confidence,
|
|
1438
|
+
rationale,
|
|
1439
|
+
judgedAgainst,
|
|
1440
|
+
unmetCriteria,
|
|
1441
|
+
authority: deriveAuthority(verdict, confidence)
|
|
1442
|
+
};
|
|
1443
|
+
}
|
|
1444
|
+
/**
|
|
1445
|
+
* Map an OutcomeVerdict + OutcomeEvalInput to the connector's ExecutionOutcome.
|
|
1446
|
+
* - result: SATISFIED -> 'success'; otherwise 'failure'. INCONCLUSIVE is
|
|
1447
|
+
* 'failure' for type-validity but omits agentPersona/affected systems so
|
|
1448
|
+
* the effectiveness scorer ignores it (plan D2).
|
|
1449
|
+
* - linkedSpecId: input.specPath (metadata only; no spec edge — plan D1).
|
|
1450
|
+
* - affectedSystemNodeIds: [] in v1 (not available from OutcomeEvalInput — D4).
|
|
1451
|
+
* - id: one node per EVALUATION. GraphStore.addNode upserts by id, so the id
|
|
1452
|
+
* carries a collision-free randomUUID() — two evaluate() calls in the same
|
|
1453
|
+
* millisecond can never overwrite each other (data-loss fix). specPath is
|
|
1454
|
+
* included for human readability only.
|
|
1455
|
+
* - taskType: OMITTED. The outcome-eval judge has no task categorization, and
|
|
1456
|
+
* asserting a false 'feature' would mislead specialization analytics (SUG-2).
|
|
1457
|
+
* - metadata: verdict-specific signal carried through the connector's
|
|
1458
|
+
* additive pass-through (verdict/confidence/judgedAgainst/source) so the
|
|
1459
|
+
* true 3-valued verdict is durable on the node (Truth 3).
|
|
1460
|
+
*/
|
|
1461
|
+
toExecutionOutcome(verdict, input) {
|
|
1462
|
+
const timestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
1463
|
+
return {
|
|
1464
|
+
id: `outcome:outcome-eval:${input.specPath}:${(0, import_node_crypto2.randomUUID)()}`,
|
|
1465
|
+
issueId: "outcome-eval",
|
|
1466
|
+
identifier: `outcome-eval:${input.specPath}`,
|
|
1467
|
+
result: verdict.verdict === "SATISFIED" ? "success" : "failure",
|
|
1468
|
+
retryCount: 0,
|
|
1469
|
+
failureReasons: verdict.unmetCriteria,
|
|
1470
|
+
// 0 means "not applicable to outcome-eval" — the judge does not time work.
|
|
1471
|
+
durationMs: 0,
|
|
1472
|
+
linkedSpecId: input.specPath,
|
|
1473
|
+
affectedSystemNodeIds: [],
|
|
1474
|
+
timestamp,
|
|
1475
|
+
metadata: {
|
|
1476
|
+
verdict: verdict.verdict,
|
|
1477
|
+
confidence: verdict.confidence,
|
|
1478
|
+
judgedAgainst: verdict.judgedAgainst,
|
|
1479
|
+
source: "outcome-eval"
|
|
1480
|
+
}
|
|
1481
|
+
};
|
|
1482
|
+
}
|
|
1483
|
+
/**
|
|
1484
|
+
* Phase 4: writes exactly one execution_outcome node via
|
|
1485
|
+
* ExecutionOutcomeConnector. Degrade-safe (plan D3): a graph-write failure is
|
|
1486
|
+
* swallowed-and-logged, never thrown — the verdict is already computed before
|
|
1487
|
+
* this runs, so swallowing keeps evaluate() total. No secrets/stack frames in
|
|
1488
|
+
* the log message.
|
|
1489
|
+
*/
|
|
1490
|
+
async persistOutcome(verdict, input) {
|
|
1491
|
+
try {
|
|
1492
|
+
const connector = new ExecutionOutcomeConnector(this.store);
|
|
1493
|
+
connector.ingest(this.toExecutionOutcome(verdict, input));
|
|
1494
|
+
} catch {
|
|
1495
|
+
console.warn("[outcome-eval] execution_outcome persistence failed; verdict unaffected.");
|
|
1496
|
+
}
|
|
1497
|
+
}
|
|
1498
|
+
};
|
|
1499
|
+
|
|
1115
1500
|
// src/pipeline.ts
|
|
1116
1501
|
var IntelligencePipeline = class {
|
|
1117
1502
|
provider;
|
|
@@ -1589,16 +1974,21 @@ function refreshProfiles(projectRoot, graphStore, opts) {
|
|
|
1589
1974
|
ExecutionOutcomeConnector,
|
|
1590
1975
|
GraphValidator,
|
|
1591
1976
|
IntelligencePipeline,
|
|
1977
|
+
OUTCOME_EVAL_SYSTEM_PROMPT,
|
|
1592
1978
|
OpenAICompatibleAnalysisProvider,
|
|
1979
|
+
OutcomeEvaluator,
|
|
1593
1980
|
PeslSimulator,
|
|
1594
1981
|
buildSpecializationProfile,
|
|
1982
|
+
buildUserPrompt,
|
|
1595
1983
|
computeExpertiseLevel,
|
|
1596
1984
|
computeHistoricalComplexity,
|
|
1597
1985
|
computePersonaEffectiveness,
|
|
1598
1986
|
computeSemanticComplexity,
|
|
1599
1987
|
computeSpecialization,
|
|
1600
1988
|
computeStructuralComplexity,
|
|
1989
|
+
createCanaryAdapter,
|
|
1601
1990
|
decayWeight,
|
|
1991
|
+
deriveAuthority,
|
|
1602
1992
|
detectBlindSpots,
|
|
1603
1993
|
enrich,
|
|
1604
1994
|
githubToRawWorkItem,
|
|
@@ -1608,6 +1998,7 @@ function refreshProfiles(projectRoot, graphStore, opts) {
|
|
|
1608
1998
|
manualToRawWorkItem,
|
|
1609
1999
|
recommendPersona,
|
|
1610
2000
|
refreshProfiles,
|
|
2001
|
+
resolveSection,
|
|
1611
2002
|
runGraphOnlyChecks,
|
|
1612
2003
|
runLlmSimulation,
|
|
1613
2004
|
saveProfiles,
|
|
@@ -1615,5 +2006,6 @@ function refreshProfiles(projectRoot, graphStore, opts) {
|
|
|
1615
2006
|
scoreToConcernSignals,
|
|
1616
2007
|
temporalSuccessRate,
|
|
1617
2008
|
toRawWorkItem,
|
|
2009
|
+
verdictSchema,
|
|
1618
2010
|
weightedRecommendPersona
|
|
1619
2011
|
});
|