@harness-engineering/intelligence 0.2.6 → 0.3.0

package/dist/index.mjs

@@ -106,6 +106,105 @@ function manualToRawWorkItem(input) {
   };
 }
 
+// src/adapters/canary.ts
+import { z } from "zod";
+import { execFile } from "child_process";
+var frameworkRecommendationSchema = z.object({
+  status: z.string(),
+  test_type: z.string(),
+  framework: z.string(),
+  file_extension: z.string(),
+  reasoning: z.array(z.string()),
+  alternatives: z.array(z.string())
+});
+var canaryFindingSchema = z.object({
+  file: z.string(),
+  line: z.number(),
+  rule: z.string(),
+  severity: z.string(),
+  message: z.string(),
+  suggestion: z.string()
+});
+var canaryFindingsSchema = z.array(canaryFindingSchema);
+var EXEC_TIMEOUT_MS = 3e4;
+var EXEC_MAX_BUFFER = 16 * 1024 * 1024;
+var defaultExec = (cmd, args) => new Promise((resolve, reject) => {
+  execFile(
+    cmd,
+    args,
+    { encoding: "utf8", timeout: EXEC_TIMEOUT_MS, maxBuffer: EXEC_MAX_BUFFER },
+    (err, stdout) => {
+      if (err) {
+        reject(err);
+        return;
+      }
+      resolve({ stdout });
+    }
+  );
+});
+function canaryInvocation(subArgs) {
+  return ["canary", subArgs];
+}
+function parseVersion(stdout) {
+  return stdout.match(/\d+\.\d+\.\d+/)?.[0];
+}
+function safeJson(stdout) {
+  try {
+    return JSON.parse(stdout);
+  } catch {
+    return void 0;
+  }
+}
+function degradedRecommendation() {
+  return {
+    status: "degraded",
+    test_type: "",
+    framework: "",
+    file_extension: "",
+    reasoning: [],
+    alternatives: []
+  };
+}
+function createCanaryAdapter(exec = defaultExec) {
+  async function execCanary(subArgs) {
+    const [cmd, args] = canaryInvocation(subArgs);
+    try {
+      const { stdout } = await exec(cmd, args);
+      return { ok: true, stdout };
+    } catch (err) {
+      const e = err;
+      if (e.code === "ENOENT") return { ok: false, reason: "not-installed" };
+      if (e.code === 1 && /canary binary not found/i.test(e.stderr ?? "")) {
+        return { ok: false, reason: "binary-missing" };
+      }
+      return { ok: false, reason: "exec-failed" };
+    }
+  }
+  let cachedProbe;
+  const probe = () => cachedProbe ??= (async () => {
+    const res = await execCanary(["version"]);
+    if (!res.ok) return { status: "degraded", reason: res.reason };
+    if (res.stdout.trim() === "") return { status: "degraded", reason: "bad-output" };
+    const version = parseVersion(res.stdout);
+    return version ? { status: "available", version } : { status: "available" };
+  })();
+  const recommendFramework = async (prompt) => {
+    const res = await execCanary(["recommend", prompt, "--json"]);
+    if (!res.ok) return degradedRecommendation();
+    const parsed = frameworkRecommendationSchema.safeParse(safeJson(res.stdout));
+    return parsed.success ? parsed.data : degradedRecommendation();
+  };
+  const reviewTest = async (path2, framework) => {
+    const args = ["review-test", path2, "--json"];
+    if (framework) args.push("--framework", framework);
+    const res = await execCanary(args);
+    if (!res.ok) return [];
+    const parsed = canaryFindingsSchema.safeParse(safeJson(res.stdout));
+    return parsed.success ? parsed.data : [];
+  };
+  return { probe, recommendFramework, reviewTest };
+}
+
 // src/analysis-provider/anthropic.ts
 import Anthropic from "@anthropic-ai/sdk";
 
@@ -397,7 +496,7 @@ ${request.prompt}` : request.prompt;
 };
 
 // src/sel/prompts.ts
-import { z } from "zod";
+import { z as z2 } from "zod";
 var SEL_SYSTEM_PROMPT = `You are a spec enrichment agent. Your job is to analyze a work item (feature request, bug report, task, etc.) and produce structured output that captures the full engineering intent.
 
 Analyze the provided work item and extract:
@@ -461,22 +560,22 @@ ${metaEntries.map(([k, v]) => `- **${k}:** ${String(v)}`).join("\n")}`
   }
   return parts.join("\n");
 }
-var selResponseSchema = z.object({
-  intent: z.string(),
-  summary: z.string(),
-  affectedSystems: z.array(z.object({ name: z.string() })),
-  functionalRequirements: z.array(z.string()),
-  nonFunctionalRequirements: z.array(z.string()),
-  apiChanges: z.array(z.string()),
-  dbChanges: z.array(z.string()),
-  integrationPoints: z.array(z.string()),
-  assumptions: z.array(z.string()),
-  unknowns: z.array(z.string()),
-  ambiguities: z.array(z.string()),
-  riskSignals: z.array(z.string()),
-  initialComplexityHints: z.object({
-    textualComplexity: z.number().min(0).max(1),
-    structuralComplexity: z.number().min(0).max(1)
+var selResponseSchema = z2.object({
+  intent: z2.string(),
+  summary: z2.string(),
+  affectedSystems: z2.array(z2.object({ name: z2.string() })),
+  functionalRequirements: z2.array(z2.string()),
+  nonFunctionalRequirements: z2.array(z2.string()),
+  apiChanges: z2.array(z2.string()),
+  dbChanges: z2.array(z2.string()),
+  integrationPoints: z2.array(z2.string()),
+  assumptions: z2.array(z2.string()),
+  unknowns: z2.array(z2.string()),
+  ambiguities: z2.array(z2.string()),
+  riskSignals: z2.array(z2.string()),
+  initialComplexityHints: z2.object({
+    textualComplexity: z2.number().min(0).max(1),
+    structuralComplexity: z2.number().min(0).max(1)
   })
 });
 
@@ -850,7 +949,7 @@ function runGraphOnlyChecks(spec, score2, store) {
 }
 
 // src/pesl/prompts.ts
-import { z as z2 } from "zod";
+import { z as z3 } from "zod";
 var PESL_SYSTEM_PROMPT = `You are a pre-execution simulation agent. Your job is to analyze an enriched specification and its complexity assessment, then simulate what would happen if an autonomous coding agent attempted to implement this change.
 
 Your simulation should:
@@ -862,13 +961,13 @@ Your simulation should:
 Be realistic and specific. Reference actual system names from the spec. Err on the side of flagging potential issues -- it is better to over-predict failures than to miss them.
 
 Return your analysis using the structured_output tool.`;
-var peslResponseSchema = z2.object({
-  simulatedPlan: z2.array(z2.string()).describe("Ordered implementation steps the agent would take"),
-  predictedFailures: z2.array(z2.string()).describe("Likely failure modes during implementation"),
-  riskHotspots: z2.array(z2.string()).describe("Files or modules that are high-risk change points"),
-  missingSteps: z2.array(z2.string()).describe("Steps the agent might miss or overlook"),
-  testGaps: z2.array(z2.string()).describe("Tests that should exist but likely do not"),
-  recommendedChanges: z2.array(z2.string()).describe("Adjustments to improve success likelihood")
+var peslResponseSchema = z3.object({
+  simulatedPlan: z3.array(z3.string()).describe("Ordered implementation steps the agent would take"),
+  predictedFailures: z3.array(z3.string()).describe("Likely failure modes during implementation"),
+  riskHotspots: z3.array(z3.string()).describe("Files or modules that are high-risk change points"),
+  missingSteps: z3.array(z3.string()).describe("Steps the agent might miss or overlook"),
+  testGaps: z3.array(z3.string()).describe("Tests that should exist but likely do not"),
+  recommendedChanges: z3.array(z3.string()).describe("Adjustments to improve success likelihood")
 });
 function appendSection(parts, heading, items) {
   if (items.length === 0) return;
@@ -1006,6 +1105,30 @@ var PeslSimulator = class {
 };
 
 // src/outcome/connector.ts
+var RESERVED_METADATA_KEYS = /* @__PURE__ */ new Set([
+  "id",
+  "identifier",
+  "type",
+  "name",
+  "result",
+  "retryCount",
+  "failureReasons",
+  "durationMs",
+  "linkedSpecId",
+  "timestamp",
+  "issueId",
+  "agentPersona",
+  "taskType",
+  "affectedSystemNodeIds",
+  "edges"
+]);
+function stripReservedKeys(metadata) {
+  const safe = {};
+  for (const [key, value] of Object.entries(metadata)) {
+    if (!RESERVED_METADATA_KEYS.has(key)) safe[key] = value;
+  }
+  return safe;
+}
 var ExecutionOutcomeConnector = class {
   constructor(store) {
     this.store = store;
@@ -1018,6 +1141,11 @@ var ExecutionOutcomeConnector = class {
       type: "execution_outcome",
       name: `${outcome.result}: ${outcome.identifier}`,
       metadata: {
+        // Caller-supplied extras are STRIPPED of all reserved/core keys before
+        // merge, so they can never shadow a core field — even conditionally-
+        // written ones like agentPersona/taskType. Only genuinely additive
+        // keys (e.g. verdict/confidence/source) survive.
+        ...stripReservedKeys(outcome.metadata ?? {}),
         issueId: outcome.issueId,
         identifier: outcome.identifier,
         result: outcome.result,
@@ -1045,6 +1173,256 @@ var ExecutionOutcomeConnector = class {
   }
 };
 
+// src/outcome-eval/authority.ts
+function deriveAuthority(verdict, confidence) {
+  return verdict === "NOT_SATISFIED" && confidence === "high" ? "blocking" : "advisory";
+}
+
+// src/outcome-eval/prompts.ts
+import { z as z4 } from "zod";
+var verdictSchema = z4.object({
+  verdict: z4.enum(["SATISFIED", "NOT_SATISFIED", "INCONCLUSIVE"]).describe("Whether the change satisfies the judged spec section"),
+  confidence: z4.enum(["low", "medium", "high"]).describe("Confidence in the verdict; high requires a named criterion"),
+  rationale: z4.string().describe("Cites specific met / unmet criteria"),
+  unmetCriteria: z4.array(z4.string()).describe("Unmet criteria; empty when SATISFIED")
+}).strict();
+var OUTCOME_EVAL_SYSTEM_PROMPT = `You are a post-execution outcome judge. Given a spec acceptance section, a unified diff, and test output, decide whether the change SATISFIED, NOT_SATISFIED, or is INCONCLUSIVE against that section.
+
+Confidence calibration (be conservative \u2014 false alarms are costly):
+- Default to "medium" confidence.
+- Use "high" ONLY when you can name a SPECIFIC criterion from the section that the diff and test output clearly met or clearly failed to meet, and quote or paraphrase it in the rationale.
+- Use "low" when the diff or test output is ambiguous, partial, or insufficient to judge.
+- When the change only PARTIALLY meets the criteria, do not exceed "medium" confidence.
+- Bias toward advisory caution: if unsure between two confidence levels, choose the lower one.
+
+Rules:
+- The rationale MUST cite specific met or unmet criteria from the section.
+- "unmetCriteria" lists the section criteria the change failed to meet; it is empty when the verdict is SATISFIED.
+- Do NOT emit an "authority" field. Authority is computed downstream in TypeScript from (verdict, confidence) and must never come from you.
+
+Return your judgment using the structured_output tool.`;
+var PROMPT_FIELD_MAX_CHARS = 12e3;
+var FENCE = "````";
+function clampField(text) {
+  const trimmed = text.trim();
+  if (trimmed.length <= PROMPT_FIELD_MAX_CHARS) return trimmed;
+  const dropped = trimmed.length - PROMPT_FIELD_MAX_CHARS;
+  return `${trimmed.slice(0, PROMPT_FIELD_MAX_CHARS)}
+\u2026 [truncated ${dropped} chars]`;
+}
+function buildUserPrompt2(section, diff, testOutput) {
+  return [
+    "## Spec Acceptance Criteria (judge against this section)",
+    section.trim() || "(empty \u2014 treat as inconclusive)",
+    "",
+    "## Change Diff",
+    `${FENCE}diff`,
+    clampField(diff) || "(empty diff)",
+    FENCE,
+    "",
+    "## Test Output",
+    FENCE,
+    clampField(testOutput) || "(no test output captured)",
+    FENCE,
+    "",
+    "## Instructions",
+    "Judge whether the diff satisfies the acceptance criteria above. Calibrate confidence conservatively per your system instructions. Cite specific criteria in the rationale."
+  ].join("\n");
+}
+
+// src/outcome-eval/section-resolver.ts
+var CHAIN = [
+  { tag: "success-criteria", matches: (h) => h === "success criteria" },
+  { tag: "user-visible-behavior", matches: (h) => h === "user visible behavior" },
+  { tag: "overview", matches: (h) => h === "overview" }
+];
+var HEADING_RE = /^(#{1,6})\s+(.*\S)\s*$/;
+var FENCE_RE = /^\s*(```|~~~)/;
+var normalizeHeading = (text) => text.toLowerCase().replace(/-/g, " ").replace(/\s+/g, " ").trim();
+var parseHeading = (line, index) => {
+  const m = HEADING_RE.exec(line);
+  if (!m || m[1] === void 0 || m[2] === void 0) return null;
+  const normalized = normalizeHeading(m[2]);
+  const entry = CHAIN.find((c) => c.matches(normalized));
+  return { index, level: m[1].length, tag: entry ? entry.tag : null };
+};
+function resolveSection(markdown) {
+  const lines = markdown.split(/\r?\n/);
+  const headings = [];
+  let inFence = false;
+  lines.forEach((line, index) => {
+    if (FENCE_RE.test(line)) {
+      inFence = !inFence;
+      return;
+    }
+    if (inFence) return;
+    const heading = parseHeading(line, index);
+    if (heading) headings.push(heading);
+  });
+  for (const { tag } of CHAIN) {
+    const start = headings.find((h) => h.tag === tag);
+    if (!start) continue;
+    const next = headings.find((h) => h.index > start.index && h.level <= start.level);
+    const endExclusive = next ? next.index : lines.length;
+    const body = lines.slice(start.index + 1, endExclusive).join("\n").trim();
+    return { judgedAgainst: tag, body };
+  }
+  return null;
+}
+
+// src/outcome-eval/evaluator.ts
+import { readFile } from "fs/promises";
+import { randomUUID as randomUUID2 } from "crypto";
+var OutcomeEvaluator = class {
+  provider;
+  store;
+  options;
+  constructor(provider, store, options = {}) {
+    this.provider = provider;
+    this.store = store;
+    this.options = options;
+  }
+  async evaluate(input) {
+    let resolved;
+    try {
+      resolved = await this.resolveJudgmentSection(input);
+    } catch {
+      return this.finish(this.degradedVerdict("overview"), input);
+    }
+    if (resolved === null) {
+      const verdict2 = this.buildVerdict(
+        "INCONCLUSIVE",
+        "low",
+        "No judgable spec section found.",
+        "overview",
+        []
+      );
+      return this.finish(verdict2, input);
+    }
+    const verdict = await this.judge(resolved, input);
+    return this.finish(verdict, input);
+  }
+  /**
+   * Run the provider call and strict re-parse. ANY failure here — provider
+   * rejection (rate limit/network), or a strict-parse rejection of a malformed
+   * or authority-injected payload — degrades safely to INCONCLUSIVE/low/
+   * advisory rather than throwing. This reconciles Criterion 3 (never block on
+   * noise) with Criterion 4: an injected `authority` key is discarded by the
+   * .strict() re-parse, and the degraded verdict's authority is DERIVED from
+   * INCONCLUSIVE/low = advisory — so the LLM gains nothing by injecting it.
+   */
+  async judge(resolved, input) {
+    try {
+      const response = await this.provider.analyze({
+        prompt: buildUserPrompt2(resolved.body, input.diff, input.testOutput),
+        systemPrompt: OUTCOME_EVAL_SYSTEM_PROMPT,
+        responseSchema: verdictSchema,
+        ...this.options.model !== void 0 && { model: this.options.model }
+      });
+      const llm = verdictSchema.parse(response.result);
+      return this.buildVerdict(
+        llm.verdict,
+        llm.confidence,
+        llm.rationale,
+        resolved.judgedAgainst,
+        llm.unmetCriteria
+      );
+    } catch {
+      return this.degradedVerdict(resolved.judgedAgainst);
+    }
+  }
+  /**
+   * Build the safe-degradation verdict. INCONCLUSIVE/low yields advisory
+   * authority via deriveAuthority — never blocking. The rationale names only a
+   * coarse reason category, never a stack trace or secret.
+   */
+  degradedVerdict(judgedAgainst) {
+    return this.buildVerdict(
+      "INCONCLUSIVE",
+      "low",
+      "Evaluation could not be completed; defaulting to an inconclusive, advisory verdict.",
+      judgedAgainst,
+      []
+    );
+  }
+  /** Persist (Phase 4 seam) then return the verdict. */
+  async finish(verdict, input) {
+    await this.persistOutcome(verdict, input);
+    return verdict;
+  }
+  async resolveJudgmentSection(input) {
+    if (input.specSection !== void 0) {
+      return input.specSection.trim() === "" ? null : { judgedAgainst: "success-criteria", body: input.specSection };
+    }
+    const markdown = await readFile(input.specPath, "utf8");
+    return resolveSection(markdown);
+  }
+  buildVerdict(verdict, confidence, rationale, judgedAgainst, unmetCriteria) {
+    return {
+      verdict,
+      confidence,
+      rationale,
+      judgedAgainst,
+      unmetCriteria,
+      authority: deriveAuthority(verdict, confidence)
+    };
+  }
+  /**
+   * Map an OutcomeVerdict + OutcomeEvalInput to the connector's ExecutionOutcome.
+   * - result: SATISFIED -> 'success'; otherwise 'failure'. INCONCLUSIVE is
+   *   'failure' for type-validity but omits agentPersona/affected systems so
+   *   the effectiveness scorer ignores it (plan D2).
+   * - linkedSpecId: input.specPath (metadata only; no spec edge — plan D1).
+   * - affectedSystemNodeIds: [] in v1 (not available from OutcomeEvalInput — D4).
+   * - id: one node per EVALUATION. GraphStore.addNode upserts by id, so the id
+   *   carries a collision-free randomUUID() — two evaluate() calls in the same
+   *   millisecond can never overwrite each other (data-loss fix). specPath is
+   *   included for human readability only.
+   * - taskType: OMITTED. The outcome-eval judge has no task categorization, and
+   *   asserting a false 'feature' would mislead specialization analytics (SUG-2).
+   * - metadata: verdict-specific signal carried through the connector's
+   *   additive pass-through (verdict/confidence/judgedAgainst/source) so the
+   *   true 3-valued verdict is durable on the node (Truth 3).
+   */
+  toExecutionOutcome(verdict, input) {
+    const timestamp = (/* @__PURE__ */ new Date()).toISOString();
+    return {
+      id: `outcome:outcome-eval:${input.specPath}:${randomUUID2()}`,
+      issueId: "outcome-eval",
+      identifier: `outcome-eval:${input.specPath}`,
+      result: verdict.verdict === "SATISFIED" ? "success" : "failure",
+      retryCount: 0,
+      failureReasons: verdict.unmetCriteria,
+      // 0 means "not applicable to outcome-eval" — the judge does not time work.
+      durationMs: 0,
+      linkedSpecId: input.specPath,
+      affectedSystemNodeIds: [],
+      timestamp,
+      metadata: {
+        verdict: verdict.verdict,
+        confidence: verdict.confidence,
+        judgedAgainst: verdict.judgedAgainst,
+        source: "outcome-eval"
+      }
+    };
+  }
+  /**
+   * Phase 4: writes exactly one execution_outcome node via
+   * ExecutionOutcomeConnector. Degrade-safe (plan D3): a graph-write failure is
+   * swallowed-and-logged, never thrown — the verdict is already computed before
+   * this runs, so swallowing keeps evaluate() total. No secrets/stack frames in
+   * the log message.
+   */
+  async persistOutcome(verdict, input) {
+    try {
+      const connector = new ExecutionOutcomeConnector(this.store);
+      connector.ingest(this.toExecutionOutcome(verdict, input));
+    } catch {
+      console.warn("[outcome-eval] execution_outcome persistence failed; verdict unaffected.");
+    }
+  }
+};
+
 // src/pipeline.ts
 var IntelligencePipeline = class {
   provider;
@@ -1521,16 +1899,21 @@ export {
   ExecutionOutcomeConnector,
   GraphValidator,
   IntelligencePipeline,
+  OUTCOME_EVAL_SYSTEM_PROMPT,
   OpenAICompatibleAnalysisProvider,
+  OutcomeEvaluator,
   PeslSimulator,
   buildSpecializationProfile,
+  buildUserPrompt2 as buildUserPrompt,
   computeExpertiseLevel,
   computeHistoricalComplexity,
   computePersonaEffectiveness,
   computeSemanticComplexity,
   computeSpecialization,
   computeStructuralComplexity,
+  createCanaryAdapter,
   decayWeight,
+  deriveAuthority,
   detectBlindSpots,
   enrich,
   githubToRawWorkItem,
@@ -1540,6 +1923,7 @@ export {
   manualToRawWorkItem,
   recommendPersona,
   refreshProfiles,
+  resolveSection,
   runGraphOnlyChecks,
   runLlmSimulation,
   saveProfiles,
@@ -1547,5 +1931,6 @@ export {
   scoreToConcernSignals,
   temporalSuccessRate,
   toRawWorkItem,
+  verdictSchema,
   weightedRecommendPersona
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@harness-engineering/intelligence",
-  "version": "0.2.6",
+  "version": "0.3.0",
   "description": "Intelligence pipeline for spec enrichment, complexity modeling, and pre-execution simulation",
   "main": "./dist/index.js",
   "module": "./dist/index.mjs",
@@ -40,8 +40,11 @@
     "@anthropic-ai/sdk": "^0.95.1",
     "openai": "^6.0.0",
     "zod": "^3.25.76",
-    "@harness-engineering/graph": "0.10.0",
-    "@harness-engineering/types": "0.15.0"
+    "@harness-engineering/graph": "0.11.0",
+    "@harness-engineering/types": "0.16.0"
+  },
+  "optionalDependencies": {
+    "canary-test-cli": "^5.4.0"
   },
   "devDependencies": {
     "@types/node": "^22.19.15",