forgedock 1.30.0 → 1.32.0

package/bin/engine/outcome-codes.mjs

@@ -68,6 +68,7 @@ export const OUTCOME_CODES = Object.freeze({
   BUILDER_BLOCK_UNTYPED: "builder.block_untyped",
   BUILDER_NO_EVIDENCE: "builder.no_evidence",
   BUILDER_NO_COMMIT: "builder.no_commit",
+  BUDGET_EXCEEDED: "budget.exceeded", // forge#4392: the implement turn budget for the lane's size was spent and the wrap-up did not finish; digest in detail
   REVIEW_FAILED: "review.failed",
   REVIEW_NO_VERDICT: "review.no_verdict",
   // Core work-on conformance gate (bin/lean/conformance.mjs).
@@ -182,13 +183,13 @@ export function codeForTerminalReason(reason) {
  * Batch recovery classes of a lane result (forge#4267). The batch runner recovers `recoverable` rows once
  * (`orchestration.recovery.max_attempts`) and re-queues them; only `human` rows are escalations.
  */
-export const RECOVERABLE_CODES = Object.freeze([OUTCOME_CODES.CI_RED, OUTCOME_CODES.BASE_CONFLICT, OUTCOME_CODES.BUILDER_BLOCK_UNTYPED, OUTCOME_CODES.CONTRACT_INVALID, OUTCOME_CODES.ENV_TOOL_MISSING, OUTCOME_CODES.ENV_REPRO_UNRUNNABLE]);
+export const RECOVERABLE_CODES = Object.freeze([OUTCOME_CODES.CI_RED, OUTCOME_CODES.BASE_CONFLICT, OUTCOME_CODES.BUILDER_BLOCK_UNTYPED, OUTCOME_CODES.CONTRACT_INVALID, OUTCOME_CODES.ENV_TOOL_MISSING, OUTCOME_CODES.ENV_REPRO_UNRUNNABLE, OUTCOME_CODES.BUDGET_EXCEEDED]);
 /** `builder.blocked` is only reached by a typed human block: credentials, spend, promotion, customer, product-decision. */
 export const HUMAN_CODES = Object.freeze([OUTCOME_CODES.BUILDER_BLOCKED]);
 
 /**
  * `recoverable` | `human` | `terminal` for a lane result row ({outcome|terminalReason, code, issueEdited?, conflicting?}).
- * Recoverable: needs-human `ci.red`, `base.conflict`, `builder.block_untyped`, `env.tool_missing`, `env.repro_unrunnable`, `contract.invalid` after a mid-run issue
+ * Recoverable: needs-human `ci.red`, `base.conflict`, `builder.block_untyped`, `env.tool_missing`, `env.repro_unrunnable`, `budget.exceeded`, `contract.invalid` after a mid-run issue
  * edit (`issueEdited`), and `ci-pending` on a conflicting PR (`conflicting`). Human: `builder.blocked`. Anything else
  * (another code, or not a needs-human/ci-pending row) is terminal. Pure, never throws.
  */

package/bin/engine/runlog.mjs

@@ -1,17 +1,21 @@
 /**
  * Append-only per-issue run-log (the crash-safe local hot path).
- * One JSON event per line at {dir}/{issue}.jsonl. seq is assigned monotonically.
+ * One JSON event per line at {dir}/{owner}__{repo}__{issue}.jsonl (#4432: issue numbers are per repo), or the legacy
+ * {dir}/{issue}.jsonl when no repo is given. seq is assigned monotonically.
  */
-import { readFileSync, appendFileSync, writeFileSync, existsSync, mkdirSync } from "node:fs";
+import { readFileSync, appendFileSync, writeFileSync, existsSync, mkdirSync, readdirSync, renameSync, unlinkSync } from "node:fs";
 import { join } from "node:path";
 
-function logPath(dir, issue) { return join(dir, `${issue}.jsonl`); }
+const segment = (s) => { const v = String(s ?? "").replace(/[^A-Za-z0-9._-]/g, "_"); return !v || /^\.+$/.test(v) ? "_" : v; };
+/** #4432: `<owner>__<repo>__<issue>` with a repo (same scheme as pitfalls.mjs), else the legacy `<issue>`. */
+export const runLogKey = (issue, repo = null) => (repo ? `${String(repo).split("/").map(segment).join("__")}__${issue}` : String(issue));
+export function runLogPath(dir, issue, repo = null) { return join(dir, `${runLogKey(issue, repo)}.jsonl`); }
 
-/** Append an event; assigns the next seq based on the current line count. */
-export function appendEvent(dir, issue, event) {
+/** Append an event; assigns the next seq based on the current line count (of the repo file only). */
+export function appendEvent(dir, issue, event, { repo = null } = {}) {
   mkdirSync(dir, { recursive: true });
-  const path = logPath(dir, issue);
-  const seq = readLog(dir, issue).length + 1;
+  const path = runLogPath(dir, issue, repo);
+  const seq = parseLog(path).length + 1;
   // forge#3696: wall-clock ISO-8601 `ts` on every appended event. Placed before the
   // spread so a caller-supplied `ts` wins. rewriteLog() deliberately does not stamp:
   // a hydrated rewrite time would be a false timestamp. Old logs lack `ts` and still parse.
@@ -23,16 +27,39 @@ export function appendEvent(dir, issue, event) {
  * Used to reconstruct the local log from a remote FORGE:STATE index on
  * hydrate (C2), so the local log and the GitHub-wins state stay consistent.
  */
-export function rewriteLog(dir, issue, events) {
+export function rewriteLog(dir, issue, events, { repo = null } = {}) {
   mkdirSync(dir, { recursive: true });
-  const path = logPath(dir, issue);
+  const path = runLogPath(dir, issue, repo);
   const lines = events.map((e, i) => JSON.stringify({ seq: i + 1, ...e }));
   writeFileSync(path, lines.length ? lines.join("\n") + "\n" : "");
 }
 
-/** Read all complete events; a truncated final line (crash mid-write) is ignored. */
-export function readLog(dir, issue) {
-  const path = logPath(dir, issue);
+/**
+ * Read all complete events; a truncated final line (crash mid-write) is ignored.
+ * #4432: with a repo whose file does not exist yet, the legacy `<issue>.jsonl` slices of that repo are returned
+ * (read fallback for one release: remove after one release, together with migrateRunLogs).
+ */
+export function readLog(dir, issue, { repo = null } = {}) {
+  const path = runLogPath(dir, issue, repo);
+  if (repo && !existsSync(path)) return repoSlices(parseLog(runLogPath(dir, issue)), repo);
+  return parseLog(path);
+}
+
+/** The RUN_START slices of `events` whose RUN_START.repo is `repo` (events before the first RUN_START excluded). */
+function repoSlices(events, repo) {
+  let cur = null;
+  const out = [];
+  for (const e of events) {
+    if (e.event === "RUN_START") cur = e.repo ?? null;
+    if (cur != null && cur === repo) out.push(e);
+  }
+  return out;
+}
+
+/** #4432: the events of one run-log file by path (for enumerators that list the dir); same parse rules as readLog. */
+export const readLogFile = (path) => parseLog(path);
+
+function parseLog(path) {
   if (!existsSync(path)) return [];
   const lines = readFileSync(path, "utf-8").split("\n");
   // Drop trailing empty lines to find the last non-empty line.
@@ -55,6 +82,49 @@ export function readLog(dir, issue) {
   return out;
 }
 
+/**
+ * #4432: split each legacy `<issue>.jsonl` (shared by every repo on the host) into the repo files by RUN_START.repo.
+ * The file is claimed by an atomic rename (one winner); migrated slices go before any events already in the repo
+ * file, seq renumbered 1..n (appendEvent derives seq from the line count), other fields kept. Slices without a repo
+ * (and events before the first RUN_START) stay in `<issue>.jsonl`. Idempotent, never throws. Remove after one release.
+ */
+export function migrateRunLogs(dir) {
+  const res = { files: 0, slices: 0, kept: 0 };
+  let names = [];
+  try { names = readdirSync(dir); } catch { return res; }
+  for (const f of names) {
+    const m = /^(\d+)\.jsonl$/.exec(f);
+    if (!m) continue;
+    const legacy = join(dir, f), claim = `${legacy}.migrating-${process.pid}`;
+    try { renameSync(legacy, claim); } catch { continue; } // another process won, or it is gone
+    try {
+      let events = [];
+      try { events = parseLog(claim); } catch { renameSync(claim, legacy); continue; } // corrupt: leave as it was
+      const byRepo = new Map(), kept = [];
+      let cur = null;
+      for (const e of events) {
+        if (e.event === "RUN_START") { cur = e.repo ?? null; if (cur) { byRepo.set(cur, byRepo.get(cur) ?? []); res.slices++; } else res.kept++; }
+        (cur ? byRepo.get(cur) : kept).push(e);
+      }
+      for (const [repo, evs] of byRepo) {
+        const target = runLogPath(dir, m[1], repo), tmp = `${target}.tmp-${process.pid}`;
+        const all = [...evs, ...parseLog(target)].map(({ seq, ...e }, i) => JSON.stringify({ seq: i + 1, ...e }));
+        writeFileSync(tmp, all.join("\n") + "\n");
+        renameSync(tmp, target);
+      }
+      if (kept.length) {
+        const n = parseLog(legacy).length; // a repo-less writer may have recreated it meanwhile
+        appendFileSync(legacy, kept.map(({ seq, ...e }, i) => JSON.stringify({ seq: n + i + 1, ...e })).join("\n") + "\n");
+      }
+      unlinkSync(claim);
+      res.files++;
+    } catch {
+      try { if (existsSync(claim)) renameSync(claim, legacy); } catch { /* best effort */ }
+    }
+  }
+  return res;
+}
+
 /** Fold events into a RunState. The commit rule lives here. */
 /**
  * #4109: the run log is shared by the full and the core path. Keep only the events of runs whose `path` (from RUN_START;

package/bin/engine/telemetry-seam.mjs

@@ -3,8 +3,10 @@
 /**
  * forge#3835: the engine's `emitTelemetry` seam writer. Maps the run-log shaped records the engine hands to the
  * seam onto FORGE:TELEMETRY events so `forgedock stats` can rebuild Cost, Time and outcome codes from GitHub alone:
- *   PHASE_ATTEMPT_USAGE -> `phase_attempt` (attempt, model, backend, turns, duration_ms, cost_usd, *_tokens)
- *   RUN_TERMINAL        -> `run_terminal`  (reason, code, wall_ms; lean adds milestone, conformance_mode, shadow_would and the pr)
+ *   PHASE_ATTEMPT_USAGE -> `phase_attempt` (attempt, model, backend, turns, duration_ms, cost_usd, cost_usd_delta, cost_usd_session, *_tokens;
+ *                          lean adds baseline_tokens and lane_profile, forge#4395)
+ *   RUN_TERMINAL        -> `run_terminal`  (reason, code, wall_ms; lean adds milestone, conformance_mode, shadow_would, the pr and
+ *                          builder_tier/class/model/escalated 0|1, forge#4394)
  *   SANDBOX_USAGE       -> `sandbox_usage` (provider, sandbox_id, minutes, cost_usd, batch; one per settled remote lane, forge#4255)
  *   STRUCTURE_DELTA     -> `structure_delta` (edges_added/removed, import_files_added/removed, boundary_changed/loosened 0|1; forge#4159)
  *                          lean's merge-time record (forge#4165) adds milestone, the pr and string lists: deps_added and
@@ -51,15 +53,20 @@ export function mapRecord(rec, wallMs) {
   if (!rec || typeof rec !== "object") return null;
   if (rec.event === "PHASE_ATTEMPT_USAGE") {
     return { event: "phase_attempt", phase: rec.phase, payload: compact({
-      attempt: rec.attempt, model: rec.model, backend: rec.backend ?? undefined, stages: rec.stages, turns: rec.turns, duration_ms: rec.duration_ms,
+      attempt: rec.attempt, trigger: rec.trigger ?? undefined, restart_of: rec.restart_of ?? undefined, model: rec.model, backend: rec.backend ?? undefined, stages: rec.stages, turns: rec.turns, duration_ms: rec.duration_ms,
       cost_usd: Number.isFinite(rec.cost_usd) ? Math.round(rec.cost_usd * 1e6) / 1e6 : undefined,
+      cost_usd_delta: Number.isFinite(rec.cost_usd_delta) ? Math.round(rec.cost_usd_delta * 1e6) / 1e6 : undefined, // #4407
+      cost_usd_session: Number.isFinite(rec.cost_usd_session) ? Math.round(rec.cost_usd_session * 1e6) / 1e6 : undefined,
       input_tokens: tok(rec.usage, "input_tokens"), output_tokens: tok(rec.usage, "output_tokens"),
       cache_read_tokens: tok(rec.usage, "cache_read_input_tokens"), cache_creation_tokens: tok(rec.usage, "cache_creation_input_tokens"),
+      baseline_tokens: rec.baseline_tokens, lane_profile: rec.lane_profile,
     }) };
   }
   if (rec.event === "RUN_TERMINAL") {
     return { event: "run_terminal", phase: null, ...prOf(rec), payload: compact({ reason: rec.reason, code: rec.code, wall_ms: wallMs,
-      milestone: rec.milestone, conformance_mode: rec.conformance_mode, shadow_would: rec.shadow_would }) };
+      milestone: rec.milestone, conformance_mode: rec.conformance_mode, shadow_would: rec.shadow_would,
+      builder_tier: rec.builder_tier, builder_class: rec.builder_class, builder_model: rec.builder_model, // #4394: the model per merge
+      builder_escalated: typeof rec.builder_escalated === "boolean" ? Number(rec.builder_escalated) : undefined }) };
   }
   if (rec.event === "SANDBOX_USAGE") {
     return { event: "sandbox_usage", phase: null, payload: compact({ provider: rec.provider, sandbox_id: rec.sandbox_id, minutes: rec.minutes,

package/bin/engine/telemetry-seam.test.mjs

@@ -14,8 +14,26 @@ function harness(over = {}) {
 }
 
 describe("engine telemetry seam", () => {
+  it("#4362: phase_attempt carries the trigger, omitted when absent", () => {
+    assert.equal(mapRecord({ event: "PHASE_ATTEMPT_USAGE", phase: "implement", attempt: 3, trigger: "ci_red" }, 0).payload.trigger, "ci_red");
+    assert.equal("trigger" in mapRecord({ event: "PHASE_ATTEMPT_USAGE", phase: "implement", attempt: 1, trigger: null }, 0).payload, false);
+    assert.equal("trigger" in mapRecord({ event: "PHASE_ATTEMPT_USAGE", phase: "implement", attempt: 1 }, 0).payload, false);
+  });
+
+  it("forge#4395: phase_attempt carries baseline_tokens and lane_profile, omitted when absent", () => {
+    const p = mapRecord({ event: "PHASE_ATTEMPT_USAGE", phase: "implement", attempt: 1, baseline_tokens: 21000, lane_profile: "lane" }, 0).payload;
+    assert.equal(p.baseline_tokens, 21000); assert.equal(p.lane_profile, "lane");
+    const q = mapRecord({ event: "PHASE_ATTEMPT_USAGE", phase: "implement", attempt: 1 }, 0).payload;
+    assert.equal("baseline_tokens" in q, false); assert.equal("lane_profile" in q, false);
+  });
+
   it("forge#4257: phase_attempt carries the backend, omitted when unreported", () => {
     assert.equal(mapRecord({ event: "PHASE_ATTEMPT_USAGE", phase: "build", attempt: 1, backend: "api" }, 0).payload.backend, "api");
+    // #4407: the per-attempt cost and the session running total ride along; absent fields are omitted
+    const c = mapRecord({ event: "PHASE_ATTEMPT_USAGE", phase: "implement", attempt: 3, cost_usd: 2.5, cost_usd_delta: 0.123456789, cost_usd_session: 2.5 }, 0).payload;
+    assert.deepEqual([c.cost_usd, c.cost_usd_delta, c.cost_usd_session], [2.5, 0.123457, 2.5]);
+    const none = mapRecord({ event: "PHASE_ATTEMPT_USAGE", phase: "implement", attempt: 1, cost_usd: 1, cost_usd_delta: null }, 0).payload;
+    assert.equal("cost_usd_delta" in none, false); assert.equal("cost_usd_session" in none, false);
     assert.equal("backend" in mapRecord({ event: "PHASE_ATTEMPT_USAGE", phase: "build", attempt: 1, backend: null }, 0).payload, false);
     assert.equal("backend" in mapRecord({ event: "PHASE_ATTEMPT_USAGE", phase: "build", attempt: 1 }, 0).payload, false);
   });