forgedock 1.34.0 → 1.36.0

package/bin/engine/events.mjs

@@ -0,0 +1,152 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Versioned outbound lifecycle events (#4630, docs/EVENTS.md). The engine hands the same run-log shaped records it gives
+ * the telemetry seam to `makeEventEmitter`, which maps them to a stable envelope, appends it to a local JSONL file and,
+ * when forge.yaml `events.webhook_url` and the secret env var are both set, POSTs it signed with HMAC-SHA256. No model call.
+ * The emitter NEVER throws; FORGE_EVENTS=off disables it. Values are ids, counts, codes and costs only.
+ */
+import { createHmac, randomBytes } from "node:crypto";
+import { appendFileSync, mkdirSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+export const EVENT_SCHEMA = "forgedock.event";
+export const EVENT_VERSION = 1;
+export const DEFAULT_SECRET_ENV = "FORGE_EVENTS_SECRET";
+export const WEBHOOK_TIMEOUT_MS = 5000;
+
+/** type -> data keys (documented in docs/EVENTS.md; the contract test pins them). */
+export const EVENT_TYPES = {
+  "lane.picked_up": ["lane", "path"],
+  "contract.frozen": ["hash", "requirements", "deferred", "derived"],
+  "pr.opened": [],
+  "pr.merged": ["milestone", "code"],
+  "release.promoted": ["fix", "default_branch"],
+  "probe.passed": ["detail", "fix"],
+  "probe.failed": ["detail", "fix"],
+  "regression.detected": ["subject", "metric", "detail"],
+  "experiment.verdict": ["experiment", "verdict", "detail"],
+};
+
+const scalar = (v) => (typeof v === "number" ? Number.isFinite(v) : typeof v === "boolean" || (typeof v === "string" && v !== ""));
+const round6 = (n) => Math.round(n * 1e6) / 1e6;
+const pickData = (type, src) => Object.fromEntries(EVENT_TYPES[type].filter((k) => scalar(src?.[k])).map((k) => [k, typeof src[k] === "string" ? src[k].slice(0, 200) : src[k]]));
+
+/** Envelope for one event; null for an unknown type. `ctx` = {repo, issue, run, pr, cost_usd, wall_ms}. */
+export function buildEvent(type, data = {}, ctx = {}, now = new Date()) {
+  if (!Object.hasOwn(EVENT_TYPES, type)) return null;
+  const int = (v) => (Number.isSafeInteger(v) && v > 0 ? v : null);
+  return {
+    schema: EVENT_SCHEMA, v: EVENT_VERSION, id: `${Math.floor(now.getTime() / 1000)}-${randomBytes(6).toString("hex")}`, type,
+    ts: now.toISOString().replace(/\.\d+Z$/, "Z"), repo: ctx.repo ?? null, issue: int(ctx.issue), pr: int(ctx.pr), run: ctx.run ?? null,
+    ...(Number.isFinite(ctx.cost_usd) ? { cost_usd: round6(ctx.cost_usd) } : {}), ...(Number.isFinite(ctx.wall_ms) ? { wall_ms: Math.round(ctx.wall_ms) } : {}),
+    data: pickData(type, data),
+  };
+}
+
+/** Probe sweep row ({issue, outcome, detail, fix}) -> event type + data; null unless pass or fail. */
+export function probeEvent(row) {
+  if (row?.outcome !== "pass" && row?.outcome !== "fail") return null;
+  return { type: row.outcome === "pass" ? "probe.passed" : "probe.failed", data: { detail: row.detail, fix: row.fix }, ctx: { issue: row.issue } };
+}
+
+/** Monitor sweep row ({issue, outcome, detail, regression}) -> regression.detected; null unless a fail that filed a regression issue. */
+export function regressionEvent(row) {
+  if (row?.outcome !== "fail" || !Number.isSafeInteger(row.regression) || row.regression < 1) return null;
+  return { type: "regression.detected", data: { subject: `post-deploy check of #${row.issue}`, metric: "post_deploy_check", detail: row.detail }, ctx: { issue: row.issue } };
+}
+
+/** Experiment report ({id, targets, targets_met}) -> experiment.verdict (met | missed | inconclusive); null without an id. */
+export function experimentEvent(report) {
+  if (!report || typeof report.id !== "string" || !report.id) return null;
+  const passes = Object.values(report.targets ?? {}).map((t) => t?.pass);
+  const verdict = report.targets_met === true ? "met" : passes.includes(false) ? "missed" : "inconclusive";
+  return { type: "experiment.verdict", data: { experiment: report.id, verdict, detail: `${passes.length} target(s)` }, ctx: {} };
+}
+
+/** A fix reached the default branch -> release.promoted; null without a fix. Reserved: no detection emits it yet. */
+export function promotedEvent({ fix, default_branch } = {}) {
+  return typeof fix === "string" && fix ? { type: "release.promoted", data: { fix, default_branch }, ctx: {} } : null;
+}
+
+/**
+ * Pure mapping of one run-log record to {type, data, ctx} or null. `cost` is the lane's running spend, kept by the caller
+ * from PHASE_ATTEMPT_USAGE (`cost_usd_delta` when present, else `cost_usd`).
+ */
+export function mapRunRecord(rec, { cost = 0 } = {}) {
+  if (!rec || typeof rec !== "object") return null;
+  switch (rec.event) {
+    case "RUN_START": return { type: "lane.picked_up", data: rec, ctx: {} };
+    case "CONTRACT_FROZEN": return { type: "contract.frozen", data: rec, ctx: {} };
+    case "PR_OPENED": return { type: "pr.opened", data: {}, ctx: { pr: rec.pr } };
+    case "RUN_TERMINAL":
+      if (rec.reason !== "merged") return null;
+      return { type: "pr.merged", data: rec, ctx: { pr: rec.pr, cost_usd: cost, wall_ms: rec.wall_ms } };
+    default: return null;
+  }
+}
+
+/** HMAC-SHA256 hex of `${ts}.${body}`; the receiver recomputes it from X-Forge-Timestamp and the raw body. */
+export const sign = (secret, ts, body) => createHmac("sha256", secret).update(`${ts}.${body}`).digest("hex");
+
+/** https, or http only for a loopback host; anything else is not a usable webhook. */
+export function webhookUrlOk(u) {
+  try {
+    const url = new URL(u);
+    return url.protocol === "https:" || (url.protocol === "http:" && ["localhost", "127.0.0.1", "[::1]"].includes(url.hostname));
+  } catch { return false; }
+}
+
+/** POST one envelope; resolves true on 2xx. Never throws. */
+export async function postWebhook(event, { url, secret, fetchFn = globalThis.fetch, timeoutMs = WEBHOOK_TIMEOUT_MS, now = Date.now } = {}) {
+  try {
+    if (!url || !secret || !webhookUrlOk(url) || typeof fetchFn !== "function") return false;
+    const body = JSON.stringify(event);
+    const ts = String(Math.floor(now() / 1000));
+    const res = await fetchFn(url, { method: "POST", redirect: "error", signal: AbortSignal.timeout(timeoutMs),
+      headers: { "content-type": "application/json", "x-forge-event": event.type, "x-forge-timestamp": ts, "x-forge-signature": `sha256=${sign(secret, ts, body)}` }, body });
+    return Boolean(res?.ok);
+  } catch { return false; }
+}
+
+/** Local sink path: `dir` (config or FORGE_EVENTS_DIR), else ~/.forge/events; one file per repo. */
+export const sinkPath = (repo, dir) => join(dir || join(homedir(), ".forge", "events"), `${String(repo || "unknown").replace(/[^\w.-]/g, "_")}.jsonl`);
+
+/**
+ * @param {{repo?: string|null, config?: {webhookUrl?: string, secretEnv?: string, dir?: string}, env?: object, fetchFn?: Function, now?: () => Date, run?: string}} o
+ * @returns {{emit: (rec: object) => Promise<void>, emitEvent: (type: string, data?: object, ctx?: object) => Promise<object|null>}}
+ */
+export function makeEventEmitter({ repo = null, config = {}, env = process.env, fetchFn, now = () => new Date(), run = null } = {}) {
+  let cost = 0, runId = run;
+  const off = () => (env.FORGE_EVENTS || "on") === "off";
+  async function emitEvent(type, data = {}, ctx = {}) {
+    try {
+      if (off()) return null;
+      const ev = buildEvent(type, data, { repo, run: runId, ...ctx }, now());
+      if (!ev) return null;
+      try {
+        const file = sinkPath(repo, env.FORGE_EVENTS_DIR || config.dir);
+        mkdirSync(join(file, ".."), { recursive: true });
+        appendFileSync(file, `${JSON.stringify(ev)}\n`);
+      } catch { /* a lost local line costs completeness only */ }
+      const secret = env[config.secretEnv || DEFAULT_SECRET_ENV];
+      if (config.webhookUrl && secret) await postWebhook(ev, { url: config.webhookUrl, secret, fetchFn, now: () => now().getTime() });
+      return ev;
+    } catch { return null; }
+  }
+  async function emit(rec) {
+    try {
+      if (rec?.event === "PHASE_ATTEMPT_USAGE") {
+        const c = Number.isFinite(rec.cost_usd_delta) ? rec.cost_usd_delta : rec.cost_usd;
+        if (Number.isFinite(c)) cost += c;
+        return;
+      }
+      if (rec?.event === "RUN_START" && typeof rec.run === "string") runId = rec.run;
+      const m = mapRunRecord(rec, { cost });
+      if (!m) return;
+      await emitEvent(m.type, m.data, { issue: rec.issue, ...m.ctx });
+    } catch { /* events never fail a run */ }
+  }
+  return { emit, emitEvent };
+}

package/bin/engine/events.test.mjs

@@ -0,0 +1,128 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// Contract test for the outbound event stream (#4630, docs/EVENTS.md): changing a type, key or the signature is a breaking change.
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { readFileSync, existsSync, writeFileSync } from "node:fs";
+import { tmpDir } from "../tests/_tmp.mjs";
+import { join } from "node:path";
+import { createHmac } from "node:crypto";
+import { EVENT_TYPES, EVENT_VERSION, buildEvent, mapRunRecord, makeEventEmitter, probeEvent, regressionEvent, experimentEvent, promotedEvent, sign, webhookUrlOk, postWebhook } from "./events.mjs";
+
+const tmp = () => tmpDir("forge-events-");
+const fixed = () => new Date("2026-10-26T12:00:00Z");
+const lines = (dir) => readFileSync(join(dir, "o_r.jsonl"), "utf8").trim().split("\n").map((l) => JSON.parse(l));
+
+describe("event contract (v1)", () => {
+  it("documents exactly the named lifecycle types", () => {
+    assert.deepEqual(Object.keys(EVENT_TYPES), ["lane.picked_up", "contract.frozen", "pr.opened", "pr.merged", "release.promoted", "probe.passed", "probe.failed", "regression.detected", "experiment.verdict"]);
+  });
+
+  it("every envelope has the same top-level keys and only documented data keys", () => {
+    for (const [type, keys] of Object.entries(EVENT_TYPES)) {
+      const data = Object.fromEntries(keys.map((k) => [k, "x"]));
+      const e = buildEvent(type, { ...data, secret: "no" }, { repo: "o/r", issue: 3, pr: 4, run: "core_3_1", cost_usd: 1.23456789, wall_ms: 10.4 }, fixed());
+      assert.deepEqual(Object.keys(e), ["schema", "v", "id", "type", "ts", "repo", "issue", "pr", "run", "cost_usd", "wall_ms", "data"]);
+      assert.equal(e.schema, "forgedock.event"); assert.equal(e.v, EVENT_VERSION); assert.equal(e.ts, "2026-10-26T12:00:00Z");
+      assert.equal(e.cost_usd, 1.234568); assert.equal(e.wall_ms, 10);
+      assert.deepEqual(Object.keys(e.data), keys);
+    }
+    assert.equal(buildEvent("nope", {}, {}, fixed()), null);
+  });
+
+  it("maps run-log records and sums attempt cost onto pr.merged", async () => {
+    const dir = tmp();
+    const { emit } = makeEventEmitter({ repo: "o/r", config: { dir }, env: {}, now: fixed });
+    await emit({ event: "RUN_START", run: "core_9_5", issue: 9, lane: "staging", path: "core" });
+    await emit({ event: "CONTRACT_FROZEN", issue: 9, hash: "abc", requirements: 2, deferred: 1 });
+    await emit({ event: "PHASE_ATTEMPT_USAGE", issue: 9, cost_usd: 5, cost_usd_delta: 0.5 });
+    await emit({ event: "PHASE_ATTEMPT_USAGE", issue: 9, cost_usd: 0.25 });
+    await emit({ event: "PR_OPENED", issue: 9, pr: 40 });
+    await emit({ event: "RUN_TERMINAL", issue: 9, reason: "needs-human", wall_ms: 1 });
+    await emit({ event: "RUN_TERMINAL", issue: 9, reason: "merged", pr: 40, wall_ms: 90000, code: "ok", milestone: "m1" });
+    const got = lines(dir);
+    assert.deepEqual(got.map((e) => e.type), ["lane.picked_up", "contract.frozen", "pr.opened", "pr.merged"]);
+    assert.deepEqual(got[0].data, { lane: "staging", path: "core" });
+    assert.equal(got[0].run, "core_9_5"); assert.equal(got[3].run, "core_9_5");
+    assert.deepEqual(got[1].data, { hash: "abc", requirements: 2, deferred: 1 });
+    assert.equal(got[2].pr, 40);
+    assert.equal(got[3].cost_usd, 0.75); assert.equal(got[3].wall_ms, 90000); assert.equal(got[3].issue, 9);
+    assert.equal(mapRunRecord({ event: "CI_PENDING_WAIT" }), null);
+  });
+
+  it("maps probe rows: only pass and fail", () => {
+    assert.equal(probeEvent({ issue: 1, outcome: "awaiting-promotion" }), null);
+    assert.equal(probeEvent({ issue: 1, outcome: "pass", detail: "exit 0", fix: "abc" }).type, "probe.passed");
+    assert.equal(probeEvent({ issue: 1, outcome: "fail" }).type, "probe.failed");
+  });
+});
+
+describe("regression, experiment and promoted builders (#4632)", () => {
+  it("regressionEvent: only a fail row that filed a regression issue", () => {
+    assert.equal(regressionEvent({ issue: 5, outcome: "pass" }), null);
+    assert.equal(regressionEvent({ issue: 5, outcome: "fail" }), null);
+    const r = regressionEvent({ issue: 5, outcome: "fail", regression: 7, detail: "broke" });
+    assert.equal(r.type, "regression.detected"); assert.equal(r.ctx.issue, 5);
+    assert.deepEqual(r.data, { subject: "post-deploy check of #5", metric: "post_deploy_check", detail: "broke" });
+  });
+  it("experimentEvent: met, missed, inconclusive", () => {
+    const t = (...p) => Object.fromEntries(p.map((pass, i) => [`t${i}`, { pass }]));
+    assert.equal(experimentEvent({ id: "e", targets: t(true), targets_met: true }).data.verdict, "met");
+    assert.equal(experimentEvent({ id: "e", targets: t(true, false), targets_met: false }).data.verdict, "missed");
+    assert.equal(experimentEvent({ id: "e", targets: t(null), targets_met: false }).data.verdict, "inconclusive");
+    assert.equal(experimentEvent({}), null);
+  });
+  it("promotedEvent needs a fix; types and keys stay pinned", () => {
+    assert.equal(promotedEvent({}), null);
+    assert.deepEqual(promotedEvent({ fix: "abc", default_branch: "main" }).data, { fix: "abc", default_branch: "main" });
+    assert.deepEqual(EVENT_TYPES["regression.detected"], ["subject", "metric", "detail"]);
+    assert.deepEqual(EVENT_TYPES["experiment.verdict"], ["experiment", "verdict", "detail"]);
+  });
+  it("a regression row is written to the sink", async () => {
+    const dir = tmp();
+    const p = regressionEvent({ issue: 5, outcome: "fail", regression: 7, detail: "d" });
+    await makeEventEmitter({ repo: "o/r", config: { dir }, env: {} }).emitEvent(p.type, p.data, p.ctx);
+    const ev = JSON.parse(readFileSync(join(dir, "o_r.jsonl"), "utf8"));
+    assert.equal(ev.type, "regression.detected"); assert.equal(ev.issue, 5);
+  });
+});
+
+describe("webhook", () => {
+  const cfg = { webhookUrl: "https://hooks.example/x", secretEnv: "MY_SECRET", dir: undefined };
+  it("signs `${timestamp}.${body}` with HMAC-SHA256 from the env secret", async () => {
+    const calls = [];
+    const fetchFn = async (url, init) => { calls.push([url, init]); return { ok: true }; };
+    const { emitEvent } = makeEventEmitter({ repo: "o/r", config: { ...cfg, dir: tmp() }, env: { MY_SECRET: "s3" }, fetchFn, now: fixed });
+    const ev = await emitEvent("pr.opened", {}, { issue: 1, pr: 2 });
+    assert.equal(calls.length, 1);
+    const [url, init] = calls[0];
+    assert.equal(url, cfg.webhookUrl); assert.equal(init.method, "POST");
+    assert.equal(init.body, JSON.stringify(ev));
+    const ts = init.headers["x-forge-timestamp"];
+    assert.equal(ts, String(Math.floor(fixed().getTime() / 1000)));
+    assert.equal(init.headers["x-forge-signature"], `sha256=${createHmac("sha256", "s3").update(`${ts}.${init.body}`).digest("hex")}`);
+    assert.equal(init.headers["x-forge-event"], "pr.opened");
+    assert.equal(sign("s3", ts, init.body), init.headers["x-forge-signature"].slice(7));
+  });
+
+  it("sends nothing without the secret, with a non-https URL, or when off", async () => {
+    let n = 0;
+    const fetchFn = async () => { n++; return { ok: true }; };
+    for (const [config, env] of [[cfg, {}], [{ ...cfg, webhookUrl: "http://evil.example/x" }, { MY_SECRET: "s" }], [cfg, { MY_SECRET: "s", FORGE_EVENTS: "off" }]]) {
+      await makeEventEmitter({ repo: "o/r", config: { ...config, dir: tmp() }, env, fetchFn }).emitEvent("pr.opened", {}, { issue: 1 });
+    }
+    assert.equal(n, 0);
+    assert.equal(webhookUrlOk("http://localhost:3000/h"), true); assert.equal(webhookUrlOk("ftp://x"), false);
+  });
+
+  it("off writes no file; sink and webhook failures never throw", async () => {
+    const dir = tmp();
+    await makeEventEmitter({ repo: "o/r", config: { dir }, env: { FORGE_EVENTS: "off" } }).emitEvent("pr.opened", {}, { issue: 1 });
+    assert.equal(existsSync(join(dir, "o_r.jsonl")), false);
+    const boom = async () => { throw new Error("net"); };
+    writeFileSync(join(dir, "f"), ""); const bad = makeEventEmitter({ repo: "o/r", config: { ...cfg, dir: join(dir, "f", "x") }, env: { MY_SECRET: "s" }, fetchFn: boom });
+    assert.ok(await bad.emitEvent("pr.opened", {}, { issue: 1 }));
+    await bad.emit({ event: "PR_OPENED", pr: 1, issue: 1 });
+    assert.equal(await postWebhook({ type: "x" }, { url: cfg.webhookUrl, secret: "s", fetchFn: boom }), false);
+  });
+});

package/bin/engine/telemetry-seam.mjs

@@ -3,7 +3,7 @@
 /**
  * forge#3835: the engine's `emitTelemetry` seam writer. Maps the run-log shaped records the engine hands to the
  * seam onto FORGE:TELEMETRY events so `forgedock stats` can rebuild Cost, Time and outcome codes from GitHub alone:
- *   PHASE_ATTEMPT_USAGE -> `phase_attempt` (attempt, model, backend, turns, duration_ms, cost_usd, cost_usd_delta, cost_usd_session, *_tokens;
+ *   PHASE_ATTEMPT_USAGE -> `phase_attempt` (experiment + arm when forge.yaml declares one, forge#4629; attempt, model, backend, turns, duration_ms, cost_usd, cost_usd_delta, cost_usd_session, *_tokens;
  *                          lean adds baseline_tokens and lane_profile, forge#4395)
  *   RUN_TERMINAL        -> `run_terminal`  (reason, code, wall_ms; lean adds milestone, conformance_mode, shadow_would, the pr and
  *                          builder_tier/class/model/escalated 0|1, forge#4394)
@@ -59,14 +59,15 @@ export function mapRecord(rec, wallMs) {
       cost_usd_session: Number.isFinite(rec.cost_usd_session) ? Math.round(rec.cost_usd_session * 1e6) / 1e6 : undefined,
       input_tokens: tok(rec.usage, "input_tokens"), output_tokens: tok(rec.usage, "output_tokens"),
       cache_read_tokens: tok(rec.usage, "cache_read_input_tokens"), cache_creation_tokens: tok(rec.usage, "cache_creation_input_tokens"),
-      baseline_tokens: rec.baseline_tokens, lane_profile: rec.lane_profile,
+      baseline_tokens: rec.baseline_tokens, lane_profile: rec.lane_profile, experiment: rec.experiment, arm: rec.arm, // #4629
     }) };
   }
   if (rec.event === "RUN_TERMINAL") {
     return { event: "run_terminal", phase: null, ...prOf(rec), payload: compact({ reason: rec.reason, code: rec.code, wall_ms: wallMs,
       milestone: rec.milestone, conformance_mode: rec.conformance_mode, shadow_would: rec.shadow_would,
       builder_tier: rec.builder_tier, builder_class: rec.builder_class, builder_model: rec.builder_model, // #4394: the model per merge
-      builder_escalated: typeof rec.builder_escalated === "boolean" ? Number(rec.builder_escalated) : undefined }) };
+      builder_escalated: typeof rec.builder_escalated === "boolean" ? Number(rec.builder_escalated) : undefined,
+      experiment: rec.experiment, arm: rec.arm }) }; // #4629
   }
   if (rec.event === "SANDBOX_USAGE") {
     return { event: "sandbox_usage", phase: null, payload: compact({ provider: rec.provider, sandbox_id: rec.sandbox_id, minutes: rec.minutes,