forgedock 1.16.2 → 1.17.0

package/bin/engine/telemetry.mjs

@@ -0,0 +1,148 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Node twin of scripts/lib/telemetry.sh: same record shape, same upsert into the trusted
+ * `<!-- FORGE:TELEMETRY v1 -->` comment, same rollover and local buffer. emit() NEVER throws and never
+ * rejects its caller; FORGE_TELEMETRY=off disables it. Records hold ids, counts and codes only.
+ */
+import { spawnSync } from "node:child_process";
+import { randomBytes } from "node:crypto";
+import { accessSync, appendFileSync, constants, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { basename, dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const here = dirname(fileURLToPath(import.meta.url));
+const ROOT = join(here, "..", "..");
+const RESERVED = new Set(["v", "ts", "forge_version", "repo", "issue", "pr", "run", "phase", "event", "eid"]);
+const MARKER_RE = /^<!-- FORGE:TELEMETRY v1( part=(\d+))? -->/;
+const REVIEW_SIDE = new Set(["review_panel", "finding_disposition", "ci_gate"]);
+
+const clean = (v) => String(v).replace(/[\n\r\t]/g, " ").replace(/[^A-Za-z0-9 _.:/#@+=,-]/g, "_").slice(0, 200);
+const num = (s) => (/^[0-9]+$/.test(s ?? "") ? Number(s) : null);
+const str = (s) => (s ? String(s) : null);
+
+function forgeVersion() {
+  try { return JSON.parse(readFileSync(join(ROOT, "package.json"), "utf8")).version || null; } catch { return null; }
+}
+
+/** Same JSON line the shell emitter writes (key order included). `ctx` overrides the FORGE_* env. */
+export function buildRecord(event, payload = {}, ctx = {}, now = new Date()) {
+  const env = process.env;
+  const c = {
+    repo: ctx.repo ?? env.FORGE_REPO, issue: ctx.issue ?? env.FORGE_ISSUE, pr: ctx.pr ?? env.FORGE_PR,
+    run: ctx.run ?? env.FORGE_RUN, phase: ctx.phase ?? env.FORGE_PHASE,
+  };
+  const extra = {};
+  for (const [k, raw] of Object.entries(payload)) {
+    if (!/^[A-Za-z0-9_]+$/.test(k) || RESERVED.has(k)) continue;
+    const v = clean(raw);
+    extra[k] = /^-?[0-9]+$/.test(v) ? Number(v) : v === "true" ? true : v === "false" ? false : v;
+  }
+  return {
+    v: 1, ts: now.toISOString().replace(/\.\d+Z$/, "Z"), forge_version: forgeVersion(), repo: str(c.repo),
+    issue: num(String(c.issue ?? "")), pr: num(String(c.pr ?? "")), run: str(c.run), phase: str(c.phase), event,
+    eid: `${Math.floor(now.getTime() / 1000)}-${process.pid}-${randomBytes(4).toString("hex")}`, ...extra,
+  };
+}
+
+function target(event, rec) {
+  const order = REVIEW_SIDE.has(event) ? [rec.pr, rec.issue] : [rec.issue, rec.pr];
+  const n = order.find((x) => x != null);
+  return n != null && rec.repo && rec.repo.includes("/") ? n : null;
+}
+
+function gh(args) {
+  const bin = process.env.FORGE_TELEMETRY_GH || "gh";
+  const r = spawnSync(bin, args, { encoding: "utf8", timeout: 30000, maxBuffer: 64 * 1024 * 1024 });
+  if (r.error || r.status !== 0) throw new Error("gh failed");
+  return r.stdout;
+}
+
+function trusted(comment) {
+  const tc = join(ROOT, "scripts", "trusted-comments.sh");
+  const r = spawnSync("bash", [tc, "count", MARKER_RE.source], { input: JSON.stringify([comment]), encoding: "utf8" });
+  return r.status === 0 && r.stdout.trim() === "1";
+}
+
+function upsert(repo, n, lines) {
+  const max = Number(process.env.FORGE_TELEMETRY_MAX_CHARS) || 60000;
+  const raw = gh(["api", "--paginate", `repos/${repo}/issues/${n}/comments`]).trim();
+  const pages = raw ? JSON.parse(`[${raw.replace(/\]\s*\[/g, "],[")}]`) : [[]];
+  let best = null;
+  const parts = [];
+  for (const c of pages.flat()) {
+    const m = MARKER_RE.exec(c.body || "");
+    if (!m || !trusted(c)) continue;
+    const part = m[2] ? Number(m[2]) : 1;
+    parts.push({ body: c.body });
+    if (!best || part >= best.part) best = { id: c.id, part, body: c.body };
+  }
+  const all = parts.map((p) => p.body).join("\n");
+  const fresh = lines.filter((l) => !all.includes(l));
+  if (fresh.length === 0) return;
+  if (best) {
+    const cut = best.body.lastIndexOf("\n```");
+    const out = `${cut >= 0 ? best.body.slice(0, cut) : best.body}\n${fresh.join("\n")}\n\`\`\``;
+    if (out.length <= max) { gh(["api", "-X", "PATCH", `repos/${repo}/issues/comments/${best.id}`, "-f", `body=${out}`]); return; }
+  }
+  const hdr = best ? `<!-- FORGE:TELEMETRY v1 part=${best.part + 1} -->` : "<!-- FORGE:TELEMETRY v1 -->";
+  gh(["api", "-X", "POST", `repos/${repo}/issues/${n}/comments`, "-f", `body=${hdr}\n\`\`\`jsonl\n${fresh.join("\n")}\n\`\`\``]);
+}
+
+/** Emit one event. Returns the record when written or buffered, null when disabled/dropped. Never throws. */
+export function emit(event, payload = {}, ctx = {}) {
+  try {
+    if ((process.env.FORGE_TELEMETRY || "on") === "off") return null;
+    if (!/^[A-Za-z0-9_.-]+$/.test(event || "")) return null;
+    const rec = buildRecord(event, payload, ctx);
+    const n = target(event, rec);
+    if (n == null) return null;
+    const dir = process.env.FORGE_TELEMETRY_BUFFER_DIR || join(homedir(), ".forge", "telemetry-buffer");
+    const buf = join(dir, `${rec.repo.replace("/", "_")}__${n}.jsonl`);
+    const line = JSON.stringify(rec);
+    // Per-target lock (mkdir is atomic): serializes emitters on this host so the unversioned GitHub read-modify-write
+    // cannot lose a record or double-POST a part. Bounded wait (50 x 0.1s); a lock over 5 min old is stale and is
+    // reclaimed by an atomic rename (one winner); the holder writes an owner token and releases only its own lock.
+    // A non-writable buffer dir skips locking; on timeout the record is buffered.
+    let canLock = true;
+    try { mkdirSync(dir, { recursive: true }); accessSync(dir, constants.W_OK); } catch { canLock = false; }
+    const lock = join(dir, `.lock-${rec.repo.replace("/", "_")}__${n}`);
+    const token = `${process.pid}.${randomBytes(4).toString("hex")}`;
+    let got = !canLock;
+    for (let i = 0; i < 50 && !got; i++) {
+      try { mkdirSync(lock); writeFileSync(join(lock, "owner"), token); got = true; } catch {
+        let stale = false;
+        try { stale = Date.now() - statSync(lock).mtimeMs > 300000; } catch { /* raced */ }
+        if (stale) { try { renameSync(lock, `${lock}.stale.${token}`); rmSync(`${lock}.stale.${token}`, { recursive: true, force: true }); } catch { /* lost the race */ } }
+        else spawnSync("sleep", ["0.1"]);
+      }
+    }
+    if (!got) { appendFileSync(buf, `${line}\n`); return rec; }
+    const unlock = () => { if (!canLock) return; try { if (readFileSync(join(lock, "owner"), "utf8") === token) rmSync(lock, { recursive: true, force: true }); } catch { /* already gone */ } };
+    // Claim the buffer by renaming it: records appended meanwhile land in a fresh file and are never deleted. Holding
+    // the lock, any other "<buf>.flush.*" is an orphan of a killed emitter: fold it in.
+    const claimed = `${buf}.flush.${token}`;
+    let pending = [line];
+    try {
+      if (existsSync(buf)) renameSync(buf, claimed);
+      for (const f of readdirSync(dir)) {
+        const full = join(dir, f);
+        if (f.startsWith(`${basename(buf)}.flush.`) && full !== claimed) { appendFileSync(claimed, readFileSync(full, "utf8")); rmSync(full, { force: true }); }
+      }
+      if (existsSync(claimed)) pending = [...readFileSync(claimed, "utf8").split("\n").filter(Boolean), line];
+    } catch { /* nothing claimed */ }
+    try {
+      upsert(rec.repo, n, pending);
+      rmSync(claimed, { force: true });
+    } catch {
+      try {
+        if (existsSync(claimed)) { appendFileSync(buf, readFileSync(claimed, "utf8")); rmSync(claimed, { force: true }); }
+        appendFileSync(buf, `${line}\n`);
+      } catch { /* lost record costs completeness only */ }
+    } finally { unlock(); }
+    return rec;
+  } catch {
+    return null;
+  }
+}

package/bin/engine/telemetry.test.mjs

@@ -0,0 +1,120 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it, beforeEach, afterEach } from "node:test";
+import assert from "node:assert/strict";
+import { chmodSync, existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { spawnSync } from "node:child_process";
+import { buildRecord, emit } from "./telemetry.mjs";
+
+let td; let saved;
+const KEYS = ["FORGE_TELEMETRY", "FORGE_TELEMETRY_GH", "FORGE_TELEMETRY_BUFFER_DIR", "FORGE_TELEMETRY_MAX_CHARS", "FORGE_REPO", "FORGE_ISSUE", "FORGE_PR", "FORGE_RUN", "FORGE_PHASE", "TD"];
+
+// Stub gh: lists $TD/comments.json, records the last PATCH/POST body in $TD/patched|posted, logs calls.
+function stubGh(fail = false) {
+  const p = join(td, "gh");
+  writeFileSync(p, `#!/usr/bin/env bash
+echo "$*" >> "$TD/calls"
+${fail ? "exit 1" : ""}
+case "$*" in
+  "api --paginate "*) if [ -f "$TD/comments.json" ]; then cat "$TD/comments.json"; else echo '[]'; fi ;;
+  "api -X PATCH"*) b="\${@: -1}"; printf '%s' "\${b#body=}" > "$TD/patched" ;;
+  "api -X POST"*) b="\${@: -1}"; printf '%s' "\${b#body=}" > "$TD/posted" ;;
+esac
+`);
+  chmodSync(p, 0o755);
+  process.env.FORGE_TELEMETRY_GH = p;
+}
+const setComments = (bodies) => writeFileSync(join(td, "comments.json"),
+  JSON.stringify(bodies.map((b, i) => ({ id: 100 + i, body: b, user: { type: "Bot" }, author_association: "NONE" }))));
+
+beforeEach(() => {
+  saved = Object.fromEntries(KEYS.map((k) => [k, process.env[k]]));
+  td = mkdtempSync(join(tmpdir(), "tel-"));
+  Object.assign(process.env, { TD: td, FORGE_TELEMETRY_BUFFER_DIR: join(td, "buf"), FORGE_REPO: "o/r", FORGE_ISSUE: "5" });
+  for (const k of ["FORGE_TELEMETRY", "FORGE_PR", "FORGE_RUN", "FORGE_PHASE", "FORGE_TELEMETRY_MAX_CHARS"]) delete process.env[k];
+});
+afterEach(() => {
+  for (const k of KEYS) { if (saved[k] === undefined) delete process.env[k]; else process.env[k] = saved[k]; }
+  rmSync(td, { recursive: true, force: true });
+});
+
+describe("telemetry emitter (node twin)", () => {
+  it("builds the shared record shape with null context and typed payload", () => {
+    const r = buildRecord("diff_size", { lines: "10", excluded: 0, threshold: 0, result: "OK", bad: "a`b\nc" });
+    assert.deepEqual(Object.keys(r).slice(0, 9), ["v", "ts", "forge_version", "repo", "issue", "pr", "run", "phase", "event"]);
+    assert.equal(r.v, 1); assert.equal(r.issue, 5); assert.equal(r.pr, null); assert.equal(r.run, null);
+    assert.equal(r.lines, 10); assert.equal(r.result, "OK"); assert.equal(r.bad, "a_b c");
+    assert.match(r.forge_version, /^\d+\.\d+\.\d+/);
+  });
+
+  it("drops reserved and malformed keys", () => {
+    const r = buildRecord("x", { repo: "evil", "a b": 1, ok: 1 });
+    assert.equal(r.repo, "o/r"); assert.equal(r.ok, 1); assert.equal("a b" in r, false);
+  });
+
+  it("creates then upserts one trusted comment", () => {
+    stubGh();
+    emit("ci_gate", { result: "PASS", rc: 0, checks: 3 });
+    const posted = readFileSync(join(td, "posted"), "utf8");
+    assert.match(posted, /^<!-- FORGE:TELEMETRY v1 -->\n```jsonl\n\{.*"event":"ci_gate".*\}\n```$/);
+    setComments([posted]);
+    emit("liveness", { decision: "DEFER", why: "lease" });
+    const patched = readFileSync(join(td, "patched"), "utf8");
+    assert.equal(patched.split("\n").filter((l) => l.startsWith("{")).length, 2);
+    assert.deepEqual(existsSync(join(td, "buf")) ? readdirSync(join(td, "buf")) : [], []);
+  });
+
+  it("ignores an untrusted telemetry comment", () => {
+    stubGh();
+    writeFileSync(join(td, "comments.json"), JSON.stringify([{ id: 1, body: "<!-- FORGE:TELEMETRY v1 -->\n```jsonl\n```", user: { type: "User" }, author_association: "NONE" }]));
+    emit("ci_gate", { result: "PASS" });
+    assert.ok(existsSync(join(td, "posted")));
+    assert.equal(existsSync(join(td, "patched")), false);
+  });
+
+  it("rolls over to part=N near the size limit", () => {
+    stubGh();
+    process.env.FORGE_TELEMETRY_MAX_CHARS = "300";
+    setComments(["<!-- FORGE:TELEMETRY v1 -->\n```jsonl\n" + '{"a":"' + "x".repeat(250) + '"}\n```']);
+    emit("ci_gate", { result: "PASS" });
+    assert.match(readFileSync(join(td, "posted"), "utf8"), /^<!-- FORGE:TELEMETRY v1 part=2 -->/);
+  });
+
+  it("buffers on a failed write and flushes on the next emit", () => {
+    stubGh(true);
+    assert.doesNotThrow(() => emit("diff_size", { lines: 1 }));
+    const files = readdirSync(join(td, "buf"));
+    assert.deepEqual(files, ["o_r__5.jsonl"]);
+    stubGh(false);
+    emit("diff_size", { lines: 2 });
+    const posted = readFileSync(join(td, "posted"), "utf8");
+    assert.equal(posted.split("\n").filter((l) => l.startsWith("{")).length, 2);
+    assert.equal(existsSync(join(td, "buf", "o_r__5.jsonl")), false);
+  });
+
+  it("never throws with an unwritable buffer dir and no gh", () => {
+    process.env.FORGE_TELEMETRY_GH = "/nonexistent/gh";
+    writeFileSync(join(td, "afile"), "x");
+    process.env.FORGE_TELEMETRY_BUFFER_DIR = join(td, "afile", "buf"); // parent is a file: mkdir fails
+    assert.doesNotThrow(() => emit("diff_size", { lines: 1 }));
+  });
+
+  it("honours FORGE_TELEMETRY=off and drops records with no target", () => {
+    stubGh();
+    process.env.FORGE_TELEMETRY = "off";
+    assert.equal(emit("ci_gate", { result: "PASS" }), null);
+    assert.equal(existsSync(join(td, "calls")), false);
+    delete process.env.FORGE_TELEMETRY; delete process.env.FORGE_ISSUE;
+    assert.equal(emit("ci_gate", { result: "PASS" }), null);
+  });
+
+  it("shell and node emitters produce the same keys", () => {
+    process.env.FORGE_PR = "9"; process.env.FORGE_RUN = "r1"; process.env.FORGE_PHASE = "build";
+    const sh = spawnSync("bash", ["-c", `. "${join(import.meta.dirname, "..", "..", "scripts", "lib", "telemetry.sh")}"; _forge_tel_record diff_size lines=10 result=OK`], { encoding: "utf8", env: process.env });
+    const a = JSON.parse(sh.stdout); const b = buildRecord("diff_size", { lines: 10, result: "OK" });
+    assert.deepEqual(Object.keys(a), Object.keys(b));
+    for (const k of Object.keys(a)) if (k !== "ts" && k !== "eid") assert.deepEqual(a[k], b[k], k);
+  });
+});

package/bin/engine-cli.mjs

@@ -13,6 +13,7 @@ import { runIssue, DEFAULT_MAX_ATTEMPTS, sanitizeLeaseHost } from "./engine.mjs"
 import { makeProjector } from "./engine/projector.mjs";
 import { readLog, deriveState } from "./engine/runlog.mjs";
 import { assertOpenCodeNativeRuntime } from "./runner.mjs";
+import { makeEmitTelemetry } from "./engine/telemetry-seam.mjs";
 
 /**
  * Workflow labels that mark an issue as "in the pipeline" (not yet terminal).
@@ -77,8 +78,16 @@ export function runDir() { return join(homedir(), ".forge", "runs"); }
 export function aggregateUsage(events) {
   let found = false;
   const total = { input_tokens: 0, output_tokens: 0, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 };
+  // forge#3696: PHASE_COMMIT.usage is cumulative over all attempts, so summing it with
+  // PHASE_FAILED would double count. For a phase that has PHASE_ATTEMPT_USAGE events,
+  // sum ONLY those; otherwise (old logs) fall back to the legacy COMMIT + FAILED sum.
+  const attemptPhases = new Set(events.filter((e) => e.event === "PHASE_ATTEMPT_USAGE").map((e) => e.phase));
   for (const e of events) {
-    if (e.event !== "PHASE_COMMIT" && e.event !== "PHASE_FAILED") continue;
+    if (e.event === "PHASE_ATTEMPT_USAGE") {
+      // counted as-is (usage null => nothing to add)
+    } else if (e.event === "PHASE_COMMIT" || e.event === "PHASE_FAILED") {
+      if (attemptPhases.has(e.phase)) continue;
+    } else continue;
     if (!e.usage || typeof e.usage !== "object") continue;
     found = true;
     total.input_tokens += e.usage.input_tokens ?? 0;
@@ -393,6 +402,8 @@ export async function runFromCli(argv, deps = {}) {
     ...(repo ? { repo } : {}),
     runner: (await import("./runner.mjs")).runCommand, now: () => Date.now(),
     onProgress,
+    // forge#3835: phase_attempt / run_terminal records reach GitHub so `forgedock stats` works from any machine.
+    emitTelemetry: deps.emitTelemetry ?? makeEmitTelemetry({ repo }),
     // Only forwarded when explicitly provided — omitting them preserves
     // runIssue's/runner.mjs's existing defaults (forge#2028).
     ...(backend ? { backend } : {}),