@cohortapp/agent-sdk 2.3.1 → 2.4.0

package/lib/kpi.mjs

@@ -0,0 +1,469 @@
+/**
+ * lib/kpi.mjs — measurement, the time series, and the gap.
+ *
+ * This is the missing half of "the agent has objectives": neither repo has ever
+ * held a KPI TIME SERIES, so no agent could tell whether it was improving, flat
+ * or sliding — which means no agent could decide whether to do more work, ask a
+ * question, or leave the number alone. `gapFor()` is that decision's input.
+ *
+ * Three rules the rest of the system leans on:
+ *
+ *   1. **Honesty is a property of the SAMPLE, not of the caller.** A sample's
+ *      `source` is `method` only when a real capability actually executed and
+ *      returned a number; `human` when a person supplied it; `llm` when a model
+ *      guessed. An `llm`-sourced sample is advisory and can NEVER create work
+ *      (enforced in goals/admission.mjs). This is the Goodhart gate.
+ *   2. **An unreachable sensor is a measurement, not an exception.** It returns
+ *      `{ok:false, reason}` and is logged; it never throws into the daemon and
+ *      it never quietly records a zero.
+ *   3. **Idempotent on `(objectiveKey, window)`.** Re-running the steward inside
+ *      the same week does not double-count; the same window is overwritten only
+ *      when a better source arrives (`llm` → `human` → `method`).
+ *
+ * Injectable everywhere: the clock, the sensor registry, and the ledger path all
+ * arrive on `deps`, so tests are hermetic and the network is never touched.
+ *
+ * @module lib/kpi
+ */
+
+"use strict";
+
+import { existsSync, readFileSync, mkdirSync } from "node:fs";
+import { join, dirname } from "node:path";
+
+import { resolveAgentRoot } from "./agent-root.mjs";
+import { appendJsonl } from "./fs-atomic.mjs";
+
+export const MEASUREMENTS_REL = join("state", "kpi", "measurements.jsonl");
+export const INTERVENTIONS_REL = join("state", "kpi", "interventions.jsonl");
+
+/** Sample sources, weakest → strongest. A stronger source may overwrite a window. */
+export const SOURCE_RANK = Object.freeze({ llm: 0, human: 1, method: 2 });
+
+/** Sources a sample must carry to be allowed to CREATE work. */
+export const WORK_CREATING_SOURCES = Object.freeze(["method", "human"]);
+
+/** How many periods of no movement counts as "flat". */
+export const FLAT_PERIODS = 2;
+
+function nowMs(deps) {
+  return deps && typeof deps.now === "function" ? deps.now() : Date.now();
+}
+
+function logOf(deps) {
+  return deps && typeof deps.log === "function" ? deps.log : () => {};
+}
+
+// ---------------------------------------------------------------------------
+// Windows
+// ---------------------------------------------------------------------------
+
+/** ISO-8601 week number + ISO week-year for a date. Pure. */
+export function isoWeek(date) {
+  const d = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()));
+  // Thursday of the current week determines the ISO year.
+  const dayNum = (d.getUTCDay() + 6) % 7;
+  d.setUTCDate(d.getUTCDate() - dayNum + 3);
+  const isoYear = d.getUTCFullYear();
+  const firstThursday = new Date(Date.UTC(isoYear, 0, 4));
+  const firstDayNum = (firstThursday.getUTCDay() + 6) % 7;
+  firstThursday.setUTCDate(firstThursday.getUTCDate() - firstDayNum + 3);
+  const week = 1 + Math.round((d - firstThursday) / (7 * 24 * 3600 * 1000));
+  return { isoYear, week };
+}
+
+/**
+ * The measurement window key for a cadence at a point in time. This is the
+ * idempotency key half of `(objectiveKey, window)`.
+ *   weekly    → "2026-W33"
+ *   monthly   → "2026-08"
+ *   quarterly → "2026-Q3"
+ *   daily/any → "2026-08-11"
+ * @param {string} cadence @param {number|Date} at @returns {string}
+ */
+export function windowFor(cadence, at) {
+  const d = at instanceof Date ? at : new Date(at || Date.now());
+  const y = d.getUTCFullYear();
+  switch (String(cadence || "").toLowerCase()) {
+    case "weekly": {
+      const { isoYear, week } = isoWeek(d);
+      return `${isoYear}-W${String(week).padStart(2, "0")}`;
+    }
+    case "monthly":
+      return `${y}-${String(d.getUTCMonth() + 1).padStart(2, "0")}`;
+    case "quarterly":
+      return `${y}-Q${Math.floor(d.getUTCMonth() / 3) + 1}`;
+    default:
+      return d.toISOString().slice(0, 10);
+  }
+}
+
+/**
+ * Is this objective due to be measured? True when its current window has no
+ * sample yet. An objective with no cadence is never due (and is warned about by
+ * `model.validateMandateBody`) — silence there would be the bug.
+ * @param {object} objective @param {object[]} series @param {object} [deps] { now }
+ * @returns {{due:boolean, window:string|null, reason:string}}
+ */
+export function isDue(objective, series, deps) {
+  if (!objective || !objective.cadence) {
+    return { due: false, window: null, reason: "objective has no measure cadence" };
+  }
+  const window = windowFor(objective.cadence, nowMs(deps));
+  const hit = (series || []).find((s) => s && s.window === window);
+  if (hit) return { due: false, window, reason: `already sampled for ${window}` };
+  return { due: true, window, reason: `no sample for ${window}` };
+}
+
+// ---------------------------------------------------------------------------
+// The ledger
+// ---------------------------------------------------------------------------
+
+/** Absolute path of the measurement ledger. */
+export function measurementsPath(agentRoot) {
+  return join(resolveAgentRoot(agentRoot), MEASUREMENTS_REL);
+}
+
+/** Absolute path of the intervention (learning-signal) ledger. */
+export function interventionsPath(agentRoot) {
+  return join(resolveAgentRoot(agentRoot), INTERVENTIONS_REL);
+}
+
+function readJsonl(path) {
+  if (!existsSync(path)) return [];
+  let body;
+  try { body = readFileSync(path, "utf-8"); } catch { return []; }
+  const out = [];
+  for (const line of body.split("\n")) {
+    if (!line.trim()) continue;
+    try { out.push(JSON.parse(line)); } catch { /* skip malformed */ }
+  }
+  return out;
+}
+
+/**
+ * Every sample for an objective, oldest first. When two rows share a window the
+ * strongest source wins (a `method` re-measure supersedes an `llm` guess).
+ * @param {string} agentRoot @param {string} objectiveKey
+ * @returns {object[]}
+ */
+export function readSeries(agentRoot, objectiveKey) {
+  const rows = readJsonl(measurementsPath(agentRoot)).filter(
+    (r) => r && (!objectiveKey || r.objectiveKey === objectiveKey)
+  );
+  const byWindow = new Map();
+  for (const r of rows) {
+    const prev = byWindow.get(r.window);
+    if (!prev) { byWindow.set(r.window, r); continue; }
+    const better =
+      (SOURCE_RANK[r.source] ?? -1) > (SOURCE_RANK[prev.source] ?? -1) ||
+      ((SOURCE_RANK[r.source] ?? -1) === (SOURCE_RANK[prev.source] ?? -1) &&
+        String(r.at || "") >= String(prev.at || ""));
+    if (better) byWindow.set(r.window, r);
+  }
+  return [...byWindow.values()].sort((a, b) => String(a.window).localeCompare(String(b.window)));
+}
+
+/**
+ * Append a sample. Idempotent on `(objectiveKey, window)`: a duplicate of equal
+ * or weaker source is refused (and reported — never silently swallowed).
+ *
+ * @param {string} agentRoot
+ * @param {{objectiveKey:string, value:number, window:string, source:string, evidence?:object}} sample
+ * @param {object} [deps] { now, log }
+ * @returns {{recorded:boolean, reason:string, sample:object|null}}
+ */
+export function recordMeasurement(agentRoot, sample, deps = {}) {
+  const key = sample && sample.objectiveKey;
+  const value = Number(sample && sample.value);
+  if (!key) return { recorded: false, reason: "no objectiveKey", sample: null };
+  if (!Number.isFinite(value)) {
+    logOf(deps)("warn", `[kpi] refusing a non-numeric sample for "${key}" (value=${sample && sample.value})`);
+    return { recorded: false, reason: "non-numeric value", sample: null };
+  }
+  const source = String((sample && sample.source) || "llm");
+  if (!(source in SOURCE_RANK)) {
+    logOf(deps)("warn", `[kpi] unknown sample source "${source}" for "${key}" — treated as llm (advisory only)`);
+  }
+  const window = String((sample && sample.window) || windowFor("daily", nowMs(deps)));
+
+  const existing = readSeries(agentRoot, key).find((s) => s.window === window);
+  if (existing && (SOURCE_RANK[existing.source] ?? -1) >= (SOURCE_RANK[source] ?? -1)) {
+    logOf(deps)(
+      "info",
+      `[kpi] "${key}" already has a ${existing.source} sample for ${window} — refusing a ${source} duplicate (idempotent)`
+    );
+    return { recorded: false, reason: "duplicate-window", sample: existing };
+  }
+
+  const rec = {
+    objectiveKey: key,
+    objectiveId: (sample && sample.objectiveId) || null,
+    value,
+    window,
+    at: new Date(nowMs(deps)).toISOString(),
+    source: source in SOURCE_RANK ? source : "llm",
+    evidence: (sample && sample.evidence) || null,
+  };
+  try {
+    const p = measurementsPath(agentRoot);
+    mkdirSync(dirname(p), { recursive: true });
+    appendJsonl(p, rec);
+  } catch (err) {
+    logOf(deps)("error", `[kpi] could not append a measurement for "${key}": ${err && err.message ? err.message : err}`);
+    return { recorded: false, reason: "append-failed", sample: null };
+  }
+  return { recorded: true, reason: existing ? "superseded-weaker-source" : "recorded", sample: rec };
+}
+
+/**
+ * Record the realized delta of an intervention — the learning signal. Step 4 of
+ * the loop feeds the last three of these back into the planner so an
+ * intervention that made a number WORSE is down-weighted next time.
+ * @param {string} agentRoot @param {object} rec @param {object} [deps]
+ * @returns {{recorded:boolean, record:object|null}}
+ */
+export function recordIntervention(agentRoot, rec, deps = {}) {
+  const row = {
+    at: new Date(nowMs(deps)).toISOString(),
+    objectiveKey: (rec && rec.objectiveKey) || null,
+    taskId: (rec && rec.taskId) || null,
+    obligationKey: (rec && rec.obligationKey) || null,
+    expectedDelta: Number.isFinite(Number(rec && rec.expectedDelta)) ? Number(rec.expectedDelta) : null,
+    realizedDelta: Number.isFinite(Number(rec && rec.realizedDelta)) ? Number(rec.realizedDelta) : null,
+    measuredAt: (rec && rec.measuredAt) || null,
+  };
+  try {
+    const p = interventionsPath(agentRoot);
+    mkdirSync(dirname(p), { recursive: true });
+    appendJsonl(p, row);
+  } catch (err) {
+    logOf(deps)("error", `[kpi] could not append an intervention: ${err && err.message ? err.message : err}`);
+    return { recorded: false, record: null };
+  }
+  return { recorded: true, record: row };
+}
+
+/**
+ * The last N interventions against an objective, newest last. Fed to the
+ * planning sub-session so it can see what it already tried.
+ */
+export function readInterventions(agentRoot, objectiveKey, limit = 3) {
+  const rows = readJsonl(interventionsPath(agentRoot)).filter(
+    (r) => r && (!objectiveKey || r.objectiveKey === objectiveKey)
+  );
+  return limit > 0 ? rows.slice(-limit) : rows;
+}
+
+// ---------------------------------------------------------------------------
+// Measurement
+// ---------------------------------------------------------------------------
+
+/**
+ * Execute an objective's sensor.
+ *
+ * `deps.sensors` is a registry `{ [capabilityId]: async ({params, objective}) =>
+ * number | {value, rowCount?, …} }`. `deps.reachable` is the set/array of
+ * capability ids the manifest says are actually reachable — an entry that is
+ * NOT reachable may never be cited (SPEC §5.1), so it fails here rather than
+ * producing a fabricated number.
+ *
+ * Never throws. Every failure path returns `{ok:false, reason}` AND logs.
+ *
+ * @param {object} objective
+ * @param {object} deps - { sensors, reachable?, now, log }
+ * @returns {Promise<{ok:boolean, value:number|null, source:string, evidence:object|null, reason:string}>}
+ */
+export async function measureKpi(objective, deps = {}) {
+  const key = objective && objective.key;
+  const sensor = objective && objective.sensor;
+  if (!sensor || !sensor.capability) {
+    const reason = "no-sensor";
+    logOf(deps)("info", `[kpi] "${key}" has no method sensor (source=${(sensor && sensor.source) || "none"}) — measurement is a human obligation`);
+    return { ok: false, value: null, source: (sensor && sensor.source) || "human", evidence: null, reason };
+  }
+
+  const reachable = deps.reachable instanceof Set ? deps.reachable : new Set(deps.reachable || []);
+  if (deps.reachable !== undefined && !reachable.has(sensor.capability)) {
+    logOf(deps)(
+      "warn",
+      `[kpi] "${key}" cites capability "${sensor.capability}" which is NOT reachable in the manifest — refusing to measure (this is a drift, not a zero)`
+    );
+    return { ok: false, value: null, source: "method", evidence: null, reason: "unreachable-capability" };
+  }
+
+  const fn = deps.sensors && deps.sensors[sensor.capability];
+  if (typeof fn !== "function") {
+    logOf(deps)("warn", `[kpi] no sensor implementation registered for "${sensor.capability}" (objective "${key}")`);
+    return { ok: false, value: null, source: "method", evidence: null, reason: "no-implementation" };
+  }
+
+  let raw;
+  try {
+    raw = await fn({ params: sensor.params || {}, objective });
+  } catch (err) {
+    logOf(deps)("error", `[kpi] sensor "${sensor.capability}" threw for "${key}": ${err && err.message ? err.message : err}`);
+    return { ok: false, value: null, source: "method", evidence: null, reason: "sensor-threw" };
+  }
+
+  const value = Number(raw && typeof raw === "object" ? raw.value : raw);
+  if (!Number.isFinite(value)) {
+    logOf(deps)("warn", `[kpi] sensor "${sensor.capability}" returned a non-numeric result for "${key}"`);
+    return { ok: false, value: null, source: "method", evidence: null, reason: "non-numeric" };
+  }
+
+  return {
+    ok: true,
+    value,
+    source: "method",
+    evidence: {
+      method: sensor.capability,
+      params: sensor.params || {},
+      rowCount: raw && typeof raw === "object" && Number.isFinite(Number(raw.rowCount)) ? Number(raw.rowCount) : null,
+      sessionId: (raw && typeof raw === "object" && raw.sessionId) || null,
+    },
+    reason: "measured",
+  };
+}
+
+// ---------------------------------------------------------------------------
+// The gap
+// ---------------------------------------------------------------------------
+
+/** Signed shortfall for a direction. Positive = "we are short of target". */
+function rawGap(direction, value, target, tolerance) {
+  switch (String(direction || "up")) {
+    case "down":
+      return value - target;
+    case "hold":
+    case "band":
+      return Math.max(0, Math.abs(value - target) - Math.max(0, tolerance || 0));
+    case "up":
+    default:
+      return target - value;
+  }
+}
+
+/**
+ * Compute the gap for an objective from its series.
+ *
+ * @param {object} objective
+ * @param {object[]} series - readSeries() output, oldest first
+ * @param {object} [deps] { now }
+ * @returns {{
+ *   objectiveKey:string, value:number|null, target:number|null, gap:number|null,
+ *   normalizedGap:number, withinTolerance:boolean, trend:'improving'|'flat'|'worsening'|'unknown',
+ *   flatPeriods:number, source:string|null, window:string|null,
+ *   stalenessPeriods:number|null, stale:boolean, reason:string
+ * }}
+ */
+export function gapFor(objective, series, deps = {}) {
+  const key = (objective && objective.key) || null;
+  // `Number(null)` is 0 — an absent target must NOT read as a target of zero.
+  const rawTarget = objective ? objective.target : null;
+  const target = rawTarget == null || rawTarget === "" ? NaN : Number(rawTarget);
+  const tolerance = Number.isFinite(Number(objective && objective.tolerance)) ? Number(objective.tolerance) : 0;
+  const direction = (objective && objective.direction) || "up";
+  const rows = (series || []).filter((s) => s && Number.isFinite(Number(s.value)));
+  const latest = rows[rows.length - 1] || null;
+
+  const base = {
+    objectiveKey: key,
+    value: latest ? Number(latest.value) : null,
+    target: Number.isFinite(target) ? target : null,
+    gap: null,
+    normalizedGap: 0,
+    withinTolerance: false,
+    trend: "unknown",
+    flatPeriods: 0,
+    source: latest ? latest.source : null,
+    window: latest ? latest.window : null,
+    stalenessPeriods: null,
+    stale: false,
+    reason: "",
+  };
+
+  if (!latest) return { ...base, reason: "no samples" };
+  if (!Number.isFinite(target)) return { ...base, reason: "objective has no numeric target" };
+
+  const value = Number(latest.value);
+  const gap = rawGap(direction, value, target, tolerance);
+  const denom = Math.max(Math.abs(target), 1e-9);
+  const normalizedGap = Math.max(0, gap) / denom;
+
+  // Trend over the gap, not the raw value, so `direction` is honoured.
+  const gaps = rows.slice(-(FLAT_PERIODS + 1)).map((r) => rawGap(direction, Number(r.value), target, tolerance));
+  let trend = "unknown";
+  let flatPeriods = 0;
+  if (gaps.length >= 2) {
+    const eps = Math.max(Math.abs(target) * 0.01, 1e-9);
+    let improving = 0;
+    let worsening = 0;
+    for (let i = 1; i < gaps.length; i++) {
+      const delta = gaps[i] - gaps[i - 1];
+      if (delta < -eps) improving += 1;
+      else if (delta > eps) worsening += 1;
+      else flatPeriods += 1;
+    }
+    if (improving > worsening) trend = "improving";
+    else if (worsening > improving) trend = "worsening";
+    else trend = "flat";
+  }
+
+  // Staleness: how many cadence periods since the latest sample.
+  let stalenessPeriods = null;
+  let stale = false;
+  if (objective && objective.cadence && latest.at) {
+    const per = periodSeconds(objective.cadence);
+    const age = Math.max(0, (nowMs(deps) - Date.parse(latest.at)) / 1000);
+    if (Number.isFinite(age) && per > 0) {
+      stalenessPeriods = age / per;
+      stale = stalenessPeriods >= 2; // 2× cadence with no sample is itself a gap
+    }
+  }
+
+  return {
+    ...base,
+    value,
+    target,
+    gap: +gap.toFixed(6),
+    normalizedGap: +normalizedGap.toFixed(6),
+    withinTolerance: gap <= tolerance,
+    trend,
+    flatPeriods,
+    stalenessPeriods,
+    stale,
+    reason: gap <= tolerance ? "within tolerance" : "gap exceeds tolerance",
+  };
+}
+
+/** Seconds in one cadence period. */
+export function periodSeconds(cadence) {
+  switch (String(cadence || "").toLowerCase()) {
+    case "weekly": return 7 * 24 * 3600;
+    case "monthly": return 30 * 24 * 3600;
+    case "quarterly": return 91 * 24 * 3600;
+    case "daily": return 24 * 3600;
+    default: return 24 * 3600;
+  }
+}
+
+export default {
+  MEASUREMENTS_REL,
+  INTERVENTIONS_REL,
+  SOURCE_RANK,
+  WORK_CREATING_SOURCES,
+  FLAT_PERIODS,
+  isoWeek,
+  windowFor,
+  isDue,
+  measurementsPath,
+  interventionsPath,
+  readSeries,
+  recordMeasurement,
+  recordIntervention,
+  readInterventions,
+  measureKpi,
+  gapFor,
+  periodSeconds,
+};