oracular 0.2.0 → 0.3.0

package/dist/cli.js

@@ -0,0 +1,656 @@
+#!/usr/bin/env node
+
+// cli/main.ts
+import { readFileSync } from "fs";
+
+// cli/run.ts
+import { createRequire } from "module";
+import { extname, resolve } from "path";
+import { pathToFileURL } from "url";
+import { createState as createState2 } from "oracular";
+
+// cli/log.ts
+import { RECORD_VERSION } from "oracular";
+var LogError = class extends Error {
+  constructor() {
+    super(...arguments);
+    this.name = "LogError";
+  }
+};
+function isObject(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function parseLog(text, source) {
+  const entries = [];
+  const unknownKinds = /* @__PURE__ */ new Map();
+  const lines = text.split(/\r?\n/);
+  for (const [i, raw] of lines.entries()) {
+    if (raw.trim() === "") continue;
+    const where = `${source}:${i + 1}`;
+    let record;
+    try {
+      record = JSON.parse(raw);
+    } catch {
+      throw new LogError(`${where}: not valid JSON`);
+    }
+    if (!isObject(record)) throw new LogError(`${where}: a record must be a JSON object`);
+    if (record.v !== RECORD_VERSION) {
+      throw new LogError(
+        `${where}: recorder format v${String(record.v)} is not supported (this replay reads v${RECORD_VERSION})`
+      );
+    }
+    const { installId, mode, kind } = record;
+    if (typeof installId !== "string" || installId === "") {
+      throw new LogError(`${where}: installId must be a non-empty string`);
+    }
+    if (mode !== "shadow" && mode !== "live") {
+      throw new LogError(`${where}: mode must be "shadow" or "live"`);
+    }
+    if (kind === "event") {
+      const event = record.event;
+      if (!isObject(event) || typeof event.name !== "string" || event.name === "") {
+        throw new LogError(`${where}: event.name must be a non-empty string`);
+      }
+      if (typeof event.at !== "number" || !Number.isFinite(event.at)) {
+        throw new LogError(`${where}: event.at must be a finite number`);
+      }
+      entries.push({ kind, installId, mode, name: event.name, at: event.at });
+    } else if (kind === "end") {
+      if (typeof record.at !== "number" || !Number.isFinite(record.at)) {
+        throw new LogError(`${where}: at must be a finite number`);
+      }
+      entries.push({ kind, installId, mode, at: record.at });
+    } else {
+      const label = typeof kind === "string" ? kind : String(kind);
+      unknownKinds.set(label, (unknownKinds.get(label) ?? 0) + 1);
+    }
+  }
+  const warnings = [...unknownKinds].map(
+    ([kind, n]) => `${source}: skipped ${n} record${n === 1 ? "" : "s"} of unknown kind "${kind}"`
+  );
+  return { entries, warnings };
+}
+
+// cli/replay.ts
+import {
+  createState,
+  endEpisode,
+  markRevealed,
+  observe,
+  predict,
+  shouldReveal
+} from "oracular";
+var DEFAULT_THRESHOLDS = [0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9];
+var DEFAULT_THRESHOLD = 0.5;
+function replay(entries, config, options = {}) {
+  const base = createState(config);
+  const normalized = base.config;
+  const includeLive = options.includeLive === true;
+  const bins = options.bins ?? 10;
+  const skipped = [];
+  const names = options.intents ?? Object.keys(normalized.intents);
+  for (const name of names) {
+    if (!Object.hasOwn(normalized.intents, name)) {
+      throw new RangeError(
+        `unknown intent ${JSON.stringify(name)} (the config declares: ${Object.keys(normalized.intents).join(", ")})`
+      );
+    }
+  }
+  const scored = names.filter((name) => {
+    if (normalized.intents[name].outcome !== void 0) return true;
+    skipped.push({
+      intent: name,
+      reason: "declares no outcome event, so there is nothing to score"
+    });
+    return false;
+  });
+  const plans = scored.map((name) => {
+    const intent = normalized.intents[name];
+    const configured = intent.reveal?.threshold ?? normalized.reveal?.threshold ?? DEFAULT_THRESHOLD;
+    const list = options.thresholds ?? [...DEFAULT_THRESHOLDS, configured];
+    const thresholds = [...new Set(list)].sort((a, b) => a - b);
+    const cooldown = predict(markRevealed(base, name, 0), name, 0).suppressedUntil ?? 0;
+    const window = options.window ?? cooldown;
+    if (!(window > 0)) {
+      throw new RangeError(
+        options.window === void 0 ? `intent ${JSON.stringify(name)} has a reveal cooldown of 0, so there is no default scoring window; pass --window (for example --window 7d)` : "the scoring window must be greater than 0"
+      );
+    }
+    return {
+      name,
+      outcome: intent.outcome,
+      configured,
+      thresholds,
+      window,
+      windowIsCooldown: options.window === void 0
+    };
+  });
+  const allThresholds = [...new Set(plans.flatMap((p) => p.thresholds))].sort((a, b) => a - b);
+  const byInstall = /* @__PURE__ */ new Map();
+  for (const entry of entries) {
+    const list = byInstall.get(entry.installId);
+    if (list === void 0) byInstall.set(entry.installId, [entry]);
+    else list.push(entry);
+  }
+  let start = null;
+  let end = null;
+  let shadow = 0;
+  let live = 0;
+  let mixedInstalls = 0;
+  for (const list of byInstall.values()) {
+    list.sort((a, b) => a.at - b.at);
+    const modes = new Set(list.map((e) => e.mode));
+    if (modes.size > 1) mixedInstalls++;
+    for (const e of list) {
+      if (e.mode === "shadow") shadow++;
+      else live++;
+      start = start === null ? e.at : Math.min(start, e.at);
+      end = end === null ? e.at : Math.max(end, e.at);
+    }
+  }
+  const logEnd = end ?? 0;
+  const tallies = plans.map((plan) => ({
+    plan,
+    installs: 0,
+    totals: plan.thresholds.map(
+      (threshold) => ({
+        threshold,
+        reveals: 0,
+        correct: 0,
+        outcomes: 0,
+        caught: 0,
+        leads: [],
+        censored: 0
+      })
+    ),
+    points: [],
+    censored: 0
+  }));
+  for (const list of byInstall.values()) {
+    if (!includeLive && !list.some((e) => e.mode === "shadow")) continue;
+    const liveTimes = list.filter((e) => e.mode === "live").map((e) => e.at);
+    const shadowSpan = (from, to) => includeLive || !liveTimes.some((t) => t >= from && t <= to);
+    const states = new Map(allThresholds.map((t) => [t, base]));
+    const tracks = tallies.map((tally) => ({
+      tally,
+      outcomes: [],
+      points: [],
+      reveals: tally.plan.thresholds.map((threshold) => ({ threshold, times: [] }))
+    }));
+    for (const entry of list) {
+      for (const [t, s] of states) {
+        states.set(
+          t,
+          entry.kind === "end" ? endEpisode(s, entry.at) : observe(s, { name: entry.name, at: entry.at })
+        );
+      }
+      if (entry.kind === "end") continue;
+      for (const track of tracks) {
+        const { plan } = track.tally;
+        if (entry.name === plan.outcome) {
+          track.outcomes.push(entry.at);
+        } else {
+          const any = states.get(allThresholds[0]);
+          track.points.push({ at: entry.at, p: predict(any, plan.name, entry.at).p });
+        }
+        for (const slot of track.reveals) {
+          const s = states.get(slot.threshold);
+          if (shouldReveal(s, plan.name, entry.at, { threshold: slot.threshold })) {
+            slot.times.push(entry.at);
+            states.set(slot.threshold, markRevealed(s, plan.name, entry.at));
+          }
+        }
+      }
+    }
+    for (const { tally, outcomes, points, reveals } of tracks) {
+      const { window } = tally.plan;
+      tally.installs++;
+      const firstOutcomeAfter = (t) => outcomes.find((o) => o > t && o <= t + window);
+      for (const point of points) {
+        if (!shadowSpan(point.at, point.at + window)) continue;
+        if (point.at + window > logEnd) {
+          tally.censored++;
+          continue;
+        }
+        tally.points.push({ p: point.p, y: firstOutcomeAfter(point.at) === void 0 ? 0 : 1 });
+      }
+      for (const [k, slot] of reveals.entries()) {
+        const sum = tally.totals[k];
+        for (const r of slot.times) {
+          if (!shadowSpan(r, r + window)) continue;
+          if (r + window > logEnd) {
+            sum.censored++;
+            continue;
+          }
+          sum.reveals++;
+          const o = firstOutcomeAfter(r);
+          if (o !== void 0) {
+            sum.correct++;
+            sum.leads.push(o - r);
+          }
+        }
+        for (const o of outcomes) {
+          if (!shadowSpan(o - window, o)) continue;
+          sum.outcomes++;
+          if (slot.times.some((r) => r >= o - window && r < o)) sum.caught++;
+        }
+      }
+    }
+  }
+  const intents = tallies.map(({ plan, installs, totals, points, censored }) => {
+    const n = points.length;
+    const hits = points.reduce((acc, pt) => acc + pt.y, 0);
+    const baseRate = n === 0 ? null : hits / n;
+    const reliability = [];
+    for (let b = 0; b < bins; b++) {
+      const inBin = points.filter((pt) => Math.min(Math.floor(pt.p * bins), bins - 1) === b);
+      if (inBin.length === 0) continue;
+      reliability.push({
+        from: b / bins,
+        to: (b + 1) / bins,
+        count: inBin.length,
+        meanP: inBin.reduce((acc, pt) => acc + pt.p, 0) / inBin.length,
+        observed: inBin.reduce((acc, pt) => acc + pt.y, 0) / inBin.length
+      });
+    }
+    return {
+      intent: plan.name,
+      outcome: plan.outcome,
+      window: plan.window,
+      windowIsCooldown: plan.windowIsCooldown,
+      installs,
+      thresholds: totals.map((s) => {
+        return {
+          threshold: s.threshold,
+          configured: s.threshold === plan.configured,
+          reveals: s.reveals,
+          correct: s.correct,
+          precision: s.reveals === 0 ? null : s.correct / s.reveals,
+          outcomes: s.outcomes,
+          caught: s.caught,
+          recall: s.outcomes === 0 ? null : s.caught / s.outcomes,
+          leadTime: median(s.leads),
+          revealsPerUser: installs === 0 ? null : s.reveals / installs,
+          censored: s.censored
+        };
+      }),
+      points: n,
+      censored,
+      reliability,
+      brier: n === 0 ? null : points.reduce((acc, pt) => acc + (pt.p - pt.y) ** 2, 0) / n,
+      baseRate,
+      brierBaseline: baseRate === null ? null : baseRate * (1 - baseRate)
+    };
+  });
+  return {
+    installs: byInstall.size,
+    records: entries.length,
+    shadow,
+    live,
+    mixedInstalls,
+    start,
+    end,
+    scored: includeLive ? "all" : "shadow",
+    intents,
+    skipped
+  };
+}
+function median(values) {
+  if (values.length === 0) return null;
+  const sorted = [...values].sort((a, b) => a - b);
+  const mid = Math.floor(sorted.length / 2);
+  const at = (i) => sorted[i];
+  return sorted.length % 2 === 1 ? at(mid) : (at(mid - 1) + at(mid)) / 2;
+}
+
+// cli/report.ts
+var SECOND = 1e3;
+var MINUTE = 60 * SECOND;
+var HOUR = 60 * MINUTE;
+var DAY = 24 * HOUR;
+function formatDuration(ms) {
+  if (ms === null) return "\u2013";
+  if (ms < SECOND) return `${Math.round(ms)}ms`;
+  const units = [
+    [DAY, "d"],
+    [HOUR, "h"],
+    [MINUTE, "m"],
+    [SECOND, "s"]
+  ];
+  const rounded = Math.round(ms / SECOND) * SECOND;
+  const i = units.findIndex(([size2]) => rounded >= size2);
+  const [size, label] = units[i];
+  const n = Math.floor(rounded / size);
+  const next = units[i + 1];
+  if (next === void 0) return `${n}${label}`;
+  const m = Math.floor((rounded - n * size) / next[0]);
+  return m > 0 ? `${n}${label} ${m}${next[1]}` : `${n}${label}`;
+}
+function formatTime(at) {
+  if (at === null) return "\u2013";
+  if (at >= 1e11 && at <= 1e14)
+    return `${new Date(at).toISOString().slice(0, 16).replace("T", " ")} UTC`;
+  return String(at);
+}
+var pct = (x) => x === null ? "\u2013" : `${(x * 100).toFixed(1)}%`;
+var num = (x, digits) => x === null ? "\u2013" : x.toFixed(digits);
+function table(headers, rows) {
+  const widths = headers.map(
+    (h, i) => Math.max(h.length, ...rows.map((r) => r[i].length))
+  );
+  const line = (cells) => `| ${cells.map((c, i) => c.padEnd(widths[i])).join(" | ")} |`;
+  return [line(headers), line(widths.map((w) => "-".repeat(w))), ...rows.map(line)].join("\n");
+}
+function formatIntent(intent) {
+  const window = formatDuration(intent.window);
+  const source = intent.windowIsCooldown ? "the intent's reveal cooldown" : "as given";
+  const out = [];
+  out.push(
+    `${intent.intent}: outcome ${intent.outcome}, window ${window} (${source}), ${intent.installs} install${intent.installs === 1 ? "" : "s"}`
+  );
+  out.push("");
+  out.push(
+    table(
+      ["threshold", "reveals", "per user", "precision", "recall", "median lead time"],
+      intent.thresholds.map((t) => [
+        `${t.threshold}${t.configured ? " *" : ""}`,
+        String(t.reveals),
+        num(t.revealsPerUser, 2),
+        pct(t.precision),
+        `${pct(t.recall)} of ${t.outcomes}`,
+        formatDuration(t.leadTime)
+      ])
+    )
+  );
+  const censored = intent.thresholds.reduce((acc, t) => Math.max(acc, t.censored), 0);
+  out.push(
+    `* the threshold your config resolves for this intent.${censored > 0 ? ` Reveals whose window runs past the end of the log are not scored (up to ${censored} per threshold).` : ""}`
+  );
+  out.push("");
+  out.push(
+    `Reliability: p after every event except the outcome, against "the outcome follows within ${window}".`
+  );
+  if (intent.points === 0) {
+    out.push("No scored points.");
+  } else {
+    out.push(
+      table(
+        ["p", "points", "mean p", "observed"],
+        intent.reliability.map((b) => [
+          `${b.from.toFixed(1)}\u2013${b.to.toFixed(1)}`,
+          String(b.count),
+          b.meanP.toFixed(3),
+          pct(b.observed)
+        ])
+      )
+    );
+    out.push(
+      `Brier score ${num(intent.brier, 3)} over ${intent.points} points (always predicting the base rate of ${pct(intent.baseRate)}: ${num(intent.brierBaseline, 3)}). Lower is better.${intent.censored > 0 ? ` ${intent.censored} points too close to the end of the log are not scored.` : ""}`
+    );
+  }
+  return out.join("\n");
+}
+function formatText(report) {
+  const out = [];
+  out.push(
+    `oracular replay: ${report.installs} install${report.installs === 1 ? "" : "s"}, ${report.records} records (${report.shadow} shadow, ${report.live} live), ${formatTime(report.start)} to ${formatTime(report.end)}`
+  );
+  out.push(
+    `Scored: ${report.scored === "shadow" ? "shadow periods only" : "every record, live included"}. Durations assume \`at\` is in milliseconds.`
+  );
+  for (const intent of report.intents) {
+    out.push("");
+    out.push(formatIntent(intent));
+  }
+  for (const { intent, reason } of report.skipped) {
+    out.push("");
+    out.push(`${intent}: skipped, ${reason}.`);
+  }
+  return `${out.join("\n")}
+`;
+}
+function formatJson(report) {
+  return `${JSON.stringify(report, null, 2)}
+`;
+}
+
+// cli/run.ts
+var UsageError = class extends Error {
+};
+var HELP = `Usage: oracular replay <log.ndjson>... --config <file> [options]
+
+Replays a recorder log (see the Shadow mode guide) through the engine under a config, and
+reports for every intent and threshold: precision, recall, lead time from reveal to outcome,
+reveals per user, and a reliability table with a Brier score.
+
+Options:
+  --config <file>        the config to replay under: a .json file, or a .js, .mjs or .cjs
+                         module whose default (or \`config\`) export is the config
+  --thresholds <list>    thresholds to score, comma separated
+                         (default: 0.3 to 0.9 by 0.1, plus each intent's configured threshold)
+  --window <duration>    how long after a reveal an outcome still counts for it: 3d, 12h, 90m
+                         (default: each intent's reveal cooldown; needed when that is 0).
+                         A plain number is in the unit of \`at\`; the suffixes ms, s, m, h, d
+                         and w assume milliseconds
+  --intent <name>        score only this intent; repeat for more
+                         (default: every intent that declares an outcome)
+  --include-live         also score live periods. While reveals are shown they cause outcomes,
+                         so precision and recall there overstate the engine
+  --json                 print the report as JSON
+  -h, --help             show this help
+  -v, --version          print the version
+
+Only shadow periods are scored by default; a log with live records gets a warning, and a log
+with nothing but live records is refused unless --include-live is given.
+
+Exit codes: 0 report printed, 1 unreadable input or nothing to score, 2 usage error.
+Docs: https://yuribenjamin.github.io/oracular/guide/shadow-mode
+`;
+var UNITS = {
+  ms: 1,
+  s: 1e3,
+  m: 6e4,
+  h: 36e5,
+  d: 864e5,
+  w: 6048e5
+};
+function parseDuration(text, flag) {
+  const match = /^(\d+(?:\.\d+)?)(ms|s|m|h|d|w)?$/.exec(text);
+  const value = match === null ? Number.NaN : Number(match[1]) * UNITS[match[2] ?? "ms"];
+  if (!(value > 0) || !Number.isFinite(value)) {
+    throw new UsageError(
+      `${flag} must be a positive duration such as 3d, 12h, 90m or 5000, got ${JSON.stringify(text)}`
+    );
+  }
+  return value;
+}
+function parseThresholds(text) {
+  return text.split(",").map((part) => {
+    const t = Number(part.trim());
+    if (part.trim() === "" || !Number.isFinite(t) || t < 0 || t > 1) {
+      throw new UsageError(
+        `--thresholds must be numbers between 0 and 1, got ${JSON.stringify(part)}`
+      );
+    }
+    return t;
+  });
+}
+function parseReplayArgs(argv) {
+  const out = { logs: [], includeLive: false, json: false, help: false };
+  for (let i = 0; i < argv.length; i++) {
+    let flag = argv[i];
+    let inline;
+    const eq = flag.indexOf("=");
+    if (flag.startsWith("--") && eq > 0) {
+      inline = flag.slice(eq + 1);
+      flag = flag.slice(0, eq);
+    }
+    const value = () => {
+      if (inline !== void 0) return inline;
+      const next = argv[i + 1];
+      if (next === void 0 || next.startsWith("--"))
+        throw new UsageError(`${flag} needs a value`);
+      i++;
+      return next;
+    };
+    const noValue = () => {
+      if (inline !== void 0) throw new UsageError(`${flag} takes no value`);
+    };
+    switch (flag) {
+      case "--config":
+        out.config = value();
+        break;
+      case "--thresholds":
+        out.thresholds = parseThresholds(value());
+        break;
+      case "--window":
+        out.window = parseDuration(value(), "--window");
+        break;
+      case "--intent":
+        out.intents = [...out.intents ?? [], value()];
+        break;
+      case "--include-live":
+        noValue();
+        out.includeLive = true;
+        break;
+      case "--json":
+        noValue();
+        out.json = true;
+        break;
+      case "--help":
+      case "-h":
+        out.help = true;
+        break;
+      default:
+        if (flag.startsWith("-")) throw new UsageError(`unknown option ${flag}`);
+        out.logs.push(flag);
+    }
+  }
+  if (!out.help) {
+    if (out.logs.length === 0) throw new UsageError("replay needs at least one log file");
+    if (out.config === void 0) throw new UsageError("replay needs --config <file>");
+  }
+  return out;
+}
+function version() {
+  return createRequire(import.meta.url)("../package.json").version;
+}
+async function loadConfig(path, io) {
+  const ext = extname(path);
+  let value;
+  if (ext === ".json") {
+    try {
+      value = JSON.parse(io.readFile(path));
+    } catch (error) {
+      throw new Error(`${path}: ${error.message}`);
+    }
+  } else if (ext === ".js" || ext === ".mjs" || ext === ".cjs") {
+    const mod = await import(pathToFileURL(path).href);
+    value = mod.default ?? mod.config;
+    if (typeof value === "object" && value !== null && !("intents" in value) && "config" in value) {
+      value = value.config;
+    }
+  } else {
+    throw new UsageError("--config must be a .json, .js, .mjs or .cjs file");
+  }
+  if (typeof value !== "object" || value === null) {
+    throw new Error(`${path} holds no config object`);
+  }
+  return value;
+}
+async function replayCommand(argv, io) {
+  const args = parseReplayArgs(argv);
+  if (args.help) {
+    io.stdout(HELP);
+    return 0;
+  }
+  const configPath = resolve(io.cwd, args.config);
+  const config = await loadConfig(configPath, io);
+  let declared;
+  try {
+    declared = Object.keys(createState2(config).config.intents);
+  } catch (error) {
+    throw new Error(
+      `${configPath} is not a valid config: ${error.message.replace(/^oracular: /, "")}`
+    );
+  }
+  for (const name of args.intents ?? []) {
+    if (!declared.includes(name)) {
+      throw new UsageError(`--intent ${name}: the config declares ${declared.join(", ")}`);
+    }
+  }
+  const entries = [];
+  for (const file of args.logs) {
+    const parsed = parseLog(io.readFile(resolve(io.cwd, file)), file);
+    for (const warning of parsed.warnings) io.stderr(`oracular: warning: ${warning}
+`);
+    entries.push(...parsed.entries);
+  }
+  if (entries.length === 0) throw new Error("the log holds no records");
+  const live = entries.filter((e) => e.mode === "live").length;
+  const shadow = entries.length - live;
+  if (shadow === 0 && !args.includeLive) {
+    throw new Error(
+      "every record in the log is live, so there is nothing to measure. Replay measures the engine where reveals were not shown; once they are, the reveal itself causes outcomes. Pass --include-live to score the live records anyway, knowing precision and recall are contaminated."
+    );
+  }
+  if (live > 0 && !args.includeLive) {
+    io.stderr(
+      `oracular: warning: the log mixes shadow and live records (${live} of ${entries.length} live). Only shadow periods are scored: a reveal, point or outcome whose window touches a live record is left out. Pass --include-live to score everything.
+`
+    );
+  }
+  if (live > 0 && args.includeLive) {
+    io.stderr(
+      `oracular: warning: scoring ${live} live records as asked. While reveals are shown they cause outcomes, so precision and recall over live periods overstate the engine.
+`
+    );
+  }
+  const options = { includeLive: args.includeLive };
+  if (args.thresholds !== void 0) options.thresholds = args.thresholds;
+  if (args.window !== void 0) options.window = args.window;
+  if (args.intents !== void 0) options.intents = args.intents;
+  let report;
+  try {
+    report = replay(entries, config, options);
+  } catch (error) {
+    if (error instanceof RangeError) throw new UsageError(error.message);
+    throw error;
+  }
+  io.stdout(args.json ? formatJson(report) : formatText(report));
+  return 0;
+}
+async function run(argv, io) {
+  try {
+    const [command, ...rest] = argv;
+    if (command === void 0 || command === "--help" || command === "-h" || command === "help") {
+      io.stdout(HELP);
+      return 0;
+    }
+    if (command === "--version" || command === "-v") {
+      io.stdout(`${version()}
+`);
+      return 0;
+    }
+    if (command !== "replay") throw new UsageError(`unknown command ${JSON.stringify(command)}`);
+    return await replayCommand(rest, io);
+  } catch (error) {
+    if (error instanceof UsageError) {
+      io.stderr(`oracular: ${error.message}
+Run "oracular --help" for usage.
+`);
+      return 2;
+    }
+    io.stderr(`oracular: ${error.message}
+`);
+    return 1;
+  }
+}
+
+// cli/main.ts
+process.exitCode = await run(process.argv.slice(2), {
+  stdout: (text) => process.stdout.write(text),
+  stderr: (text) => process.stderr.write(text),
+  readFile: (path) => readFileSync(path, "utf8"),
+  cwd: process.cwd()
+});