@rol1n/memory-log 0.0.0-stage → 0.1.0

package/extensions/memory-log/runtime.ts

@@ -0,0 +1,127 @@
+import { type Config, DEFAULTS, loadConfig } from "./config.ts";
+import { foldLedger, poolTokens, rawTokensSinceObservationCoverage, sumSessionCost, type Entry } from "./ledger/index.ts";
+import { StatusController } from "./ui/status-controller.ts";
+
+/**
+ * In-process orchestrator state.
+ */
+export class Runtime {
+  config: Config = { ...DEFAULTS };
+  configLoaded = false;
+
+  /** The per-session on/off gate (uses config.defaultEnabled when no ledger entry exists). */
+  enabled = false;
+
+  /**
+   * Absolute `.memory/<sessionId>/` root for this session.
+   */
+  memoryRoot = "";
+
+  /**
+   * In-flight observer subprocesses, keyed by runId.
+   */
+  readonly observersInFlight = new Map<string, { controller: AbortController; coversUpToId: string }>();
+
+  /** In-flight observer async tasks. */
+  readonly observerTasks = new Set<Promise<void>>();
+
+  consolidatorInFlight = false;
+  consolidatorController: AbortController | undefined;
+
+  dispatchedCoversUpToId: string | undefined;
+
+  compactInFlight = false;
+  compactHookInFlight = false;
+  autoCompactionController: AbortController | undefined;
+
+  lastWorkerError: string | undefined;
+
+  lastCompactionObserverWait: "skipped" | "waited" | undefined;
+
+  readonly status = new StatusController();
+
+  private pendingInfoToastLines: string[] = [];
+  private infoToastFlushTimer: ReturnType<typeof setTimeout> | undefined;
+
+  queueToast(
+    line: string,
+    level: "info" | "warning" | "error",
+    notify: (message: string, level: "info" | "warning" | "error") => void,
+  ): void {
+    if (level !== "info") {
+      notify(line, level);
+      return;
+    }
+    this.pendingInfoToastLines.push(line);
+    if (this.infoToastFlushTimer !== undefined) return;
+    this.infoToastFlushTimer = setTimeout(() => {
+      this.infoToastFlushTimer = undefined;
+      const lines = this.pendingInfoToastLines.splice(0);
+      if (lines.length > 0) notify(lines.join("\n"), "info");
+    }, 0);
+    (this.infoToastFlushTimer as any)?.unref?.();
+  }
+
+  cancelPendingToasts(): void {
+    if (this.infoToastFlushTimer !== undefined) {
+      clearTimeout(this.infoToastFlushTimer);
+      this.infoToastFlushTimer = undefined;
+    }
+    this.pendingInfoToastLines = [];
+  }
+
+  ensureConfig(cwd: string): void {
+    if (this.configLoaded) return;
+    this.config = loadConfig(cwd);
+    this.configLoaded = true;
+  }
+
+  refreshFooterGauges(branch: Entry[], contextTokens?: number | null): void {
+    if (!this.enabled) return;
+    const folded = foldLedger(branch);
+    this.status.setGauges({
+      nextValue: rawTokensSinceObservationCoverage(branch),
+      nextMax: this.config.chunkTokens,
+      poolValue: poolTokens(folded.activeObservations),
+      poolMax: this.config.consolidateAtPoolTokens,
+      ctxValue: contextTokens ?? 0,
+      ctxMax: this.config.compactAtContextTokens,
+    });
+  }
+
+  refreshCost(allEntries: Entry[]): void {
+    if (!this.enabled) return;
+    const { costUsd, runs } = sumSessionCost(allEntries);
+    this.status.setCost(costUsd, runs);
+  }
+
+  abortAllWorkers(): void {
+    this.cancelPendingToasts();
+    for (const { controller } of this.observersInFlight.values()) {
+      controller.abort();
+    }
+    this.observersInFlight.clear();
+    this.consolidatorController?.abort();
+    this.consolidatorController = undefined;
+    this.consolidatorInFlight = false;
+    if (this.autoCompactionController) {
+      this.autoCompactionController.abort();
+      this.autoCompactionController = undefined;
+    }
+  }
+
+  trackObserverTask(task: Promise<void>): void {
+    this.observerTasks.add(task);
+    void task.finally(() => this.observerTasks.delete(task));
+  }
+
+  async whenObserversIdle(): Promise<void> {
+    while (this.observerTasks.size > 0) {
+      await Promise.allSettled([...this.observerTasks]);
+    }
+  }
+
+  get observerSlotsAvailable(): number {
+    return Math.max(0, this.config.observerConcurrency - this.observersInFlight.size);
+  }
+}

package/extensions/memory-log/spawn/launch.ts

@@ -0,0 +1,133 @@
+import { spawn } from "node:child_process";
+import { mkdirSync, realpathSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { dirname, join, resolve } from "node:path";
+import type { ConfiguredModel } from "../config.ts";
+import { runCostPath, runResultPath } from "./runs.ts";
+
+/** Path to agent/index.ts relative to spawn directory */
+export const AGENT_EXTENSION_PATH = resolve(dirname(fileURLToPath(import.meta.url)), "..", "agent", "index.ts");
+
+export function modelArg(model: ConfiguredModel): string {
+  return model.provider && model.id ? `${model.provider}/${model.id}` : (model.id || "");
+}
+
+/** Resolve the `pi` entry point, falling back to `pi` on PATH. */
+export function resolvePiBinary(): { command: string; baseArgs: string[] } {
+  const entry = process.argv[1];
+  if (entry) {
+    try {
+      const realEntry = realpathSync(entry);
+      if (/\.(?:mjs|cjs|js|ts)$/i.test(realEntry)) {
+        return { command: process.execPath, baseArgs: [realEntry] };
+      }
+    } catch {
+      // fall through
+    }
+  }
+  const fallbackCmd = process.platform === "win32" ? "pi.cmd" : "pi";
+  return { command: fallbackCmd, baseArgs: [] };
+}
+
+export function buildWorkerArgv(opts: {
+  model: ConfiguredModel;
+  sessionName: string;
+  kickoffPrompt: string;
+  promptPath?: string;
+  agentExtensionPath?: string;
+}): string[] {
+  const pi = resolvePiBinary();
+  const args = [
+    ...pi.baseArgs,
+    "--no-skills",
+    "--no-prompt-templates",
+    "--no-context-files",
+    "--no-builtin-tools",
+  ];
+  if (opts.model.provider) {
+    args.push("--provider", opts.model.provider);
+  }
+  if (opts.model.id) {
+    args.push("--model", opts.model.id);
+  }
+  if (opts.model.thinking) {
+    args.push("--thinking", opts.model.thinking);
+  }
+  args.push("-e", opts.agentExtensionPath ?? AGENT_EXTENSION_PATH);
+  args.push("-n", opts.sessionName);
+  if (opts.promptPath) {
+    args.push("-p", `@${resolve(opts.promptPath)}`);
+  } else {
+    args.push("-p", opts.kickoffPrompt);
+  }
+  return [pi.command, ...args];
+}
+
+export type WorkerExit = { code: number | null; signal: NodeJS.Signals | null; stderr: string };
+
+/**
+ * Spawn a headless worker; resolve when it exits.
+ */
+export function spawnWorker(opts: {
+  argv: string[];
+  cwd: string;
+  env: NodeJS.ProcessEnv;
+  signal?: AbortSignal;
+}): Promise<WorkerExit> {
+  const [command, ...rest] = opts.argv;
+  mkdirSync(opts.cwd, { recursive: true });
+  return new Promise<WorkerExit>((resolvePromise) => {
+    const isWindowsCmd = process.platform === "win32" && (command.endsWith(".cmd") || command.endsWith(".bat"));
+    let proc;
+    try {
+      proc = spawn(command, rest, {
+        cwd: opts.cwd,
+        env: opts.env,
+        stdio: ["ignore", "ignore", "pipe"],
+        shell: isWindowsCmd,
+      });
+    } catch (err: any) {
+      resolvePromise({ code: 1, signal: null, stderr: `spawn error: ${err?.message || String(err)}` });
+      return;
+    }
+
+    let stderr = "";
+    proc.stderr?.on("data", (d: Buffer | string) => {
+      stderr += d.toString();
+    });
+    proc.on("error", (err: any) => resolvePromise({ code: 1, signal: null, stderr: stderr || `spawn error: ${err?.message || String(err)}` }));
+    proc.on("close", (code: number | null, signal: NodeJS.Signals | null) => resolvePromise({ code, signal, stderr }));
+
+    if (opts.signal) {
+      const kill = () => {
+        proc.kill("SIGTERM");
+        const timer = setTimeout(() => {
+          if (!proc.killed) proc.kill("SIGKILL");
+        }, 3000);
+        (timer as any).unref?.();
+      };
+      if (opts.signal.aborted) kill();
+      else opts.signal.addEventListener("abort", kill, { once: true });
+    }
+  });
+}
+
+export type ObserverLaunchEnv = {
+  /** Absolute `.memory/<sessionId>/` root. */
+  memoryRoot: string;
+  runId: string;
+};
+
+/**
+ * Build the env a worker subprocess needs to write its result file.
+ */
+export function buildWorkerEnv(role: "observer" | "consolidator", opts: ObserverLaunchEnv): NodeJS.ProcessEnv {
+  return {
+    ...process.env,
+    OM_WORKER: role,
+    OM_RUN_ID: opts.runId,
+    OM_RESULT_PATH: runResultPath(opts.memoryRoot, opts.runId),
+    OM_COST_PATH: runCostPath(opts.memoryRoot, opts.runId),
+    OM_MEMORY_DIR: opts.memoryRoot,
+  };
+}

package/extensions/memory-log/spawn/runs.ts

@@ -0,0 +1,83 @@
+import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+
+/** What the observer model emits, before the orchestrator re-derives precise timestamp-ids. */
+export type RawObservation = {
+  timestamp: string; // "YYYY-MM-DD HH:MM"
+  content: string;
+};
+
+export type ObserverRunResult = {
+  observations: RawObservation[];
+};
+
+export function runsDir(root: string): string {
+  return resolve(root, ".runs");
+}
+
+export function runResultPath(root: string, runId: string): string {
+  return resolve(runsDir(root), `${runId}.result.json`);
+}
+
+/**
+ * Per-run cost handoff file. Written by the worker EXTENSION from pi's built-in usage.cost.total.
+ */
+export function runCostPath(root: string, runId: string): string {
+  return resolve(runsDir(root), `${runId}.cost.json`);
+}
+
+/**
+ * Per-run prompt file for workers to avoid command line length limits (spawn ENAMETOOLONG on Windows).
+ */
+export function runPromptPath(root: string, runId: string): string {
+  return resolve(runsDir(root), `${runId}.prompt.txt`);
+}
+
+export type WorkerCostResult = {
+  costUsd: number;
+};
+
+export function writeWorkerCost(path: string, cost: WorkerCostResult): void {
+  atomicWrite(path, JSON.stringify(cost));
+}
+
+/** Best-effort read of a worker cost file; returns undefined on missing/malformed input. */
+export function readWorkerCost(path: string): WorkerCostResult | undefined {
+  try {
+    const raw = JSON.parse(readFileSync(path, "utf-8")) as unknown;
+    if (!raw || typeof raw !== "object") return undefined;
+    const cost = (raw as { costUsd?: unknown }).costUsd;
+    if (typeof cost !== "number" || !Number.isFinite(cost) || cost < 0) return undefined;
+    return { costUsd: cost };
+  } catch {
+    return undefined;
+  }
+}
+
+/** Atomic write (temp + rename) so a reader never sees a half-written file. */
+export function atomicWrite(path: string, content: string): void {
+  mkdirSync(dirname(path), { recursive: true });
+  const tmp = `${path}.tmp-${process.pid}-${Date.now()}`;
+  writeFileSync(tmp, content, "utf-8");
+  renameSync(tmp, path);
+}
+
+function isRawObservation(value: unknown): value is RawObservation {
+  if (!value || typeof value !== "object") return false;
+  const v = value as Record<string, unknown>;
+  return typeof v.timestamp === "string" && typeof v.content === "string" && v.content.trim().length > 0;
+}
+
+/** Parse + validate an observer result file. Throws on malformed input. */
+export function readObserverResult(path: string): ObserverRunResult {
+  const raw = JSON.parse(readFileSync(path, "utf-8")) as unknown;
+  if (!raw || typeof raw !== "object" || !Array.isArray((raw as { observations?: unknown }).observations)) {
+    throw new Error("observer result missing observations array");
+  }
+  const observations = (raw as { observations: unknown[] }).observations.filter(isRawObservation);
+  return { observations };
+}
+
+export function writeObserverResult(path: string, result: ObserverRunResult): void {
+  atomicWrite(path, JSON.stringify(result));
+}

package/extensions/memory-log/tokens.ts

@@ -0,0 +1,66 @@
+/** Rough char-based token estimate. Matches OM's heuristic (≈4 chars/token). */
+export function estimateStringTokens(text: string): number {
+  return Math.ceil(text.length / 4);
+}
+
+export function estimateMessageTokens(message: unknown): number {
+  if (!message || typeof message !== "object") return 0;
+  const msg = message as { content?: unknown };
+  if (typeof msg.content === "string") {
+    return estimateStringTokens(msg.content);
+  }
+  if (Array.isArray(msg.content)) {
+    let total = 0;
+    for (const part of msg.content) {
+      if (part && typeof part === "object") {
+        if ("text" in part && typeof (part as { text?: unknown }).text === "string") {
+          total += estimateStringTokens((part as { text: string }).text);
+        } else if ("thinking" in part && typeof (part as { thinking?: unknown }).thinking === "string") {
+          total += estimateStringTokens((part as { thinking: string }).thinking);
+        }
+      }
+    }
+    return total;
+  }
+  try {
+    return estimateStringTokens(JSON.stringify(message));
+  } catch {
+    return 0;
+  }
+}
+
+/**
+ * Estimate the token cost of a single ledger entry. Only "source" entries
+ * (message / custom_message / branch_summary) carry conversational text; every
+ * other entry type (memory ledger records, compaction, etc.) contributes 0 so
+ * the raw-token clocks measure only new conversation.
+ */
+export function estimateEntryTokens(entry: {
+  type: string;
+  message?: unknown;
+  content?: unknown;
+  summary?: unknown;
+}): number {
+  if (entry.type === "message" && entry.message) {
+    return estimateMessageTokens(
+      entry.message as Parameters<typeof estimateMessageTokens>[0],
+    );
+  }
+  if (entry.type === "custom_message" && entry.content) {
+    const content = entry.content;
+    if (typeof content === "string") return estimateStringTokens(content);
+    if (Array.isArray(content)) {
+      let total = 0;
+      for (const block of content) {
+        if (block.type === "text" && block.text) {
+          total += estimateStringTokens(block.text);
+        }
+      }
+      return total;
+    }
+  }
+  if (entry.type === "branch_summary" && typeof entry.summary === "string") {
+    return estimateStringTokens(entry.summary);
+  }
+  return 0;
+}

package/extensions/memory-log/ui/status-controller.ts

@@ -0,0 +1,225 @@
+/**
+ * TUI observability for SRP Observational Memory.
+ */
+
+export type WorkerType = "observer" | "consolidator";
+
+export interface FooterGauges {
+  /** Raw tokens accrued toward the next observer chunk. */
+  nextValue: number;
+  nextMax: number;
+  /** Active pool tokens accrued toward the consolidation threshold. */
+  poolValue: number;
+  poolMax: number;
+  /** Live context-window tokens toward the compaction threshold. */
+  ctxValue: number;
+  ctxMax: number;
+}
+
+interface Theme {
+  fg(color: string, text: string): string;
+  bg?(color: string, text: string): string;
+  bold?(text: string): string;
+}
+
+export interface StatusUI {
+  setStatus(key: string, text: string | undefined): void;
+  setWidget(
+    key: string,
+    content: string[] | undefined,
+    options?: { placement?: "aboveEditor" | "belowEditor" },
+  ): void;
+  theme: Theme;
+}
+
+type WorkerState =
+  | { kind: "running" }
+  | { kind: "done"; delta?: number }
+  | { kind: "error" };
+
+const WIDGET_KEY = "memory-log";
+const LEGACY_WIDGET_KEY = "srp-memory";
+const LEGACY_WORKERS_WIDGET_KEY = "srp-memory-workers";
+const LEGACY_STATUS_KEY = "srp-memory";
+const SPINNER_FRAMES = ["◐", "◓", "◑", "◒"] as const;
+const WORKER_SEP = "   ";
+
+export interface StatusControllerOptions {
+  spinnerIntervalMs?: number;
+  settleMs?: number;
+}
+
+interface WorkerEntry {
+  type: WorkerType;
+  state: WorkerState;
+  settleTimer?: ReturnType<typeof setTimeout>;
+}
+
+export class StatusController {
+  private ui: StatusUI | undefined;
+  private frame = 0;
+  private readonly workers = new Map<string, WorkerEntry>();
+  private spinnerTimer: ReturnType<typeof setInterval> | undefined;
+  private gauges: FooterGauges | undefined;
+  private cost: { costUsd: number; runs: number } | undefined;
+  private readonly spinnerIntervalMs: number;
+  private readonly settleMs: number;
+
+  constructor(options: StatusControllerOptions = {}) {
+    this.spinnerIntervalMs = options.spinnerIntervalMs ?? 120;
+    this.settleMs = options.settleMs ?? 5000;
+  }
+
+  attach(ui: StatusUI): void {
+    this.ui = ui;
+    this.ui.setStatus(LEGACY_STATUS_KEY, undefined);
+    this.ui.setStatus(WIDGET_KEY, undefined);
+    this.ui.setWidget(LEGACY_WORKERS_WIDGET_KEY, undefined);
+    this.ui.setWidget(LEGACY_WIDGET_KEY, undefined);
+    this.renderWidget();
+  }
+
+  detach(): void {
+    this.stopSpinner();
+    for (const entry of this.workers.values()) {
+      if (entry.settleTimer) clearTimeout(entry.settleTimer);
+    }
+    this.workers.clear();
+    this.gauges = undefined;
+    this.cost = undefined;
+    this.ui?.setWidget(WIDGET_KEY, undefined);
+    this.ui?.setWidget(LEGACY_WORKERS_WIDGET_KEY, undefined);
+    this.ui?.setWidget(LEGACY_WIDGET_KEY, undefined);
+    if (this.ui) this.ui.setStatus(LEGACY_STATUS_KEY, undefined);
+    if (this.ui) this.ui.setStatus(WIDGET_KEY, undefined);
+    this.ui = undefined;
+  }
+
+  setGauges(gauges: FooterGauges | undefined): void {
+    this.gauges = gauges;
+    this.renderWidget();
+  }
+
+  setCost(costUsd: number, runs: number): void {
+    this.cost = { costUsd, runs };
+    this.renderWidget();
+  }
+
+  workerStart(type: WorkerType, runId: string): void {
+    if (!this.ui) return;
+    const existing = this.workers.get(runId);
+    if (existing?.settleTimer) clearTimeout(existing.settleTimer);
+    this.workers.set(runId, { type, state: { kind: "running" } });
+    this.startSpinner();
+    this.renderWidget();
+  }
+
+  workerDone(runId: string, delta?: number): void {
+    this.settle(runId, { kind: "done", delta });
+  }
+
+  workerError(runId: string): void {
+    this.settle(runId, { kind: "error" });
+  }
+
+  private settle(runId: string, state: WorkerState): void {
+    if (!this.ui) return;
+    const entry = this.workers.get(runId);
+    if (!entry) return;
+    if (entry.settleTimer) clearTimeout(entry.settleTimer);
+    entry.state = state;
+    this.renderWidget();
+    entry.settleTimer = setTimeout(() => {
+      this.workers.delete(runId);
+      this.renderWidget();
+      if (!this.hasRunningWorker()) this.stopSpinner();
+    }, this.settleMs);
+    (entry.settleTimer as any)?.unref?.();
+    if (!this.hasRunningWorker()) this.stopSpinner();
+  }
+
+  private hasRunningWorker(): boolean {
+    for (const entry of this.workers.values()) {
+      if (entry.state.kind === "running") return true;
+    }
+    return false;
+  }
+
+  private startSpinner(): void {
+    if (this.spinnerTimer) return;
+    this.spinnerTimer = setInterval(() => {
+      this.frame = (this.frame + 1) % SPINNER_FRAMES.length;
+      if (this.hasRunningWorker()) this.renderWidget();
+    }, this.spinnerIntervalMs);
+    (this.spinnerTimer as any)?.unref?.();
+  }
+
+  private stopSpinner(): void {
+    if (!this.spinnerTimer) return;
+    clearInterval(this.spinnerTimer);
+    this.spinnerTimer = undefined;
+  }
+
+  private gaugeBar(value: number, max: number, cells = 6): string {
+    const theme = this.ui!.theme;
+    const frac = max <= 0 ? 0 : Math.max(0, value / max);
+    const filled = Math.min(cells, Math.round(Math.min(1, frac) * cells));
+    const fillColor = frac >= 1 ? "warning" : "dim";
+    return (
+      theme.fg(fillColor, "▕") +
+      theme.fg(fillColor, "█".repeat(filled)) +
+      theme.fg(fillColor, "░".repeat(cells - filled)) +
+      theme.fg(fillColor, "▏")
+    );
+  }
+
+  private renderWorkersLine(): string | undefined {
+    if (this.workers.size === 0) return undefined;
+    const theme = this.ui!.theme;
+    const parts: string[] = [];
+    for (const entry of this.workers.values()) {
+      if (entry.state.kind === "running") {
+        parts.push(`${theme.fg("accent", SPINNER_FRAMES[this.frame])} ${theme.fg("accent", `[${entry.type}]`)}`);
+      } else if (entry.state.kind === "error") {
+        parts.push(`${theme.fg("error", "✗")} ${theme.fg("muted", `[${entry.type}]`)}`);
+      } else {
+        const delta =
+          entry.state.delta && entry.state.delta > 0
+            ? ` ${theme.fg("success", `+${entry.state.delta}`)}`
+            : "";
+        parts.push(`${theme.fg("success", "✓")} ${theme.fg("muted", `[${entry.type}]`)}${delta}`);
+      }
+    }
+    return parts.length > 0 ? parts.join(WORKER_SEP) : undefined;
+  }
+
+  private renderMemLine(): string {
+    const theme = this.ui?.theme;
+    if (!theme) return "memory-log";
+    const base = `${theme.fg("success", "● mem")}`;
+    const g = this.gauges;
+    if (!g) return base;
+    const next = `${theme.fg("muted", "O")}${this.gaugeBar(g.nextValue, g.nextMax)}`;
+    const pool = `${theme.fg("muted", "C")}${this.gaugeBar(g.poolValue, g.poolMax)}`;
+    const ctx = `${theme.fg("muted", "X")}${this.gaugeBar(g.ctxValue, g.ctxMax)}`;
+    const cost = this.cost && this.cost.costUsd > 0 ? ` ${theme.fg("dim", `$${this.cost.costUsd.toFixed(3)}`)}` : "";
+    return `${base} ${next} ${pool} ${ctx}${cost}`;
+  }
+
+  private renderWidget(): void {
+    const ui = this.ui;
+    if (!ui) return;
+
+    const lines: string[] = [];
+    const workersLine = this.renderWorkersLine();
+    if (workersLine) {
+      lines.push(workersLine);
+    }
+    const memLine = this.renderMemLine();
+    if (memLine) {
+      lines.push(memLine);
+    }
+
+    ui.setWidget(WIDGET_KEY, lines.length > 0 ? lines : undefined, { placement: "aboveEditor" });
+  }
+}