@rol1n/memory-log 0.0.0-stage → 0.1.0

package/extensions/memory-log/commands/status.ts

@@ -0,0 +1,50 @@
+import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
+import { foldLedger, poolTokens, rawTokensSinceObservationCoverage, sumSessionCost, type Entry } from "../ledger/index.ts";
+import { listTopics, readJourney } from "../memory/paths.ts";
+import { estimateStringTokens } from "../tokens.ts";
+import type { Runtime } from "../runtime.ts";
+import { renderTimeline } from "../ui/timeline.ts";
+
+export async function handleStatusCommand(_args: string, ctx: ExtensionContext, runtime: Runtime): Promise<void> {
+  if (!ctx.hasUI) return;
+  if (!runtime.enabled) {
+    ctx.ui.notify("memory-log is disabled (run /mem-log on to enable)", "info");
+    return;
+  }
+  runtime.ensureConfig(ctx.cwd);
+  const branch = ctx.sessionManager.getBranch() as Entry[];
+  const folded = foldLedger(branch);
+  const sinceObservation = rawTokensSinceObservationCoverage(branch);
+  const contextTokens = ctx.getContextUsage?.()?.tokens ?? null;
+  const pool = poolTokens(folded.activeObservations);
+  const topicCount = listTopics(runtime.memoryRoot).length;
+  const journey = readJourney(runtime.memoryRoot);
+  const { costUsd, runs } = sumSessionCost(ctx.sessionManager.getEntries() as Entry[]);
+
+  const lines = [
+    `─── Observational Memory Log Status ───`,
+    `  • Active observers      : ${runtime.observersInFlight.size} / ${runtime.config.observerConcurrency}`,
+    `  • Active observations   : ${folded.activeObservations.length}`,
+    `  • Next chunk progress   : ${sinceObservation.toLocaleString()} / ${runtime.config.chunkTokens.toLocaleString()} tok`,
+    `  • Active observation pool: ${pool.toLocaleString()} tok (target: ${runtime.config.poolTargetTokens.toLocaleString()}, trigger: ${runtime.config.consolidateAtPoolTokens.toLocaleString()})`,
+    `  • Consolidator status   : ${runtime.consolidatorInFlight ? "running" : "idle"}` ,
+    `  • Compaction wait status : ${runtime.lastCompactionObserverWait ?? "n/a"}`,
+    `  • Topic files count     : ${topicCount} (.memory/<sessionId>/)`,
+    `  • Journey history       : ${journey ? `~${estimateStringTokens(journey).toLocaleString()} / ${runtime.config.journeyTargetTokens.toLocaleString()} tok` : "none"}`,
+    `  • Context window usage  : ${contextTokens != null ? contextTokens.toLocaleString() : "?"} / ${runtime.config.compactAtContextTokens.toLocaleString()} tok`,
+    `  • Session total cost    : $${costUsd.toFixed(4)} (${runs} run${runs === 1 ? "" : "s"})`,
+    `  • Last worker error     : ${runtime.lastWorkerError || "none"}`,
+    ``,
+    renderTimeline(branch, runtime.config),
+  ];
+  ctx.ui.notify(lines.join("\n"), "info");
+}
+
+export function registerStatusCommand(pi: ExtensionAPI, runtime: Runtime): void {
+  const handler = async (args: string, ctx: ExtensionContext) => handleStatusCommand(args, ctx, runtime);
+
+  pi.registerCommand("mem-log:status", {
+    description: "View observational memory log running status",
+    handler,
+  });
+}

package/extensions/memory-log/config.ts

@@ -0,0 +1,253 @@
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import type { ModelThinkingLevel } from "@earendil-works/pi-ai";
+import { getAgentDir } from "@earendil-works/pi-coding-agent";
+
+const CONFIG_DIR_NAME = ".pi";
+
+export interface ConfiguredModel {
+  provider?: string;
+  id?: string;
+  thinking?: ModelThinkingLevel;
+}
+
+export interface Config {
+  /** Whether new sessions start with observational memory enabled. */
+  defaultEnabled: boolean;
+  /** Raw-history token size of one observation chunk (fixed boundary). */
+  chunkTokens: number;
+  /** Overlap between adjacent chunks; default 0 in v1. */
+  chunkOverlapTokens: number;
+  /** Target size of the active observation pool; the buffer drains back toward this after consolidation. */
+  poolTargetTokens: number;
+  /** Active-pool token count that triggers a consolidation (150% - 200% of target). */
+  consolidateAtPoolTokens: number;
+  /** Live context-window usage that triggers compaction. */
+  compactAtContextTokens: number;
+  /** Verbatim raw tail kept after the cutoff; snaps to a chunk boundary. */
+  tailTokens: number;
+  /**
+   * Target size of `.memory/JOURNEY.md`, the running descriptive project history the
+   * consolidator appends to and pushes into every compaction block.
+   */
+  journeyTargetTokens: number;
+  /** Max simultaneous in-flight observer subprocesses. */
+  observerConcurrency: number;
+  models: {
+    observer: ConfiguredModel;
+    consolidator: ConfiguredModel;
+  };
+  /**
+   * Pi settings default model (`defaultProvider`/`defaultModel` from settings.json),
+   * used as the last-resort fallback when no session model is available.
+   */
+  settingsDefaultModel?: { provider?: string; id?: string };
+  /** Resume the agent automatically after a mid-run compaction. */
+  resumeAfterMidRunCompaction: boolean;
+  /** Power-user setting: disable all triggers (distinct from the on/off gate). */
+  passive: boolean;
+  /** Emit the NDJSON debug log. */
+  debugLog: boolean;
+}
+
+export const DEFAULTS: Config = {
+  defaultEnabled: false,
+  chunkTokens: 10_000,
+  chunkOverlapTokens: 0,
+  poolTargetTokens: 10_000,
+  consolidateAtPoolTokens: 15_000,
+  compactAtContextTokens: 150_000,
+  tailTokens: 20_000,
+  journeyTargetTokens: 1_000,
+  observerConcurrency: 4,
+  resumeAfterMidRunCompaction: true,
+  models: {
+    observer: { thinking: "low" },
+    consolidator: { thinking: "medium" },
+  },
+  passive: false,
+  debugLog: false,
+};
+
+const THINKING_LEVEL_VALUES: readonly ModelThinkingLevel[] = [
+  "off",
+  "minimal",
+  "low",
+  "medium",
+  "high",
+  "xhigh",
+] as const;
+
+const SETTINGS_KEYS = ["memoryLog", "memory-log", "memLog", "mem-log", "srpMemory", "srp-memory", "observational-memory"] as const;
+const PASSIVE_ENV = "PI_OM_PASSIVE";
+
+function positiveIntegerOrUndefined(value: unknown): number | undefined {
+  return Number.isInteger(value) && typeof value === "number" && value > 0 ? value : undefined;
+}
+
+function isThinkingLevel(value: unknown): value is ModelThinkingLevel {
+  return typeof value === "string" && (THINKING_LEVEL_VALUES as readonly string[]).includes(value);
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function nonEmptyString(value: unknown): string | undefined {
+  return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+
+function normalizeModel(value: unknown, fallback: ConfiguredModel): ConfiguredModel {
+  if (!isRecord(value)) return fallback;
+  const provider = nonEmptyString(value.provider) ?? fallback.provider;
+  const id = nonEmptyString(value.id) ?? fallback.id;
+  const model: ConfiguredModel = {};
+  if (provider) model.provider = provider;
+  if (id) model.id = id;
+  const thinking = isThinkingLevel(value.thinking) ? value.thinking : fallback.thinking;
+  if (thinking) model.thinking = thinking;
+  return model;
+}
+
+/**
+ * Resolve the model a worker subprocess should run on.
+ *
+ * Priority: memory-log configured model > current session model > pi settings
+ * default model. When nothing resolves, return an empty model so the worker
+ * omits `--provider`/`--model` and pi resolves its own default — never pin a
+ * hardcoded model here.
+ */
+export function resolveEffectiveModel(
+  configured?: ConfiguredModel,
+  sessionModel?: { provider?: string; id?: string },
+  settingsDefault?: { provider?: string; id?: string },
+): ConfiguredModel {
+  const model: ConfiguredModel = {};
+  const provider = configured?.provider || sessionModel?.provider || settingsDefault?.provider;
+  const id = configured?.id || sessionModel?.id || settingsDefault?.id;
+  if (provider) model.provider = provider;
+  if (id) model.id = id;
+  if (configured?.thinking) model.thinking = configured.thinking;
+  return model;
+}
+
+function normalizeSettingsConfig(value: Record<string, unknown>, base: Config): Partial<Config> {
+  const normalized: Partial<Config> = {};
+  if (typeof value.default_enabled === "boolean") {
+    normalized.defaultEnabled = value.default_enabled;
+  }
+  if (typeof value.defaultEnabled === "boolean") {
+    normalized.defaultEnabled = value.defaultEnabled;
+  }
+  const numberKeys = [
+    "chunkTokens",
+    "chunkOverlapTokens",
+    "poolTargetTokens",
+    "consolidateAtPoolTokens",
+    "compactAtContextTokens",
+    "tailTokens",
+    "journeyTargetTokens",
+    "observerConcurrency",
+  ] as const;
+  for (const key of numberKeys) {
+    const normalizedValue = positiveIntegerOrUndefined(value[key]);
+    if (normalizedValue !== undefined) normalized[key] = normalizedValue;
+  }
+  if (value.chunkOverlapTokens === 0) normalized.chunkOverlapTokens = 0;
+  if (typeof value.resumeAfterMidRunCompaction === "boolean") {
+    normalized.resumeAfterMidRunCompaction = value.resumeAfterMidRunCompaction;
+  }
+  if (typeof value.passive === "boolean") normalized.passive = value.passive;
+  if (typeof value.debugLog === "boolean") normalized.debugLog = value.debugLog;
+  if (isRecord(value.models)) {
+    normalized.models = {
+      observer: normalizeModel(value.models.observer, base.models.observer),
+      consolidator: normalizeModel(value.models.consolidator, base.models.consolidator),
+    };
+  }
+  return normalized;
+}
+
+function readJsonFile(path: string): Record<string, unknown> {
+  try {
+    if (!existsSync(path)) return {};
+    const parsed = JSON.parse(readFileSync(path, "utf-8"));
+    return isRecord(parsed) ? parsed : {};
+  } catch {
+    return {};
+  }
+}
+
+function extractDefaultModel(settings: Record<string, unknown>): { provider?: string; id?: string } | undefined {
+  const provider = nonEmptyString(settings.defaultProvider);
+  const id = nonEmptyString(settings.defaultModel);
+  if (!provider && !id) return undefined;
+  const model: { provider?: string; id?: string } = {};
+  if (provider) model.provider = provider;
+  if (id) model.id = id;
+  return model;
+}
+
+function findSection(settings: Record<string, unknown>): Record<string, unknown> | undefined {
+  for (const key of SETTINGS_KEYS) {
+    if (isRecord(settings[key])) {
+      return settings[key] as Record<string, unknown>;
+    }
+  }
+  return undefined;
+}
+
+export function loadConfig(cwd: string = process.cwd()): Config {
+  let merged: Config = {
+    ...DEFAULTS,
+    models: {
+      observer: { ...DEFAULTS.models.observer },
+      consolidator: { ...DEFAULTS.models.consolidator },
+    },
+  };
+
+  const globalSettings = readJsonFile(join(getAgentDir(), "settings.json"));
+  const globalSection = findSection(globalSettings);
+  if (globalSection) {
+    const globalNormalized = normalizeSettingsConfig(globalSection, merged);
+    merged = { ...merged, ...globalNormalized };
+  }
+  const globalDefaultModel = extractDefaultModel(globalSettings);
+  if (globalDefaultModel) merged.settingsDefaultModel = globalDefaultModel;
+
+  const projectSettings = readJsonFile(join(cwd, CONFIG_DIR_NAME, "settings.json"));
+  const projectSection = findSection(projectSettings);
+  if (projectSection) {
+    const projectNormalized = normalizeSettingsConfig(projectSection, merged);
+    merged = { ...merged, ...projectNormalized };
+  }
+  const projectDefaultModel = extractDefaultModel(projectSettings);
+  if (projectDefaultModel) merged.settingsDefaultModel = projectDefaultModel;
+
+  // Environment variable overrides
+  if (process.env[PASSIVE_ENV] === "1" || process.env[PASSIVE_ENV]?.toLowerCase() === "true") {
+    merged.passive = true;
+  }
+
+  if (process.env.PI_OM_OBSERVER_PROVIDER) {
+    merged.models.observer.provider = process.env.PI_OM_OBSERVER_PROVIDER;
+  }
+  if (process.env.PI_OM_OBSERVER_MODEL) {
+    merged.models.observer.id = process.env.PI_OM_OBSERVER_MODEL;
+  }
+  if (isThinkingLevel(process.env.PI_OM_OBSERVER_THINKING)) {
+    merged.models.observer.thinking = process.env.PI_OM_OBSERVER_THINKING as ModelThinkingLevel;
+  }
+
+  if (process.env.PI_OM_CONSOLIDATOR_PROVIDER) {
+    merged.models.consolidator.provider = process.env.PI_OM_CONSOLIDATOR_PROVIDER;
+  }
+  if (process.env.PI_OM_CONSOLIDATOR_MODEL) {
+    merged.models.consolidator.id = process.env.PI_OM_CONSOLIDATOR_MODEL;
+  }
+  if (isThinkingLevel(process.env.PI_OM_CONSOLIDATOR_THINKING)) {
+    merged.models.consolidator.thinking = process.env.PI_OM_CONSOLIDATOR_THINKING as ModelThinkingLevel;
+  }
+
+  return merged;
+}

package/extensions/memory-log/debug-log.ts

@@ -0,0 +1,76 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import { existsSync, mkdirSync, renameSync, statSync, unlinkSync, appendFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { getAgentDir } from "@earendil-works/pi-coding-agent";
+
+export const DEBUG_LOG_MAX_BYTES = 10 * 1024 * 1024;
+export const DEBUG_LOG_RELATIVE_PATH = join("memory-log", "debug.ndjson");
+export const DEBUG_LOG_SESSION_DIR_RELATIVE_PATH = join("memory-log", "debug");
+
+export interface DebugLogContext {
+  enabled: boolean;
+  cwd?: string;
+  sessionId?: string;
+  sessionFile?: string;
+  runId?: string;
+}
+
+const storage = new AsyncLocalStorage<DebugLogContext>();
+
+export function withDebugLogContext<T>(context: DebugLogContext, fn: () => T): T {
+  const parent = storage.getStore();
+  return storage.run({ ...parent, ...context }, fn);
+}
+
+export function safeDebugLogSessionId(sessionId: string | undefined): string | undefined {
+  const trimmed = sessionId?.trim();
+  if (!trimmed) return undefined;
+  const sanitized = trimmed
+    .replace(/[^A-Za-z0-9._-]+/g, "_")
+    .replace(/^_+|_+$/g, "")
+    .slice(0, 128);
+  if (!/[A-Za-z0-9]/.test(sanitized)) return undefined;
+  return sanitized;
+}
+
+export function debugLogRelativePath(context: Pick<DebugLogContext, "sessionId">): string {
+  const safeSessionId = safeDebugLogSessionId(context.sessionId);
+  return safeSessionId
+    ? join(DEBUG_LOG_SESSION_DIR_RELATIVE_PATH, `${safeSessionId}.ndjson`)
+    : DEBUG_LOG_RELATIVE_PATH;
+}
+
+export function debugLog(event: string, data: Record<string, unknown> = {}): void {
+  const context = storage.getStore();
+  if (context?.enabled !== true) return;
+
+  try {
+    const path = join(getAgentDir(), debugLogRelativePath(context));
+    mkdirSync(dirname(path), { recursive: true });
+    rotateIfNeeded(path);
+    const payload = {
+      ts: new Date().toISOString(),
+      event,
+      cwd: context.cwd,
+      sessionId: context.sessionId,
+      sessionFile: context.sessionFile,
+      runId: context.runId,
+      data,
+    };
+    appendFileSync(path, `${JSON.stringify(payload)}\n`, "utf-8");
+  } catch {
+    // Debug logging must never affect memory behavior.
+  }
+}
+
+function rotateIfNeeded(path: string): void {
+  if (!existsSync(path)) return;
+  try {
+    if (statSync(path).size < DEBUG_LOG_MAX_BYTES) return;
+    const backupPath = `${path}.1`;
+    if (existsSync(backupPath)) unlinkSync(backupPath);
+    renameSync(path, backupPath);
+  } catch {
+    // ignore
+  }
+}

package/extensions/memory-log/hooks/compaction-hook.ts

@@ -0,0 +1,161 @@
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { renderMemoryMap } from "../memory/index-render.ts";
+import { listTopics, readJourney } from "../memory/paths.ts";
+import type { Runtime } from "../runtime.ts";
+import {
+  buildCompactionProjection,
+  entryIndexById,
+  findLastCompactionIndex,
+  isObservationsRecordedEntry,
+  isSourceEntry,
+  isValidCutPoint,
+  rawTokensAfterIndex,
+  renderSummary,
+  type Entry,
+} from "../ledger/index.ts";
+
+/** Distinct, branch-resolved coversUpToId indices of committed observation chunks, ascending. */
+function chunkBoundaryIndices(branch: Entry[]): number[] {
+  const indexes = entryIndexById(branch);
+  const set = new Set<number>();
+  for (const entry of branch) {
+    if (!isObservationsRecordedEntry(entry)) continue;
+    const idx = indexes.get(entry.data.coversUpToId);
+    if (idx !== undefined) set.add(idx);
+  }
+  return Array.from(set).sort((a, b) => a - b);
+}
+
+/** First source entry after `boundaryIndex` that is a valid cut point, or undefined. */
+function firstKeptAfterBoundary(branch: Entry[], boundaryIndex: number): Entry | undefined {
+  for (let i = boundaryIndex + 1; i < branch.length; i++) {
+    if (!isSourceEntry(branch[i])) continue;
+    return isValidCutPoint(branch[i]) ? branch[i] : undefined;
+  }
+  return undefined;
+}
+
+export function snapCutoff(
+  branch: Entry[],
+  proposedFirstKeptId: string,
+  tailTokens: number,
+): { firstKeptId: string; tail: number | undefined } {
+  const indexes = entryIndexById(branch);
+  const lastCompactionIdx = findLastCompactionIndex(branch);
+  const boundaries = chunkBoundaryIndices(branch);
+  let bestId: string | undefined;
+  let bestTail: number | undefined;
+  let bestDelta = Number.POSITIVE_INFINITY;
+
+  for (const boundaryIndex of boundaries) {
+    if (lastCompactionIdx !== -1 && boundaryIndex <= lastCompactionIdx) {
+      continue;
+    }
+    const firstKept = firstKeptAfterBoundary(branch, boundaryIndex);
+    if (!firstKept) continue;
+
+    const firstKeptIdx = indexes.get(firstKept.id);
+    if (lastCompactionIdx !== -1 && (firstKeptIdx === undefined || firstKeptIdx <= lastCompactionIdx)) {
+      continue;
+    }
+
+    const tail = rawTokensAfterIndex(branch, boundaryIndex);
+    const delta = Math.abs(tail - tailTokens);
+    if (delta < bestDelta) {
+      bestDelta = delta;
+      bestId = firstKept.id;
+      bestTail = tail;
+    }
+  }
+
+  let finalFirstKeptId = bestId;
+  let finalTail = bestTail;
+
+  // Defensive verification: ensure the returned firstKeptId is NEVER at or before any existing compaction entry on branch.
+  if (finalFirstKeptId !== undefined) {
+    const finalIdx = indexes.get(finalFirstKeptId);
+    if (finalIdx === undefined || (lastCompactionIdx !== -1 && finalIdx <= lastCompactionIdx)) {
+      finalFirstKeptId = undefined;
+      finalTail = undefined;
+    }
+  }
+
+  if (!finalFirstKeptId) {
+    finalFirstKeptId = proposedFirstKeptId;
+    finalTail = undefined;
+  }
+
+  return { firstKeptId: finalFirstKeptId, tail: finalTail };
+}
+
+export function snapFirstKeptEntryId(branch: Entry[], proposedFirstKeptId: string, tailTokens: number): string {
+  return snapCutoff(branch, proposedFirstKeptId, tailTokens).firstKeptId;
+}
+
+export function canSkipObserverWait(
+  branch: Entry[],
+  snappedFirstKeptId: string,
+  snappedTail: number | undefined,
+  tailTokens: number,
+  observersInFlight: Iterable<{ coversUpToId: string }>,
+): boolean {
+  if (snappedTail === undefined || snappedTail > tailTokens) return false;
+
+  const indexes = entryIndexById(branch);
+  const cutoffIndex = indexes.get(snappedFirstKeptId);
+  if (cutoffIndex === undefined) return false;
+
+  for (const { coversUpToId } of observersInFlight) {
+    const idx = indexes.get(coversUpToId);
+    if (idx === undefined || idx < cutoffIndex) return false;
+  }
+  return true;
+}
+
+export function registerCompactionHook(pi: ExtensionAPI, runtime: Runtime): void {
+  pi.on("session_before_compact", async (event: any, ctx: any) => {
+    if (!runtime.enabled || runtime.config.passive) return undefined;
+
+    const hasUI = ctx.hasUI;
+    if (runtime.compactHookInFlight) {
+      if (hasUI) ctx.ui.notify("memory-log: another compaction is already in progress, skipping duplicate trigger", "warning");
+      return { cancel: true };
+    }
+
+    runtime.compactHookInFlight = true;
+    try {
+      runtime.ensureConfig(ctx.cwd);
+      const tailTokens = runtime.config.tailTokens;
+      const { firstKeptEntryId, tokensBefore } = event.preparation;
+
+      let branch = (ctx.sessionManager?.getBranch?.() as Entry[] | undefined) ?? (event.branchEntries as Entry[]);
+      let snap = snapCutoff(branch, firstKeptEntryId, tailTokens);
+
+      const skip = canSkipObserverWait(branch, snap.firstKeptId, snap.tail, tailTokens, runtime.observersInFlight.values());
+      runtime.lastCompactionObserverWait = skip ? "skipped" : "waited";
+      if (!skip) {
+        if (hasUI) ctx.ui.notify("memory-log: waiting for in-flight observers to finish before compaction...", "info");
+        await runtime.whenObserversIdle();
+        branch = (ctx.sessionManager?.getBranch?.() as Entry[] | undefined) ?? (event.branchEntries as Entry[]);
+        snap = snapCutoff(branch, firstKeptEntryId, tailTokens);
+      }
+
+      const snapped = snap.firstKeptId;
+      const projection = buildCompactionProjection(branch, snapped);
+      const journey = readJourney(runtime.memoryRoot);
+      const map = renderMemoryMap(listTopics(runtime.memoryRoot));
+      const summary = renderSummary(journey, map, projection.observations);
+
+      return {
+        compaction: {
+          summary,
+          firstKeptEntryId: snapped,
+          tokensBefore,
+          details: projection.details,
+        },
+      };
+    } finally {
+      runtime.compactHookInFlight = false;
+    }
+  });
+}