@rol1n/memory-log 0.0.0-stage → 0.1.0

package/extensions/memory-log/ledger/serialize.ts

@@ -0,0 +1,161 @@
+import type { Message, TextContent, ToolResultMessage } from "@earendil-works/pi-ai";
+
+function pad(n: number): string {
+  return n.toString().padStart(2, "0");
+}
+
+function fmtLocal(d: Date): string {
+  return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}`;
+}
+
+function formatTimestamp(v: number | string | undefined): string {
+  if (v === undefined) return "????-??-?? ??:??";
+  const d = new Date(v);
+  return Number.isNaN(d.getTime()) ? "????-??-?? ??:??" : fmtLocal(d);
+}
+
+export function nowTimestamp(): string {
+  return fmtLocal(new Date());
+}
+
+export const MAX_RECORD_CONTENT_CHARS = 10_000;
+
+export function truncateRecordContent(content: string): string {
+  if (content.length <= MAX_RECORD_CONTENT_CHARS) return content;
+  const head = content.slice(0, MAX_RECORD_CONTENT_CHARS);
+  const dropped = content.length - MAX_RECORD_CONTENT_CHARS;
+  return `${head} … [truncated ${dropped} chars]`;
+}
+
+function textAndPlaceholders(content: unknown, options: { includeThinking?: boolean } = {}): string {
+  if (typeof content === "string") return content;
+  if (!Array.isArray(content)) return "[non-text content omitted]";
+
+  const parts: string[] = [];
+  for (const block of content as Array<Record<string, unknown>>) {
+    if (!block || typeof block !== "object") {
+      parts.push("[non-text content omitted]");
+      continue;
+    }
+    if (block.type === "text" && typeof block.text === "string") {
+      parts.push(block.text);
+      continue;
+    }
+    if (block.type === "thinking") {
+      if (options.includeThinking && typeof block.thinking === "string") {
+        parts.push(`[thinking: ${block.thinking}]`);
+        continue;
+      }
+      parts.push("[non-text content omitted]");
+      continue;
+    }
+    if (block.type === "toolCall" && typeof block.name === "string") {
+      parts.push(`[${block.name}(${JSON.stringify(block.arguments ?? {})})]`);
+      continue;
+    }
+    parts.push("[non-text content omitted]");
+  }
+  return parts.join("\n");
+}
+
+function textOnly(content: unknown): string {
+  if (content == null) return "";
+  if (typeof content === "string") return content;
+  if (!Array.isArray(content)) return "";
+  return content
+    .filter((b): b is TextContent => b?.type === "text" && typeof b.text === "string")
+    .map((b) => b.text)
+    .join("\n");
+}
+
+export function serializeConversation(messages: Message[]): string {
+  return messages
+    .map((msg): string | null => {
+      const time = formatTimestamp(msg.timestamp);
+      if (msg.role === "user") {
+        return `[User @ ${time}]: ${textOnly(msg.content)}`;
+      }
+      if (msg.role === "assistant") {
+        const body = textAndPlaceholders(msg.content, { includeThinking: true })
+          .split("\n")
+          .filter(Boolean)
+          .join("\n");
+        if (!body) return null;
+        return `[Assistant @ ${time}]: ${body}`;
+      }
+      return `[Tool result for ${(msg as ToolResultMessage).toolName} @ ${time}]: ${textOnly(msg.content)}`;
+    })
+    .filter((line): line is string => line !== null)
+    .join("\n\n");
+}
+
+export type RenderableEntry = {
+  type: string;
+  id?: string;
+  timestamp?: string;
+  message?: unknown;
+  customType?: string;
+  content?: unknown;
+  summary?: unknown;
+};
+
+function renderCustomMessage(entry: RenderableEntry): string {
+  const time = formatTimestamp(entry.timestamp);
+  const text =
+    typeof entry.content === "string"
+      ? entry.content
+      : Array.isArray(entry.content)
+        ? (entry.content as Array<{ type?: string; text?: string }>)
+            .filter((b) => b?.type === "text" && typeof b.text === "string")
+            .map((b) => b.text as string)
+            .join("\n")
+        : "";
+  const tag = entry.customType ? `Custom (${entry.customType})` : "Custom";
+  return `[${tag} @ ${time}]: ${text}`;
+}
+
+export function serializeBranchEntries(entries: RenderableEntry[]): string {
+  const blocks: string[] = [];
+  for (const entry of entries) {
+    if (entry.type === "message" && entry.message) {
+      const part = serializeConversation([entry.message as Message]);
+      if (part) blocks.push(part);
+      continue;
+    }
+    if (entry.type === "custom_message") {
+      blocks.push(renderCustomMessage(entry));
+      continue;
+    }
+    if (entry.type === "branch_summary" && typeof entry.summary === "string") {
+      const time = formatTimestamp(entry.timestamp);
+      blocks.push(`[Branch summary @ ${time}]: ${entry.summary}`);
+    }
+  }
+  return blocks.join("\n\n");
+}
+
+export type SourceAddressedSerialization = {
+  text: string;
+  sourceEntryIds: string[];
+};
+
+function isSourceRenderableEntry(entry: RenderableEntry): boolean {
+  return entry.type === "message" || entry.type === "custom_message" || entry.type === "branch_summary";
+}
+
+/**
+ * Serialize a slice of source entries into an observer prompt chunk, each block prefixed
+ * with its source entry id.
+ */
+export function serializeSourceAddressedBranchEntries(entries: RenderableEntry[]): SourceAddressedSerialization {
+  const blocks: string[] = [];
+  const sourceEntryIds: string[] = [];
+  for (const entry of entries) {
+    if (!entry.id || !isSourceRenderableEntry(entry)) continue;
+    const rendered = serializeBranchEntries([entry]);
+    if (!rendered.trim()) continue;
+    sourceEntryIds.push(entry.id);
+    blocks.push(`[Source entry id: ${entry.id}]\n${rendered}`);
+  }
+  return { text: blocks.join("\n\n"), sourceEntryIds };
+}

package/extensions/memory-log/ledger/types.ts

@@ -0,0 +1,201 @@
+/**
+ * Ledger custom-type vocabulary for observational memory.
+ */
+
+/** Observer output committed by the orchestrator; the buffer tier. */
+export const OM_OBSERVATIONS_RECORDED = "om.observations.recorded";
+/** Promotion tombstones written by the orchestrator after a consolidator run (Phase B). */
+export const OM_OBSERVATIONS_DROPPED = "om.observations.dropped";
+/** Compaction details type stamped into the compaction entry's `details`. */
+export const OM_FOLDED = "om.folded";
+/** Per-session on/off gate state. When absent, defaultEnabled applies. */
+export const OM_ENABLED = "om.enabled";
+/**
+ * Per-worker cost record (one per finished worker run). The orchestrator appends these from
+ * pi's built-in `usage.cost.total`, reported back by the worker extension via the cost file.
+ * Summed across the WHOLE session (every branch), so spend never rolls back under /tree.
+ */
+export const OM_COST = "om.cost";
+/**
+ * Synthetic continuation message used to resume the agent loop after a mid-run compaction.
+ */
+export const OM_RESUME = "om.resume";
+
+export type Entry = {
+  type: string;
+  id: string;
+  timestamp?: string;
+  message?: unknown;
+  content?: unknown;
+  customType?: string;
+  summary?: unknown;
+  fromId?: string;
+  data?: unknown;
+  details?: unknown;
+  firstKeptEntryId?: string;
+};
+
+/**
+ * Minimal observation unit.
+ * - `timestamp`: the orchestrator-assigned precise, unique id-timestamp
+ *   ("YYYY-MM-DDTHH:MM:SS" with an optional ".NN" disambiguator). Doubles as the id.
+ * - `content`: single-line plain prose.
+ * - `tokenCount`: computed in code (never by the model).
+ */
+export type Observation = {
+  timestamp: string;
+  content: string;
+  tokenCount: number;
+};
+
+export type ObservationsRecordedEntryData = {
+  observations: Observation[];
+  coversUpToId: string;
+};
+
+export type ObservationsDroppedEntryData = {
+  observationTimestamps: string[];
+  coversUpToId: string;
+};
+
+export type CostEntryData = {
+  costUsd: number;
+  role: "observer" | "consolidator";
+  runId: string;
+};
+
+/** Stamped into the compaction entry's `details` so a future visible projection can read it back. */
+export type MemoryDetails = {
+  type: typeof OM_FOLDED;
+  version: 1;
+  observations: Observation[];
+};
+
+export type MemoryCustomType = typeof OM_OBSERVATIONS_RECORDED | typeof OM_OBSERVATIONS_DROPPED;
+
+export function isNonEmptyString(value: unknown): value is string {
+  return typeof value === "string" && value.length > 0;
+}
+
+export function isNonEmptyStringArray(value: unknown): value is string[] {
+  return Array.isArray(value) && value.length > 0 && value.every(isNonEmptyString);
+}
+
+function isTokenCount(value: unknown): value is number {
+  return typeof value === "number" && Number.isFinite(value) && value >= 0;
+}
+
+function isPlainRecord(value: unknown): value is Record<string, unknown> {
+  return !!value && typeof value === "object";
+}
+
+export function isObservation(value: unknown): value is Observation {
+  if (!isPlainRecord(value)) return false;
+  return (
+    isNonEmptyString(value.timestamp) &&
+    isNonEmptyString(value.content) &&
+    !/\r|\n/.test(value.content) &&
+    isTokenCount(value.tokenCount)
+  );
+}
+
+export function isObservationsRecordedData(value: unknown): value is ObservationsRecordedEntryData {
+  if (!isPlainRecord(value)) return false;
+  return (
+    Array.isArray(value.observations) &&
+    value.observations.length > 0 &&
+    value.observations.every(isObservation) &&
+    isNonEmptyString(value.coversUpToId)
+  );
+}
+
+export function isObservationsDroppedData(value: unknown): value is ObservationsDroppedEntryData {
+  if (!isPlainRecord(value)) return false;
+  return isNonEmptyStringArray(value.observationTimestamps) && isNonEmptyString(value.coversUpToId);
+}
+
+export function isMemoryDetails(value: unknown): value is MemoryDetails {
+  if (!isPlainRecord(value)) return false;
+  return (
+    value.type === OM_FOLDED &&
+    value.version === 1 &&
+    Array.isArray(value.observations) &&
+    value.observations.every(isObservation)
+  );
+}
+
+export function isCostData(value: unknown): value is CostEntryData {
+  if (!isPlainRecord(value)) return false;
+  return (
+    typeof value.costUsd === "number" &&
+    Number.isFinite(value.costUsd) &&
+    value.costUsd >= 0 &&
+    (value.role === "observer" || value.role === "consolidator") &&
+    isNonEmptyString(value.runId)
+  );
+}
+
+export function isObservationsRecordedEntry(
+  entry: Entry,
+): entry is Entry & { type: "custom"; customType: typeof OM_OBSERVATIONS_RECORDED; data: ObservationsRecordedEntryData } {
+  return (
+    entry.type === "custom" &&
+    entry.customType === OM_OBSERVATIONS_RECORDED &&
+    isObservationsRecordedData(entry.data)
+  );
+}
+
+export function isObservationsDroppedEntry(
+  entry: Entry,
+): entry is Entry & { type: "custom"; customType: typeof OM_OBSERVATIONS_DROPPED; data: ObservationsDroppedEntryData } {
+  return (
+    entry.type === "custom" &&
+    entry.customType === OM_OBSERVATIONS_DROPPED &&
+    isObservationsDroppedData(entry.data)
+  );
+}
+
+export function isCostEntry(
+  entry: Entry,
+): entry is Entry & { type: "custom"; customType: typeof OM_COST; data: CostEntryData } {
+  if (entry.type !== "custom" || entry.customType !== OM_COST) return false;
+  const data = entry.data as Record<string, unknown> | undefined;
+  return (
+    !!data &&
+    typeof data.costUsd === "number" &&
+    Number.isFinite(data.costUsd as number) &&
+    (data.costUsd as number) >= 0
+  );
+}
+
+/**
+ * Sum every `om.cost` entry across the WHOLE session. Callers MUST pass all entries
+ * (`getEntries()`), NOT a single branch (`getBranch()`).
+ */
+export function sumSessionCost(allEntries: Entry[]): { costUsd: number; runs: number } {
+  let costUsd = 0;
+  let runs = 0;
+  for (const entry of allEntries) {
+    if (isCostEntry(entry)) {
+      costUsd += (entry.data as CostEntryData).costUsd;
+      runs += 1;
+    }
+  }
+  return { costUsd, runs };
+}
+
+export function buildObservationsRecordedData(
+  observations: Observation[],
+  coversUpToId: string,
+): ObservationsRecordedEntryData | undefined {
+  if (observations.length === 0 || !isNonEmptyString(coversUpToId)) return undefined;
+  return { observations, coversUpToId };
+}
+
+export function buildObservationsDroppedData(
+  observationTimestamps: string[],
+  coversUpToId: string,
+): ObservationsDroppedEntryData | undefined {
+  if (observationTimestamps.length === 0 || !isNonEmptyString(coversUpToId)) return undefined;
+  return { observationTimestamps, coversUpToId };
+}

package/extensions/memory-log/memory/index-render.ts

@@ -0,0 +1,45 @@
+import type { Topic } from "./paths.ts";
+
+function summaryOf(topic: Topic): string {
+  const s = (topic.summary ?? "").trim();
+  return s.length > 0 ? s : "(no summary)";
+}
+
+function titleOf(topic: Topic): string {
+  const t = (topic.title ?? "").trim();
+  return t.length > 0 ? t : topic.filename;
+}
+
+/** The on-disk INDEX.md content. Orchestrator-owned; the consolidator never writes it. */
+export function renderIndexFile(topics: Topic[]): string {
+  const parts: string[] = ["# Memory index", ""];
+  if (topics.length === 0) {
+    parts.push("_No topics yet._");
+    return `${parts.join("\n")}\n`;
+  }
+  parts.push("Durable memory topics for this project. Read a file for its full current state.", "");
+  for (const topic of topics) {
+    const updated = topic.updated ? ` · updated ${topic.updated}` : "";
+    parts.push(`## ${titleOf(topic)}`);
+    parts.push(`- \`${topic.path}\`${updated}`);
+    parts.push(`- ${summaryOf(topic)}`);
+    parts.push("");
+  }
+  return `${parts.join("\n").trimEnd()}\n`;
+}
+
+/**
+ * The compaction injection block's memory-map section. Returns undefined when there are no topics.
+ */
+export function renderMemoryMap(topics: Topic[]): string | undefined {
+  if (topics.length === 0) return undefined;
+  const lines: string[] = [
+    "## Memory map",
+    "Durable long-term notes live in `.memory/`. Read a file when a topic below looks relevant; these summaries are intentionally terse.",
+  ];
+  for (const topic of topics) {
+    const updated = topic.updated ? ` (updated ${topic.updated})` : "";
+    lines.push(`- \`${topic.path}\` — ${summaryOf(topic)}${updated}`);
+  }
+  return lines.join("\n");
+}

package/extensions/memory-log/memory/paths.ts

@@ -0,0 +1,120 @@
+import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, writeFileSync } from "node:fs";
+import { dirname, join, relative, resolve } from "node:path";
+
+export const INDEX_FILENAME = "INDEX.md";
+export const JOURNEY_FILENAME = "JOURNEY.md";
+
+/** The project-level `.memory/` base. Per-session roots live one level below it. */
+export function memoryBaseDir(cwd: string): string {
+  return join(cwd, ".memory");
+}
+
+/**
+ * The per-session memory root: `.memory/<sessionId>/`.
+ */
+export function sessionMemoryRoot(cwd: string, sessionId: string): string {
+  return join(memoryBaseDir(cwd), sessionId);
+}
+
+export function indexPath(root: string): string {
+  return join(root, INDEX_FILENAME);
+}
+
+export function journeyPath(root: string): string {
+  return join(root, JOURNEY_FILENAME);
+}
+
+/** Read `.memory/JOURNEY.md` body, trimmed. Returns undefined when missing or effectively empty. */
+export function readJourney(root: string): string | undefined {
+  const path = journeyPath(root);
+  if (!existsSync(path)) return undefined;
+  try {
+    const body = readFileSync(path, "utf-8").trim();
+    return body.length > 0 ? body : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+/** Atomic write (temp + rename). Creates parent dirs as needed. */
+export function atomicWrite(path: string, content: string): void {
+  mkdirSync(dirname(path), { recursive: true });
+  const tmp = `${path}.tmp-${process.pid}-${Date.now()}`;
+  writeFileSync(tmp, content, "utf-8");
+  renameSync(tmp, path);
+}
+
+/**
+ * Resolve a (possibly relative) path and confirm it stays inside `.memory/`.
+ */
+export function resolveWithinMemory(root: string, requestedPath: string): string | undefined {
+  const base = resolve(root);
+  const abs = resolve(base, requestedPath);
+  const rel = relative(base, abs);
+  if (rel === "" || rel === ".") return abs;
+  if (rel.startsWith("..") || resolve(base, rel) !== abs) return undefined;
+  return abs;
+}
+
+export type TopicFrontMatter = {
+  id?: string;
+  title?: string;
+  summary?: string;
+  updated?: string;
+};
+
+export type Topic = TopicFrontMatter & {
+  /** Path relative to the project root, e.g. ".memory/<sessionId>/auth.md". */
+  path: string;
+  /** Bare filename, e.g. "auth.md". */
+  filename: string;
+};
+
+const FRONT_MATTER_RE = /^---\n([\s\S]*?)\n---\n?/;
+
+/**
+ * Parse leading YAML-ish front-matter.
+ */
+export function parseFrontMatter(content: string): { front: TopicFrontMatter; body: string } {
+  const match = FRONT_MATTER_RE.exec(content);
+  if (!match) return { front: {}, body: content };
+  const front: TopicFrontMatter = {};
+  for (const line of match[1].split("\n")) {
+    const idx = line.indexOf(":");
+    if (idx < 0) continue;
+    const key = line.slice(0, idx).trim();
+    let value = line.slice(idx + 1).trim();
+    if (
+      (value.startsWith('"') && value.endsWith('"')) ||
+      (value.startsWith("'") && value.endsWith("'"))
+    ) {
+      value = value.slice(1, -1);
+    }
+    if (key === "id" || key === "title" || key === "summary" || key === "updated") {
+      front[key] = value;
+    }
+  }
+  return { front, body: content.slice(match[0].length) };
+}
+
+/**
+ * List parsed topic files under a session memory root, sorted by filename.
+ */
+export function listTopics(root: string): Topic[] {
+  if (!existsSync(root)) return [];
+  const cwd = resolve(root, "..", "..");
+  const topics: Topic[] = [];
+  for (const filename of readdirSync(root)) {
+    if (!filename.endsWith(".md") || filename === INDEX_FILENAME || filename === JOURNEY_FILENAME) continue;
+    let content: string;
+    try {
+      content = readFileSync(join(root, filename), "utf-8");
+    } catch {
+      continue;
+    }
+    const { front } = parseFrontMatter(content);
+    topics.push({ ...front, path: relative(cwd, join(root, filename)), filename });
+  }
+  topics.sort((a, b) => (a.filename < b.filename ? -1 : a.filename > b.filename ? 1 : 0));
+  return topics;
+}

package/extensions/memory-log/memory/session.ts

@@ -0,0 +1,64 @@
+import { cpSync, existsSync, readFileSync, renameSync, rmSync } from "node:fs";
+import { basename, sep } from "node:path";
+import { sessionMemoryRoot } from "./paths.ts";
+
+type SessionCtx = {
+  cwd: string;
+  sessionManager: {
+    getSessionId: () => string;
+    getHeader?: () => { id?: string; cwd?: string; parentSession?: string } | null | undefined;
+  };
+};
+
+/** Read a session file's header id (first JSONL line). Undefined on any parse/IO failure. */
+function readSessionHeaderId(file: string): string | undefined {
+  try {
+    const firstLine = readFileSync(file, "utf-8").split("\n", 1)[0] ?? "";
+    const header = JSON.parse(firstLine) as { type?: string; id?: string } | undefined;
+    return typeof header?.id === "string" ? header.id : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * Resolve the parent session's memory root for fork/clone seeding.
+ */
+function parentMemoryRoot(ctx: SessionCtx): string | undefined {
+  const parentFile = ctx.sessionManager.getHeader?.()?.parentSession;
+  if (!parentFile) return undefined;
+  const parentId = readSessionHeaderId(parentFile);
+  if (!parentId) return undefined;
+  const root = sessionMemoryRoot(ctx.cwd, parentId);
+  return existsSync(root) ? root : undefined;
+}
+
+/** True for any path inside a `.runs` directory (transient IPC; never seeded). */
+function isRunsPath(p: string): boolean {
+  return basename(p) === ".runs" || p.includes(`${sep}.runs${sep}`);
+}
+
+/**
+ * Resolve this session's `.memory/<sessionId>/` root, seeding it from the parent session on first touch.
+ */
+export function ensureSessionMemory(ctx: SessionCtx): string {
+  const sessionId = ctx.sessionManager.getSessionId();
+  const root = sessionMemoryRoot(ctx.cwd, sessionId);
+  if (existsSync(root)) return root;
+
+  const parent = parentMemoryRoot(ctx);
+  if (parent) {
+    const tmp = `${root}.seed-tmp-${process.pid}-${Date.now()}`;
+    try {
+      cpSync(parent, tmp, { recursive: true, filter: (src: string) => !isRunsPath(src) });
+      renameSync(tmp, root);
+    } catch {
+      try {
+        rmSync(tmp, { recursive: true, force: true });
+      } catch {
+        // best-effort cleanup
+      }
+    }
+  }
+  return root;
+}