mirror-mind 0.35.0 → 0.36.1

package/ts/src/conversation/backfill.ts

@@ -1,271 +0,0 @@
-// Session backfills from runtime-written JSONL (CV22.DS7.US10 slice E).
-//
-// Ports `backfill_pi_sessions` from `src/memory/cli/conversation_logger.py`:
-// it imports closed sessions the live hooks never saw, through the atomic
-// seam in `sessionImport.ts`, and is graded against
-// `ts/test/goldens/backfill.golden.json`. (The Codex importer,
-// `backfill_codex_session`, was ported beside it and retired with the
-// v0.32.1 Codex wrapper by CV21.DS4.TS5: its only caller was the wrapper's
-// session-end hook, and the plugin logs Codex turns live.)
-//
-// Pi titles through `set_provisional_title` (whitespace collapse plus
-// `title_source`/`title_status` provenance in metadata), tolerates a corrupt
-// or structurally invalid file by skipping THAT file and continuing the
-// walk, and parses every timestamp form (millisecond epoch, string, absent).
-//
-// Nothing here touches a provider or the clock directly: `now` is injected
-// through `LoggerDeps`, which is what makes "no timestamp -> now" pinnable.
-//
-// CR121 (2026-10-11): a home imports only the Pi sessions newer than itself.
-// Python imported every untracked session on the machine, which for a new
-// home meant the person's whole Pi history, copied and spent on before the
-// first prompt. The floor is explicit (`notBefore`), never defaulted, and
-// the rule fails closed: there is no value that means "everything" except
-// the epoch, which says so.
-
-import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
-import { join, relative, sep } from "node:path";
-import { type LoggerDeps, setProvisionalTitle } from "#conversation/logger.ts";
-import { type ImportedMessage, importClosedConversation } from "#conversation/sessionImport.ts";
-import type { WritableDatabase } from "#db/database.ts";
-import { expandHome } from "#util/paths.ts";
-import { comparePathComponents, sliceCodePoints } from "#util/pythonText.ts";
-
-/** Python `_DEFAULT_PI_SESSIONS_DIR`: `~/.pi/agent/sessions`. */
-export function defaultPiSessionsDir(homeDir: string): string {
-  return join(homeDir, ".pi", "agent", "sessions");
-}
-
-/**
- * Python's source-directory resolution for the Pi backfill, in order:
- * explicit argument -> `PI_SESSIONS_DIR` -> `~/.pi/agent/sessions`.
- * (Python's module-level `_PI_SESSIONS_DIR` override exists only for its own
- * test monkey-patching and has no runtime input; it is not a contract.)
- */
-export function resolvePiSessionsDir(
-  explicit: string | null | undefined,
-  env: { PI_SESSIONS_DIR?: string },
-  homeDir: string,
-): string {
-  if (explicit) return expandHome(explicit);
-  if (env.PI_SESSIONS_DIR) return expandHome(env.PI_SESSIONS_DIR);
-  return defaultPiSessionsDir(homeDir);
-}
-
-function isRecord(value: unknown): value is Record<string, unknown> {
-  return typeof value === "object" && value !== null && !Array.isArray(value);
-}
-
-/**
- * Python `dict.get(key, default)` on a value that MUST be a dict: anything
- * else raises AttributeError, which each parser turns into its own failure
- * mode. The default applies only when the key is ABSENT -- a present `null`
- * is returned as null, exactly as Python returns None, so a later `.get` on
- * it raises rather than silently reading from the default.
- */
-function getOr(container: unknown, key: string, fallback: unknown): unknown {
-  if (!isRecord(container)) throw new TypeError(`expected an object with '${key}'`);
-  return key in container ? container[key] : fallback;
-}
-
-function getFrom(container: unknown, key: string): unknown {
-  return getOr(container, key, undefined);
-}
-
-/** Python `_parse_pi_content`: a string as-is, text blocks joined by `\n`, else `""`. */
-export function parsePiContent(content: unknown): string {
-  if (typeof content === "string") return content;
-  if (Array.isArray(content)) {
-    const parts: string[] = [];
-    for (const block of content) {
-      if (!isRecord(block) || block.type !== "text") continue;
-      const text = block.text ?? "";
-      // `"\n".join(...)` raises on a non-str item; the file is skipped.
-      if (typeof text !== "string") throw new TypeError("text block is not a string");
-      parts.push(text);
-    }
-    return parts.join("\n");
-  }
-  return "";
-}
-
-/**
- * Python `datetime.fromtimestamp(ms / 1000, tz=utc).isoformat()`:
- * `+00:00` suffix (not `Z`), microseconds only when non-zero, rounded from
- * the fractional second the way CPython does (`round(frac * 1e6)`).
- */
-function pythonUtcIsoFromEpochMs(epochMs: number): string {
-  const seconds = epochMs / 1000;
-  let whole = Math.floor(seconds);
-  let micros = Math.round((seconds - whole) * 1e6);
-  if (micros >= 1_000_000) {
-    whole += 1;
-    micros -= 1_000_000;
-  }
-  const base = new Date(whole * 1000).toISOString().slice(0, 19);
-  const fraction = micros === 0 ? "" : `.${String(micros).padStart(6, "0")}`;
-  return `${base}${fraction}+00:00`;
-}
-
-/**
- * Python `_parse_pi_timestamp`: a number above 1e10 is a millisecond epoch;
- * a string is stored verbatim; anything else (absent, null, a seconds epoch,
- * a boolean -- which is an int in Python but never exceeds 1e10) is `now`.
- */
-export function parsePiTimestamp(value: unknown, now: () => string): string {
-  if (typeof value === "number" && value > 1e10) return pythonUtcIsoFromEpochMs(value);
-  if (typeof value === "string") return value;
-  return now();
-}
-
-/**
- * Parse one Pi session file into importable messages. Throws on any line
- * that is not valid JSON or not the expected shape -- Python wraps the whole
- * file loop in one `try` and skips the file on the first failure, even after
- * earlier lines parsed cleanly.
- */
-function parsePiSessionFile(path: string, now: () => string): ImportedMessage[] {
-  const messages: ImportedMessage[] = [];
-  for (const line of readFileSync(path, "utf8").split(/\r?\n|\r/)) {
-    if (!line.trim()) continue;
-    const entry: unknown = JSON.parse(line);
-    if (getFrom(entry, "type") !== "message") continue;
-    const message = getOr(entry, "message", {});
-    const role = getOr(message, "role", "");
-    if (role !== "user" && role !== "assistant") continue;
-    const content = parsePiContent(getOr(message, "content", ""));
-    if (!content.trim()) continue;
-    const createdAt = parsePiTimestamp(getFrom(message, "timestamp"), now);
-    messages.push({ role, content, createdAt });
-  }
-  return messages;
-}
-
-/**
- * Python `sorted(pi_sessions_dir.rglob("*.jsonl"))`: every `.jsonl` file
- * under the directory, ordered by path components (see
- * `comparePathComponents` for why that is not a string sort).
- */
-function listPiSessionFiles(sessionsDir: string): string[] {
-  const entries = readdirSync(sessionsDir, { recursive: true, withFileTypes: true });
-  const found: string[][] = [];
-  for (const entry of entries) {
-    if (!entry.name.endsWith(".jsonl")) continue;
-    const parents = relative(sessionsDir, entry.parentPath).split(sep).filter(Boolean);
-    found.push([...parents, entry.name]);
-  }
-  found.sort(comparePathComponents);
-  return found.map((components) => join(sessionsDir, ...components));
-}
-
-export interface PiBackfillOptions {
-  /** Already resolved through `resolvePiSessionsDir`. */
-  sessionsDir: string;
-  /**
-   * An ISO instant: a session that ended at or before it is not imported
-   * (CR121). Production passes the home's birth (`homeBornAt`); a caller
-   * that wants no floor passes the epoch and says so.
-   */
-  notBefore: string;
-}
-
-/**
- * When this home was born: the first `applied_at` in the `_migrations`
- * ledger. A fresh database stamps its whole ledger at its first open, and a
- * database from any earlier era carries the ledger from its own first open,
- * so the minimum is the birth of every home there is -- with no new table,
- * column, or marker. Read once per maintenance run.
- *
- * The ledger means "which migrations ran, when"; reading its minimum as a
- * birth is an inference, accepted because every plausible edit to the ledger
- * moves the minimum forward and imports less, never more. `migrations.ts`
- * names this consumer at the ledger's definition. A second consumer of
- * "birth" is the trigger to model it as a row of its own.
- *
- * Fails closed: an empty ledger is an error, not a home born at the epoch.
- */
-export function homeBornAt(db: WritableDatabase): string {
-  const row = db.prepare("SELECT MIN(applied_at) AS born FROM _migrations").get() as
-    | { born: string | null }
-    | undefined;
-  if (typeof row?.born !== "string" || !row.born) {
-    throw new Error("the _migrations ledger is empty: this home's birth is unknown");
-  }
-  return row.born;
-}
-
-/** The instant of an ISO timestamp, or NaN for anything `Date.parse` rejects. */
-function instant(iso: string): number {
-  return Date.parse(iso);
-}
-
-/**
- * Python `backfill_pi_sessions`: import every untracked Pi session with at
- * least two messages as a closed `pi` conversation, titled provisionally from
- * the first user line. Returns the number imported.
- *
- * The session id is the session file's path, exactly as Python's
- * `str(session_file)` -- which is also what the live Pi extension binds, so a
- * session the hooks already track is skipped before its file is even read.
- *
- * CR121: a session is Mirror's only if it ended after `notBefore`. Two
- * checks, in cost order. A file whose mtime is at or before the floor is
- * skipped unread: its last message was written at or before its last write,
- * so it cannot be newer. (This is what keeps a skipped session, which is
- * never tracked, from being parsed again at every start for the rest of the
- * home's life.) A file written after the floor is parsed and judged by its
- * last message, the same instant it stores as `endedAt` -- a copy or a sync
- * can touch a file long after its conversation ended. A session whose
- * messages carry no parseable instant dates to `now` under the existing rule
- * and is imported: undated is newer, not older.
- */
-export function backfillPiSessions(
-  db: WritableDatabase,
-  options: PiBackfillOptions,
-  deps: LoggerDeps,
-): number {
-  const floor = instant(options.notBefore);
-  if (Number.isNaN(floor)) {
-    throw new Error(`notBefore is not an instant: ${JSON.stringify(options.notBefore)}`);
-  }
-  if (!existsSync(options.sessionsDir)) return 0;
-
-  let count = 0;
-  for (const sessionFile of listPiSessionFiles(options.sessionsDir)) {
-    const sessionId = sessionFile;
-    if (db.prepare("SELECT 1 FROM runtime_sessions WHERE session_id = ?").get(sessionId)) {
-      continue;
-    }
-    if (statSync(sessionFile).mtimeMs <= floor) continue;
-    let messages: ImportedMessage[];
-    try {
-      messages = parsePiSessionFile(sessionFile, deps.nowIso);
-    } catch {
-      continue;
-    }
-    if (messages.length < 2) continue;
-    const endedAt = instant((messages.at(-1) as ImportedMessage).createdAt);
-    if (endedAt <= floor) continue;
-
-    const conversationId = importClosedConversation(
-      db,
-      sessionId,
-      { interface: "pi", messages, endedAt: (messages.at(-1) as ImportedMessage).createdAt },
-      deps,
-    );
-    if (conversationId === null) {
-      // A concurrent writer (a live hook) bound this session while the import
-      // waited for the write lock -- their binding wins.
-      continue;
-    }
-    const firstUser = messages.find((message) => message.role === "user");
-    if (firstUser) {
-      // Python: `content.strip().split("\n")[0][:60]` -- cut BEFORE the
-      // provisional-title whitespace collapse, by code point.
-      const title = sliceCodePoints(firstUser.content.trim().split("\n")[0] ?? "", 60);
-      setProvisionalTitle(db, conversationId, title);
-    }
-    count += 1;
-  }
-  return count;
-}

package/ts/src/conversation/sessionImport.ts

@@ -1,90 +0,0 @@
-// Atomic closed-session import (CV22.DS7.US10 slice E).
-//
-// Ports `RuntimeSessionService.import_closed_conversation` from
-// `src/memory/services/runtime_session.py`, the seam both backfills write
-// through. It is the sibling of `getOrCreateSessionConversation` in
-// `logger.ts` and shares its transaction discipline for the same reason:
-// a backfill imports against the SAME `runtime_sessions` key a live hook
-// binds (Pi's live session id is the session file path), and the two can
-// interleave.
-//
-// Resolved decision 4B (plan.md): the binding check runs INSIDE the
-// `BEGIN IMMEDIATE` transaction, so a binding a concurrent hook committed
-// while this import waited for the write lock is seen and honored -- the
-// import returns null instead of clobbering a live session with
-// `active=0`/`closed_at`. The same atomicity removes the crash-window
-// duplicate-import path: either the conversation, every message, and the
-// binding all land, or nothing does.
-//
-// Titles are deliberately NOT set here. The two callers title differently
-// (Pi through `setProvisionalTitle`, Codex through a bare column write) and
-// Python applies both after commit; reproducing that asymmetry is the
-// callers' job.
-
-import type { LoggerDeps } from "#conversation/logger.ts";
-import { type WritableDatabase, withTransaction } from "#db/database.ts";
-
-export interface ImportedMessage {
-  role: string;
-  content: string;
-  createdAt: string;
-}
-
-export interface ClosedSessionImport {
-  interface: string;
-  messages: readonly ImportedMessage[];
-  endedAt: string;
-}
-
-/**
- * Import a closed session transcript as one conversation bound to
- * `sessionId`. Returns the new conversation id, or null when the session is
- * already tracked -- including by a writer that won the binding while this
- * call waited for the lock.
- */
-export function importClosedConversation(
-  db: WritableDatabase,
-  sessionId: string,
-  session: ClosedSessionImport,
-  deps: LoggerDeps,
-): string | null {
-  return withTransaction(db, () => {
-    const tracked = db
-      .prepare("SELECT 1 FROM runtime_sessions WHERE session_id = ?")
-      .get(sessionId);
-    if (tracked) return null;
-
-    // Python's Conversation(...) draws its id before its started_at; keep the
-    // same generator order so injected sequences line up with the oracle.
-    const conversationId = deps.newId();
-    const startedAt = deps.nowIso();
-    db.prepare(
-      `INSERT INTO conversations
-         (id, title, started_at, ended_at, interface, persona, journey, summary, tags, metadata)
-       VALUES (?, NULL, ?, ?, ?, NULL, NULL, NULL, NULL, NULL)`,
-    ).run(conversationId, startedAt, session.endedAt, session.interface);
-
-    const insertMessage = db.prepare(
-      `INSERT INTO messages (id, conversation_id, role, content, created_at, token_count, metadata)
-       VALUES (?, ?, ?, ?, ?, NULL, NULL)`,
-    );
-    for (const message of session.messages) {
-      insertMessage.run(
-        deps.newId(),
-        conversationId,
-        message.role,
-        message.content,
-        message.createdAt,
-      );
-    }
-
-    db.prepare(
-      `INSERT INTO runtime_sessions
-         (session_id, conversation_id, interface, mirror_active, persona, journey,
-          hook_injected, active, started_at, updated_at, closed_at, metadata)
-       VALUES (?, ?, ?, 0, NULL, NULL, 0, 0, ?, ?, ?, NULL)`,
-    ).run(sessionId, conversationId, session.interface, startedAt, startedAt, session.endedAt);
-
-    return conversationId;
-  });
-}