mirror-mind 0.0.1 → 0.33.0

package/ts/src/conversation/extraction.ts

@@ -0,0 +1,333 @@
+import { metadataDict } from "#conversation/closeTail.ts";
+import type { WritableDatabase } from "#db/database.ts";
+import { embeddingToBytes } from "#db/decode.ts";
+import { optionalString, requireString } from "#db/rowDecode.ts";
+import {
+  curateAgainstExisting,
+  type ExistingMemoryForCuration,
+  type ExtractedMemory,
+  type ExtractionMessage,
+  type ExtractionStatus,
+  extractMemoriesWithStatus,
+  extractTasks,
+  formatTranscript,
+  naiveSummary,
+  type OnLlmCall,
+} from "#extraction/conversation.ts";
+import { createMemoryRow } from "#memory/memoryWrite.ts";
+import { chatLedgerHook, embeddingLedgerHook } from "#observability/ledgerHooks.ts";
+import { resolveExtractionModel } from "#providers/config.ts";
+import {
+  addEmbeddingProvenance,
+  type EmbeddingAttemptInfo,
+  type EmbeddingProvider,
+  generateEmbeddingSafely,
+} from "#providers/embedding.ts";
+import type { LlmProvider } from "#providers/llm.ts";
+import { newId, nowIso, pythonJsonDumps, pythonJsonDumpsEnsureAscii } from "#util/pyGenerators.ts";
+
+export interface ConversationExtractionOptions {
+  llm: LlmProvider;
+  embeddings: EmbeddingProvider;
+  now?: () => string;
+  id?: () => string;
+  summarize?: boolean;
+  twoPass?: boolean;
+  curationExisting?: readonly ExistingMemoryForCuration[];
+  /**
+   * Model-call ledger (CV9.E2.S13/S14). Python's `_make_logger(role,
+   * conversation_id)` writes one `llm_calls` row per successful call in
+   * `extraction`, `curation`, `task_extraction`, and `summary`; the embedding
+   * rows were already logged. Off by default so the DS5 unit surface stays
+   * ledger-free; the front door turns it on.
+   */
+  ledger?: boolean;
+}
+
+export interface ConversationExtractionResult {
+  memoryIds: string[];
+  taskIds: string[];
+  extracted: number;
+}
+
+interface ConversationRow {
+  id: string;
+  persona: string | null;
+  journey: string | null;
+  metadata: string | null;
+}
+
+export async function runConversationExtraction(
+  db: WritableDatabase,
+  conversationId: string,
+  options: ConversationExtractionOptions,
+): Promise<ConversationExtractionResult> {
+  const conv = getConversation(db, conversationId);
+  const messages = getMessages(db, conversationId);
+  if (!conv?.journey || messages.length < 4) {
+    return { memoryIds: [], taskIds: [], extracted: 0 };
+  }
+
+  const now = options.now ?? nowIso;
+  const id = options.id ?? newId;
+  const userName = resolveUserName(db);
+  const ledger = options.ledger ? llmLedger(db, conversationId, now) : undefined;
+
+  // A failure anywhere below propagates unmodified. Python records it -- the
+  // `llm_failed` status, the attempt counter, and quarantine at the max --
+  // OUTSIDE this orchestration, in `_run_extraction`; the port keeps that
+  // accounting in `extractionRun.ts` for the same reason: the stored metadata
+  // key order depends on it being written once, there, never partly here.
+  const outcome = await extractMemoriesWithStatus(options.llm, messages, {
+    persona: conv.persona,
+    journey: conv.journey,
+    userName,
+    onLlmCall: ledger?.("extraction"),
+  });
+  let extractedMemories: ExtractedMemory[] = outcome.memories;
+  const extractionStatus: ExtractionStatus = outcome.status;
+
+  if (options.twoPass && extractedMemories.length > 0) {
+    extractedMemories = await curateAgainstExisting(
+      options.llm,
+      extractedMemories,
+      options.curationExisting ?? [],
+      { onLlmCall: ledger?.("curation") },
+    );
+  }
+
+  const taskIds = await persistExtractedTasks(db, options.llm, messages, {
+    journey: conv.journey,
+    userName,
+    now,
+    id,
+    onLlmCall: ledger?.("task_extraction"),
+  });
+
+  const summaryText =
+    options.summarize === false
+      ? naiveSummary(messages)
+      : await replayedSummary(options.llm, messages, userName, ledger?.("summary"));
+  const finalSummary = summaryText || naiveSummary(messages);
+
+  // Stage EVERY network embedding first -- the summary and each memory -- so a
+  // failure leaves nothing persisted. The retry then starts clean and does not
+  // duplicate rows or re-spend on embeddings (AI-03 / CV9.E2.S9).
+  //
+  // This mirrors Python's `_extract_and_persist` deliberately. The port
+  // originally interleaved embed -> insert per memory, which is invisible
+  // under replay -- replayed embeddings never fail -- and becomes a partial
+  // write the moment a live provider is on the other end: memories 1..n-1
+  // persisted, `extracted` unset, and a retry that inserts them a second time
+  // with a second paid embedding each (CV22.DS8.US2 plan review).
+  const summaryEmbeddingBytes = finalSummary
+    ? embeddingToBytes(
+        // Logged to the ledger (AI-09/D-003), but NOT provenance-stamped:
+        // conversation_embeddings has no metadata column, matching Python's own
+        // narrower scope (add_embedding_provenance is called from
+        // add_memory/add_attachment only, never from the summary path).
+        await generateEmbeddingSafely(options.embeddings, finalSummary, {
+          onAttempt: logEmbeddingAttempt(db, conversationId, now),
+        }),
+      )
+    : null;
+
+  const stagedMemories: { memory: (typeof extractedMemories)[number]; embedding: Uint8Array }[] =
+    [];
+  for (const memory of extractedMemories) {
+    const embeddingText = `${memory.title}. ${memory.content}${memory.context ? ` Context: ${memory.context}` : ""}`;
+    const embedding = await generateEmbeddingSafely(options.embeddings, embeddingText, {
+      onAttempt: logEmbeddingAttempt(db, conversationId, now),
+    });
+    stagedMemories.push({ memory, embedding: embeddingToBytes(embedding) });
+  }
+
+  // All embeddings succeeded -- only local writes remain.
+  if (summaryEmbeddingBytes !== null && finalSummary) {
+    db.prepare(
+      `INSERT INTO conversation_embeddings (conversation_id, summary_embedding) VALUES (?, ?) ` +
+        `ON CONFLICT(conversation_id) DO UPDATE SET summary_embedding = excluded.summary_embedding`,
+    ).run(conversationId, summaryEmbeddingBytes);
+    db.prepare("UPDATE conversations SET summary = ? WHERE id = ?").run(
+      finalSummary.slice(0, 1000),
+      conversationId,
+    );
+  }
+
+  const memoryIds: string[] = [];
+  for (const { memory, embedding } of stagedMemories) {
+    const memoryId = id();
+    insertMemory(db, memoryId, conversationId, memory, embedding, now());
+    memoryIds.push(memoryId);
+  }
+
+  const metadata = metadataDict(conv.metadata);
+  metadata.extracted = true;
+  metadata.extraction_status = extractionStatus.status;
+  if (
+    extractionStatus.dropped &&
+    Object.values(extractionStatus.dropped).some((count) => count > 0)
+  ) {
+    metadata.extraction_dropped = extractionStatus.dropped;
+  }
+  db.prepare("UPDATE conversations SET metadata = ? WHERE id = ?").run(
+    pythonJsonDumps(metadata),
+    conversationId,
+  );
+
+  return { memoryIds, taskIds, extracted: memoryIds.length };
+}
+
+async function persistExtractedTasks(
+  db: WritableDatabase,
+  llm: LlmProvider,
+  messages: readonly ExtractionMessage[],
+  options: {
+    journey: string;
+    userName: string;
+    now: () => string;
+    id: () => string;
+    onLlmCall?: OnLlmCall;
+  },
+): Promise<string[]> {
+  try {
+    const tasks = await extractTasks(llm, messages, {
+      journey: options.journey,
+      userName: options.userName,
+      onLlmCall: options.onLlmCall,
+    });
+    const inserted: string[] = [];
+    for (const task of tasks) {
+      const existing = db
+        .prepare("SELECT id FROM tasks WHERE title LIKE ? AND journey = ? ORDER BY created_at DESC")
+        .all(`%${task.title}%`, task.journey ?? options.journey);
+      if (existing.length > 0) continue;
+      const taskId = options.id();
+      const now = options.now();
+      db.prepare(
+        `INSERT INTO tasks ` +
+          `(id, journey, title, status, due_date, scheduled_at, time_hint, stage, context, source, created_at, updated_at, completed_at, metadata) ` +
+          `VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      ).run(
+        taskId,
+        task.journey,
+        task.title,
+        "todo",
+        task.due_date,
+        null,
+        null,
+        task.stage,
+        task.context,
+        "conversation",
+        now,
+        now,
+        null,
+        null,
+      );
+      inserted.push(taskId);
+    }
+    return inserted;
+  } catch {
+    return [];
+  }
+}
+
+async function replayedSummary(
+  llm: LlmProvider,
+  messages: readonly ExtractionMessage[],
+  userName: string,
+  onLlmCall?: OnLlmCall,
+): Promise<string> {
+  try {
+    const prompt = formatTranscript(messages, userName);
+    const response = await llm.complete({
+      role: "summary",
+      prompt,
+      model: resolveExtractionModel(),
+      temperature: 0.3,
+    });
+    onLlmCall?.(response, prompt);
+    return response.content.trim();
+  } catch {
+    return naiveSummary(messages);
+  }
+}
+
+function getConversation(db: WritableDatabase, conversationId: string): ConversationRow | null {
+  const row = db
+    .prepare("SELECT id, persona, journey, metadata FROM conversations WHERE id = ?")
+    .get(conversationId);
+  if (!row) return null;
+  return {
+    id: requireString(row, "id"),
+    persona: optionalString(row, "persona"),
+    journey: optionalString(row, "journey"),
+    metadata: optionalString(row, "metadata"),
+  };
+}
+
+function getMessages(db: WritableDatabase, conversationId: string): ExtractionMessage[] {
+  return db
+    .prepare("SELECT role, content FROM messages WHERE conversation_id = ? ORDER BY created_at ASC")
+    .all(conversationId)
+    .map((row) => ({ role: requireString(row, "role"), content: requireString(row, "content") }));
+}
+
+function resolveUserName(db: WritableDatabase): string {
+  const row = db
+    .prepare("SELECT content FROM identity WHERE layer = ? AND key = ? LIMIT 1")
+    .get("user", "identity");
+  const content = row ? optionalString(row, "content") : null;
+  const match = content?.match(
+    /(?:You are talking to|Você está falando com) ([A-Z][a-zA-Záéíóúãõ]+)/,
+  );
+  return match?.[1] ?? "User";
+}
+
+/**
+ * The close tail's two ledger hooks. Roles match Python's `build_llm_logger`
+ * roles so the `llm_calls` ledger agrees across engines; the orchestration's
+ * clock stamps every row.
+ */
+function llmLedger(
+  db: WritableDatabase,
+  conversationId: string,
+  now: () => string,
+): (role: string) => OnLlmCall {
+  return (role) => chatLedgerHook(db, role, { conversationId, now });
+}
+
+function logEmbeddingAttempt(
+  db: WritableDatabase,
+  conversationId: string,
+  now: () => string,
+): (info: EmbeddingAttemptInfo) => void {
+  return embeddingLedgerHook(db, { conversationId, now });
+}
+
+function insertMemory(
+  db: WritableDatabase,
+  id: string,
+  conversationId: string,
+  memory: ExtractedMemory,
+  embedding: Uint8Array,
+  createdAt: string,
+): void {
+  createMemoryRow(db, {
+    id,
+    conversationId,
+    memoryType: memory.memory_type,
+    layer: memory.layer,
+    title: memory.title,
+    content: memory.content,
+    context: memory.context,
+    journey: memory.journey,
+    persona: memory.persona,
+    // Memory tags go through `services/memory.py`'s plain `json.dumps(tags)`,
+    // which keeps `ensure_ascii=True` -- unlike conversation tags above.
+    tags: memory.tags.length > 0 ? pythonJsonDumpsEnsureAscii(memory.tags) : null,
+    createdAt,
+    embedding,
+    metadata: addEmbeddingProvenance(null),
+  });
+}

package/ts/src/conversation/extractionDriver.ts

@@ -0,0 +1,130 @@
+// CV22.DS7.US5 slice C — the budgeted extraction driver.
+//
+// Ports `extract_pending` and the maintenance counters from
+// `src/memory/cli/conversation_logger.py`, plus the eligibility query in
+// `ConversationStore.get_unextracted_conversations`.
+//
+// The extraction call is injected rather than imported: it is the DS5
+// orchestration (`runConversationExtraction`) behind the replay transport, so
+// this module — the part that decides *what* gets extracted and *how failure
+// behaves* — stays deterministic and provider-free. Two released properties
+// live here and must not regress:
+//
+//   * AI-05 (CV9.E2.S26): a backlog can never turn one session start into an
+//     unbounded spend loop. At most `limit` conversations per call,
+//     oldest-ended first, remainder visibly carried over.
+//   * CV9.E2.S7: one poison-pill conversation is recorded and skipped, never
+//     allowed to crash the batch or block the queue behind it.
+
+import type { WritableDatabase } from "#db/database.ts";
+
+/** Python's `MEMORY_MAINTENANCE_MAX_EXTRACTIONS` default. */
+export const DEFAULT_MAINTENANCE_MAX_EXTRACTIONS = 10;
+
+/**
+ * Eligibility, byte-for-byte from `ConversationStore._UNEXTRACTED_WHERE`.
+ * `json_extract` returns NULL for malformed metadata, and `IS NOT 1` keeps
+ * such a row eligible rather than silently dropping it from the queue.
+ */
+const UNEXTRACTED_WHERE = `
+  WHERE c.ended_at IS NOT NULL
+    AND c.journey IS NOT NULL
+    AND (c.metadata IS NULL OR json_extract(c.metadata, '$.extracted') IS NOT 1)
+    AND (c.metadata IS NULL
+         OR json_extract(c.metadata, '$.extraction_quarantined') IS NOT 1)
+    AND (SELECT COUNT(*) FROM messages m WHERE m.conversation_id = c.id) >= 4
+`;
+
+export interface UnextractedConversation {
+  id: string;
+  journey: string | null;
+  persona: string | null;
+  metadata: string | null;
+}
+
+/** Ended, eligible conversations, oldest-ended first (FIFO drain). */
+export function selectUnextractedConversations(
+  db: WritableDatabase,
+  limit: number | null,
+): UnextractedConversation[] {
+  const base = `SELECT c.id, c.journey, c.persona, c.metadata FROM conversations c${UNEXTRACTED_WHERE}ORDER BY c.ended_at ASC`;
+  const rows = limit === null ? db.prepare(base).all() : db.prepare(`${base} LIMIT ?`).all(limit);
+  return rows.map((row) => ({
+    id: String(row.id),
+    journey: typeof row.journey === "string" ? row.journey : null,
+    persona: typeof row.persona === "string" ? row.persona : null,
+    metadata: typeof row.metadata === "string" ? row.metadata : null,
+  }));
+}
+
+export interface ExtractPendingOptions {
+  /** Defaults to the released maintenance budget. */
+  limit?: number;
+  /**
+   * The DS5 orchestration behind replay, wrapped in Python's `_run_extraction`
+   * accounting (`extractionRun.ts`); a throw or a rejection marks this one
+   * failed. Async because the orchestration awaits providers -- a synchronous
+   * signature here would let a rejected extraction escape the isolation below
+   * and count as a success.
+   */
+  runExtraction: (db: WritableDatabase, conversationId: string) => Promise<void> | void;
+}
+
+/**
+ * Extract memories from ended conversations not yet processed, one at a time
+ * in queue order.
+ *
+ * Returns the number of conversations successfully extracted. Each is isolated:
+ * a failure is skipped so it cannot crash the batch or block the queue behind
+ * it, matching Python's bare `except Exception: continue`.
+ */
+export async function extractPending(
+  db: WritableDatabase,
+  options: ExtractPendingOptions,
+): Promise<number> {
+  const limit = options.limit ?? DEFAULT_MAINTENANCE_MAX_EXTRACTIONS;
+  if (limit <= 0) return 0;
+
+  let extracted = 0;
+  for (const conversation of selectUnextractedConversations(db, limit)) {
+    try {
+      await options.runExtraction(db, conversation.id);
+      extracted += 1;
+    } catch {}
+  }
+  return extracted;
+}
+
+function countScalar(db: WritableDatabase, sql: string, ...params: string[]): number {
+  const row = db.prepare(sql).get(...params);
+  const value = row?.n;
+  if (typeof value === "bigint") return Number(value);
+  return typeof value === "number" ? value : 0;
+}
+
+/** Conversations quarantined after repeated extraction failure (CV9.E2.S7). */
+export function countQuarantinedConversations(db: WritableDatabase): number {
+  return countScalar(
+    db,
+    `SELECT COUNT(*) AS n FROM conversations c
+     WHERE json_extract(c.metadata, '$.extraction_quarantined') IS 1`,
+  );
+}
+
+/** Conversations whose recorded extraction status matches (AI-10). */
+export function countConversationsWithExtractionStatus(
+  db: WritableDatabase,
+  status: string,
+): number {
+  return countScalar(
+    db,
+    `SELECT COUNT(*) AS n FROM conversations c
+     WHERE json_extract(c.metadata, '$.extraction_status') = ?`,
+    status,
+  );
+}
+
+/** Conversations still eligible after a capped run (CV9.E2.S26, AI-05). */
+export function countCarriedOverConversations(db: WritableDatabase): number {
+  return countScalar(db, `SELECT COUNT(*) AS n FROM conversations c${UNEXTRACTED_WHERE}`);
+}

package/ts/src/conversation/extractionRun.ts

@@ -0,0 +1,70 @@
+// Extraction failure accounting (CV22.DS7.US10 slice F).
+//
+// Ports `ConversationService._run_extraction` -- the wrapper around the DS5
+// orchestration that Python's `extract_conversation` and `end_conversation`
+// both go through -- and `_record_failed_extraction_attempt` (CV9.E2.S7).
+//
+// The orchestration itself (`runConversationExtraction`) is DS5's; what it
+// lacked was the accounting Python keeps OUTSIDE it: every failed attempt
+// increments `extraction_attempts`, stamps `extraction_status=llm_failed`,
+// and quarantines the conversation at `EXTRACTION_MAX_ATTEMPTS` so a
+// poison-pill transcript stops being retried at every session start. Without
+// this wrapper a conversation whose extraction failed under TypeScript would
+// be retried forever and the maintenance report's quarantine tail would never
+// appear -- a divergence `session-maintenance` parity cannot tolerate.
+//
+// The metadata key ORDER is part of the stored bytes: Python appends
+// `extraction_attempts` before `extraction_status` when neither exists yet, so
+// the write happens here, once, rather than partly inside the orchestration.
+
+import { metadataDict } from "#conversation/closeTail.ts";
+import type { WritableDatabase } from "#db/database.ts";
+import { pythonJsonDumps } from "#util/pyGenerators.ts";
+
+/** Python's `MEMORY_EXTRACTION_MAX_ATTEMPTS` default. */
+export const DEFAULT_EXTRACTION_MAX_ATTEMPTS = 3;
+
+/**
+ * Python `_record_failed_extraction_attempt`: bump the counter in the
+ * conversation's metadata JSON, mark the failure, quarantine at the max.
+ * Returns the new attempt count (0 when the conversation does not exist).
+ */
+export function recordFailedExtractionAttempt(
+  db: WritableDatabase,
+  conversationId: string,
+  maxAttempts: number = DEFAULT_EXTRACTION_MAX_ATTEMPTS,
+): number {
+  const row = db.prepare("SELECT metadata FROM conversations WHERE id = ?").get(conversationId);
+  if (!row) return 0;
+  const metadata = metadataDict(typeof row.metadata === "string" ? row.metadata : null);
+  // Python: `int(meta.get("extraction_attempts", 0)) + 1`.
+  const attempts = Math.trunc(Number(metadata.extraction_attempts ?? 0)) + 1;
+  metadata.extraction_attempts = attempts;
+  metadata.extraction_status = "llm_failed";
+  if (attempts >= maxAttempts) metadata.extraction_quarantined = true;
+  db.prepare("UPDATE conversations SET metadata = ? WHERE id = ?").run(
+    pythonJsonDumps(metadata),
+    conversationId,
+  );
+  return attempts;
+}
+
+/**
+ * Python `_run_extraction`: run the orchestration; on ANY failure record the
+ * attempt and re-raise, so the caller's per-conversation loop can isolate it.
+ * The eligibility gate (journey, four messages) lives in the orchestration and
+ * returns before touching the row, exactly as Python's does.
+ */
+export async function runExtractionWithAccounting(
+  db: WritableDatabase,
+  conversationId: string,
+  run: (db: WritableDatabase, conversationId: string) => Promise<unknown> | unknown,
+  maxAttempts: number = DEFAULT_EXTRACTION_MAX_ATTEMPTS,
+): Promise<void> {
+  try {
+    await run(db, conversationId);
+  } catch (error) {
+    recordFailedExtractionAttempt(db, conversationId, maxAttempts);
+    throw error;
+  }
+}

package/ts/src/conversation/hookPayload.ts

@@ -0,0 +1,51 @@
+// The fields a Claude-schema hook payload carries, in both spellings
+// (CV21.DS2.US8).
+//
+// Three runtimes hand the shared hooks a payload: Claude Code and Codex in
+// snake_case, Grok in camelCase with a snake_case alias on some fields but
+// not all (DS12.TS4 L8). The turn is correlated under three names: Claude
+// Code's `prompt_id`, Codex's `turn_id`, Grok's `promptId`. One reader, so
+// the logger and the hook entries cannot disagree about where a field lives.
+//
+// Every reader returns "" for a missing or non-string field, and the parser
+// returns null rather than throwing: a hook that threw on a payload it did
+// not recognize would break the user's turn (the `extract_prompt` posture).
+
+export type HookPayloadObject = Record<string, unknown>;
+
+/** Parse a hook payload. Null for anything but a JSON object. */
+export function parseHookPayloadObject(raw: string): HookPayloadObject | null {
+  try {
+    const value: unknown = JSON.parse(raw);
+    if (typeof value !== "object" || value === null || Array.isArray(value)) return null;
+    return value as HookPayloadObject;
+  } catch {
+    return null;
+  }
+}
+
+/** The first of the named fields that holds a string, else "". */
+export function payloadText(payload: HookPayloadObject, ...names: string[]): string {
+  for (const name of names) {
+    const value = payload[name];
+    if (typeof value === "string") return value;
+  }
+  return "";
+}
+
+export const sessionIdOf = (payload: HookPayloadObject): string =>
+  payloadText(payload, "session_id", "sessionId");
+
+export const promptOf = (payload: HookPayloadObject): string => payloadText(payload, "prompt");
+
+/** The runtime's correlation id for the turn, whatever it calls it. */
+export const turnIdOf = (payload: HookPayloadObject): string =>
+  payloadText(payload, "prompt_id", "turn_id", "promptId");
+
+/** The turn's final answer, as the Stop hook receives it. */
+export const lastAssistantMessageOf = (payload: HookPayloadObject): string =>
+  payloadText(payload, "last_assistant_message", "lastAssistantMessage");
+
+/** Set only inside a subagent (Grok names it; Claude Code and Codex fire SubagentStop instead). */
+export const subagentTypeOf = (payload: HookPayloadObject): string =>
+  payloadText(payload, "subagentType", "subagent_type");