@remnic/core 9.3.712 → 9.3.714

package/src/chat/chat-session.ts

@@ -0,0 +1,296 @@
+/**
+ * Chat session state — transcript + pending-plan pointer persisted under
+ * `<memoryDir>/state/chat/<chatSessionId>.jsonl` (issue #1583).
+ *
+ * Atomic appends (rule 54).  Chat transcripts are excluded from memory
+ * extraction — the work-task exclusion tag is stamped on creation so talking
+ * *about* memories does not create memories of the conversation itself.
+ */
+
+import { mkdir, readFile, appendFile, readdir, stat, unlink } from "node:fs/promises";
+import { join } from "node:path";
+import { randomUUID } from "node:crypto";
+
+import type { ChatSessionState, ChatTranscriptEntry } from "./chat-types.js";
+
+const CHAT_DIR_NAME = "state/chat";
+
+/**
+ * Resolve the chat-session directory for a given memory dir.
+ */
+export function chatSessionDir(memoryDir: string): string {
+  return join(memoryDir, CHAT_DIR_NAME);
+}
+
+/**
+ * Resolve the transcript file path for a chat session id.
+ */
+/**
+ * Validate a chat session id is safe for filesystem use (no path traversal).
+ * Accepts UUIDs and alphanumeric+dash identifiers only (P1 — path safety).
+ */
+export function isSafeChatSessionId(chatSessionId: string): boolean {
+  return /^[A-Za-z0-9]{8,128}$/.test(chatSessionId.replace(/-/g, ""));
+}
+
+export function chatSessionFile(memoryDir: string, chatSessionId: string): string {
+  if (!isSafeChatSessionId(chatSessionId)) {
+    throw new Error("Invalid chat session id");
+  }
+  return join(chatSessionDir(memoryDir), `${chatSessionId}.jsonl`);
+}
+
+/**
+ * Create a new chat session.  The session binds the caller's principal and
+ * sessionKey at creation (rule 42); every tool call flows through
+ * access-service with that identity.
+ */
+export async function createChatSession(
+  memoryDir: string,
+  opts: { principal?: string; sessionKey?: string; namespace?: string },
+): Promise<ChatSessionState> {
+  const id = randomUUID();
+  const now = new Date().toISOString();
+  const state: ChatSessionState = {
+    id,
+    principal: opts.principal,
+    sessionKey: opts.sessionKey,
+    namespace: opts.namespace,
+    transcript: [],
+    confirmedPlanIds: new Set(),
+    createdAt: now,
+  };
+  // Ensure the directory exists.
+  await mkdir(chatSessionDir(memoryDir), { recursive: true });
+  // Write a metadata header line so the file is never empty and carries
+  // the principal/scope binding (used for per-session isolation checks).
+  await appendFile(
+    chatSessionFile(memoryDir, id),
+    JSON.stringify({
+      seq: 0,
+      ts: now,
+      role: "system" as const,
+      content: `Chat session created. Principal: ${opts.principal ?? "default"}. Namespace: ${opts.namespace ?? "default"}. work_task: Excluded from memory extraction`,
+      // Structured binding fields so loadChatSession doesn't need to regex-parse
+      // prose (robust against dotted principals like alice@example.com).
+      ...(opts.principal ? { principal: opts.principal } : {}),
+      ...(opts.namespace ? { namespace: opts.namespace } : {}),
+      ...(opts.sessionKey ? { sessionKey: opts.sessionKey } : {}),
+    }) + "\n",
+    "utf8",
+  );
+  return state;
+}
+
+/**
+ * Load a chat session from disk, including its full transcript.
+ * Returns `null` if the session file does not exist.
+ *
+ * Per-session isolation: the caller MUST verify that the loaded session's
+ * principal matches the requesting principal before operating on it.
+ */
+export async function loadChatSession(
+  memoryDir: string,
+  chatSessionId: string,
+): Promise<ChatSessionState | null> {
+  let raw: string;
+  try {
+    raw = await readFile(chatSessionFile(memoryDir, chatSessionId), "utf8");
+  } catch {
+    return null;
+  }
+  const lines = raw.split("\n").filter((l) => l.trim().length > 0);
+  const transcript: ChatTranscriptEntry[] = [];
+  let principal: string | undefined;
+  let sessionKey: string | undefined;
+  let namespace: string | undefined;
+  let createdAt = "";
+  let pendingPlanId: string | undefined;
+  let pendingPromotionId: string | undefined;
+  for (const line of lines) {
+    try {
+      const entry = JSON.parse(line) as ChatTranscriptEntry & Record<string, unknown>;
+      transcript.push(entry);
+      // Extract binding from the system header line.
+      if (entry.role === "system" && entry.seq === 0) {
+        // Prefer structured fields (robust against dotted principals).
+        const metaPrincipal = typeof entry.principal === "string" ? entry.principal : undefined;
+        const metaNamespace = typeof entry.namespace === "string" ? entry.namespace : undefined;
+        const metaSessionKey = typeof entry.sessionKey === "string" ? entry.sessionKey : undefined;
+        if (metaPrincipal) principal = metaPrincipal;
+        else {
+          // Greedy regex fallback for older session files (captures until ". Namespace").
+          const pMatch = entry.content.match(/Principal: (.+?)\.\s/);
+          if (pMatch) principal = pMatch[1]?.trim() === "default" ? undefined : pMatch[1]?.trim();
+        }
+        if (metaNamespace) namespace = metaNamespace;
+        else {
+          const nsMatch = entry.content.match(/Namespace: (.+?)\.\s/);
+          if (nsMatch) namespace = nsMatch[1]?.trim() === "default" ? undefined : nsMatch[1]?.trim();
+        }
+        if (metaSessionKey) sessionKey = metaSessionKey;
+        createdAt = entry.ts;
+      }
+      // Scan for pending-plan / pending-promotion markers (append-only state).
+      if (entry.role === "system") {
+        const pm = entry.content.match(/^pending_plan:(.+)$/);
+        if (pm) pendingPlanId = pm[1];
+        const pp = entry.content.match(/^pending_promotion:(.+)$/);
+        if (pp) pendingPromotionId = pp[1];
+        const resolved = entry.content.match(/^(plan_applied|plan_cancelled|promotion_applied|promotion_cancelled):/);
+        if (resolved) {
+          // Only clear the matching pending — a plan resolution must not drop
+          // an active promotion (and vice versa) (cursor Medium OlACs).
+          if (entry.content.startsWith("plan_")) {
+            pendingPlanId = undefined;
+          }
+          if (entry.content.startsWith("promotion_")) {
+            pendingPromotionId = undefined;
+          }
+        }
+      }
+    } catch {
+      // Skip malformed lines (rule 54 — atomic appends; partial lines are
+      // tolerated, never crash the engine).
+    }
+  }
+  return {
+    id: chatSessionId,
+    principal,
+    sessionKey,
+    namespace,
+    transcript,
+    confirmedPlanIds: new Set(),
+    ...(pendingPlanId ? { pendingPlanId } : {}),
+    ...(pendingPromotionId ? { pendingPromotionId } : {}),
+    createdAt,
+  };
+}
+
+/**
+ * Append a transcript entry to the session file (atomic append, rule 54).
+ */
+export async function appendTranscriptEntry(
+  memoryDir: string,
+  chatSessionId: string,
+  entry: Omit<ChatTranscriptEntry, "seq" | "ts">,
+): Promise<ChatTranscriptEntry> {
+  const full: ChatTranscriptEntry = {
+    ...entry,
+    seq: Date.now(),
+    ts: new Date().toISOString(),
+  };
+  await appendFile(
+    chatSessionFile(memoryDir, chatSessionId),
+    JSON.stringify(full) + "\n",
+    "utf8",
+  );
+  return full;
+}
+
+/**
+ * Append a pending-plan marker so a later turn can confirm the plan
+ * (append-only state — loadChatSession scans for the latest unresolved one).
+ */
+export async function markPendingPlan(
+  memoryDir: string,
+  chatSessionId: string,
+  planId: string,
+): Promise<void> {
+  await appendTranscriptEntry(memoryDir, chatSessionId, {
+    role: "system",
+    content: `pending_plan:${planId}`,
+  });
+}
+
+/**
+ * Append a pending-promotion marker so a later turn can confirm the promotion.
+ */
+export async function markPendingPromotion(
+  memoryDir: string,
+  chatSessionId: string,
+  memoryId: string,
+): Promise<void> {
+  await appendTranscriptEntry(memoryDir, chatSessionId, {
+    role: "system",
+    content: `pending_promotion:${memoryId}`,
+  });
+}
+
+/**
+ * Record that a pending plan was applied or cancelled.
+ */
+export async function markPlanResolved(
+  memoryDir: string,
+  chatSessionId: string,
+  planId: string,
+): Promise<void> {
+  await appendTranscriptEntry(memoryDir, chatSessionId, {
+    role: "system",
+    content: `plan_applied:${planId}`,
+  });
+}
+
+/**
+ * Record that a pending promotion was applied or cancelled.
+ */
+export async function markPromotionResolved(
+  memoryDir: string,
+  chatSessionId: string,
+  memoryId: string,
+): Promise<void> {
+  await appendTranscriptEntry(memoryDir, chatSessionId, {
+    role: "system",
+    content: `promotion_applied:${memoryId}`,
+  });
+}
+
+/**
+ * Clean up expired chat sessions based on TTL.
+ */
+export async function cleanupExpiredChatSessions(
+  memoryDir: string,
+  ttlHours: number,
+): Promise<number> {
+  const dir = chatSessionDir(memoryDir);
+  let files: string[];
+  try {
+    files = await readdir(dir);
+  } catch {
+    return 0;
+  }
+  const now = Date.now();
+  let removed = 0;
+  for (const file of files) {
+    if (!file.endsWith(".jsonl")) continue;
+    try {
+      const filePath = join(dir, file);
+      const st = await stat(filePath);
+      const ageHours = (now - st.mtimeMs) / (1000 * 60 * 60);
+      // TTL of 0 = expire everything immediately. For positive TTLs the
+      // boundary is inclusive; sub-millisecond APFS mtime skew is negligible
+      // against an hour-scale threshold (rule 54 — deterministic cleanup).
+      if (ttlHours <= 0 || ageHours >= ttlHours) {
+        await unlink(filePath);
+        removed++;
+      }
+    } catch {
+      // Best-effort cleanup; skip files we can't stat/unlink.
+    }
+  }
+  return removed;
+}
+
+/**
+ * Verify that a session belongs to the given principal (rule 42/47 —
+ * per-session isolation, no cross-visibility).
+ */
+export function sessionBelongsToPrincipal(
+  session: ChatSessionState,
+  principal?: string,
+): boolean {
+  // A session with no recorded principal is accessible to anyone (legacy/
+  // single-user case).  A session with a principal requires an exact match.
+  if (!session.principal) return true;
+  return session.principal === principal;
+}

package/src/chat/chat-system-prompt.ts

@@ -0,0 +1,76 @@
+/**
+ * Sealed system prompt for the chat engine (issue #1583).
+ *
+ * The prompt encodes the core rules that every turn enforces:
+ * - Always cite memory ids/handles and source quotes when asserting what's
+ *   remembered.
+ * - NEVER apply a mutation without explicit user confirmation in the
+ *   conversation.
+ * - Render plans as the diff the Correction Contract returns.
+ * - Prefer "I found nothing" over speculation.
+ *
+ * The engine supplements the prompt with a tool-citation guard at runtime
+ * (the model must ground assertions in tool results from the current turn).
+ */
+
+/**
+ * Build the system prompt for the chat engine.  Parameterised by whether the
+ * correction (mutation) tools are available — when #1580 is not yet merged,
+ * correction_plan/apply are omitted and the prompt reflects that.
+ */
+export function buildChatSystemPrompt(opts: {
+  correctionAvailable: boolean;
+}): string {
+  const lines: string[] = [
+    "You are Remnic Chat — a conversational memory inspector and corrector.",
+    "You help the user understand, explore, and fix what Remnic remembers.",
+    "",
+    "## Core rules",
+    "",
+    "1. GROUND EVERY ASSERTION. When you state what Remnic remembers, you MUST",
+    "   cite the memory id or handle, include the source quote, and the date.",
+    "   If no tool result in this turn supports the claim, say",
+    "   \"I found nothing\" instead of guessing.",
+    "",
+    "2. NO MUTATION WITHOUT CONFIRMATION. You may propose a correction plan",
+    "   (correction_plan), but you must NEVER apply it (correction_apply) until",
+    "   the user explicitly confirms in the conversation.",
+    "",
+    "3. RENDER PLANS AS DIFFS. When you surface a correction plan, render it",
+    "   as the diff preview the plan returns — before/after, affected memory",
+    "   ids, and the supersession reason.",
+    "",
+    "4. BE HONEST ABOUT LIMITS. If a tool returns no results, say so. If you",
+    "   are unsure, say so. Never fabricate memory content.",
+    "",
+    "5. RESPECT SCOPE. You operate within the caller's namespace and principal.",
+    "   Do not attempt to access memories outside the caller's scope.",
+  ];
+
+  if (opts.correctionAvailable) {
+    lines.push(
+      "",
+      "## Mutation protocol",
+      "",
+      "When the user asks to change or forget memories:",
+      "1. Call correction_plan to generate a diff preview.",
+      "2. Present the diff to the user and ask them to reply 'apply' to confirm.",
+      "3. Only after explicit confirmation, call correction_apply.",
+      "",
+      "For bulk/destructive requests ('forget everything about X'), always go",
+      "through correction_plan first — there is no direct-delete tool.",
+    );
+  } else {
+    lines.push(
+      "",
+      "## Mutation status",
+      "",
+      "Correction (mutation) tools are not yet available in this build.",
+      "You can inspect and search memories but cannot apply corrections.",
+      "When the user asks to change memories, explain that correction tools",
+      "require the Correction Contract (#1580) which is pending.",
+    );
+  }
+
+  return lines.join("\n");
+}

package/src/chat/chat-tools.ts

@@ -0,0 +1,255 @@
+/**
+ * Chat tools — the ONLY tools available to the chat engine (issue #1583).
+ *
+ * Each tool is a thin adapter over an existing access-service method (rule 22
+ * — zero new business logic here).  Mutating tools (correction_apply,
+ * memory_promote) are marked and go through the confirmation protocol in the
+ * engine before reaching the executor.
+ */
+
+import type { ChatToolSchema } from "./chat-types.js";
+
+// ---------------------------------------------------------------------------
+// Tool identifiers
+// ---------------------------------------------------------------------------
+
+export type ChatToolName =
+  | "memory_search"
+  | "memory_get"
+  | "memory_timeline"
+  | "recall_explain"
+  | "entity_get"
+  | "stats"
+  | "correction_plan"
+  | "correction_apply"
+  | "memory_promote"
+  | "review_list"
+  | "scope_inspect";
+
+/** Tools that mutate memory — require confirmation before execution. */
+export const MUTATING_TOOLS: ReadonlySet<ChatToolName> = new Set([
+  "correction_apply",
+  "memory_promote",
+]);
+
+// ---------------------------------------------------------------------------
+// Tool schema definitions (sent to the LLM as the function-calling schema)
+// ---------------------------------------------------------------------------
+
+const MEMORY_SEARCH_SCHEMA: ChatToolSchema = {
+  type: "function",
+  function: {
+    name: "memory_search",
+    description:
+      "Semantic search across stored memories. Returns matching memory paths, scores, and snippets. Use this to find what Remnic remembers about a topic.",
+    parameters: {
+      type: "object",
+      properties: {
+        query: { type: "string", description: "The search query." },
+        maxResults: { type: "number", description: "Maximum results (default 8)." },
+      },
+      required: ["query"],
+      additionalProperties: false,
+    },
+  },
+};
+
+const MEMORY_GET_SCHEMA: ChatToolSchema = {
+  type: "function",
+  function: {
+    name: "memory_get",
+    description:
+      "Fetch a single memory by its id. Returns the full memory content, category, tags, and source attribution.",
+    parameters: {
+      type: "object",
+      properties: {
+        memoryId: { type: "string", description: "The memory id to fetch." },
+      },
+      required: ["memoryId"],
+      additionalProperties: false,
+    },
+  },
+};
+
+const MEMORY_TIMELINE_SCHEMA: ChatToolSchema = {
+  type: "function",
+  function: {
+    name: "memory_timeline",
+    description:
+      "Read the lifecycle timeline of a single memory — creation, updates, consolidations, and tier transitions.",
+    parameters: {
+      type: "object",
+      properties: {
+        memoryId: { type: "string", description: "The memory id." },
+        limit: { type: "number", description: "Maximum rows (default 200)." },
+      },
+      required: ["memoryId"],
+      additionalProperties: false,
+    },
+  },
+};
+
+const RECALL_EXPLAIN_SCHEMA: ChatToolSchema = {
+  type: "function",
+  function: {
+    name: "recall_explain",
+    description:
+      "Explain why Remnic would recall certain memories for a given query. Returns tier scoring and matching details. Use this to answer 'why did you remember this?'",
+    parameters: {
+      type: "object",
+      properties: {
+        query: { type: "string", description: "Optional query to explain recall for. When omitted, explains the most recent recall." },
+      },
+      required: [],
+      additionalProperties: false,
+    },
+  },
+};
+
+const ENTITY_GET_SCHEMA: ChatToolSchema = {
+  type: "function",
+  function: {
+    name: "entity_get",
+    description:
+      "Fetch the compiled entity record for a named entity (person, project, tool, etc.). Returns traits, timeline, and associated memories.",
+    parameters: {
+      type: "object",
+      properties: {
+        name: { type: "string", description: "Entity name." },
+      },
+      required: ["name"],
+      additionalProperties: false,
+    },
+  },
+};
+
+const STATS_SCHEMA: ChatToolSchema = {
+  type: "function",
+  function: {
+    name: "stats",
+    description:
+      "Return aggregate memory statistics — memory count, entity count, profile summary, and open questions. Use this for high-level 'what do you know?' questions.",
+    parameters: {
+      type: "object",
+      properties: {},
+      additionalProperties: false,
+    },
+  },
+};
+
+const CORRECTION_PLAN_SCHEMA: ChatToolSchema = {
+  type: "function",
+  function: {
+    name: "correction_plan",
+    description:
+      "Generate a correction plan (diff preview) for a requested memory change. This does NOT apply the change — it previews what would happen. The user must confirm before correction_apply runs.",
+    parameters: {
+      type: "object",
+      properties: {
+        request: {
+          type: "string",
+          description:
+            "Natural-language description of the correction, e.g. 'we finished the MySQL migration in March' or 'forget the outdated database credentials'.",
+        },
+      },
+      required: ["request"],
+      additionalProperties: false,
+    },
+  },
+};
+
+const CORRECTION_APPLY_SCHEMA: ChatToolSchema = {
+  type: "function",
+  function: {
+    name: "correction_apply",
+    description:
+      "Apply a previously generated correction plan. The engine intercepts this call and requires explicit user confirmation before it executes.",
+    parameters: {
+      type: "object",
+      properties: {
+        planId: { type: "string", description: "The plan id from correction_plan." },
+      },
+      required: ["planId"],
+      additionalProperties: false,
+    },
+  },
+};
+
+const MEMORY_PROMOTE_SCHEMA: ChatToolSchema = {
+  type: "function",
+  function: {
+    name: "memory_promote",
+    description:
+      "Promote a memory's tier (e.g. from cold to hot). The engine requires user confirmation before executing.",
+    parameters: {
+      type: "object",
+      properties: {
+        memoryId: { type: "string", description: "The memory id to promote." },
+      },
+      required: ["memoryId"],
+      additionalProperties: false,
+    },
+  },
+};
+
+const REVIEW_LIST_SCHEMA: ChatToolSchema = {
+  type: "function",
+  function: {
+    name: "review_list",
+    description:
+      "List items in the contradiction/quality review queue. Returns pairs of potentially conflicting memories awaiting resolution.",
+    parameters: {
+      type: "object",
+      properties: {
+        runId: { type: "string", description: "Optional governance run id." },
+      },
+      additionalProperties: false,
+    },
+  },
+};
+
+const SCOPE_INSPECT_SCHEMA: ChatToolSchema = {
+  type: "function",
+  function: {
+    name: "scope_inspect",
+    description:
+      "Inspect the current scope/namespace resolution — which namespace the chat session operates in and how it was resolved.",
+    parameters: {
+      type: "object",
+      properties: {},
+      additionalProperties: false,
+    },
+  },
+};
+
+/**
+ * All read-only tool schemas, always available when chat is enabled.
+ */
+const READONLY_SCHEMAS: ChatToolSchema[] = [
+  MEMORY_SEARCH_SCHEMA,
+  MEMORY_GET_SCHEMA,
+  MEMORY_TIMELINE_SCHEMA,
+  RECALL_EXPLAIN_SCHEMA,
+  ENTITY_GET_SCHEMA,
+  STATS_SCHEMA,
+  REVIEW_LIST_SCHEMA,
+];
+
+/**
+ * Build the tool schema list for the LLM.  The correction tools are only
+ * included when the Correction Contract (#1580) is available; scope_inspect
+ * is only included when the scope resolver (#1494) is available.
+ */
+export function buildChatToolSchemas(opts: {
+  correctionAvailable: boolean;
+  scopeInspectAvailable: boolean;
+}): ChatToolSchema[] {
+  const tools = [...READONLY_SCHEMAS];
+  if (opts.correctionAvailable) {
+    tools.push(CORRECTION_PLAN_SCHEMA, CORRECTION_APPLY_SCHEMA, MEMORY_PROMOTE_SCHEMA);
+  }
+  if (opts.scopeInspectAvailable) {
+    tools.push(SCOPE_INSPECT_SCHEMA);
+  }
+  return tools;
+}