@polycode-projects/the-mechanical-code-talker 0.2.0 → 0.3.0

package/src/memory/core.mjs

@@ -0,0 +1,292 @@
+// memory/core.mjs — tmct's OWN conversational memory graph (ROADMAP item 9).
+//
+// A dedicated OWL-labelled store at <repo>/.tmct/memory/graph.json — raw JSON in
+// the exact `entities` shape buildEntities produces, so codegraph.mjs's
+// parseEntities() loads it unchanged ({ individuals, byId, relations, proseIndex }).
+// It is DISTINCT from any provider-supplied code graph: tmct never writes a
+// provider's graph (docs/adapter-contract.md); memory writes land ONLY here.
+//
+// What goes in:
+//   - every parsed inbound request becomes an "a-visitor-said" individual
+//     (class `Utterance`, role=visitor) and every response an "a-tmct-said"
+//     individual (role=tmct), each carrying text/ts/role attributes, an
+//     `mgx:saidInSession` edge to its Session anchor, and — for a response —
+//     an `mgx:inReplyTo` edge to the visitor utterance it answers;
+//   - grammar-derived OWL triples via appendFact() (subject/predicate/object +
+//     provenance), reified RDF-style (rdf:subject / rdf:predicate / rdf:object
+//     on a `Fact` individual) — the Phase-2 ACE parser's write point.
+//
+// OWL labelling: individuals are rdf-ish typed twice — the payload-level `class`
+// field (Utterance / Fact / Session, counted in `classes[]` like every other
+// graph class) AND an `rdf:type` attribute naming the OWL term
+// (owl:NamedIndividual for utterances, rdf:Statement for reified facts), with
+// the owl/rdf/rdfs prefixes declared in the payload's `prefixes` block —
+// consistent with graph-build.mjs's JSON-label-only vocabulary style.
+//
+// Every append is crash-safe (fresh read → mutate → temp-file + rename, the
+// sessions.mjs discipline) and IDEMPOTENT: utterance ids are deterministic
+// (utt:<session>#<ts>#<role>) and fact ids hash the triple, so the per-turn
+// re-append sessions.mjs performs replaces rather than duplicates.
+
+import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { proseTokensFor, buildProseIndex } from "../prose.mjs";
+
+export const MEMORY_DIR_REL = join(".tmct", "memory");
+export const MEMORY_GRAPH_REL = join(MEMORY_DIR_REL, "graph.json");
+
+export const UTTERANCE_CLASS = "Utterance";
+export const FACT_CLASS = "Fact";
+export const MEMORY_SESSION_CLASS = "Session";
+
+export const SAID_IN_SESSION_PROP = "mgx:saidInSession";
+export const IN_REPLY_TO_PROP = "mgx:inReplyTo";
+
+const ROLES = new Set(["visitor", "tmct"]);
+const LABEL_CAP = 48;    // utterance/fact labels stay skimmable in renders
+const TEXT_CAP = 2000;   // an utterance's stored text (a whole answer fits; a pasted book doesn't)
+
+/** The memory graph's vocabulary — documented in-payload exactly like
+ *  graph-build.mjs documents the code graph's. */
+const MEMORY_VOCABULARY = [
+  { prop: "rdf:type", note: "rdf-ish typing attribute: owl:NamedIndividual (utterances/sessions) or rdf:Statement (reified facts)" },
+  { prop: "mgx:utteranceRole", note: "who said it: visitor (an a-visitor-said item) or tmct (the response alongside it)" },
+  { prop: "mgx:utteranceText", note: "the utterance's normalized text (capped)" },
+  { prop: "mgx:utteranceTs", note: "when it was said, ISO-8601 (the chat turn timestamp)" },
+  { prop: "mgx:utteranceParsed", note: "optional JSON of the parse the interpretation pipeline produced for this request" },
+  { prop: SAID_IN_SESSION_PROP, predicate: "saidInSession", note: "Utterance → Session it was said in; runtime observation, owned (no SEON term)" },
+  { prop: IN_REPLY_TO_PROP, predicate: "inReplyTo", note: "tmct Utterance → the visitor Utterance it answers (the Q/A pairing)" },
+  { prop: "rdf:subject", note: "reified fact: the triple's subject term" },
+  { prop: "rdf:predicate", note: "reified fact: the triple's predicate term" },
+  { prop: "rdf:object", note: "reified fact: the triple's object term" },
+  { prop: "mgx:factProvenance", note: "where a fact came from (a session/turn ref, a corpus block id, an ACE parse)" },
+  { prop: "mgx:hasProseTokens", note: "prose tokens (prose.mjs tokenizer) backing the payload's proseIndex" },
+  { prop: "mgx:sessionStarted", note: "session anchor: when the session started, ISO-8601" },
+];
+
+/** A fresh, empty memory payload — the buildEntities shape, plus the OWL/RDF
+ *  prefixes the memory vocabulary uses. `memory: true` marks it as tmct's own
+ *  store (never a provider artifact). */
+export function emptyMemory() {
+  return {
+    generated_at: "",
+    memory: true,
+    prefixes: {
+      owl: "http://www.w3.org/2002/07/owl#",
+      rdf: "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
+      rdfs: "http://www.w3.org/2000/01/rdf-schema#",
+      mgx: "urn:tmct:mgx#",
+    },
+    vocabulary: MEMORY_VOCABULARY.map((v) => ({ ...v })),
+    classes: [],
+    objectProperties: [],
+    individuals: [],
+    proseIndex: {},
+  };
+}
+
+const memoryGraphFile = (dir) => join(dir, MEMORY_GRAPH_REL);
+
+/** Atomic JSON write (temp in the same dir + rename) — same discipline as
+ *  sessions.mjs's graph append: a crash never destroys the previous store. */
+async function atomicWriteJson(file, obj) {
+  const tmp = `${file}.tmp-${process.pid}-${Math.random().toString(36).slice(2, 10)}`;
+  await writeFile(tmp, JSON.stringify(obj));
+  await rename(tmp, file);
+}
+
+/** Load the memory graph for a repo dir. A missing store is the bootstrap:
+ *  return the empty payload (uncached — the first append creates the file).
+ *  The result is a raw entities payload; parseEntities() loads it. */
+export async function loadMemory(dir) {
+  let text;
+  try {
+    text = await readFile(memoryGraphFile(dir), "utf8");
+  } catch (e) {
+    if (e?.code === "ENOENT") return emptyMemory();
+    throw e;
+  }
+  return JSON.parse(text);
+}
+
+/** Fresh read → mutate → atomic write. Serialized per call; every public append
+ *  goes through here so a concurrent reader never sees a torn store. */
+async function mutateMemory(dir, fn) {
+  const payload = await loadMemory(dir);
+  const out = fn(payload) ?? payload;
+  out.proseIndex = buildProseIndex(out.individuals);
+  await mkdir(dirname(memoryGraphFile(dir)), { recursive: true });
+  await atomicWriteJson(memoryGraphFile(dir), out);
+  return out;
+}
+
+const normText = (t) => String(t ?? "").replace(/\s+/g, " ").trim().slice(0, TEXT_CAP);
+const labelOf = (text) => (text.length > LABEL_CAP ? text.slice(0, LABEL_CAP - 1) + "…" : text);
+
+/** Upsert an individual by id (replace-in-place keeps ordering stable). */
+function upsertIndividual(payload, ind) {
+  const i = payload.individuals.findIndex((x) => x?.id === ind.id);
+  if (i >= 0) payload.individuals[i] = ind;
+  else payload.individuals.push(ind);
+}
+
+/** Upsert one edge into the named relation group (dedupe by subject>object). */
+function upsertEdge(payload, { predicate, prop }, edge) {
+  let group = payload.objectProperties.find((g) => g?.prop === prop);
+  if (!group) {
+    group = { predicate, prop, count: 0, examples: [] };
+    payload.objectProperties.push(group);
+  }
+  group.examples = (group.examples || []).filter(
+    (e) => !(e?.subject === edge.subject && e?.object === edge.object),
+  );
+  group.examples.push(edge);
+  group.count = group.examples.length;
+}
+
+/** Recount `classes[]` from the individuals — every memory class stays counted
+ *  and sampled the way graph-build.mjs counts the code classes. */
+function recountClasses(payload) {
+  const names = [MEMORY_SESSION_CLASS, UTTERANCE_CLASS, FACT_CLASS];
+  payload.classes = payload.classes.filter((c) => !names.includes(c?.name));
+  for (const name of names) {
+    const of = payload.individuals.filter((i) => i?.class === name);
+    if (of.length) payload.classes.push({ name, count: of.length, sample: of.slice(0, 3).map((i) => i.label) });
+  }
+}
+
+/** Make sure the Session anchor individual exists (edges never dangle). */
+function ensureSession(payload, sessionId, started = "") {
+  const sid = `session:${sessionId}`;
+  if (payload.individuals.some((i) => i?.id === sid)) return sid;
+  payload.individuals.push({
+    id: sid, label: String(sessionId).slice(0, 8), class: MEMORY_SESSION_CLASS,
+    derived_from: [], mentions: [],
+    attributes: [
+      { prop: "rdf:type", key: "type", value: "owl:NamedIndividual" },
+      ...(started ? [{ prop: "mgx:sessionStarted", key: "started", value: started }] : []),
+    ],
+  });
+  return sid;
+}
+
+/** Build (don't write) one Utterance individual + its edges; shared by the
+ *  single and batch append paths. Returns the utterance id. */
+function putUtterance(payload, { role, text, ts, sessionId, sessionStarted = "", parsed = null, replyTo = null }) {
+  if (!ROLES.has(role)) throw new Error(`utterance role must be "visitor" or "tmct", got ${JSON.stringify(role)}`);
+  if (!sessionId) throw new Error("utterance needs a sessionId");
+  const cleanTs = String(ts || "");
+  const cleanText = normText(text);
+  const id = `utt:${sessionId}#${cleanTs}#${role}`;
+  const label = labelOf(cleanText) || (role === "visitor" ? "a-visitor-said" : "a-tmct-said");
+  const tokens = proseTokensFor({ doc: cleanText });
+  const ind = {
+    id, label, class: UTTERANCE_CLASS,
+    derived_from: [], mentions: [],
+    attributes: [
+      { prop: "rdf:type", key: "type", value: "owl:NamedIndividual" },
+      { prop: "mgx:utteranceRole", key: "role", value: role },
+      { prop: "mgx:utteranceText", key: "text", value: cleanText },
+      { prop: "mgx:utteranceTs", key: "ts", value: cleanTs },
+      ...(parsed != null ? [{ prop: "mgx:utteranceParsed", key: "parsed", value: JSON.stringify(parsed) }] : []),
+      ...(tokens.length ? [{ prop: "mgx:hasProseTokens", key: "prose_tokens", value: tokens.join(" ") }] : []),
+    ],
+  };
+  upsertIndividual(payload, ind);
+  const sid = ensureSession(payload, sessionId, sessionStarted);
+  upsertEdge(payload, { predicate: "saidInSession", prop: SAID_IN_SESSION_PROP }, {
+    subject: id, object: sid, subjectLabel: label, objectLabel: String(sessionId).slice(0, 8),
+  });
+  if (replyTo) {
+    const target = payload.individuals.find((i) => i?.id === replyTo);
+    if (target) { // never a dangling reply edge — honest drop, like sessions.mjs
+      upsertEdge(payload, { predicate: "inReplyTo", prop: IN_REPLY_TO_PROP }, {
+        subject: id, object: replyTo, subjectLabel: label, objectLabel: target.label,
+      });
+    }
+  }
+  if (cleanTs && cleanTs > String(payload.generated_at || "")) payload.generated_at = cleanTs;
+  return id;
+}
+
+/** Append ONE utterance (visitor request or tmct response) to the memory graph.
+ *  { role, text, ts, sessionId, sessionStarted?, parsed?, replyTo? } — `parsed`
+ *  is the interpretation pipeline's parse of the request (stored as JSON);
+ *  `replyTo` a prior utterance id (Q/A pairing). Deterministic id → idempotent.
+ *  Returns { id }. */
+export async function appendUtterance(dir, utterance) {
+  let id;
+  await mutateMemory(dir, (payload) => {
+    id = putUtterance(payload, utterance);
+    recountClasses(payload);
+  });
+  return { id };
+}
+
+/** Batch append — ONE read-modify-write for a whole turn (or session) worth of
+ *  utterances; what sessions.mjs's per-turn wiring calls. Returns { ids }. */
+export async function appendUtterances(dir, utterances) {
+  const ids = [];
+  if (!utterances?.length) return { ids };
+  await mutateMemory(dir, (payload) => {
+    for (const u of utterances) ids.push(putUtterance(payload, u));
+    recountClasses(payload);
+  });
+  return { ids };
+}
+
+/** FNV-1a 32-bit — a stable little content hash for fact ids (dedupe by triple). */
+function fnv1a(s) {
+  let h = 0x811c9dc5;
+  for (let i = 0; i < s.length; i += 1) {
+    h ^= s.charCodeAt(i);
+    h = Math.imul(h, 0x01000193);
+  }
+  return (h >>> 0).toString(16).padStart(8, "0");
+}
+
+/** Normalize a fact TERM (subject/object) so every writer converges on one
+ *  spelling and the graph stays queryable: ConceptNet's /c/en/foo_bar, a
+ *  grammar's tmct:Foo_bar and a bare "Foo bar" all become "foo bar". The
+ *  PREDICATE is deliberately NOT normalized this way - it is a controlled
+ *  vocabulary term (rdfs:subClassOf) whose casing is meaningful. */
+export function normFactTerm(t) {
+  let s = normText(t);
+  s = s.replace(/^\/c\/[a-z]{2,3}\//i, "");
+  s = s.replace(/^[a-z][\w.-]*:/i, "");
+  s = s.replace(/_/g, " ").replace(/\s+/g, " ").trim();
+  return s.toLowerCase();
+}
+
+/** Append one grammar-derived OWL triple, RDF-reified: a `Fact` individual
+ *  carrying rdf:subject / rdf:predicate / rdf:object (+ provenance). The
+ *  Phase-2 ACE parser's write point. Same (s,p,o) → same id → upsert, never a
+ *  duplicate. Returns { id }. */
+export async function appendFact(dir, { subject, predicate, object, provenance = "" } = {}) {
+  const s = normFactTerm(subject);
+  const p = normText(predicate);
+  const o = normFactTerm(object);
+  if (!s || !p || !o) throw new Error("a fact needs subject, predicate and object");
+  const id = `fact:${fnv1a(`${s}${p}${o}`)}`;
+  const text = `${s} ${p} ${o}`;
+  const tokens = proseTokensFor({ doc: text });
+  await mutateMemory(dir, (payload) => {
+    const prior = payload.individuals.find((x) => x?.id === id);
+    const priorProv = prior?.attributes?.find((a) => a?.prop === "mgx:factProvenance")?.value || "";
+    const provs = [...new Set([...priorProv.split(" | "), normText(provenance)].filter(Boolean))];
+    upsertIndividual(payload, {
+      id, label: labelOf(text), class: FACT_CLASS,
+      derived_from: [], mentions: [],
+      attributes: [
+        { prop: "rdf:type", key: "type", value: "rdf:Statement" },
+        { prop: "rdf:subject", key: "subject", value: s },
+        { prop: "rdf:predicate", key: "predicate", value: p },
+        { prop: "rdf:object", key: "object", value: o },
+        ...(provs.length ? [{ prop: "mgx:factProvenance", key: "provenance", value: provs.join(" | ") }] : []),
+        ...(tokens.length ? [{ prop: "mgx:hasProseTokens", key: "prose_tokens", value: tokens.join(" ") }] : []),
+      ],
+    });
+    recountClasses(payload);
+  });
+  return { id };
+}

package/src/memory/fold.mjs

@@ -0,0 +1,105 @@
+// memory/fold.mjs — session-log cleaning → corpus folding (ROADMAP item 9).
+//
+// A raw chat session is noisy: slash-commands and their dumps, greetings,
+// thanks, honest misses. foldSessionLogs() reads the structured sidecars
+// (.tmct/sessions/*.jsonl), pairs each turn with its answer PROSE from the
+// human transcript (.tmct/session-<id>.log — the only artifact carrying answer
+// text), cleans the turns, and writes ONE text block per session into
+// blocks.mjs's store. Cleaning rules:
+//   - slash-command turns (and therefore their outputs) are dropped;
+//   - conversational filler is dropped — both turns chat.mjs already flagged
+//     `conversational` and unflagged greeting/thanks/bye one-liners;
+//   - miss turns are dropped (their answer is the honest-miss boilerplate —
+//     the memory GRAPH still records them as utterances; the corpus doesn't);
+//   - surviving Q/A pairs are whitespace-normalized into "Q: …\nA: …" lines.
+//
+// Idempotent by construction: the block id IS the session id, and
+// blocks.saveBlock() replaces on the same id — re-folding a session updates
+// its block, never duplicates it. A session that cleans down to nothing writes
+// no block (and removes a stale one), so the corpus never holds empty blocks.
+
+import { readdir, readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { SESSIONS_DIR_REL, parseSessionJsonl, parseSessionLog, turnKey } from "../sessions.mjs";
+import { removeBlock, saveBlock } from "./blocks.mjs";
+
+// chat.mjs's SESSION_LOG_DIR — repeated here (a one-word constant) rather than
+// importing the whole chat surface into the memory layer.
+const LOG_DIR_REL = ".tmct";
+
+/** Conversational filler a transcript doesn't need: greetings, thanks, byes,
+ *  bare acknowledgements. Catches the one-liners chat.mjs did NOT flag
+ *  `conversational` (e.g. typed straight at an older tmct). */
+const FILLER_RE = new RegExp(
+  "^(?:(?:hi|hiya|hello|hey|yo|howdy)(?:\\s+there)?|good\\s*(?:morning|afternoon|evening)|" +
+    "thanks(?:\\s+(?:a\\s+lot|so\\s+much))?|thank\\s*you|thx|ty|cheers|" +
+    "ok(?:ay)?|cool|nice|great|sure|yes|no|yep|nope|" +
+    "bye(?:\\s+bye)?|goodbye|see\\s*(?:ya|you)|later)[\\s!.?,]*$",
+  "i",
+);
+
+const squash = (s) => String(s ?? "").replace(/\s+/g, " ").trim();
+
+/**
+ * Clean one parsed session record into corpus text (pure). `answers` is
+ * parseSessionLog()'s Map (may be empty — a vanished transcript degrades to
+ * question-only lines, honestly, rather than losing the session).
+ * Returns "" when nothing survives.
+ */
+export function cleanSessionText(record, answers = new Map()) {
+  const parts = [];
+  for (const t of record?.turns || []) {
+    const query = squash(t?.query);
+    if (!query) continue;
+    if (t.command || query.startsWith("/")) continue; // slash-commands + their outputs
+    if (t.conversational || FILLER_RE.test(query)) continue; // greetings/thanks/bye
+    if (t.miss) continue; // honest-miss boilerplate is not corpus content
+    const answer = squash(answers.get(turnKey(t.ts, t.query)));
+    parts.push(answer ? `Q: ${query}\nA: ${answer}` : `Q: ${query}`);
+  }
+  return parts.join("\n");
+}
+
+/**
+ * Fold recorded session logs into the text-block corpus. Reads every
+ * .tmct/sessions/*.jsonl sidecar under `repoDir` (or just `sessionId`'s),
+ * cleans it, and upserts one block per session (block id = session id).
+ * Best-effort per session — an unreadable sidecar or missing transcript never
+ * fails the fold. Returns { folded: [ids], removed: [ids], skipped: n }.
+ */
+export async function foldSessionLogs(repoDir, { sessionId = null } = {}) {
+  const dir = join(repoDir, SESSIONS_DIR_REL);
+  let names;
+  try {
+    names = (await readdir(dir)).filter((n) => n.endsWith(".jsonl")).sort();
+  } catch {
+    return { folded: [], removed: [], skipped: 0 };
+  }
+  if (sessionId) names = names.filter((n) => n === `session-${sessionId}.jsonl`);
+
+  const folded = [];
+  const removed = [];
+  let skipped = 0;
+  for (const name of names) {
+    try {
+      const record = parseSessionJsonl(await readFile(join(dir, name), "utf8"));
+      if (!record) { skipped += 1; continue; }
+      let answers = new Map();
+      try {
+        answers = parseSessionLog(await readFile(join(repoDir, LOG_DIR_REL, `session-${record.id}.log`), "utf8"));
+      } catch { /* transcript gone — fold question-only, honestly */ }
+      const text = cleanSessionText(record, answers);
+      if (text) {
+        await saveBlock(repoDir, { id: record.id, text });
+        folded.push(record.id);
+      } else if (await removeBlock(repoDir, record.id)) {
+        removed.push(record.id); // the session re-cleaned down to nothing — stale block gone
+      } else {
+        skipped += 1;
+      }
+    } catch {
+      skipped += 1; // one bad session never fails the fold
+    }
+  }
+  return { folded, removed, skipped };
+}

package/src/sessions.mjs

@@ -24,7 +24,8 @@
 // rather than re-deriving them from source.
 
 import { mkdir, readFile, readdir, rename, writeFile } from "node:fs/promises";
-import { dirname, join } from "node:path";
+import { basename, dirname, join } from "node:path";
+import { appendUtterances } from "./memory/core.mjs";
 
 export const SESSIONS_DIR_REL = join(".tmct", "sessions");
 
@@ -144,8 +145,13 @@ export function upsertSession(entities, record) {
  *  may have replaced it mid-session), upsert, write back atomically. A MISSING
  *  artifact is the empty-graph bootstrap: seed a minimal valid payload so the
  *  conversation itself becomes the first graph write. Still throws on an invalid
- *  (unparseable) artifact — the caller treats the append as best-effort. */
-export async function appendSessionToGraph(graphFile, record) {
+ *  (unparseable) artifact — the caller treats the append as best-effort.
+ *
+ *  Signature stays backward-compatible: chat.mjs passes (graphFile, record)
+ *  exactly as before. The optional third param only tunes the MEMORY side-write
+ *  (below): `repoDir` overrides the derived repo root, `memory: false` disables
+ *  the side-write entirely. */
+export async function appendSessionToGraph(graphFile, record, { memory = true, repoDir = null } = {}) {
   let text = null;
   try {
     text = await readFile(graphFile, "utf8");
@@ -161,9 +167,82 @@ export async function appendSessionToGraph(graphFile, record) {
   }
   const res = upsertSession(entities, record);
   await atomicWriteJson(graphFile, entities);
+  // ALSO record the turn(s) into tmct's OWN memory graph (.tmct/memory/ — item 9),
+  // and fold the transcript into the text-block corpus once the session has ended.
+  // Best-effort by design: memory must never degrade the graph append that already
+  // succeeded, so every failure here is swallowed (mirrors chat.mjs's own stance).
+  if (memory) {
+    try { await recordSessionMemory(graphFile, record, repoDir); } catch { /* best-effort */ }
+  }
   return res;
 }
 
+/** Derive the repo root the memory store lives under from the graph artifact's
+ *  location: the default layout is <repo>/.tmct/graph.json. A custom
+ *  TMCT_GRAPH_FILE outside a .tmct dir has no discoverable repo (and no session
+ *  transcript/sidecar layout to read), so the memory side-write is skipped —
+ *  memory writes go ONLY under a real .tmct/, never beside arbitrary files. */
+function repoDirFromGraphFile(graphFile) {
+  const tmctDir = dirname(graphFile);
+  return basename(tmctDir) === ".tmct" ? dirname(tmctDir) : null;
+}
+
+/** The memory side-write for one session append (item 9's chat wiring, placed
+ *  HERE so chat.mjs needs no change — it already calls appendSessionToGraph
+ *  every turn). Each recorded turn becomes an a-visitor-said Utterance; the
+ *  response prose is recovered from the human transcript (the only artifact
+ *  that carries answer TEXT — the sidecar records ids) and recorded alongside
+ *  as a tmct Utterance replying to it. Deterministic utterance ids make the
+ *  per-turn replay idempotent. Once the sidecar carries its end marker (chat
+ *  writes it before the final graph upsert), the session is folded into the
+ *  text-block corpus (memory/fold.mjs). */
+async function recordSessionMemory(graphFile, record, repoDirOverride = null) {
+  const repoDir = repoDirOverride ?? repoDirFromGraphFile(graphFile);
+  if (!repoDir || !record?.id) return;
+
+  let answers = new Map();
+  try {
+    answers = parseSessionLog(await readFile(join(repoDir, ".tmct", `session-${record.id}.log`), "utf8"));
+  } catch { /* no transcript (direct API callers) — record the requests alone */ }
+
+  const utterances = [];
+  for (const t of record.turns || []) {
+    const query = String(t?.query || "");
+    const ts = String(t?.ts || "");
+    if (!query || !ts) continue;
+    // the structured parse the turn produced — stored on the visitor utterance
+    const parsed = {};
+    if (t.resolvedIds?.length) parsed.resolvedIds = t.resolvedIds;
+    if (t.answeredIds?.length) parsed.answeredIds = t.answeredIds;
+    if (t.command) parsed.command = t.command;
+    if (t.miss) parsed.miss = true;
+    utterances.push({
+      role: "visitor", text: query, ts, sessionId: record.id, sessionStarted: record.started || "",
+      ...(Object.keys(parsed).length ? { parsed } : {}),
+    });
+    const answer = answers.get(turnKey(ts, query));
+    if (answer) {
+      utterances.push({
+        role: "tmct", text: answer, ts, sessionId: record.id,
+        replyTo: `utt:${record.id}#${ts}#visitor`,
+      });
+    }
+  }
+  await appendUtterances(repoDir, utterances);
+
+  // Session over? The sidecar's end marker is authoritative (chat.mjs writes it
+  // before the final upsert). Fold THIS session's transcript into the corpus.
+  let ended = false;
+  try {
+    const sidecar = await readFile(join(repoDir, SESSIONS_DIR_REL, `session-${record.id}.jsonl`), "utf8");
+    ended = Boolean(parseSessionJsonl(sidecar)?.ended);
+  } catch { /* no sidecar — nothing to fold from */ }
+  if (ended) {
+    const { foldSessionLogs } = await import("./memory/fold.mjs"); // lazy: fold imports this module
+    await foldSessionLogs(repoDir, { sessionId: record.id });
+  }
+}
+
 /** Parse one sidecar .jsonl into a session record (null if no valid header).
  *  Torn/partial trailing lines (a killed session) are skipped, not fatal. */
 export function parseSessionJsonl(text) {
@@ -181,6 +260,10 @@ export function parseSessionJsonl(text) {
       turns.push({
         ts: String(rec.ts || ""), query: String(rec.query || ""),
         resolvedIds: arr(rec.resolvedIds), answeredIds: arr(rec.answeredIds), miss: !!rec.miss,
+        // preserved for the memory fold (memory/fold.mjs): slash-command turns and
+        // conversational filler are recorded but never folded into the corpus.
+        ...(rec.command ? { command: String(rec.command) } : {}),
+        ...(rec.conversational ? { conversational: true } : {}),
       });
     } else if (rec?.type === "end") ended = String(rec.ts || "") || ended;
   }
@@ -188,6 +271,45 @@ export function parseSessionJsonl(text) {
   return { id: String(header.id), started: String(header.started || ""), ended, turns };
 }
 
+// A transcript turn opens with an ISO-8601 ms timestamp line followed by the
+// echoed "> <query>" line (chat.mjs's logLines shape).
+const LOG_TS_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/;
+
+/** Key a transcript answer by its turn: ts + query (ts alone can collide when
+ *  two instant turns land in the same millisecond). */
+export const turnKey = (ts, query) => `${ts}${query}`;
+
+/**
+ * Parse a human-readable session transcript (.tmct/session-<id>.log) into a
+ * Map of turnKey(ts, query) → answer text. The transcript is the ONLY session
+ * artifact that carries the answer PROSE (the structured sidecar records ids,
+ * not text), so the memory write-path recovers response text from here.
+ * Tolerant by design: a block is `ts` line + "> query" line + answer lines
+ * until the next block; header/footer and torn tails just don't match.
+ */
+export function parseSessionLog(text) {
+  const lines = String(text ?? "").split("\n");
+  const answers = new Map();
+  let open = null; // { ts, query, answerLines }
+  const close = () => {
+    if (!open) return;
+    while (open.answerLines.length && !open.answerLines.at(-1).trim()) open.answerLines.pop();
+    answers.set(turnKey(open.ts, open.query), open.answerLines.join("\n"));
+    open = null;
+  };
+  for (let i = 0; i < lines.length; i += 1) {
+    if (LOG_TS_RE.test(lines[i]) && lines[i + 1]?.startsWith("> ")) {
+      close();
+      open = { ts: lines[i], query: lines[i + 1].slice(2), answerLines: [] };
+      i += 1;
+    } else if (open) {
+      open.answerLines.push(lines[i]);
+    }
+  }
+  close();
+  return answers;
+}
+
 /** All recorded sessions under <rootDir>/.tmct/sessions/*.jsonl, oldest first
  *  (uuidv7 filenames sort chronologically). Best-effort: no dir → []. */
 export async function readSessionRecords(rootDir) {

package/src/source.mjs

@@ -2,16 +2,39 @@
 // layer. The tool layer takes this as an injectable dependency (so tests can
 // stub it); in production it reads the JSON artifact the deterministic indexer
 // wrote to config.graphFile. No network, no model calls.
+//
+// This module is the PROVIDER SEAM (ROADMAP item 14, docs/adapter-contract.md):
+// any graph producer can feed tmct either by writing the entities-payload JSON
+// where config.graphFile points, or by registering a custom loader with
+// registerProvider() — no indexer is ever imported here. tmct only READS
+// through this seam; its own writes go to .tmct/memory/ (src/memory/), never
+// back into a provider's artifact.
 
 import { readFile } from "node:fs/promises";
 import { ToolError } from "./config.mjs";
 
 let cache = null; // { file, payload } — one artifact per process; cheap re-reads.
+let provider = null; // registered custom loader (config) => entities payload | Promise
 
 export function clearCache() {
   cache = null;
 }
 
+/** Register a custom graph provider: an async (or sync) `(config) => payload`
+ *  returning the entities-payload shape documented in docs/adapter-contract.md.
+ *  Pass `null` to restore the default file loader. Returns the PREVIOUS
+ *  provider (so a caller can wrap or restore it). The read cache is cleared
+ *  either way — a provider swap must never serve the old source's payload. */
+export function registerProvider(fn) {
+  if (fn != null && typeof fn !== "function") {
+    throw new TypeError("registerProvider expects a function (config) => entities payload, or null");
+  }
+  const prev = provider;
+  provider = fn ?? null;
+  cache = null;
+  return prev;
+}
+
 /** The empty-graph bootstrap payload: what a repo with no artifact "contains".
  *  Shaped exactly like a buildEntities payload so parseEntities and the session
  *  upsert treat it as a normal (just empty) graph. `bootstrap: true` marks it. */
@@ -27,12 +50,28 @@ export function emptyEntities() {
   };
 }
 
-/** Read + parse the local graph artifact. Cached per file for the process.
- *  A MISSING artifact (ENOENT) is not an error: the chat surface starts from an
- *  empty graph and the first session fold-in creates the file — so we return the
- *  bootstrap payload (uncached, so the freshly written file is picked up next
- *  fetch). Every other failure still throws a clean ToolError. */
+/** Fetch the entities payload through the provider seam. With a registered
+ *  provider, its result is returned as-is (uncached — a live provider owns its
+ *  own caching/refresh policy); a non-object result is a clean ToolError.
+ *  Default: read + parse the local graph artifact, cached per file for the
+ *  process. A MISSING artifact (ENOENT) is not an error: the chat surface
+ *  starts from an empty graph and the first session fold-in creates the file —
+ *  so we return the bootstrap payload (uncached, so the freshly written file is
+ *  picked up next fetch). Every other failure still throws a clean ToolError. */
 export async function fetchEntities(config) {
+  if (provider) {
+    let payload;
+    try {
+      payload = await provider(config);
+    } catch (e) {
+      if (e instanceof ToolError) throw e;
+      throw new ToolError(`graph provider failed (${e?.message || e})`);
+    }
+    if (!payload || typeof payload !== "object") {
+      throw new ToolError("graph provider returned no entities payload");
+    }
+    return payload;
+  }
   if (cache && cache.file === config.graphFile) return cache.payload;
   let text;
   try {