@cerefox/memory 1.0.6 → 1.0.7

package/dist/bin/cerefox.js

@@ -7184,7 +7184,7 @@ var exports_meta = {};
 __export(exports_meta, {
   PKG_VERSION: () => PKG_VERSION
 });
-var PKG_VERSION = "1.0.6";
+var PKG_VERSION = "1.0.7";
 var init_meta = () => {};
 
 // ../../node_modules/.bun/tslib@2.8.1/node_modules/tslib/tslib.js
@@ -69104,6 +69104,7 @@ async function fetchAllPages(makeQuery, batchSize = 200) {
 
 // src/cli/commands/backup.ts
 init_client();
+init_meta();
 function expandHome(path) {
   if (path === "~")
     return homedir2();
@@ -69124,12 +69125,24 @@ async function action(options) {
   const filename = `cerefox-${stamp}${options.label ? "-" + options.label : ""}.json`;
   const dest = join2(outDir, filename);
   const client = getClient();
+  let schemaVersion = "unknown";
+  try {
+    schemaVersion = await client.rpc("cerefox_schema_version", {}) ?? "unknown";
+  } catch {}
   let docs;
   try {
     docs = await fetchAllPages((from, to) => client.raw.from("cerefox_documents").select("id, title, content_hash, source, metadata, total_chars, chunk_count, " + "review_status, created_at, updated_at, deleted_at").is("deleted_at", null).order("created_at", { ascending: true }).order("id", { ascending: true }).range(from, to));
   } catch (err) {
     throw systemError(`Document fetch failed: ${err instanceof Error ? err.message : String(err)}`);
   }
+  let projects = [];
+  let memberships = [];
+  try {
+    projects = await fetchAllPages((from, to) => client.raw.from("cerefox_projects").select("id, name, description, created_at, updated_at").order("id", { ascending: true }).range(from, to));
+    memberships = await fetchAllPages((from, to) => client.raw.from("cerefox_document_projects").select("document_id, project_id").order("document_id", { ascending: true }).order("project_id", { ascending: true }).range(from, to));
+  } catch (err) {
+    throw systemError(`Project/membership fetch failed: ${err instanceof Error ? err.message : String(err)}`);
+  }
   let chunkTotal = 0;
   const enriched = [];
   for (let i = 0;i < docs.length; i++) {
@@ -69152,15 +69165,21 @@ async function action(options) {
 `);
   const payload = {
     created_at: new Date().toISOString(),
-    cerefox_version: process.env.npm_package_version ?? "unknown",
+    cerefox_version: PKG_VERSION,
+    backup_format: 2,
+    schema_version: schemaVersion,
     document_count: docs.length,
     chunk_count: chunkTotal,
+    project_count: projects.length,
+    membership_count: memberships.length,
+    projects,
+    memberships,
     documents: enriched
   };
   writeFileSync(dest, JSON.stringify(payload, null, 2), "utf8");
   println("");
   println(c.green("✓ ") + `Backup written: ${dest}`);
-  println(c.dim(`  documents: ${docs.length} · chunks: ${chunkTotal}`));
+  println(c.dim(`  documents: ${docs.length} · chunks: ${chunkTotal} · ` + `projects: ${projects.length} · memberships: ${memberships.length}`));
   if (options.git) {
     println(c.yellow("⚠ ") + "--git commit is not implemented; the snapshot was written without a git checkpoint.");
     println(c.dim("  Commit the backup directory yourself if you want it version-controlled."));
@@ -75020,7 +75039,7 @@ import { homedir as homedir6 } from "node:os";
 import { join as join9 } from "node:path";
 
 // ../../_shared/ef-meta/index.ts
-var EF_VERSION = "1.0.6";
+var EF_VERSION = "1.0.7";
 var EF_LAST_CHANGED = "1.0.6";
 
 // src/cli/util/checks.ts
@@ -75370,7 +75389,7 @@ async function checkContentFormat() {
       name: CONTENT_FORMAT_CHECK_NAME,
       status: "skipped",
       detail: `${legacy} of ${total} document(s) use the legacy reconstruction format (format 1).`,
-      hint: "They auto-convert on next edit; run `cerefox server reindex` to convert all now. What this means: `cerefox guides show content-format`."
+      hint: "They auto-convert on next edit; run `cerefox server migrate-format` to convert all now (re-embeds; `--dry-run` first). What this means: `cerefox guides show content-format`."
     };
   } catch (err) {
     return {
@@ -77381,6 +77400,114 @@ function registerReindex(program2) {
   program2.command("reindex").description("Re-embed existing document chunks (v0.7+).").option("--all", "Reindex every chunk regardless of embedder.").option("--batch <n>", "Chunks per OpenAI batch call. Capped at 96 internally.", "32").option("--dry-run", "Show counts without re-embedding.").option("-i, --document-id <uuid>", "Limit reindex to a single document.").action(action27);
 }
 
+// src/cli/commands/migrate-format.ts
+init_cli_core();
+init_config();
+init_client();
+var CURRENT_FORMAT = 2;
+async function action28(options) {
+  const settings = loadSettings();
+  const client = getClient();
+  const supabase = client.raw;
+  let legacyChunkRows;
+  try {
+    legacyChunkRows = await fetchAllPages((from, to) => {
+      let q = supabase.from("cerefox_chunks").select("document_id").is("version_id", null).lt("content_format", CURRENT_FORMAT);
+      if (options.documentId)
+        q = q.eq("document_id", options.documentId);
+      return q.order("document_id", { ascending: true }).range(from, to);
+    }, 1000);
+  } catch (err) {
+    throw systemError(`Could not list legacy chunks: ${err instanceof Error ? err.message : String(err)}`);
+  }
+  const docIds = [...new Set(legacyChunkRows.map((r) => r.document_id))];
+  const limit = options.limit ? parsePositiveInt(options.limit, "--limit", docIds.length) : docIds.length;
+  const targets = docIds.slice(0, limit);
+  if (targets.length === 0) {
+    println(c.green("✓ Nothing to migrate — every document already uses the current format."));
+    return;
+  }
+  println(c.bold(`${docIds.length} document(s) on the legacy format` + (targets.length < docIds.length ? `; converting ${targets.length} (--limit)` : "")));
+  if (options.dryRun) {
+    println(c.yellow("⚠  --dry-run: nothing was written."));
+    println(c.dim("   Each document would be re-chunked and RE-EMBEDDED (embedding spend)."));
+    return;
+  }
+  println(c.dim("Each document is re-chunked and re-embedded — this costs embedding spend."));
+  println("");
+  const author = resolveAuthor(options.author);
+  const authorType = resolveAuthorType(undefined);
+  const pipeline2 = new IngestionPipeline({
+    supabase,
+    openAiApiKey: settings.openaiApiKey
+  });
+  let converted = 0;
+  let skipped = 0;
+  const duplicates = [];
+  const failures = [];
+  for (let i = 0;i < targets.length; i++) {
+    const id = targets[i];
+    if (process.stdout.isTTY) {
+      process.stderr.write(`\r  Converting ${i + 1}/${targets.length}…`);
+    }
+    let doc2 = null;
+    try {
+      const rows = await client.rpc("cerefox_get_document", { p_document_id: id, p_version_id: null });
+      doc2 = rows?.[0] ?? null;
+    } catch (err) {
+      failures.push({ document: id, reason: `read: ${err instanceof Error ? err.message : String(err)}` });
+      continue;
+    }
+    if (!doc2) {
+      failures.push({ document: id, reason: "read: document not found" });
+      continue;
+    }
+    try {
+      await pipeline2.ingestText({
+        text: doc2.full_content,
+        title: doc2.doc_title,
+        documentId: id,
+        source: "migrate-format",
+        author,
+        authorType,
+        expectedContentHash: doc2.content_hash
+      });
+      converted++;
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      if (/conflict/i.test(message)) {
+        skipped++;
+      } else if (/identical content already exists/i.test(message)) {
+        duplicates.push({ document: `${doc2.doc_title} (${id})`, reason: message });
+      } else {
+        failures.push({ document: `${doc2.doc_title} (${id})`, reason: message });
+      }
+    }
+  }
+  if (process.stdout.isTTY)
+    process.stderr.write(`
+`);
+  println("");
+  println(c.bold(`Converted ${converted} · skipped ${skipped} (changed mid-run) · failed ${failures.length}`));
+  if (targets.length < docIds.length) {
+    println(c.dim(`  ${docIds.length - targets.length} document(s) still pending — re-run to continue.`));
+  }
+  if (duplicates.length > 0) {
+    println("");
+    println(c.yellow(`⚠ ${duplicates.length} document(s) could not be converted because their content is ` + "byte-identical to another document."));
+    println(c.dim("  Re-ingesting them would collide with the content-hash dedup check. They keep working " + "on the legacy format; de-duplicate them if you want them converted."));
+    printTable(duplicates.map((d) => ({ document: d.document })));
+  }
+  if (failures.length > 0) {
+    println("");
+    printTable(failures);
+    throw systemError(`${failures.length} document(s) failed to convert.`);
+  }
+}
+function registerMigrateFormat(program2) {
+  program2.command("migrate-format").description("Convert legacy-format documents to the current chunk format (re-chunks + re-embeds).").option("--dry-run", "Report how many documents would be converted; write nothing.").option("-l, --limit <n>", "Convert at most N documents (re-run to continue).").option("--document-id <uuid>", "Convert a single document.").option("--author <name>", "Recorded in the audit log for each conversion.").action(action28);
+}
+
 // src/cli/commands/restore.ts
 init_cli_core();
 init_client();
@@ -77408,7 +77535,7 @@ function resolveBackupFile(target) {
   }
   return join12(path, candidates[0].name);
 }
-async function action28(target, options) {
+async function action29(target, options) {
   const file = resolveBackupFile(target);
   let payload;
   try {
@@ -77420,7 +77547,13 @@ async function action28(target, options) {
     throw userError(`Backup file is missing "documents" array: ${file}`);
   }
   println(c.bold(`Restoring from ${file}`));
-  println(c.dim(`  cerefox_version: ${payload.cerefox_version ?? "?"} · ` + `documents in file: ${payload.documents.length} · chunks in file: ${payload.chunk_count ?? "?"}`));
+  const hasMemberships = Array.isArray(payload.memberships);
+  println(c.dim(`  cerefox_version: ${payload.cerefox_version ?? "?"} · ` + `schema: ${payload.schema_version ?? "?"} · ` + `documents in file: ${payload.documents.length} · chunks in file: ${payload.chunk_count ?? "?"}`));
+  if (hasMemberships) {
+    println(c.dim(`  projects: ${payload.projects?.length ?? 0} · memberships: ${payload.memberships?.length ?? 0}`));
+  } else {
+    warn("This backup predates project-membership capture (format 1) — documents " + "will be restored WITHOUT their project assignments.");
+  }
   println("");
   const client = getClient();
   let restored = 0;
@@ -77455,8 +77588,47 @@ async function action28(target, options) {
     }
     restored++;
   }
+  let projectsRestored = 0;
+  let membershipsRestored = 0;
+  if (!options.dryRun && hasMemberships) {
+    const projects = payload.projects ?? [];
+    if (projects.length > 0) {
+      const { error: projErr } = await client.raw.from("cerefox_projects").upsert(projects, { onConflict: "id", ignoreDuplicates: true });
+      if (projErr) {
+        errors4++;
+        errorDetails.push({ title: "(projects)", error: projErr.message });
+      } else {
+        projectsRestored = projects.length;
+      }
+    }
+    const presentDocIds = new Set;
+    {
+      const ids = (payload.documents ?? []).map((d) => d.id);
+      for (let i = 0;i < ids.length; i += 200) {
+        const { data } = await client.raw.from("cerefox_documents").select("id").in("id", ids.slice(i, i + 200));
+        for (const row of data ?? [])
+          presentDocIds.add(row.id);
+      }
+    }
+    const links = (payload.memberships ?? []).filter((m) => presentDocIds.has(m.document_id));
+    for (let i = 0;i < links.length; i += 500) {
+      const { error: linkErr } = await client.raw.from("cerefox_document_projects").upsert(links.slice(i, i + 500), {
+        onConflict: "document_id,project_id",
+        ignoreDuplicates: true
+      });
+      if (linkErr) {
+        errors4++;
+        errorDetails.push({ title: "(memberships)", error: linkErr.message });
+        break;
+      }
+      membershipsRestored += links.slice(i, i + 500).length;
+    }
+  }
   println("");
   println((options.dryRun ? c.yellow("(dry-run) ") : "") + c.bold(`Summary: ${restored} restored · ${skipped} skipped · ${errors4} errors`));
+  if (hasMemberships && !options.dryRun) {
+    println(c.dim(`  projects: ${projectsRestored} · memberships: ${membershipsRestored}`));
+  }
   if (errors4 > 0) {
     println("");
     printTable(errorDetails);
@@ -77464,7 +77636,7 @@ async function action28(target, options) {
   }
 }
 function registerRestore(program2) {
-  program2.command("restore").description("Restore a JSON-snapshot backup into the knowledge base.").argument("<snapshot>", "Backup file (or directory; most recent is picked) produced by `cerefox backup`.").option("--dry-run", "Print what would be restored without writing.").option("-p, --project-name <name>", "Reserved for future use; currently ignored (project memberships ride along with each doc's metadata).").action(action28);
+  program2.command("restore").description("Restore a JSON-snapshot backup into the knowledge base.").argument("<snapshot>", "Backup file (or directory; most recent is picked) produced by `cerefox backup`.").option("--dry-run", "Print what would be restored without writing.").option("-p, --project-name <name>", "Reserved for future use; currently ignored. Project memberships are restored from the backup itself (format 2+).").action(action29);
 }
 
 // src/cli/commands/search.ts
@@ -77489,7 +77661,7 @@ async function embedQuery(query) {
 }
 
 // src/cli/commands/search.ts
-async function action29(query, options) {
+async function action30(query, options) {
   if (!query || query.trim() === "") {
     throw userError("Empty query.");
   }
@@ -77653,7 +77825,7 @@ async function action29(query, options) {
   }
 }
 function registerSearch(program2) {
-  program2.command("search").description("Search the knowledge base (hybrid FTS + semantic).").argument("<query>", "Natural-language search query.").option("-c, --match-count <n>", "Maximum number of documents to return.", "5").option("-p, --project-name <name>", "Filter results to a specific project.").option("-f, --metadata-filter <json>", "JSON containment filter; only docs whose metadata contains ALL pairs are returned.").option("--mode <mode>", "Search mode: docs (default), hybrid, fts.", "docs").option("--alpha <float>", "Semantic weight 0..1 (default: CEREFOX_SEARCH_ALPHA; else 0.7).").option("--min-score <float>", "Minimum cosine similarity threshold (default: CEREFOX_MIN_SEARCH_SCORE; else 0.5, or 0.6 with the local embedder).").option("--min-term-coverage <float>", "OR-fallback keyword matches must cover at least this fraction of the query's meaningful terms to count as confident hits (default: CEREFOX_MIN_TERM_COVERAGE; else the server default 0.5; needs schema ≥ 0.9.1).").option("--max-bytes <n>", "Response size budget in bytes (default: CEREFOX_MAX_RESPONSE_BYTES or 200000).").option("-r, --requestor <name>", "Agent / user name (recorded in usage log).").option("--json", "Emit machine-readable JSON instead of the default text.").option("--only-metadata", "List matching docs (id, score, chunks, chars, partial/full) WITHOUT their content — like the web UI's collapsed result list. Grab a [id:…] then `cerefox document get <id>`.").action(action29);
+  program2.command("search").description("Search the knowledge base (hybrid FTS + semantic).").argument("<query>", "Natural-language search query.").option("-c, --match-count <n>", "Maximum number of documents to return.", "5").option("-p, --project-name <name>", "Filter results to a specific project.").option("-f, --metadata-filter <json>", "JSON containment filter; only docs whose metadata contains ALL pairs are returned.").option("--mode <mode>", "Search mode: docs (default), hybrid, fts.", "docs").option("--alpha <float>", "Semantic weight 0..1 (default: CEREFOX_SEARCH_ALPHA; else 0.7).").option("--min-score <float>", "Minimum cosine similarity threshold (default: CEREFOX_MIN_SEARCH_SCORE; else 0.5, or 0.6 with the local embedder).").option("--min-term-coverage <float>", "OR-fallback keyword matches must cover at least this fraction of the query's meaningful terms to count as confident hits (default: CEREFOX_MIN_TERM_COVERAGE; else the server default 0.5; needs schema ≥ 0.9.1).").option("--max-bytes <n>", "Response size budget in bytes (default: CEREFOX_MAX_RESPONSE_BYTES or 200000).").option("-r, --requestor <name>", "Agent / user name (recorded in usage log).").option("--json", "Emit machine-readable JSON instead of the default text.").option("--only-metadata", "List matching docs (id, score, chunks, chars, partial/full) WITHOUT their content — like the web UI's collapsed result list. Grab a [id:…] then `cerefox document get <id>`.").action(action30);
 }
 
 // src/cli/commands/self-update.ts
@@ -77700,7 +77872,7 @@ async function fetchLatestVersion() {
   }
   return body.version;
 }
-async function action30(options) {
+async function action31(options) {
   let target;
   try {
     target = options.version ?? await fetchLatestVersion();
@@ -77752,7 +77924,7 @@ async function action30(options) {
 }
 function registerSelfUpdate(program2) {
   const desc = "Upgrade Cerefox in place. Alias: `cerefox upgrade`.";
-  const declaration = (cmd) => cmd.description(desc).option("--check", "Print current vs latest; do nothing.").option("--yes", "Non-interactive (skip confirmation).").option("--version <version>", "Pin a specific version (e.g. 0.5.1 or 0.6.0-rc.1).").action(action30);
+  const declaration = (cmd) => cmd.description(desc).option("--check", "Print current vs latest; do nothing.").option("--yes", "Non-interactive (skip confirmation).").option("--version <version>", "Pin a specific version (e.g. 0.5.1 or 0.6.0-rc.1).").action(action31);
   declaration(program2.command("self-update"));
   declaration(program2.command("upgrade"));
 }
@@ -77771,7 +77943,7 @@ function symbol2(status) {
       return cErr.dim("ℹ");
   }
 }
-async function action31(options) {
+async function action32(options) {
   const useSpinner = !options.json && process.stderr.isTTY;
   const spinner = useSpinner ? ora({ text: "Starting checks…", spinner: "dots", stream: process.stderr }).start() : null;
   const results = await runFastChecks({
@@ -77792,7 +77964,7 @@ async function action31(options) {
   }
 }
 function registerStatus(program2) {
-  program2.command("status").description("Quick sanity check (fast subset of `cerefox doctor`).").option("--json", "Emit machine-readable JSON.").action(action31);
+  program2.command("status").description("Quick sanity check (fast subset of `cerefox doctor`).").option("--json", "Emit machine-readable JSON.").action(action32);
 }
 
 // src/cli/commands/token.ts
@@ -77825,10 +77997,10 @@ function upsertEnvVar(path, key, value, opts = {}) {
   }
   const re = new RegExp(`^(\\s*)${escapeRegExp(key)}=.*$`, "m");
   let next;
-  let action32;
+  let action33;
   if (re.test(original)) {
     next = original.replace(re, `$1${line}`);
-    action32 = "updated";
+    action33 = "updated";
   } else {
     const base = original.endsWith(`
 `) ? original : `${original}
@@ -77836,10 +78008,10 @@ function upsertEnvVar(path, key, value, opts = {}) {
     next = `${base}
 ${header}${line}
 `;
-    action32 = "added";
+    action33 = "added";
   }
   writeFileSync5(path, next);
-  return { path, action: action32, backupPath };
+  return { path, action: action33, backupPath };
 }
 function readEnvVar(path, key) {
   if (!existsSync14(path))
@@ -83451,6 +83623,7 @@ Learn more:
   const server = program2.command("server").description("Server side: deploy, reindex.");
   moveInto(server, registerDeployServer, "deploy");
   moveInto(server, registerReindex, "reindex");
+  registerMigrateFormat(server);
   const guides = program2.command("guides").description("Bundled docs: list, open, show, ingest (into the KB).");
   registerGuides(guides);
   moveInto(guides, registerSyncSelfDocs, "ingest");

package/dist/server-assets/_shared/ef-meta/index.ts

@@ -18,7 +18,7 @@
  * doesn't touch `supabase/functions/` leaves it alone).
  */
 
-export const EF_VERSION = "1.0.6";
+export const EF_VERSION = "1.0.7";
 
 /**
  * The most recent version whose EF-side SOURCE actually changed (#127).

package/docs/guides/content-format.md

@@ -40,7 +40,14 @@ in — even after the current version has moved to format 2.
 - A document **moves to format 2 automatically the next time it is edited/saved**
   (it gets re-chunked by the new chunker).
 - If you want to convert everything now rather than on next edit, run
-  `cerefox server reindex` (re-chunks + re-embeds the whole knowledge base).
+  `cerefox server migrate-format`. It re-ingests each legacy document through
+  the normal pipeline (re-chunk + re-embed + stamp the current format), which
+  costs embedding spend — so it is opt-in, supports `--dry-run` and `--limit`,
+  and skips any document that changes mid-run rather than overwriting it.
+
+  > **Not `cerefox server reindex`.** Reindex refreshes *embeddings* on the
+  > existing chunk rows; it never re-chunks, so it cannot advance the stored
+  > format. Earlier versions of this guide said otherwise (#164).
 
 `cerefox doctor` reports how many documents still use the legacy format — purely
 informational, never a failure. A fresh install shows zero.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerefox/memory",
-  "version": "1.0.6",
+  "version": "1.0.7",
   "description": "Cerefox — user-owned shared memory for AI agents. CLI + stdio MCP server + web UI + ingestion for a knowledge base on your own Supabase project (or fully self-hosted with Cerefox Local).",
   "license": "Apache-2.0",
   "homepage": "https://github.com/fstamatelopoulos/cerefox",