@cerefox/memory 1.1.0-beta.3 → 1.1.0-beta.4

package/dist/bin/cerefox.js

@@ -7438,7 +7438,7 @@ var exports_meta = {};
 __export(exports_meta, {
   PKG_VERSION: () => PKG_VERSION
 });
-var PKG_VERSION = "1.1.0-beta.3";
+var PKG_VERSION = "1.1.0-beta.4";
 var init_meta = () => {};
 
 // ../../_shared/config/paths.ts
@@ -75712,11 +75712,10 @@ function checkConfig() {
       modeDetail = ` (mode 0${mode.toString(8)})`;
     } catch {}
   }
-  const envLabel = (process.env.CEREFOX_ENV_LABEL ?? "").trim();
   return {
     name: "config",
     status: "ok",
-    detail: `${envPath}${modeDetail}${envLabel ? ` [${envLabel.toUpperCase()}]` : ""}`
+    detail: `${envPath}${modeDetail}`
   };
 }
 async function checkSupabase() {
@@ -76231,7 +76230,8 @@ async function action15(options) {
   if (options.json) {
     printJson(results);
   } else {
-    println("Cerefox doctor");
+    const envLabel = (process.env.CEREFOX_ENV_LABEL ?? "").trim();
+    println(envLabel ? `Cerefox doctor ${c.yellow(`[${envLabel.toUpperCase()}]`)}` : "Cerefox doctor");
     println("");
     const nameWidth = Math.max(...results.map((r) => r.name.length));
     for (const r of results) {
@@ -76793,7 +76793,8 @@ class IngestionPipeline {
       author = "unknown",
       authorType = "user",
       expectedContentHash,
-      lastWriteWins = false
+      lastWriteWins = false,
+      forceRechunk = false
     } = opts;
     const listFormProvided = projectIds !== undefined && projectIds !== null || projectNames !== undefined && projectNames !== null;
     const getOrCreate = (name) => this.db.getOrCreateProject(name);
@@ -76816,7 +76817,8 @@ class IngestionPipeline {
         author,
         authorType,
         expectedContentHash,
-        lastWriteWins
+        lastWriteWins,
+        forceRechunk
       });
       if (!listFormProvided && (projectId || projectName)) {
         const singular = await resolveProjectIds({ projectId, projectName }, getOrCreate);
@@ -76941,7 +76943,8 @@ class IngestionPipeline {
       author = "unknown",
       authorType = "user",
       expectedContentHash,
-      lastWriteWins = false
+      lastWriteWins = false,
+      forceRechunk = false
     } = opts;
     const existing = await this.db.getDocumentById(documentId);
     if (!existing) {
@@ -76966,7 +76969,7 @@ class IngestionPipeline {
     }
     const actualChunks = await this.db.listChunksForDocument(documentId);
     const hasChunks = actualChunks.length > 0;
-    if (contentUnchanged && hasChunks) {
+    if (contentUnchanged && hasChunks && !forceRechunk) {
       const oldTitle = existing.title ?? "";
       const titleChanged = oldTitle !== title;
       const updates = { title };
@@ -77053,7 +77056,7 @@ class IngestionPipeline {
       sourceLabel: source,
       retentionHours: this.settings.versionRetentionHours,
       cleanupEnabled: this.settings.versionCleanupEnabled,
-      expectedContentHash: expectedContentHash ?? null,
+      expectedContentHash: expectedContentHash ?? (forceRechunk && contentUnchanged ? existing.content_hash : null),
       lastWriteWins
     });
     let finalProjectIds;
@@ -78032,6 +78035,25 @@ function registerMetadataSearch(program2) {
 init_dist4();
 init_cli_core();
 init_config();
+
+// src/cli/util/bulk-write-warning.ts
+init_cli_core();
+function warnLargeBulkWrite(opts) {
+  if (opts.count < opts.threshold)
+    return false;
+  println("");
+  println(c.yellow("⚠ ") + `${opts.count.toLocaleString()} ${opts.unit}(s) is a large bulk rewrite.`);
+  println(c.dim(`  On Supabase (especially the free tier and small compute add-ons) this can
+` + `  deplete the project's Disk IO Budget. Symptoms: slower responses, CPU
+` + `  climbing on IO wait, and in the worst case a briefly unresponsive
+` + "  instance. It recovers on its own once the budget refills."));
+  println(c.dim(`  Gentler: ${opts.batchHint}, leaving time between runs and ideally
+` + "  picking a quiet period. The work is resumable — re-run to continue."));
+  println("");
+  return true;
+}
+
+// src/cli/commands/reindex.ts
 async function action27(options) {
   const settings = loadSettings();
   if (!settings.supabaseUrl || !settings.supabaseKey) {
@@ -78070,6 +78092,12 @@ async function action27(options) {
     return;
   }
   println(c.bold(`Reindexing ${chunks.length} chunk(s) ${reindexAll ? "(--all)" : "(stale only)"}${dryRun ? " — DRY RUN" : ""}`));
+  warnLargeBulkWrite({
+    count: chunks.length,
+    threshold: 1000,
+    unit: "chunk",
+    batchHint: "reindex in stages with --document-id"
+  });
   if (dryRun) {
     const byDoc = new Map;
     for (const ch of chunks)
@@ -78139,7 +78167,16 @@ async function action28(options) {
   } catch (err) {
     throw systemError(`Could not list legacy chunks: ${err instanceof Error ? err.message : String(err)}`);
   }
-  const docIds = [...new Set(legacyChunkRows.map((r) => r.document_id))];
+  let liveIds = new Set;
+  try {
+    const liveRows = await fetchAllPages((from, to) => supabase.from("cerefox_documents").select("id").is("deleted_at", null).order("id", { ascending: true }).range(from, to));
+    liveIds = new Set(liveRows.map((r) => r.id));
+  } catch (err) {
+    throw systemError(`Could not list documents: ${err instanceof Error ? err.message : String(err)}`);
+  }
+  const allLegacy = [...new Set(legacyChunkRows.map((r) => r.document_id))];
+  const docIds = allLegacy.filter((id) => liveIds.has(id));
+  const trashedSkipped = allLegacy.length - docIds.length;
   const limit = options.limit ? parsePositiveInt(options.limit, "--limit", docIds.length) : docIds.length;
   const targets = docIds.slice(0, limit);
   if (targets.length === 0) {
@@ -78147,6 +78184,15 @@ async function action28(options) {
     return;
   }
   println(c.bold(`${docIds.length} document(s) on the legacy format` + (targets.length < docIds.length ? `; converting ${targets.length} (--limit)` : "")));
+  if (trashedSkipped > 0) {
+    println(c.dim(`  (${trashedSkipped} trashed document(s) on the legacy format ignored)`));
+  }
+  warnLargeBulkWrite({
+    count: targets.length,
+    threshold: 500,
+    unit: "document",
+    batchHint: "run it in batches with --limit 200"
+  });
   if (options.dryRun) {
     println(c.yellow("⚠  --dry-run: nothing was written."));
     println(c.dim("   Each document would be re-chunked and RE-EMBEDDED (embedding spend)."));
@@ -78182,16 +78228,24 @@ async function action28(options) {
       continue;
     }
     try {
-      await pipeline2.ingestText({
+      const result = await pipeline2.ingestText({
         text: doc2.full_content,
         title: doc2.doc_title,
         documentId: id,
         source: "migrate-format",
         author,
         authorType,
-        expectedContentHash: doc2.content_hash
+        expectedContentHash: doc2.content_hash,
+        forceRechunk: true
       });
-      converted++;
+      if (result.reindexed) {
+        converted++;
+      } else {
+        failures.push({
+          document: `${doc2.doc_title} (${id})`,
+          reason: `pipeline reported no re-chunk (action=${result.action}); format not advanced`
+        });
+      }
     } catch (err) {
       const message = err instanceof Error ? err.message : String(err);
       if (/conflict/i.test(message)) {

package/docs/guides/configuration.md

@@ -265,7 +265,7 @@ cerefox document get <document-id> --version-id <version-id>
 | Variable | Default | Description |
 |----------|---------|-------------|
 | `CEREFOX_BACKUP_DIR` | `~/.cerefox/backups` | Local directory where file system backups are stored. Created automatically if it doesn't exist. **Use an absolute path** — a relative value (such as the pre-v0.3.0 `./backups`) resolves against the current working directory, so snapshots scatter depending on where you run the command; `backup create` warns when it sees one. Does **not** follow `CEREFOX_CONFIG_DIR`, so a second environment must set it explicitly. |
-| `CEREFOX_ENV_LABEL` | _(unset)_ | Names a non-production environment (e.g. `staging`). Purely cosmetic and inert when unset. When set: the web UI shows a banner on every page, `doctor` appends `[LABEL]` to its config line, `backup create` puts the label in the snapshot filename and payload, and `backup restore` warns when a snapshot's environment differs from the target's. See [`staging-env.md`](staging-env.md). |
+| `CEREFOX_ENV_LABEL` | _(unset)_ | Names a non-production environment (e.g. `staging`). Purely cosmetic and inert when unset. When set: the web UI shows a banner on every page, `doctor` shows `[LABEL]` on its title line, `backup create` puts the label in the snapshot filename and payload, and `backup restore` warns when a snapshot's environment differs from the target's. See [`staging-env.md`](staging-env.md). |
 | `CEREFOX_VERSION_RETENTION_HOURS` | `48` | How long to retain archived document versions (hours). The most recent version is always kept regardless of this setting. |
 
 ---

package/docs/guides/content-format.md

@@ -49,6 +49,24 @@ in — even after the current version has moved to format 2.
   > existing chunk rows; it never re-chunks, so it cannot advance the stored
   > format. Earlier versions of this guide said otherwise (#164).
 
+  > **Large stores: batch it.** Converting a few hundred documents or more is a
+  > heavy bulk rewrite — each one is re-chunked, its previous chunks archived as
+  > a version snapshot, and new rows inserted. A contributor reindexing a
+  > ~1,300-document store on Supabase depleted the project's **Disk IO Budget**
+  > and got a warning email: slower responses, CPU climbing on IO wait, and
+  > potentially a briefly unresponsive instance (it recovers once the budget
+  > refills). Nothing is corrupted, but prefer `--limit 200` in stages during a
+  > quiet period. The command warns above 500 documents and is resumable.
+
+  > **Requires v1.1.0-beta.4 or later.** On **v1.0.7 through v1.1.0-beta.3**
+  > `migrate-format` printed `Converted N` while converting nothing: it
+  > re-ingests byte-identical content by design, and the pipeline answered an
+  > unchanged content hash with a metadata-only update — no re-chunk, so no
+  > format advance. If you ran it on one of those versions, the documents are
+  > untouched and still on format 1 (no data was harmed and no embedding spend
+  > was incurred); re-run it on a current build. Check with `cerefox doctor`,
+  > which reports the real count.
+
 `cerefox doctor` reports how many documents still use the legacy format — purely
 informational, never a failure. A fresh install shows zero.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerefox/memory",
-  "version": "1.1.0-beta.3",
+  "version": "1.1.0-beta.4",
   "description": "Cerefox — user-owned shared memory for AI agents. CLI + stdio MCP server + web UI + ingestion for a knowledge base on your own Supabase project (or fully self-hosted with Cerefox Local).",
   "license": "Apache-2.0",
   "homepage": "https://github.com/fstamatelopoulos/cerefox",