@cerefox/memory 1.1.0-beta.3 → 1.1.0-beta.5

package/dist/bin/cerefox.js

@@ -7438,7 +7438,7 @@ var exports_meta = {};
 __export(exports_meta, {
   PKG_VERSION: () => PKG_VERSION
 });
-var PKG_VERSION = "1.1.0-beta.3";
+var PKG_VERSION = "1.1.0-beta.5";
 var init_meta = () => {};
 
 // ../../_shared/config/paths.ts
@@ -75712,11 +75712,10 @@ function checkConfig() {
       modeDetail = ` (mode 0${mode.toString(8)})`;
     } catch {}
   }
-  const envLabel = (process.env.CEREFOX_ENV_LABEL ?? "").trim();
   return {
     name: "config",
     status: "ok",
-    detail: `${envPath}${modeDetail}${envLabel ? ` [${envLabel.toUpperCase()}]` : ""}`
+    detail: `${envPath}${modeDetail}`
   };
 }
 async function checkSupabase() {
@@ -76231,7 +76230,8 @@ async function action15(options) {
   if (options.json) {
     printJson(results);
   } else {
-    println("Cerefox doctor");
+    const envLabel = (process.env.CEREFOX_ENV_LABEL ?? "").trim();
+    println(envLabel ? `Cerefox doctor ${c.yellow(`[${envLabel.toUpperCase()}]`)}` : "Cerefox doctor");
     println("");
     const nameWidth = Math.max(...results.map((r) => r.name.length));
     for (const r of results) {
@@ -76793,7 +76793,8 @@ class IngestionPipeline {
       author = "unknown",
       authorType = "user",
       expectedContentHash,
-      lastWriteWins = false
+      lastWriteWins = false,
+      forceRechunk = false
     } = opts;
     const listFormProvided = projectIds !== undefined && projectIds !== null || projectNames !== undefined && projectNames !== null;
     const getOrCreate = (name) => this.db.getOrCreateProject(name);
@@ -76816,7 +76817,8 @@ class IngestionPipeline {
         author,
         authorType,
         expectedContentHash,
-        lastWriteWins
+        lastWriteWins,
+        forceRechunk
       });
       if (!listFormProvided && (projectId || projectName)) {
         const singular = await resolveProjectIds({ projectId, projectName }, getOrCreate);
@@ -76941,7 +76943,8 @@ class IngestionPipeline {
       author = "unknown",
       authorType = "user",
       expectedContentHash,
-      lastWriteWins = false
+      lastWriteWins = false,
+      forceRechunk = false
     } = opts;
     const existing = await this.db.getDocumentById(documentId);
     if (!existing) {
@@ -76966,7 +76969,7 @@ class IngestionPipeline {
     }
     const actualChunks = await this.db.listChunksForDocument(documentId);
     const hasChunks = actualChunks.length > 0;
-    if (contentUnchanged && hasChunks) {
+    if (contentUnchanged && hasChunks && !forceRechunk) {
       const oldTitle = existing.title ?? "";
       const titleChanged = oldTitle !== title;
       const updates = { title };
@@ -77053,7 +77056,7 @@ class IngestionPipeline {
       sourceLabel: source,
       retentionHours: this.settings.versionRetentionHours,
       cleanupEnabled: this.settings.versionCleanupEnabled,
-      expectedContentHash: expectedContentHash ?? null,
+      expectedContentHash: expectedContentHash ?? (forceRechunk && contentUnchanged ? existing.content_hash : null),
       lastWriteWins
     });
     let finalProjectIds;
@@ -78032,6 +78035,25 @@ function registerMetadataSearch(program2) {
 init_dist4();
 init_cli_core();
 init_config();
+
+// src/cli/util/bulk-write-warning.ts
+init_cli_core();
+function warnLargeBulkWrite(opts) {
+  if (opts.count < opts.threshold)
+    return false;
+  println("");
+  println(c.yellow("⚠ ") + `${opts.count.toLocaleString()} ${opts.unit}(s) is a large bulk rewrite.`);
+  println(c.dim(`  On Supabase (especially the free tier and small compute add-ons) this can
+` + `  deplete the project's Disk IO Budget. Symptoms: slower responses, CPU
+` + `  climbing on IO wait, and in the worst case a briefly unresponsive
+` + "  instance. It recovers on its own once the budget refills."));
+  println(c.dim(`  Gentler: ${opts.batchHint}, leaving time between runs and ideally
+` + "  picking a quiet period. The work is resumable — re-run to continue."));
+  println("");
+  return true;
+}
+
+// src/cli/commands/reindex.ts
 async function action27(options) {
   const settings = loadSettings();
   if (!settings.supabaseUrl || !settings.supabaseKey) {
@@ -78070,6 +78092,12 @@ async function action27(options) {
     return;
   }
   println(c.bold(`Reindexing ${chunks.length} chunk(s) ${reindexAll ? "(--all)" : "(stale only)"}${dryRun ? " — DRY RUN" : ""}`));
+  warnLargeBulkWrite({
+    count: chunks.length,
+    threshold: 1000,
+    unit: "chunk",
+    batchHint: "reindex in stages with --document-id"
+  });
   if (dryRun) {
     const byDoc = new Map;
     for (const ch of chunks)
@@ -78139,7 +78167,16 @@ async function action28(options) {
   } catch (err) {
     throw systemError(`Could not list legacy chunks: ${err instanceof Error ? err.message : String(err)}`);
   }
-  const docIds = [...new Set(legacyChunkRows.map((r) => r.document_id))];
+  let liveIds = new Set;
+  try {
+    const liveRows = await fetchAllPages((from, to) => supabase.from("cerefox_documents").select("id").is("deleted_at", null).order("id", { ascending: true }).range(from, to));
+    liveIds = new Set(liveRows.map((r) => r.id));
+  } catch (err) {
+    throw systemError(`Could not list documents: ${err instanceof Error ? err.message : String(err)}`);
+  }
+  const allLegacy = [...new Set(legacyChunkRows.map((r) => r.document_id))];
+  const docIds = allLegacy.filter((id) => liveIds.has(id));
+  const trashedSkipped = allLegacy.length - docIds.length;
   const limit = options.limit ? parsePositiveInt(options.limit, "--limit", docIds.length) : docIds.length;
   const targets = docIds.slice(0, limit);
   if (targets.length === 0) {
@@ -78147,6 +78184,15 @@ async function action28(options) {
     return;
   }
   println(c.bold(`${docIds.length} document(s) on the legacy format` + (targets.length < docIds.length ? `; converting ${targets.length} (--limit)` : "")));
+  if (trashedSkipped > 0) {
+    println(c.dim(`  (${trashedSkipped} trashed document(s) on the legacy format ignored)`));
+  }
+  warnLargeBulkWrite({
+    count: targets.length,
+    threshold: 200,
+    unit: "document",
+    batchHint: "run it in batches with --limit 100"
+  });
   if (options.dryRun) {
     println(c.yellow("⚠  --dry-run: nothing was written."));
     println(c.dim("   Each document would be re-chunked and RE-EMBEDDED (embedding spend)."));
@@ -78182,16 +78228,24 @@ async function action28(options) {
       continue;
     }
     try {
-      await pipeline2.ingestText({
+      const result = await pipeline2.ingestText({
         text: doc2.full_content,
         title: doc2.doc_title,
         documentId: id,
         source: "migrate-format",
         author,
         authorType,
-        expectedContentHash: doc2.content_hash
+        expectedContentHash: doc2.content_hash,
+        forceRechunk: true
       });
-      converted++;
+      if (result.reindexed) {
+        converted++;
+      } else {
+        failures.push({
+          document: `${doc2.doc_title} (${id})`,
+          reason: `pipeline reported no re-chunk (action=${result.action}); format not advanced`
+        });
+      }
     } catch (err) {
       const message = err instanceof Error ? err.message : String(err);
       if (/conflict/i.test(message)) {
@@ -82330,6 +82384,107 @@ function registerAuditUsageRoutes(app, ctx) {
 }
 
 // src/web/routes/config.ts
+import { homedir as homedir9 } from "node:os";
+import { sep } from "node:path";
+
+// ../../_shared/config-catalog/index.ts
+var CONFIG_CATALOG = [
+  {
+    key: "usage_tracking_enabled",
+    description: "Log reads and writes to cerefox_usage_log.",
+    kind: "boolean",
+    defaultValue: "false",
+    group: "Governance"
+  },
+  {
+    key: "require_requestor_identity",
+    description: "Require a requestor/author on MCP tool calls.",
+    kind: "boolean",
+    defaultValue: "false",
+    group: "Governance",
+    highImpact: true,
+    impactNote: "Agents that do not send a requestor will start getting errors. Confirm your MCP clients identify themselves before enabling."
+  },
+  {
+    key: "requestor_identity_format",
+    description: "Regex the requestor/author must match. Only enforced while “require requestor identity” is on.",
+    kind: "string",
+    defaultValue: "",
+    group: "Governance"
+  },
+  {
+    key: "min_search_score",
+    description: "Minimum cosine similarity for vector-side results. Use 0.6 with the local embedder.",
+    kind: "number",
+    defaultValue: "0.5",
+    min: 0,
+    max: 1,
+    group: "Retrieval"
+  },
+  {
+    key: "min_term_coverage",
+    description: "Fraction of a query's meaningful terms a keyword OR-fallback match must cover to count as confident.",
+    kind: "number",
+    defaultValue: "0.5",
+    min: 0,
+    max: 1,
+    group: "Retrieval"
+  },
+  {
+    key: "search_alpha",
+    description: "Hybrid fusion weight: 1 = pure semantic, 0 = pure keyword.",
+    kind: "number",
+    defaultValue: "0.7",
+    min: 0,
+    max: 1,
+    group: "Retrieval"
+  },
+  {
+    key: "relations_enabled",
+    description: "Expose the four document-relation tools to agents. The feature is dormant until enabled.",
+    kind: "boolean",
+    defaultValue: "false",
+    group: "Features",
+    highImpact: true,
+    impactNote: "This adds four tools (set/delete/get relations, get neighbours) to every connected agent's tool list — local MCP, remote MCP, and Edge Functions alike. Relations data and schema are always present; this switch only controls whether agents can see and use the tools. Turning it back off hides them again without deleting anything."
+  }
+];
+function configKeySpec(key) {
+  return CONFIG_CATALOG.find((k) => k.key === key);
+}
+var CONFIG_KEYS2 = CONFIG_CATALOG.map((k) => k.key);
+function validateConfigValue(key, value) {
+  const spec = configKeySpec(key);
+  if (!spec)
+    return `Unknown config key: ${key}`;
+  if (spec.kind === "boolean") {
+    return value === "true" || value === "false" ? null : `${key} must be "true" or "false" (got ${JSON.stringify(value)}).`;
+  }
+  if (spec.kind === "number") {
+    const n = Number(value);
+    if (!Number.isFinite(n)) {
+      return `${key} must be a number (got ${JSON.stringify(value)}).`;
+    }
+    if (spec.min !== undefined && n < spec.min) {
+      return `${key} must be ≥ ${spec.min} (got ${n}).`;
+    }
+    if (spec.max !== undefined && n > spec.max) {
+      return `${key} must be ≤ ${spec.max} (got ${n}).`;
+    }
+    return null;
+  }
+  if (key === "requestor_identity_format" && value.length > 0) {
+    try {
+      new RegExp(value);
+    } catch (err) {
+      return `${key} must be a valid regular expression: ${err instanceof Error ? err.message : String(err)}`;
+    }
+  }
+  return null;
+}
+
+// src/web/routes/config.ts
+init_config();
 function unwrapScalarRpc(data) {
   if (typeof data === "string")
     return data;
@@ -82339,7 +82494,43 @@ function unwrapScalarRpc(data) {
   }
   return null;
 }
+var ENV_OVERRIDES = {
+  min_search_score: "CEREFOX_MIN_SEARCH_SCORE",
+  min_term_coverage: "CEREFOX_MIN_TERM_COVERAGE",
+  search_alpha: "CEREFOX_SEARCH_ALPHA"
+};
 function registerConfigRoutes(app, ctx) {
+  app.get("/api/v1/config", async (c2) => {
+    const entries = await Promise.all(CONFIG_CATALOG.map(async (spec) => {
+      const { data, error: error3 } = await ctx.supabase.rpc("cerefox_get_config", {
+        p_key: spec.key
+      });
+      const stored = error3 ? null : unwrapScalarRpc(data);
+      const envVar = ENV_OVERRIDES[spec.key];
+      const envValue = envVar ? (process.env[envVar] ?? "").trim() : "";
+      return {
+        key: spec.key,
+        value: stored,
+        effective: stored ?? spec.defaultValue,
+        description: spec.description,
+        kind: spec.kind,
+        default: spec.defaultValue,
+        min: spec.min ?? null,
+        max: spec.max ?? null,
+        group: spec.group,
+        high_impact: spec.highImpact ?? false,
+        impact_note: spec.impactNote ?? null,
+        env_override: envValue ? { name: envVar, value: envValue } : null
+      };
+    }));
+    let configFile = null;
+    try {
+      const abs = resolveEnvFile();
+      const home = homedir9();
+      configFile = abs === home || abs.startsWith(home + sep) ? `~${abs.slice(home.length)}` : abs;
+    } catch {}
+    return c2.json({ keys: entries, config_file: configFile });
+  });
   app.get("/api/v1/config/:key", async (c2) => {
     const key = c2.req.param("key");
     const { data, error: error3 } = await ctx.supabase.rpc("cerefox_get_config", {
@@ -82358,6 +82549,9 @@ function registerConfigRoutes(app, ctx) {
       return c2.json({ detail: "Invalid JSON body" }, 400);
     }
     const value = typeof body.value === "string" ? body.value : String(body.value ?? "");
+    const invalid = validateConfigValue(key, value);
+    if (invalid)
+      return c2.json({ detail: invalid }, 400);
     const { error: error3 } = await ctx.supabase.rpc("cerefox_set_config", {
       p_key: key,
       p_value: value
@@ -84017,9 +84211,9 @@ import {
   rmSync,
   writeFileSync as writeFileSync7
 } from "node:fs";
-import { homedir as homedir9 } from "node:os";
+import { homedir as homedir10 } from "node:os";
 import { join as join18 } from "node:path";
-function resolveStateDir(override = process.env.CEREFOX_CONFIG_DIR, home = homedir9()) {
+function resolveStateDir(override = process.env.CEREFOX_CONFIG_DIR, home = homedir10()) {
   override = (override ?? "").trim();
   if (!override)
     return join18(home, ".cerefox");