@cerefox/memory 1.7.1 → 1.8.0

package/dist/bin/cerefox.js

@@ -7438,7 +7438,7 @@ var exports_meta = {};
 __export(exports_meta, {
   PKG_VERSION: () => PKG_VERSION
 });
-var PKG_VERSION = "1.7.1";
+var PKG_VERSION = "1.8.0";
 var init_meta = () => {};
 
 // ../../_shared/config/paths.ts
@@ -25743,7 +25743,7 @@ var init_bundled_docs = __esm(() => {
 });
 
 // ../../_shared/ef-meta/index.ts
-var EF_VERSION = "1.7.1", CEREFOX_VERSION = "1.7.1", EF_LAST_CHANGED = "1.7.0";
+var EF_VERSION = "1.8.0", CEREFOX_VERSION = "1.8.0", EF_LAST_CHANGED = "1.7.0";
 var init_ef_meta = () => {};
 
 // ../../_shared/compatibility/index.ts
@@ -72988,7 +72988,13 @@ async function migrationStatus(opts) {
 }
 async function runDbMigrate(opts) {
   const log = opts.log ?? (() => {});
-  const sql = src_default(opts.dbUrl, { prepare: false, onnotice: () => {} });
+  const sql = src_default(opts.dbUrl, {
+    prepare: false,
+    onnotice: (n) => {
+      if (n.message)
+        log(`   ↳ ${n.message}`);
+    }
+  });
   try {
     await sql.unsafe(BOOTSTRAP_MIGRATIONS_SQL);
     const allFiles = listMigrationFiles(opts.assets.migrationsDir);
@@ -77860,6 +77866,10 @@ async function action17(options) {
     }
     if (remediation) {
       println(cErr.yellow("→ " + remediation));
+      const configDir = (process.env.CEREFOX_CONFIG_DIR ?? "").trim();
+      if (configDir) {
+        println(cErr.dim(`  (this doctor ran against CEREFOX_CONFIG_DIR=${configDir} — ` + `prefix the command above the same way, or use your environment alias, ` + `or a bare \`cerefox\` will act on your DEFAULT environment instead)`));
+      }
       println("");
     }
   }

package/dist/server-assets/_shared/ef-meta/index.ts

@@ -18,7 +18,7 @@
  * doesn't touch `supabase/functions/` leaves it alone).
  */
 
-export const EF_VERSION = "1.7.1";
+export const EF_VERSION = "1.8.0";
 
 /**
  * The Cerefox RELEASE version — what `cerefox --version` reports and what npm
@@ -36,7 +36,7 @@ export const EF_VERSION = "1.7.1";
  * is imported by the Deno Edge Functions, which cannot reach into the npm
  * package.
  */
-export const CEREFOX_VERSION = "1.7.1";
+export const CEREFOX_VERSION = "1.8.0";
 
 /**
  * The most recent version whose EF-side SOURCE actually changed (#127).

package/dist/server-assets/db/migrations/0025_drop_orphaned_overloads.sql

@@ -5,7 +5,8 @@
 -- replaces the SAME signature), leaving long-lived databases with BOTH
 -- overloads. A named 1-arg call is then ambiguous — PostgREST PGRST203
 -- ("could not choose the best candidate") — which is how the first
--- production acceptance run failed to purge its fixtures (v1.7.0).
+-- acceptance run against a long-lived database failed to purge its
+-- fixtures (v1.7.0).
 -- Fresh databases never had the old signatures and are unaffected.
 --
 -- Schema version 0.12.0 → 0.12.1. The DROPs also run from rpcs.sql on every

package/dist/server-assets/db/migrations/0027_drop_archived_search_artifacts.sql

@@ -0,0 +1,172 @@
+-- 0027_drop_archived_search_artifacts.sql — archived chunks carry no search
+-- artifacts (#216). THE CANONICAL RATIONALE LIVES HERE; code comments
+-- reference it.
+--
+-- Search is current-chunks-only by design (every search index is partial on
+-- version_id IS NULL), version reconstruction and diffs read `content`, and
+-- restoring an old version is deliberately manual re-ingest (which
+-- re-embeds). Embeddings and fts on archived chunks were therefore never
+-- readable by anything — pure storage cost, measured at ~30-45% of the chunk
+-- relation on long-lived stores — and after a reindex they are stale for the
+-- current embedder besides. There is no config and no maintenance command:
+-- with nothing able to read the artifacts, this is an invariant, not a
+-- policy. The archived content — the actual safety copy — is untouched.
+--
+-- Four parts:
+--   1. embedding_primary becomes nullable, WITH a replacement guard: a CHECK
+--      that a CURRENT chunk always carries an embedding. Without it, a short
+--      embedding-API response could insert a silently search-invisible chunk
+--      where the old NOT NULL failed loudly (review round 6).
+--   2. The NEW cerefox_snapshot_version ships INSIDE this migration (repo
+--      precedent: 0005-0008, 0011, 0025 carry function bodies). This closes
+--      two windows where the OLD snapshot could keep archiving WITH
+--      artifacts after 0027 was stamped: `bun scripts/db_migrate.ts` (which
+--      never refreshes rpcs.sql), and a `server deploy` that failed between
+--      the migration step and the RPC refresh.
+--   3. Back-fill: strip artifacts from existing archived rows, reporting
+--      rows and bytes. embedder_upgrade is nulled with its vector;
+--      embedder_primary is deliberately kept (NOT NULL, harmless provenance).
+--   4. Space note: Postgres frees the bytes for REUSE via autovacuum rather
+--      than shrinking files immediately — growth stops even if the reported
+--      database size does not drop the same day.
+--
+-- Schema version 0.12.2 → 0.13.0.
+
+ALTER TABLE cerefox_chunks ALTER COLUMN embedding_primary DROP NOT NULL;
+
+DO $$
+BEGIN
+    IF NOT EXISTS (
+        SELECT 1 FROM pg_constraint WHERE conname = 'cerefox_chunks_current_has_embedding'
+    ) THEN
+        ALTER TABLE cerefox_chunks
+            ADD CONSTRAINT cerefox_chunks_current_has_embedding
+            CHECK (version_id IS NOT NULL OR embedding_primary IS NOT NULL);
+    END IF;
+END $$;
+
+-- The new snapshot (identical to the rpcs.sql copy this release ships):
+
+DROP FUNCTION IF EXISTS cerefox_snapshot_version(UUID, TEXT, INT);
+DROP FUNCTION IF EXISTS cerefox_snapshot_version(UUID, TEXT, INT, BOOLEAN);
+CREATE FUNCTION cerefox_snapshot_version(
+    p_document_id       UUID,
+    p_source            TEXT    DEFAULT 'manual',
+    -- NULL (the new default) means "use the store's policy from
+    -- cerefox_config". Passing a value still overrides, for deliberate one-off
+    -- admin operations — but callers no longer supply one by accident.
+    p_retention_hours   INT     DEFAULT NULL,
+    p_cleanup_enabled   BOOLEAN DEFAULT NULL
+)
+RETURNS TABLE (
+    version_id     UUID,
+    version_number INT,
+    chunk_count    INT,
+    total_chars    INT
+)
+LANGUAGE plpgsql
+SECURITY DEFINER
+SET search_path = public, pg_catalog
+AS $$
+DECLARE
+    v_version_id     UUID;
+    v_version_number INT;
+    v_chunk_count    INT;
+    v_total_chars    INT;
+    -- Resolve the retention policy from the STORE, not the caller.
+    --
+    -- These used to arrive as parameters filled from each client's own env, so
+    -- the surviving version history depended on which client wrote last: an
+    -- agent running defaults would prune versions that an operator had
+    -- configured to keep. Retention describes the data, so it belongs to the
+    -- data. Same COALESCE(param, config, default) shape the retrieval tunables
+    -- already use.
+    v_retention      INT     := COALESCE(p_retention_hours,
+                                         cerefox_config_int('version_retention_hours', 120));
+    v_cleanup        BOOLEAN := COALESCE(p_cleanup_enabled,
+                                         cerefox_config_bool('version_cleanup_enabled', TRUE));
+BEGIN
+    -- Count current chunks to record in the version metadata
+    SELECT COUNT(*), COALESCE(SUM(char_count), 0)
+    INTO v_chunk_count, v_total_chars
+    FROM cerefox_chunks c
+    WHERE c.document_id = p_document_id
+      AND c.version_id IS NULL;
+
+    -- Compute the next version number (sequential per document)
+    SELECT COALESCE(MAX(dv.version_number), 0) + 1
+    INTO v_version_number
+    FROM cerefox_document_versions dv
+    WHERE dv.document_id = p_document_id;
+
+    -- Create the version row
+    INSERT INTO cerefox_document_versions (
+        document_id, version_number, source, chunk_count, total_chars
+    ) VALUES (
+        p_document_id, v_version_number, p_source, v_chunk_count, v_total_chars
+    )
+    RETURNING id INTO v_version_id;
+
+    -- Archive all current chunks by pointing them at the new version, and
+    -- NULL their search artifacts in the same write (0.13.0, #216 — full
+    -- rationale in migration 0027). The content — the actual safety copy —
+    -- is untouched. embedder_upgrade is nulled with its vector;
+    -- embedder_primary is deliberately KEPT (it is NOT NULL, and the label
+    -- is harmless provenance for a vector that no longer exists — nothing
+    -- reads embedder columns without a version_id IS NULL filter).
+    UPDATE cerefox_chunks c
+    SET version_id = v_version_id,
+        embedding_primary = NULL,
+        embedding_upgrade = NULL,
+        embedder_upgrade = NULL,
+        fts = NULL
+    WHERE c.document_id = p_document_id
+      AND c.version_id IS NULL;
+
+    -- Lazy retention: delete versions outside the retention window,
+    -- but always keep the most recently created version (the one we just made).
+    -- Skip archived versions (archived=true) -- they are protected from cleanup.
+    -- Skip cleanup entirely if p_cleanup_enabled is false (immutable mode).
+    IF v_cleanup THEN
+        DELETE FROM cerefox_document_versions dv
+        WHERE dv.document_id = p_document_id
+          AND dv.archived IS NOT TRUE
+          AND dv.created_at < NOW() - (v_retention || ' hours')::INTERVAL
+          AND dv.id != (
+              SELECT id FROM cerefox_document_versions
+              WHERE document_id = p_document_id
+              ORDER BY created_at DESC
+              LIMIT 1
+          );
+    END IF;
+
+    RETURN QUERY SELECT v_version_id, v_version_number, v_chunk_count, v_total_chars;
+END;
+$$;
+
+DO $$
+DECLARE
+    v_rows  INT;
+    v_bytes BIGINT;
+BEGIN
+    SELECT count(*),
+           COALESCE(SUM(COALESCE(pg_column_size(embedding_primary), 0))
+                  + SUM(COALESCE(pg_column_size(embedding_upgrade), 0))
+                  + SUM(COALESCE(pg_column_size(fts), 0)), 0)
+    INTO v_rows, v_bytes
+    FROM cerefox_chunks
+    WHERE version_id IS NOT NULL
+      AND (embedding_primary IS NOT NULL OR embedding_upgrade IS NOT NULL OR fts IS NOT NULL);
+
+    UPDATE cerefox_chunks
+    SET embedding_primary = NULL,
+        embedding_upgrade = NULL,
+        embedder_upgrade = NULL,
+        fts = NULL
+    WHERE version_id IS NOT NULL
+      AND (embedding_primary IS NOT NULL OR embedding_upgrade IS NOT NULL OR fts IS NOT NULL);
+
+    RAISE NOTICE
+        'Migration 0027: stripped search artifacts from % archived chunk row(s), freeing ~% for reuse. Archived content is untouched; current chunks keep their embeddings.',
+        v_rows, pg_size_pretty(v_bytes);
+END $$;

package/dist/server-assets/db/rpcs.sql

@@ -1009,9 +1009,19 @@ BEGIN
     )
     RETURNING id INTO v_version_id;
 
-    -- Archive all current chunks by pointing them at the new version
+    -- Archive all current chunks by pointing them at the new version, and
+    -- NULL their search artifacts in the same write (0.13.0, #216 — full
+    -- rationale in migration 0027). The content — the actual safety copy —
+    -- is untouched. embedder_upgrade is nulled with its vector;
+    -- embedder_primary is deliberately KEPT (it is NOT NULL, and the label
+    -- is harmless provenance for a vector that no longer exists — nothing
+    -- reads embedder columns without a version_id IS NULL filter).
     UPDATE cerefox_chunks c
-    SET version_id = v_version_id
+    SET version_id = v_version_id,
+        embedding_primary = NULL,
+        embedding_upgrade = NULL,
+        embedder_upgrade = NULL,
+        fts = NULL
     WHERE c.document_id = p_document_id
       AND c.version_id IS NULL;
 
@@ -1248,7 +1258,7 @@ $$;
 DROP FUNCTION IF EXISTS cerefox_restore_document(UUID, TEXT, TEXT, TEXT);
 DROP FUNCTION IF EXISTS cerefox_restore_document(UUID, TEXT, TEXT);
 -- 0.12.1: the pre-author 1-arg overload survived every CREATE OR REPLACE
--- since the signature grew (same orphan class as purge; found live on prod).
+-- since the signature grew (same orphan class as purge; found live on a long-lived database).
 DROP FUNCTION IF EXISTS cerefox_restore_document(UUID);
 CREATE FUNCTION cerefox_restore_document(
     p_document_id   UUID,
@@ -1829,9 +1839,9 @@ $$;
 -- Reads chunk title and content directly from the DB -- caller only needs to
 -- supply the new document title.
 --
--- Only affects current chunks (version_id IS NULL). Archived chunks retain their
--- original tsvectors (they are excluded from all search indexes and require
--- re-ingestion to restore anyway).
+-- Only affects current chunks (version_id IS NULL). Archived chunks carry NO
+-- tsvector at all since 0.13.0 (#216) — fts is nulled at archive time, and a
+-- restore is a re-ingest that recomputes everything.
 
 DROP FUNCTION IF EXISTS cerefox_update_chunk_fts(UUID, TEXT);
 CREATE FUNCTION cerefox_update_chunk_fts(
@@ -2788,6 +2798,10 @@ SET search_path = public, pg_catalog
 AS $$
     -- Keep in lockstep with the `@version:` marker in schema.sql (cut_release.ts
     -- enforces it). Bump whenever schema.sql OR rpcs.sql changes.
+    -- 0.13.0 (#216): archived chunks carry no search artifacts —
+    -- cerefox_snapshot_version nulls embedding_primary/embedding_upgrade/fts
+    -- at archive time; embedding_primary becomes nullable; migration 0027
+    -- strips existing archived rows.
     -- 0.12.2 (#212, #214): metadata must be a JSON object (ingest input guard
     -- + set_document_metadata stored-state merge guard); cerefox_find_dead_links
     -- (link-integrity phase-2 sweep); cerefox_metadata_health (doctor check).
@@ -2806,7 +2820,7 @@ AS $$
     -- 0.11.0 supersedes 0.10.6 (v1.2.1, #191): this branch carries that fix plus
     -- the partial-edit surface, and both migrations (0019, 0020) are in the
     -- sequence, so a store deploying this gets everything from both lines.
-    SELECT '0.12.2'::TEXT;
+    SELECT '0.13.0'::TEXT;
 $$;
 
 -- ── cerefox_find_dead_links ──────────────────────────────────────────────────

package/dist/server-assets/db/schema.sql

@@ -5,7 +5,7 @@
 -- Requires extensions: vector (pgvector), uuid-ossp
 -- These are enabled at the top of db_deploy.py before this file is applied.
 --
--- @version: 0.12.2
+-- @version: 0.13.0
 -- The `@version` marker above is read by the schema-version-mismatch banner
 -- (see /api/v1/schema-version). Bump it whenever schema.sql OR rpcs.sql
 -- changes in a way that requires `cerefox server deploy` to be re-run —
@@ -212,8 +212,16 @@ CREATE TABLE IF NOT EXISTS cerefox_chunks (
     -- reconstructs with its OWN format. See docs/guides/content-format.md.
     content_format  SMALLINT    NOT NULL DEFAULT 1,
 
-    -- Primary embedding: always computed, cloud API (default: OpenAI text-embedding-3-small)
-    embedding_primary  VECTOR(768) NOT NULL,
+    -- Primary embedding: always computed on write, cloud API (default: OpenAI
+    -- text-embedding-3-small). NULL only on ARCHIVED rows (0.13.0, #216 —
+    -- rationale in migration 0027): archiving nulls the search artifacts.
+    -- The CHECK below preserves the pre-0.13.0 guarantee that a CURRENT
+    -- chunk always has an embedding — without it, a short embedding-API
+    -- response could insert a silently search-invisible chunk where the old
+    -- NOT NULL failed loudly.
+    embedding_primary  VECTOR(768)
+                       CONSTRAINT cerefox_chunks_current_has_embedding
+                       CHECK (version_id IS NOT NULL OR embedding_primary IS NOT NULL),
     -- Upgrade embedding: optional, alternative model (Fireworks, Vertex, etc.)
     embedding_upgrade  VECTOR(768),
 

package/docs/guides/upgrading.md

@@ -150,6 +150,15 @@ knowing about:
   you're coming from a pre-installer 0.1.x clone, see the "old pre-installer
   clone" note above: install the package and run `cerefox init`.
 
+## Notable: v1.8.0 storage reclaim (migration 0027)
+
+Upgrading to v1.8.0 strips never-read search artifacts from archived version
+chunks (rationale: #216 / the migration's own header) and the migration
+prints what it freed. Postgres releases the bytes for **reuse** via
+autovacuum rather than shrinking files immediately, so expect growth to stop
+rather than the reported database size to drop the same day. Archived
+version *content* is untouched.
+
 ## After upgrading: AI agents
 
 New tools and updated tool signatures are picked up by MCP clients in **new

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerefox/memory",
-  "version": "1.7.1",
+  "version": "1.8.0",
   "description": "Cerefox — user-owned shared memory for AI agents. CLI + stdio MCP server + web UI + ingestion for a knowledge base on your own Supabase project (or fully self-hosted with Cerefox Local).",
   "license": "Apache-2.0",
   "homepage": "https://github.com/fstamatelopoulos/cerefox",
@@ -65,7 +65,7 @@
     "typescript": "^6.0.3"
   },
   "scripts": {
-    "build": "bun build src/bin/cerefox.ts --outdir dist/bin --target node --format esm --external @huggingface/transformers --external onnxruntime-node",
+    "build": "bun run bundle-server-assets && bun build src/bin/cerefox.ts --outdir dist/bin --target node --format esm --external @huggingface/transformers --external onnxruntime-node",
     "clean": "rm -rf dist docs AGENT_GUIDE.md AGENT_QUICK_REFERENCE.md",
     "bundle-docs": "bun run ../../scripts/bundle_package_docs.ts",
     "bundle-server-assets": "bun run ../../scripts/bundle_server_assets.ts",