@cerefox/memory 1.15.3 → 1.16.0

package/dist/server-assets/db/rpcs.sql

@@ -1103,6 +1103,82 @@ AS $$
     ORDER BY created_at DESC;
 $$;
 
+-- ── cerefox_purge_expired_trash (#251, 0.17.0) ───────────────────────────────
+-- Permanently purges documents that have been in the trash longer than
+-- `trash_retention_days`, when `trash_auto_purge_enabled` is exactly 'true'.
+-- Called by cerefox_delete_document after every real soft delete: the write
+-- that adds to the trash also sweeps it. There is no scheduler, by decision
+-- (docs/specs/trash-auto-purge.md): the trash only grows through deletes, so
+-- this keeps it bounded on every deployment without pg_cron or a daemon.
+--
+-- Safeguards, each load-bearing:
+--   * at most p_max documents per call, oldest first (partial index on
+--     deleted_at), so a large first backlog drains over several deletes;
+--   * FOR UPDATE SKIP LOCKED, so concurrent deletes sweep disjoint rows;
+--   * an unparsable or < 1 retention purges NOTHING (never "purge everything");
+--   * one audit entry per purge, authored 'trash-retention' / 'user': the
+--     authority is the operator's setting, not whoever's delete triggered it.
+-- Not exposed on MCP, the Edge Functions or the CLI: there is still no
+-- agent-callable purge (access-paths.md → trust model).
+CREATE OR REPLACE FUNCTION cerefox_purge_expired_trash(
+    p_max INT DEFAULT 100
+)
+RETURNS INT
+LANGUAGE plpgsql
+SECURITY DEFINER
+SET search_path = public, pg_catalog
+AS $$
+DECLARE
+    v_enabled  TEXT;
+    v_days_txt TEXT;
+    v_days     INT;
+    v_cutoff   TIMESTAMPTZ;
+    v_count    INT := 0;
+    r          RECORD;
+BEGIN
+    SELECT value INTO v_enabled FROM cerefox_config WHERE key = 'trash_auto_purge_enabled';
+    IF v_enabled IS DISTINCT FROM 'true' THEN
+        RETURN 0;
+    END IF;
+
+    SELECT value INTO v_days_txt FROM cerefox_config WHERE key = 'trash_retention_days';
+    IF v_days_txt IS NULL OR BTRIM(v_days_txt) !~ '^[0-9]{1,6}$' THEN
+        RETURN 0;
+    END IF;
+    v_days := BTRIM(v_days_txt)::INT;
+    IF v_days < 1 THEN
+        RETURN 0;
+    END IF;
+    v_cutoff := NOW() - make_interval(days => v_days);
+
+    FOR r IN
+        SELECT id, title, total_chars, deleted_at
+        FROM cerefox_documents
+        WHERE deleted_at IS NOT NULL AND deleted_at < v_cutoff
+        ORDER BY deleted_at
+        LIMIT GREATEST(COALESCE(p_max, 0), 0)
+        FOR UPDATE SKIP LOCKED
+    LOOP
+        PERFORM cerefox_create_audit_entry(
+            p_document_id := r.id,
+            p_operation   := 'delete',
+            p_author      := 'trash-retention',
+            p_author_type := 'user',
+            p_size_before := r.total_chars,
+            p_size_after  := 0,
+            p_description := 'Auto-purged from the trash (trash_retention_days = ' || v_days ||
+                             '; trashed ' || to_char(r.deleted_at AT TIME ZONE 'UTC', 'YYYY-MM-DD') ||
+                             '): ' || COALESCE(r.title, '(untitled)') ||
+                             ' (' || COALESCE(r.total_chars, 0) || ' chars)'
+        );
+        DELETE FROM cerefox_documents WHERE id = r.id;
+        v_count := v_count + 1;
+    END LOOP;
+
+    RETURN v_count;
+END;
+$$;
+
 -- ── cerefox_delete_document (soft delete) ────────────────────────────────────
 -- Soft-deletes a document by setting deleted_at = NOW(). The document, its
 -- chunks, and versions remain in the database but are excluded from search.
@@ -1123,6 +1199,12 @@ $$;
 --
 -- Deleting an already-deleted document is a reported no-op: the original
 -- deleted_at is preserved and no duplicate audit entry is written.
+--
+-- 0.17.0 (#251): a real soft delete then sweeps the trash
+-- (cerefox_purge_expired_trash) and reports how many documents it purged as
+-- `auto_purged`. The sweep runs in its own sub-block: if it fails, only its
+-- savepoint rolls back, a WARNING is raised, and the delete commits exactly as
+-- it would with auto-purge off. A sweep can never fail a delete.
 
 DROP FUNCTION IF EXISTS cerefox_delete_document(UUID, TEXT, TEXT, TEXT, TEXT);
 DROP FUNCTION IF EXISTS cerefox_delete_document(UUID, TEXT, TEXT);
@@ -1144,6 +1226,7 @@ DECLARE
     v_total_chars  INT;
     v_current_hash TEXT;
     v_deleted_at   TIMESTAMPTZ;
+    v_auto_purged  INT := 0;
 BEGIN
     -- FOR UPDATE: makes the hash check atomic with the delete — a concurrent
     -- content update serializes here, and a stale deleter sees its hash.
@@ -1202,12 +1285,21 @@ BEGIN
                          COALESCE('; reason: ' || NULLIF(BTRIM(p_reason), ''), '')
     );
 
+    BEGIN
+        v_auto_purged := cerefox_purge_expired_trash(100);
+    EXCEPTION WHEN OTHERS THEN
+        v_auto_purged := 0;
+        RAISE WARNING 'cerefox: trash auto-purge skipped after deleting %: % (%)',
+            p_document_id, SQLERRM, SQLSTATE;
+    END;
+
     RETURN jsonb_build_object(
         'document_id', p_document_id,
         'title', v_title,
         'total_chars', v_total_chars,
         'deleted_at', v_deleted_at,
-        'already_deleted', FALSE
+        'already_deleted', FALSE,
+        'auto_purged', v_auto_purged
     );
 END;
 $$;
@@ -2742,7 +2834,10 @@ DECLARE
         -- (0 = off). Partial edits make writes cheap, so an insert-only agent
         -- never assembles the document and never sees it grow past its split
         -- point. A signal in the write's response, never a refusal.
-        'document_size_warning_chars'
+        'document_size_warning_chars',
+        -- #251: trash auto-purge, off by default; read by
+        -- cerefox_purge_expired_trash on every soft delete.
+        'trash_auto_purge_enabled', 'trash_retention_days'
     ];
     v_old TEXT;
 BEGIN
@@ -3091,6 +3186,9 @@ SET search_path = public, pg_catalog
 AS $$
     -- Keep in lockstep with the `@version:` marker in schema.sql (cut_release.ts
     -- enforces it). Bump whenever schema.sql OR rpcs.sql changes.
+    -- 0.17.0 (#251): trash auto-purge. New cerefox_purge_expired_trash, called
+    -- by cerefox_delete_document after a real soft delete (which now returns
+    -- `auto_purged`); two config keys, off by default. Migration 0033.
     -- 0.16.2 (#26 follow-up): the Data API grant list is DERIVED from pg_class
     -- rather than a hand-written array that has to match the set of tables. The
     -- array had drifted (cerefox_document_relations), which only a FRESH cloud
@@ -3129,7 +3227,7 @@ AS $$
     -- 0.11.0 supersedes 0.10.6 (v1.2.1, #191): this branch carries that fix plus
     -- the partial-edit surface, and both migrations (0019, 0020) are in the
     -- sequence, so a store deploying this gets everything from both lines.
-    SELECT '0.16.2'::TEXT;
+    SELECT '0.17.0'::TEXT;
 $$;
 
 -- ── cerefox_find_dead_links ──────────────────────────────────────────────────

package/dist/server-assets/db/schema.sql

@@ -5,7 +5,7 @@
 -- Requires extensions: vector (pgvector), uuid-ossp
 -- These are enabled at the top of db_deploy.py before this file is applied.
 --
--- @version: 0.16.2
+-- @version: 0.17.0
 -- The `@version` marker above is read by the schema-version-mismatch banner
 -- (see /api/v1/schema-version). Bump it whenever schema.sql OR rpcs.sql
 -- changes in a way that requires `cerefox server deploy` to be re-run —
@@ -413,6 +413,15 @@ ON CONFLICT (key) DO NOTHING;
 INSERT INTO cerefox_config (key, value)
 VALUES ('review_workflow_enabled', 'false')
 ON CONFLICT (key) DO NOTHING;
+-- Trash auto-purge (#251): OFF by default. When on, the soft delete that adds
+-- to the trash also purges documents trashed more than trash_retention_days
+-- ago (cerefox_purge_expired_trash). Design: docs/specs/trash-auto-purge.md.
+INSERT INTO cerefox_config (key, value)
+VALUES ('trash_auto_purge_enabled', 'false')
+ON CONFLICT (key) DO NOTHING;
+INSERT INTO cerefox_config (key, value)
+VALUES ('trash_retention_days', '60')
+ON CONFLICT (key) DO NOTHING;
 
 
 -- ── Usage log ────────────────────────────────────────────────────────────────

package/docs/api/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "Cerefox /api/v1",
-    "version": "1.1.0",
+    "version": "1.2.0",
     "description": "The HTTP API served by `cerefox web`. Intended for callers that embed Cerefox rather than\nusing MCP or the CLI — MCP remains the recommended path for AI agents.\n\nGENERATED by `bun scripts/gen_openapi.ts`; do not edit by hand. Paths and every input a route\nreads come from the route handlers, summaries from the endpoint table in docs/guides/api.md,\nand body shapes from the zod schemas in `_shared/schemas/` (responses verified against a live\nserver by `api-schema-truth.test.ts`).\n\nAUTH: a request arriving on loopback needs no credential. From any other interface, an\n`X-API-Key` (or `Authorization: Bearer`) matching `CEREFOX_API_KEY` is required.\nSee docs/guides/securing-local-access.md.\n\nIDENTITY: send `X-Cerefox-Author` (and optionally `X-Cerefox-Author-Type`) to be recorded as\nyourself. Without them you are recorded as the bundled web UI.\n\nCONCURRENCY: read before you write. A content update (`POST /ingest` with `document_id` or\n`update_if_exists`, `POST /documents/{id}/edit`, `/upload`) needs the `content_hash` you read\nas `expected_content_hash`, or an explicit `last_write_wins`. `DELETE /documents/{id}` needs it\ntoo when you identify yourself, as `X-Cerefox-Expected-Content-Hash` or a query parameter.\nNeither is a 400 (`CEREFOX_TOKEN_REQUIRED`); a stale hash is a 409 carrying `current_hash`.\nRestore and purge take no hash.\n\nDISCOVERY: this document is served at `GET /api/v1/openapi.json` by the server it describes,\nand every /api/v1 response carries `Link: </api/v1/openapi.json>; rel=\"service-desc\"` (RFC 8631)."
   },
   "servers": [
@@ -1289,6 +1289,11 @@
                     },
                     "already_deleted": {
                       "type": "boolean"
+                    },
+                    "auto_purged": {
+                      "type": "integer",
+                      "minimum": -9007199254740991,
+                      "maximum": 9007199254740991
                     }
                   },
                   "required": [
@@ -3096,7 +3101,7 @@
     },
     "/api/v1/documents/trash": {
       "get": {
-        "summary": "Soft-deleted documents, newest first, `limit` ≤ 500. The exact total is in the `X-Total-Count` response header (v1.14.1).",
+        "summary": "Soft-deleted documents, newest first, `limit` ≤ 500. The exact total is in the `X-Total-Count` response header (v1.14.1). `deleted_before=<ISO timestamp>` restricts both to documents trashed before then.",
         "operationId": "get_documents_trash",
         "parameters": [
           {
@@ -3110,6 +3115,16 @@
               "maximum": 500,
               "default": 50
             }
+          },
+          {
+            "name": "deleted_before",
+            "in": "query",
+            "required": false,
+            "description": "Only documents trashed before this instant; `X-Total-Count` then counts exactly those. With now minus `trash_retention_days`, it is what trash auto-purge would make eligible.",
+            "schema": {
+              "type": "string",
+              "format": "date-time"
+            }
           }
         ],
         "responses": {
@@ -3194,6 +3209,16 @@
               }
             }
           },
+          "400": {
+            "description": "The request is malformed or incomplete: a bad parameter or body, an unknown body field, a blank identity header, or a content update with no `expected_content_hash` and no `last_write_wins` (`CEREFOX_TOKEN_REQUIRED`).",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/Error"
+                }
+              }
+            }
+          },
           "401": {
             "description": "An API key is required: a non-loopback request while `CEREFOX_API_KEY` is set. See docs/guides/securing-local-access.md.",
             "content": {
@@ -5777,7 +5802,7 @@
   "x-cerefox-coverage": {
     "routes": 40,
     "withResponseSchema": 36,
-    "queryParameters": 41,
+    "queryParameters": 42,
     "note": "Routes without a response schema are listed with their parameters and summary only. Nothing is invented; see `uncovered`.",
     "nonJson": [
       "GET /openapi.json",

package/docs/guides/access-paths.md

@@ -295,6 +295,22 @@ states the count, with a Stop button and a per-document audit entry; there is no
 server-side "purge everything" endpoint, on purpose, because `/api/v1` is
 reachable by anything holding the local key.
 
+**Trash auto-purge (v1.16.0, opt-in) does not weaken this either, and here is exactly
+why.** With `trash_auto_purge_enabled` on, every soft delete (including an agent's) also
+purges documents that have been in the trash longer than `trash_retention_days`. So an
+agent's action can *trigger* a purge. It stays within the model because:
+
+- **a human turned the policy on and chose the period**: that period is the review
+  window of step 2, decided in advance instead of per document;
+- the sweep purges only documents **older than that period**, never the one being
+  deleted and never anything an agent selects;
+- there is still **no agent-callable purge**: the sweep function is not exposed over MCP
+  or the Edge Functions, has no CLI verb, and, like every RPC, is callable over the
+  Data API only with the service-role key.
+
+With the setting off (the default), nothing here changes. Design:
+[`docs/specs/trash-auto-purge.md`](../specs/trash-auto-purge.md).
+
 A `cerefox purge-doc` CLI command, a `cerefox_purge_document` MCP tool, or a
 `/documents/{id}/purge` HTTP endpoint accessible via the Cerefox token would each break this
 property. **Do not add them without a governance design that replaces the human-in-the-

package/docs/guides/api.md

@@ -189,7 +189,7 @@ default; Cerefox Local picks its own port and `cerefox-local status` prints it).
 | `GET /documents/{id}/versions` | Version history. |
 | `POST /documents/{id}/versions/{version_id}/archive` | Archive or unarchive one stored version (body `{ "archived": true\|false }`). Archiving strips a version's search artifacts; its content is kept as the safety copy. |
 | `GET /documents/{id}/download` | Raw markdown. |
-| `GET /documents/trash` | Soft-deleted documents, newest first, `limit` ≤ 500. The exact total is in the `X-Total-Count` response header (v1.14.1). |
+| `GET /documents/trash` | Soft-deleted documents, newest first, `limit` ≤ 500. The exact total is in the `X-Total-Count` response header (v1.14.1). `deleted_before=<ISO timestamp>` restricts both to documents trashed before then. |
 | `POST /documents/metadata-search` | Query by metadata / project / time, no text query. |
 | `GET /metadata-keys` | Metadata keys with counts and example values. |
 | `GET /resolve-link`, `GET /check-filename` | Link and title resolution helpers. |
@@ -197,7 +197,7 @@ default; Cerefox Local picks its own port and `cerefox-local status` prints it).
 | `POST /ingest/file` | Create from a multipart file upload. |
 | `POST /documents/{id}/upload` | Replace a document's content from a file. |
 | `POST /documents/{id}/edit` | Update title, content, metadata, projects. Unknown fields are a `400`. |
-| `DELETE /documents/{id}` | Soft-delete (to trash). |
+| `DELETE /documents/{id}` | Soft-delete (to trash). Returns `auto_purged`: documents this delete purged because they had outlived `trash_retention_days` (only while trash auto-purge is on). |
 | `POST /documents/{id}/restore` | Restore from trash. |
 | `DELETE /documents/{id}/purge` | Permanent delete. Irreversible. |
 | `POST /documents/{id}/review-status` | Approve or re-queue for review. `404` while the review workflow is off. |

package/docs/guides/configuration.md

@@ -270,6 +270,53 @@ cerefox document get <document-id> --version-id <version-id>
 
 ---
 
+## Trash auto-purge
+
+Deleting a document moves it to the trash, where it stays until someone restores or
+purges it. On a store where agents create and delete a lot, the trash grows without
+bound. Since v1.16.0 you can have Cerefox purge old trash automatically. It is
+**off by default**.
+
+| Setting | Default | What it does |
+|---|---|---|
+| `trash_auto_purge_enabled` | `false` | Turn automatic purging on or off. |
+| `trash_retention_days` | `60` | How long a document stays in the trash before it may be purged: your window to restore it. Whole days, 1 to 3650. |
+
+Set them from **Settings** (Retention) or the CLI:
+
+```bash
+cerefox config set trash_retention_days 60
+cerefox config set trash_auto_purge_enabled true
+```
+
+**When it runs.** Cerefox has no scheduler. The trash only grows when a document is
+deleted, so that is when it is swept: each delete also purges documents that have been
+in the trash longer than `trash_retention_days`, up to 100 per delete, oldest first.
+A document therefore becomes *eligible* after the retention period and is removed by
+**the next delete after that**. If nothing is deleted, expired documents stay, and
+nothing new arrives in the trash either.
+
+**What you can rely on.**
+
+- Only documents already older than the retention period are purged, never the one
+  being deleted.
+- A failure while sweeping never fails the delete: the delete completes, and the sweep
+  is retried on the next one.
+- Every purge is in the audit log (author `trash-retention`), naming the retention in
+  force and the day the document was trashed.
+- A delete reports how many documents it purged: `auto_purged` in the
+  `/api/v1` response, a line in the CLI and MCP output.
+- `cerefox doctor` shows whether it is on and with which period.
+- Settings asks for confirmation and tells you how many documents already in the trash
+  the next delete would purge. Shortening the period while it is on gets the same
+  confirmation.
+
+Purged documents cannot be restored. Turning this on for a store with a large, old
+trash purges it over the next few deletes; check the Trash page first if anything
+there matters.
+
+---
+
 ## Storage & Backup
 
 | Variable | Default | Description |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cerefox/memory",
-  "version": "1.15.3",
+  "version": "1.16.0",
   "description": "Cerefox \u2014 user-owned shared memory for AI agents. CLI + stdio MCP server + web UI + ingestion for a knowledge base on your own Supabase project (or fully self-hosted with Cerefox Local).",
   "license": "Apache-2.0",
   "homepage": "https://github.com/fstamatelopoulos/cerefox",
@@ -65,7 +65,7 @@
     "typescript": "^6.0.3"
   },
   "scripts": {
-    "build": "bun run bundle-server-assets && bun build src/bin/cerefox.ts --outdir dist/bin --target node --format esm --external @huggingface/transformers --external onnxruntime-node",
+    "build": "bun run bundle-server-assets && bun run bundle-docs && bun build src/bin/cerefox.ts --outdir dist/bin --target node --format esm --external @huggingface/transformers --external onnxruntime-node",
     "clean": "rm -rf dist docs AGENT_GUIDE.md AGENT_QUICK_REFERENCE.md",
     "bundle-docs": "bun run ../../scripts/bundle_package_docs.ts",
     "bundle-server-assets": "bun run ../../scripts/bundle_server_assets.ts",