@happyvertical/smrt-core 0.41.0 → 0.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/agents/schema-paths.md +79 -8
- package/dist/change-feed.d.ts.map +1 -1
- package/dist/change-feed.js +2 -2
- package/dist/change-feed.js.map +1 -1
- package/dist/change-signals.d.ts.map +1 -1
- package/dist/change-signals.js +8 -1
- package/dist/change-signals.js.map +1 -1
- package/dist/class.d.ts.map +1 -1
- package/dist/class.js +10 -9
- package/dist/class.js.map +1 -1
- package/dist/collection.d.ts.map +1 -1
- package/dist/collection.js +2 -2
- package/dist/collection.js.map +1 -1
- package/dist/dispatch/collections/Dispatches.d.ts.map +1 -1
- package/dist/dispatch/collections/Dispatches.js +5 -5
- package/dist/dispatch/collections/Dispatches.js.map +1 -1
- package/dist/dispatch/models/Dispatch.d.ts +1 -1
- package/dist/dispatch/models/Dispatch.d.ts.map +1 -1
- package/dist/dispatch/models/Dispatch.js +3 -2
- package/dist/dispatch/models/Dispatch.js.map +1 -1
- package/dist/dispatch/models/DispatchSubscription.d.ts +1 -1
- package/dist/dispatch/models/DispatchSubscription.d.ts.map +1 -1
- package/dist/dispatch/models/DispatchSubscription.js +2 -1
- package/dist/dispatch/models/DispatchSubscription.js.map +1 -1
- package/dist/embeddings/storage.d.ts.map +1 -1
- package/dist/embeddings/storage.js +2 -1
- package/dist/embeddings/storage.js.map +1 -1
- package/dist/learning/memory.d.ts.map +1 -1
- package/dist/learning/memory.js +5 -4
- package/dist/learning/memory.js.map +1 -1
- package/dist/manifest/static-manifest.js +1 -1
- package/dist/manifest/static-manifest.js.map +1 -1
- package/dist/manifest/store.js +1 -1
- package/dist/manifest.json +1 -1
- package/dist/migrations/differ.js +2 -2
- package/dist/migrations/differ.js.map +1 -1
- package/dist/migrations/index.d.ts +1 -0
- package/dist/migrations/index.d.ts.map +1 -1
- package/dist/migrations/index.js +2 -1
- package/dist/migrations/integer-width.d.ts +99 -0
- package/dist/migrations/integer-width.d.ts.map +1 -0
- package/dist/migrations/integer-width.js +237 -0
- package/dist/migrations/integer-width.js.map +1 -0
- package/dist/migrations/minor-units.d.ts +2 -2
- package/dist/migrations/minor-units.d.ts.map +1 -1
- package/dist/migrations/minor-units.js +16 -14
- package/dist/migrations/minor-units.js.map +1 -1
- package/dist/migrations/tracker.d.ts.map +1 -1
- package/dist/migrations/tracker.js +28 -20
- package/dist/migrations/tracker.js.map +1 -1
- package/dist/migrations.js +2 -1
- package/dist/schema/ddl/duckdb-strategy.d.ts.map +1 -1
- package/dist/schema/ddl/duckdb-strategy.js +1 -0
- package/dist/schema/ddl/duckdb-strategy.js.map +1 -1
- package/dist/schema/ddl/postgres-strategy.d.ts.map +1 -1
- package/dist/schema/ddl/postgres-strategy.js +2 -0
- package/dist/schema/ddl/postgres-strategy.js.map +1 -1
- package/dist/schema/generator.d.ts +1 -33
- package/dist/schema/generator.d.ts.map +1 -1
- package/dist/schema/generator.js +1 -150
- package/dist/schema/generator.js.map +1 -1
- package/dist/schema/index.d.ts +1 -2
- package/dist/schema/index.d.ts.map +1 -1
- package/dist/schema/index.js +1 -2
- package/dist/schema/live-parity.d.ts +3 -1
- package/dist/schema/live-parity.d.ts.map +1 -1
- package/dist/schema/live-parity.js +19 -0
- package/dist/schema/live-parity.js.map +1 -1
- package/dist/schema/types.d.ts +17 -10
- package/dist/schema/types.d.ts.map +1 -1
- package/dist/schema.js +1 -2
- package/dist/smrt-knowledge.json +7 -7
- package/dist/system/retention.d.ts.map +1 -1
- package/dist/system/retention.js +3 -3
- package/dist/system/retention.js.map +1 -1
- package/dist/system/schema.d.ts +2 -1
- package/dist/system/schema.d.ts.map +1 -1
- package/dist/system/schema.js +4 -2
- package/dist/system/schema.js.map +1 -1
- package/dist/utils/safe-integer.d.ts +19 -0
- package/dist/utils/safe-integer.d.ts.map +1 -0
- package/dist/utils/safe-integer.js +31 -0
- package/dist/utils/safe-integer.js.map +1 -0
- package/dist/utils.d.ts +2 -1
- package/dist/utils.d.ts.map +1 -1
- package/dist/utils.js +6 -3
- package/dist/utils.js.map +1 -1
- package/dist/vite-plugin/index.d.ts.map +1 -1
- package/dist/vite-plugin/index.js +0 -53
- package/dist/vite-plugin/index.js.map +1 -1
- package/package.json +4 -4
- package/dist/schema/override-system.d.ts +0 -43
- package/dist/schema/override-system.d.ts.map +0 -1
- package/dist/schema/override-system.js +0 -205
- package/dist/schema/override-system.js.map +0 -1
package/dist/smrt-knowledge.json
CHANGED
|
@@ -3,17 +3,17 @@
|
|
|
3
3
|
"sensitiveFieldsExcluded": true,
|
|
4
4
|
"generatedAt": "1970-01-01T00:00:00.000Z",
|
|
5
5
|
"packageName": "@happyvertical/smrt-core",
|
|
6
|
-
"packageVersion": "0.
|
|
6
|
+
"packageVersion": "0.42.0",
|
|
7
7
|
"sourceManifestPath": "dist/manifest.json",
|
|
8
8
|
"agentDocPath": "AGENTS.md",
|
|
9
9
|
"sourceHashes": {
|
|
10
|
-
"manifest": "
|
|
11
|
-
"packageJson": "
|
|
12
|
-
"agents": "
|
|
10
|
+
"manifest": "faa718cd7951ef50fb3d97d41e277f40f3718f88b53f6714e66b7816588accd9",
|
|
11
|
+
"packageJson": "f55459b145dcf91bd02e5087f2bed5ddbce339b92143ec2de216b4187470aec7",
|
|
12
|
+
"agents": "26f3bcb1b417610ecd69ef7426bb065ca5a9cde3055ff982c04cb6699ac26f72",
|
|
13
13
|
"moduleDoc:agents/change-feed.md": "1530ded9ed605aa9b8a772b3ba4dd992a3f797f4d7c9ede3b3389dd7bfb401fa",
|
|
14
14
|
"moduleDoc:agents/change-signals.md": "d9cb6a5541728ffea46607a6b1d4fa61d4621849f2b4ea86a0645fbb0af892e9",
|
|
15
15
|
"moduleDoc:agents/generators.md": "2b6ccd5ff557293f2ce84d25cc254ceb518f4c274d93b7e3175985b18f36f564",
|
|
16
|
-
"moduleDoc:agents/schema-paths.md": "
|
|
16
|
+
"moduleDoc:agents/schema-paths.md": "ffb1cbdd9429c0115ec414425598c6599ff7022d82c2b2a99e68854f43207ec0",
|
|
17
17
|
"moduleDoc:agents/query-bounds.md": "3a0601ddaf2bea4e90e3f22e16a2eb3508a6fb8c019e3fb2c1f930c90a377cd5"
|
|
18
18
|
},
|
|
19
19
|
"exports": [
|
|
@@ -913,7 +913,7 @@
|
|
|
913
913
|
"polymorphicAssociations": 1,
|
|
914
914
|
"uuidColumns": 3
|
|
915
915
|
},
|
|
916
|
-
"agentDoc": "# @happyvertical/smrt-core\n\nORM, code generation, AI integration, and the DispatchBus. Everything else builds on this.\n\nKey surfaces are `SmrtObject`, `SmrtCollection`, `ObjectRegistry`,\n`DispatchBus`, `GlobalInterceptors`, and `LearningMemory`; this file documents\ntheir invariants and source locations, and the module docs below cover the\nper-subsystem semantics.\n\n## Modules\n\nSubsystem semantics live in sibling module docs — read the one for the\nsubsystem you are editing. This file keeps what holds across all of them.\n\n| Module | Scope | Module doc |\n|---|---|---|\n| `src/change-feed.ts` | the adapter-agnostic change-observation spine — `_smrt_changes`, cursors, table versions, generated `_changes` routes, retention | [agents/change-feed.md](agents/change-feed.md) |\n| `src/change-signals.ts` + the generated `_events` SSE route | the push companion to the change feed — the signal bus, cross-replica fan-out, the SSE route, and its documented gaps | [agents/change-signals.md](agents/change-signals.md) |\n| `src/generators/` + `src/vite-plugin/web-collections.ts` | REST/CLI/MCP/web-collection generation, the `manifestHash` emission sites, and generated conditional-GET / ETag v2 semantics | [agents/generators.md](agents/generators.md) |\n| `src/schema/` | the five `SchemaGenerator` entry points, which two reach production, why schema drift stayed invisible, and the #2382 index/tenancy rules | [agents/schema-paths.md](agents/schema-paths.md) |\n\n## SmrtObject Lifecycle\n\n`constructor(options)` → `initialize()` → ready for `save()`/`delete()`/`loadFromId()`\n\n- `initialize()`: loads field initializers, applies option values (options override initializers), loads from DB if id/slug provided\n- `save()`: upsert with STI validation, interceptor execution, auto-embeddings. Persisted objects (`isPersisted` — set by DB hydration and successful saves) upsert on `['id']` so natural-key edits (e.g. slug renames) update in place; new objects upsert on the natural-key conflict columns for ingestion-style dedup (#1472)\n- `is(criteria)` / `do(instructions)` / `describe()`: AI operations via function calling. They inject the object's own `toPublicJSON()` (sensitive fields stripped) as a \"content body\" so the model reasons over the instance. Options: `includeData: false` skips injection (for callers that already curate the relevant fields into the instruction); `maxDataLength` overrides the truncation budget. Neither key is forwarded to `ai.message()`. (#1567)\n- `save()` error contract (#2366): unique/PK violation → `ValidationError` `VALIDATION_UNIQUE_CONSTRAINT`, NOT NULL → `VALIDATION_REQUIRED_FIELD`, both on the first attempt on every adapter; any other database failure → `DatabaseError` with the driver error on `cause`\n- `getSlug()`: auto-generates from name → title → label → id\n- `loadRelated(fieldName)`: lazy-loads relationships (cached in `_loadedRelationships` Map)\n\n## LearningMemory (#1886)\n\n`LearningMemory` provides tenant-isolated, confidence-scored recall over\n`_smrt_contexts` plus optional injected semantic search. `capture()` reinforces\nsuccesses and decays failures while updating outcome counters; `recall()`\napplies confidence, expiry, time-decay, and hierarchical-scope filters and\nrefreshes `last_used_at`. Keep semantic search behind the\n`SmrtCollection.semanticSearch`-compatible injection boundary.\n\n## SmrtCollection Query\n\n```typescript\nawait collection.list({\n where: { status: 'active', 'price >': 10 },\n limit: 50, offset: 0, orderBy: 'created_at DESC'\n});\n```\n\nProjection primitive (#1902): pass `select: ['id', 'title', 'tenantId']` to\n`list()` when an admin/list workflow needs compact rows. `select` uses SMRT\nfield names, maps them to DB columns internally, and returns plain objects keyed\nby the same SMRT field names without hydrating `SmrtObject` instances. It\ncomposes with `where`, `orderBy`, `limit`, and `offset`; `beforeList`\ninterceptors still run. It is for column-backed fields only and cannot combine\nwith `include`/relationship eager loading.\n\n`list()` and `query()` hydrate model instances serially in result order because\nan `initialize()` hook may query through the same transaction-bound PostgreSQL\nclient. Keep this serialization invariant; use `select` when callers need plain\nrows without model hydration.\n\n**WHERE operators**: `=`, `>`, `<`, `>=`, `<=`, `!=`, `in`, `not in`, `like`.\nArrays auto-detect `IN`. NULL is a value, not an operator: `{ deletedAt: null }`\nrenders `IS NULL` and `{ 'deletedAt !=': null }` renders `IS NOT NULL`.\n\nThis list is the set `@happyvertical/sql`'s `buildWhere` can execute, and\n`convertWhereKeys` accepts nothing outside it — an operator accepted here but\nunknown there fails inside the query builder, after the API said the query was\nvalid (#2276). Two entries were removed for that reason and now reject at the\nAPI boundary: `contains` (never existed in the SQL layer; use `like` with\nexplicit wildcards) and dot-notation JSON paths such as `metadata.userId` (never\nrewritten into an extraction expression, so they reached SQL as qualified column\nreferences). Re-adding either requires the query builder to support it first;\n`src/__tests__/issue-2276-where-contract.test.ts` executes every accepted\noperator against a database to keep the two in step.\n\nSTI child collections auto-filter by `_meta_type`. Query bounds — `LIMIT 1` on `get()`, the `limit`/`offset` parser, the `orderBy` whitelist and sensitive/permission refusals, and the deterministic generated-list ordering (#2367) — are in [agents/query-bounds.md](agents/query-bounds.md).\n\n## Bounded Collection Read Plans\n\nUse `executeCollectionReadPlan()` when one operation needs several independent\ncollections. It bounds top-level `collection.list()` concurrency while keeping\nall reads on the normal registry/collection path. Callers must choose an\nexplicit positive `maxConcurrency` and pass their normal shared\n`collectionOptions` when database or tenant context matters.\n\nThe executor deliberately does not compose SQL, cache the plan, or change pool\ndefaults. On failure it stops starting queued entries, drains operations already\nin flight, and rethrows the first error.\n\n## Object Memory & Semantic Search\n\nTwo persistence primitives every `SmrtObject`/`SmrtCollection` inherits — load-bearing for learning agents, usable by any object. Full guide: `docs/content/core.md` → \"Context Memory System\".\n\n- **Context memory** (`remember`/`recall`/`recallAll`/`forget`/`forgetScope`, table `_smrt_contexts`): stores any JSON value keyed by `(owner_class, owner_id, scope, key, version)` with a `confidence` score (0–1) and a stored `expiresAt` (metadata — `recall()` does **not** filter expired rows; expiry is caller-managed). `recall()` returns the highest-confidence match with an optional `minConfidence` floor and **opt-in** hierarchical scope fallback (`includeAncestors: true` → `'a/b/c' → 'a/b' → 'a' → 'global'`; default off); `recallAll()` returns a `Map`. Typical use: cache a learned strategy (e.g. a working selector per host) and reuse it across sessions. `success_count`/`failure_count` columns exist for outcome-weighting: `SmrtObject.remember()` leaves them untouched, `SmrtCollection.remember()` resets them to zero, and neither recall path updates them. `LearningMemory` is the layer that maintains them (and that does filter expired rows).\n- **Semantic search** (on `SmrtCollection`, table `_smrt_embeddings`): `semanticSearch(query)`, `findSimilar(object)`, `findSimilarToEmbedding(vector)` — cosine ranking over embeddings of the fields declared in `@smrt({ embeddings })`. Native pgvector/HNSW when configured, in-memory `CosineSimilarity` fallback otherwise; default local model `Xenova/bge-base-en-v1.5` (768-dim) or AI `text-embedding-3-small`. Hits hydrate via `list({ 'id in': … })`, so `@TenantScoped` isolation applies to results.\n\n## @smrt() Decorator Options\n\nKey options: `tableName`, `tableStrategy` ('cti'|'sti'), `conflictColumns`, `indexes` (declared multi-column indexes, #2357 — see \"Schema paths\"), `api`/`mcp`/`cli` (generation config), `ai` (callable methods), `hooks` (beforeSave/afterSave/beforeDelete/afterDelete), `embeddings` (auto-generate), `tenantScoped`, `agent`, `ui` (`{ icon, label, description }` — nav/help hints round-tripped through the manifest as plain data; `description` is the object-level seed for form-level help, #2046).\n\nRegistration sets `SMRT_TABLE_NAME` static property (survives minification).\n\n## @field() UI hints (#2046)\n\n`@field({ ui: { basic, group, order, locked } })` — a static, presentation-only\nseed for the field-policy rail (epic #2045). Carried in the manifest under the\nfield's `_meta.ui` (never a top-level `FieldDefinition` key), readable at\nruntime via `getAllFields()` at `field._meta.ui`, and emitted (sanitized) with\n`description` into generated web-collection definitions and browser MCP tool\nschemas. No schema/persistence/security effect — `sensitive`/`readPermission`\nstay the security rail, and `sensitive`/`transient` fields never emit to the\nclient at all.\n\n## Domain Knowledge Artifacts\n\n`smrtPlugin()` writes runtime manifests and agent/developer knowledge artifacts:\n\n- local dev/build: `.smrt/manifest.json` and `.smrt/smrt-knowledge.json`\n- package build: `dist/manifest.json` and `dist/smrt-knowledge.json`\n\nKeep `manifest.json` runtime-focused. `smrt-knowledge.json` is the deterministic\nagent contract for downstream review and architecture tools.\n\nThe schema-version-1 object projection is additive and high-signal: it retains\nnormalized tenant mode/field, explicit `cti`/`sti` strategy, conflict columns,\nmethod signatures, and field defaults/constraints/readonly/transient flags.\nSensitive fields are removed before both `fields` and `relationships` are\nderived, including legacy flags stored under `_meta`; matching field and\nsnake-case column names are also removed from projected conflict columns, and a\nsensitive custom tenant field is omitted while retaining scope and mode.\nGenerated artifacts assert this boundary with `sensitiveFieldsExcluded: true`;\nthe optional marker keeps schema version 1 additive while letting readers\nidentify older artifacts that require raw-manifest corroboration.\n\nConfig precedence for knowledge is defaults → top-level `knowledge` in\n`smrt.config.ts` → `packages[packageName].knowledge` → plugin option →\nobject-level `@smrt({ knowledge })`.\n\nObject-level `knowledge: false` excludes an object from authored context only;\nit must not change runtime manifest registration. Use\n`knowledge: { tags, summary, risks }` for review-sensitive domain objects.\n\nHTTP knowledge routes are disabled by default. If `knowledge.api.enabled` is\ntrue, generated SvelteKit routes must stay GET-only and guarded by dev mode or\nadmin auth.\n\n## DispatchBus\n\n- `emit(signalType, payload, metadata)` → creates persistent Dispatch record\n- `on(pattern, handler)` → in-memory handler (immediate)\n- `subscribe({ signalType, subscriber })` → persistent subscription (survives restarts)\n- `process(subscriberName, handler)` → process pending dispatches\n- Wildcards: `campaign.*` matches `campaign.completed` (single segment only)\n- Tables: `_smrt_dispatch`, `_smrt_dispatch_subscriptions`\n- Status: `pending → processing → completed` (or `failed`)\n\n## Single Table Inheritance (STI)\n\n- Base: `@smrt({ tableStrategy: 'sti' })` — children inherit, share one table\n- Discriminator: `_meta_type` column with qualified names (`@happyvertical/smrt-content:Article`)\n- Child fields: `@meta()` decorator → stored in `_meta_data` JSONB (not as columns)\n- Polymorphic queries: collection loads `_meta_type`, creates correct subclass dynamically\n- Validation: fail-fast on save if `_meta_type` missing or mismatched\n\n## Child Accessors (R10)\n\n`src/child-accessors.ts` installs a consistent `get<FieldName>()` instance method for every `@oneToMany` field at `@smrt()` registration time (e.g. `@oneToMany('OrderItem') items` → `order.getItems()`), delegating to `loadRelatedMany`. Two invariants:\n\n- **Additive** — never overwrites a hand-rolled method of the same name (checks the whole prototype chain). `Profile.getMetadata()` (key-value) and `ProfileRelationship.getTerms()` are preserved.\n- **Runtime-only** — attached to the prototype, invisible to the build-time manifest, so it never leaks into the REST/CLI/MCP surface.\n\nWhen the target declares multiple FKs back to the parent, annotate `@oneToMany(Target, { foreignKey: '<inverseField>' })`; `loadRelatedMany` and the eager `include:` loader both honor it (else first-match).\n\n## Vite Plugin\n\n```typescript\n// vite.config.ts — required for @smrt() decorators (Vite 8+, oxc transform)\nexport default defineConfig({\n oxc: {\n decorator: {\n legacy: true,\n emitDecoratorMetadata: true,\n },\n },\n});\n```\n\nUnder Vite 8 the oxc transform does not honor the pre-Vite-8 `esbuild.tsconfigRaw`\nrecipe (or tsconfig `experimentalDecorators` reached through SvelteKit's\n`extends \"./.svelte-kit/tsconfig.json\"` chain), so that recipe throws\n`SyntaxError: Invalid or unexpected token` on the first SSR request. Configure\ndecorators through `oxc.decorator` instead. Consumers still pinned on vite<8 need\nthe legacy `esbuild.tsconfigRaw` form with `experimentalDecorators: true,\nemitDecoratorMetadata: true`.\n\nFor independent CI invocations, both `smrtPlugin()` and `smrtConsumer()` accept\nthe same `generationSnapshot: { path, sha256, provenance, sourceRoot }`. The\nschema-v1 snapshot produced by `serializeSmrtGenerationSnapshot()` contains the\nmerged project/dependency manifest, portable source paths, and source-file\ndigests; each plugin selects its own view. Reuse mode fails closed on\nbyte/provenance/path/content drift, skips scans and manifest writes, and still\ngenerates routes, types, registration, and virtual modules. Omit it for normal\nlocal development and watch mode.\n\n## Schema paths (#2382)\n\nProduction DDL comes from the **manifest** paths\n(`generateSTISchemaFromManifest`/`generateCTISchemaFromManifest`, selected in\n`src/scanner/manifest-generator.ts` → registered `schema` → `db:migrate`). The\n**registry** paths feed `getTestDatabase()` and emit foreign-key indexes\nproduction never gets: the suite runs on a richer schema than it ships.\n\n- Change column/index emission on every shipping path, proven by the path-parity\n test `src/schema/schema-path-parity.test.ts` (#2359; index rules in the module doc). A \"same as migrations\" comment is a claim to check.\n- Every new query predicate ships with its index, or a reason it doesn't.\n- Numeric types, uuid casts, conflict targets, timestamps, migrations: run the\n `test:postgres` lane — SQLite affinity accepts what PostgreSQL rejects.\n- Read `dist/manifest.json`/regenerated schemas for what a decorator produced;\n count across all packages instead of sampling.\n- Tenant scoping is whole-path: every unique constraint and conflict target on a\n tenant-scoped table carries the tenant column, and every read path — not only\n `list()` — is interceptor-aware.\n- Rolling indexes out is part of the change: a bulk `CREATE INDEX` batch needs\n the bounded, concurrent migrate path (#2362, Gotchas), or it takes production\n down on deploy.\n\n## Gotchas\n\n- **Filesystem support is a lazy boundary (#1979)**: `SmrtClass` acquires `options.fs` adapters via `createFilesystemAdapter()` (`src/filesystem-loader.ts`), never a static `@happyvertical/files` import — the files SDK statically pulls @aws-sdk/client-s3 and reaches googleapis, and a static edge here would land it in every downstream SSR bundle. Node/tsx/vite-dev runtimes resolve it on first use; fully-bundled deployments import `@happyvertical/smrt-core/filesystem` at startup. Use `importOptionalDependency()` (`src/lazy-external.ts`) for any similar optional heavyweight dependency.\n- **Never override toJSON()** — handles STI discriminator + meta field extraction. Use `transformJSON()`\n- **Property init order**: TypeScript initializers run first, then `initialize()` applies option values (options win)\n- **No runtime schema creation**: application tables must be prepared explicitly via migrations/tooling; runtime verification is `tableExists()` only (`src/schema/table-verifier.ts`) — no column, type, or index check\n- **PostgreSQL migrate batches are always time-bounded (#2362)**: `MigrationTracker.applyAll({ atomic: true })` emits `SET LOCAL lock_timeout`/`statement_timeout` before any DDL, so a batch blocked on one table cannot hold its earlier locks indefinitely. `postgresSafe: true` adds concurrent-index mode — non-index DDL commits atomically, then index DDL runs `CONCURRENTLY` on a session pinned via `db.acquireSession()` (a pooled `db.query` would not keep the `SET` and the DDL on one connection). That mode is deliberately **not atomic**: unfinished index migrations are recorded `failed`, not `running`, and their `error_message` carries a `[smrt: concurrent-index phase 1 committed]` marker so a reconciling re-run resumes at the index build instead of replaying committed DDL. INVALID indexes are found via `pg_index.indisvalid` (`pg_indexes` reports them as present) and dropped before rebuild. Operational detail: `packages/cli/AGENTS.md`.\n- **Retry logic is transient-only (#2366)**: `db.get()`/`db.upsert()` retry 4× total (initial + 3), but `ErrorUtils.withRetry` classifies via the cause chain (`src/db-errors.ts`) and rethrows deterministic failures immediately — constraint violations, bad input syntax, missing tables, aborted PG tx (`25P02`). `@happyvertical/sql` stringifies the driver text into `context.originalError`, so **never match `error.message`**; use `classifyDatabaseError()` / `isUniqueViolationError()` / `isAbortedTransactionError()`.\n- **Field caching**: `_cachedFields` populated during `Collection.create()` — eliminates async `getFields()` per query\n- **Smart cloning**: arrays/objects shallow-cloned in property init to prevent aliasing (Issue #22)\n- **Table verification cache**: `isTableVerified(dbUrl, tableName)` avoids redundant `tableExists()` calls\n- **Manifest required**: build-time AST scanning creates manifest. Without vitest plugin → \"No field metadata\"\n- **ManifestBuilder fails on scanner errors**: every production manifest path\n must abort before adapting partial scan results. A syntax error or unresolved\n `@smrt()` config spread cannot be allowed to emit a default-open manifest.\n- **Vite plugin loads scanner from `dist/` first**: `src/vite-plugin/import-build-aware.ts` prefers `dist/` when it exists on disk; it only falls back to `src/` on fresh clones. So if you edit `src/scanner/*.ts` or `src/schema/generator.ts` and want those edits reflected in consumer manifest generation, you must rebuild (`pnpm build` or have `pnpm dev` / `pnpm build:watch` running in core). This is intentional — sniffing `.ts` vs `.js` via `import.meta.url` was non-deterministic under tsx and broke 12–13 publishes (#1139).\n- **Bundled registry ownership**: flattened production bundles can rewrite constructor names and make decorator-time stack inference attribute provider code to the consumer. Generated registration repairs identity only from the exact imported constructor plus an explicit package and isolated one-object manifest; never infer ownership from output paths, simple names, or table names. Distinct packages may export the same simple name under qualified keys. The production-consumer gate lives in `packages/bundle-gate/src/__tests__/registry-identity.spec.ts` (#2308).\n",
|
|
916
|
+
"agentDoc": "# @happyvertical/smrt-core\n\nORM, code generation, AI integration, and the DispatchBus. Everything else builds on this.\n\nKey surfaces are `SmrtObject`, `SmrtCollection`, `ObjectRegistry`,\n`DispatchBus`, `GlobalInterceptors`, and `LearningMemory`; this file documents\ntheir invariants and source locations, and the module docs below cover the\nper-subsystem semantics.\n\n## Modules\n\nSubsystem semantics live in sibling module docs — read the one for the\nsubsystem you are editing. This file keeps what holds across all of them.\n\n| Module | Scope | Module doc |\n|---|---|---|\n| `src/change-feed.ts` | the adapter-agnostic change-observation spine — `_smrt_changes`, cursors, table versions, generated `_changes` routes, retention | [agents/change-feed.md](agents/change-feed.md) |\n| `src/change-signals.ts` + the generated `_events` SSE route | the push companion to the change feed — the signal bus, cross-replica fan-out, the SSE route, and its documented gaps | [agents/change-signals.md](agents/change-signals.md) |\n| `src/generators/` + `src/vite-plugin/web-collections.ts` | REST/CLI/MCP/web-collection generation, the `manifestHash` emission sites, and generated conditional-GET / ETag v2 semantics | [agents/generators.md](agents/generators.md) |\n| `src/schema/` | the four `SchemaGenerator` entry points, which two reach production, why schema drift stayed invisible, and the #2382 index/tenancy rules | [agents/schema-paths.md](agents/schema-paths.md) |\n\n## SmrtObject Lifecycle\n\n`constructor(options)` → `initialize()` → ready for `save()`/`delete()`/`loadFromId()`\n\n- `initialize()`: loads field initializers, applies option values (options override initializers), loads from DB if id/slug provided\n- `save()`: upsert with STI validation, interceptor execution, auto-embeddings. Persisted objects (`isPersisted` — set by DB hydration and successful saves) upsert on `['id']` so natural-key edits (e.g. slug renames) update in place; new objects upsert on the natural-key conflict columns for ingestion-style dedup (#1472)\n- `is(criteria)` / `do(instructions)` / `describe()`: AI operations via function calling. They inject the object's own `toPublicJSON()` (sensitive fields stripped) as a \"content body\" so the model reasons over the instance. Options: `includeData: false` skips injection (for callers that already curate the relevant fields into the instruction); `maxDataLength` overrides the truncation budget. Neither key is forwarded to `ai.message()`. (#1567)\n- `save()` error contract (#2366): unique/PK violation → `ValidationError` `VALIDATION_UNIQUE_CONSTRAINT`, NOT NULL → `VALIDATION_REQUIRED_FIELD`, both on the first attempt on every adapter; any other database failure → `DatabaseError` with the driver error on `cause`\n- `getSlug()`: auto-generates from name → title → label → id\n- `loadRelated(fieldName)`: lazy-loads relationships (cached in `_loadedRelationships` Map)\n\n## LearningMemory (#1886)\n\n`LearningMemory` provides tenant-isolated, confidence-scored recall over\n`_smrt_contexts` plus optional injected semantic search. `capture()` reinforces\nsuccesses and decays failures while updating outcome counters; `recall()`\napplies confidence, expiry, time-decay, and hierarchical-scope filters and\nrefreshes `last_used_at`. Keep semantic search behind the\n`SmrtCollection.semanticSearch`-compatible injection boundary.\n\n## SmrtCollection Query\n\n```typescript\nawait collection.list({\n where: { status: 'active', 'price >': 10 },\n limit: 50, offset: 0, orderBy: 'created_at DESC'\n});\n```\n\nProjection primitive (#1902): pass `select: ['id', 'title', 'tenantId']` to\n`list()` when an admin/list workflow needs compact rows. `select` uses SMRT\nfield names, maps them to DB columns internally, and returns plain objects keyed\nby the same SMRT field names without hydrating `SmrtObject` instances. It\ncomposes with `where`, `orderBy`, `limit`, and `offset`; `beforeList`\ninterceptors still run. It is for column-backed fields only and cannot combine\nwith `include`/relationship eager loading.\n\n`list()` and `query()` hydrate model instances serially in result order because\nan `initialize()` hook may query through the same transaction-bound PostgreSQL\nclient. Keep this serialization invariant; use `select` when callers need plain\nrows without model hydration.\n\n**WHERE operators**: `=`, `>`, `<`, `>=`, `<=`, `!=`, `in`, `not in`, `like`.\nArrays auto-detect `IN`. NULL is a value, not an operator: `{ deletedAt: null }`\nrenders `IS NULL` and `{ 'deletedAt !=': null }` renders `IS NOT NULL`.\n\nThis list is the set `@happyvertical/sql`'s `buildWhere` can execute, and\n`convertWhereKeys` accepts nothing outside it — an operator accepted here but\nunknown there fails inside the query builder, after the API said the query was\nvalid (#2276). Two entries were removed for that reason and now reject at the\nAPI boundary: `contains` (never existed in the SQL layer; use `like` with\nexplicit wildcards) and dot-notation JSON paths such as `metadata.userId` (never\nrewritten into an extraction expression, so they reached SQL as qualified column\nreferences). Re-adding either requires the query builder to support it first;\n`src/__tests__/issue-2276-where-contract.test.ts` executes every accepted\noperator against a database to keep the two in step.\n\nSTI child collections auto-filter by `_meta_type`. Query bounds — `LIMIT 1` on `get()`, the `limit`/`offset` parser, the `orderBy` whitelist and sensitive/permission refusals, and the deterministic generated-list ordering (#2367) — are in [agents/query-bounds.md](agents/query-bounds.md).\n\n## Bounded Collection Read Plans\n\nUse `executeCollectionReadPlan()` when one operation needs several independent\ncollections. It bounds top-level `collection.list()` concurrency while keeping\nall reads on the normal registry/collection path. Callers must choose an\nexplicit positive `maxConcurrency` and pass their normal shared\n`collectionOptions` when database or tenant context matters.\n\nThe executor deliberately does not compose SQL, cache the plan, or change pool\ndefaults. On failure it stops starting queued entries, drains operations already\nin flight, and rethrows the first error.\n\n## Object Memory & Semantic Search\n\nTwo persistence primitives every `SmrtObject`/`SmrtCollection` inherits — load-bearing for learning agents, usable by any object. Full guide: `docs/content/core.md` → \"Context Memory System\".\n\n- **Context memory** (`remember`/`recall`/`recallAll`/`forget`/`forgetScope`, table `_smrt_contexts`): stores any JSON value keyed by `(owner_class, owner_id, scope, key, version)` with a `confidence` score (0–1) and a stored `expiresAt` (metadata — `recall()` does **not** filter expired rows; expiry is caller-managed). `recall()` returns the highest-confidence match with an optional `minConfidence` floor and **opt-in** hierarchical scope fallback (`includeAncestors: true` → `'a/b/c' → 'a/b' → 'a' → 'global'`; default off); `recallAll()` returns a `Map`. Typical use: cache a learned strategy (e.g. a working selector per host) and reuse it across sessions. `success_count`/`failure_count` columns exist for outcome-weighting: `SmrtObject.remember()` leaves them untouched, `SmrtCollection.remember()` resets them to zero, and neither recall path updates them. `LearningMemory` is the layer that maintains them (and that does filter expired rows).\n- **Semantic search** (on `SmrtCollection`, table `_smrt_embeddings`): `semanticSearch(query)`, `findSimilar(object)`, `findSimilarToEmbedding(vector)` — cosine ranking over embeddings of the fields declared in `@smrt({ embeddings })`. Native pgvector/HNSW when configured, in-memory `CosineSimilarity` fallback otherwise; default local model `Xenova/bge-base-en-v1.5` (768-dim) or AI `text-embedding-3-small`. Hits hydrate via `list({ 'id in': … })`, so `@TenantScoped` isolation applies to results.\n\n## @smrt() Decorator Options\n\nKey options: `tableName`, `tableStrategy` ('cti'|'sti'), `conflictColumns`, `indexes` (declared multi-column indexes, #2357 — see \"Schema paths\"), `api`/`mcp`/`cli` (generation config), `ai` (callable methods), `hooks` (beforeSave/afterSave/beforeDelete/afterDelete), `embeddings` (auto-generate), `tenantScoped`, `agent`, `ui` (`{ icon, label, description }` — nav/help hints round-tripped through the manifest as plain data; `description` is the object-level seed for form-level help, #2046).\n\nRegistration sets `SMRT_TABLE_NAME` static property (survives minification).\n\n## @field() UI hints (#2046)\n\n`@field({ ui: { basic, group, order, locked } })` — a static, presentation-only\nseed for the field-policy rail (epic #2045). Carried in the manifest under the\nfield's `_meta.ui` (never a top-level `FieldDefinition` key), readable at\nruntime via `getAllFields()` at `field._meta.ui`, and emitted (sanitized) with\n`description` into generated web-collection definitions and browser MCP tool\nschemas. No schema/persistence/security effect — `sensitive`/`readPermission`\nstay the security rail, and `sensitive`/`transient` fields never emit to the\nclient at all.\n\n## Domain Knowledge Artifacts\n\n`smrtPlugin()` writes runtime manifests and agent/developer knowledge artifacts:\n\n- local dev/build: `.smrt/manifest.json` and `.smrt/smrt-knowledge.json`\n- package build: `dist/manifest.json` and `dist/smrt-knowledge.json`\n\nKeep `manifest.json` runtime-focused. `smrt-knowledge.json` is the deterministic\nagent contract for downstream review and architecture tools.\n\nThe schema-version-1 object projection is additive and high-signal: it retains\nnormalized tenant mode/field, explicit `cti`/`sti` strategy, conflict columns,\nmethod signatures, and field defaults/constraints/readonly/transient flags.\nSensitive fields are removed before both `fields` and `relationships` are\nderived, including legacy flags stored under `_meta`; matching field and\nsnake-case column names are also removed from projected conflict columns, and a\nsensitive custom tenant field is omitted while retaining scope and mode.\nGenerated artifacts assert this boundary with `sensitiveFieldsExcluded: true`;\nthe optional marker keeps schema version 1 additive while letting readers\nidentify older artifacts that require raw-manifest corroboration.\n\nConfig precedence for knowledge is defaults → top-level `knowledge` in\n`smrt.config.ts` → `packages[packageName].knowledge` → plugin option →\nobject-level `@smrt({ knowledge })`.\n\nObject-level `knowledge: false` excludes an object from authored context only;\nit must not change runtime manifest registration. Use\n`knowledge: { tags, summary, risks }` for review-sensitive domain objects.\n\nHTTP knowledge routes are disabled by default. If `knowledge.api.enabled` is\ntrue, generated SvelteKit routes must stay GET-only and guarded by dev mode or\nadmin auth.\n\n## DispatchBus\n\n- `emit(signalType, payload, metadata)` → creates persistent Dispatch record\n- `on(pattern, handler)` → in-memory handler (immediate)\n- `subscribe({ signalType, subscriber })` → persistent subscription (survives restarts)\n- `process(subscriberName, handler)` → process pending dispatches\n- Wildcards: `campaign.*` matches `campaign.completed` (single segment only)\n- Tables: `_smrt_dispatch`, `_smrt_dispatch_subscriptions`\n- Status: `pending → processing → completed` (or `failed`)\n\n## Single Table Inheritance (STI)\n\n- Base: `@smrt({ tableStrategy: 'sti' })` — children inherit, share one table\n- Discriminator: `_meta_type` column with qualified names (`@happyvertical/smrt-content:Article`)\n- Child fields: `@meta()` decorator → stored in `_meta_data` JSONB (not as columns)\n- Polymorphic queries: collection loads `_meta_type`, creates correct subclass dynamically\n- Validation: fail-fast on save if `_meta_type` missing or mismatched\n\n## Child Accessors (R10)\n\n`src/child-accessors.ts` installs a consistent `get<FieldName>()` instance method for every `@oneToMany` field at `@smrt()` registration time (e.g. `@oneToMany('OrderItem') items` → `order.getItems()`), delegating to `loadRelatedMany`. Two invariants:\n\n- **Additive** — never overwrites a hand-rolled method of the same name (checks the whole prototype chain). `Profile.getMetadata()` (key-value) and `ProfileRelationship.getTerms()` are preserved.\n- **Runtime-only** — attached to the prototype, invisible to the build-time manifest, so it never leaks into the REST/CLI/MCP surface.\n\nWhen the target declares multiple FKs back to the parent, annotate `@oneToMany(Target, { foreignKey: '<inverseField>' })`; `loadRelatedMany` and the eager `include:` loader both honor it (else first-match).\n\n## Vite Plugin\n\n```typescript\n// vite.config.ts — required for @smrt() decorators (Vite 8+, oxc transform)\nexport default defineConfig({\n oxc: {\n decorator: {\n legacy: true,\n emitDecoratorMetadata: true,\n },\n },\n});\n```\n\nUnder Vite 8 the oxc transform does not honor the pre-Vite-8 `esbuild.tsconfigRaw`\nrecipe (or tsconfig `experimentalDecorators` reached through SvelteKit's\n`extends \"./.svelte-kit/tsconfig.json\"` chain), so that recipe throws\n`SyntaxError: Invalid or unexpected token` on the first SSR request. Configure\ndecorators through `oxc.decorator` instead. Consumers still pinned on vite<8 need\nthe legacy `esbuild.tsconfigRaw` form with `experimentalDecorators: true,\nemitDecoratorMetadata: true`.\n\nFor independent CI invocations, both `smrtPlugin()` and `smrtConsumer()` accept\nthe same `generationSnapshot: { path, sha256, provenance, sourceRoot }`. The\nschema-v1 snapshot produced by `serializeSmrtGenerationSnapshot()` contains the\nmerged project/dependency manifest, portable source paths, and source-file\ndigests; each plugin selects its own view. Reuse mode fails closed on\nbyte/provenance/path/content drift, skips scans and manifest writes, and still\ngenerates routes, types, registration, and virtual modules. Omit it for normal\nlocal development and watch mode.\n\n## Schema paths (#2382)\n\nProduction DDL comes from the **manifest** paths\n(`generateSTISchemaFromManifest`/`generateCTISchemaFromManifest`, selected in\n`src/scanner/manifest-generator.ts` → registered `schema` → `db:migrate`). The\n**registry** paths feed `getTestDatabase()` and emit foreign-key indexes\nproduction never gets: the suite runs on a richer schema than it ships.\n\n- Change column/index emission on every shipping path, proven by the path-parity\n test `src/schema/schema-path-parity.test.ts` (#2359; index rules in the module doc). A \"same as migrations\" comment is a claim to check.\n- Every new query predicate ships with its index, or a reason it doesn't.\n- Numeric types, uuid casts, conflict targets, timestamps, migrations: run the\n `test:postgres` lane — SQLite affinity accepts what PostgreSQL rejects.\n- Read `dist/manifest.json`/regenerated schemas for what a decorator produced;\n count across all packages instead of sampling.\n- Tenant scoping is whole-path: every unique constraint and conflict target on a\n tenant-scoped table carries the tenant column, and every read path — not only\n `list()` — is interceptor-aware.\n- Rolling indexes out is part of the change: a bulk `CREATE INDEX` batch needs\n the bounded, concurrent migrate path (#2362, Gotchas), or it takes production\n down on deploy.\n\n## Gotchas\n\n- **Filesystem support is a lazy boundary (#1979)**: `SmrtClass` acquires `options.fs` adapters via `createFilesystemAdapter()` (`src/filesystem-loader.ts`), never a static `@happyvertical/files` import — the files SDK statically pulls @aws-sdk/client-s3 and reaches googleapis, and a static edge here would land it in every downstream SSR bundle. Node/tsx/vite-dev runtimes resolve it on first use; fully-bundled deployments import `@happyvertical/smrt-core/filesystem` at startup. Use `importOptionalDependency()` (`src/lazy-external.ts`) for any similar optional heavyweight dependency.\n- **Never override toJSON()** — handles STI discriminator + meta field extraction. Use `transformJSON()`\n- **Property init order**: TypeScript initializers run first, then `initialize()` applies option values (options win)\n- **No runtime schema creation**: application tables must be prepared explicitly via migrations/tooling; runtime verification is `tableExists()` only (`src/schema/table-verifier.ts`) — no column, type, or index check\n- **PostgreSQL migrate batches are always time-bounded (#2362)**: `MigrationTracker.applyAll({ atomic: true })` emits `SET LOCAL lock_timeout`/`statement_timeout` before any DDL, so a batch blocked on one table cannot hold its earlier locks indefinitely. `postgresSafe: true` adds concurrent-index mode — non-index DDL commits atomically, then index DDL runs `CONCURRENTLY` on a session pinned via `db.acquireSession()` (a pooled `db.query` would not keep the `SET` and the DDL on one connection). That mode is deliberately **not atomic**: unfinished index migrations are recorded `failed`, not `running`, and their `error_message` carries a `[smrt: concurrent-index phase 1 committed]` marker so a reconciling re-run resumes at the index build instead of replaying committed DDL. INVALID indexes are found via `pg_index.indisvalid` (`pg_indexes` reports them as present) and dropped before rebuild. Operational detail: `packages/cli/AGENTS.md`.\n- **Retry logic is transient-only (#2366)**: `db.get()`/`db.upsert()` retry 4× total (initial + 3), but `ErrorUtils.withRetry` classifies via the cause chain (`src/db-errors.ts`) and rethrows deterministic failures immediately — constraint violations, bad input syntax, missing tables, aborted PG tx (`25P02`). `@happyvertical/sql` stringifies the driver text into `context.originalError`, so **never match `error.message`**; use `classifyDatabaseError()` / `isUniqueViolationError()` / `isAbortedTransactionError()`.\n- **Field caching**: `_cachedFields` populated during `Collection.create()` — eliminates async `getFields()` per query\n- **Smart cloning**: arrays/objects shallow-cloned in property init to prevent aliasing (Issue #22)\n- **Table verification cache**: `isTableVerified(dbUrl, tableName)` avoids redundant `tableExists()` calls\n- **Manifest required**: build-time AST scanning creates manifest. Without vitest plugin → \"No field metadata\"\n- **ManifestBuilder fails on scanner errors**: every production manifest path\n must abort before adapting partial scan results. A syntax error or unresolved\n `@smrt()` config spread cannot be allowed to emit a default-open manifest.\n- **Vite plugin loads scanner from `dist/` first**: `src/vite-plugin/import-build-aware.ts` prefers `dist/` when it exists on disk; it only falls back to `src/` on fresh clones. So if you edit `src/scanner/*.ts` or `src/schema/generator.ts` and want those edits reflected in consumer manifest generation, you must rebuild (`pnpm build` or have `pnpm dev` / `pnpm build:watch` running in core). This is intentional — sniffing `.ts` vs `.js` via `import.meta.url` was non-deterministic under tsx and broke 12–13 publishes (#1139).\n- **Bundled registry ownership**: flattened production bundles can rewrite constructor names and make decorator-time stack inference attribute provider code to the consumer. Generated registration repairs identity only from the exact imported constructor plus an explicit package and isolated one-object manifest; never infer ownership from output paths, simple names, or table names. Distinct packages may export the same simple name under qualified keys. The production-consumer gate lives in `packages/bundle-gate/src/__tests__/registry-identity.spec.ts` (#2308).\n",
|
|
917
917
|
"moduleDocs": [
|
|
918
918
|
{
|
|
919
919
|
"path": "agents/change-feed.md",
|
|
@@ -933,7 +933,7 @@
|
|
|
933
933
|
{
|
|
934
934
|
"path": "agents/schema-paths.md",
|
|
935
935
|
"module": "schema-paths",
|
|
936
|
-
"content": "# smrt-core/schema paths\n\nModule semantics for `src/schema/` — which `SchemaGenerator` entry point reaches\na real database, what each one emits, and the rules that keep them in step.\nPackage orientation, the cross-module invariants, and the traps that apply\nbefore editing anything live in [../AGENTS.md](../AGENTS.md) — read that first;\nits \"Schema paths\" section is the short form of everything below.\n\nWritten from the 2026-08-17 database-layer gap assessment (epic #2382). Symbol\nnames here are stable; the line numbers the assessment quotes are not, so trust\nthis call graph and re-grep before citing a location.\n\n## Five entry points, two of which ship\n\n`src/schema/generator.ts` exposes five index-emitting entry points. They do not\nproduce the same schema for the same class.\n\n| Entry point | Selected by | Status |\n|---|---|---|\n| `generateSTISchemaFromManifest` | `src/scanner/manifest-generator.ts` | **production** |\n| `generateCTISchemaFromManifest` | `src/scanner/manifest-generator.ts` | **production** |\n| `generateSTISchemaFromRegistry` | `src/testing/database.ts` (`getTestDatabase()`), `src/schema/utils.ts` (`generateSchema`; `ensureSchema` only as a fallback) | tests + runtime helpers |\n| `generateSchemaFromRegistry` | the same two callers | tests + runtime helpers |\n| `generateSchema` (AST) | the `smrt:schema` virtual module, which has no consumer | dead (#2380) |\n\nProduction DDL takes the manifest route:\n\n```\n@smrt() class ─▶ scanner ─▶ manifest.json ─▶ generate{STI,CTI}SchemaFromManifest\n ─▶ registered `schema` ─▶ ObjectRegistry.getAllSchemasAsDefinitions()\n ├─▶ smrt db:migrate | db:diff | db:status\n │ (the CLI drives SchemaComparer + MigrationTracker directly)\n └─▶ migrateSmrtSchemas() / getPendingSchemaStatements()\n (src/migrations/orchestrate.ts — exported for programmatic\n use; no in-repo caller outside its own tests)\n```\n\nThe suite takes the registry route. Before #2359 the registry route emitted\nindexes the manifest route did not — per-column foreign-key indexes, and STI\npartial FK indexes filtered by `_meta_type` — so tests ran against a richer\nschema than any deployment received, the manifest STI path populated a\n`fkColumnsByClass` map it never read, and the manifest CTI path had no FK loop\nat all. `src/testing/database.ts`'s \"same as migrations\" comment described an\nintent, not the code.\n\nSince #2359 the two families share one set of index helpers and\n`src/schema/schema-path-parity.test.ts` runs the same fixture manifest through\nthe manifest paths, through `ObjectRegistry.registerFromManifest()` + the\nregistry paths, and through `getAllSchemasAsDefinitions()`, asserting identical\ncolumn and index sets. Extend that fixture with every generator change; a\ndivergence is a bug in the generator, not an exception to add to the test.\n\n### Index rules (#2359)\n\n- **Reference columns are always indexed.** `ensureReferenceColumnIndexes()`\n runs last on every path and gives each `@foreignKey`, `@crossPackageRef` and\n tenant column `<table>_<column>_idx` unless an UNQUALIFIED index (no `WHERE`,\n no JSON path) already leads with it — the `conflictColumns` unique index or an\n `indexed: true` opt-in, or the column's own inline UNIQUE. A partial\n `WHERE _meta_type = …` index does not count: base-class polymorphic queries\n carry no discriminator predicate. `indexed: true` on a reference column is\n redundant. Roll the index wave out to production with\n `smrt db:migrate --postgres-safe` (concurrent-index mode, #2362): a plain\n atomic batch takes SHARE/ACCESS EXCLUSIVE locks for ~230 index builds. STI FK indexes are plain, one per\n column, not per-class partial.\n- **No index on the primary key.** `<table>_id_idx` is gone from every path,\n and `conflictColumns` equal to the PK column set emit no conflict index\n (`ON CONFLICT (id)` binds to the PK constraint). `SchemaComparer` drops the\n legacy non-unique single-column PK index from existing databases without\n `--drop-indexes` when the live table reports that column as its sole primary\n key (never a UNIQUE one — on PostgreSQL that may back a custom-named PRIMARY\n KEY constraint, and `DROP INDEX` on it would fail the atomic batch).\n- **Slug loading keeps its index.** Custom `conflictColumns` replace the\n `(slug, context)` unique index; `loadFromSlug()`/`getId()`/`getSavedId()`\n still filter on slug/context, so a plain `<table>_slug_context_idx` is kept\n (additive; routing those lookups through the conflict key would change which\n row a slug resolves to). The tenant-led default key below counts as serving\n it (`servesSlugLookup()`): a tenant-scoped slug lookup carries the tenant\n predicate (#2365) and is served by the prefix, so no second index.\n- **Tenant-scoped tables key per tenant (#2360).** A tenant-scoped class with\n no explicit `conflictColumns` upserts on, and indexes,\n `(tenant_id, slug, context)` — `(tenant_id, slug, context, _meta_type)` for\n an STI hierarchy — resolved by one rule on both paths:\n `ManifestGenerator.normalizeConflictColumns()` materializes it into\n `decoratorConfig.conflictColumns` for the manifest paths (so the manifest,\n the schema, `smrt-knowledge.json` and the runtime read one value), and\n `ObjectRegistry.getConflictColumns()` derives the same value at runtime from\n the schema owner's `tenantScoped` config (`ObjectRegistry.getTenantColumn()`;\n an STI child resolves through its root; a `@report` class through its\n group/bucket columns; a custom primary key through that key). Explicit\n `conflictColumns` are never rewritten. `src/schema/conflict-target.ts` holds\n the shared helpers. Consequences: the index NAME stays\n `<table>_slug_context_idx` / `_slug_context_meta_type_idx`, so the differ\n swaps the columns of an existing global unique in place by name (a superset\n key — creating it cannot fail on existing rows); the tenant-led key also\n serves the tenant column, so `<table>_tenant_id_idx` is no longer emitted\n for those tables (an existing one is an orphan the differ drops only with\n `--drop-indexes`); NULL-tenant rows (`mode: 'optional'` outside a tenant\n context) dedup among themselves through the SDK's null-aware upsert\n (`IS NOT DISTINCT FROM` under a PostgreSQL advisory lock / an in-process\n lock on SQLite) — application-enforced now, where the old global index was\n database-enforced: the tenant-led index treats NULLs as distinct, so raw SQL\n can insert two global rows with one slug, and a raw\n `ON CONFLICT (slug, context…)` against such a table no longer binds (use\n `WHERE NOT EXISTS`, plus an advisory lock on PostgreSQL). Emitting\n `NULLS NOT DISTINCT` on PostgreSQL ≥ 15 (the SDK already detects it) would\n restore the database arbiter — a follow-up. The `save()` path serializes an\n unset tenant field as an explicit `NULL` whatever its registered type,\n because the SDK rejects an upsert whose conflict column is missing from the\n row.\n- **Rolling the tenant-led key out (#2360).** There is no mixed-version state:\n new code against the old index fails every NEW-object create on a\n tenant-scoped default-key table (PostgreSQL 42P10, SQLite \"ON CONFLICT\n clause does not match…\"), and old code against the new index fails the same\n way, because the conflict target must match the unique index's column set\n exactly; only persisted objects (upsert on `id`) keep saving. Deploy the code\n and run `smrt db:migrate` in the same maintenance step. The plan is one\n `DROP INDEX` + `CREATE UNIQUE INDEX` per table under the SAME name (a\n superset key, so the build cannot fail when the old same-name index was a\n valid UNIQUE over the subset key; a #1165-class table whose old index was\n non-unique or missing may hold duplicates that a superset UNIQUE rejects —\n `db:diff` shows which tables' old index is non-unique or missing; dedupe\n those rows before migrating). Atomic mode swaps every table in one\n transaction: `DROP INDEX` takes ACCESS EXCLUSIVE and holds it until commit,\n which blocks ALL access to those tables — reads included — for the batch;\n size `statementTimeout` for the largest tenant-scoped table. That is the\n maintenance window this rollout requires anyway (no mixed-version state), so\n run this wave — the #2359 index wave included — in atomic mode inside it;\n the \"roll out with `--postgres-safe`\" advice above applies to a #2359-only\n wave, because `--postgres-safe` runs the two statements sequentially per\n table, so each table has NO conflict index between them and a failed rebuild\n leaves it without one until the re-run. The recreate has no automatic\n DOWN: reverting the code means re-creating the old index by hand. And\n legacy NULL-tenant rows fork rather than get adopted — a tenant-context save\n whose slug matches a `(NULL, slug, ctx)` row now inserts `(tenant, slug,\n ctx)` beside it, and that tenant no longer sees the legacy row — so backfill\n `tenant_id` (anytown: `SET tenant_id = context::uuid`) BEFORE this release.\n Ingestion that relied on natural-key dedup across tenants now inserts one\n row per tenant (release note).\n- **STI `@field({ unique: true })` is enforced through indexes** (the differ can\n add an index to an existing table, never a column constraint): a full\n `<table>_<col>_unique_idx` when the STI base declares it, one\n `<table>_<col>_<class>_unique_idx WHERE _meta_type = '<qualified>'` per class\n when only descendants do — uniqueness per concrete class, not across the\n subtree. DuckDB/JSON have no partial indexes, so the descendant-scoped shape\n (`isStiSubtypeUniqueIndex`) is not emitted there — degrading it to a full\n UNIQUE would constrain every subtype; the DDL strategy and the differ both\n skip it, while other partial indexes keep degrading to full ones as before. Remember the\n framework serializes an unset text field as `''`, so a unique optional text\n field must be `nullable: true` with a `null` initializer or every unset row\n collides.\n- **Every class in an STI hierarchy carries the schema of the one shared\n table**, generated from the root base (`ManifestGenerator.generateSchemas()`\n resolves the root through `findSTIBaseInfo`), so a child never treats its own\n descendant-only unique field as base-declared.\n\n`src/schema/utils.ts` sits in between, and the two exports differ:\n\n- `generateSchema()` (reached from `SmrtCollection.generateSchema()`) always\n rebuilds from the registry and writes the result back into the registry,\n replacing whatever the manifest registered for that class.\n- `ensureSchema()` (reached from the deprecated `smrt db:setup`) is\n manifest-first: it takes `ObjectRegistry.getSchema()` plus the merged\n `getAllSchemasAsDefinitions()` table definition, and only falls back to\n `generateSchema()` when no schema is registered at all.\n\nSo a normal build keeps the manifest schema through `db:setup`, and a\nregistry-derived schema is a dev/test artifact. `smrt-content` shows what one\nlooks like: `packages/content/src/hooks.server.ts` `bootstrapSchema()` calls\n`generateSchema()` for every registered class and then `ensureSchema()` from the\nSvelteKit `handle` hook on any `/api/*` request, so that process holds\nregistry-derived schemas rather than the manifest ones. It reaches only that\npackage's own `vite dev` app — the library build excludes the file and the\npackage never exports it — but it is the shape to recognize. Check which route a\nprocess actually took before trusting a reproduction.\n\n## Why the drift stayed invisible\n\nEvery drift oracle compares a database with the same artifact that dropped the\nindex:\n\n- `verifyPersistenceTable()` (`src/schema/table-verifier.ts`) calls\n `db.tableExists()` and nothing else. \"Runtime verifies schema\" has always meant\n existence-only — no column, type, constraint, or index comparison.\n- `smrt doctor` never opens a database connection.\n- `db:status` and `db:diff` diff the live database against\n `getAllSchemasAsDefinitions()`, i.e. the manifest projection.\n\nAn index the manifest never emitted is \"in sync\" by construction. That is how a\nproduction database reached 164 unindexed `tenant_id` columns while `db:status`\nreported no drift (#2356 → #2359). The assessment's other counts — 196/231\n`@foreignKey` and 91/92 `@crossPackageRef` columns with no production index,\n238/238 tables carrying a redundant index on the primary key, zero DB-level\nforeign-key constraints on any engine — come from regenerating every package's\nschema against a live database, so re-measure rather than quote them once the\nepic's fixes land.\n\n## Rules\n\n### 1. Verify against the production path, not the test path\n\nAny change to column or index emission goes on **all** paths that ship and is\nproven by the path-parity test (`src/schema/schema-path-parity.test.ts`, #2359)\n— extend its fixture; a green suite otherwise proves the registry paths only.\nRead the call graph before believing a comment: \"same as migrations\" was wrong\nfor years.\n\n### 2. Every new query predicate ships with its index\n\nCollection methods, poll loops, auth lookups, junction right-side filters, and\npolymorphic owner lookups all count — or write down why the predicate does not\nneed one. For list workloads, EXPLAIN on a PostgreSQL snapshot; the measured\nspread on the assessed workload was 21 ms → 0.1 ms.\n\n### 3. Run the PostgreSQL lane\n\nAnything touching numeric types, uuid casts, upsert conflict targets, timestamps,\nor migrations runs the package's `test:postgres` script:\n\n```bash\npnpm --filter @happyvertical/smrt-<pkg> test:postgres\n```\n\ncore, cli, users, sales, marketing, analytics, and vitest carry the lane.\nSQLite's type affinity accepts values PostgreSQL rejects — a money field declared\n`number = 0` compiles to INTEGER and only fails on PG (#2361).\n\n### 4. Read the built artifact, not the source\n\nWhat a decorator produced is in `dist/manifest.json` and in regenerated schemas:\n`integer` vs `decimal`, the actual index list, the actual conflict columns. When\nthe question is \"how many tables/columns/indexes\", regenerate and count across\nevery package; do not sample a few and extrapolate.\n\n### 5. Index intent belongs on both the constraint and the read path\n\nA conflict target is not automatically a unique index, and a unique index is not\nautomatically the index a read path uses. Custom `conflictColumns` used to\nreplace the `(slug, context)` index while `loadFromSlug`/`getId` still queried\nslug+context, and STI dropped `@field({ unique: true })` — both fixed in #2359,\nsee \"Index rules\" above. Check the pair, not the declaration.\n\n### 6. Multi-tenancy is a whole-path property\n\nEvery unique constraint and every conflict target on a tenant-scoped table\nincludes the tenant column — otherwise a second tenant's `save()` of the same\nnatural key updates the first tenant's row through `DO UPDATE SET` (#2360; the\ndefault key now does, see \"Index rules\" — an explicit `conflictColumns` that\nomits the tenant column is the class author's own key and is not rewritten).\nAnd every read path is interceptor-aware: hydration\n(`loadFromId`/`loadFromSlug`), get-by-slug, vector search, and collection\nmemory, not only `list()` (#2365).\n\n### 7. Retry only transient errors\n\nClassify through the cause chain (SQLSTATE), never on a message substring, and\nnever retry inside an aborted PostgreSQL transaction (`25P02`). Test the\ncontract end to end against a real database, not only the classifier (#2366).\n\n### 8. Thread new decorator options through every config-rebuild site\n\nA new `@smrt()` or `@field()` option that affects schema must reach the\n`SchemaGeneratorConfig` type in `src/schema/generator.ts` and every site that\nrebuilds that config — `src/schema/utils.ts` and `src/testing/database.ts` — or\nit is silently dropped on the paths that rebuild it (#2357).\n\n### 9. Delete or wire dead paths, and write docs to what the code does\n\nDead code that looks canonical misleads the next agent: the AST `generateSchema`\npath, `SchemaOverrideSystem`, and the never-emitted `triggers: []` all read as\nsupported surfaces (#2380). Documentation follows the implementation, not the\nintent — say \"verifies the table exists\" when that is what runs.\n\n### 10. Untracked \"known limitation\" comments are bugs nobody will read\n\nFile the issue and link it from the comment. A `products` comment explaining why\na conflict-column change was refrained from sat there for months — and\nmisdescribed the failure mode the whole time.\n\n### 11. Consumer repair scripts are signals\n\nDownstream repair tooling (anytown's `db-repair-plan.ts` carried column-type\nrepairs, missing STI columns and indexes, and `tenant_id` backfills since April)\nis the consumer-side record of framework gaps. Mine it during triage.\n\n### 12. Try to falsify before filing, and treat operations as correctness\n\nRe-verify a finding at source before it becomes an issue — one assessment\ncandidate claimed conflict indexes past two columns were narrowed to two\ncolumns, when only the index *name* is shortened. And an index fix that ships\nwithout a bounded-timeout, `CONCURRENTLY`-capable migrate path can take\nproduction down on rollout (#2362).\n\n### 13. Composite indexes are declared, not inferred (#2357)\n\nThe generated set only covers foreign keys, unique/conflict columns, the STI\ndiscriminator, reference columns (#2359), the default list ordering (rule 18\nbelow), and single columns opted in with `@field({ indexed: true })`. A list\nworkload's access path is composite, so declare it:\n\n```ts\n@smrt({\n indexes: [\n { name: 'contents_tenant_id_publish_date_idx',\n columns: ['tenantId', 'publish_date'] },\n ],\n})\n```\n\n`columns` takes field names or column names in access-path order — filter\ncolumns first, sort column last. Declare columns, not a direction: PostgreSQL\nscans a btree either way, so an ascending index also serves the matching\n`ORDER BY ... DESC` as an ordered scan with no Sort node. `unique` and `where`\n(partial index) are honoured.\n\n`appendDeclaredIndexes()` runs first on all five entry points, ahead of\n`ensureDefaultListOrderingIndex()` (rule 18) and `ensureReferenceColumnIndexes()`,\nso a declared composite leading with the tenant column (or any reference column)\nreplaces the automatic standalone index rather than duplicating it.\nUnknown columns, malformed entries, and a name collision with a different index\nall fail generation — a silently dropped index only surfaces later as a\nproduction slowdown. Rule 8 above is why this works at runtime at all.\n\n### 14. Relationship targets resolve to a class name on both paths\n\n`@foreignKey`/`@oneToMany`/`@manyToMany` accept a class, a name string, or a\n`() => Target` thunk. The decorator invokes the thunk and throws when the target\ncannot be resolved (never `related: ''`); the scanner unwraps the same thunk\nfrom raw source (never `related: '() => Target'`). An unresolved target silently\ncosts the relationship edge, `loadRelated()`, and the FK-derived index (#2379).\nA thunk resolves at decoration time, so a target declared later in the same\nmodule is still in its temporal dead zone — use the string form there.\n\n### 15. A SQLite type change is a table rebuild (#2370)\n\nSQLite has no `ALTER TABLE ... ALTER COLUMN ... TYPE`, so\n`src/migrations/sqlite-rebuild.ts` answers a `type_upgrade` on SQLite with the\nstatement list SQLite's own docs prescribe: stage a new table under\n`_smrt_rebuild_<table>`, copy, drop, rename, replay the indexes and triggers.\n`SchemaComparer.compareTable` swaps that plan in for the differ's\n\"requires table recreation\" placeholder, so `db:migrate` applies it inside the\nnormal atomic batch instead of exiting 1 forever.\n\nFour properties of that module are load-bearing; keep them if you touch it:\n\n- **The target shape comes from the live `sqlite_master` DDL**, retyping only\n the drifted columns. It is not regenerated from the manifest, so the rebuild\n never becomes an implicit `DROP COLUMN`, and it preserves table constraints,\n `CHECK`s, and `WITHOUT ROWID`/`STRICT`.\n- **The rebuild is hoisted ahead of the table's other column changes.** Its\n staging DDL and copy list are captured at diff time, and the differ emits\n changes in manifest field order, so a new field declared above the retyped\n one would otherwise run `ALTER TABLE ... ADD COLUMN` first and have the\n rebuild silently drop it — both statements succeed and the batch commits.\n Rebuild first, then add columns to the rebuilt table.\n- **The copy carries no `CAST`.** SQLite applies the destination column's\n affinity on insert — the same conversion a fresh table performs. An explicit\n cast is worse: non-numeric TEXT cast to REAL/INTEGER silently becomes `0`,\n and an ISO timestamp cast to NUMERIC-affinity `DATETIME` becomes its year.\n- **It refuses when any table has a foreign key onto the target and\n `PRAGMA foreign_keys` is ON** (the SMRT adapter's default). `DROP TABLE`\n performs an implicit `DELETE FROM` that fires `ON DELETE CASCADE` on\n children, and `defer_foreign_keys` defers constraint *checks*, not FK\n *actions* — verified: the child rows go. The target's own self-reference\n counts, because the staging table copies that clause and becomes a child of\n the table being dropped (verified: a two-row self-referencing table finishes\n the rebuild holding one row). Such a column stays manual drift.\n- **`PRAGMA legacy_alter_table` brackets the rename**, because SQLite ≥ 3.25\n re-parses the schema on `ALTER TABLE ... RENAME` and a view still pointing at\n the just-dropped table makes it fail outright. It is restored immediately\n after; a rolled-back batch leaves it set on that connection, which is inert\n here only because nothing else in SMRT renames a table.\n\nAll the drifted columns of one table share a single rebuild: the first change\ncarries the plan and the rest become `no change needed` comments that the CLI\nclassifies as no-ops.\n\n## What the differ compares (#2369)\n\n`SchemaComparer` (`src/migrations/differ.ts`) compares each manifest column's\ntype, then — unless the type itself is drifting — its nullability and default,\nand always reports what it will not touch:\n\n- **Strengthening** (`SET NOT NULL`, `SET DEFAULT`) is executable on\n PostgreSQL/DuckDB. `SET NOT NULL` is preceded by an `UPDATE … WHERE c IS NULL`\n backfill of the manifest default; without a default the live data is probed\n and, if NULLs exist, the change is reported (comment SQL + `advisory`) instead\n of emitting an ALTER that would abort the atomic batch.\n- **Relaxing** (`DROP NOT NULL`, `DROP DEFAULT`) is a report-only advisory until\n the caller passes `relaxColumns` (`db:migrate --relax-columns`). The manifest\n can be under-specified (#2372 registration-order weakness), so a live column\n that is stricter than the manifest is never weakened silently.\n- **Orphans** — DB columns absent from the manifest, DB tables no manifest\n declares (`SchemaDiff.orphan_tables`), and unclaimed `*_key` unique constraint\n indexes — are always reported. A NOT NULL orphan without a default is a\n `warning` advisory (every ORM insert fails on it); `includeDroppedColumns`\n (`--drop-columns`) drops it, `relaxColumns` relaxes it. Advisory-only changes\n carry no SQL, never reach the tracker, and do not fail `db:migrate`.\n- **ADD COLUMN** is planned per engine: DuckDB rejects every inline constraint\n (add with `DEFAULT`, then `SET NOT NULL`, `CREATE UNIQUE INDEX`); SQLite\n rejects inline `UNIQUE` (separate `CREATE UNIQUE INDEX <table>_<col>_key`, the\n PostgreSQL constraint-index name, so the orphan sweep leaves it alone) and\n `NOT NULL` without a default on a populated table; PostgreSQL keeps constraints\n inline. DuckDB has no `ADD CONSTRAINT`, so the separate index is the only\n way to add uniqueness there; the bundled DuckDB 1.4.x resolves\n `ON CONFLICT (col)` through that index (the old #12684 limitation the DuckDB\n strategy's `requiresInlineUnique()` note describes no longer reproduces —\n the #2369 DuckDB test pins the upsert), older DuckDB builds may not. A required column with no default is enforced only on an empty table;\n on a populated one it is added nullable and the `NOT NULL` is reported as a\n manual follow-up on every engine.\n- **SQLite** has no `ALTER COLUMN`: nullability/default alterations are manual\n (comment SQL → `db:migrate` exit 1). The #2370 rebuild (rule 15) consumes\n only `type_upgrade` placeholders today; extending it to rewrite constraints\n would lift this.\n- Defaults compare through `canonicalizeDefault()`, which folds engine\n renderings (`'x'::text`, `CAST('t' AS BOOLEAN)`, `CURRENT_TIMESTAMP` vs\n `now()`) by manifest type; an unclassifiable rendering skips the comparison\n rather than risking a false positive that would churn every run. The\n round-trip test (create from each DDL strategy → compare → zero changes) in\n `src/migrations/__tests__/issue-2369-*.test.ts` guards this.\n\n### 16. `schema.ddl` is a preview, not the table\n\n`SchemaDefinition.ddl` / `manifest.json` `schema.ddl` is the engine-neutral\nCREATE TABLE string from `SchemaGenerator.generateSQL()` with no engine: no\nindexes, no triggers, abstract `REAL`/`JSON`/`UUID`/`TIMESTAMP`. It is kept for\nbackward compatibility only. Everything that needs an executable table renders\n`columns` + `indexes` through `getDDLStrategy(engine)` — `db:migrate`\n(`migrations/orchestrate.ts`), `MigrationGenerator` (default\n`materializeStructuredSchema: true`; `false` is a deprecated opt-out),\n`SchemaAggregator`, and `createIsolatedTestDbFromManifest` in smrt-vitest, the\nlast two via `src/schema/manifest-schema.ts` (`collectManifestTables` /\n`renderCollectedManifestTable`). The cached string is merged in only for a\ntable whose contributors expose no structured columns (hand-authored\nmanifests); table constraints that exist only in the string are dropped with a\nwarning, as `db:migrate` drops them. Do not add a new consumer of the\nstring, and do not write a private CREATE INDEX renderer — the retired ones\ndropped `where` and `jsonPath` (#2358). Every DDL strategy also spells out\n`PRIMARY KEY NOT NULL`: SQLite lets a bare non-INTEGER PRIMARY KEY hold NULL.\n\n### 17. The merged table shape is registration-order independent (#2372)\n\n`getAllSchemas()` and `getAllSchemasAsDefinitions()` fold every class that\nshares a physical table — the whole STI hierarchy — into one shape. Both route\nthrough `buildMergedTableSchemas()`, which groups contributors by table and\nthen merges them in a **deterministic** order: the STI base first, then\nancestors before descendants, then by qualified name.\n\nThat order matters because the first contributor seeds the table: it supplies\nthe fallback base columns, the `idType`, the conflict columns and the cached\nDDL, and its columns win every merge conflict. When registration order decided\nit, an STI child that carries no manifest `schema` — the external- and\nconsumer-manifest case — seeded the table from bare fallback columns and the\nbase class's richer ones were skipped when it registered later, yielding\n`context TEXT` instead of `context TEXT NOT NULL DEFAULT ''` and timestamps\nwith no NOT NULL/DEFAULT. The shipped content manifest lists `Article` before\n`Content`, so the losing order was the one that shipped, and the differ\ncompares types only, so the weak fresh-create was never repaired.\n\nTwo invariants keep the two assembly paths agreeing:\n\n- `createBaseColumns()` mirrors what `generateSchemaFromManifest` /\n `generateSTISchemaFromManifest` emit for the same table, so a table built\n from runtime field metadata alone has the same NOT NULL/DEFAULT shape as one\n built from a manifest. Note `_meta_type` is `TEXT NOT NULL` with **no**\n default, matching the generator.\n- `fieldsToColumns()` reads `required`, `default`, and `description` from the\n top level *or* `_meta`. Registry fields normalize them into `_meta`\n (`manifest-field-merge.ts`), so reading only the top level silently dropped\n NOT NULL and DEFAULT for every registry-sourced field.\n\nSTI columns stay nullable regardless of the field's `required` flag\n(`fieldsToColumns(fields, { stiUnionColumns: true })`): the table holds the\nunion of all subtypes' fields, so a column only one subtype declares is never\npopulated on a sibling's row. Declared defaults are still emitted. This matches\n`generateSTISchemaFromManifest`, which sets `notNull: false` on every non-system\nSTI column.\n\nWhen adding a class-level input to the merged shape, take it from the seeding\ncontributor rather than \"whichever class arrives first\", and cover it with a\nchild-first/base-first equality test.\n\n### 18. The generator owns the index for its own default ordering (#2363)\n\nEvery generated list surface — REST, MCP, the SvelteKit list route — pages with\n`ORDER BY created_at DESC, <pk> ASC` (`DEFAULT_LIST_ORDER_BY`, #2367), and\nuntil #2363 no schema path indexed `created_at` (the dead AST path indexed\n`updated_at`), so the framework's own default page was a sequential scan plus a\ntop-N sort. `ensureDefaultListOrderingIndex()` now runs on all five paths and\nemits:\n\n- `(<tenant column>, created_at)` on a tenant-scoped table — the tenancy\n interceptor puts `tenant_id = ?` in front of every list, so the tenant column\n leads and `created_at` orders within it. This composite **replaces** the\n standalone tenant index from #2359: a B-tree serves every prefix of its\n column list, so `ensureDefaultListOrderingIndex()` is called first and\n `ensureReferenceColumnIndexes()` then sees the column as already served. The\n tenant column is found by `referenceKind === 'tenantId'`, never by the\n `tenant_id` spelling — `@smrt({ tenantScoped: { field } })` renames it.\n- `(created_at)` otherwise.\n\nThree deliberate omissions, so nobody \"fixes\" them later:\n\n- **No `DESC`.** `IndexDefinition` carries no per-column direction and\n PostgreSQL scans a B-tree backwards just as cheaply.\n- **No primary-key tiebreak column.** The default order mixes directions\n (`created_at DESC, id ASC`), so no single-direction index satisfies the whole\n key; the leading columns already turn a full sort into an index scan plus an\n incremental sort over rows sharing a timestamp.\n- **Not scoped per STI subtype.** `(_meta_type, created_at)` would serve a\n child collection's list but not the base class's polymorphic one, which\n carries no discriminator predicate — the same reasoning that keeps STI\n reference indexes plain (#2359). One unqualified index per shared table.\n\nAn existing UNQUALIFIED index that already leads with the same columns\nsuppresses it — a partial or JSON-path index never counts. That is how a\ndeclared `@smrt({ indexes: [...] })` composite (#2357) takes over: declaring\n`(tenant_id, created_at, status)` replaces the generated pair, while declaring\na different sort column such as `(tenant_id, publish_date)` sits **beside** it,\nbecause that index cannot order the default page. Declared indexes are appended\nbefore this helper for exactly that reason; anything that appends an index in\nfuture goes in the same slot, ahead of `ensureDefaultListOrderingIndex()` and\n`ensureReferenceColumnIndexes()`.\n\n### 19. One conflict-target rule, applied on every producer\n\n`save()` upserts on `ObjectRegistry.getConflictColumns()`; the schema must\ncarry exactly one unique index over those columns (or they must be the\nprimary key). Keep the derivation in `src/schema/conflict-target.ts` and let\nevery producer call it — the three manifest pipelines share\n`ManifestGenerator.applyGenerationPasses()` since #2360 because\n`ManifestBuilder` had silently skipped the report passes for months. When you\nadd a way for the key to vary (a new decorator option, a new class kind),\nthread it through `getConflictColumns()`, `normalizeConflictColumns()` and the\ngenerator's `resolveConflictTarget()` together, and extend the parity test's\n\"unique index == conflict target\" assertion; a key the runtime uses and the\nschema does not index is a hard PostgreSQL error (42P10) on the first save,\nand a key the schema indexes without the tenant column is the silent\ncross-tenant overwrite this rule exists for.\n\n### 20. Every generated index name is length-guarded before it leaves a path (#2374)\n\nPostgreSQL truncates any identifier past 63 **bytes** and reports nothing;\nSQLite and DuckDB do not, so the entire test suite was blind to it. The 66-byte\n`content_contribution_revisions_contribution_id_revision_number_idx` shipped\nthat way — only the differ's signature-equivalence check kept it from emitting\n`add_index` on every run. Two names agreeing for 63 bytes is the real hazard:\n`CREATE INDEX IF NOT EXISTS` no-ops against the wrong index, and the second\nindex is never created.\n\n`schema/index-utils.ts` owns the guard, and it splits by who owns the name:\n\n- **Generated index, trigger and PL/pgSQL function names** →\n `shortenIdentifier()`. Deterministic `<head>_<digest><suffix>`, digest taken\n over the **full** original so a shared prefix still yields distinct names, and\n a recognised suffix (`_idx`, `_unique_idx`, `_key`, `_pkey`) preserved.\n- **Hand-declared `@smrt({ indexes: [{ name }] })`** → `assertIdentifierFits()`,\n a hard error in `validateDeclaredIndex()`. Renaming what a developer wrote is\n worse than refusing it, and `SchemaComparer` matches indexes **by name**\n first, so a 70-byte declaration could never match the 63-byte index\n PostgreSQL stored and `db:migrate` would emit `add_index` forever.\n- **Table and column names** → deliberately **not** guarded. PostgreSQL\n truncates identifiers *consistently on every reference*: `CREATE TABLE\n \"<80 bytes>\"` and a later `SELECT ... FROM \"<the same 80 bytes>\"` both resolve\n to the same stored 63-byte name, so one long name round-trips fine end to end.\n `smrt-users` depends on this — it ships an intentional 80-byte\n `@smrt({ tableName })` (`permission_policy_table_name_that_is_far_too_long…`)\n and derives unique Postgres RLS policy names from it. An earlier revision of\n this rule hard-errored here on the theory that the runtime resolves tables by\n name and would break; that theory is wrong for the reason above, and the error\n broke `packages/users`. The residual collision risk is over a name the\n developer chose, not one the generator manufactured.\n\n`enforceIdentifierLimits()` is the single call site per path, placed **after**\n`ensureReferenceColumnIndexes()` — nothing may lengthen a name after it. Doing\nthe shortening at the end rather than at each `indexes.push()` is safe because\nthe digest covers the whole original name, so entries distinct before shortening\nstay distinct after; the helper still throws if two ever collide. The migrate\nleg's `withConflictIndex()` (`registry/schema-builder.ts`) and the PostgreSQL\ntrigger-function name call `shortenIdentifier()` directly, because they compose\na name outside the generator's index list. Note that an over-long *table* name\nstill yields in-limit, distinct *index* names, because the shortening runs over\nthe whole composed name.\n\nThe digest is FNV-1a, not `node:crypto`: `index-utils.ts` is re-exported from\n`schema/utils.ts`, which exists to keep Node built-ins out of browser bundles.\nIt only has to be *stable* — a shortened name that changed between releases\nwould make every deployment drop and recreate the index — so the parity and\nunit tests pin the literal output rather than recomputing it. Unpaired\nsurrogates are folded to U+FFFD before both counting and hashing, so the digest\nis taken over exactly the bytes the driver transmits.\n\nExisting databases migrate **by name swap, without a rebuild**: the live index\nstill carries the name PostgreSQL truncated it to, the manifest now carries the\nshortened one, and the differ claims it by signature (columns + uniqueness +\npredicate), emitting nothing — including under `includeDroppedIndexes`. See\n`migrations/__tests__/index-drift.test.ts` and the PostgreSQL lane test\n`schema/issue-2374-identifier-length-postgres.optional.test.ts`.\n\nOut of scope, deliberately: constraint names PostgreSQL invents for itself. A\nCTI table's inline `UNIQUE` produces an implicit `<table>_<column>_key`, which\ncan exceed 63 bytes even when the table and column each fit. SMRT never names\nit, and PostgreSQL disambiguates its own truncations by appending a counter\nrather than collapsing them, so there is no silent-collision hazard there.\n\n### 21. The `_smrt_` prefix does not mean \"system table\" (#2376)\n\n`bootstrapSystemTables()` owns nine hand-written tables; ~25 more `_smrt_*`\ntables belong to `@smrt()` models and are created by `db:migrate` (feature\nflags, prompt overrides, subscription plans, report schedules, field policies,\njobs). Never classify by prefix — use `SYSTEM_TABLE_NAMES`\n(`schema/system-table-shapes.ts`, derived from the DDL parse) plus\n`FRAMEWORK_OPERATIONAL_TABLES` / `RETIRED_SYSTEM_TABLES` in `system/schema.ts`.\nThe change-feed writer skipped by prefix, so clients syncing those domain\ntables through `_changes` never saw an update.\n\nEditing `ALL_SYSTEM_TABLES` requires bumping `SMRT_SCHEMA_VERSION` *and*\nappending to `SMRT_SCHEMA_DDL_CHECKSUMS` — the version gates the DDL replay, so\nwithout a bump no existing database ever applies the change. A new **column**\nadditionally needs an `addColumnIfMissing()` entry in `system/compatibility.ts`\n(`CREATE TABLE IF NOT EXISTS` is a no-op on an existing table).\n`system-schema-evolution.test.ts` enforces both, and asserts a legacy database\nupgrades to exactly the shape a fresh install gets.\n\n`_smrt_jobs` / `_smrt_job_events` are dual-owned: `db:migrate` creates them,\nthe compatibility pass reshapes them. On a fresh install bootstrap runs first,\nso their pass is deferred — `ensureDeferredSystemTableCompatibility()` re-runs\nuntil the tables exist, then stamps a `<version>+deferred-compat` marker. It\nruns OUTSIDE the bootstrap lock and swallows its own failures: those statements\ntarget tables the framework does not own, and inside the PostgreSQL transaction\none failure would roll back system-table creation with it. Only\n`ensureBootstrapSystemTableCompatibility()` (the tables the DDL itself creates)\nbelongs inside the lock.\n\nReconciling `_smrt_jobs.task_id` uniqueness reads the live index catalog, which\nis implemented for PostgreSQL and SQLite only; DuckDB and the JSON adapter keep\nthe redundant compat index rather than risk dropping the one that enforces the\nupsert conflict target. When reading a PostgreSQL catalog array, cast it\n(`attname::text`) and parse both shapes — a driver with no parser registered for\nthe array OID returns the raw `{a,b}` literal, and reading that as \"no columns\"\nsilently inverts an index-existence decision.\n\n## Referential integrity lives in `delete()`, not in the DDL\n\nNo schema path emits a `FOREIGN KEY` clause on any engine — grep `schema/ddl/*`,\n`schema-manager.ts` and `differ.ts` for `REFERENCES` and you get nothing. That is\na deliberate position, not an oversight: emitting constraints changes delete\nsemantics for every consumer, requires topological table ordering in both\nmigrate paths (neither orders today), and needs a plan for the orphans already in\nproduction databases. Emitting them is tracked separately.\n\nWhat `@foreignKey(..., { onDelete })` therefore means is *application* behaviour,\napplied by `SmrtObject.delete()` through `src/cascade.ts` (#2371):\n\n| Reference | Default when `onDelete` is absent |\n|---|---|\n| Column is part of the referencing class's `conflictColumns`, and is not a `@tenantId()` field | `CASCADE` |\n| Polymorphic `(metaType, metaId)` association row | `CASCADE` |\n| Anything else, including every `@tenantId()` field | `NO ACTION` — the row is left alone |\n\nThe natural-key rule is what cleans junction rows up without any per-package\nannotation: a junction declares\n`@smrt({ conflictColumns: ['content_id', 'asset_id', 'relationship'] })`, so the\nrow is *identified* by the content and cannot outlive it. An ordinary child\n(`Order.customerId`) is keyed by `(slug, context)` and keeps its pre-#2371\nbehaviour unless it opts in explicitly.\n\n**`@tenantId()` is excluded even though it lands in `conflictColumns`.**\n#2360 leads every tenant-scoped class's *default* natural key with the\ntenant column, so without this exclusion, deleting one `Tenant` row would\nrecursively CASCADE through every tenant-scoped table in the schema that has\nnot declared its own `conflictColumns` — the overwhelming majority. The\ntenant column scopes ownership; it does not identify the row the way a\njunction's foreign key does. Detected via the `__tenancy.isTenantIdField`\nmarker on `FieldMeta` (`smrt-core` reads it structurally so it never depends\non `smrt-tenancy`). `@tenantId()` exposes no `onDelete` option today, so\nthis cannot currently be overridden per field — found in review before this\nlanded (originally reachable, untested, and undocumented).\n\nProperties to keep if you touch that module:\n\n- **The plan is registry-derived and rebuilt per delete.** Registration is\n incremental — manifests load lazily and tests register classes between cases —\n so a cached plan would silently skip a table that registered later. Cache it\n only behind an invalidation hook that every registration path calls.\n- **A class with nothing pointing at it skips the transaction entirely — but\n `CascadePlan.isEmpty` requires no polymorphic association class anywhere in\n the process, not just no typed references.** `buildCascadePlan()` pushes\n *every* registered `SmrtPolymorphicAssociation` subclass into\n `plan.polymorphic` unconditionally (`cascade.ts` around\n `isPolymorphicAssociationClass`): a `metaType` column can point at any class\n at runtime, so there is no static metadata to scope it by the target being\n deleted. One registered polymorphic class anywhere makes `isEmpty` false for\n every delete in that process — do not read \"the common case skips the\n transaction\" as \"most deletes in a real app skip it\"; in a multi-package app\n that registers even one polymorphic association, almost none do.\n `runCascadeDelete()` builds the plan for `getResolvedQualifiedName()` (not the\n bare constructor name — two packages can register the same simple name).\n- **Cascaded rows are removed set-based.** Their `beforeDelete`/`afterDelete`\n hooks and interceptors do not run and no change-feed tombstone is written for\n them, which is exactly what a DB-level `ON DELETE CASCADE` does. Only the\n object `delete()` was called on runs the lifecycle. Do not \"improve\" this into\n a per-row model delete without deciding what that means for sync consumers.\n- **Everything is one transaction where the adapter has one**, including the\n object's own `DELETE`, whenever there is anything to cascade. The `RESTRICT`\n checks run first, before any mutation, so a refusal costs nothing; the\n transaction is what makes a refusal *deeper* in the graph safe.\n- **`_smrt_embeddings` and `_smrt_contexts` are matched by id *and* a\n class-name candidate set, not id alone.** Their class columns store the\n *runtime* constructor name, which for an STI hierarchy is a concrete\n subclass rather than the class the cascade planned from — id-alone matching\n looked STI-safe, but let two unrelated classes using `idType: 'text'`\n (non-UUID, not guaranteed globally unique) collide on a shared id value and\n delete each other's rows (review fix). `ownerClassCandidates()` expands to\n every STI hierarchy member of the class the ids actually belong to, in both\n qualified and simple form. A failure to clean them is logged, never raised —\n an application database may predate the table, and losing derived rows must\n not fail a valid delete.\n\n`_smrt_changes`, `_smrt_ai_usage`, `_smrt_signals` and the dispatch tables are\ndeliberately **not** cascaded. They are append-only logs; the change feed in\nparticular receives the delete's own tombstone, so cascading it would erase the\nrecord that tells sync clients the row is gone.\n\n### 22. System tables get a retention policy, not just a prune function (#2375)\n\nFour framework-owned tables grow with traffic and nothing used to remove a row:\n`_smrt_changes` (one per save/delete), `_smrt_ai_usage` (one per AI call, and\npersistence is on by default), `_smrt_contexts` (whose `expires_at` nothing\nenforced) and `_smrt_dispatch` (an operator-only `dispatch:cleanup`).\n`src/system/retention.ts` is now the single place that bounds them.\n\n- **`runRetentionSweep(db, policy)` is the entry point.** It runs the four\n built-in tasks in a fixed order, then every task other packages contributed\n via `registerRetentionTask()` — `@happyvertical/smrt-jobs` registers\n `_smrt_jobs`/`_smrt_job_events`, `@happyvertical/smrt-users` registers\n session/magic-link/CLI-auth expiry. A task that throws is recorded on its own\n result and the sweep continues; a missing table reports `unavailable`, so a\n sweep is safe against a partially bootstrapped database.\n- **A contributed task only exists in a process that loaded its package.** Both\n packages register on import from their entry point, and the registry lives on\n `globalThis` (like `ObjectRegistry`) so a duplicated `smrt-core` resolution\n cannot split it. `smrt db:prune` optionally imports both packages for exactly\n this reason — a project that installs neither correctly gets neither task.\n- **Defaults are opt-out, not opt-in.** `DEFAULT_RETENTION_POLICY` covers the\n four built-in tables (changes 30 days, AI usage 90 days, dispatch 30 days\n completed / 90 days failed, contexts strictly by their own `expires_at`), and\n those are the ones `smrt.configure({ retention })` tunes. Contributed tasks\n carry their own defaults and their own window options —\n `DEFAULT_JOB_RETENTION` (7 days terminal / 30 days failed / 30 days events,\n set through `registerJobRetentionTasks()` or the runner's `retention.jobs`),\n and expired credentials, which have no window because an expired credential\n has nothing worth retaining. Every task, built-in or contributed, can be\n turned off: a table set to `false`, a task set to `false` under `tasks`, or\n `enabled: false` for the whole sweep — through `smrt.configure`,\n `smrt db:prune --skip`, or the runner's `retention` config.\n- **Contributed task names are prefixed with the owning package's short name**\n (`jobs-records`, `jobs-events`, `users-sessions`, …) because the registry is\n one process-global namespace.\n- **Scheduling lives outside core.** A running `TaskRunner` sweeps every six\n hours (`retention: false` opts out) and `smrt db:prune` is the cron entry\n point. The first runner sweep is one interval after `start()`, never at\n start: a crash-looping worker must not become a delete loop.\n- **Every prune counts before it deletes.** `rowCount` is not reliably\n populated across the engines SMRT supports, so counting is both what gives a\n usable figure and what lets `dryRun` preview the *same* predicate rather than\n an approximation of it. Count and delete are two statements and deliberately\n not one transaction — a maintenance pass must not hold a write lock over a\n large delete — so the figure is approximate under concurrent writers. Where\n two bounds can select the same row (`pruneChangeFeed`, `pruneAiUsage`), the\n second bound excludes what the first already accounted for, so a dry run does\n not count an entry twice.\n- **Every retention predicate ships with its index** (rule 2 applies to\n maintenance SQL too): `_smrt_contexts(expires_at)`,\n `_smrt_ai_usage(tenant_id, created_at)` — which is also the subscriptions\n billing meter's range scan — `_smrt_dispatch(status, processed_at)` and\n `(status, updated_at)` come from the system DDL, so they reach existing\n databases through the `SMRT_SCHEMA_VERSION` bump that replays it.\n `_smrt_jobs(status, completed_at)` comes from\n `ensureJobsSystemTableCompatibility()` instead, because `_smrt_jobs` is\n generated from a decorated class and does not exist yet when bootstrap runs;\n the jobs collection calls that path on every `initialize()`.\n- **Expiry enforcement is prune-side only.** `recall()`/`recallAll()` keep\n their documented \"expiry is not applied at read time\" contract — changing it\n would change read semantics for existing callers, which is a different issue\n from bounding storage. `LearningMemory` filters expired rows itself.\n"
|
|
936
|
+
"content": "# smrt-core/schema paths\n\nModule semantics for `src/schema/` — which `SchemaGenerator` entry point reaches\na real database, what each one emits, and the rules that keep them in step.\nPackage orientation, the cross-module invariants, and the traps that apply\nbefore editing anything live in [../AGENTS.md](../AGENTS.md) — read that first;\nits \"Schema paths\" section is the short form of everything below.\n\nWritten from the 2026-08-17 database-layer gap assessment (epic #2382). Symbol\nnames here are stable; the line numbers the assessment quotes are not, so trust\nthis call graph and re-grep before citing a location.\n\n## Four entry points, two of which ship\n\n`src/schema/generator.ts` exposes four index-emitting entry points. They do not\nproduce the same schema for the same class.\n\n| Entry point | Selected by | Status |\n|---|---|---|\n| `generateSTISchemaFromManifest` | `src/scanner/manifest-generator.ts` | **production** |\n| `generateCTISchemaFromManifest` | `src/scanner/manifest-generator.ts` | **production** |\n| `generateSTISchemaFromRegistry` | `src/testing/database.ts` (`getTestDatabase()`), `src/schema/utils.ts` (`generateSchema`; `ensureSchema` only as a fallback) | tests + runtime helpers |\n| `generateSchemaFromRegistry` | the same two callers | tests + runtime helpers |\n\nA fifth entry point, the build-time AST `generateSchema(objectDef)`, existed\nuntil #2380: it fed only the `smrt:schema` virtual module, which had no\nconsumer, had rotted relative to the four paths above (an `idx_`-prefixed\nnaming scheme none of the others use, and no conflict-index emission at all),\nand was deleted rather than wired up. `SchemaOverrideSystem`\n(`schema/override-system.ts`) — unwired, and its two non-generic methods\nhard-coded a schema extension for a project outside this monorepo — was\ndeleted alongside it. See rule 9 and the new rule at the end of this file.\n\nProduction DDL takes the manifest route:\n\n```\n@smrt() class ─▶ scanner ─▶ manifest.json ─▶ generate{STI,CTI}SchemaFromManifest\n ─▶ registered `schema` ─▶ ObjectRegistry.getAllSchemasAsDefinitions()\n ├─▶ smrt db:migrate | db:diff | db:status\n │ (the CLI drives SchemaComparer + MigrationTracker directly)\n └─▶ migrateSmrtSchemas() / getPendingSchemaStatements()\n (src/migrations/orchestrate.ts — exported for programmatic\n use; no in-repo caller outside its own tests)\n```\n\nThe suite takes the registry route. Before #2359 the registry route emitted\nindexes the manifest route did not — per-column foreign-key indexes, and STI\npartial FK indexes filtered by `_meta_type` — so tests ran against a richer\nschema than any deployment received, the manifest STI path populated a\n`fkColumnsByClass` map it never read, and the manifest CTI path had no FK loop\nat all. `src/testing/database.ts`'s \"same as migrations\" comment described an\nintent, not the code.\n\nSince #2359 the two families share one set of index helpers and\n`src/schema/schema-path-parity.test.ts` runs the same fixture manifest through\nthe manifest paths, through `ObjectRegistry.registerFromManifest()` + the\nregistry paths, and through `getAllSchemasAsDefinitions()`, asserting identical\ncolumn and index sets. Extend that fixture with every generator change; a\ndivergence is a bug in the generator, not an exception to add to the test.\n\n### Index rules (#2359)\n\n- **Reference columns are always indexed.** `ensureReferenceColumnIndexes()`\n runs last on every path and gives each `@foreignKey`, `@crossPackageRef` and\n tenant column `<table>_<column>_idx` unless an UNQUALIFIED index (no `WHERE`,\n no JSON path) already leads with it — the `conflictColumns` unique index or an\n `indexed: true` opt-in, or the column's own inline UNIQUE. A partial\n `WHERE _meta_type = …` index does not count: base-class polymorphic queries\n carry no discriminator predicate. `indexed: true` on a reference column is\n redundant. Roll the index wave out to production with\n `smrt db:migrate --postgres-safe` (concurrent-index mode, #2362): a plain\n atomic batch takes SHARE/ACCESS EXCLUSIVE locks for ~230 index builds. STI FK indexes are plain, one per\n column, not per-class partial.\n- **No index on the primary key.** `<table>_id_idx` is gone from every path,\n and `conflictColumns` equal to the PK column set emit no conflict index\n (`ON CONFLICT (id)` binds to the PK constraint). `SchemaComparer` drops the\n legacy non-unique single-column PK index from existing databases without\n `--drop-indexes` when the live table reports that column as its sole primary\n key (never a UNIQUE one — on PostgreSQL that may back a custom-named PRIMARY\n KEY constraint, and `DROP INDEX` on it would fail the atomic batch).\n- **Slug loading keeps its index.** Custom `conflictColumns` replace the\n `(slug, context)` unique index; `loadFromSlug()`/`getId()`/`getSavedId()`\n still filter on slug/context, so a plain `<table>_slug_context_idx` is kept\n (additive; routing those lookups through the conflict key would change which\n row a slug resolves to). The tenant-led default key below counts as serving\n it (`servesSlugLookup()`): a tenant-scoped slug lookup carries the tenant\n predicate (#2365) and is served by the prefix, so no second index.\n- **Tenant-scoped tables key per tenant (#2360).** A tenant-scoped class with\n no explicit `conflictColumns` upserts on, and indexes,\n `(tenant_id, slug, context)` — `(tenant_id, slug, context, _meta_type)` for\n an STI hierarchy — resolved by one rule on both paths:\n `ManifestGenerator.normalizeConflictColumns()` materializes it into\n `decoratorConfig.conflictColumns` for the manifest paths (so the manifest,\n the schema, `smrt-knowledge.json` and the runtime read one value), and\n `ObjectRegistry.getConflictColumns()` derives the same value at runtime from\n the schema owner's `tenantScoped` config (`ObjectRegistry.getTenantColumn()`;\n an STI child resolves through its root; a `@report` class through its\n group/bucket columns; a custom primary key through that key). Explicit\n `conflictColumns` are never rewritten. `src/schema/conflict-target.ts` holds\n the shared helpers. Consequences: the index NAME stays\n `<table>_slug_context_idx` / `_slug_context_meta_type_idx`, so the differ\n swaps the columns of an existing global unique in place by name (a superset\n key — creating it cannot fail on existing rows); the tenant-led key also\n serves the tenant column, so `<table>_tenant_id_idx` is no longer emitted\n for those tables (an existing one is an orphan the differ drops only with\n `--drop-indexes`); NULL-tenant rows (`mode: 'optional'` outside a tenant\n context) dedup among themselves through the SDK's null-aware upsert\n (`IS NOT DISTINCT FROM` under a PostgreSQL advisory lock / an in-process\n lock on SQLite) — application-enforced now, where the old global index was\n database-enforced: the tenant-led index treats NULLs as distinct, so raw SQL\n can insert two global rows with one slug, and a raw\n `ON CONFLICT (slug, context…)` against such a table no longer binds (use\n `WHERE NOT EXISTS`, plus an advisory lock on PostgreSQL). Emitting\n `NULLS NOT DISTINCT` on PostgreSQL ≥ 15 (the SDK already detects it) would\n restore the database arbiter — a follow-up. The `save()` path serializes an\n unset tenant field as an explicit `NULL` whatever its registered type,\n because the SDK rejects an upsert whose conflict column is missing from the\n row.\n- **Rolling the tenant-led key out (#2360).** There is no mixed-version state:\n new code against the old index fails every NEW-object create on a\n tenant-scoped default-key table (PostgreSQL 42P10, SQLite \"ON CONFLICT\n clause does not match…\"), and old code against the new index fails the same\n way, because the conflict target must match the unique index's column set\n exactly; only persisted objects (upsert on `id`) keep saving. Deploy the code\n and run `smrt db:migrate` in the same maintenance step. The plan is one\n `DROP INDEX` + `CREATE UNIQUE INDEX` per table under the SAME name (a\n superset key, so the build cannot fail when the old same-name index was a\n valid UNIQUE over the subset key; a #1165-class table whose old index was\n non-unique or missing may hold duplicates that a superset UNIQUE rejects —\n `db:diff` shows which tables' old index is non-unique or missing; dedupe\n those rows before migrating). Atomic mode swaps every table in one\n transaction: `DROP INDEX` takes ACCESS EXCLUSIVE and holds it until commit,\n which blocks ALL access to those tables — reads included — for the batch;\n size `statementTimeout` for the largest tenant-scoped table. That is the\n maintenance window this rollout requires anyway (no mixed-version state), so\n run this wave — the #2359 index wave included — in atomic mode inside it;\n the \"roll out with `--postgres-safe`\" advice above applies to a #2359-only\n wave, because `--postgres-safe` runs the two statements sequentially per\n table, so each table has NO conflict index between them and a failed rebuild\n leaves it without one until the re-run. The recreate has no automatic\n DOWN: reverting the code means re-creating the old index by hand. And\n legacy NULL-tenant rows fork rather than get adopted — a tenant-context save\n whose slug matches a `(NULL, slug, ctx)` row now inserts `(tenant, slug,\n ctx)` beside it, and that tenant no longer sees the legacy row — so backfill\n `tenant_id` (anytown: `SET tenant_id = context::uuid`) BEFORE this release.\n Ingestion that relied on natural-key dedup across tenants now inserts one\n row per tenant (release note).\n- **STI `@field({ unique: true })` is enforced through indexes** (the differ can\n add an index to an existing table, never a column constraint): a full\n `<table>_<col>_unique_idx` when the STI base declares it, one\n `<table>_<col>_<class>_unique_idx WHERE _meta_type = '<qualified>'` per class\n when only descendants do — uniqueness per concrete class, not across the\n subtree. DuckDB/JSON have no partial indexes, so the descendant-scoped shape\n (`isStiSubtypeUniqueIndex`) is not emitted there — degrading it to a full\n UNIQUE would constrain every subtype; the DDL strategy and the differ both\n skip it, while other partial indexes keep degrading to full ones as before. Remember the\n framework serializes an unset text field as `''`, so a unique optional text\n field must be `nullable: true` with a `null` initializer or every unset row\n collides.\n- **Every class in an STI hierarchy carries the schema of the one shared\n table**, generated from the root base (`ManifestGenerator.generateSchemas()`\n resolves the root through `findSTIBaseInfo`), so a child never treats its own\n descendant-only unique field as base-declared.\n\n`src/schema/utils.ts` sits in between, and the two exports differ:\n\n- `generateSchema()` (reached from `SmrtCollection.generateSchema()`) always\n rebuilds from the registry and writes the result back into the registry,\n replacing whatever the manifest registered for that class.\n- `ensureSchema()` (reached from the deprecated `smrt db:setup`) is\n manifest-first: it takes `ObjectRegistry.getSchema()` plus the merged\n `getAllSchemasAsDefinitions()` table definition, and only falls back to\n `generateSchema()` when no schema is registered at all.\n\nSo a normal build keeps the manifest schema through `db:setup`, and a\nregistry-derived schema is a dev/test artifact. `smrt-content` shows what one\nlooks like: `packages/content/src/hooks.server.ts` `bootstrapSchema()` calls\n`generateSchema()` for every registered class and then `ensureSchema()` from the\nSvelteKit `handle` hook on any `/api/*` request, so that process holds\nregistry-derived schemas rather than the manifest ones. It reaches only that\npackage's own `vite dev` app — the library build excludes the file and the\npackage never exports it — but it is the shape to recognize. Check which route a\nprocess actually took before trusting a reproduction.\n\n## Why the drift stayed invisible\n\nEvery drift oracle compares a database with the same artifact that dropped the\nindex:\n\n- `verifyPersistenceTable()` (`src/schema/table-verifier.ts`) calls\n `db.tableExists()` and nothing else. \"Runtime verifies schema\" has always meant\n existence-only — no column, type, constraint, or index comparison.\n- `smrt doctor` never opens a database connection.\n- `db:status` and `db:diff` diff the live database against\n `getAllSchemasAsDefinitions()`, i.e. the manifest projection.\n\nAn index the manifest never emitted is \"in sync\" by construction. That is how a\nproduction database reached 164 unindexed `tenant_id` columns while `db:status`\nreported no drift (#2356 → #2359). The assessment's other counts — 196/231\n`@foreignKey` and 91/92 `@crossPackageRef` columns with no production index,\n238/238 tables carrying a redundant index on the primary key, zero DB-level\nforeign-key constraints on any engine — come from regenerating every package's\nschema against a live database, so re-measure rather than quote them once the\nepic's fixes land.\n\n## Rules\n\n### 1. Verify against the production path, not the test path\n\nAny change to column or index emission goes on **all** paths that ship and is\nproven by the path-parity test (`src/schema/schema-path-parity.test.ts`, #2359)\n— extend its fixture; a green suite otherwise proves the registry paths only.\nRead the call graph before believing a comment: \"same as migrations\" was wrong\nfor years.\n\n### 2. Every new query predicate ships with its index\n\nCollection methods, poll loops, auth lookups, junction right-side filters, and\npolymorphic owner lookups all count — or write down why the predicate does not\nneed one. For list workloads, EXPLAIN on a PostgreSQL snapshot; the measured\nspread on the assessed workload was 21 ms → 0.1 ms.\n\n### 3. Run the PostgreSQL lane\n\nAnything touching numeric types, uuid casts, upsert conflict targets, timestamps,\nor migrations runs the package's `test:postgres` script:\n\n```bash\npnpm --filter @happyvertical/smrt-<pkg> test:postgres\n```\n\ncore, cli, users, sales, marketing, analytics, and vitest carry the lane.\nSQLite's type affinity accepts values PostgreSQL rejects — a money field declared\n`number = 0` compiles to INTEGER and only fails on PG (#2361).\n\n### 4. Read the built artifact, not the source\n\nWhat a decorator produced is in `dist/manifest.json` and in regenerated schemas:\n`integer` vs `decimal`, the actual index list, the actual conflict columns. When\nthe question is \"how many tables/columns/indexes\", regenerate and count across\nevery package; do not sample a few and extrapolate.\n\n### 5. Index intent belongs on both the constraint and the read path\n\nA conflict target is not automatically a unique index, and a unique index is not\nautomatically the index a read path uses. Custom `conflictColumns` used to\nreplace the `(slug, context)` index while `loadFromSlug`/`getId` still queried\nslug+context, and STI dropped `@field({ unique: true })` — both fixed in #2359,\nsee \"Index rules\" above. Check the pair, not the declaration.\n\n### 6. Multi-tenancy is a whole-path property\n\nEvery unique constraint and every conflict target on a tenant-scoped table\nincludes the tenant column — otherwise a second tenant's `save()` of the same\nnatural key updates the first tenant's row through `DO UPDATE SET` (#2360; the\ndefault key now does, see \"Index rules\" — an explicit `conflictColumns` that\nomits the tenant column is the class author's own key and is not rewritten).\nAnd every read path is interceptor-aware: hydration\n(`loadFromId`/`loadFromSlug`), get-by-slug, vector search, and collection\nmemory, not only `list()` (#2365).\n\n### 7. Retry only transient errors\n\nClassify through the cause chain (SQLSTATE), never on a message substring, and\nnever retry inside an aborted PostgreSQL transaction (`25P02`). Test the\ncontract end to end against a real database, not only the classifier (#2366).\n\n### 8. Thread new decorator options through every config-rebuild site\n\nA new `@smrt()` or `@field()` option that affects schema must reach the\n`SchemaGeneratorConfig` type in `src/schema/generator.ts` and every site that\nrebuilds that config — `src/schema/utils.ts` and `src/testing/database.ts` — or\nit is silently dropped on the paths that rebuild it (#2357).\n\n### 9. Delete or wire dead paths, and write docs to what the code does\n\nDead code that looks canonical misleads the next agent: the AST `generateSchema`\npath, `SchemaOverrideSystem`, and the never-emitted `triggers: []` all read as\nsupported surfaces (#2380). Documentation follows the implementation, not the\nintent — say \"verifies the table exists\" when that is what runs.\n\n### 10. Untracked \"known limitation\" comments are bugs nobody will read\n\nFile the issue and link it from the comment. A `products` comment explaining why\na conflict-column change was refrained from sat there for months — and\nmisdescribed the failure mode the whole time.\n\n### 11. Consumer repair scripts are signals\n\nDownstream repair tooling (anytown's `db-repair-plan.ts` carried column-type\nrepairs, missing STI columns and indexes, and `tenant_id` backfills since April)\nis the consumer-side record of framework gaps. Mine it during triage.\n\n### 12. Try to falsify before filing, and treat operations as correctness\n\nRe-verify a finding at source before it becomes an issue — one assessment\ncandidate claimed conflict indexes past two columns were narrowed to two\ncolumns, when only the index *name* is shortened. And an index fix that ships\nwithout a bounded-timeout, `CONCURRENTLY`-capable migrate path can take\nproduction down on rollout (#2362).\n\n### 13. Composite indexes are declared, not inferred (#2357)\n\nThe generated set only covers foreign keys, unique/conflict columns, the STI\ndiscriminator, reference columns (#2359), the default list ordering (rule 18\nbelow), and single columns opted in with `@field({ indexed: true })`. A list\nworkload's access path is composite, so declare it:\n\n```ts\n@smrt({\n indexes: [\n { name: 'contents_tenant_id_publish_date_idx',\n columns: ['tenantId', 'publish_date'] },\n ],\n})\n```\n\n`columns` takes field names or column names in access-path order — filter\ncolumns first, sort column last. Declare columns, not a direction: PostgreSQL\nscans a btree either way, so an ascending index also serves the matching\n`ORDER BY ... DESC` as an ordered scan with no Sort node. `unique` and `where`\n(partial index) are honoured.\n\n`appendDeclaredIndexes()` runs first on all four entry points, ahead of\n`ensureDefaultListOrderingIndex()` (rule 18) and `ensureReferenceColumnIndexes()`,\nso a declared composite leading with the tenant column (or any reference column)\nreplaces the automatic standalone index rather than duplicating it.\nUnknown columns, malformed entries, and a name collision with a different index\nall fail generation — a silently dropped index only surfaces later as a\nproduction slowdown. Rule 8 above is why this works at runtime at all.\n\n### 14. Relationship targets resolve to a class name on both paths\n\n`@foreignKey`/`@oneToMany`/`@manyToMany` accept a class, a name string, or a\n`() => Target` thunk. The decorator invokes the thunk and throws when the target\ncannot be resolved (never `related: ''`); the scanner unwraps the same thunk\nfrom raw source (never `related: '() => Target'`). An unresolved target silently\ncosts the relationship edge, `loadRelated()`, and the FK-derived index (#2379).\nA thunk resolves at decoration time, so a target declared later in the same\nmodule is still in its temporal dead zone — use the string form there.\n\n### 15. A SQLite type change is a table rebuild (#2370)\n\nSQLite has no `ALTER TABLE ... ALTER COLUMN ... TYPE`, so\n`src/migrations/sqlite-rebuild.ts` answers a `type_upgrade` on SQLite with the\nstatement list SQLite's own docs prescribe: stage a new table under\n`_smrt_rebuild_<table>`, copy, drop, rename, replay the indexes and triggers.\n`SchemaComparer.compareTable` swaps that plan in for the differ's\n\"requires table recreation\" placeholder, so `db:migrate` applies it inside the\nnormal atomic batch instead of exiting 1 forever.\n\nFour properties of that module are load-bearing; keep them if you touch it:\n\n- **The target shape comes from the live `sqlite_master` DDL**, retyping only\n the drifted columns. It is not regenerated from the manifest, so the rebuild\n never becomes an implicit `DROP COLUMN`, and it preserves table constraints,\n `CHECK`s, and `WITHOUT ROWID`/`STRICT`.\n- **The rebuild is hoisted ahead of the table's other column changes.** Its\n staging DDL and copy list are captured at diff time, and the differ emits\n changes in manifest field order, so a new field declared above the retyped\n one would otherwise run `ALTER TABLE ... ADD COLUMN` first and have the\n rebuild silently drop it — both statements succeed and the batch commits.\n Rebuild first, then add columns to the rebuilt table.\n- **The copy carries no `CAST`.** SQLite applies the destination column's\n affinity on insert — the same conversion a fresh table performs. An explicit\n cast is worse: non-numeric TEXT cast to REAL/INTEGER silently becomes `0`,\n and an ISO timestamp cast to NUMERIC-affinity `DATETIME` becomes its year.\n- **It refuses when any table has a foreign key onto the target and\n `PRAGMA foreign_keys` is ON** (the SMRT adapter's default). `DROP TABLE`\n performs an implicit `DELETE FROM` that fires `ON DELETE CASCADE` on\n children, and `defer_foreign_keys` defers constraint *checks*, not FK\n *actions* — verified: the child rows go. The target's own self-reference\n counts, because the staging table copies that clause and becomes a child of\n the table being dropped (verified: a two-row self-referencing table finishes\n the rebuild holding one row). Such a column stays manual drift.\n- **`PRAGMA legacy_alter_table` brackets the rename**, because SQLite ≥ 3.25\n re-parses the schema on `ALTER TABLE ... RENAME` and a view still pointing at\n the just-dropped table makes it fail outright. It is restored immediately\n after; a rolled-back batch leaves it set on that connection, which is inert\n here only because nothing else in SMRT renames a table.\n\nAll the drifted columns of one table share a single rebuild: the first change\ncarries the plan and the rest become `no change needed` comments that the CLI\nclassifies as no-ops.\n\n## What the differ compares (#2369)\n\n`SchemaComparer` (`src/migrations/differ.ts`) compares each manifest column's\ntype, then — unless the type itself is drifting — its nullability and default,\nand always reports what it will not touch:\n\n- **Strengthening** (`SET NOT NULL`, `SET DEFAULT`) is executable on\n PostgreSQL/DuckDB. `SET NOT NULL` is preceded by an `UPDATE … WHERE c IS NULL`\n backfill of the manifest default; without a default the live data is probed\n and, if NULLs exist, the change is reported (comment SQL + `advisory`) instead\n of emitting an ALTER that would abort the atomic batch.\n- **Relaxing** (`DROP NOT NULL`, `DROP DEFAULT`) is a report-only advisory until\n the caller passes `relaxColumns` (`db:migrate --relax-columns`). The manifest\n can be under-specified (#2372 registration-order weakness), so a live column\n that is stricter than the manifest is never weakened silently.\n- **Orphans** — DB columns absent from the manifest, DB tables no manifest\n declares (`SchemaDiff.orphan_tables`), and unclaimed `*_key` unique constraint\n indexes — are always reported. A NOT NULL orphan without a default is a\n `warning` advisory (every ORM insert fails on it); `includeDroppedColumns`\n (`--drop-columns`) drops it, `relaxColumns` relaxes it. Advisory-only changes\n carry no SQL, never reach the tracker, and do not fail `db:migrate`.\n- **ADD COLUMN** is planned per engine: DuckDB rejects every inline constraint\n (add with `DEFAULT`, then `SET NOT NULL`, `CREATE UNIQUE INDEX`); SQLite\n rejects inline `UNIQUE` (separate `CREATE UNIQUE INDEX <table>_<col>_key`, the\n PostgreSQL constraint-index name, so the orphan sweep leaves it alone) and\n `NOT NULL` without a default on a populated table; PostgreSQL keeps constraints\n inline. DuckDB has no `ADD CONSTRAINT`, so the separate index is the only\n way to add uniqueness there; the bundled DuckDB 1.4.x resolves\n `ON CONFLICT (col)` through that index (the old #12684 limitation the DuckDB\n strategy's `requiresInlineUnique()` note describes no longer reproduces —\n the #2369 DuckDB test pins the upsert), older DuckDB builds may not. A required column with no default is enforced only on an empty table;\n on a populated one it is added nullable and the `NOT NULL` is reported as a\n manual follow-up on every engine.\n- **SQLite** has no `ALTER COLUMN`: nullability/default alterations are manual\n (comment SQL → `db:migrate` exit 1). The #2370 rebuild (rule 15) consumes\n only `type_upgrade` placeholders today; extending it to rewrite constraints\n would lift this.\n- Defaults compare through `canonicalizeDefault()`, which folds engine\n renderings (`'x'::text`, `CAST('t' AS BOOLEAN)`, `CURRENT_TIMESTAMP` vs\n `now()`) by manifest type; an unclassifiable rendering skips the comparison\n rather than risking a false positive that would churn every run. The\n round-trip test (create from each DDL strategy → compare → zero changes) in\n `src/migrations/__tests__/issue-2369-*.test.ts` guards this.\n\n### 16. `schema.ddl` is a preview, not the table\n\n`SchemaDefinition.ddl` / `manifest.json` `schema.ddl` is the engine-neutral\nCREATE TABLE string from `SchemaGenerator.generateSQL()` with no engine: no\nindexes, no triggers, abstract `REAL`/`JSON`/`UUID`/`TIMESTAMP`. It is kept for\nbackward compatibility only. Everything that needs an executable table renders\n`columns` + `indexes` through `getDDLStrategy(engine)` — `db:migrate`\n(`migrations/orchestrate.ts`), `MigrationGenerator` (default\n`materializeStructuredSchema: true`; `false` is a deprecated opt-out),\n`SchemaAggregator`, and `createIsolatedTestDbFromManifest` in smrt-vitest, the\nlast two via `src/schema/manifest-schema.ts` (`collectManifestTables` /\n`renderCollectedManifestTable`). The cached string is merged in only for a\ntable whose contributors expose no structured columns (hand-authored\nmanifests); table constraints that exist only in the string are dropped with a\nwarning, as `db:migrate` drops them. Do not add a new consumer of the\nstring, and do not write a private CREATE INDEX renderer — the retired ones\ndropped `where` and `jsonPath` (#2358). Every DDL strategy also spells out\n`PRIMARY KEY NOT NULL`: SQLite lets a bare non-INTEGER PRIMARY KEY hold NULL.\n\n### 17. The merged table shape is registration-order independent (#2372)\n\n`getAllSchemas()` and `getAllSchemasAsDefinitions()` fold every class that\nshares a physical table — the whole STI hierarchy — into one shape. Both route\nthrough `buildMergedTableSchemas()`, which groups contributors by table and\nthen merges them in a **deterministic** order: the STI base first, then\nancestors before descendants, then by qualified name.\n\nThat order matters because the first contributor seeds the table: it supplies\nthe fallback base columns, the `idType`, the conflict columns and the cached\nDDL, and its columns win every merge conflict. When registration order decided\nit, an STI child that carries no manifest `schema` — the external- and\nconsumer-manifest case — seeded the table from bare fallback columns and the\nbase class's richer ones were skipped when it registered later, yielding\n`context TEXT` instead of `context TEXT NOT NULL DEFAULT ''` and timestamps\nwith no NOT NULL/DEFAULT. The shipped content manifest lists `Article` before\n`Content`, so the losing order was the one that shipped, and the differ\ncompares types only, so the weak fresh-create was never repaired.\n\nTwo invariants keep the two assembly paths agreeing:\n\n- `createBaseColumns()` mirrors what `generateSchemaFromManifest` /\n `generateSTISchemaFromManifest` emit for the same table, so a table built\n from runtime field metadata alone has the same NOT NULL/DEFAULT shape as one\n built from a manifest. Note `_meta_type` is `TEXT NOT NULL` with **no**\n default, matching the generator.\n- `fieldsToColumns()` reads `required`, `default`, and `description` from the\n top level *or* `_meta`. Registry fields normalize them into `_meta`\n (`manifest-field-merge.ts`), so reading only the top level silently dropped\n NOT NULL and DEFAULT for every registry-sourced field.\n\nSTI columns stay nullable regardless of the field's `required` flag\n(`fieldsToColumns(fields, { stiUnionColumns: true })`): the table holds the\nunion of all subtypes' fields, so a column only one subtype declares is never\npopulated on a sibling's row. Declared defaults are still emitted. This matches\n`generateSTISchemaFromManifest`, which sets `notNull: false` on every non-system\nSTI column.\n\nWhen adding a class-level input to the merged shape, take it from the seeding\ncontributor rather than \"whichever class arrives first\", and cover it with a\nchild-first/base-first equality test.\n\n### 18. The generator owns the index for its own default ordering (#2363)\n\nEvery generated list surface — REST, MCP, the SvelteKit list route — pages with\n`ORDER BY created_at DESC, <pk> ASC` (`DEFAULT_LIST_ORDER_BY`, #2367), and\nuntil #2363 no schema path indexed `created_at` (the AST path, deleted in\n#2380, indexed `updated_at`), so the framework's own default page was a\nsequential scan plus a top-N sort. `ensureDefaultListOrderingIndex()` now runs\non all four entry points and emits:\n\n- `(<tenant column>, created_at)` on a tenant-scoped table — the tenancy\n interceptor puts `tenant_id = ?` in front of every list, so the tenant column\n leads and `created_at` orders within it. This composite **replaces** the\n standalone tenant index from #2359: a B-tree serves every prefix of its\n column list, so `ensureDefaultListOrderingIndex()` is called first and\n `ensureReferenceColumnIndexes()` then sees the column as already served. The\n tenant column is found by `referenceKind === 'tenantId'`, never by the\n `tenant_id` spelling — `@smrt({ tenantScoped: { field } })` renames it.\n- `(created_at)` otherwise.\n\nThree deliberate omissions, so nobody \"fixes\" them later:\n\n- **No `DESC`.** `IndexDefinition` carries no per-column direction and\n PostgreSQL scans a B-tree backwards just as cheaply.\n- **No primary-key tiebreak column.** The default order mixes directions\n (`created_at DESC, id ASC`), so no single-direction index satisfies the whole\n key; the leading columns already turn a full sort into an index scan plus an\n incremental sort over rows sharing a timestamp.\n- **Not scoped per STI subtype.** `(_meta_type, created_at)` would serve a\n child collection's list but not the base class's polymorphic one, which\n carries no discriminator predicate — the same reasoning that keeps STI\n reference indexes plain (#2359). One unqualified index per shared table.\n\nAn existing UNQUALIFIED index that already leads with the same columns\nsuppresses it — a partial or JSON-path index never counts. That is how a\ndeclared `@smrt({ indexes: [...] })` composite (#2357) takes over: declaring\n`(tenant_id, created_at, status)` replaces the generated pair, while declaring\na different sort column such as `(tenant_id, publish_date)` sits **beside** it,\nbecause that index cannot order the default page. Declared indexes are appended\nbefore this helper for exactly that reason; anything that appends an index in\nfuture goes in the same slot, ahead of `ensureDefaultListOrderingIndex()` and\n`ensureReferenceColumnIndexes()`.\n\n### 19. One conflict-target rule, applied on every producer\n\n`save()` upserts on `ObjectRegistry.getConflictColumns()`; the schema must\ncarry exactly one unique index over those columns (or they must be the\nprimary key). Keep the derivation in `src/schema/conflict-target.ts` and let\nevery producer call it — the three manifest pipelines share\n`ManifestGenerator.applyGenerationPasses()` since #2360 because\n`ManifestBuilder` had silently skipped the report passes for months. When you\nadd a way for the key to vary (a new decorator option, a new class kind),\nthread it through `getConflictColumns()`, `normalizeConflictColumns()` and the\ngenerator's `resolveConflictTarget()` together, and extend the parity test's\n\"unique index == conflict target\" assertion; a key the runtime uses and the\nschema does not index is a hard PostgreSQL error (42P10) on the first save,\nand a key the schema indexes without the tenant column is the silent\ncross-tenant overwrite this rule exists for.\n\n### 20. Every generated index name is length-guarded before it leaves a path (#2374)\n\nPostgreSQL truncates any identifier past 63 **bytes** and reports nothing;\nSQLite and DuckDB do not, so the entire test suite was blind to it. The 66-byte\n`content_contribution_revisions_contribution_id_revision_number_idx` shipped\nthat way — only the differ's signature-equivalence check kept it from emitting\n`add_index` on every run. Two names agreeing for 63 bytes is the real hazard:\n`CREATE INDEX IF NOT EXISTS` no-ops against the wrong index, and the second\nindex is never created.\n\n`schema/index-utils.ts` owns the guard, and it splits by who owns the name:\n\n- **Generated index, trigger and PL/pgSQL function names** →\n `shortenIdentifier()`. Deterministic `<head>_<digest><suffix>`, digest taken\n over the **full** original so a shared prefix still yields distinct names, and\n a recognised suffix (`_idx`, `_unique_idx`, `_key`, `_pkey`) preserved.\n- **Hand-declared `@smrt({ indexes: [{ name }] })`** → `assertIdentifierFits()`,\n a hard error in `validateDeclaredIndex()`. Renaming what a developer wrote is\n worse than refusing it, and `SchemaComparer` matches indexes **by name**\n first, so a 70-byte declaration could never match the 63-byte index\n PostgreSQL stored and `db:migrate` would emit `add_index` forever.\n- **Table and column names** → deliberately **not** guarded. PostgreSQL\n truncates identifiers *consistently on every reference*: `CREATE TABLE\n \"<80 bytes>\"` and a later `SELECT ... FROM \"<the same 80 bytes>\"` both resolve\n to the same stored 63-byte name, so one long name round-trips fine end to end.\n `smrt-users` depends on this — it ships an intentional 80-byte\n `@smrt({ tableName })` (`permission_policy_table_name_that_is_far_too_long…`)\n and derives unique Postgres RLS policy names from it. An earlier revision of\n this rule hard-errored here on the theory that the runtime resolves tables by\n name and would break; that theory is wrong for the reason above, and the error\n broke `packages/users`. The residual collision risk is over a name the\n developer chose, not one the generator manufactured.\n\n`enforceIdentifierLimits()` is the single call site per path, placed **after**\n`ensureReferenceColumnIndexes()` — nothing may lengthen a name after it. Doing\nthe shortening at the end rather than at each `indexes.push()` is safe because\nthe digest covers the whole original name, so entries distinct before shortening\nstay distinct after; the helper still throws if two ever collide. The migrate\nleg's `withConflictIndex()` (`registry/schema-builder.ts`) and the PostgreSQL\ntrigger-function name call `shortenIdentifier()` directly, because they compose\na name outside the generator's index list. Note that an over-long *table* name\nstill yields in-limit, distinct *index* names, because the shortening runs over\nthe whole composed name.\n\nThe digest is FNV-1a, not `node:crypto`: `index-utils.ts` is re-exported from\n`schema/utils.ts`, which exists to keep Node built-ins out of browser bundles.\nIt only has to be *stable* — a shortened name that changed between releases\nwould make every deployment drop and recreate the index — so the parity and\nunit tests pin the literal output rather than recomputing it. Unpaired\nsurrogates are folded to U+FFFD before both counting and hashing, so the digest\nis taken over exactly the bytes the driver transmits.\n\nExisting databases migrate **by name swap, without a rebuild**: the live index\nstill carries the name PostgreSQL truncated it to, the manifest now carries the\nshortened one, and the differ claims it by signature (columns + uniqueness +\npredicate), emitting nothing — including under `includeDroppedIndexes`. See\n`migrations/__tests__/index-drift.test.ts` and the PostgreSQL lane test\n`schema/issue-2374-identifier-length-postgres.optional.test.ts`.\n\nOut of scope, deliberately: constraint names PostgreSQL invents for itself. A\nCTI table's inline `UNIQUE` produces an implicit `<table>_<column>_key`, which\ncan exceed 63 bytes even when the table and column each fit. SMRT never names\nit, and PostgreSQL disambiguates its own truncations by appending a counter\nrather than collapsing them, so there is no silent-collision hazard there.\n\n### 21. The `_smrt_` prefix does not mean \"system table\" (#2376)\n\n`bootstrapSystemTables()` owns nine hand-written tables; ~25 more `_smrt_*`\ntables belong to `@smrt()` models and are created by `db:migrate` (feature\nflags, prompt overrides, subscription plans, report schedules, field policies,\njobs). Never classify by prefix — use `SYSTEM_TABLE_NAMES`\n(`schema/system-table-shapes.ts`, derived from the DDL parse) plus\n`FRAMEWORK_OPERATIONAL_TABLES` / `RETIRED_SYSTEM_TABLES` in `system/schema.ts`.\nThe change-feed writer skipped by prefix, so clients syncing those domain\ntables through `_changes` never saw an update.\n\nEditing `ALL_SYSTEM_TABLES` requires bumping `SMRT_SCHEMA_VERSION` *and*\nappending to `SMRT_SCHEMA_DDL_CHECKSUMS` — the version gates the DDL replay, so\nwithout a bump no existing database ever applies the change. A new **column**\nadditionally needs an `addColumnIfMissing()` entry in `system/compatibility.ts`\n(`CREATE TABLE IF NOT EXISTS` is a no-op on an existing table).\n`system-schema-evolution.test.ts` enforces both, and asserts a legacy database\nupgrades to exactly the shape a fresh install gets.\n\n`_smrt_jobs` / `_smrt_job_events` are dual-owned: `db:migrate` creates them,\nthe compatibility pass reshapes them. On a fresh install bootstrap runs first,\nso their pass is deferred — `ensureDeferredSystemTableCompatibility()` re-runs\nuntil the tables exist, then stamps a `<version>+deferred-compat` marker. It\nruns OUTSIDE the bootstrap lock and swallows its own failures: those statements\ntarget tables the framework does not own, and inside the PostgreSQL transaction\none failure would roll back system-table creation with it. Only\n`ensureBootstrapSystemTableCompatibility()` (the tables the DDL itself creates)\nbelongs inside the lock.\n\nReconciling `_smrt_jobs.task_id` uniqueness reads the live index catalog, which\nis implemented for PostgreSQL and SQLite only; DuckDB and the JSON adapter keep\nthe redundant compat index rather than risk dropping the one that enforces the\nupsert conflict target. When reading a PostgreSQL catalog array, cast it\n(`attname::text`) and parse both shapes — a driver with no parser registered for\nthe array OID returns the raw `{a,b}` literal, and reading that as \"no columns\"\nsilently inverts an index-existence decision.\n\n## Referential integrity lives in `delete()`, not in the DDL\n\nNo schema path emits a `FOREIGN KEY` clause on any engine — grep `schema/ddl/*`,\n`schema-manager.ts` and `differ.ts` for `REFERENCES` and you get nothing. That is\na deliberate position, not an oversight: emitting constraints changes delete\nsemantics for every consumer, requires topological table ordering in both\nmigrate paths (neither orders today), and needs a plan for the orphans already in\nproduction databases. Emitting them is tracked separately.\n\nWhat `@foreignKey(..., { onDelete })` therefore means is *application* behaviour,\napplied by `SmrtObject.delete()` through `src/cascade.ts` (#2371):\n\n| Reference | Default when `onDelete` is absent |\n|---|---|\n| Column is part of the referencing class's `conflictColumns`, and is not a `@tenantId()` field | `CASCADE` |\n| Polymorphic `(metaType, metaId)` association row | `CASCADE` |\n| Anything else, including every `@tenantId()` field | `NO ACTION` — the row is left alone |\n\nThe natural-key rule is what cleans junction rows up without any per-package\nannotation: a junction declares\n`@smrt({ conflictColumns: ['content_id', 'asset_id', 'relationship'] })`, so the\nrow is *identified* by the content and cannot outlive it. An ordinary child\n(`Order.customerId`) is keyed by `(slug, context)` and keeps its pre-#2371\nbehaviour unless it opts in explicitly.\n\n**`@tenantId()` is excluded even though it lands in `conflictColumns`.**\n#2360 leads every tenant-scoped class's *default* natural key with the\ntenant column, so without this exclusion, deleting one `Tenant` row would\nrecursively CASCADE through every tenant-scoped table in the schema that has\nnot declared its own `conflictColumns` — the overwhelming majority. The\ntenant column scopes ownership; it does not identify the row the way a\njunction's foreign key does. Detected via the `__tenancy.isTenantIdField`\nmarker on `FieldMeta` (`smrt-core` reads it structurally so it never depends\non `smrt-tenancy`). `@tenantId()` exposes no `onDelete` option today, so\nthis cannot currently be overridden per field — found in review before this\nlanded (originally reachable, untested, and undocumented).\n\nProperties to keep if you touch that module:\n\n- **The plan is registry-derived and rebuilt per delete.** Registration is\n incremental — manifests load lazily and tests register classes between cases —\n so a cached plan would silently skip a table that registered later. Cache it\n only behind an invalidation hook that every registration path calls.\n- **A class with nothing pointing at it skips the transaction entirely — but\n `CascadePlan.isEmpty` requires no polymorphic association class anywhere in\n the process, not just no typed references.** `buildCascadePlan()` pushes\n *every* registered `SmrtPolymorphicAssociation` subclass into\n `plan.polymorphic` unconditionally (`cascade.ts` around\n `isPolymorphicAssociationClass`): a `metaType` column can point at any class\n at runtime, so there is no static metadata to scope it by the target being\n deleted. One registered polymorphic class anywhere makes `isEmpty` false for\n every delete in that process — do not read \"the common case skips the\n transaction\" as \"most deletes in a real app skip it\"; in a multi-package app\n that registers even one polymorphic association, almost none do.\n `runCascadeDelete()` builds the plan for `getResolvedQualifiedName()` (not the\n bare constructor name — two packages can register the same simple name).\n- **Cascaded rows are removed set-based.** Their `beforeDelete`/`afterDelete`\n hooks and interceptors do not run and no change-feed tombstone is written for\n them, which is exactly what a DB-level `ON DELETE CASCADE` does. Only the\n object `delete()` was called on runs the lifecycle. Do not \"improve\" this into\n a per-row model delete without deciding what that means for sync consumers.\n- **Everything is one transaction where the adapter has one**, including the\n object's own `DELETE`, whenever there is anything to cascade. The `RESTRICT`\n checks run first, before any mutation, so a refusal costs nothing; the\n transaction is what makes a refusal *deeper* in the graph safe.\n- **`_smrt_embeddings` and `_smrt_contexts` are matched by id *and* a\n class-name candidate set, not id alone.** Their class columns store the\n *runtime* constructor name, which for an STI hierarchy is a concrete\n subclass rather than the class the cascade planned from — id-alone matching\n looked STI-safe, but let two unrelated classes using `idType: 'text'`\n (non-UUID, not guaranteed globally unique) collide on a shared id value and\n delete each other's rows (review fix). `ownerClassCandidates()` expands to\n every STI hierarchy member of the class the ids actually belong to, in both\n qualified and simple form. A failure to clean them is logged, never raised —\n an application database may predate the table, and losing derived rows must\n not fail a valid delete.\n\n`_smrt_changes`, `_smrt_ai_usage`, `_smrt_signals` and the dispatch tables are\ndeliberately **not** cascaded. They are append-only logs; the change feed in\nparticular receives the delete's own tombstone, so cascading it would erase the\nrecord that tells sync clients the row is gone.\n\n### 22. System tables get a retention policy, not just a prune function (#2375)\n\nFour framework-owned tables grow with traffic and nothing used to remove a row:\n`_smrt_changes` (one per save/delete), `_smrt_ai_usage` (one per AI call, and\npersistence is on by default), `_smrt_contexts` (whose `expires_at` nothing\nenforced) and `_smrt_dispatch` (an operator-only `dispatch:cleanup`).\n`src/system/retention.ts` is now the single place that bounds them.\n\n- **`runRetentionSweep(db, policy)` is the entry point.** It runs the four\n built-in tasks in a fixed order, then every task other packages contributed\n via `registerRetentionTask()` — `@happyvertical/smrt-jobs` registers\n `_smrt_jobs`/`_smrt_job_events`, `@happyvertical/smrt-users` registers\n session/magic-link/CLI-auth expiry. A task that throws is recorded on its own\n result and the sweep continues; a missing table reports `unavailable`, so a\n sweep is safe against a partially bootstrapped database.\n- **A contributed task only exists in a process that loaded its package.** Both\n packages register on import from their entry point, and the registry lives on\n `globalThis` (like `ObjectRegistry`) so a duplicated `smrt-core` resolution\n cannot split it. `smrt db:prune` optionally imports both packages for exactly\n this reason — a project that installs neither correctly gets neither task.\n- **Defaults are opt-out, not opt-in.** `DEFAULT_RETENTION_POLICY` covers the\n four built-in tables (changes 30 days, AI usage 90 days, dispatch 30 days\n completed / 90 days failed, contexts strictly by their own `expires_at`), and\n those are the ones `smrt.configure({ retention })` tunes. Contributed tasks\n carry their own defaults and their own window options —\n `DEFAULT_JOB_RETENTION` (7 days terminal / 30 days failed / 30 days events,\n set through `registerJobRetentionTasks()` or the runner's `retention.jobs`),\n and expired credentials, which have no window because an expired credential\n has nothing worth retaining. Every task, built-in or contributed, can be\n turned off: a table set to `false`, a task set to `false` under `tasks`, or\n `enabled: false` for the whole sweep — through `smrt.configure`,\n `smrt db:prune --skip`, or the runner's `retention` config.\n- **Contributed task names are prefixed with the owning package's short name**\n (`jobs-records`, `jobs-events`, `users-sessions`, …) because the registry is\n one process-global namespace.\n- **Scheduling lives outside core.** A running `TaskRunner` sweeps every six\n hours (`retention: false` opts out) and `smrt db:prune` is the cron entry\n point. The first runner sweep is one interval after `start()`, never at\n start: a crash-looping worker must not become a delete loop.\n- **Every prune counts before it deletes.** `rowCount` is not reliably\n populated across the engines SMRT supports, so counting is both what gives a\n usable figure and what lets `dryRun` preview the *same* predicate rather than\n an approximation of it. Count and delete are two statements and deliberately\n not one transaction — a maintenance pass must not hold a write lock over a\n large delete — so the figure is approximate under concurrent writers. Where\n two bounds can select the same row (`pruneChangeFeed`, `pruneAiUsage`), the\n second bound excludes what the first already accounted for, so a dry run does\n not count an entry twice.\n- **Every retention predicate ships with its index** (rule 2 applies to\n maintenance SQL too): `_smrt_contexts(expires_at)`,\n `_smrt_ai_usage(tenant_id, created_at)` — which is also the subscriptions\n billing meter's range scan — `_smrt_dispatch(status, processed_at)` and\n `(status, updated_at)` come from the system DDL, so they reach existing\n databases through the `SMRT_SCHEMA_VERSION` bump that replays it.\n `_smrt_jobs(status, completed_at)` comes from\n `ensureJobsSystemTableCompatibility()` instead, because `_smrt_jobs` is\n generated from a decorated class and does not exist yet when bootstrap runs;\n the jobs collection calls that path on every `initialize()`.\n- **Expiry enforcement is prune-side only.** `recall()`/`recallAll()` keep\n their documented \"expiry is not applied at read time\" contract — changing it\n would change read semantics for existing callers, which is a different issue\n from bounding storage. `LearningMemory` filters expired rows itself.\n\n### 23. Dead generation surfaces were deleted, not wired (#2380)\n\nRule 9 named three surfaces that read as canonical but were not: the AST\n`generateSchema(objectDef)` entry point, `SchemaOverrideSystem`, and the\nnever-emitted `triggers: []`. Resolution, so a future agent does not re-open\nwhat was deliberately decided:\n\n- **The AST path is gone.** `SchemaGenerator.generateSchema(objectDef)` and its\n AST-only private helpers (`generateIndexes`, `generateTriggers`,\n `extractDependencies`, `generateVersion`, `getTableName`,\n `extractPackageName`) were deleted from `schema/generator.ts`, along with\n their sole caller, `generateSchemaModule()` in `vite-plugin/index.ts`, and the\n `smrt:schema` / `@happyvertical/smrt-virt-schema` virtual module registration\n that fed. Nothing else called it — grep the deleted method's exact name\n before assuming a caller was missed; the path-parity fixture and every other\n rule above already speak only of the four surviving entry points.\n- **`SchemaOverrideSystem` is gone**, file and all\n (`schema/override-system.ts` no longer exists). It was never called from\n anywhere in this repository outside its own now-deleted exports, and two of\n its five public methods (`createPraecoContentOverride`,\n `createPraecoMeetingOverride`) hard-coded a schema extension for a\n consuming project outside this monorepo — scaffolding that never belonged in\n the framework, not a generic feature with a missing caller. `SchemaOverride`\n (the type) went with it; `ColumnDefinition`/`IndexDefinition`/\n `TriggerDefinition`, which it merely referenced, did not.\n- **The DDL-strategy trigger machinery was kept, not deleted.**\n `TriggerDefinition`, `SchemaDefinition.triggers`, and every DDL strategy's\n `generateTriggers()` / `generateTriggerStatement()` / `supportsTriggers()`\n (`schema/ddl/*.ts`) are real, engine-uniform, directly-tested rendering code\n that runs on **every** table creation via `strategy.generateTriggers(schema)`\n — unlike the AST path, this is not an orphaned call graph. It is kept for the\n same reason rule 16 keeps the cached `schema.ddl` string: `SchemaDefinition`\n is part of the shape third-party tooling and published manifests may already\n depend on, and `EngineSpecificDDL`/`MultiEngineDDL` (`schema/ddl/types.ts`)\n carry `triggers` as part of that same contract. Deleting a published field is\n a different (and unjustified) risk from deleting a virtual module nothing\n ever imported.\n- **What changed is what is documented, not what runs.** `schema.triggers` is\n now explicitly documented (`schema/types.ts`) as always `[]` on every schema\n a `@smrt()` class can produce, and why: there is no `@smrt()`/`@field()`\n option that populates it (unlike `indexes`, #2357), `updated_at` is\n maintained at the application layer (`SmrtObject.save()`), and\n `migrations/differ.ts` never diffs triggers — so even a hand-populated one\n would only apply to a newly `CREATE TABLE`d table and never retrofit an\n existing one. Wiring live trigger emission was considered and rejected for\n this issue: it is a migration-rollout feature (retrofitting 238+ existing\n production tables needs the same `SMRT_SCHEMA_VERSION`-replay or differ\n support rule 21/rule 22's system-table work required), not a cleanup, and\n nothing in the epic depended on it the way #2359 depended on FK indexes\n actually shipping.\n- **`_smrt_signals` and `ObjectRegistry.persistToDatabase()`/`loadFromDatabase()`**\n — named in the original finding alongside triggers — were already handled by\n #2376 before this issue landed: see rule 21 and `system/schema.ts`'s\n `RETIRED_SYSTEM_TABLES`. Nothing further to do there.\n- **The two config-rebuild-site comments** (`schema/utils.ts`,\n `testing/database.ts`) rule 8 requires were already in place, added by\n #2357/#2360; the `testing/database.ts` \"same as migrations\" overclaim rule 1\n quotes was already corrected by #2359, and doctor's `experimentalDecorators`\n check was already fixed by #2368/#2399 (see `packages/cli/AGENTS.md`\n Gotchas). Re-verify against current source before repeating any of these —\n the epic's PRs landed across one evening and a stale assessment line is not\n proof a fix is still needed.\n"
|
|
937
937
|
},
|
|
938
938
|
{
|
|
939
939
|
"path": "agents/query-bounds.md",
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"retention.d.ts","sourceRoot":"","sources":["../../src/system/retention.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AAGH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"retention.d.ts","sourceRoot":"","sources":["../../src/system/retention.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AAGH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAc5D,mEAAmE;AACnE,MAAM,WAAW,yBAAyB;IACxC,sCAAsC;IACtC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,2DAA2D;IAC3D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+EAA+E;IAC/E,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,2DAA2D;AAC3D,MAAM,WAAW,sBAAsB;IACrC,sCAAsC;IACtC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,2DAA2D;IAC3D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+EAA+E;IAC/E,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,mEAAmE;AACnE,MAAM,WAAW,uBAAuB;IACtC,sCAAsC;IACtC,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,2DAA2D;AAC3D,MAAM,WAAW,uBAAuB;IACtC,sCAAsC;IACtC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,qFAAqF;IACrF,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,qFAAqF;IACrF,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC9B,6CAA6C;IAC7C,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,+EAA+E;IAC/E,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,qDAAqD;IACrD,OAAO,CAAC,EAAE,yBAAyB,GAAG,KAAK,CAAC;IAC5C,sDAAsD;IACtD,OAAO,CAAC,EAAE,sBAAsB,GAAG,KAAK,CAAC;IACzC,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,uBAAuB,GAAG,KAAK,CAAC;IAC3C,sDAAsD;IACtD,QAAQ,CAAC,EAAE,uBAAuB,GAAG,KAAK,CAAC;IAC3C;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,wBAAwB,EAAE;IACrC,OAAO,EAAE,yBAAyB,CAAC;IACnC,OAAO,EAAE,sBAAsB,CAAC;IAChC,QAAQ,EAAE,uBAAuB,CAAC;IAClC,QAAQ,EAAE,uBAAuB,CAAC;CAMnC,CAAC;AAMF,qDAAqD;AACrD,MAAM,WAAW,oBAAoB;IACnC,0DAA0D;IAC1D,MAAM,EAAE,OAAO,CAAC;IAChB,6EAA6E;IAC7E,GAAG,EAAE,IAAI,CAAC;CACX;AAED,iDAAiD;AACjD,MAAM,WAAW,mBAAmB;IAClC,wCAAwC;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,sEAAsE;IACtE,MAAM,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,OAAO,CAAC,EAAE,UAAU,GAAG,aAAa,CAAC;IACrC,0DAA0D;IAC1D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,2EAA2E;IAC3E,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED,wEAAwE;AACxE,MAAM,MAAM,oBAAoB,GAC5B,MAAM,GACN;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,CAAC;AAEzD;;;;;GAKG;AACH,MAAM,WAAW,aAAa;IAC5B,uEAAuE;IACvE,IAAI,EAAE,MAAM,CAAC;IACb,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wDAAwD;IACxD,GAAG,CACD,EAAE,EAAE,iBAAiB,EACrB,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,oBAAoB,CAAC,CAAC;CAClC;AAuCD;;;;;;;;;;;GAWG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,aAAa,GAAG,IAAI,CAU/D;AAED,6EAA6E;AAC7E,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAE7D;AAED,4EAA4E;AAC5E,wBAAgB,iBAAiB,IAAI,aAAa,EAAE,CAInD;AAED,0DAA0D;AAC1D,wBAAgB,mBAAmB,IAAI,IAAI,CAE1C;AAMD,uEAAuE;AACvE,MAAM,WAAW,gBAAgB;IAC/B,wDAAwD;IACxD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,oDAAoD;IACpD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,+CAA+C;IAC/C,GAAG,CAAC,EAAE,IAAI,CAAC;CACZ;AAQD;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,YAAY,CAChC,EAAE,EAAE,iBAAiB,EACrB,SAAS,EAAE,gBAAgB,GAC1B,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CA0E7B;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,oBAAoB,CACxC,EAAE,EAAE,iBAAiB,EACrB,OAAO,GAAE;IAAE,GAAG,CAAC,EAAE,IAAI,CAAC;IAAC,MAAM,CAAC,EAAE,OAAO,CAAA;CAAO,GAC7C,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAU7B;AAMD,+DAA+D;AAC/D,MAAM,WAAW,oBAAoB;IACnC,0CAA0C;IAC1C,MAAM,EAAE,OAAO,CAAC;IAChB,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,uDAAuD;IACvD,MAAM,EAAE,MAAM,CAAC;IACf,6CAA6C;IAC7C,KAAK,EAAE,mBAAmB,EAAE,CAAC;IAC7B,2EAA2E;IAC3E,MAAM,EAAE,OAAO,CAAC;CACjB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,iBAAiB,EACrB,MAAM,GAAE,eAAoB,GAC3B,OAAO,CAAC,oBAAoB,CAAC,CAoD/B"}
|
package/dist/system/retention.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { toSafeInteger } from "../utils/safe-integer.js";
|
|
1
2
|
import { detectEngine } from "../schema/ddl/index.js";
|
|
2
3
|
import { pruneChangeFeed } from "../change-feed.js";
|
|
3
4
|
import { createLogger } from "@happyvertical/logger";
|
|
@@ -471,9 +472,8 @@ async function selectOne(db, sql, params) {
|
|
|
471
472
|
*/
|
|
472
473
|
async function deleteFrom(db, table, conditions, params, dryRun) {
|
|
473
474
|
const where = whereClause(conditions);
|
|
474
|
-
const
|
|
475
|
-
|
|
476
|
-
if (!Number.isFinite(total) || total <= 0) return 0;
|
|
475
|
+
const total = toSafeInteger((await selectOne(db, `SELECT COUNT(*) AS total FROM ${table} ${where}`, params))?.total ?? 0, "Retention count");
|
|
476
|
+
if (total <= 0) return 0;
|
|
477
477
|
if (!dryRun) await db.query(bindPlaceholders(db, `DELETE FROM ${table} ${where}`), ...params);
|
|
478
478
|
return total;
|
|
479
479
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"retention.js","names":[],"sources":["../../src/system/retention.ts"],"sourcesContent":["/**\n * System-table retention — bounded growth for framework-owned tables (#2375).\n *\n * Several `_smrt_*` tables are append-only by construction and were, until\n * this module, unbounded by default: `_smrt_changes` grew one row per\n * framework save/delete, `_smrt_ai_usage` one row per AI call (persistence is\n * on by default, see `config.ts`), `_smrt_contexts` accumulated rows whose\n * `expires_at` nothing ever enforced, and `_smrt_dispatch` retained completed\n * work until an operator remembered to run `smrt dispatch:cleanup`.\n *\n * This module supplies the missing half:\n *\n * - {@link pruneAiUsage} — the retention API `_smrt_ai_usage` never had.\n * - {@link pruneExpiredContexts} — enforcement for `_smrt_contexts.expires_at`.\n * - {@link runRetentionSweep} — one entry point that applies a documented,\n * opt-out {@link RetentionPolicy} across every framework-owned table, plus\n * any task other packages contribute via {@link registerRetentionTask}\n * (`@happyvertical/smrt-jobs` registers job/job-event cleanup;\n * `@happyvertical/smrt-users` registers session and token expiry).\n *\n * Scheduling is the caller's: `smrt db:prune` is the cron entry point, and a\n * running `TaskRunner` sweeps on an interval unless configured otherwise.\n *\n * ## Failure policy\n *\n * A sweep is maintenance, not a transaction. One failing task never aborts the\n * rest — the failure is captured on that task's {@link RetentionTaskResult}\n * and the sweep continues. Callers decide what a partial sweep means; the CLI\n * exits non-zero when any task failed.\n *\n * ## Predicate indexes\n *\n * Every predicate below is indexed by the system DDL in `system/schema.ts`\n * (`_smrt_contexts(expires_at)`, `_smrt_ai_usage(tenant_id, created_at)`,\n * `_smrt_dispatch(status, processed_at)` / `(status, updated_at)`) or by the\n * jobs compatibility path in `system/compatibility.ts`\n * (`_smrt_jobs(status, completed_at)`, `_smrt_job_events(created_at)`).\n * A new retention predicate must ship with its index.\n *\n * @see https://github.com/happyvertical/smrt/issues/2375\n * @packageDocumentation\n */\n\nimport { createLogger } from '@happyvertical/logger';\nimport type { DatabaseInterface } from '@happyvertical/sql';\nimport { pruneChangeFeed } from '../change-feed.js';\nimport { detectEngine } from '../schema/ddl/index.js';\n\nconst logger = createLogger({ level: 'info' });\n\n/** Milliseconds in a day — the unit every retention policy is expressed in. */\nconst MS_PER_DAY = 24 * 60 * 60 * 1000;\n\n// ============================================================================\n// Policy\n// ============================================================================\n\n/** Retention for the append-only change feed (`_smrt_changes`). */\nexport interface ChangeFeedRetentionPolicy {\n /** Run this task (default `true`). */\n enabled?: boolean;\n /** Drop entries older than this many days (default 30). */\n maxAgeDays?: number;\n /** Additionally keep at most this many newest entries (default: unbounded). */\n maxRows?: number;\n}\n\n/** Retention for AI usage telemetry (`_smrt_ai_usage`). */\nexport interface AiUsageRetentionPolicy {\n /** Run this task (default `true`). */\n enabled?: boolean;\n /** Drop records older than this many days (default 90). */\n maxAgeDays?: number;\n /** Additionally keep at most this many newest records (default: unbounded). */\n maxRows?: number;\n}\n\n/** Retention for remembered context entries (`_smrt_contexts`). */\nexport interface ContextsRetentionPolicy {\n /** Run this task (default `true`). */\n enabled?: boolean;\n}\n\n/** Retention for the dispatch queue (`_smrt_dispatch`). */\nexport interface DispatchRetentionPolicy {\n /** Run this task (default `true`). */\n enabled?: boolean;\n /** Drop completed dispatches processed more than this many days ago (default 30). */\n completedOlderThanDays?: number;\n /** Drop failed dispatches last touched more than this many days ago (default 90). */\n failedOlderThanDays?: number;\n}\n\n/**\n * Retention policy for a sweep.\n *\n * Every table is pruned by default; set a table's entry to `false` (or\n * `{ enabled: false }`) to opt that table out, or `enabled: false` at the top\n * level to opt the whole sweep out.\n */\nexport interface RetentionPolicy {\n /** Run the sweep at all (default `true`). */\n enabled?: boolean;\n /** Report what would be pruned without deleting anything (default `false`). */\n dryRun?: boolean;\n /** `_smrt_changes` retention, or `false` to skip. */\n changes?: ChangeFeedRetentionPolicy | false;\n /** `_smrt_ai_usage` retention, or `false` to skip. */\n aiUsage?: AiUsageRetentionPolicy | false;\n /** `_smrt_contexts` expiry enforcement, or `false` to skip. */\n contexts?: ContextsRetentionPolicy | false;\n /** `_smrt_dispatch` retention, or `false` to skip. */\n dispatch?: DispatchRetentionPolicy | false;\n /**\n * Enable/disable individual tasks contributed through\n * {@link registerRetentionTask}, keyed by task name. Unlisted tasks run.\n */\n tasks?: Record<string, boolean>;\n}\n\n/**\n * Documented retention defaults.\n *\n * Chosen to be generous relative to every in-tree consumer: change-feed\n * cursors resync automatically past the window, AI usage outlives a monthly\n * billing cycle by two months, and dispatch keeps a full quarter of failures\n * for forensics. Applications that need longer history raise these; those that\n * need none set the table to `false`.\n */\nexport const DEFAULT_RETENTION_POLICY: {\n changes: ChangeFeedRetentionPolicy;\n aiUsage: AiUsageRetentionPolicy;\n contexts: ContextsRetentionPolicy;\n dispatch: DispatchRetentionPolicy;\n} = {\n changes: { maxAgeDays: 30 },\n aiUsage: { maxAgeDays: 90 },\n contexts: {},\n dispatch: { completedOlderThanDays: 30, failedOlderThanDays: 90 },\n};\n\n// ============================================================================\n// Tasks\n// ============================================================================\n\n/** Per-run inputs handed to every retention task. */\nexport interface RetentionTaskContext {\n /** Count what would be deleted instead of deleting it. */\n dryRun: boolean;\n /** Sweep start time — every task derives its cutoffs from this one clock. */\n now: Date;\n}\n\n/** What one task did (or why it did nothing). */\nexport interface RetentionTaskResult {\n /** Task name, unique within a sweep. */\n task: string;\n /** Rows deleted, or — under `dryRun` — rows that would be deleted. */\n pruned: number;\n /**\n * Why the task did no work: `disabled` (policy opted it out) or\n * `unavailable` (the table does not exist in this database).\n */\n skipped?: 'disabled' | 'unavailable';\n /** Message of the error this task failed with, if any. */\n error?: string;\n /** Optional per-bucket breakdown (e.g. completed vs failed dispatches). */\n details?: Record<string, number>;\n}\n\n/** What a task returns; a bare number is shorthand for `{ pruned }`. */\nexport type RetentionTaskOutcome =\n | number\n | { pruned: number; details?: Record<string, number> };\n\n/**\n * A retention task contributed by a package that owns its own tables.\n *\n * Tasks must honour `context.dryRun` — a task that deletes during a dry run\n * makes the whole preview a lie.\n */\nexport interface RetentionTask {\n /** Unique name; also the {@link RetentionPolicy.tasks} opt-out key. */\n name: string;\n /** One-line description surfaced by `smrt db:prune --json`. */\n description?: string;\n /** Perform (or, under `dryRun`, count) the deletion. */\n run(\n db: DatabaseInterface,\n context: RetentionTaskContext,\n ): Promise<RetentionTaskOutcome>;\n}\n\nconst RETENTION_TASKS_KEY = Symbol.for('smrt.retention-tasks');\n\n/**\n * Names {@link builtInTasks} always uses for the four framework-owned tables.\n *\n * Reserved against contributed tasks: `RetentionTaskResult.task` is\n * documented unique within a sweep, and a contributed task sharing one of\n * these names would collide with a built-in result — silently, since nothing\n * else in the sweep loop checks for it — and would also be unreachable\n * through {@link RetentionPolicy.tasks}, which only ever sees the *last*\n * task run under that name, never the built-in one (opting a built-in table\n * out goes through its own dedicated policy field instead).\n */\nconst RESERVED_TASK_NAMES = new Set([\n 'changes',\n 'ai-usage',\n 'contexts',\n 'dispatch',\n]);\n\n/**\n * The task registry, held on `globalThis` rather than in module scope.\n *\n * Contributing packages resolve their own copy of `@happyvertical/smrt-core`\n * whenever a consumer's install is not fully deduped, and the CLI resolves a\n * third. A module-local `Map` would give each of them a private, empty\n * registry — the task would register in one and the sweep would run in\n * another. `ObjectRegistry` is on `globalThis` for the same reason.\n */\nfunction registeredTasks(): Map<string, RetentionTask> {\n const root = globalThis as typeof globalThis & {\n [RETENTION_TASKS_KEY]?: Map<string, RetentionTask>;\n };\n root[RETENTION_TASKS_KEY] ??= new Map<string, RetentionTask>();\n return root[RETENTION_TASKS_KEY];\n}\n\n/**\n * Register a retention task so {@link runRetentionSweep} includes it.\n *\n * Registering a task does not schedule anything — a task only runs when\n * something calls {@link runRetentionSweep}, and the policy can still turn it\n * off by name. Re-registering the same name replaces the previous task, which\n * keeps module re-evaluation (HMR, repeated test imports) idempotent.\n *\n * @throws If `task.name` is empty, or is one of the four built-in table\n * names (`changes`, `ai-usage`, `contexts`, `dispatch`) — see\n * {@link RESERVED_TASK_NAMES}.\n */\nexport function registerRetentionTask(task: RetentionTask): void {\n if (!task.name) {\n throw new Error('registerRetentionTask requires a non-empty task name');\n }\n if (RESERVED_TASK_NAMES.has(task.name)) {\n throw new Error(\n `registerRetentionTask: \"${task.name}\" is a built-in task name (reserved: ${[...RESERVED_TASK_NAMES].join(', ')})`,\n );\n }\n registeredTasks().set(task.name, task);\n}\n\n/** Remove a registered retention task. Returns `true` if one was removed. */\nexport function unregisterRetentionTask(name: string): boolean {\n return registeredTasks().delete(name);\n}\n\n/** Registered retention tasks, ordered by name for deterministic sweeps. */\nexport function getRetentionTasks(): RetentionTask[] {\n return [...registeredTasks().values()].sort((left, right) =>\n left.name.localeCompare(right.name),\n );\n}\n\n/** Drop every registered retention task (test helper). */\nexport function clearRetentionTasks(): void {\n registeredTasks().clear();\n}\n\n// ============================================================================\n// AI usage retention\n// ============================================================================\n\n/** Bounds for {@link pruneAiUsage}. At least one bound is required. */\nexport interface AiUsageRetention {\n /** Delete records older than this many milliseconds. */\n maxAgeMs?: number;\n /**\n * Keep at most this many newest records (by `created_at`).\n *\n * A ceiling, not a floor: combined with `maxAgeMs`, the two bounds union\n * — a row past the age cutoff is deleted regardless of `maxRows`, so\n * `maxRows` never resurrects rows the age bound already excludes. If every\n * row is already older than `maxAgeMs`, a finite `maxRows` keeps none of\n * them; it does not guarantee `maxRows` survivors independent of age. (A\n * `maxRows` that overrode age this way would defeat the reason `maxAgeMs`\n * usually exists in the first place — a compliance/privacy deletion\n * deadline that a row-count floor should not be able to postpone.)\n *\n * Rows sharing the cutoff timestamp all survive, so the retained count can\n * exceed `maxRows` by the size of one timestamp tie.\n */\n maxRows?: number;\n /**\n * Restrict the prune to one tenant: a string matches that tenant, `null`\n * matches the global (tenant-less) records. Omit to prune every tenant.\n */\n tenantId?: string | null;\n /** Count matching records without deleting them. */\n dryRun?: boolean;\n /** Clock for the age cutoff (default: now). */\n now?: Date;\n}\n\n/** Name of the AI usage telemetry table. */\nconst AI_USAGE_TABLE = '_smrt_ai_usage';\n\n/** Name of the remembered-context table. */\nconst CONTEXTS_TABLE = '_smrt_contexts';\n\n/**\n * Prune AI usage telemetry to bound `_smrt_ai_usage` growth.\n *\n * `_smrt_ai_usage` gains one row per AI call and persistence is on by default,\n * so a busy deployment writes millions of rows a year with nothing to remove\n * them. Both bounds may be combined; each is applied independently and the\n * returned count is the total deleted.\n *\n * Predicates are covered by `idx_smrt_ai_usage_tenant_created`\n * (`tenant_id, created_at`) — the same index the subscriptions billing meter\n * needs for its `tenant_id + created_at` range scan.\n *\n * @param db - Database holding the system tables.\n * @param retention - At least one of `maxAgeMs` / `maxRows`.\n * @returns Number of records deleted (or, under `dryRun`, matched).\n */\nexport async function pruneAiUsage(\n db: DatabaseInterface,\n retention: AiUsageRetention,\n): Promise<{ pruned: number }> {\n const { maxAgeMs, maxRows, dryRun = false } = retention;\n\n if (maxAgeMs == null && maxRows == null) {\n throw new Error('pruneAiUsage requires maxAgeMs and/or maxRows');\n }\n if (maxAgeMs != null && (!Number.isFinite(maxAgeMs) || maxAgeMs < 0)) {\n throw new Error(`pruneAiUsage maxAgeMs must be >= 0, got ${maxAgeMs}`);\n }\n if (maxRows != null && (!Number.isFinite(maxRows) || maxRows < 0)) {\n throw new Error(`pruneAiUsage maxRows must be >= 0, got ${maxRows}`);\n }\n\n const now = retention.now ?? new Date();\n const scope = tenantScopeClause(retention);\n let pruned = 0;\n\n // Rows the age bound already accounted for. Redundant when they were really\n // deleted, load-bearing under `dryRun`, where they were not — without it the\n // row bound would count the same records a second time.\n const alreadyCounted: { conditions: string[]; params: unknown[] } = {\n conditions: [],\n params: [],\n };\n\n if (maxAgeMs != null) {\n const cutoff = new Date(now.getTime() - maxAgeMs).toISOString();\n pruned += await deleteFrom(\n db,\n AI_USAGE_TABLE,\n [...scope.conditions, 'created_at < ?'],\n [...scope.params, cutoff],\n dryRun,\n );\n alreadyCounted.conditions.push('created_at >= ?');\n alreadyCounted.params.push(cutoff);\n }\n\n if (maxRows != null) {\n const keep = Math.floor(maxRows);\n if (keep === 0) {\n pruned += await deleteFrom(\n db,\n AI_USAGE_TABLE,\n [...scope.conditions, ...alreadyCounted.conditions],\n [...scope.params, ...alreadyCounted.params],\n dryRun,\n );\n } else {\n // The oldest retained row's timestamp is the cutoff. `LIMIT 1 OFFSET n`\n // is portable across every supported engine, unlike the\n // `LIMIT ALL`/`LIMIT -1` spellings of \"all rows past an offset\".\n const boundary = await selectOne(\n db,\n `SELECT created_at FROM ${AI_USAGE_TABLE}\n ${whereClause(scope.conditions)}\n ORDER BY created_at DESC\n LIMIT 1 OFFSET ${keep - 1}`,\n scope.params,\n );\n const cutoff = boundary?.created_at;\n if (cutoff != null) {\n pruned += await deleteFrom(\n db,\n AI_USAGE_TABLE,\n [...scope.conditions, 'created_at < ?', ...alreadyCounted.conditions],\n [...scope.params, cutoff, ...alreadyCounted.params],\n dryRun,\n );\n }\n }\n }\n\n return { pruned };\n}\n\n/**\n * Delete context entries whose `expires_at` has passed.\n *\n * `remember({ expiresAt })` has always stored the expiry and never acted on\n * it. Enforcement is deliberately prune-side only: `recall()`/`recallAll()`\n * keep their documented \"expiry is not applied at read time\" contract, so this\n * bounds storage without changing read semantics for existing callers.\n * (`LearningMemory` filters expired rows itself.)\n *\n * Rows with a `NULL` `expires_at` never expire and are never touched.\n * The predicate is covered by `idx_smrt_contexts_expires_at`.\n *\n * @param db - Database holding the system tables.\n * @param options.now - Clock for the expiry comparison (default: now).\n * @param options.dryRun - Count matching rows without deleting them.\n * @returns Number of expired entries deleted (or, under `dryRun`, matched).\n */\nexport async function pruneExpiredContexts(\n db: DatabaseInterface,\n options: { now?: Date; dryRun?: boolean } = {},\n): Promise<{ pruned: number }> {\n const now = (options.now ?? new Date()).toISOString();\n const pruned = await deleteFrom(\n db,\n CONTEXTS_TABLE,\n ['expires_at IS NOT NULL', 'expires_at < ?'],\n [now],\n options.dryRun ?? false,\n );\n return { pruned };\n}\n\n// ============================================================================\n// Sweep\n// ============================================================================\n\n/** Aggregate outcome of one {@link runRetentionSweep} call. */\nexport interface RetentionSweepResult {\n /** Whether this run only counted rows. */\n dryRun: boolean;\n /** Sweep start time, ISO-8601. */\n startedAt: string;\n /** Wall-clock duration of the sweep. */\n durationMs: number;\n /** Total rows pruned (or matched) across all tasks. */\n pruned: number;\n /** Per-task outcomes, in execution order. */\n tasks: RetentionTaskResult[];\n /** Whether any task failed — the sweep never throws for a task failure. */\n failed: boolean;\n}\n\n/**\n * Apply a {@link RetentionPolicy} across every framework-owned table.\n *\n * Built-in tasks run first, in a fixed order (`changes`, `ai-usage`,\n * `contexts`, `dispatch`), followed by tasks registered via\n * {@link registerRetentionTask} in name order. Tables that do not exist in\n * this database are reported as `skipped: 'unavailable'` rather than failing —\n * a sweep must be safe to run against a partially bootstrapped database.\n *\n * @param db - Database holding the system tables.\n * @param policy - Overrides on {@link DEFAULT_RETENTION_POLICY}.\n * @returns Per-task and aggregate counts; never throws for a task failure.\n */\nexport async function runRetentionSweep(\n db: DatabaseInterface,\n policy: RetentionPolicy = {},\n): Promise<RetentionSweepResult> {\n const startedAt = new Date();\n const dryRun = policy.dryRun ?? false;\n\n if (policy.enabled === false) {\n return {\n dryRun,\n startedAt: startedAt.toISOString(),\n durationMs: 0,\n pruned: 0,\n tasks: [],\n failed: false,\n };\n }\n\n const context: RetentionTaskContext = { dryRun, now: startedAt };\n const tasks: RetentionTaskResult[] = [];\n\n for (const builtIn of builtInTasks(policy)) {\n if (!builtIn.task) {\n tasks.push({ task: builtIn.name, pruned: 0, skipped: 'disabled' });\n continue;\n }\n tasks.push(await runTask(db, builtIn.table, builtIn.task, context));\n }\n\n for (const task of getRetentionTasks()) {\n if (policy.tasks?.[task.name] === false) {\n tasks.push({ task: task.name, pruned: 0, skipped: 'disabled' });\n continue;\n }\n tasks.push(await runTask(db, null, task, context));\n }\n\n const pruned = tasks.reduce((total, task) => total + task.pruned, 0);\n const failed = tasks.some((task) => task.error !== undefined);\n\n logger.info('[smrt] retention sweep complete', {\n dryRun,\n pruned,\n failed,\n tasks: tasks.map((task) => `${task.task}=${task.pruned}`).join(' '),\n });\n\n return {\n dryRun,\n startedAt: startedAt.toISOString(),\n durationMs: Date.now() - startedAt.getTime(),\n pruned,\n tasks,\n failed,\n };\n}\n\n/**\n * Run one task, converting an absent table or a thrown error into a result.\n *\n * `requiredTable` is checked before the task runs so a sweep against a\n * database that never bootstrapped a given system table reports\n * `unavailable` instead of surfacing an engine-specific \"no such table\".\n */\nasync function runTask(\n db: DatabaseInterface,\n requiredTable: string | null,\n task: RetentionTask,\n context: RetentionTaskContext,\n): Promise<RetentionTaskResult> {\n try {\n if (requiredTable && !(await tableExists(db, requiredTable))) {\n return { task: task.name, pruned: 0, skipped: 'unavailable' };\n }\n\n const outcome = await task.run(db, context);\n return typeof outcome === 'number'\n ? { task: task.name, pruned: outcome }\n : { task: task.name, pruned: outcome.pruned, details: outcome.details };\n } catch (error) {\n const message = error instanceof Error ? error.message : String(error);\n logger.warn(`[smrt] retention task \"${task.name}\" failed: ${message}`);\n return { task: task.name, pruned: 0, error: message };\n }\n}\n\n/** Resolve a per-table policy entry, or `null` when it is opted out. */\nfunction resolveEntry<T extends { enabled?: boolean }>(\n entry: T | false | undefined,\n defaults: T,\n): T | null {\n if (entry === false) return null;\n const merged = { ...defaults, ...(entry ?? {}) };\n return merged.enabled === false ? null : merged;\n}\n\n/** A built-in task plus the table it needs and whether policy enabled it. */\ninterface BuiltInRetentionTask {\n name: string;\n table: string;\n task: RetentionTask | null;\n}\n\n/** The framework-owned tasks, in their fixed execution order. */\nfunction builtInTasks(policy: RetentionPolicy): BuiltInRetentionTask[] {\n return [\n { name: 'changes', table: '_smrt_changes', task: changeFeedTask(policy) },\n { name: 'ai-usage', table: AI_USAGE_TABLE, task: aiUsageTask(policy) },\n { name: 'contexts', table: CONTEXTS_TABLE, task: contextsTask(policy) },\n { name: 'dispatch', table: '_smrt_dispatch', task: dispatchTask(policy) },\n ];\n}\n\nfunction changeFeedTask(policy: RetentionPolicy): RetentionTask | null {\n const config = resolveEntry<ChangeFeedRetentionPolicy>(\n policy.changes,\n DEFAULT_RETENTION_POLICY.changes,\n );\n if (!config) return null;\n\n return {\n name: 'changes',\n description: 'Prune the append-only change feed (_smrt_changes)',\n run: async (db, context) => {\n const { pruned } = await pruneChangeFeed(db, {\n maxAgeMs:\n config.maxAgeDays == null\n ? undefined\n : config.maxAgeDays * MS_PER_DAY,\n maxRows: config.maxRows,\n dryRun: context.dryRun,\n });\n return pruned;\n },\n };\n}\n\nfunction aiUsageTask(policy: RetentionPolicy): RetentionTask | null {\n const config = resolveEntry<AiUsageRetentionPolicy>(\n policy.aiUsage,\n DEFAULT_RETENTION_POLICY.aiUsage,\n );\n if (!config) return null;\n\n return {\n name: 'ai-usage',\n description: 'Prune AI usage telemetry (_smrt_ai_usage)',\n run: async (db, context) => {\n const { pruned } = await pruneAiUsage(db, {\n maxAgeMs:\n config.maxAgeDays == null\n ? undefined\n : config.maxAgeDays * MS_PER_DAY,\n maxRows: config.maxRows,\n dryRun: context.dryRun,\n now: context.now,\n });\n return pruned;\n },\n };\n}\n\nfunction contextsTask(policy: RetentionPolicy): RetentionTask | null {\n const config = resolveEntry<ContextsRetentionPolicy>(\n policy.contexts,\n DEFAULT_RETENTION_POLICY.contexts,\n );\n if (!config) return null;\n\n return {\n name: 'contexts',\n description: 'Delete expired context entries (_smrt_contexts)',\n run: async (db, context) => {\n const { pruned } = await pruneExpiredContexts(db, {\n now: context.now,\n dryRun: context.dryRun,\n });\n return pruned;\n },\n };\n}\n\nfunction dispatchTask(policy: RetentionPolicy): RetentionTask | null {\n const config = resolveEntry<DispatchRetentionPolicy>(\n policy.dispatch,\n DEFAULT_RETENTION_POLICY.dispatch,\n );\n if (!config) return null;\n\n return {\n name: 'dispatch',\n description: 'Prune processed dispatch queue rows (_smrt_dispatch)',\n run: async (db, context) => {\n // Imported lazily: the dispatch collection pulls in the whole dispatch\n // module graph, which this module must not force onto callers that only\n // prune system tables.\n const { DispatchCollection } = await import(\n '../dispatch/collections/Dispatches.js'\n );\n const result = await DispatchCollection.cleanup(db, {\n completedOlderThanDays: config.completedOlderThanDays,\n failedOlderThanDays: config.failedOlderThanDays,\n dryRun: context.dryRun,\n });\n return {\n pruned: result.completedDeleted + result.failedDeleted,\n details: {\n completed: result.completedDeleted,\n failed: result.failedDeleted,\n },\n };\n },\n };\n}\n\n// ============================================================================\n// SQL helpers\n// ============================================================================\n\nfunction getQueryRows(result: unknown): Record<string, unknown>[] {\n if (Array.isArray(result)) {\n return result as Record<string, unknown>[];\n }\n if (result && typeof result === 'object' && 'rows' in result) {\n const rows = (result as { rows?: unknown }).rows;\n if (Array.isArray(rows)) {\n return rows as Record<string, unknown>[];\n }\n }\n return [];\n}\n\nfunction getDatabaseUrl(db: DatabaseInterface): string {\n return db.url || (db as { config?: { url?: string } }).config?.url || '';\n}\n\n/**\n * Rewrite `?` placeholders to `$n` on PostgreSQL.\n *\n * Retention SQL is written once in the portable `?` form; only the binding\n * syntax differs between engines.\n */\nfunction bindPlaceholders(db: DatabaseInterface, sql: string): string {\n const engine = detectEngine(\n getDatabaseUrl(db),\n (db as { type?: string }).type,\n );\n if (engine !== 'postgres') return sql;\n\n let index = 0;\n return sql.replace(/\\?/g, () => `$${++index}`);\n}\n\nfunction whereClause(conditions: string[]): string {\n return conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';\n}\n\nfunction tenantScopeClause(retention: AiUsageRetention): {\n conditions: string[];\n params: unknown[];\n} {\n if (!('tenantId' in retention)) return { conditions: [], params: [] };\n if (retention.tenantId === undefined) return { conditions: [], params: [] };\n if (retention.tenantId === null) {\n return { conditions: ['tenant_id IS NULL'], params: [] };\n }\n return { conditions: ['tenant_id = ?'], params: [retention.tenantId] };\n}\n\nasync function selectOne(\n db: DatabaseInterface,\n sql: string,\n params: unknown[],\n): Promise<Record<string, unknown> | undefined> {\n const rows = getQueryRows(\n await db.query(bindPlaceholders(db, sql), ...params),\n );\n return rows[0];\n}\n\n/**\n * Count matching rows, then delete them unless this is a dry run.\n *\n * Counting first gives a usable number on every adapter — `rowCount` is not\n * reliably populated across the engines SMRT supports — and is what makes\n * `dryRun` a genuine preview of the same predicate rather than an estimate of\n * a different one.\n *\n * The count and the delete are two statements and deliberately not a\n * transaction: a maintenance sweep must not hold a write lock over a large\n * delete. The returned figure is therefore approximate under concurrent\n * writers, exactly as `pruneChangeFeed` documents.\n */\nasync function deleteFrom(\n db: DatabaseInterface,\n table: string,\n conditions: string[],\n params: unknown[],\n dryRun: boolean,\n): Promise<number> {\n const where = whereClause(conditions);\n const countRow = await selectOne(\n db,\n `SELECT COUNT(*) AS total FROM ${table} ${where}`,\n params,\n );\n const total = Number(countRow?.total ?? 0);\n if (!Number.isFinite(total) || total <= 0) return 0;\n\n if (!dryRun) {\n await db.query(\n bindPlaceholders(db, `DELETE FROM ${table} ${where}`),\n ...params,\n );\n }\n\n return total;\n}\n\n/** Best-effort table probe used to report `unavailable` rather than throwing. */\nasync function tableExists(\n db: DatabaseInterface,\n table: string,\n): Promise<boolean> {\n try {\n if (typeof db.tableExists === 'function') {\n return await db.tableExists(table);\n }\n } catch {\n return false;\n }\n\n try {\n await db.query(`SELECT 1 FROM ${table} LIMIT 1`);\n return true;\n } catch {\n return false;\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgDA,IAAM,SAAS,aAAa,EAAE,OAAO,OAAO,CAAC;;AAG7C,IAAM,aAAa,OAAU,KAAK;;;;;;;;;;AA8ElC,IAAa,2BAKT;CACF,SAAS,EAAE,YAAY,GAAG;CAC1B,SAAS,EAAE,YAAY,GAAG;CAC1B,UAAU,CAAC;CACX,UAAU;EAAE,wBAAwB;EAAI,qBAAqB;CAAG;AAClE;AAsDA,IAAM,sBAAsB,OAAO,IAAI,sBAAsB;;;;;;;;;;;;AAa7D,IAAM,sCAAsB,IAAI,IAAI;CAClC;CACA;CACA;CACA;AACF,CAAC;;;;;;;;;;AAWD,SAAS,kBAA8C;CACrD,MAAM,OAAO;CAGb,KAAK,yCAAyB,IAAI,IAA2B;CAC7D,OAAO,KAAK;AACd;;;;;;;;;;;;;AAcA,SAAgB,sBAAsB,MAA2B;CAC/D,IAAI,CAAC,KAAK,MACR,MAAM,IAAI,MAAM,sDAAsD;CAExE,IAAI,oBAAoB,IAAI,KAAK,IAAI,GACnC,MAAM,IAAI,MACR,2BAA2B,KAAK,KAAK,uCAAuC,CAAC,GAAG,mBAAmB,CAAC,CAAC,KAAK,IAAI,EAAE,EAClH;CAEF,gBAAgB,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI;AACvC;;AAGA,SAAgB,wBAAwB,MAAuB;CAC7D,OAAO,gBAAgB,CAAC,CAAC,OAAO,IAAI;AACtC;;AAGA,SAAgB,oBAAqC;CACnD,OAAO,CAAC,GAAG,gBAAgB,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,MAAM,UACjD,KAAK,KAAK,cAAc,MAAM,IAAI,CACpC;AACF;;AAGA,SAAgB,sBAA4B;CAC1C,gBAAgB,CAAC,CAAC,MAAM;AAC1B;;AAsCA,IAAM,iBAAiB;;AAGvB,IAAM,iBAAiB;;;;;;;;;;;;;;;;;AAkBvB,eAAsB,aACpB,IACA,WAC6B;CAC7B,MAAM,EAAE,UAAU,SAAS,SAAS,UAAU;CAE9C,IAAI,YAAY,QAAQ,WAAW,MACjC,MAAM,IAAI,MAAM,+CAA+C;CAEjE,IAAI,YAAY,SAAS,CAAC,OAAO,SAAS,QAAQ,KAAK,WAAW,IAChE,MAAM,IAAI,MAAM,2CAA2C,UAAU;CAEvE,IAAI,WAAW,SAAS,CAAC,OAAO,SAAS,OAAO,KAAK,UAAU,IAC7D,MAAM,IAAI,MAAM,0CAA0C,SAAS;CAGrE,MAAM,MAAM,UAAU,uBAAO,IAAI,KAAK;CACtC,MAAM,QAAQ,kBAAkB,SAAS;CACzC,IAAI,SAAS;CAKb,MAAM,iBAA8D;EAClE,YAAY,CAAC;EACb,QAAQ,CAAC;CACX;CAEA,IAAI,YAAY,MAAM;EACpB,MAAM,SAAS,IAAI,KAAK,IAAI,QAAQ,IAAI,QAAQ,CAAC,CAAC,YAAY;EAC9D,UAAU,MAAM,WACd,IACA,gBACA,CAAC,GAAG,MAAM,YAAY,gBAAgB,GACtC,CAAC,GAAG,MAAM,QAAQ,MAAM,GACxB,MACF;EACA,eAAe,WAAW,KAAK,iBAAiB;EAChD,eAAe,OAAO,KAAK,MAAM;CACnC;CAEA,IAAI,WAAW,MAAM;EACnB,MAAM,OAAO,KAAK,MAAM,OAAO;EAC/B,IAAI,SAAS,GACX,UAAU,MAAM,WACd,IACA,gBACA,CAAC,GAAG,MAAM,YAAY,GAAG,eAAe,UAAU,GAClD,CAAC,GAAG,MAAM,QAAQ,GAAG,eAAe,MAAM,GAC1C,MACF;OACK;GAYL,MAAM,UAAS,MARQ,UACrB,IACA,0BAA0B,eAAe;WACtC,YAAY,MAAM,UAAU,EAAE;;0BAEf,OAAO,KACzB,MAAM,MACR,EAAA,EACyB;GACzB,IAAI,UAAU,MACZ,UAAU,MAAM,WACd,IACA,gBACA;IAAC,GAAG,MAAM;IAAY;IAAkB,GAAG,eAAe;GAAU,GACpE;IAAC,GAAG,MAAM;IAAQ;IAAQ,GAAG,eAAe;GAAM,GAClD,MACF;EAEJ;CACF;CAEA,OAAO,EAAE,OAAO;AAClB;;;;;;;;;;;;;;;;;;AAmBA,eAAsB,qBACpB,IACA,UAA4C,CAAC,GAChB;CAS7B,OAAO,EAAE,QAAA,MAPY,WACnB,IACA,gBACA,CAAC,0BAA0B,gBAAgB,GAC3C,EALW,QAAQ,uBAAO,IAAI,KAAK,EAAA,CAAG,YAKrC,CAAG,GACJ,QAAQ,UAAU,KACpB,EACgB;AAClB;;;;;;;;;;;;;;AAmCA,eAAsB,kBACpB,IACA,SAA0B,CAAC,GACI;CAC/B,MAAM,4BAAY,IAAI,KAAK;CAC3B,MAAM,SAAS,OAAO,UAAU;CAEhC,IAAI,OAAO,YAAY,OACrB,OAAO;EACL;EACA,WAAW,UAAU,YAAY;EACjC,YAAY;EACZ,QAAQ;EACR,OAAO,CAAC;EACR,QAAQ;CACV;CAGF,MAAM,UAAgC;EAAE;EAAQ,KAAK;CAAU;CAC/D,MAAM,QAA+B,CAAC;CAEtC,KAAK,MAAM,WAAW,aAAa,MAAM,GAAG;EAC1C,IAAI,CAAC,QAAQ,MAAM;GACjB,MAAM,KAAK;IAAE,MAAM,QAAQ;IAAM,QAAQ;IAAG,SAAS;GAAW,CAAC;GACjE;EACF;EACA,MAAM,KAAK,MAAM,QAAQ,IAAI,QAAQ,OAAO,QAAQ,MAAM,OAAO,CAAC;CACpE;CAEA,KAAK,MAAM,QAAQ,kBAAkB,GAAG;EACtC,IAAI,OAAO,QAAQ,KAAK,UAAU,OAAO;GACvC,MAAM,KAAK;IAAE,MAAM,KAAK;IAAM,QAAQ;IAAG,SAAS;GAAW,CAAC;GAC9D;EACF;EACA,MAAM,KAAK,MAAM,QAAQ,IAAI,MAAM,MAAM,OAAO,CAAC;CACnD;CAEA,MAAM,SAAS,MAAM,QAAQ,OAAO,SAAS,QAAQ,KAAK,QAAQ,CAAC;CACnE,MAAM,SAAS,MAAM,MAAM,SAAS,KAAK,UAAU,KAAA,CAAS;CAE5D,OAAO,KAAK,mCAAmC;EAC7C;EACA;EACA;EACA,OAAO,MAAM,KAAK,SAAS,GAAG,KAAK,KAAK,GAAG,KAAK,QAAQ,CAAC,CAAC,KAAK,GAAG;CACpE,CAAC;CAED,OAAO;EACL;EACA,WAAW,UAAU,YAAY;EACjC,YAAY,KAAK,IAAI,IAAI,UAAU,QAAQ;EAC3C;EACA;EACA;CACF;AACF;;;;;;;;AASA,eAAe,QACb,IACA,eACA,MACA,SAC8B;CAC9B,IAAI;EACF,IAAI,iBAAiB,CAAE,MAAM,YAAY,IAAI,aAAa,GACxD,OAAO;GAAE,MAAM,KAAK;GAAM,QAAQ;GAAG,SAAS;EAAc;EAG9D,MAAM,UAAU,MAAM,KAAK,IAAI,IAAI,OAAO;EAC1C,OAAO,OAAO,YAAY,WACtB;GAAE,MAAM,KAAK;GAAM,QAAQ;EAAQ,IACnC;GAAE,MAAM,KAAK;GAAM,QAAQ,QAAQ;GAAQ,SAAS,QAAQ;EAAQ;CAC1E,SAAS,OAAO;EACd,MAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;EACrE,OAAO,KAAK,0BAA0B,KAAK,KAAK,YAAY,SAAS;EACrE,OAAO;GAAE,MAAM,KAAK;GAAM,QAAQ;GAAG,OAAO;EAAQ;CACtD;AACF;;AAGA,SAAS,aACP,OACA,UACU;CACV,IAAI,UAAU,OAAO,OAAO;CAC5B,MAAM,SAAS;EAAE,GAAG;EAAU,GAAI,SAAS,CAAC;CAAG;CAC/C,OAAO,OAAO,YAAY,QAAQ,OAAO;AAC3C;;AAUA,SAAS,aAAa,QAAiD;CACrE,OAAO;EACL;GAAE,MAAM;GAAW,OAAO;GAAiB,MAAM,eAAe,MAAM;EAAE;EACxE;GAAE,MAAM;GAAY,OAAO;GAAgB,MAAM,YAAY,MAAM;EAAE;EACrE;GAAE,MAAM;GAAY,OAAO;GAAgB,MAAM,aAAa,MAAM;EAAE;EACtE;GAAE,MAAM;GAAY,OAAO;GAAkB,MAAM,aAAa,MAAM;EAAE;CAC1E;AACF;AAEA,SAAS,eAAe,QAA+C;CACrE,MAAM,SAAS,aACb,OAAO,SACP,yBAAyB,OAC3B;CACA,IAAI,CAAC,QAAQ,OAAO;CAEpB,OAAO;EACL,MAAM;EACN,aAAa;EACb,KAAK,OAAO,IAAI,YAAY;GAC1B,MAAM,EAAE,WAAW,MAAM,gBAAgB,IAAI;IAC3C,UACE,OAAO,cAAc,OACjB,KAAA,IACA,OAAO,aAAa;IAC1B,SAAS,OAAO;IAChB,QAAQ,QAAQ;GAClB,CAAC;GACD,OAAO;EACT;CACF;AACF;AAEA,SAAS,YAAY,QAA+C;CAClE,MAAM,SAAS,aACb,OAAO,SACP,yBAAyB,OAC3B;CACA,IAAI,CAAC,QAAQ,OAAO;CAEpB,OAAO;EACL,MAAM;EACN,aAAa;EACb,KAAK,OAAO,IAAI,YAAY;GAC1B,MAAM,EAAE,WAAW,MAAM,aAAa,IAAI;IACxC,UACE,OAAO,cAAc,OACjB,KAAA,IACA,OAAO,aAAa;IAC1B,SAAS,OAAO;IAChB,QAAQ,QAAQ;IAChB,KAAK,QAAQ;GACf,CAAC;GACD,OAAO;EACT;CACF;AACF;AAEA,SAAS,aAAa,QAA+C;CAKnE,IAAI,CAJW,aACb,OAAO,UACP,yBAAyB,QAEtB,GAAQ,OAAO;CAEpB,OAAO;EACL,MAAM;EACN,aAAa;EACb,KAAK,OAAO,IAAI,YAAY;GAC1B,MAAM,EAAE,WAAW,MAAM,qBAAqB,IAAI;IAChD,KAAK,QAAQ;IACb,QAAQ,QAAQ;GAClB,CAAC;GACD,OAAO;EACT;CACF;AACF;AAEA,SAAS,aAAa,QAA+C;CACnE,MAAM,SAAS,aACb,OAAO,UACP,yBAAyB,QAC3B;CACA,IAAI,CAAC,QAAQ,OAAO;CAEpB,OAAO;EACL,MAAM;EACN,aAAa;EACb,KAAK,OAAO,IAAI,YAAY;GAI1B,MAAM,EAAE,uBAAuB,MAAM,OACnC;GAEF,MAAM,SAAS,MAAM,mBAAmB,QAAQ,IAAI;IAClD,wBAAwB,OAAO;IAC/B,qBAAqB,OAAO;IAC5B,QAAQ,QAAQ;GAClB,CAAC;GACD,OAAO;IACL,QAAQ,OAAO,mBAAmB,OAAO;IACzC,SAAS;KACP,WAAW,OAAO;KAClB,QAAQ,OAAO;IACjB;GACF;EACF;CACF;AACF;AAMA,SAAS,aAAa,QAA4C;CAChE,IAAI,MAAM,QAAQ,MAAM,GACtB,OAAO;CAET,IAAI,UAAU,OAAO,WAAW,YAAY,UAAU,QAAQ;EAC5D,MAAM,OAAQ,OAA8B;EAC5C,IAAI,MAAM,QAAQ,IAAI,GACpB,OAAO;CAEX;CACA,OAAO,CAAC;AACV;AAEA,SAAS,eAAe,IAA+B;CACrD,OAAO,GAAG,OAAQ,GAAqC,QAAQ,OAAO;AACxE;;;;;;;AAQA,SAAS,iBAAiB,IAAuB,KAAqB;CAKpE,IAJe,aACb,eAAe,EAAE,GAChB,GAAyB,IAExB,MAAW,YAAY,OAAO;CAElC,IAAI,QAAQ;CACZ,OAAO,IAAI,QAAQ,aAAa,IAAI,EAAE,OAAO;AAC/C;AAEA,SAAS,YAAY,YAA8B;CACjD,OAAO,WAAW,SAAS,IAAI,SAAS,WAAW,KAAK,OAAO,MAAM;AACvE;AAEA,SAAS,kBAAkB,WAGzB;CACA,IAAI,EAAE,cAAc,YAAY,OAAO;EAAE,YAAY,CAAC;EAAG,QAAQ,CAAC;CAAE;CACpE,IAAI,UAAU,aAAa,KAAA,GAAW,OAAO;EAAE,YAAY,CAAC;EAAG,QAAQ,CAAC;CAAE;CAC1E,IAAI,UAAU,aAAa,MACzB,OAAO;EAAE,YAAY,CAAC,mBAAmB;EAAG,QAAQ,CAAC;CAAE;CAEzD,OAAO;EAAE,YAAY,CAAC,eAAe;EAAG,QAAQ,CAAC,UAAU,QAAQ;CAAE;AACvE;AAEA,eAAe,UACb,IACA,KACA,QAC8C;CAI9C,OAHa,aACX,MAAM,GAAG,MAAM,iBAAiB,IAAI,GAAG,GAAG,GAAG,MAAM,CAE9C,CAAA,CAAK;AACd;;;;;;;;;;;;;;AAeA,eAAe,WACb,IACA,OACA,YACA,QACA,QACiB;CACjB,MAAM,QAAQ,YAAY,UAAU;CACpC,MAAM,WAAW,MAAM,UACrB,IACA,iCAAiC,MAAM,GAAG,SAC1C,MACF;CACA,MAAM,QAAQ,OAAO,UAAU,SAAS,CAAC;CACzC,IAAI,CAAC,OAAO,SAAS,KAAK,KAAK,SAAS,GAAG,OAAO;CAElD,IAAI,CAAC,QACH,MAAM,GAAG,MACP,iBAAiB,IAAI,eAAe,MAAM,GAAG,OAAO,GACpD,GAAG,MACL;CAGF,OAAO;AACT;;AAGA,eAAe,YACb,IACA,OACkB;CAClB,IAAI;EACF,IAAI,OAAO,GAAG,gBAAgB,YAC5B,OAAO,MAAM,GAAG,YAAY,KAAK;CAErC,QAAQ;EACN,OAAO;CACT;CAEA,IAAI;EACF,MAAM,GAAG,MAAM,iBAAiB,MAAM,SAAS;EAC/C,OAAO;CACT,QAAQ;EACN,OAAO;CACT;AACF"}
|
|
1
|
+
{"version":3,"file":"retention.js","names":[],"sources":["../../src/system/retention.ts"],"sourcesContent":["/**\n * System-table retention — bounded growth for framework-owned tables (#2375).\n *\n * Several `_smrt_*` tables are append-only by construction and were, until\n * this module, unbounded by default: `_smrt_changes` grew one row per\n * framework save/delete, `_smrt_ai_usage` one row per AI call (persistence is\n * on by default, see `config.ts`), `_smrt_contexts` accumulated rows whose\n * `expires_at` nothing ever enforced, and `_smrt_dispatch` retained completed\n * work until an operator remembered to run `smrt dispatch:cleanup`.\n *\n * This module supplies the missing half:\n *\n * - {@link pruneAiUsage} — the retention API `_smrt_ai_usage` never had.\n * - {@link pruneExpiredContexts} — enforcement for `_smrt_contexts.expires_at`.\n * - {@link runRetentionSweep} — one entry point that applies a documented,\n * opt-out {@link RetentionPolicy} across every framework-owned table, plus\n * any task other packages contribute via {@link registerRetentionTask}\n * (`@happyvertical/smrt-jobs` registers job/job-event cleanup;\n * `@happyvertical/smrt-users` registers session and token expiry).\n *\n * Scheduling is the caller's: `smrt db:prune` is the cron entry point, and a\n * running `TaskRunner` sweeps on an interval unless configured otherwise.\n *\n * ## Failure policy\n *\n * A sweep is maintenance, not a transaction. One failing task never aborts the\n * rest — the failure is captured on that task's {@link RetentionTaskResult}\n * and the sweep continues. Callers decide what a partial sweep means; the CLI\n * exits non-zero when any task failed.\n *\n * ## Predicate indexes\n *\n * Every predicate below is indexed by the system DDL in `system/schema.ts`\n * (`_smrt_contexts(expires_at)`, `_smrt_ai_usage(tenant_id, created_at)`,\n * `_smrt_dispatch(status, processed_at)` / `(status, updated_at)`) or by the\n * jobs compatibility path in `system/compatibility.ts`\n * (`_smrt_jobs(status, completed_at)`, `_smrt_job_events(created_at)`).\n * A new retention predicate must ship with its index.\n *\n * @see https://github.com/happyvertical/smrt/issues/2375\n * @packageDocumentation\n */\n\nimport { createLogger } from '@happyvertical/logger';\nimport type { DatabaseInterface } from '@happyvertical/sql';\nimport { pruneChangeFeed } from '../change-feed.js';\nimport { detectEngine } from '../schema/ddl/index.js';\nimport { toSafeInteger } from '../utils/safe-integer.js';\n\nconst logger = createLogger({ level: 'info' });\n\n/** Milliseconds in a day — the unit every retention policy is expressed in. */\nconst MS_PER_DAY = 24 * 60 * 60 * 1000;\n\n// ============================================================================\n// Policy\n// ============================================================================\n\n/** Retention for the append-only change feed (`_smrt_changes`). */\nexport interface ChangeFeedRetentionPolicy {\n /** Run this task (default `true`). */\n enabled?: boolean;\n /** Drop entries older than this many days (default 30). */\n maxAgeDays?: number;\n /** Additionally keep at most this many newest entries (default: unbounded). */\n maxRows?: number;\n}\n\n/** Retention for AI usage telemetry (`_smrt_ai_usage`). */\nexport interface AiUsageRetentionPolicy {\n /** Run this task (default `true`). */\n enabled?: boolean;\n /** Drop records older than this many days (default 90). */\n maxAgeDays?: number;\n /** Additionally keep at most this many newest records (default: unbounded). */\n maxRows?: number;\n}\n\n/** Retention for remembered context entries (`_smrt_contexts`). */\nexport interface ContextsRetentionPolicy {\n /** Run this task (default `true`). */\n enabled?: boolean;\n}\n\n/** Retention for the dispatch queue (`_smrt_dispatch`). */\nexport interface DispatchRetentionPolicy {\n /** Run this task (default `true`). */\n enabled?: boolean;\n /** Drop completed dispatches processed more than this many days ago (default 30). */\n completedOlderThanDays?: number;\n /** Drop failed dispatches last touched more than this many days ago (default 90). */\n failedOlderThanDays?: number;\n}\n\n/**\n * Retention policy for a sweep.\n *\n * Every table is pruned by default; set a table's entry to `false` (or\n * `{ enabled: false }`) to opt that table out, or `enabled: false` at the top\n * level to opt the whole sweep out.\n */\nexport interface RetentionPolicy {\n /** Run the sweep at all (default `true`). */\n enabled?: boolean;\n /** Report what would be pruned without deleting anything (default `false`). */\n dryRun?: boolean;\n /** `_smrt_changes` retention, or `false` to skip. */\n changes?: ChangeFeedRetentionPolicy | false;\n /** `_smrt_ai_usage` retention, or `false` to skip. */\n aiUsage?: AiUsageRetentionPolicy | false;\n /** `_smrt_contexts` expiry enforcement, or `false` to skip. */\n contexts?: ContextsRetentionPolicy | false;\n /** `_smrt_dispatch` retention, or `false` to skip. */\n dispatch?: DispatchRetentionPolicy | false;\n /**\n * Enable/disable individual tasks contributed through\n * {@link registerRetentionTask}, keyed by task name. Unlisted tasks run.\n */\n tasks?: Record<string, boolean>;\n}\n\n/**\n * Documented retention defaults.\n *\n * Chosen to be generous relative to every in-tree consumer: change-feed\n * cursors resync automatically past the window, AI usage outlives a monthly\n * billing cycle by two months, and dispatch keeps a full quarter of failures\n * for forensics. Applications that need longer history raise these; those that\n * need none set the table to `false`.\n */\nexport const DEFAULT_RETENTION_POLICY: {\n changes: ChangeFeedRetentionPolicy;\n aiUsage: AiUsageRetentionPolicy;\n contexts: ContextsRetentionPolicy;\n dispatch: DispatchRetentionPolicy;\n} = {\n changes: { maxAgeDays: 30 },\n aiUsage: { maxAgeDays: 90 },\n contexts: {},\n dispatch: { completedOlderThanDays: 30, failedOlderThanDays: 90 },\n};\n\n// ============================================================================\n// Tasks\n// ============================================================================\n\n/** Per-run inputs handed to every retention task. */\nexport interface RetentionTaskContext {\n /** Count what would be deleted instead of deleting it. */\n dryRun: boolean;\n /** Sweep start time — every task derives its cutoffs from this one clock. */\n now: Date;\n}\n\n/** What one task did (or why it did nothing). */\nexport interface RetentionTaskResult {\n /** Task name, unique within a sweep. */\n task: string;\n /** Rows deleted, or — under `dryRun` — rows that would be deleted. */\n pruned: number;\n /**\n * Why the task did no work: `disabled` (policy opted it out) or\n * `unavailable` (the table does not exist in this database).\n */\n skipped?: 'disabled' | 'unavailable';\n /** Message of the error this task failed with, if any. */\n error?: string;\n /** Optional per-bucket breakdown (e.g. completed vs failed dispatches). */\n details?: Record<string, number>;\n}\n\n/** What a task returns; a bare number is shorthand for `{ pruned }`. */\nexport type RetentionTaskOutcome =\n | number\n | { pruned: number; details?: Record<string, number> };\n\n/**\n * A retention task contributed by a package that owns its own tables.\n *\n * Tasks must honour `context.dryRun` — a task that deletes during a dry run\n * makes the whole preview a lie.\n */\nexport interface RetentionTask {\n /** Unique name; also the {@link RetentionPolicy.tasks} opt-out key. */\n name: string;\n /** One-line description surfaced by `smrt db:prune --json`. */\n description?: string;\n /** Perform (or, under `dryRun`, count) the deletion. */\n run(\n db: DatabaseInterface,\n context: RetentionTaskContext,\n ): Promise<RetentionTaskOutcome>;\n}\n\nconst RETENTION_TASKS_KEY = Symbol.for('smrt.retention-tasks');\n\n/**\n * Names {@link builtInTasks} always uses for the four framework-owned tables.\n *\n * Reserved against contributed tasks: `RetentionTaskResult.task` is\n * documented unique within a sweep, and a contributed task sharing one of\n * these names would collide with a built-in result — silently, since nothing\n * else in the sweep loop checks for it — and would also be unreachable\n * through {@link RetentionPolicy.tasks}, which only ever sees the *last*\n * task run under that name, never the built-in one (opting a built-in table\n * out goes through its own dedicated policy field instead).\n */\nconst RESERVED_TASK_NAMES = new Set([\n 'changes',\n 'ai-usage',\n 'contexts',\n 'dispatch',\n]);\n\n/**\n * The task registry, held on `globalThis` rather than in module scope.\n *\n * Contributing packages resolve their own copy of `@happyvertical/smrt-core`\n * whenever a consumer's install is not fully deduped, and the CLI resolves a\n * third. A module-local `Map` would give each of them a private, empty\n * registry — the task would register in one and the sweep would run in\n * another. `ObjectRegistry` is on `globalThis` for the same reason.\n */\nfunction registeredTasks(): Map<string, RetentionTask> {\n const root = globalThis as typeof globalThis & {\n [RETENTION_TASKS_KEY]?: Map<string, RetentionTask>;\n };\n root[RETENTION_TASKS_KEY] ??= new Map<string, RetentionTask>();\n return root[RETENTION_TASKS_KEY];\n}\n\n/**\n * Register a retention task so {@link runRetentionSweep} includes it.\n *\n * Registering a task does not schedule anything — a task only runs when\n * something calls {@link runRetentionSweep}, and the policy can still turn it\n * off by name. Re-registering the same name replaces the previous task, which\n * keeps module re-evaluation (HMR, repeated test imports) idempotent.\n *\n * @throws If `task.name` is empty, or is one of the four built-in table\n * names (`changes`, `ai-usage`, `contexts`, `dispatch`) — see\n * {@link RESERVED_TASK_NAMES}.\n */\nexport function registerRetentionTask(task: RetentionTask): void {\n if (!task.name) {\n throw new Error('registerRetentionTask requires a non-empty task name');\n }\n if (RESERVED_TASK_NAMES.has(task.name)) {\n throw new Error(\n `registerRetentionTask: \"${task.name}\" is a built-in task name (reserved: ${[...RESERVED_TASK_NAMES].join(', ')})`,\n );\n }\n registeredTasks().set(task.name, task);\n}\n\n/** Remove a registered retention task. Returns `true` if one was removed. */\nexport function unregisterRetentionTask(name: string): boolean {\n return registeredTasks().delete(name);\n}\n\n/** Registered retention tasks, ordered by name for deterministic sweeps. */\nexport function getRetentionTasks(): RetentionTask[] {\n return [...registeredTasks().values()].sort((left, right) =>\n left.name.localeCompare(right.name),\n );\n}\n\n/** Drop every registered retention task (test helper). */\nexport function clearRetentionTasks(): void {\n registeredTasks().clear();\n}\n\n// ============================================================================\n// AI usage retention\n// ============================================================================\n\n/** Bounds for {@link pruneAiUsage}. At least one bound is required. */\nexport interface AiUsageRetention {\n /** Delete records older than this many milliseconds. */\n maxAgeMs?: number;\n /**\n * Keep at most this many newest records (by `created_at`).\n *\n * A ceiling, not a floor: combined with `maxAgeMs`, the two bounds union\n * — a row past the age cutoff is deleted regardless of `maxRows`, so\n * `maxRows` never resurrects rows the age bound already excludes. If every\n * row is already older than `maxAgeMs`, a finite `maxRows` keeps none of\n * them; it does not guarantee `maxRows` survivors independent of age. (A\n * `maxRows` that overrode age this way would defeat the reason `maxAgeMs`\n * usually exists in the first place — a compliance/privacy deletion\n * deadline that a row-count floor should not be able to postpone.)\n *\n * Rows sharing the cutoff timestamp all survive, so the retained count can\n * exceed `maxRows` by the size of one timestamp tie.\n */\n maxRows?: number;\n /**\n * Restrict the prune to one tenant: a string matches that tenant, `null`\n * matches the global (tenant-less) records. Omit to prune every tenant.\n */\n tenantId?: string | null;\n /** Count matching records without deleting them. */\n dryRun?: boolean;\n /** Clock for the age cutoff (default: now). */\n now?: Date;\n}\n\n/** Name of the AI usage telemetry table. */\nconst AI_USAGE_TABLE = '_smrt_ai_usage';\n\n/** Name of the remembered-context table. */\nconst CONTEXTS_TABLE = '_smrt_contexts';\n\n/**\n * Prune AI usage telemetry to bound `_smrt_ai_usage` growth.\n *\n * `_smrt_ai_usage` gains one row per AI call and persistence is on by default,\n * so a busy deployment writes millions of rows a year with nothing to remove\n * them. Both bounds may be combined; each is applied independently and the\n * returned count is the total deleted.\n *\n * Predicates are covered by `idx_smrt_ai_usage_tenant_created`\n * (`tenant_id, created_at`) — the same index the subscriptions billing meter\n * needs for its `tenant_id + created_at` range scan.\n *\n * @param db - Database holding the system tables.\n * @param retention - At least one of `maxAgeMs` / `maxRows`.\n * @returns Number of records deleted (or, under `dryRun`, matched).\n */\nexport async function pruneAiUsage(\n db: DatabaseInterface,\n retention: AiUsageRetention,\n): Promise<{ pruned: number }> {\n const { maxAgeMs, maxRows, dryRun = false } = retention;\n\n if (maxAgeMs == null && maxRows == null) {\n throw new Error('pruneAiUsage requires maxAgeMs and/or maxRows');\n }\n if (maxAgeMs != null && (!Number.isFinite(maxAgeMs) || maxAgeMs < 0)) {\n throw new Error(`pruneAiUsage maxAgeMs must be >= 0, got ${maxAgeMs}`);\n }\n if (maxRows != null && (!Number.isFinite(maxRows) || maxRows < 0)) {\n throw new Error(`pruneAiUsage maxRows must be >= 0, got ${maxRows}`);\n }\n\n const now = retention.now ?? new Date();\n const scope = tenantScopeClause(retention);\n let pruned = 0;\n\n // Rows the age bound already accounted for. Redundant when they were really\n // deleted, load-bearing under `dryRun`, where they were not — without it the\n // row bound would count the same records a second time.\n const alreadyCounted: { conditions: string[]; params: unknown[] } = {\n conditions: [],\n params: [],\n };\n\n if (maxAgeMs != null) {\n const cutoff = new Date(now.getTime() - maxAgeMs).toISOString();\n pruned += await deleteFrom(\n db,\n AI_USAGE_TABLE,\n [...scope.conditions, 'created_at < ?'],\n [...scope.params, cutoff],\n dryRun,\n );\n alreadyCounted.conditions.push('created_at >= ?');\n alreadyCounted.params.push(cutoff);\n }\n\n if (maxRows != null) {\n const keep = Math.floor(maxRows);\n if (keep === 0) {\n pruned += await deleteFrom(\n db,\n AI_USAGE_TABLE,\n [...scope.conditions, ...alreadyCounted.conditions],\n [...scope.params, ...alreadyCounted.params],\n dryRun,\n );\n } else {\n // The oldest retained row's timestamp is the cutoff. `LIMIT 1 OFFSET n`\n // is portable across every supported engine, unlike the\n // `LIMIT ALL`/`LIMIT -1` spellings of \"all rows past an offset\".\n const boundary = await selectOne(\n db,\n `SELECT created_at FROM ${AI_USAGE_TABLE}\n ${whereClause(scope.conditions)}\n ORDER BY created_at DESC\n LIMIT 1 OFFSET ${keep - 1}`,\n scope.params,\n );\n const cutoff = boundary?.created_at;\n if (cutoff != null) {\n pruned += await deleteFrom(\n db,\n AI_USAGE_TABLE,\n [...scope.conditions, 'created_at < ?', ...alreadyCounted.conditions],\n [...scope.params, cutoff, ...alreadyCounted.params],\n dryRun,\n );\n }\n }\n }\n\n return { pruned };\n}\n\n/**\n * Delete context entries whose `expires_at` has passed.\n *\n * `remember({ expiresAt })` has always stored the expiry and never acted on\n * it. Enforcement is deliberately prune-side only: `recall()`/`recallAll()`\n * keep their documented \"expiry is not applied at read time\" contract, so this\n * bounds storage without changing read semantics for existing callers.\n * (`LearningMemory` filters expired rows itself.)\n *\n * Rows with a `NULL` `expires_at` never expire and are never touched.\n * The predicate is covered by `idx_smrt_contexts_expires_at`.\n *\n * @param db - Database holding the system tables.\n * @param options.now - Clock for the expiry comparison (default: now).\n * @param options.dryRun - Count matching rows without deleting them.\n * @returns Number of expired entries deleted (or, under `dryRun`, matched).\n */\nexport async function pruneExpiredContexts(\n db: DatabaseInterface,\n options: { now?: Date; dryRun?: boolean } = {},\n): Promise<{ pruned: number }> {\n const now = (options.now ?? new Date()).toISOString();\n const pruned = await deleteFrom(\n db,\n CONTEXTS_TABLE,\n ['expires_at IS NOT NULL', 'expires_at < ?'],\n [now],\n options.dryRun ?? false,\n );\n return { pruned };\n}\n\n// ============================================================================\n// Sweep\n// ============================================================================\n\n/** Aggregate outcome of one {@link runRetentionSweep} call. */\nexport interface RetentionSweepResult {\n /** Whether this run only counted rows. */\n dryRun: boolean;\n /** Sweep start time, ISO-8601. */\n startedAt: string;\n /** Wall-clock duration of the sweep. */\n durationMs: number;\n /** Total rows pruned (or matched) across all tasks. */\n pruned: number;\n /** Per-task outcomes, in execution order. */\n tasks: RetentionTaskResult[];\n /** Whether any task failed — the sweep never throws for a task failure. */\n failed: boolean;\n}\n\n/**\n * Apply a {@link RetentionPolicy} across every framework-owned table.\n *\n * Built-in tasks run first, in a fixed order (`changes`, `ai-usage`,\n * `contexts`, `dispatch`), followed by tasks registered via\n * {@link registerRetentionTask} in name order. Tables that do not exist in\n * this database are reported as `skipped: 'unavailable'` rather than failing —\n * a sweep must be safe to run against a partially bootstrapped database.\n *\n * @param db - Database holding the system tables.\n * @param policy - Overrides on {@link DEFAULT_RETENTION_POLICY}.\n * @returns Per-task and aggregate counts; never throws for a task failure.\n */\nexport async function runRetentionSweep(\n db: DatabaseInterface,\n policy: RetentionPolicy = {},\n): Promise<RetentionSweepResult> {\n const startedAt = new Date();\n const dryRun = policy.dryRun ?? false;\n\n if (policy.enabled === false) {\n return {\n dryRun,\n startedAt: startedAt.toISOString(),\n durationMs: 0,\n pruned: 0,\n tasks: [],\n failed: false,\n };\n }\n\n const context: RetentionTaskContext = { dryRun, now: startedAt };\n const tasks: RetentionTaskResult[] = [];\n\n for (const builtIn of builtInTasks(policy)) {\n if (!builtIn.task) {\n tasks.push({ task: builtIn.name, pruned: 0, skipped: 'disabled' });\n continue;\n }\n tasks.push(await runTask(db, builtIn.table, builtIn.task, context));\n }\n\n for (const task of getRetentionTasks()) {\n if (policy.tasks?.[task.name] === false) {\n tasks.push({ task: task.name, pruned: 0, skipped: 'disabled' });\n continue;\n }\n tasks.push(await runTask(db, null, task, context));\n }\n\n const pruned = tasks.reduce((total, task) => total + task.pruned, 0);\n const failed = tasks.some((task) => task.error !== undefined);\n\n logger.info('[smrt] retention sweep complete', {\n dryRun,\n pruned,\n failed,\n tasks: tasks.map((task) => `${task.task}=${task.pruned}`).join(' '),\n });\n\n return {\n dryRun,\n startedAt: startedAt.toISOString(),\n durationMs: Date.now() - startedAt.getTime(),\n pruned,\n tasks,\n failed,\n };\n}\n\n/**\n * Run one task, converting an absent table or a thrown error into a result.\n *\n * `requiredTable` is checked before the task runs so a sweep against a\n * database that never bootstrapped a given system table reports\n * `unavailable` instead of surfacing an engine-specific \"no such table\".\n */\nasync function runTask(\n db: DatabaseInterface,\n requiredTable: string | null,\n task: RetentionTask,\n context: RetentionTaskContext,\n): Promise<RetentionTaskResult> {\n try {\n if (requiredTable && !(await tableExists(db, requiredTable))) {\n return { task: task.name, pruned: 0, skipped: 'unavailable' };\n }\n\n const outcome = await task.run(db, context);\n return typeof outcome === 'number'\n ? { task: task.name, pruned: outcome }\n : { task: task.name, pruned: outcome.pruned, details: outcome.details };\n } catch (error) {\n const message = error instanceof Error ? error.message : String(error);\n logger.warn(`[smrt] retention task \"${task.name}\" failed: ${message}`);\n return { task: task.name, pruned: 0, error: message };\n }\n}\n\n/** Resolve a per-table policy entry, or `null` when it is opted out. */\nfunction resolveEntry<T extends { enabled?: boolean }>(\n entry: T | false | undefined,\n defaults: T,\n): T | null {\n if (entry === false) return null;\n const merged = { ...defaults, ...(entry ?? {}) };\n return merged.enabled === false ? null : merged;\n}\n\n/** A built-in task plus the table it needs and whether policy enabled it. */\ninterface BuiltInRetentionTask {\n name: string;\n table: string;\n task: RetentionTask | null;\n}\n\n/** The framework-owned tasks, in their fixed execution order. */\nfunction builtInTasks(policy: RetentionPolicy): BuiltInRetentionTask[] {\n return [\n { name: 'changes', table: '_smrt_changes', task: changeFeedTask(policy) },\n { name: 'ai-usage', table: AI_USAGE_TABLE, task: aiUsageTask(policy) },\n { name: 'contexts', table: CONTEXTS_TABLE, task: contextsTask(policy) },\n { name: 'dispatch', table: '_smrt_dispatch', task: dispatchTask(policy) },\n ];\n}\n\nfunction changeFeedTask(policy: RetentionPolicy): RetentionTask | null {\n const config = resolveEntry<ChangeFeedRetentionPolicy>(\n policy.changes,\n DEFAULT_RETENTION_POLICY.changes,\n );\n if (!config) return null;\n\n return {\n name: 'changes',\n description: 'Prune the append-only change feed (_smrt_changes)',\n run: async (db, context) => {\n const { pruned } = await pruneChangeFeed(db, {\n maxAgeMs:\n config.maxAgeDays == null\n ? undefined\n : config.maxAgeDays * MS_PER_DAY,\n maxRows: config.maxRows,\n dryRun: context.dryRun,\n });\n return pruned;\n },\n };\n}\n\nfunction aiUsageTask(policy: RetentionPolicy): RetentionTask | null {\n const config = resolveEntry<AiUsageRetentionPolicy>(\n policy.aiUsage,\n DEFAULT_RETENTION_POLICY.aiUsage,\n );\n if (!config) return null;\n\n return {\n name: 'ai-usage',\n description: 'Prune AI usage telemetry (_smrt_ai_usage)',\n run: async (db, context) => {\n const { pruned } = await pruneAiUsage(db, {\n maxAgeMs:\n config.maxAgeDays == null\n ? undefined\n : config.maxAgeDays * MS_PER_DAY,\n maxRows: config.maxRows,\n dryRun: context.dryRun,\n now: context.now,\n });\n return pruned;\n },\n };\n}\n\nfunction contextsTask(policy: RetentionPolicy): RetentionTask | null {\n const config = resolveEntry<ContextsRetentionPolicy>(\n policy.contexts,\n DEFAULT_RETENTION_POLICY.contexts,\n );\n if (!config) return null;\n\n return {\n name: 'contexts',\n description: 'Delete expired context entries (_smrt_contexts)',\n run: async (db, context) => {\n const { pruned } = await pruneExpiredContexts(db, {\n now: context.now,\n dryRun: context.dryRun,\n });\n return pruned;\n },\n };\n}\n\nfunction dispatchTask(policy: RetentionPolicy): RetentionTask | null {\n const config = resolveEntry<DispatchRetentionPolicy>(\n policy.dispatch,\n DEFAULT_RETENTION_POLICY.dispatch,\n );\n if (!config) return null;\n\n return {\n name: 'dispatch',\n description: 'Prune processed dispatch queue rows (_smrt_dispatch)',\n run: async (db, context) => {\n // Imported lazily: the dispatch collection pulls in the whole dispatch\n // module graph, which this module must not force onto callers that only\n // prune system tables.\n const { DispatchCollection } = await import(\n '../dispatch/collections/Dispatches.js'\n );\n const result = await DispatchCollection.cleanup(db, {\n completedOlderThanDays: config.completedOlderThanDays,\n failedOlderThanDays: config.failedOlderThanDays,\n dryRun: context.dryRun,\n });\n return {\n pruned: result.completedDeleted + result.failedDeleted,\n details: {\n completed: result.completedDeleted,\n failed: result.failedDeleted,\n },\n };\n },\n };\n}\n\n// ============================================================================\n// SQL helpers\n// ============================================================================\n\nfunction getQueryRows(result: unknown): Record<string, unknown>[] {\n if (Array.isArray(result)) {\n return result as Record<string, unknown>[];\n }\n if (result && typeof result === 'object' && 'rows' in result) {\n const rows = (result as { rows?: unknown }).rows;\n if (Array.isArray(rows)) {\n return rows as Record<string, unknown>[];\n }\n }\n return [];\n}\n\nfunction getDatabaseUrl(db: DatabaseInterface): string {\n return db.url || (db as { config?: { url?: string } }).config?.url || '';\n}\n\n/**\n * Rewrite `?` placeholders to `$n` on PostgreSQL.\n *\n * Retention SQL is written once in the portable `?` form; only the binding\n * syntax differs between engines.\n */\nfunction bindPlaceholders(db: DatabaseInterface, sql: string): string {\n const engine = detectEngine(\n getDatabaseUrl(db),\n (db as { type?: string }).type,\n );\n if (engine !== 'postgres') return sql;\n\n let index = 0;\n return sql.replace(/\\?/g, () => `$${++index}`);\n}\n\nfunction whereClause(conditions: string[]): string {\n return conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';\n}\n\nfunction tenantScopeClause(retention: AiUsageRetention): {\n conditions: string[];\n params: unknown[];\n} {\n if (!('tenantId' in retention)) return { conditions: [], params: [] };\n if (retention.tenantId === undefined) return { conditions: [], params: [] };\n if (retention.tenantId === null) {\n return { conditions: ['tenant_id IS NULL'], params: [] };\n }\n return { conditions: ['tenant_id = ?'], params: [retention.tenantId] };\n}\n\nasync function selectOne(\n db: DatabaseInterface,\n sql: string,\n params: unknown[],\n): Promise<Record<string, unknown> | undefined> {\n const rows = getQueryRows(\n await db.query(bindPlaceholders(db, sql), ...params),\n );\n return rows[0];\n}\n\n/**\n * Count matching rows, then delete them unless this is a dry run.\n *\n * Counting first gives a usable number on every adapter — `rowCount` is not\n * reliably populated across the engines SMRT supports — and is what makes\n * `dryRun` a genuine preview of the same predicate rather than an estimate of\n * a different one.\n *\n * The count and the delete are two statements and deliberately not a\n * transaction: a maintenance sweep must not hold a write lock over a large\n * delete. The returned figure is therefore approximate under concurrent\n * writers, exactly as `pruneChangeFeed` documents.\n */\nasync function deleteFrom(\n db: DatabaseInterface,\n table: string,\n conditions: string[],\n params: unknown[],\n dryRun: boolean,\n): Promise<number> {\n const where = whereClause(conditions);\n const countRow = await selectOne(\n db,\n `SELECT COUNT(*) AS total FROM ${table} ${where}`,\n params,\n );\n const total = toSafeInteger(countRow?.total ?? 0, 'Retention count');\n if (total <= 0) return 0;\n\n if (!dryRun) {\n await db.query(\n bindPlaceholders(db, `DELETE FROM ${table} ${where}`),\n ...params,\n );\n }\n\n return total;\n}\n\n/** Best-effort table probe used to report `unavailable` rather than throwing. */\nasync function tableExists(\n db: DatabaseInterface,\n table: string,\n): Promise<boolean> {\n try {\n if (typeof db.tableExists === 'function') {\n return await db.tableExists(table);\n }\n } catch {\n return false;\n }\n\n try {\n await db.query(`SELECT 1 FROM ${table} LIMIT 1`);\n return true;\n } catch {\n return false;\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiDA,IAAM,SAAS,aAAa,EAAE,OAAO,OAAO,CAAC;;AAG7C,IAAM,aAAa,OAAU,KAAK;;;;;;;;;;AA8ElC,IAAa,2BAKT;CACF,SAAS,EAAE,YAAY,GAAG;CAC1B,SAAS,EAAE,YAAY,GAAG;CAC1B,UAAU,CAAC;CACX,UAAU;EAAE,wBAAwB;EAAI,qBAAqB;CAAG;AAClE;AAsDA,IAAM,sBAAsB,OAAO,IAAI,sBAAsB;;;;;;;;;;;;AAa7D,IAAM,sCAAsB,IAAI,IAAI;CAClC;CACA;CACA;CACA;AACF,CAAC;;;;;;;;;;AAWD,SAAS,kBAA8C;CACrD,MAAM,OAAO;CAGb,KAAK,yCAAyB,IAAI,IAA2B;CAC7D,OAAO,KAAK;AACd;;;;;;;;;;;;;AAcA,SAAgB,sBAAsB,MAA2B;CAC/D,IAAI,CAAC,KAAK,MACR,MAAM,IAAI,MAAM,sDAAsD;CAExE,IAAI,oBAAoB,IAAI,KAAK,IAAI,GACnC,MAAM,IAAI,MACR,2BAA2B,KAAK,KAAK,uCAAuC,CAAC,GAAG,mBAAmB,CAAC,CAAC,KAAK,IAAI,EAAE,EAClH;CAEF,gBAAgB,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI;AACvC;;AAGA,SAAgB,wBAAwB,MAAuB;CAC7D,OAAO,gBAAgB,CAAC,CAAC,OAAO,IAAI;AACtC;;AAGA,SAAgB,oBAAqC;CACnD,OAAO,CAAC,GAAG,gBAAgB,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,MAAM,UACjD,KAAK,KAAK,cAAc,MAAM,IAAI,CACpC;AACF;;AAGA,SAAgB,sBAA4B;CAC1C,gBAAgB,CAAC,CAAC,MAAM;AAC1B;;AAsCA,IAAM,iBAAiB;;AAGvB,IAAM,iBAAiB;;;;;;;;;;;;;;;;;AAkBvB,eAAsB,aACpB,IACA,WAC6B;CAC7B,MAAM,EAAE,UAAU,SAAS,SAAS,UAAU;CAE9C,IAAI,YAAY,QAAQ,WAAW,MACjC,MAAM,IAAI,MAAM,+CAA+C;CAEjE,IAAI,YAAY,SAAS,CAAC,OAAO,SAAS,QAAQ,KAAK,WAAW,IAChE,MAAM,IAAI,MAAM,2CAA2C,UAAU;CAEvE,IAAI,WAAW,SAAS,CAAC,OAAO,SAAS,OAAO,KAAK,UAAU,IAC7D,MAAM,IAAI,MAAM,0CAA0C,SAAS;CAGrE,MAAM,MAAM,UAAU,uBAAO,IAAI,KAAK;CACtC,MAAM,QAAQ,kBAAkB,SAAS;CACzC,IAAI,SAAS;CAKb,MAAM,iBAA8D;EAClE,YAAY,CAAC;EACb,QAAQ,CAAC;CACX;CAEA,IAAI,YAAY,MAAM;EACpB,MAAM,SAAS,IAAI,KAAK,IAAI,QAAQ,IAAI,QAAQ,CAAC,CAAC,YAAY;EAC9D,UAAU,MAAM,WACd,IACA,gBACA,CAAC,GAAG,MAAM,YAAY,gBAAgB,GACtC,CAAC,GAAG,MAAM,QAAQ,MAAM,GACxB,MACF;EACA,eAAe,WAAW,KAAK,iBAAiB;EAChD,eAAe,OAAO,KAAK,MAAM;CACnC;CAEA,IAAI,WAAW,MAAM;EACnB,MAAM,OAAO,KAAK,MAAM,OAAO;EAC/B,IAAI,SAAS,GACX,UAAU,MAAM,WACd,IACA,gBACA,CAAC,GAAG,MAAM,YAAY,GAAG,eAAe,UAAU,GAClD,CAAC,GAAG,MAAM,QAAQ,GAAG,eAAe,MAAM,GAC1C,MACF;OACK;GAYL,MAAM,UAAS,MARQ,UACrB,IACA,0BAA0B,eAAe;WACtC,YAAY,MAAM,UAAU,EAAE;;0BAEf,OAAO,KACzB,MAAM,MACR,EAAA,EACyB;GACzB,IAAI,UAAU,MACZ,UAAU,MAAM,WACd,IACA,gBACA;IAAC,GAAG,MAAM;IAAY;IAAkB,GAAG,eAAe;GAAU,GACpE;IAAC,GAAG,MAAM;IAAQ;IAAQ,GAAG,eAAe;GAAM,GAClD,MACF;EAEJ;CACF;CAEA,OAAO,EAAE,OAAO;AAClB;;;;;;;;;;;;;;;;;;AAmBA,eAAsB,qBACpB,IACA,UAA4C,CAAC,GAChB;CAS7B,OAAO,EAAE,QAAA,MAPY,WACnB,IACA,gBACA,CAAC,0BAA0B,gBAAgB,GAC3C,EALW,QAAQ,uBAAO,IAAI,KAAK,EAAA,CAAG,YAKrC,CAAG,GACJ,QAAQ,UAAU,KACpB,EACgB;AAClB;;;;;;;;;;;;;;AAmCA,eAAsB,kBACpB,IACA,SAA0B,CAAC,GACI;CAC/B,MAAM,4BAAY,IAAI,KAAK;CAC3B,MAAM,SAAS,OAAO,UAAU;CAEhC,IAAI,OAAO,YAAY,OACrB,OAAO;EACL;EACA,WAAW,UAAU,YAAY;EACjC,YAAY;EACZ,QAAQ;EACR,OAAO,CAAC;EACR,QAAQ;CACV;CAGF,MAAM,UAAgC;EAAE;EAAQ,KAAK;CAAU;CAC/D,MAAM,QAA+B,CAAC;CAEtC,KAAK,MAAM,WAAW,aAAa,MAAM,GAAG;EAC1C,IAAI,CAAC,QAAQ,MAAM;GACjB,MAAM,KAAK;IAAE,MAAM,QAAQ;IAAM,QAAQ;IAAG,SAAS;GAAW,CAAC;GACjE;EACF;EACA,MAAM,KAAK,MAAM,QAAQ,IAAI,QAAQ,OAAO,QAAQ,MAAM,OAAO,CAAC;CACpE;CAEA,KAAK,MAAM,QAAQ,kBAAkB,GAAG;EACtC,IAAI,OAAO,QAAQ,KAAK,UAAU,OAAO;GACvC,MAAM,KAAK;IAAE,MAAM,KAAK;IAAM,QAAQ;IAAG,SAAS;GAAW,CAAC;GAC9D;EACF;EACA,MAAM,KAAK,MAAM,QAAQ,IAAI,MAAM,MAAM,OAAO,CAAC;CACnD;CAEA,MAAM,SAAS,MAAM,QAAQ,OAAO,SAAS,QAAQ,KAAK,QAAQ,CAAC;CACnE,MAAM,SAAS,MAAM,MAAM,SAAS,KAAK,UAAU,KAAA,CAAS;CAE5D,OAAO,KAAK,mCAAmC;EAC7C;EACA;EACA;EACA,OAAO,MAAM,KAAK,SAAS,GAAG,KAAK,KAAK,GAAG,KAAK,QAAQ,CAAC,CAAC,KAAK,GAAG;CACpE,CAAC;CAED,OAAO;EACL;EACA,WAAW,UAAU,YAAY;EACjC,YAAY,KAAK,IAAI,IAAI,UAAU,QAAQ;EAC3C;EACA;EACA;CACF;AACF;;;;;;;;AASA,eAAe,QACb,IACA,eACA,MACA,SAC8B;CAC9B,IAAI;EACF,IAAI,iBAAiB,CAAE,MAAM,YAAY,IAAI,aAAa,GACxD,OAAO;GAAE,MAAM,KAAK;GAAM,QAAQ;GAAG,SAAS;EAAc;EAG9D,MAAM,UAAU,MAAM,KAAK,IAAI,IAAI,OAAO;EAC1C,OAAO,OAAO,YAAY,WACtB;GAAE,MAAM,KAAK;GAAM,QAAQ;EAAQ,IACnC;GAAE,MAAM,KAAK;GAAM,QAAQ,QAAQ;GAAQ,SAAS,QAAQ;EAAQ;CAC1E,SAAS,OAAO;EACd,MAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;EACrE,OAAO,KAAK,0BAA0B,KAAK,KAAK,YAAY,SAAS;EACrE,OAAO;GAAE,MAAM,KAAK;GAAM,QAAQ;GAAG,OAAO;EAAQ;CACtD;AACF;;AAGA,SAAS,aACP,OACA,UACU;CACV,IAAI,UAAU,OAAO,OAAO;CAC5B,MAAM,SAAS;EAAE,GAAG;EAAU,GAAI,SAAS,CAAC;CAAG;CAC/C,OAAO,OAAO,YAAY,QAAQ,OAAO;AAC3C;;AAUA,SAAS,aAAa,QAAiD;CACrE,OAAO;EACL;GAAE,MAAM;GAAW,OAAO;GAAiB,MAAM,eAAe,MAAM;EAAE;EACxE;GAAE,MAAM;GAAY,OAAO;GAAgB,MAAM,YAAY,MAAM;EAAE;EACrE;GAAE,MAAM;GAAY,OAAO;GAAgB,MAAM,aAAa,MAAM;EAAE;EACtE;GAAE,MAAM;GAAY,OAAO;GAAkB,MAAM,aAAa,MAAM;EAAE;CAC1E;AACF;AAEA,SAAS,eAAe,QAA+C;CACrE,MAAM,SAAS,aACb,OAAO,SACP,yBAAyB,OAC3B;CACA,IAAI,CAAC,QAAQ,OAAO;CAEpB,OAAO;EACL,MAAM;EACN,aAAa;EACb,KAAK,OAAO,IAAI,YAAY;GAC1B,MAAM,EAAE,WAAW,MAAM,gBAAgB,IAAI;IAC3C,UACE,OAAO,cAAc,OACjB,KAAA,IACA,OAAO,aAAa;IAC1B,SAAS,OAAO;IAChB,QAAQ,QAAQ;GAClB,CAAC;GACD,OAAO;EACT;CACF;AACF;AAEA,SAAS,YAAY,QAA+C;CAClE,MAAM,SAAS,aACb,OAAO,SACP,yBAAyB,OAC3B;CACA,IAAI,CAAC,QAAQ,OAAO;CAEpB,OAAO;EACL,MAAM;EACN,aAAa;EACb,KAAK,OAAO,IAAI,YAAY;GAC1B,MAAM,EAAE,WAAW,MAAM,aAAa,IAAI;IACxC,UACE,OAAO,cAAc,OACjB,KAAA,IACA,OAAO,aAAa;IAC1B,SAAS,OAAO;IAChB,QAAQ,QAAQ;IAChB,KAAK,QAAQ;GACf,CAAC;GACD,OAAO;EACT;CACF;AACF;AAEA,SAAS,aAAa,QAA+C;CAKnE,IAAI,CAJW,aACb,OAAO,UACP,yBAAyB,QAEtB,GAAQ,OAAO;CAEpB,OAAO;EACL,MAAM;EACN,aAAa;EACb,KAAK,OAAO,IAAI,YAAY;GAC1B,MAAM,EAAE,WAAW,MAAM,qBAAqB,IAAI;IAChD,KAAK,QAAQ;IACb,QAAQ,QAAQ;GAClB,CAAC;GACD,OAAO;EACT;CACF;AACF;AAEA,SAAS,aAAa,QAA+C;CACnE,MAAM,SAAS,aACb,OAAO,UACP,yBAAyB,QAC3B;CACA,IAAI,CAAC,QAAQ,OAAO;CAEpB,OAAO;EACL,MAAM;EACN,aAAa;EACb,KAAK,OAAO,IAAI,YAAY;GAI1B,MAAM,EAAE,uBAAuB,MAAM,OACnC;GAEF,MAAM,SAAS,MAAM,mBAAmB,QAAQ,IAAI;IAClD,wBAAwB,OAAO;IAC/B,qBAAqB,OAAO;IAC5B,QAAQ,QAAQ;GAClB,CAAC;GACD,OAAO;IACL,QAAQ,OAAO,mBAAmB,OAAO;IACzC,SAAS;KACP,WAAW,OAAO;KAClB,QAAQ,OAAO;IACjB;GACF;EACF;CACF;AACF;AAMA,SAAS,aAAa,QAA4C;CAChE,IAAI,MAAM,QAAQ,MAAM,GACtB,OAAO;CAET,IAAI,UAAU,OAAO,WAAW,YAAY,UAAU,QAAQ;EAC5D,MAAM,OAAQ,OAA8B;EAC5C,IAAI,MAAM,QAAQ,IAAI,GACpB,OAAO;CAEX;CACA,OAAO,CAAC;AACV;AAEA,SAAS,eAAe,IAA+B;CACrD,OAAO,GAAG,OAAQ,GAAqC,QAAQ,OAAO;AACxE;;;;;;;AAQA,SAAS,iBAAiB,IAAuB,KAAqB;CAKpE,IAJe,aACb,eAAe,EAAE,GAChB,GAAyB,IAExB,MAAW,YAAY,OAAO;CAElC,IAAI,QAAQ;CACZ,OAAO,IAAI,QAAQ,aAAa,IAAI,EAAE,OAAO;AAC/C;AAEA,SAAS,YAAY,YAA8B;CACjD,OAAO,WAAW,SAAS,IAAI,SAAS,WAAW,KAAK,OAAO,MAAM;AACvE;AAEA,SAAS,kBAAkB,WAGzB;CACA,IAAI,EAAE,cAAc,YAAY,OAAO;EAAE,YAAY,CAAC;EAAG,QAAQ,CAAC;CAAE;CACpE,IAAI,UAAU,aAAa,KAAA,GAAW,OAAO;EAAE,YAAY,CAAC;EAAG,QAAQ,CAAC;CAAE;CAC1E,IAAI,UAAU,aAAa,MACzB,OAAO;EAAE,YAAY,CAAC,mBAAmB;EAAG,QAAQ,CAAC;CAAE;CAEzD,OAAO;EAAE,YAAY,CAAC,eAAe;EAAG,QAAQ,CAAC,UAAU,QAAQ;CAAE;AACvE;AAEA,eAAe,UACb,IACA,KACA,QAC8C;CAI9C,OAHa,aACX,MAAM,GAAG,MAAM,iBAAiB,IAAI,GAAG,GAAG,GAAG,MAAM,CAE9C,CAAA,CAAK;AACd;;;;;;;;;;;;;;AAeA,eAAe,WACb,IACA,OACA,YACA,QACA,QACiB;CACjB,MAAM,QAAQ,YAAY,UAAU;CAMpC,MAAM,QAAQ,eAAc,MALL,UACrB,IACA,iCAAiC,MAAM,GAAG,SAC1C,MACF,EAAA,EACsC,SAAS,GAAG,iBAAiB;CACnE,IAAI,SAAS,GAAG,OAAO;CAEvB,IAAI,CAAC,QACH,MAAM,GAAG,MACP,iBAAiB,IAAI,eAAe,MAAM,GAAG,OAAO,GACpD,GAAG,MACL;CAGF,OAAO;AACT;;AAGA,eAAe,YACb,IACA,OACkB;CAClB,IAAI;EACF,IAAI,OAAO,GAAG,gBAAgB,YAC5B,OAAO,MAAM,GAAG,YAAY,KAAK;CAErC,QAAQ;EACN,OAAO;CACT;CAEA,IAAI;EACF,MAAM,GAAG,MAAM,iBAAiB,MAAM,SAAS;EAC/C,OAAO;CACT,QAAQ;EACN,OAAO;CACT;AACF"}
|