@venturekit/data 0.0.32 → 0.0.34

package/src/sql/vk_data_001_idempotency.sql

@@ -0,0 +1,48 @@
+-- @venturekit/data — idempotency records.
+--
+-- Table created by this migration:
+--   vk_idempotency_record — stored responses for replayed unsafe requests
+--
+-- Backs `createPostgresIdempotencyStore()` (`src/idempotency/postgres.ts`),
+-- which plugs into `@venturekit/runtime`'s `idempotencyMiddleware`. That
+-- middleware has always taken a pluggable store and shipped two — in-memory
+-- (per-process, so wrong for Lambda) and DynamoDB — leaving a Postgres-only
+-- project to stand up a DynamoDB table for four columns.
+--
+-- Unlike `0000_vk_data_foundation.sql` this file keeps the
+-- conventional `vk_data_001_` name: nothing in a consumer's own migration
+-- references the table, so sorting after the project's `0xx_*` files is fine.
+--
+-- Additive for existing projects: one new table, no existing file edited, and
+-- nothing writes to it until a project wires the middleware.
+
+CREATE TABLE IF NOT EXISTS vk_idempotency_record (
+  -- The idempotency key, optionally namespaced by the store's `scope` as
+  -- `<scope>:<key>`. Scoping matters: the middleware's default extractor reads
+  -- a CLIENT-supplied header, so without it two callers sending the same key
+  -- collide and the second is served the first's cached response — across
+  -- tenants, a cross-tenant read. See the `scope` option.
+  key         text        PRIMARY KEY,
+  -- The serialized handler response, replayed verbatim on a duplicate.
+  -- Empty while status is 'pending'.
+  response    text        NOT NULL DEFAULT '',
+  -- 'pending' is written before the handler runs and is what makes a
+  -- concurrent duplicate detectable; 'completed' once the response is stored.
+  -- A CHECK rather than an enum: enums are a migration to extend, and these
+  -- two values are the middleware's whole vocabulary.
+  status      text        NOT NULL CHECK (status IN ('pending', 'completed')),
+  -- Absolute expiry. Reads filter on this, so an expired row is already
+  -- invisible whether or not anything has deleted it.
+  expires_at  timestamptz NOT NULL,
+  created_at  timestamptz NOT NULL DEFAULT now(),
+  updated_at  timestamptz NOT NULL DEFAULT now()
+);
+
+-- Postgres has no TTL, so expired rows are deleted by
+-- `purgeExpiredIdempotencyRecords()` on a schedule. This index is what keeps
+-- that sweep from scanning the table.
+CREATE INDEX IF NOT EXISTS vk_idempotency_record_expires_at_idx
+  ON vk_idempotency_record (expires_at);
+
+COMMENT ON TABLE vk_idempotency_record IS
+  'Replay protection for unsafe requests. Backs @venturekit/data''s createPostgresIdempotencyStore().';

package/src/sql/vk_data_002_outbox.sql

@@ -0,0 +1,99 @@
+-- @venturekit/data — transactional outbox.
+--
+-- Table created by this migration:
+--   vk_outbox — domain events awaiting publication
+--
+-- Backs `createPostgresOutboxStore()` (`src/outbox/postgres.ts`), the storage
+-- half of `@venturekit/runtime/patterns`' outbox relay. `publishEvent()` puts an
+-- event on EventBridge now; this table is how an event becomes part of the
+-- transaction that caused it, so the two can never disagree about whether the
+-- state change happened.
+--
+-- Additive for existing projects: one new table, no existing file edited, and
+-- nothing writes to it until a project calls `appendToOutbox()`.
+
+CREATE TABLE IF NOT EXISTS vk_outbox (
+  -- Supplied by the writer, not defaulted, and expected to be a time-ordered
+  -- uuid v7 (`newEventId()` in the runtime, `vk_uuid_generate_v7()` in SQL).
+  -- This is the primary key of an append-only table AND every consumer's dedupe
+  -- key: a v4 scatters the index it is clustered on as the table grows, and an
+  -- append-only table only grows. No DEFAULT because an id the writer already
+  -- put on the wire must be the id stored here.
+  id              uuid        PRIMARY KEY,
+
+  -- The ordering key. A sequence, so ordering is assigned at insert by the
+  -- database rather than by a clock — two events a millisecond apart on
+  -- different connections still get a total order, which `occurred_at` cannot
+  -- promise. The relay sorts by this and the consumer keeps it as the source
+  -- version.
+  seq             bigserial   NOT NULL UNIQUE,
+
+  type            text        NOT NULL,
+
+  -- Ordering is guaranteed PER AGGREGATE, so these two columns are what the
+  -- relay's blocking is keyed on, not decoration.
+  aggregate_type  text        NOT NULL,
+  aggregate_id    text        NOT NULL,
+
+  -- NULL for platform-global facts that belong to no tenant.
+  tenant_id       uuid,
+
+  payload         jsonb       NOT NULL DEFAULT '{}'::jsonb,
+  -- Payload schema version. The envelope calls it `version`; the column is
+  -- `event_version` because `version` is a word Postgres tooling likes to use.
+  event_version   integer     NOT NULL DEFAULT 1,
+  -- When the business fact occurred, which is not when the row was inserted.
+  occurred_at     timestamptz NOT NULL DEFAULT now(),
+  -- `{ type: 'user' | 'system', id }`. jsonb rather than two columns so the
+  -- envelope round-trips without the adapter reassembling it.
+  actor           jsonb       NOT NULL,
+
+  correlation_id  text,
+  causation_id    text,
+
+  -- Publication state. NULL means pending; the relay sets it AFTER the bus has
+  -- accepted the event, never before — marking first turns a crash into an
+  -- event that no longer exists anywhere.
+  published_at    timestamptz,
+
+  -- Retry accounting. `attempts >= maxAttempts` is the quarantine, and the
+  -- relay's claim excludes both the quarantined row and anything queued behind
+  -- it for the same aggregate.
+  attempts        integer     NOT NULL DEFAULT 0,
+  last_error      text,
+  failed_at       timestamptz,
+
+  -- An operator deciding the event will never publish. Separate from `attempts`
+  -- on purpose: collapsing the two means raising the quarantine bound later
+  -- drags retired events back onto the bus. A discard also RELEASES the
+  -- successors held behind it, which is what makes quarantine a state an
+  -- aggregate can leave.
+  discarded_at    timestamptz,
+  discard_reason  text,
+
+  created_at      timestamptz NOT NULL DEFAULT now()
+);
+
+-- The relay's claim: pending, not discarded, not quarantined, oldest first.
+-- Partial, so it indexes only the rows that are actually claimable — which in a
+-- healthy system is a handful out of however many million have been published.
+CREATE INDEX IF NOT EXISTS vk_outbox_pending_seq_idx
+  ON vk_outbox (seq)
+  WHERE published_at IS NULL AND discarded_at IS NULL;
+
+-- The `blocked` CTE in `claim()`: which aggregates have an abandoned event.
+CREATE INDEX IF NOT EXISTS vk_outbox_blocked_idx
+  ON vk_outbox (aggregate_type, aggregate_id)
+  WHERE published_at IS NULL AND discarded_at IS NULL;
+
+-- "What is stuck, and why" — the operator's query, and the one a dashboard runs.
+CREATE INDEX IF NOT EXISTS vk_outbox_quarantined_idx
+  ON vk_outbox (failed_at DESC)
+  WHERE published_at IS NULL AND discarded_at IS NULL AND attempts > 0;
+
+COMMENT ON TABLE vk_outbox IS
+  'Transactional outbox. Events are appended in the caller''s transaction and relayed by @venturekit/runtime''s relayOnce().';
+COMMENT ON COLUMN vk_outbox.seq IS
+  'Ordering key assigned at insert. The relay guarantees per-aggregate order by this, and never publishes an event while an earlier one for the same aggregate is unpublished.';
+COMMENT ON COLUMN vk_outbox.discarded_at IS
+  'Operator retired this event. Leaves attempts untouched and releases successors of the same aggregate.';

package/src/sql/vk_data_003_jobs.sql

@@ -0,0 +1,114 @@
+-- @venturekit/data — durable background jobs.
+--
+-- Table created by this migration:
+--   vk_job — queued work, in Postgres rather than SQS
+--
+-- Backs `@venturekit/data/jobs`.
+--
+-- # Why this exists next to the SQS `queues` intent rather than instead of it
+--
+-- SQS is the right tool for fire-and-forget fan-out and it stays the default.
+-- What it cannot do is answer a question about work in flight. "Which imports
+-- are still geocoding for this tenant" is a screen in the product, and a queue
+-- has no query surface: messages are invisible until received, and received
+-- messages are invisible to everyone else. Rebuilding that visibility on top of
+-- SQS means a second table tracking what you put in the queue — at which point
+-- the queue is the redundant half, because the table can be claimed from
+-- directly.
+--
+-- So: reach for `queues` when the work is opaque and throughput matters, and for
+-- this when the work is a thing the product talks about — an import, a export, a
+-- recompute someone is waiting on.
+--
+-- Claiming uses FOR UPDATE SKIP LOCKED, which lets several workers share the
+-- table without a lock convoy: each transaction takes rows nobody else holds and
+-- skips the rest rather than queueing behind them.
+--
+-- Additive for existing projects: one new table, no existing file edited, and
+-- nothing writes to it until a project calls `enqueueJob()`.
+
+CREATE TABLE IF NOT EXISTS vk_job (
+  id            uuid        PRIMARY KEY DEFAULT gen_random_uuid(),
+
+  -- NULL for platform-wide work (partition maintenance, a global recompute).
+  -- Deliberately no foreign key: the table a tenant lives in is the consumer's
+  -- (`vk_tenants` only exists if @venturekit-pro/tenancy is installed), and a
+  -- framework-owned FK into an optional package's table would make this
+  -- migration fail for everyone who does not use it.
+  tenant_id     uuid,
+
+  -- Dispatch key. `runJobsOnce()` maps this to a handler, so it is the contract
+  -- between whoever enqueues and whoever runs.
+  kind          text        NOT NULL,
+  payload       jsonb       NOT NULL DEFAULT '{}'::jsonb,
+
+  -- text + CHECK rather than an enum: an enum is a new type in the consumer's
+  -- schema and a migration to extend, and this vocabulary is closed anyway.
+  status        text        NOT NULL DEFAULT 'queued'
+                            CHECK (status IN ('queued', 'running', 'succeeded', 'failed', 'cancelled')),
+
+  -- Scheduling and retry are the SAME mechanism: a failed job is re-queued with
+  -- a later `run_after`, and a job scheduled for tomorrow is just one with a
+  -- distant `run_after`. One column, so there is no way for the two to disagree.
+  run_after     timestamptz NOT NULL DEFAULT now(),
+
+  attempts      integer     NOT NULL DEFAULT 0,
+  max_attempts  integer     NOT NULL DEFAULT 5,
+
+  -- Held while a worker owns the row. A worker that crashes leaves these set
+  -- and the row in 'running' forever, which is what `reclaimStuckJobs()` looks
+  -- for: 'running' plus an old `locked_at` is indistinguishable from a dead
+  -- worker, and treating it as one is the only way the work resumes.
+  locked_at     timestamptz,
+  locked_by     text,
+
+  last_error    text,
+  created_at    timestamptz NOT NULL DEFAULT now(),
+  updated_at    timestamptz NOT NULL DEFAULT now(),
+  finished_at   timestamptz,
+
+  CONSTRAINT vk_job_attempts_within_max CHECK (attempts <= max_attempts),
+  -- At least one, because the claim increments `attempts` before the handler
+  -- runs: with a bound of zero the first claim would violate the check above
+  -- and the job could never be taken at all.
+  CONSTRAINT vk_job_max_attempts_positive CHECK (max_attempts >= 1)
+);
+
+-- The claim: due, runnable, oldest first. Partial, so it indexes only the rows
+-- a worker can take rather than every job ever run.
+CREATE INDEX IF NOT EXISTS vk_job_runnable_idx
+  ON vk_job (run_after)
+  WHERE status = 'queued';
+
+-- The product's per-tenant job list — the query this table exists to make
+-- possible.
+CREATE INDEX IF NOT EXISTS vk_job_tenant_status_idx
+  ON vk_job (tenant_id, status, created_at DESC);
+
+-- Finding work abandoned by a dead worker.
+CREATE INDEX IF NOT EXISTS vk_job_stuck_idx
+  ON vk_job (locked_at)
+  WHERE status = 'running';
+
+-- # On row-level security
+--
+-- `vk_install_tenant_guards()` is deliberately NOT applied here, for two
+-- reasons that are both about the worker rather than the reader:
+--
+--   1. `tenant_id` is nullable, and RLS would make platform-wide rows invisible
+--      to the very process that has to run them;
+--   2. a worker runs outside any request, so there is no acting tenant to scope
+--      to — it processes every tenant's work by design.
+--
+-- Reads that serve a REQUEST must therefore carry the scope themselves, which
+-- `listJobs()` does (`tenant_id = ANY (vk_tenant_scope())` when a scope is
+-- engaged). That is the same predicate the policy would have applied, so a
+-- caller who goes through the helper cannot see another tenant's work; a caller
+-- who writes their own SQL against this table is responsible for the clause.
+
+COMMENT ON TABLE vk_job IS
+  'Durable background work, claimable with FOR UPDATE SKIP LOCKED. In Postgres so that work in flight is queryable; see @venturekit/data/jobs.';
+COMMENT ON COLUMN vk_job.run_after IS
+  'Earliest time this job may run. Retry backoff and future scheduling are the same mechanism.';
+COMMENT ON COLUMN vk_job.locked_at IS
+  'Set while a worker holds the row. Old + status running = a dead worker; reclaimStuckJobs() re-queues it.';

package/src/sql/vk_data_004_file_object.sql

@@ -0,0 +1,112 @@
+-- @venturekit/data — uploaded-object metadata.
+--
+-- Table created by this migration:
+--   vk_file_object — the row that makes an S3 object findable and deletable
+--
+-- Backs `@venturekit/data/files`.
+--
+-- # Why a table when @venturekit/storage already talks to S3
+--
+-- That package is complete on the object: put, get, head, presign, copy, list,
+-- image optimisation. What a bucket cannot tell you is anything a product needs
+-- to know about an upload — which tenant owns it, what it is for, who put it
+-- there, whether the same bytes are already stored, and when it may be deleted.
+-- `ListObjectsV2` plus a key-naming convention is the usual substitute, and it
+-- fails the first time somebody needs "every document for this tenant, newest
+-- first" or a retention rule that differs per purpose.
+--
+-- # The deletion order is the point
+--
+-- Rows are never hard-deleted on the request path. `archived_at` is set, and a
+-- retention sweep removes the object and the row together — but "together" is
+-- not available: there is no transaction spanning S3 and Postgres, so one of the
+-- two failure modes has to be chosen deliberately.
+--
+--   * row first, then object → an orphaned object nobody can account for. Only
+--     discoverable by diffing the entire bucket against this table.
+--   * object first, then row → a row naming a key that 404s. Discoverable by
+--     walking rows, which is cheap, and invisible to users anyway because the
+--     row is already archived.
+--
+-- So the sweep deletes objects first and rows second, and a crash in between
+-- leaves the recoverable state rather than the silent one. `sweepArchivedFiles`
+-- is re-runnable for exactly this reason.
+--
+-- Additive for existing projects: one new table, no existing file edited, and
+-- nothing writes to it until a project calls `recordFileObject()`.
+
+CREATE TABLE IF NOT EXISTS vk_file_object (
+  id            uuid        PRIMARY KEY DEFAULT gen_random_uuid(),
+
+  -- NULL for platform-wide artefacts belonging to no tenant. No foreign key:
+  -- the table a tenant lives in is the consumer's, and `vk_tenants` only exists
+  -- with @venturekit-pro/tenancy installed.
+  tenant_id     uuid,
+
+  -- What the file is FOR, which is what decides its retention and who may read
+  -- it. Free text rather than an enum because the vocabulary is the product's —
+  -- 'student_photo' carries obligations 'report_artefact' does not, and only the
+  -- consumer knows which of theirs is which.
+  purpose       text        NOT NULL,
+
+  -- Stored, not derived: the bucket changes per stage and per region, and a row
+  -- written in one stage must stay resolvable after a restore into another.
+  bucket        text        NOT NULL,
+  object_key    text        NOT NULL,
+
+  content_type  text        NOT NULL,
+  size_bytes    bigint      NOT NULL CHECK (size_bytes >= 0),
+
+  -- SHA-256 hex. Lets a re-upload of identical bytes be recognised instead of
+  -- duplicated, and lets a restore be verified.
+  checksum      char(64),
+
+  -- The name the user's file had. Never the storage key: a key built from user
+  -- input is a path-traversal question and a collision waiting to happen.
+  original_name text,
+  uploaded_by   uuid,
+
+  created_at    timestamptz NOT NULL DEFAULT now(),
+  updated_at    timestamptz NOT NULL DEFAULT now(),
+
+  -- Soft delete. Set on the request path; the retention sweep is what actually
+  -- removes the object and then this row.
+  archived_at   timestamptz,
+
+  -- One row per object. A second row for the same key would mean two records
+  -- claiming one set of bytes, and archiving either would delete the object out
+  -- from under the other.
+  CONSTRAINT vk_file_object_bucket_key_uniq UNIQUE (bucket, object_key)
+);
+
+-- "Every live file for this tenant, of this purpose, newest first" — the query
+-- the table exists for. Partial, so archived rows do not bloat it.
+CREATE INDEX IF NOT EXISTS vk_file_object_tenant_purpose_idx
+  ON vk_file_object (tenant_id, purpose, created_at DESC)
+  WHERE archived_at IS NULL;
+
+-- De-duplication on re-upload: same bytes, same tenant, already stored.
+CREATE INDEX IF NOT EXISTS vk_file_object_checksum_idx
+  ON vk_file_object (tenant_id, checksum)
+  WHERE archived_at IS NULL AND checksum IS NOT NULL;
+
+-- The retention sweep's claim.
+CREATE INDEX IF NOT EXISTS vk_file_object_archived_idx
+  ON vk_file_object (archived_at)
+  WHERE archived_at IS NOT NULL;
+
+-- # On row-level security
+--
+-- No policy is installed here, because `vk_install_tenant_guards()` needs the
+-- name of the consumer's application role and this migration cannot know it. A
+-- project that wants RLS on this table should call the installer itself:
+--
+--   SELECT vk_install_tenant_guards('public', 'vk_file_object', 'app_user');
+--
+-- Until then, reads that serve a request must carry the scope predicate, which
+-- `listFileObjects()` and `findFileObjectByChecksum()` do by default.
+
+COMMENT ON TABLE vk_file_object IS
+  'Metadata for an object in S3. The bytes live in the bucket; this row is what makes them findable, attributable and deletable. See @venturekit/data/files.';
+COMMENT ON COLUMN vk_file_object.archived_at IS
+  'Soft delete. The retention sweep deletes the OBJECT first and then this row, so a crash leaves a recoverable dangling row rather than an unaccountable orphaned object.';