experimental-a2 0.20.0 → 0.22.0

package/examples/playground/app/a2slack/db/index.ts

@@ -0,0 +1,73 @@
+/**
+ * a2slack's database: Postgres at `DATABASE_URL`, or PGlite in development
+ * (in memory under test).
+ *
+ * Production never changes the schema at runtime: apply ./migrations with
+ * `pnpm db:migrate` before deploying. PGlite applies them when it opens, since
+ * a development database is only ever this app's.
+ */
+import 'server-only'
+import { join } from 'node:path'
+import type { PgDatabase, PgQueryResultHKT } from 'drizzle-orm/pg-core'
+import * as schema from './schema'
+
+export type Db = PgDatabase<PgQueryResultHKT, typeof schema>
+
+const url =
+  process.env.DATABASE_URL?.replace(
+    /sslmode=(require|prefer|verify-ca)/,
+    'sslmode=verify-full',
+  ) || undefined
+
+const migrations = {
+  migrationsFolder: join(process.cwd(), 'app/a2slack/db/migrations'),
+  migrationsTable: 'a2slack_migrations',
+  migrationsSchema: 'public',
+}
+
+/** A PGlite database at `dataDir`, migrated. */
+export async function openPglite(dataDir: string): Promise<Db> {
+  const [{ PGlite }, { pg_trgm }, { drizzle }, { migrate }] = await Promise.all(
+    [
+      import('@electric-sql/pglite'),
+      import('@electric-sql/pglite/contrib/pg_trgm'),
+      import('drizzle-orm/pglite'),
+      import('drizzle-orm/pglite/migrator'),
+    ],
+  )
+  const client = new PGlite(dataDir, { extensions: { pg_trgm } })
+  const db = drizzle({ client, schema, casing: 'snake_case' })
+  await migrate(db, migrations)
+  return db
+}
+
+async function open(): Promise<Db> {
+  if (url !== undefined) {
+    const { drizzle } = await import('drizzle-orm/node-postgres')
+    const db = drizzle({ connection: url, schema, casing: 'snake_case' })
+    // The server closing a connection (a restart, Neon scaling to zero) is
+    // reported on the pool when the connection is idle, and on the connection
+    // when a transaction holds it. Without a listener either one is an
+    // uncaught exception, which ends the process. The query it cut off fails
+    // either way, and its consumer retries.
+    db.$client.on('error', () => {})
+    db.$client.on('connect', (connection) => connection.on('error', () => {}))
+    return db
+  }
+  // A serverless filesystem keeps nothing, so a deployment needs Postgres.
+  if (process.env.VERCEL) throw new Error('a2slack needs DATABASE_URL')
+  return openPglite(
+    process.env.NODE_ENV === 'test' ? 'memory://' : '.a2/a2slack',
+  )
+}
+
+const shared = globalThis as { a2slackDb?: Promise<Db> }
+
+/** The database, opened once per process. */
+export function database(): Promise<Db> {
+  shared.a2slackDb ??= open()
+  shared.a2slackDb.catch(() => {
+    delete shared.a2slackDb
+  })
+  return shared.a2slackDb
+}

package/examples/playground/app/a2slack/db/migrations/0000_extensions.sql

@@ -0,0 +1,4 @@
+-- Message search falls back to trigram matching when there's no Upstash Redis
+-- Search (a2slack_messages_text_trgm, in 0001). drizzle-kit can't declare an
+-- extension, so this migration is written by hand.
+CREATE EXTENSION IF NOT EXISTS pg_trgm;

package/examples/playground/app/a2slack/db/migrations/0001_workspace.sql

@@ -0,0 +1,74 @@
+CREATE TABLE "a2slack_channels" (
+	"id" text PRIMARY KEY NOT NULL,
+	"created_by_name" text NOT NULL,
+	"created_at" timestamp with time zone NOT NULL,
+	"log_index" bigint NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "a2slack_messages" (
+	"id" text PRIMARY KEY NOT NULL,
+	"channel" text NOT NULL,
+	"log_index" bigint NOT NULL,
+	"thread_id" text,
+	"author_id" text NOT NULL,
+	"author_name" text NOT NULL,
+	"text" text NOT NULL,
+	"posted_at" timestamp with time zone NOT NULL,
+	"edited_at" timestamp with time zone,
+	"deleted" boolean DEFAULT false NOT NULL,
+	"label" text
+);
+--> statement-breakpoint
+CREATE TABLE "a2slack_notifications" (
+	"id" text PRIMARY KEY NOT NULL,
+	"user_id" text NOT NULL,
+	"thread_id" text NOT NULL,
+	"channel" text NOT NULL,
+	"reason" text NOT NULL,
+	"message_id" text NOT NULL,
+	"from_id" text NOT NULL,
+	"from_name" text NOT NULL,
+	"text" text NOT NULL,
+	"log_index" bigint NOT NULL,
+	"sent_at" timestamp with time zone NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "a2slack_people" (
+	"id" text PRIMARY KEY NOT NULL,
+	"name" text NOT NULL,
+	"log_index" bigint NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "a2slack_positions" (
+	"consumer" text PRIMARY KEY NOT NULL,
+	"log_index" bigint NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "a2slack_reactions" (
+	"message_id" text NOT NULL,
+	"emoji" text NOT NULL,
+	"author_id" text NOT NULL,
+	"author_name" text NOT NULL,
+	"log_index" bigint NOT NULL,
+	CONSTRAINT "a2slack_reactions_message_id_emoji_author_id_pk" PRIMARY KEY("message_id","emoji","author_id")
+);
+--> statement-breakpoint
+CREATE TABLE "a2slack_reads" (
+	"user_id" text NOT NULL,
+	"thread_id" text NOT NULL,
+	"through" bigint NOT NULL,
+	CONSTRAINT "a2slack_reads_user_id_thread_id_pk" PRIMARY KEY("user_id","thread_id")
+);
+--> statement-breakpoint
+CREATE TABLE "a2slack_thread_members" (
+	"thread_id" text NOT NULL,
+	"person_id" text NOT NULL,
+	"log_index" bigint NOT NULL,
+	CONSTRAINT "a2slack_thread_members_thread_id_person_id_pk" PRIMARY KEY("thread_id","person_id")
+);
+--> statement-breakpoint
+CREATE INDEX "a2slack_messages_feed" ON "a2slack_messages" USING btree ("channel","thread_id","log_index");--> statement-breakpoint
+CREATE INDEX "a2slack_messages_thread" ON "a2slack_messages" USING btree ("thread_id","log_index");--> statement-breakpoint
+CREATE INDEX "a2slack_messages_text_trgm" ON "a2slack_messages" USING gin ("text" gin_trgm_ops);--> statement-breakpoint
+CREATE INDEX "a2slack_notifications_inbox" ON "a2slack_notifications" USING btree ("user_id","thread_id","log_index");--> statement-breakpoint
+CREATE INDEX "a2slack_people_name" ON "a2slack_people" USING btree ("name");

package/examples/playground/app/a2slack/db/migrations/0002_a2.sql

@@ -0,0 +1,215 @@
+-- A2's tables: experimental-a2's sql/postgres/schema.sql, in place of the
+-- CREATE TABLEs drizzle-kit generated from db/a2.ts, so this migration also
+-- upgrades a database that already has A2's tables (from an earlier release
+-- or a schema.sql applied by hand), keeping their events.
+--
+-- A2's tables for experimental-a2/store-postgres. drizzle-schema.ts, next to
+-- this file, shows the same tables as Drizzle definitions, as an example.
+--
+-- Apply this file before the first deploy, and again when a release's
+-- changelog says the tables changed. Every statement creates only what is
+-- missing and leaves existing data alone, so the file is safe to re-run on any
+-- A2 database, whichever release created it. A2 checks nothing up front: a
+-- query that finds a table or column missing throws STORE_NOT_CONFIGURED.
+--
+-- On a database from a release that ran handlers, it drops the work those
+-- handlers hadn't finished, and the columns they dispatched by, so drain
+-- earlier deployments and apply it while none is running.
+--
+-- Each index sits behind a catalog check: `create index if not exists` takes
+-- a share lock on its table, and requires owning it, even when the index
+-- exists. The check makes a re-run on a live database wait for no writer.
+--
+-- The file works as a migration as it ships: copy it into your migration
+-- tool's folder. The breakpoint comment after each statement splits it for
+-- tools that send one statement per query, like Drizzle on PGlite; to every
+-- other tool, psql included, it's only a comment.
+
+create table if not exists a2_events (
+  session_id   text    not null,
+  idx          bigint  not null,
+  event_type   text    not null,
+  payload      jsonb   not null,
+  event_id     text    not null,
+  created_at   bigint  not null,
+  cause        jsonb,
+  primary key (session_id, idx)
+);
+--> statement-breakpoint
+
+-- One row per run of a cursor: an event cursor's one event, a lane's run of
+-- queued events, or a log cursor's position. A run is keyed by the head it
+-- started at (`epoch`), so a clear starts new runs instead of rewriting old
+-- ones.
+create table if not exists a2_cursors (
+  session_id   text    not null,
+  consumer     text    not null,
+  cursor_key   text    not null,
+  epoch        bigint  not null,
+  lane         text,
+  kind         text    not null,
+  position     bigint  not null,
+  head         bigint,
+  types        jsonb,
+  attempt_count integer not null default 0,
+  first_claimed_at bigint,
+  last_claimed_at bigint,
+  claim_holder text,
+  claim_expires_at bigint,
+  failure_count integer not null default 0,
+  last_failed_at bigint,
+  last_failed_attempt integer,
+  last_error   text,
+  failed_at    bigint,
+  completed_idx bigint,
+  completed_attempt integer,
+  returned_event_ids jsonb,
+  primary key (session_id, cursor_key, consumer, epoch)
+);
+--> statement-breakpoint
+
+-- A lane cursor's queue: the events appended for the lane, in log order.
+create table if not exists a2_lane_events (
+  session_id   text    not null,
+  consumer     text    not null,
+  lane         text    not null,
+  idx          bigint  not null,
+  done         boolean not null default false,
+  completed_attempt integer,
+  returned_event_ids jsonb,
+  primary key (session_id, consumer, lane, idx)
+);
+--> statement-breakpoint
+
+create table if not exists a2_snapshots (
+  session_id   text   not null,
+  reducer_name text   not null,
+  up_to_index  bigint not null,
+  state        jsonb  not null,
+  updated_at   bigint not null,
+  primary key (session_id, reducer_name)
+);
+--> statement-breakpoint
+
+create table if not exists a2_snapshot_history (
+  session_id   text   not null,
+  reducer_name text   not null,
+  up_to_index  bigint not null,
+  state        jsonb  not null,
+  updated_at   bigint not null,
+  primary key (session_id, reducer_name, up_to_index)
+);
+--> statement-breakpoint
+
+-- Earlier releases pinned snapshots per event; this one pins them per
+-- cursor head. Pins only hold history for work in flight, so old ones go.
+do $$ begin
+  if to_regclass('a2_snapshot_pins') is not null and not exists (
+    select 1 from pg_attribute
+     where attrelid = to_regclass('a2_snapshot_pins')
+       and attname = 'cursor_key' and not attisdropped
+  ) then
+    drop table a2_snapshot_pins;
+  end if;
+end $$;
+--> statement-breakpoint
+
+create table if not exists a2_snapshot_pins (
+  session_id   text   not null,
+  event_index  bigint not null,
+  consumer     text   not null,
+  cursor_key   text   not null,
+  reducer_name text   not null,
+  up_to_index  bigint not null,
+  primary key (session_id, event_index, consumer, cursor_key, reducer_name, up_to_index)
+);
+--> statement-breakpoint
+
+create table if not exists a2_clears (
+  session_id  text   primary key,
+  clear_index bigint not null
+);
+--> statement-breakpoint
+
+create table if not exists a2_presence (
+  ns          text   not null,
+  participant text   not null,
+  field       text   not null,
+  value       jsonb  not null,
+  seen        bigint not null,
+  at          bigint not null,
+  expires_at  bigint not null,
+  primary key (ns, participant, field)
+);
+--> statement-breakpoint
+
+-- From an earlier release: work its handlers hadn't finished is dropped,
+-- since this release runs no handlers to finish it. Every event is kept.
+-- Causes gain the dispatcher that appended them, the handlers table
+-- (`a2.handlers`), and its lane. Then the old dispatch columns go, and
+-- their indexes with them. The block locks the events first, so an earlier
+-- release still writing waits and then fails, rather than committing between
+-- its statements, and it commits whole, or not at all.
+do $$ begin
+  if exists (
+    select 1 from pg_attribute
+     where attrelid = to_regclass('a2_events')
+       and attname = 'processed_at' and not attisdropped
+  ) then
+    lock table a2_events in access exclusive mode;
+
+    update a2_events child
+       set cause = jsonb_build_object(
+             'consumer', 'a2.handlers',
+             'lane', (select parent.lane from a2_events parent
+                       where parent.session_id = child.session_id
+                         and parent.idx = (child.cause ->> 'index')::bigint)
+           ) || child.cause
+     where child.cause is not null and not child.cause ? 'consumer';
+
+    alter table a2_events
+      drop column lane,
+      drop column lane_ready,
+      drop column processed_at,
+      drop column processed_by_attempt,
+      drop column returned_event_ids,
+      drop column first_claimed_at,
+      drop column last_claimed_at,
+      drop column attempt_count,
+      drop column failure_count,
+      drop column last_failed_at,
+      drop column last_failed_attempt,
+      drop column last_error,
+      drop column failed_at,
+      drop column claim_holder,
+      drop column claim_expires_at;
+  end if;
+end $$;
+--> statement-breakpoint
+
+-- Earlier releases recorded a schema version here and checked it before
+-- their first query. This one checks nothing up front.
+drop table if exists a2_schema;
+--> statement-breakpoint
+
+do $$ begin
+  if to_regclass('a2_events_event_id') is null then
+    create unique index if not exists a2_events_event_id on a2_events (event_id);
+  end if;
+  if to_regclass('a2_events_types') is null then
+    create index if not exists a2_events_types
+      on a2_events (session_id, event_type, idx);
+  end if;
+  if to_regclass('a2_cursors_heads') is null then
+    create index if not exists a2_cursors_heads
+      on a2_cursors (session_id, head) where head is not null;
+  end if;
+  if to_regclass('a2_cursors_lanes') is null then
+    create index if not exists a2_cursors_lanes
+      on a2_cursors (session_id) where kind = 'lane';
+  end if;
+  if to_regclass('a2_snapshot_pins_checkpoint') is null then
+    create index if not exists a2_snapshot_pins_checkpoint
+      on a2_snapshot_pins (session_id, reducer_name, up_to_index);
+  end if;
+end $$;

package/examples/playground/app/a2slack/db/migrations/0003_a2_batch.sql

@@ -0,0 +1,67 @@
+-- The a2_batch function from experimental-a2's sql/postgres/schema.sql, which
+-- Drizzle can't declare: A2 runs its writes through it.
+
+-- Runs an operation's statements in one round trip: A2 sends each write that
+-- takes a session's lock as one call, the lock and then the statements that
+-- need it, so it holds the lock only while the server runs them. Each
+-- statement takes its snapshot when it starts, so those after the lock see
+-- what its last holder committed; a single statement can't, since its
+-- snapshot predates the lock. Each statement is a select of jsonb rows: a row
+-- with an `abort` key fails the call and rolls it back, and one with a `stop`
+-- key ends it. It needs read committed, the default: under a stricter level
+-- every statement would share the first one's snapshot, so it refuses to run.
+-- Created only if missing, as the indexes are: a later release that changes it
+-- adds a function with a new name.
+do $$ declare grantee text; begin
+  if to_regprocedure('a2_batch(text[], jsonb)') is null then
+    create function a2_batch(statements text[], params jsonb)
+    returns setof jsonb
+    language plpgsql
+    volatile
+    as $batch$
+    declare
+      statement text;
+      result jsonb;
+    begin
+      if current_setting('transaction_isolation') <> 'read committed' then
+        raise exception using errcode = 'A2002',
+          message = 'a2_batch needs the read committed isolation level';
+      end if;
+      foreach statement in array statements loop
+        for result in execute statement using params loop
+          if result ? 'abort' then
+            raise exception using errcode = 'A2001',
+              message = 'a2_batch aborted', detail = result::text;
+          end if;
+          return next result;
+          if result ? 'stop' then
+            return;
+          end if;
+        end loop;
+      end loop;
+    end
+    $batch$;
+    -- Default privileges can grant a new function to roles besides PUBLIC, as
+    -- platforms that serve functions over an API grant theirs: it starts with
+    -- none but its owner's. Grants made after it exist are left alone.
+    for grantee in
+      select acl.grantee::regrole::text
+        from pg_proc proc, aclexplode(proc.proacl) acl
+       where proc.oid = to_regprocedure('a2_batch(text[], jsonb)')
+         and acl.grantee not in (0, proc.proowner)
+    loop
+      execute format('revoke execute on function a2_batch(text[], jsonb) from %s', grantee);
+    end loop;
+  end if;
+  -- It runs any statement it's handed, as its caller: only its owner, the role
+  -- that applies this file, may call it, and not every role, as Postgres lets
+  -- by default. Platforms that serve functions over an API would serve it too.
+  if pg_has_role(
+    current_user,
+    (select proowner from pg_proc
+      where oid = to_regprocedure('a2_batch(text[], jsonb)')),
+    'usage'
+  ) then
+    revoke execute on function a2_batch(text[], jsonb) from public;
+  end if;
+end $$;

package/examples/playground/app/a2slack/db/migrations/0004_a2_session_event_ids.sql

@@ -0,0 +1,2 @@
+CREATE UNIQUE INDEX "a2_events_session_event_id" ON "a2_events" USING btree ("session_id","event_id");--> statement-breakpoint
+DROP INDEX "a2_events_event_id";

package/examples/playground/app/a2slack/db/migrations/meta/0000_snapshot.json

@@ -0,0 +1,18 @@
+{
+  "id": "3375264b-f603-4311-bef0-6f2a628dbdce",
+  "prevId": "00000000-0000-0000-0000-000000000000",
+  "version": "7",
+  "dialect": "postgresql",
+  "tables": {},
+  "enums": {},
+  "schemas": {},
+  "views": {},
+  "sequences": {},
+  "roles": {},
+  "policies": {},
+  "_meta": {
+    "columns": {},
+    "schemas": {},
+    "tables": {}
+  }
+}