experimental-a2 0.20.0 → 0.22.0

package/examples/playground/app/a2slack/drills/a2slack.drill.ts

@@ -0,0 +1,708 @@
+/**
+ * Drills for a2slack's consumers and projections, against a throwaway Redis
+ * and Postgres: a load run, a rebuild, a poisoned event, a Postgres outage,
+ * two versions of a consumer at once, and the load run again with A2 on
+ * Postgres. Each prints what it measured, and the whole run is written to
+ * .a2/drills/. Sizes come from DRILL_MESSAGES, DRILL_CONCURRENCY,
+ * DRILL_SUBSCRIBERS, and DRILL_SERVERS.
+ *
+ *   pnpm --filter playground drills
+ */
+// oxlint-disable no-await-in-loop, no-console -- drills run steps in order and report as they go
+import { mkdirSync, writeFileSync } from 'node:fs'
+import { sql } from 'drizzle-orm'
+import type { A2Store, Consumer } from 'experimental-a2/server'
+import { afterAll, beforeAll, expect, it } from 'vitest'
+import {
+  idle,
+  recorder,
+  sleep,
+  startServices,
+  summarize,
+  timerScheduler,
+  until,
+  type Run,
+  type Services,
+} from './harness'
+
+const MESSAGES = Number(process.env.DRILL_MESSAGES ?? 10_000)
+const CONCURRENCY = Number(process.env.DRILL_CONCURRENCY ?? 20)
+const SUBSCRIBERS = Number(process.env.DRILL_SUBSCRIBERS ?? 20)
+const SERVERS = Number(process.env.DRILL_SERVERS ?? 4)
+
+const CHANNELS = Array.from({ length: 20 }, (_, i) => `room-${i + 1}`)
+const USERS = Array.from({ length: 50 }, (_, i) => ({
+  id: `user-${i + 1}`,
+  name: `user${i + 1}`,
+}))
+const WORDS =
+  'deploy review ship merge plan draft launch bug fix test build release ' +
+  'sync cache queue index search thread channel message reply mention ' +
+  'design schema table cursor lane log event store redis postgres drizzle'
+const words = WORDS.split(' ')
+
+const pick = <T>(items: readonly T[]): T =>
+  items[Math.floor(Math.random() * items.length)]!
+
+const text = (): string =>
+  Array.from({ length: 4 + Math.floor(Math.random() * 8) }, () =>
+    pick(words),
+  ).join(' ')
+
+let services: Services
+const results: Record<string, unknown> = {}
+
+// Loaded once DATABASE_URL points at the drill's Postgres.
+let a2slack: Awaited<ReturnType<typeof load>>
+
+async function load() {
+  const [
+    { redis },
+    { postgres },
+    { consumer, createServer },
+    { drizzle },
+    { migrate },
+    model,
+    feedModule,
+    { notify },
+    { welcome },
+    queries,
+    { database },
+  ] = await Promise.all([
+    import('experimental-a2/store-redis'),
+    import('experimental-a2/store-postgres'),
+    import('experimental-a2/server'),
+    import('drizzle-orm/node-postgres'),
+    import('drizzle-orm/node-postgres/migrator'),
+    import('../model'),
+    import('../consumers/feed'),
+    import('../consumers/notify'),
+    import('../consumers/welcome'),
+    import('../db/queries'),
+    import('../db'),
+  ])
+  return {
+    redis,
+    postgres,
+    consumer,
+    createServer,
+    drizzle,
+    migrate,
+    model,
+    feed: feedModule.feed,
+    feedEvent: feedModule.feedEvent,
+    notify,
+    welcome,
+    queries,
+    database,
+  }
+}
+
+type Workspace = (typeof a2slack)['model']['workspace']
+
+const servers = (
+  store: A2Store,
+  consumers: Consumer<Workspace['events']>[],
+  telemetry: ReturnType<typeof recorder>['telemetry'],
+  many = SERVERS,
+) =>
+  Array.from({ length: many }, () => {
+    const { scheduler, deliverTo } = timerScheduler()
+    const server = a2slack.createServer({
+      contract: a2slack.model.workspace,
+      store,
+      scheduler,
+      consumers,
+      telemetry,
+    })
+    deliverTo(server)
+    return server
+  })
+
+type WorkspaceServer = ReturnType<typeof servers>[number]
+
+const session = (server: WorkspaceServer) =>
+  server.session(a2slack.model.WORKSPACE)
+
+/** Every table a2slack keeps, emptied, for a run that starts a new log. */
+async function emptyTables(): Promise<void> {
+  const db = await a2slack.database()
+  await db.execute(
+    sql.raw(
+      'truncate a2slack_channels, a2slack_messages, a2slack_notifications, a2slack_reactions, a2slack_reads, a2slack_positions, a2slack_people, a2slack_thread_members',
+    ),
+  )
+}
+
+/** The rows of a raw query; the app's database type leaves them untyped. */
+async function rows<T>(query: string): Promise<T[]> {
+  const db = await a2slack.database()
+  return ((await db.execute(sql.raw(query))) as unknown as { rows: T[] }).rows
+}
+
+async function count(table: string): Promise<number> {
+  const [row] = await rows<{ n: number }>(
+    `select count(*)::int as n from ${table}`,
+  )
+  return row!.n
+}
+
+/** Counts in the log of what the tables should hold. */
+async function logCounts(server: WorkspaceServer) {
+  const events = await session(server).history()
+  const reactions = new Set<string>()
+  for (const event of events) {
+    if (event.type === 'reaction.added')
+      reactions.add(
+        `${event.payload.messageId}|${event.payload.emoji}|${event.payload.by.id}`,
+      )
+    if (event.type === 'reaction.removed')
+      reactions.delete(
+        `${event.payload.messageId}|${event.payload.emoji}|${event.payload.by.id}`,
+      )
+  }
+  return {
+    head: events.at(-1)?.index ?? 0,
+    messages: events.filter((event) => event.type === 'message.posted').length,
+    notifications: events.filter((event) => event.type === 'notification.sent')
+      .length,
+    channels: new Set(
+      events.flatMap((event) =>
+        event.type === 'channel.created' ? [event.payload.id] : [],
+      ),
+    ).size,
+    reactions: reactions.size,
+  }
+}
+
+async function tableCounts() {
+  return {
+    messages: await count('a2slack_messages'),
+    notifications: await count('a2slack_notifications'),
+    channels: await count('a2slack_channels'),
+    reactions: await count('a2slack_reactions'),
+  }
+}
+
+/** Live subscribers to the workspace: how long each event took to reach them. */
+function subscribe(
+  from: WorkspaceServer[],
+  subscribers: number,
+  startAfter: number,
+) {
+  const delays: number[] = []
+  const last = Array.from({ length: subscribers }, () => 0)
+  const streams = last.map((_, s) => {
+    const stream = session(from[s % from.length]!).stream({ startAfter })
+    return stream[Symbol.asyncIterator]()
+  })
+  const done = Promise.all(
+    streams.map(async (stream, s) => {
+      for (;;) {
+        const next = await stream.next()
+        if (next.done) return
+        const event = next.value
+        if (event.type === 'a2.cleared') continue
+        if (event.type === 'message.posted')
+          delays.push(Date.now() - event.createdAt.getTime())
+        last[s] = event.index
+      }
+    }),
+  )
+  return {
+    delays,
+    last,
+    /** Ends every subscription, even one waiting for its next event. */
+    async close() {
+      await Promise.all(streams.map((stream) => stream.return?.()))
+      await done
+    },
+  }
+}
+
+/** Posts `messages` messages and reactions from `concurrency` writers. */
+async function post(
+  to: WorkspaceServer[],
+  messages: number,
+  concurrency: number,
+  {
+    onSent,
+    prefix = '',
+  }: { onSent?: (n: number) => Promise<void> | void; prefix?: string } = {},
+) {
+  const latencies: number[] = []
+  const roots = new Map<string, string[]>()
+  const recent: { id: string; channel: string }[] = []
+  let next = 0
+  const started = performance.now()
+  await Promise.all(
+    Array.from({ length: concurrency }, async () => {
+      for (;;) {
+        const n = next
+        next += 1
+        if (n >= messages) return
+        const target = session(to[n % to.length]!)
+        const author = pick(USERS)
+        const channel = pick(CHANNELS)
+        const t0 = performance.now()
+        if (Math.random() < 0.05 && recent.length > 0) {
+          const message = pick(recent)
+          await target.append({
+            type: 'reaction.added',
+            payload: {
+              channel: message.channel,
+              messageId: message.id,
+              emoji: pick(['👍', '🎉', '👀']),
+              by: author,
+            },
+          })
+        } else {
+          const thread = roots.get(channel) ?? []
+          const threadId =
+            Math.random() < 0.15 && thread.length > 0 ? pick(thread) : undefined
+          const mention = Math.random() < 0.1 ? ` @${pick(USERS).name}` : ''
+          const [posted] = await target.append({
+            type: 'message.posted',
+            payload: {
+              channel,
+              author,
+              text: `${prefix}${text()}${mention}`,
+              ...(threadId ? { threadId } : {}),
+            },
+          })
+          if (!threadId) {
+            thread.push(posted!.id)
+            if (thread.length > 50) thread.shift()
+            roots.set(channel, thread)
+          }
+          recent.push({ id: posted!.id, channel })
+          if (recent.length > 200) recent.shift()
+        }
+        latencies.push(performance.now() - t0)
+        await onSent?.(n)
+      }
+    }),
+  )
+  const endedAt = performance.now()
+  return { latencies, seconds: (endedAt - started) / 1000, endedAt }
+}
+
+/** Completion lag per consumer: append to the end of its successful run. */
+function lagByConsumer(runs: readonly Run[]) {
+  const by = new Map<string, number[]>()
+  const failures = new Map<string, number>()
+  for (const run of runs) {
+    if (run.failed) {
+      failures.set(run.consumer, (failures.get(run.consumer) ?? 0) + 1)
+      continue
+    }
+    const lags = by.get(run.consumer) ?? []
+    lags.push(run.ageMs + run.durationMs)
+    by.set(run.consumer, lags)
+  }
+  return Object.fromEntries(
+    [...by].map(([consumer, lags]) => [
+      consumer,
+      {
+        lagMs: summarize(lags),
+        runMs: summarize(
+          runs
+            .filter((run) => run.consumer === consumer && !run.failed)
+            .map((run) => run.durationMs),
+        ),
+        failures: failures.get(consumer) ?? 0,
+      },
+    ]),
+  )
+}
+
+async function timed(times: number, fn: () => Promise<unknown>) {
+  const durations: number[] = []
+  for (let i = 0; i < times; i += 1) {
+    const t0 = performance.now()
+    await fn()
+    durations.push(performance.now() - t0)
+  }
+  return summarize(durations)
+}
+
+/** A full load run on `store`: post, follow, catch up, check, and read. */
+async function loadRun(store: A2Store, label: string) {
+  const { telemetry, runs } = recorder()
+  const to = servers(
+    store,
+    [a2slack.welcome, a2slack.feed, a2slack.notify],
+    telemetry,
+  )
+  for (const channel of CHANNELS)
+    await session(to[0]!).append({
+      type: 'channel.created',
+      payload: { id: channel, by: pick(USERS) },
+    })
+  await until(
+    () => idle(store, ['welcome', 'a2slack-feed', 'a2slack-notify']),
+    {
+      timeoutMs: 60_000,
+    },
+  )
+  const before = await logCounts(to[0]!)
+  const followers = subscribe(to, SUBSCRIBERS, before.head)
+
+  const posted = await post(to, MESSAGES, CONCURRENCY)
+  const catchUpMs = await until(
+    () => idle(store, ['welcome', 'a2slack-feed', 'a2slack-notify']),
+    { timeoutMs: 10 * 60_000, everyMs: 100 },
+  )
+  const log = await logCounts(to[0]!)
+  const followMs = await until(
+    () => followers.last.every((index) => index >= log.head),
+    { timeoutMs: 60_000 },
+  )
+  await followers.close()
+  const tables = await tableCounts()
+  const db = await a2slack.database()
+  const feedPosition = await a2slack.queries.position(db, 'a2slack-feed')
+
+  const [root] = await rows<{ id: string }>(
+    "select id from a2slack_messages where thread_id is null and channel = 'room-1' order by log_index desc limit 1",
+  )
+  const anyRoot = root?.id
+  const reads = {
+    messagePage: await timed(10, () =>
+      a2slack.queries.messagePage(pick(CHANNELS)),
+    ),
+    threadPage: await timed(10, () =>
+      a2slack.queries.threadPage(anyRoot ?? ''),
+    ),
+    inboxPage: await timed(10, () => a2slack.queries.inboxPage(pick(USERS).id)),
+    search: await timed(10, () => a2slack.queries.searchFeed(pick(words))),
+  }
+
+  const result = {
+    messages: MESSAGES,
+    concurrency: CONCURRENCY,
+    subscribers: SUBSCRIBERS,
+    servers: SERVERS,
+    events: log.head,
+    appendsPerSecond: Math.round(MESSAGES / posted.seconds),
+    appendMs: summarize(posted.latencies),
+    catchUpAfterLastAppendMs: Math.round(catchUpMs),
+    totalSeconds: Math.round(posted.seconds + catchUpMs / 1000),
+    consumers: lagByConsumer(runs),
+    stream: {
+      deliveryMs: summarize(followers.delays),
+      allSubscribersAtHeadMs: Math.round(followMs),
+    },
+    exactlyOnce: {
+      log,
+      tables,
+      feedPosition,
+      ok:
+        tables.messages === log.messages &&
+        tables.notifications === log.notifications &&
+        tables.channels === log.channels &&
+        tables.reactions === log.reactions &&
+        feedPosition === log.head,
+    },
+    readsMs: reads,
+  }
+  results[label] = result
+  console.log(`\n${label}\n${JSON.stringify(result, null, 2)}`)
+  return { result, to, log, tables }
+}
+
+beforeAll(async () => {
+  services = await startServices()
+  process.env.DATABASE_URL = services.databaseUrl
+  a2slack = await load()
+  const db = a2slack.drizzle({ connection: services.databaseUrl })
+  await a2slack.migrate(db, {
+    migrationsFolder: new URL('../db/migrations', import.meta.url).pathname,
+    migrationsTable: 'a2slack_migrations',
+    migrationsSchema: 'public',
+  })
+  await db.$client.end()
+})
+
+afterAll(() => {
+  const dir = new URL('../../../.a2/drills/', import.meta.url).pathname
+  mkdirSync(dir, { recursive: true })
+  const file = `${dir}a2slack-${new Date().toISOString().replace(/[:.]/g, '-')}.json`
+  writeFileSync(file, JSON.stringify(results, null, 2))
+  console.log(`\nresults  ${file}`)
+  services?.close()
+})
+
+let redisStore: A2Store
+
+beforeAll(() => {
+  redisStore = a2slack.redis({ url: services.redisUrl, keyPrefix: 'drill' })
+})
+
+it('load: A2 on Redis, the tables in Postgres', async () => {
+  const { result } = await loadRun(redisStore, 'load (Redis)')
+  expect(result.exactlyOnce.ok).toBe(true)
+})
+
+it('rebuild: a renamed feed fills empty tables from the whole log', async () => {
+  const before = await tableCounts()
+  const db = await a2slack.database()
+  await db.execute(
+    sql.raw(
+      "truncate a2slack_channels, a2slack_messages, a2slack_notifications, a2slack_reactions, a2slack_reads; delete from a2slack_positions where consumer = 'a2slack-feed'",
+    ),
+  )
+  const { telemetry, runs } = recorder()
+  const feedV2 = a2slack
+    .consumer({
+      contract: a2slack.model.workspace,
+      name: 'a2slack-feed-v2',
+      lane: 'feed-v2',
+      backfill: true,
+    })
+    .run(async ({ event }) => {
+      await a2slack.feedEvent(await a2slack.database(), event)
+    })
+  const [server] = servers(redisStore, [feedV2], telemetry, 1)
+  const t0 = performance.now()
+  const consumed = await session(server!).consume(feedV2)
+  const ms = performance.now() - t0
+  const after = await tableCounts()
+  const result = {
+    events: consumed.index,
+    ms: Math.round(ms),
+    eventsPerSecond: Math.round(consumed.index / (ms / 1000)),
+    runMs: summarize(runs.map((run) => run.durationMs)),
+    before,
+    after,
+    ok: JSON.stringify(before) === JSON.stringify(after),
+  }
+  results.rebuild = result
+  console.log(`\nrebuild\n${JSON.stringify(result, null, 2)}`)
+  expect(result.ok).toBe(true)
+})
+
+it('poison: one bad event dead-letters its consumer alone, and redrive resumes it', async () => {
+  const db = await a2slack.database()
+  await db.execute(
+    sql.raw(
+      'create table if not exists drill_seen (event_id text primary key)',
+    ),
+  )
+  let fixed = false
+  const seen = a2slack
+    .consumer({
+      contract: a2slack.model.workspace,
+      name: 'drill-seen',
+      types: ['message.posted'],
+      lane: 'seen',
+    })
+    .run(async ({ event }) => {
+      if (event.payload.text.includes('POISON') && !fixed)
+        throw new Error('poisoned')
+      await (
+        await a2slack.database()
+      ).execute(
+        sql`insert into drill_seen (event_id) values (${event.id}) on conflict do nothing`,
+      )
+    })
+  const { telemetry, runs } = recorder()
+  const to = servers(
+    redisStore,
+    [a2slack.feed, a2slack.notify, seen],
+    telemetry,
+    2,
+  )
+  await session(to[0]!).consume(seen, { through: 0 })
+  // The consumer starts with the first event appended after it exists.
+  const { index: startedAt } = await session(to[0]!).state()
+  await post(to, 50, 5)
+  const [poison] = await session(to[0]!).append({
+    type: 'message.posted',
+    payload: { channel: 'room-1', author: USERS[0]!, text: 'POISON pill' },
+  })
+  await post(to, 50, 5)
+
+  const t0 = performance.now()
+  await until(() => idle(redisStore, ['a2slack-feed', 'a2slack-notify']), {
+    timeoutMs: 60_000,
+  })
+  const othersCaughtUpMs = performance.now() - t0
+  const stuck = (await redisStore.cursors('a2slack\u001fworkspace')).find(
+    (cursor) => cursor.consumer === 'drill-seen',
+  )
+  let probes = 0
+  const deadLetteredMs = await until(
+    async () => {
+      probes += 1
+      if (process.env.DRILL_DEBUG && probes % 10 === 0) {
+        const cursor = (
+          await redisStore.cursors('a2slack\u001fworkspace')
+        ).find((c) => c.consumer === 'drill-seen')
+        console.log(
+          'drill-seen',
+          JSON.stringify({
+            position: cursor?.position,
+            head: cursor?.head,
+            attempts: cursor?.attemptCount,
+            failures: cursor?.failureCount,
+            holder: cursor?.claimHolder,
+            expires: cursor?.claimExpiresAt,
+            now: new Date(),
+          }),
+        )
+      }
+      return (await session(to[0]!).deadLetters()).some(
+        (letter) => letter.consumer === 'drill-seen',
+      )
+    },
+    { timeoutMs: 10 * 60_000, everyMs: 1000 },
+  )
+  const consumeError = await session(to[0]!)
+    .consume(seen)
+    .then(() => null)
+    .catch((error: { code?: string }) => error.code)
+
+  fixed = true
+  const redriven = await session(to[0]!).redrive(
+    { consumer: 'drill-seen', index: poison!.index },
+    { action: 'retry' },
+  )
+  await session(to[0]!).consume(seen)
+  const seenRows = await count('drill_seen')
+  // Every message posted once the consumer existed, the poison included.
+  const posted = (await session(to[0]!).history({ gte: startedAt + 1 })).filter(
+    (event) => event.type === 'message.posted',
+  ).length
+  const result = {
+    othersCaughtUpWhileStuckMs: Math.round(othersCaughtUpMs),
+    stuckAt: {
+      position: stuck?.position,
+      head: stuck?.head,
+      poison: poison!.index,
+    },
+    deadLetteredAfterMs: Math.round(deadLetteredMs + othersCaughtUpMs),
+    failedAttempts: runs.filter(
+      (run) => run.consumer === 'drill-seen' && run.failed,
+    ).length,
+    consumeWhileDeadLettered: consumeError,
+    redriven: redriven.redriven,
+    rowsAfterRedrive: seenRows,
+    messagesPosted: posted,
+    ok:
+      seenRows === posted &&
+      redriven.redriven &&
+      consumeError === 'CONSUMER_DEAD_LETTERED',
+  }
+  results.poison = result
+  console.log(`\npoison\n${JSON.stringify(result, null, 2)}`)
+  expect(result.ok).toBe(true)
+})
+
+it('outage: Postgres goes away mid-load, and every event still lands once', async () => {
+  const { telemetry, runs } = recorder()
+  const to = servers(
+    redisStore,
+    [a2slack.welcome, a2slack.feed, a2slack.notify],
+    telemetry,
+  )
+  let downAt = 0
+  let upAt = 0
+  const outage = (async () => {
+    await sleep(2_000)
+    services.stopPostgres()
+    downAt = performance.now()
+    await sleep(15_000)
+    services.startPostgres()
+    upAt = performance.now()
+  })()
+  // About 100 appends a second for 30s: they go on through the outage and
+  // after it, so drains keep starting.
+  const posted = await post(to, 3_000, 10, {
+    onSent: async () => {
+      await sleep(100)
+    },
+  })
+  await outage
+  await until(
+    () => idle(redisStore, ['welcome', 'a2slack-feed', 'a2slack-notify']),
+    { timeoutMs: 5 * 60_000, everyMs: 200 },
+  )
+  const idleAt = performance.now()
+  const log = await logCounts(to[0]!)
+  const tables = await tableCounts()
+  const db = await a2slack.database()
+  const feedPosition = await a2slack.queries.position(db, 'a2slack-feed')
+  const deadLetters = await session(to[0]!).deadLetters()
+  const result = {
+    appended: 3_000,
+    appendMs: summarize(posted.latencies),
+    postgresDownMs: Math.round(upAt - downAt),
+    appendsEndedAfterRestartMs: Math.round(posted.endedAt - upAt),
+    idleAfterRestartMs: Math.round(idleAt - upAt),
+    failedRuns: runs.filter((run) => run.failed).length,
+    deadLetters: deadLetters.length,
+    exactlyOnce: {
+      log,
+      tables,
+      feedPosition,
+      ok:
+        tables.messages === log.messages &&
+        tables.notifications === log.notifications &&
+        tables.reactions === log.reactions &&
+        feedPosition === log.head,
+    },
+  }
+  results.outage = result
+  console.log(`\noutage\n${JSON.stringify(result, null, 2)}`)
+  expect(result.exactlyOnce.ok).toBe(true)
+  expect(deadLetters).toEqual([])
+})
+
+it('versions: two deployments run different code under one consumer name', async () => {
+  const db = await a2slack.database()
+  await db.execute(
+    sql.raw(
+      'create table if not exists drill_versions (event_id text primary key, version text not null)',
+    ),
+  )
+  const version = (tag: string) =>
+    a2slack
+      .consumer({
+        contract: a2slack.model.workspace,
+        name: 'drill-version',
+        types: ['message.posted'],
+        lane: 'version',
+      })
+      .run(async ({ event }) => {
+        await (
+          await a2slack.database()
+        ).execute(
+          sql`insert into drill_versions (event_id, version) values (${event.id}, ${tag}) on conflict do nothing`,
+        )
+      })
+  const { telemetry } = recorder()
+  const v1 = version('v1')
+  const [oldServer] = servers(redisStore, [v1], telemetry, 1)
+  const [newServer] = servers(redisStore, [version('v2')], telemetry, 1)
+  await session(oldServer!).consume(v1, { through: 0 })
+  await post([oldServer!, newServer!], 200, 10)
+  await until(() => idle(redisStore, ['drill-version']), { timeoutMs: 60_000 })
+  const handled = await rows<{ version: string; n: number }>(
+    'select version, count(*)::int as n from drill_versions group by version order by version',
+  )
+  const result = {
+    events: 200,
+    handledBy: Object.fromEntries(handled.map((row) => [row.version, row.n])),
+  }
+  results.versions = result
+  console.log(`\nversions\n${JSON.stringify(result, null, 2)}`)
+  expect(handled.reduce((sum, row) => sum + row.n, 0)).toBeGreaterThan(0)
+})
+
+it('load: A2 on Postgres, beside its tables', async () => {
+  await emptyTables()
+  const store = a2slack.postgres({ connectionString: services.databaseUrl })
+  const { result } = await loadRun(store, 'load (Postgres)')
+  expect(result.exactlyOnce.ok).toBe(true)
+})