@weft-js/start 0.0.0-stage → 0.1.1

package/src/changes-sql.mjs

@@ -0,0 +1,99 @@
+// The change feed's triggers, as SQL (`weft db live`, and tests). A trigger says which table and which primary keys
+// changed, never a column of the row: a notification can be read by anyone who can LISTEN, and the hub only needs tags.
+//
+//   postgresql  statement triggers with transition tables, so 10k rows in one statement are one NOTIFY: up to 32 ids, or
+//               none past that, which means "this whole table". NOTIFY is in the transaction: a rollback sends nothing.
+//   sqlite      no notify across processes: rows in a capped log, which the watcher reads when `data_version` moves.
+//   mysql       the same log, read on a timer. A MySQL trigger is per row and can't reach out.
+
+export const CHANNEL = 'weft_changes'
+export const LOG = '__weft_changes'
+/** More ids than this in one statement is a table-level change. */
+export const MAX_IDS = 32
+/** Rows SQLite's log keeps. A reader further behind than this has lost its place, and pokes every table. */
+export const LOG_ROWS = 4096
+export const BREAK = '\n--> statement-breakpoint\n'
+
+const pgName = n => `"${n.replaceAll('"', '""')}"`
+const myName = n => `\`${n.replaceAll('`', '``')}\``
+const literal = s => `'${s.replaceAll("'", "''")}'`
+const safe = t => t.replace(/\W/g, '_')
+
+function postgres(tables) {
+  const fn = `CREATE OR REPLACE FUNCTION weft_changes_notify() RETURNS trigger AS $weft$
+DECLARE
+  ids text[];
+  body text;
+BEGIN
+  IF TG_OP <> 'TRUNCATE' AND TG_NARGS > 0 THEN
+    EXECUTE format('SELECT array_agg(id) FROM (SELECT concat_ws(%L, %s) AS id FROM weft_rows LIMIT ${MAX_IDS + 1}) s', ':', (
+      SELECT string_agg(format('%I', a), ', ') FROM unnest(TG_ARGV) a
+    )) INTO ids;
+    IF ids IS NULL THEN RETURN NULL; END IF;
+    IF cardinality(ids) <= ${MAX_IDS} THEN
+      body := json_build_object('t', TG_TABLE_NAME, 'i', ids)::text;
+    END IF;
+  END IF;
+  IF body IS NULL OR octet_length(body) > 7900 THEN
+    body := json_build_object('t', TG_TABLE_NAME)::text;
+  END IF;
+  PERFORM pg_notify(${literal(CHANNEL)}, body);
+  RETURN NULL;
+END;
+$weft$ LANGUAGE plpgsql`
+  const list = tables.flatMap(({ name, pk }) => {
+    const args = pk.map(literal).join(', ')
+    const on = pgName(name)
+    const trigger = (suffix, event, rows) => [
+      `DROP TRIGGER IF EXISTS weft_changes_${suffix} ON ${on}`,
+      `CREATE TRIGGER weft_changes_${suffix} AFTER ${event} ON ${on} ${rows ? `REFERENCING ${rows} TABLE AS weft_rows ` : ''}FOR EACH STATEMENT EXECUTE FUNCTION weft_changes_notify(${args})`,
+    ]
+    return [...trigger('ins', 'INSERT', 'NEW'), ...trigger('upd', 'UPDATE', 'NEW'), ...trigger('del', 'DELETE', 'OLD'), ...trigger('trn', 'TRUNCATE')]
+  })
+  return [fn, ...list]
+}
+
+function sqlite(tables) {
+  const log = `CREATE TABLE IF NOT EXISTS ${LOG} (seq INTEGER PRIMARY KEY AUTOINCREMENT, tbl TEXT NOT NULL, pk TEXT)`
+  const list = tables.flatMap(({ name, pk }) => {
+    const id = row => (pk.length ? pk.map(c => `${row}.${pgName(c)}`).join(` || ':' || `) : 'NULL')
+    const trigger = (suffix, event, row) => [
+      `DROP TRIGGER IF EXISTS weft_changes_${safe(name)}_${suffix}`,
+      `CREATE TRIGGER weft_changes_${safe(name)}_${suffix} AFTER ${event} ON ${pgName(name)} BEGIN
+  INSERT INTO ${LOG} (tbl, pk) VALUES (${literal(name)}, ${id(row)});
+  DELETE FROM ${LOG} WHERE seq <= last_insert_rowid() - ${LOG_ROWS};
+END`,
+    ]
+    return [...trigger('ins', 'INSERT', 'NEW'), ...trigger('upd', 'UPDATE', 'NEW'), ...trigger('del', 'DELETE', 'OLD')]
+  })
+  return [log, ...list]
+}
+
+function mysql(tables) {
+  const log = `CREATE TABLE IF NOT EXISTS ${LOG} (seq BIGINT AUTO_INCREMENT PRIMARY KEY, tbl VARCHAR(64) NOT NULL, pk VARCHAR(255), at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, KEY at_idx (at))`
+  const list = tables.flatMap(({ name, pk }) => {
+    const id = row => (pk.length ? `CONCAT_WS(':', ${pk.map(c => `${row}.${myName(c)}`).join(', ')})` : 'NULL')
+    const trigger = (suffix, event, row) => [
+      `DROP TRIGGER IF EXISTS weft_changes_${safe(name)}_${suffix}`,
+      `CREATE TRIGGER weft_changes_${safe(name)}_${suffix} AFTER ${event} ON ${myName(name)} FOR EACH ROW INSERT INTO ${LOG} (tbl, pk) VALUES (${literal(name)}, ${id(row)})`,
+    ]
+    return [...trigger('ins', 'INSERT', 'NEW'), ...trigger('upd', 'UPDATE', 'NEW'), ...trigger('del', 'DELETE', 'OLD')]
+  })
+  return [log, ...list]
+}
+
+/**
+ * The statements that make `tables` report their changes. Each can run twice.
+ * @param {'postgresql' | 'sqlite' | 'mysql'} dialect
+ * @param {{ name: string, pk: string[] }[]} tables
+ * @returns {string[]}
+ */
+export function changeStatements(dialect, tables) {
+  if (dialect === 'postgresql') return postgres(tables)
+  if (dialect === 'sqlite') return sqlite(tables)
+  if (dialect === 'mysql') return mysql(tables)
+  throw new Error(`no change triggers for ${String(dialect)}`)
+}
+
+/** A migration file, in drizzle-kit's format. */
+export const changeSql = (dialect, tables) => `-- weft: report changes to ${tables.map(t => t.name).join(', ')} (\`weft db live\`). Ids only, never a row.\n${changeStatements(dialect, tables).join(`;${BREAK}`)};\n`

package/src/changes.ts

@@ -0,0 +1,253 @@
+// Writes from outside the app reach `.live` queries. A database tells this module which table and ids changed; it turns
+// those into tags and the hub pokes whoever subscribed to them. Clients refetch through their own `query`, so `auth`
+// runs per user and no row travels with a poke: a user who can't see a row gets nothing new.
+//
+//   hold     A change waits `hold` ms. A mutation's poke covers what committed before it, so an app write needs no
+//            second poke: before poking, the mutation sends a barrier down the same feed, which arrives after every
+//            change committed earlier. Whatever a barrier's tags cover is dropped. No clock is involved, and a change
+//            the barrier doesn't cover (cron, psql, a job with no `invalidates`) is delivered. Too slow a barrier costs
+//            one extra refetch, never a missed one.
+//   local    Every instance listens and pokes only its own connections: each client gets one poke with no leader to
+//            elect. `listener: 'external'` is the other way round: one process listens, and publishes to the bus.
+//   rate     After a flush the next waits `throttle` ms, so a writer in a loop can't drive refetches at its own pace.
+//
+// `q.live()` shows the mode (`liveMode`) and `degraded` or `reconnecting`, because a page that merely looks live is worse
+// than one that says it isn't. Sources are in db/changes-*.ts; each probes what it can do before it says so.
+
+import { StartError } from './data'
+import type { DbHandle } from './db/core'
+
+export type { LiveMode } from './data'
+
+/** What changed: never what it changed to. No `id`: the whole table. */
+export interface Change { table: string; id?: string }
+
+/** What a source learned about this database, by trying it. `feed` sees every change; `poll` only estimates. */
+export interface Probe {
+  mode: 'feed' | 'poll' | 'off'
+  /** `postgres-notify`, `sqlite-data-version`, `mysql-log`, `mongo-change-stream`, `postgres-stats`… */
+  via: string
+  /** Why this and not something better. */
+  why?: string
+  fix?: string
+}
+
+/** Where a source says what changed. */
+export interface Sink {
+  changes(list: Change[]): void
+  /** A barrier arrived: what came before it and these tags cover is on its way as a poke. */
+  settled(tags: string[]): void
+  /** Changes may have gone unseen (a reconnect): every table is poked. */
+  lost(): void
+  down(why: string): void
+  up(): void
+}
+
+/** A change feed: what reports writes to the database, whoever made them. */
+export interface Source {
+  readonly name: string
+  readonly tables: string[]
+  /** `listener: 'external'`: a separate process listens, and this app only reads the bus. */
+  readonly external?: boolean | undefined
+  /** Dev: the tags of one change. Default `[table, `${table}:${id}`]`. */
+  readonly tagged?: ((change: Change) => string[]) | undefined
+  /** Ms a change waits for its barrier, when a feed delivers in slower batches than the default 25. */
+  readonly hold?: number
+  /** The migration's input (`weft db live`). */
+  defs?(): Promise<TableDef[]>
+  readonly dialect?: string
+  probe(): Promise<Probe>
+  /** Starts from `probe`'s answer; resolves listening, with the way to stop. */
+  start(sink: Sink, probe: Probe): Promise<() => unknown>
+  /** Send a barrier through the feed. Omitted: only the hold window dedupes. */
+  settle?(tags: string[]): Promise<void>
+}
+
+/** A table as `Source.defs` reports it. */
+export interface TableDef { name: string; pk: string[] }
+
+/** `dbChanges`: how to listen, and how a change turns into tags. */
+export interface ChangeOptions {
+  /** Direct `postgres://…` for the listener, when the app's URL goes through a pooler. */
+  listenUrl?: string
+  /** `'external'`: a separately deployed listener publishes to the bus (`listen` from `@weft-js/start/listen`). */
+  listener?: 'external'
+  /** Map a change to tags your queries use. Default `[table, `${table}:${id}`]`. */
+  tagged?: (change: Change) => string[]
+  /** Ms between looks where the database can't push: SQLite's `data_version` (100), MySQL's log (500), Postgres's stats (2000). */
+  interval?: number
+}
+
+const NAME = Symbol.for('drizzle:Name')
+const nameOf = (t: unknown): string => (typeof t === 'string' ? t : (t as Record<symbol, string> | null)?.[NAME] ?? (() => { throw new StartError('R017', `\`dbChanges\` got something that isn't a table: ${String(t)}`, 'pass Drizzle tables, or their names') })())
+
+const CORE = { sqlite: 'sqlite-core', postgresql: 'pg-core', mysql: 'mysql-core' }
+
+/** A table's primary key columns, from its Drizzle definition. */
+interface Key { name: string; primary?: boolean }
+
+async function pkOf(table: unknown, dialect: DbHandle['dialect']): Promise<string[]> {
+  const { getTableConfig } = await import(/* @vite-ignore */ `drizzle-orm/${CORE[dialect]}`)
+  const { columns, primaryKeys } = getTableConfig(table)
+  const inline = columns.filter((c: Key) => c.primary).map((c: Key) => c.name)
+  return inline.length ? inline : (primaryKeys[0]?.columns ?? []).map((c: Key) => c.name)
+}
+
+/** `realtime({ changes: dbChanges(db, [todos, notes]) })`: these tables' writes reach `.live`, whoever made them. */
+export function dbChanges(db: { $weft?: DbHandle }, tables: unknown[], options: ChangeOptions = {}): Source {
+  const handle = db?.$weft
+  if (!handle?.changes) throw new StartError('R017', '`dbChanges` needs a client from `@weft-js/start/db/sqlite`, `/postgres` or `/mysql`', 'for MongoDB use `mongoChanges` from `@weft-js/db/mongo`; for another client, write a `Source`')
+  const names = tables.map(nameOf)
+  let source: Promise<Omit<Source, 'name' | 'tables'>> | undefined
+  const load = () => (source ??= handle.changes!(names, options))
+  return {
+    name: handle.dialect,
+    dialect: handle.dialect,
+    tables: names,
+    external: options.listener === 'external',
+    tagged: options.tagged,
+    defs: () => Promise.all(tables.map(async t => ({ name: nameOf(t), pk: typeof t === 'string' ? [] : await pkOf(t, handle.dialect) }))),
+    probe: () => load().then(s => s.probe()),
+    start: (sink, probe) => load().then(s => s.start(sink, probe)),
+    settle: tags => load().then(s => s.settle?.(tags)),
+  }
+}
+
+/** Set by a platform, read at run time: this process lives for one request. */
+export function functionHost(env: Record<string, string | undefined> = (globalThis as unknown as { process?: { env: Record<string, string | undefined> } }).process?.env ?? {}): string | undefined {
+  if (env.VERCEL) return 'Vercel'
+  if (env.NETLIFY || env.NETLIFY_LOCAL) return 'Netlify'
+  if (env.AWS_LAMBDA_FUNCTION_NAME) return 'AWS Lambda'
+  if (env.AZURE_FUNCTIONS_ENVIRONMENT) return 'Azure Functions'
+  if (env.FUNCTION_TARGET) return 'Cloud Functions'
+  if (globalThis.navigator?.userAgent === 'Cloudflare-Workers') return 'Cloudflare Workers'
+  return undefined
+}
+
+/** What a change sends to live queries, as tags to refetch. */
+export interface Poke { t: string[]; /** Tag prefixes: `todos:` for every row of a table-level change. */ p?: string[]; /** Tags a barrier covered: a subscription with one of them is poked already. */ k?: string[] }
+
+/** Where a feed's pokes go: this instance, and the rest through the bus. */
+export interface FeedOut {
+  emit(poke: Poke): void
+  /** Bumps the shared cache, so a refetch can't be answered with what the change made stale. */
+  bump?(tags: string[]): void
+  status(down: string | undefined): void
+}
+
+/** A running feed, between its source and the hub. */
+export interface Feed {
+  start(probe: Probe): Promise<void>
+  stop(): Promise<void>
+  /** Before a mutation's poke. Never throws: a dead feed must not fail a write that committed. */
+  settle(tags: string[]): Promise<void>
+  covers(tag: string): boolean
+  readonly running: boolean
+}
+
+const unref = <T,>(t: T) => ((t as { unref?(): void } | null | undefined)?.unref?.(), t)
+
+/** Batches a source's changes into pokes, held and throttled so a burst of writes is one refetch. */
+export function createFeed(source: Source, out: FeedOut, { hold = source.hold ?? 25, throttle = 250, barrier = 1000 } = {}): Feed {
+  const known = new Set(source.tables)
+  let pending = new Set<string>(), prefixes = new Set<string>(), covered = new Set<string>()
+  let timer: ReturnType<typeof setTimeout> | undefined
+  let last = 0
+  let first: number | undefined
+  let stopping: (() => unknown) | undefined
+  let running = false
+  let isDown = false
+
+  const flush = () => {
+    timer = first = undefined
+    last = Date.now()
+    const t = [...pending], p = [...prefixes], k = [...covered]
+    pending = new Set()
+    prefixes = new Set()
+    covered = new Set()
+    if (t.length || p.length) out.emit({ t, ...(p.length && { p }), ...(k.length && { k }) })
+  }
+  /** Each change gets its `hold`, for a barrier to reach it; a writer that never stops gets a flush per `throttle`. */
+  const arm = () => {
+    const now = Date.now()
+    first ??= now
+    clearTimeout(timer)
+    timer = unref(setTimeout(flush, Math.max(Math.min(hold, first + throttle - now), last + throttle - now)))
+  }
+
+  const tagsOf = (change: Change) => source.tagged?.(change) ?? (change.id === undefined ? [change.table] : [change.table, `${change.table}:${change.id}`])
+
+  const sink: Sink = {
+    changes(list) {
+      const seen: string[] = []
+      for (const c of list) {
+        if (!known.has(c.table)) continue
+        for (const t of tagsOf(c)) {
+          pending.add(t)
+          covered.delete(t) // a change after the barrier: no poke covers it
+          seen.push(t)
+        }
+        if (c.id === undefined) prefixes.add(`${c.table}:`)
+      }
+      if (!seen.length) return
+      out.bump?.(seen)
+      arm()
+    },
+    settled(tags) {
+      for (const t of tags) {
+        if (pending.delete(t)) covered.add(t)
+        else if (prefixes.delete(`${t}:`)) covered.add(t)
+      }
+      if (pending.size || prefixes.size) return
+      clearTimeout(timer)
+      timer = first = undefined
+      covered = new Set()
+    },
+    lost() {
+      sink.changes(source.tables.map(table => ({ table })))
+    },
+    down(why) {
+      if (isDown) return
+      isDown = true
+      out.status(why)
+    },
+    up() {
+      if (!isDown) return
+      isDown = false
+      out.status(undefined)
+    },
+  }
+
+  return {
+    get running() { return running },
+    covers(tag) {
+      const at = tag.indexOf(':')
+      return known.has(at < 0 ? tag : tag.slice(0, at))
+    },
+    async start(probe) {
+      if (running) return
+      running = true
+      try {
+        stopping = await source.start(sink, probe)
+      } catch (e) {
+        running = false
+        throw e
+      }
+    },
+    async stop() {
+      clearTimeout(timer)
+      timer = undefined
+      running = false
+      const stop = stopping
+      stopping = undefined
+      await stop?.()
+    },
+    async settle(tags) {
+      const ours = tags.filter(t => this.covers(t))
+      if (!ours.length || !source.settle) return
+      let late: ReturnType<typeof setTimeout> | undefined
+      await Promise.race([source.settle(ours), new Promise<void>(ok => { late = unref(setTimeout(ok, barrier)) })]).catch(() => {})
+      clearTimeout(late)
+    },
+  }
+}

package/src/channel-fake.ts

@@ -0,0 +1,55 @@
+// @weft-js/core/test's socket (research 32 §7): the real socket.ts and hub, a memory bus, frames through JSON.
+
+import { fakeScope, fns, net, runtime, track } from './data'
+import type { Frame, SocketLink } from './channel'
+
+/** A socket to the real server code in this process. */
+const exec_ = async () => (runtime.exec ??= await import('./exec'))
+
+export async function open(frame: (d: Frame) => void, done: (code: number) => void, up: () => void): Promise<SocketLink> {
+  const n = net()!
+  const [{ hub, resetHub }, { attach, publish }, exec] = await Promise.all([import('./hub'), import('./socket'), exec_()])
+  if (!n.online()) throw new Error('offline')
+  if (!n.resets.includes(resetHub)) n.resets.push(resetHub)
+  const h = hub()
+  h.dev = true
+  runtime.poke ??= (tags, origin) => hub().poke(tags, origin)
+  const request = new Request('http://localhost/_weft/ws')
+  const rooms = new Map<number, string>()
+  const joining = new Map<number, () => void>()
+  let closed = false
+  const peer = {
+    send(t: string) {
+      const d = JSON.parse(t)
+      frame(d)
+      if (d[0] === 'j' || d[0] === 'e') joining.get(d[1])?.()
+    },
+    close(code: number) {
+      if (closed) return
+      closed = true
+      n.ws = undefined
+      for (const go of joining.values()) go()
+      done(code)
+    },
+    backlog: () => 0,
+  }
+  const socket = await attach(peer, request, fakeScope(exec, n, request), h)
+  n.ws = {
+    publish: (id, params, name, msg) => publish(fns.get(id)!, params, name, msg),
+    drop: () => { peer.close(1006); socket.close() }, // both ends, as a real cut: the server runs `left`
+    recheck: () => socket.recheck(),
+  }
+  up()
+  return {
+    send(text) {
+      const f = JSON.parse(text)
+      if (f[0] === 'm') n.sent.push({ id: rooms.get(f[1]) ?? '', name: f[3], msg: f[4] })
+      if (f[0] === 'j') {
+        rooms.set(f[1], f[2])
+        void track(new Promise<void>(ok => { joining.set(f[1], ok) })) // `net.settle()` waits for the join's answer
+      }
+      socket.message(text)
+    },
+    close: () => socket.close(),
+  }
+}