@weft-js/start 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +16 -0
- package/README.md +41 -2
- package/package.json +107 -4
- package/src/app.mjs +93 -0
- package/src/auth/better-auth.ts +58 -0
- package/src/auth/drizzle.ts +17 -0
- package/src/auth.ts +47 -0
- package/src/boot.ts +44 -0
- package/src/builtin.tsx +19 -0
- package/src/bus/postgres.ts +91 -0
- package/src/bus.ts +48 -0
- package/src/capture.d.mts +1 -0
- package/src/capture.mjs +15 -0
- package/src/changes-sql.d.mts +14 -0
- package/src/changes-sql.mjs +99 -0
- package/src/changes.ts +253 -0
- package/src/channel-fake.ts +55 -0
- package/src/channel.ts +622 -0
- package/src/client.tsx +20 -0
- package/src/contract.mjs +312 -0
- package/src/data.ts +929 -0
- package/src/db/bun-sqlite.d.ts +10 -0
- package/src/db/changes-mysql.ts +86 -0
- package/src/db/changes-postgres.ts +159 -0
- package/src/db/changes-sqlite.ts +116 -0
- package/src/db/core.ts +63 -0
- package/src/db/index.ts +27 -0
- package/src/db/mysql.ts +67 -0
- package/src/db/postgres.ts +65 -0
- package/src/db/sqlite.ts +139 -0
- package/src/derive.d.mts +16 -0
- package/src/derive.mjs +140 -0
- package/src/dev-runtime.mjs +42 -0
- package/src/exec.ts +519 -0
- package/src/extension.ts +38 -0
- package/src/first-paint.ts +57 -0
- package/src/form.tsx +134 -0
- package/src/gate.css +6 -0
- package/src/hub.ts +822 -0
- package/src/index.ts +106 -0
- package/src/inline.mjs +2 -0
- package/src/inspect.mjs +280 -0
- package/src/link.ts +461 -0
- package/src/list-drizzle.ts +50 -0
- package/src/list.ts +224 -0
- package/src/listen.ts +28 -0
- package/src/live-fake.ts +42 -0
- package/src/live-target.mjs +147 -0
- package/src/live.ts +499 -0
- package/src/media.ts +43 -0
- package/src/migrations.mjs +19 -0
- package/src/mux-session.ts +6 -0
- package/src/mux.ts +135 -0
- package/src/og/DefaultCard.tsx +60 -0
- package/src/og/fonts.mjs +64 -0
- package/src/og/images.mjs +54 -0
- package/src/og/index.mjs +146 -0
- package/src/og/native.mjs +29 -0
- package/src/range.mjs +19 -0
- package/src/router.ts +149 -0
- package/src/routes.mjs +142 -0
- package/src/run.tsx +202 -0
- package/src/rust.ts +333 -0
- package/src/server.tsx +1021 -0
- package/src/shape.d.mts +3 -0
- package/src/shape.mjs +73 -0
- package/src/share-server.ts +208 -0
- package/src/share.ts +503 -0
- package/src/sheets.mjs +75 -0
- package/src/sidecar.mjs +503 -0
- package/src/socket.ts +389 -0
- package/src/stream.ts +13 -0
- package/src/verify.ts +251 -0
- package/src/view.tsx +47 -0
- package/src/vite.mjs +899 -0
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
// Bun's built-in driver has no types on Node, where this package is checked: only what `open` calls.
|
|
2
|
+
declare module 'bun:sqlite' {
|
|
3
|
+
/** One open database file. */
|
|
4
|
+
export class Database {
|
|
5
|
+
constructor(path: string, options: { create: boolean })
|
|
6
|
+
prepare(text: string): { run(...params: unknown[]): unknown; values(...params: unknown[]): unknown[][] }
|
|
7
|
+
run(text: string): unknown
|
|
8
|
+
close(): void
|
|
9
|
+
}
|
|
10
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
// The change feed on MySQL. A trigger can't reach a client and the binlog needs replication rights, so triggers write a
|
|
2
|
+
// log and this reads it on a timer: polling, and it says so. An AUTO_INCREMENT id is taken before commit, so a long
|
|
3
|
+
// transaction can land behind the mark: each look re-reads a window behind it.
|
|
4
|
+
|
|
5
|
+
import type { Pool } from 'mysql2/promise'
|
|
6
|
+
import { LOG, MAX_IDS } from '../changes-sql.mjs'
|
|
7
|
+
import type { Change, Probe, Sink } from '../changes'
|
|
8
|
+
|
|
9
|
+
const BEHIND = 512
|
|
10
|
+
const KEEP_HOURS = 1
|
|
11
|
+
const PRUNE = 60_000
|
|
12
|
+
|
|
13
|
+
interface Options { pool: Pool; tables: string[]; interval?: number }
|
|
14
|
+
|
|
15
|
+
/** A bigint column: the driver's number, or its string once past 2^53. */
|
|
16
|
+
type Seq = number | string
|
|
17
|
+
const said = (e: unknown) => (e as Error | null)?.message ?? String(e)
|
|
18
|
+
|
|
19
|
+
export function mysqlChanges({ pool, tables, interval = 500 }: Options) {
|
|
20
|
+
async function probe(): Promise<Probe> {
|
|
21
|
+
try {
|
|
22
|
+
const [rows] = await pool.query(`select event_object_table as t from information_schema.triggers where trigger_schema = database() and trigger_name like 'weft\\_changes\\_%\\_upd'`)
|
|
23
|
+
const have = (rows as { t: string }[]).map(r => r.t)
|
|
24
|
+
const missing = tables.filter(t => !have.includes(t))
|
|
25
|
+
if (!missing.length) return { mode: 'feed', via: 'mysql-log', why: `read every ${interval} ms: MySQL can't push` }
|
|
26
|
+
const [grants] = await pool.query('show grants').catch(() => [[]])
|
|
27
|
+
const may = (grants as Record<string, string>[]).some(g => /ALL PRIVILEGES|\bTRIGGER\b/.test(Object.values(g)[0] ?? ''))
|
|
28
|
+
return { mode: 'off', via: 'none', why: `no trigger on ${missing.map(t => `\`${t}\``).join(', ')}, and MySQL has no counter to fall back on`, fix: may ? 'run `weft db live`, then apply the migration' : 'run `weft db live`, and apply the migration as a user with the TRIGGER privilege' }
|
|
29
|
+
} catch (e) {
|
|
30
|
+
return { mode: 'off', via: 'none', why: `the database didn't answer: ${said(e)}`, fix: 'check `DATABASE_URL`' }
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
async function start(sink: Sink, how: Probe) {
|
|
35
|
+
if (how.mode === 'off') return () => {}
|
|
36
|
+
let stopped = false, timer: ReturnType<typeof setTimeout> | undefined, failed = false, pruned = Date.now()
|
|
37
|
+
const [[{ top }]] = (await pool.query(`select coalesce(max(seq), 0) as top from ${LOG}`)) as unknown as [[{ top: number | string }]]
|
|
38
|
+
let mark = Number(top)
|
|
39
|
+
// What's already there is old news, and needs no poke.
|
|
40
|
+
const [old] = (await pool.query(`select seq from ${LOG} where seq > ?`, [Math.max(0, mark - BEHIND)])) as unknown as [{ seq: Seq }[]]
|
|
41
|
+
const seen = new Set<number>(old.map(r => Number(r.seq)))
|
|
42
|
+
const look = async () => {
|
|
43
|
+
if (stopped) return
|
|
44
|
+
try {
|
|
45
|
+
const [rows] = (await pool.query(`select seq, tbl, pk from ${LOG} where seq > ? order by seq limit 5000`, [Math.max(0, mark - BEHIND)])) as unknown as [{ seq: Seq; tbl: string; pk: string | null }[]]
|
|
46
|
+
if (failed) {
|
|
47
|
+
failed = false
|
|
48
|
+
sink.lost()
|
|
49
|
+
sink.up()
|
|
50
|
+
}
|
|
51
|
+
const fresh = rows.filter(r => !seen.has(Number(r.seq)))
|
|
52
|
+
const byTable = new Map<string, Set<string> | null>()
|
|
53
|
+
for (const r of fresh) {
|
|
54
|
+
seen.add(Number(r.seq))
|
|
55
|
+
if (r.pk === null) byTable.set(r.tbl, null)
|
|
56
|
+
else if (byTable.get(r.tbl) !== null) {
|
|
57
|
+
const ids = byTable.get(r.tbl) ?? new Set()
|
|
58
|
+
ids.add(r.pk)
|
|
59
|
+
byTable.set(r.tbl, ids.size > MAX_IDS ? null : ids)
|
|
60
|
+
}
|
|
61
|
+
mark = Math.max(mark, Number(r.seq))
|
|
62
|
+
}
|
|
63
|
+
for (const s of seen) if (s <= mark - BEHIND) seen.delete(s)
|
|
64
|
+
const list: Change[] = []
|
|
65
|
+
for (const [table, ids] of byTable) ids ? list.push(...[...ids].map(id => ({ table, id }))) : list.push({ table })
|
|
66
|
+
if (list.length) sink.changes(list)
|
|
67
|
+
if (Date.now() - pruned > PRUNE) {
|
|
68
|
+
pruned = Date.now()
|
|
69
|
+
await pool.query(`delete from ${LOG} where at < now() - interval ${KEEP_HOURS} hour limit 10000`)
|
|
70
|
+
}
|
|
71
|
+
} catch (e) {
|
|
72
|
+
failed = true
|
|
73
|
+
sink.down(`MySQL watcher: ${said(e)}`)
|
|
74
|
+
}
|
|
75
|
+
timer = setTimeout(() => void look(), interval)
|
|
76
|
+
;(timer as { unref?(): void }).unref?.()
|
|
77
|
+
}
|
|
78
|
+
await look()
|
|
79
|
+
return () => {
|
|
80
|
+
stopped = true
|
|
81
|
+
clearTimeout(timer)
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
return { probe, start }
|
|
86
|
+
}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
// The change feed on Postgres: triggers NOTIFY, one dedicated connection LISTENs. The probe tries the real thing before it
|
|
2
|
+
// claims anything, because a pooler in transaction mode accepts LISTEN and then delivers nothing.
|
|
3
|
+
|
|
4
|
+
import postgres from 'postgres'
|
|
5
|
+
import { CHANNEL } from '../changes-sql.mjs'
|
|
6
|
+
import type { Probe, Sink } from '../changes'
|
|
7
|
+
|
|
8
|
+
type Sql = ReturnType<typeof postgres>
|
|
9
|
+
|
|
10
|
+
const SECRET = /\/\/[^@/]*@/
|
|
11
|
+
const opts = { onnotice: () => {}, connect_timeout: 5 }
|
|
12
|
+
const PROBE = 1500
|
|
13
|
+
const HEARTBEAT = 5_000
|
|
14
|
+
const MAX_BARRIER = 7_800
|
|
15
|
+
|
|
16
|
+
interface Options { url: string; listenUrl?: string; client: Sql; tables: string[]; interval?: number; heartbeat?: number }
|
|
17
|
+
|
|
18
|
+
export function postgresChanges({ url, listenUrl, client, tables, interval = 2000, heartbeat = HEARTBEAT }: Options) {
|
|
19
|
+
const direct = listenUrl ?? url
|
|
20
|
+
const late = heartbeat * 2
|
|
21
|
+
|
|
22
|
+
/** Sends a NOTIFY and waits to hear it on a listening connection: the only proof a pooler can't fake. */
|
|
23
|
+
async function hears(): Promise<boolean | string> {
|
|
24
|
+
const sql = postgres(direct, { ...opts, max: 1 })
|
|
25
|
+
const token = `?${crypto.randomUUID()}`
|
|
26
|
+
let heard!: () => void
|
|
27
|
+
const got = new Promise<void>(ok => { heard = ok })
|
|
28
|
+
try {
|
|
29
|
+
await sql.listen(CHANNEL, p => { if (p === token) heard() })
|
|
30
|
+
await client.notify(CHANNEL, token)
|
|
31
|
+
return await Promise.race([got.then(() => true), new Promise<boolean>(ok => { setTimeout(ok, PROBE, false) })])
|
|
32
|
+
} catch (e) {
|
|
33
|
+
return (e as Error | null)?.message ?? String(e)
|
|
34
|
+
} finally {
|
|
35
|
+
void sql.end({ timeout: 0 }).catch(() => {})
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async function triggers(): Promise<string[]> {
|
|
40
|
+
const rows = await client`select c.relname from pg_trigger t join pg_class c on c.oid = t.tgrelid where t.tgname = 'weft_changes_upd' and c.relname = any(${tables})`
|
|
41
|
+
return rows.map(r => r.relname as string)
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Whether this role may add them, for the error's fix. */
|
|
45
|
+
async function mayCreate(table: string): Promise<boolean> {
|
|
46
|
+
const [r] = await client`select has_table_privilege(current_user, ${table}, 'TRIGGER') as ok`.catch(() => [{ ok: false }])
|
|
47
|
+
return !!r?.ok
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
async function stats(): Promise<boolean> {
|
|
51
|
+
const [r] = await client`select current_setting('track_counts') = 'on' as ok`.catch(() => [{ ok: false }])
|
|
52
|
+
return !!r?.ok
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const poll = async (why: string, fix: string): Promise<Probe> =>
|
|
56
|
+
(await stats())
|
|
57
|
+
? { mode: 'poll', via: 'postgres-stats', why: `${why}. Reading table counters every ${interval / 1000} s: writes show up late, and by table only`, fix }
|
|
58
|
+
: { mode: 'off', via: 'none', why: `${why}, and \`track_counts\` is off, so there is nothing to poll`, fix }
|
|
59
|
+
|
|
60
|
+
async function probe(): Promise<Probe> {
|
|
61
|
+
let have: string[]
|
|
62
|
+
try {
|
|
63
|
+
have = await triggers()
|
|
64
|
+
} catch (e) {
|
|
65
|
+
return { mode: 'off', via: 'none', why: `the database didn't answer: ${((e as Error | null)?.message ?? String(e)).replace(SECRET, '//…@')}`, fix: 'check `DATABASE_URL`' }
|
|
66
|
+
}
|
|
67
|
+
const missing = tables.filter(t => !have.includes(t))
|
|
68
|
+
if (missing.length) {
|
|
69
|
+
const allowed = await mayCreate(missing[0]!)
|
|
70
|
+
return poll(`no trigger on ${missing.map(t => `\`${t}\``).join(', ')}`, allowed ? 'run `weft db live`, then apply the migration' : `run \`weft db live\`, and apply the migration as a role that may create triggers: this one has no TRIGGER on \`${missing[0]}\``)
|
|
71
|
+
}
|
|
72
|
+
const ok = await hears()
|
|
73
|
+
if (ok === true) return { mode: 'feed', via: 'postgres-notify' }
|
|
74
|
+
const pooler = /:6432\b|-pooler\.|pooler\./.test(direct)
|
|
75
|
+
const why = typeof ok === 'string' ? `LISTEN failed: ${ok.replace(SECRET, '//…@')}` : `a NOTIFY never reached a LISTEN on this connection${pooler ? ', which looks like a pooler' : ''}: a pooler in transaction mode does that`
|
|
76
|
+
return poll(why, 'give `dbChanges` a direct connection: `{ listenUrl }`')
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** A self-addressed NOTIFY proves the listener is alive: a half-open TCP connection looks fine until then. */
|
|
80
|
+
async function listen(sink: Sink) {
|
|
81
|
+
let sql: Sql | undefined, started = false, stopped = false, seen = Date.now(), timer: ReturnType<typeof setTimeout> | undefined
|
|
82
|
+
const token = `?${crypto.randomUUID()}`
|
|
83
|
+
const heard = (payload: string) => {
|
|
84
|
+
seen = Date.now()
|
|
85
|
+
if (payload[0] === '?') return
|
|
86
|
+
const m = JSON.parse(payload)
|
|
87
|
+
if (m.b) return sink.settled(m.b)
|
|
88
|
+
sink.changes(m.i ? m.i.map((id: string) => ({ table: m.t, id })) : [{ table: m.t }])
|
|
89
|
+
}
|
|
90
|
+
const connect = async () => {
|
|
91
|
+
const mine = (sql = postgres(direct, { ...opts, max: 1, idle_timeout: 0 }))
|
|
92
|
+
await mine.listen(CHANNEL, heard, () => {
|
|
93
|
+
seen = Date.now()
|
|
94
|
+
if (started) sink.lost() // what committed while it was gone is unknown
|
|
95
|
+
started = true
|
|
96
|
+
sink.up()
|
|
97
|
+
})
|
|
98
|
+
}
|
|
99
|
+
const check = async () => {
|
|
100
|
+
if (stopped) return
|
|
101
|
+
if (Date.now() - seen > late) {
|
|
102
|
+
sink.down('the database listener is not receiving')
|
|
103
|
+
const old = sql
|
|
104
|
+
sql = undefined
|
|
105
|
+
void old?.end({ timeout: 0 }).catch(() => {})
|
|
106
|
+
await connect().catch(() => {}) // the next check tries again
|
|
107
|
+
}
|
|
108
|
+
await client.notify(CHANNEL, token).catch(() => sink.down('the database is not answering'))
|
|
109
|
+
if (stopped) return
|
|
110
|
+
timer = setTimeout(() => void check(), heartbeat)
|
|
111
|
+
;(timer as { unref?(): void }).unref?.()
|
|
112
|
+
}
|
|
113
|
+
await connect().catch(e => sink.down(`the database listener can't connect: ${String(e?.message ?? e).replace(SECRET, '//…@')}`))
|
|
114
|
+
timer = setTimeout(() => void check(), heartbeat)
|
|
115
|
+
;(timer as { unref?(): void }).unref?.()
|
|
116
|
+
return async () => {
|
|
117
|
+
stopped = true
|
|
118
|
+
clearTimeout(timer)
|
|
119
|
+
await sql?.end({ timeout: 0 }).catch(() => {})
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** `postgres-stats`: counters that only ever grow, per table, by the second. Nothing to install. */
|
|
124
|
+
function counters(sink: Sink) {
|
|
125
|
+
const seen = new Map<string, string>()
|
|
126
|
+
let stopped = false, timer: ReturnType<typeof setTimeout> | undefined
|
|
127
|
+
const look = async () => {
|
|
128
|
+
if (stopped) return
|
|
129
|
+
try {
|
|
130
|
+
const rows = await client`select relname, n_tup_ins + n_tup_upd + n_tup_del + n_mod_since_analyze as n from pg_stat_user_tables where relname = any(${tables})`
|
|
131
|
+
sink.up()
|
|
132
|
+
const changed = rows.filter(r => {
|
|
133
|
+
const was = seen.get(r.relname)
|
|
134
|
+
seen.set(r.relname, String(r.n))
|
|
135
|
+
return was !== undefined && was !== String(r.n)
|
|
136
|
+
})
|
|
137
|
+
if (changed.length) sink.changes(changed.map(r => ({ table: r.relname as string })))
|
|
138
|
+
} catch {
|
|
139
|
+
sink.down('the database is not answering')
|
|
140
|
+
}
|
|
141
|
+
timer = setTimeout(() => void look(), interval)
|
|
142
|
+
;(timer as { unref?(): void }).unref?.()
|
|
143
|
+
}
|
|
144
|
+
return look().then(() => () => {
|
|
145
|
+
stopped = true
|
|
146
|
+
clearTimeout(timer)
|
|
147
|
+
})
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
return {
|
|
151
|
+
probe,
|
|
152
|
+
start: (sink: Sink, how: Probe) => (how.via === 'postgres-notify' ? listen(sink) : how.mode === 'poll' ? counters(sink) : Promise.resolve(() => {})),
|
|
153
|
+
/** After the mutation committed, so it arrives after that commit's changes, and before the poke it stands for. */
|
|
154
|
+
async settle(tags: string[]) {
|
|
155
|
+
const text = JSON.stringify({ b: tags })
|
|
156
|
+
await client.notify(CHANNEL, text.length < MAX_BARRIER ? text : JSON.stringify({ b: [...new Set(tags.map(t => t.split(':')[0]))] }))
|
|
157
|
+
},
|
|
158
|
+
}
|
|
159
|
+
}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
// The change feed on SQLite. There is no notify across processes, so this is polling, of the cheapest thing there is:
|
|
2
|
+
// `PRAGMA data_version` on a second connection moves when any connection commits, and costs no file read. Only then is
|
|
3
|
+
// the triggers' log read. Same host only: WAL's shared memory doesn't cross a network file system.
|
|
4
|
+
|
|
5
|
+
import { execFileSync } from 'node:child_process'
|
|
6
|
+
import { existsSync, statfsSync } from 'node:fs'
|
|
7
|
+
import { LOG, MAX_IDS } from '../changes-sql.mjs'
|
|
8
|
+
import type { Change, Probe, Sink } from '../changes'
|
|
9
|
+
import type { Method } from './sqlite'
|
|
10
|
+
|
|
11
|
+
interface Conn {
|
|
12
|
+
query(sql: string, params: unknown[], method: Method): unknown
|
|
13
|
+
exec(sql: string): void
|
|
14
|
+
close?(): void
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
// statfs(2) magic numbers: https://man7.org/linux/man-pages/man2/statfs.2.html
|
|
18
|
+
const NETWORK = new Set([0x6969 /* nfs */, 0x517b /* smb */, 0xfe534d42 /* smb2 */, 0xff534d42 /* cifs */, 0x65735546 /* fuse */, 0x5346414f /* afs */, 0xc36400 /* ceph */, 0x47504653 /* gpfs */])
|
|
19
|
+
|
|
20
|
+
/** Whether a file is on a network file system. `exec` and `statfs` are the test's way in. */
|
|
21
|
+
export function onNetwork(path: string, { platform = process.platform, statfs = statfsSync, df = (p: string) => execFileSync('df', ['-P', p], { encoding: 'utf8' }) } = {}): boolean {
|
|
22
|
+
try {
|
|
23
|
+
if (platform === 'linux') return NETWORK.has(statfs(path).type)
|
|
24
|
+
const device = df(path).trim().split('\n').at(-1)!.split(/\s+/)[0] ?? ''
|
|
25
|
+
return device.startsWith('//') || /^[^/]+:\//.test(device)
|
|
26
|
+
} catch {
|
|
27
|
+
return false
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
interface Options { path: string; open: (path: string) => Promise<Conn>; tables: string[]; interval?: number }
|
|
32
|
+
|
|
33
|
+
export function sqliteChanges({ path, open, tables, interval = 100 }: Options) {
|
|
34
|
+
const log = (c: Conn) => (c.query(`select count(*) from sqlite_master where name = '${LOG}'`, [], 'values') as number[][])[0]![0]! > 0
|
|
35
|
+
const triggers = (c: Conn) => (c.query(`select tbl_name from sqlite_master where type = 'trigger' and name like 'weft\\_changes\\_%\\_upd' escape '\\'`, [], 'values') as string[][]).map(r => r[0])
|
|
36
|
+
let tick: (() => void) | undefined, out: Sink | undefined
|
|
37
|
+
|
|
38
|
+
async function probe(): Promise<Probe> {
|
|
39
|
+
if (path === ':memory:') return { mode: 'off', via: 'none', why: 'an in-memory database has no other writers, and no second connection could see it', fix: 'use a file to have writes from other processes reach `.live`' }
|
|
40
|
+
if (onNetwork(existsSync(path) ? path : '.')) return { mode: 'off', via: 'none', why: `\`${path}\` is on a network file system: \`data_version\` and WAL don't work across hosts`, fix: 'put the database on a local disk, or move to Postgres' }
|
|
41
|
+
const c = await open(path)
|
|
42
|
+
try {
|
|
43
|
+
const have = log(c) ? triggers(c) : []
|
|
44
|
+
const missing = tables.filter(t => !have.includes(t))
|
|
45
|
+
if (missing.length) return { mode: 'poll', via: 'sqlite-data-version', why: `no trigger on ${missing.map(t => `\`${t}\``).join(', ')}: any commit pokes every table, unrelated ones included`, fix: 'run `weft db live`' }
|
|
46
|
+
return { mode: 'feed', via: 'sqlite-data-version' }
|
|
47
|
+
} finally {
|
|
48
|
+
c.close?.()
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
async function start(sink: Sink, how: Probe) {
|
|
53
|
+
if (how.mode === 'off') return () => {}
|
|
54
|
+
const c = await open(path)
|
|
55
|
+
const version = () => (c.query('pragma data_version', [], 'values') as number[][])[0]![0]!
|
|
56
|
+
const usable = how.mode === 'feed'
|
|
57
|
+
const newest = () => (c.query(`select coalesce(max(seq), 0) from ${LOG}`, [], 'values') as number[][])[0]![0]!
|
|
58
|
+
let seq = usable ? newest() : 0
|
|
59
|
+
let at = version()
|
|
60
|
+
let timer: ReturnType<typeof setTimeout> | undefined
|
|
61
|
+
const read = () => {
|
|
62
|
+
const rows = c.query(`select seq, tbl, pk from ${LOG} where seq > ? order by seq`, [seq], 'values') as [number, string, string | null][]
|
|
63
|
+
if (!rows.length) return
|
|
64
|
+
if (rows[0]![0] > seq + 1) sink.lost() // the log moved past us
|
|
65
|
+
const byTable = new Map<string, Set<string> | null>()
|
|
66
|
+
for (const [, table, id] of rows) {
|
|
67
|
+
if (id === null) byTable.set(table, null)
|
|
68
|
+
else if (byTable.get(table) !== null) {
|
|
69
|
+
const ids = byTable.get(table) ?? new Set()
|
|
70
|
+
ids.add(id)
|
|
71
|
+
byTable.set(table, ids.size > MAX_IDS ? null : ids)
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
seq = rows.at(-1)![0]
|
|
75
|
+
const list: Change[] = []
|
|
76
|
+
for (const [table, ids] of byTable) ids ? list.push(...[...ids].map(id => ({ table, id }))) : list.push({ table })
|
|
77
|
+
sink.changes(list)
|
|
78
|
+
}
|
|
79
|
+
const look = () => {
|
|
80
|
+
try {
|
|
81
|
+
const now = version()
|
|
82
|
+
if (now === at) return
|
|
83
|
+
at = now
|
|
84
|
+
if (usable) read()
|
|
85
|
+
else sink.changes(tables.map(table => ({ table })))
|
|
86
|
+
sink.up()
|
|
87
|
+
} catch (e) {
|
|
88
|
+
sink.down(`SQLite watcher: ${(e as Error | null)?.message ?? String(e)}`)
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
tick = look
|
|
92
|
+
out = sink
|
|
93
|
+
const every = () => {
|
|
94
|
+
look()
|
|
95
|
+
timer = setTimeout(every, interval)
|
|
96
|
+
;(timer as { unref?(): void }).unref?.()
|
|
97
|
+
}
|
|
98
|
+
every()
|
|
99
|
+
return () => {
|
|
100
|
+
tick = out = undefined
|
|
101
|
+
clearTimeout(timer)
|
|
102
|
+
c.close?.()
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
return {
|
|
107
|
+
probe,
|
|
108
|
+
start,
|
|
109
|
+
/** In this process, so the barrier needs no trip: read what's committed now, and let the hub drop what the poke covers. */
|
|
110
|
+
settle(tags: string[]) {
|
|
111
|
+
tick?.()
|
|
112
|
+
out?.settled(tags)
|
|
113
|
+
return Promise.resolve()
|
|
114
|
+
},
|
|
115
|
+
}
|
|
116
|
+
}
|
package/src/db/core.ts
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
// What every data layer shares (research 28 §2, 34 §4): the gate that applies the server's migrations before a database's
|
|
2
|
+
// first statement, and the touched-tables record behind Q011. A provider is any client with a `$weft` handle made through
|
|
3
|
+
// `register`. No driver imports, so `@weft-js/start/db/sqlite` doesn't load Postgres, or the other way round.
|
|
4
|
+
|
|
5
|
+
import { currentScope, runtime, StartError } from '../data'
|
|
6
|
+
import type { Probe, Sink } from '../changes'
|
|
7
|
+
|
|
8
|
+
/** One migration file: its name and the SQL. */
|
|
9
|
+
export interface Migration { tag: string; sql: string }
|
|
10
|
+
|
|
11
|
+
/** What every Weft database carries on `$weft`, whichever driver is under it. */
|
|
12
|
+
export interface DbHandle {
|
|
13
|
+
dialect: 'sqlite' | 'postgresql' | 'mysql'
|
|
14
|
+
/** Apply what's new in `list`, in order. */
|
|
15
|
+
migrate(list: Migration[]): Promise<void>
|
|
16
|
+
/** Tables Q011 doesn't report (Better Auth's). */
|
|
17
|
+
exempt: Set<string>
|
|
18
|
+
/** `dbChanges`'s source: loaded on the first use, so a database with no live tables loads none of it. */
|
|
19
|
+
changes?(tables: string[], options: { listenUrl?: string; interval?: number }): Promise<{ probe(): Promise<Probe>; start(sink: Sink, how: Probe): Promise<() => unknown>; settle?(tags: string[]): Promise<void> }>
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** Attach weft's handle to a client. */
|
|
23
|
+
export const register = <D extends object>(db: D, handle: DbHandle): D & { $weft: DbHandle } => Object.assign(db, { $weft: handle })
|
|
24
|
+
|
|
25
|
+
/** The database can't be used: its connection, or a migration it needs. The client sees this; the server log gets the cause. */
|
|
26
|
+
export const unavailable = (cause: unknown) =>
|
|
27
|
+
new StartError('Q016', 'the database is unavailable, or a migration failed', 'start it and check `DATABASE_URL`; the server log has the driver\'s error', { status: 503, cause })
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* What a provider awaits before each statement, or its connection call: resolved once the list the server's first request
|
|
31
|
+
* set (`runtime.migrations`) is applied, so a request that touches no database never needs one. Concurrent callers share
|
|
32
|
+
* one run; a failed run is retried by the next caller, so a database that comes back needs no restart. Before a request
|
|
33
|
+
* (`weft build`, scripts) and with `migrate: false` there's nothing to wait for.
|
|
34
|
+
*/
|
|
35
|
+
export function gate(handle: DbHandle, migrate: boolean) {
|
|
36
|
+
let applied: unknown
|
|
37
|
+
let run: Promise<void> = Promise.resolve()
|
|
38
|
+
return (): Promise<void> | undefined => {
|
|
39
|
+
const m = runtime.migrations
|
|
40
|
+
if (!migrate || !m) return undefined
|
|
41
|
+
if (m !== applied) {
|
|
42
|
+
applied = m
|
|
43
|
+
run = run.catch(() => {}).then(() => handle.migrate(m.list)).catch(e => {
|
|
44
|
+
if (applied === m) applied = undefined
|
|
45
|
+
m.report(e)
|
|
46
|
+
throw unavailable(e)
|
|
47
|
+
})
|
|
48
|
+
}
|
|
49
|
+
return run
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Dev: record a statement's tables in the running function's frame. Identifiers may be quoted, as Drizzle's always are. */
|
|
54
|
+
export function touch(handle: DbHandle, sql: string) {
|
|
55
|
+
const t = currentScope()?.touched
|
|
56
|
+
if (!t) return
|
|
57
|
+
const target = /^\s*(?:insert\s+into|update|delete\s+from)\s+(?:["`]?\w+["`]?\.)?["`]?([\w$]+)/i.exec(sql)?.[1]
|
|
58
|
+
if (target && !handle.exempt.has(target)) t.write.add(target)
|
|
59
|
+
for (const m of sql.matchAll(/\b(?:from|join)\s+(?:["`]?\w+["`]?\.)?["`]?([\w$]+)/gi)) if (m[1] !== target && !handle.exempt.has(m[1]!)) t.read.add(m[1]!)
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** A migration file's statements (drizzle-kit separates them with this marker). */
|
|
63
|
+
export const statements = (sql: string) => sql.split('--> statement-breakpoint').map(s => s.trim()).filter(Boolean)
|
package/src/db/index.ts
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
// `@weft-js/start/db`: what doesn't depend on the driver (research 28 §2). `schemasOf(schema).todos.insert` is a Zod
|
|
2
|
+
// schema per table, so a table is its own validator: `input: schemas.todos.insert.pick({ title: true })`.
|
|
3
|
+
|
|
4
|
+
import { getTableName, isTable, type Table } from 'drizzle-orm'
|
|
5
|
+
import { createInsertSchema, createSelectSchema, createUpdateSchema, type BuildSchema } from 'drizzle-zod'
|
|
6
|
+
|
|
7
|
+
export type { DbHandle, Migration } from './core'
|
|
8
|
+
|
|
9
|
+
type TablesOf<S> = { [K in keyof S as S[K] extends Table ? K : never]: S[K] }
|
|
10
|
+
type Select<T extends Table> = BuildSchema<'select', T['_']['columns'], undefined, undefined>
|
|
11
|
+
type Insert<T extends Table> = BuildSchema<'insert', T['_']['columns'], undefined, undefined>
|
|
12
|
+
type Update<T extends Table> = BuildSchema<'update', T['_']['columns'], undefined, undefined>
|
|
13
|
+
/** The select and insert types of each table in a schema. */
|
|
14
|
+
export type Schemas<S> = { [K in keyof TablesOf<S>]: TablesOf<S>[K] extends infer T extends Table ? { select: Select<T>; insert: Insert<T>; update: Update<T> } : never }
|
|
15
|
+
|
|
16
|
+
/** `refine`: per table, per column, what drizzle-zod takes: `{ todos: { title: s => s.trim().min(1) } }`. */
|
|
17
|
+
export function schemasOf<S extends Record<string, unknown>>(schema: S, refine: { [K in keyof TablesOf<S>]?: Record<string, (s: never) => unknown> } = {}): Schemas<S> {
|
|
18
|
+
const out: Record<string, unknown> = {}
|
|
19
|
+
for (const [k, t] of Object.entries(schema)) {
|
|
20
|
+
if (!isTable(t)) continue
|
|
21
|
+
const r = (refine as Record<string, never>)[k]
|
|
22
|
+
out[k] = { select: createSelectSchema(t, r), insert: createInsertSchema(t, r), update: createUpdateSchema(t, r) }
|
|
23
|
+
}
|
|
24
|
+
return out as Schemas<S>
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export { getTableName }
|
package/src/db/mysql.ts
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
// MySQL for `@weft-js/start` (research 34 §4): Drizzle on mysql2, pure JS. The pool connects on the first statement, so
|
|
2
|
+
// `weft build` and a page with no queries need no database. MySQL commits around DDL, so a failed migration can't roll back; a named lock keeps
|
|
3
|
+
// instances starting together from applying a file twice.
|
|
4
|
+
|
|
5
|
+
import { drizzle, type MySql2Database } from 'drizzle-orm/mysql2'
|
|
6
|
+
import { createPool } from 'mysql2/promise'
|
|
7
|
+
import { StartError } from '../data'
|
|
8
|
+
import { gate, register, statements, touch, type DbHandle, type Migration } from './core'
|
|
9
|
+
|
|
10
|
+
/** `createDb` on MySQL: the schema, the URL, and whether to migrate on first use. */
|
|
11
|
+
export interface MysqlOptions<S extends Record<string, unknown>> {
|
|
12
|
+
schema: S
|
|
13
|
+
/** `mysql://user:pass@host:3306/db`. */
|
|
14
|
+
url: string
|
|
15
|
+
/** Apply `drizzle/` before the first statement a request runs (default true). */
|
|
16
|
+
migrate?: boolean
|
|
17
|
+
/** Pool size (default 10). */
|
|
18
|
+
max?: number
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** Drizzle on mysql2, with `$weft` for the framework's own needs. */
|
|
22
|
+
export type MysqlDb<S extends Record<string, unknown>> = MySql2Database<S> & { $weft: DbHandle }
|
|
23
|
+
|
|
24
|
+
const LOCK = 'weft_migrations'
|
|
25
|
+
|
|
26
|
+
/** A pooled Drizzle client on MySQL, with this app's migrations and write tracking. */
|
|
27
|
+
export function createDb<S extends Record<string, unknown>>({ schema, url, migrate = true, max = 10 }: MysqlOptions<S>): MysqlDb<S> {
|
|
28
|
+
if (!url && process.env.WEFT_BUILD) url = 'mysql://localhost/build' // a build loads this module and never connects
|
|
29
|
+
if (!url?.startsWith('mysql://'))
|
|
30
|
+
throw new StartError('Q013', `\`@weft-js/start/db/mysql\` needs a \`mysql://\` URL, not \`${(url ?? '').replace(/\/\/.*@/, '//…@')}\``, 'set `DATABASE_URL`, or `createDb` from `@weft-js/start/db/sqlite`')
|
|
31
|
+
// No `maxIdle` below the limit: that starts mysql2's reaper, a timer that keeps a finished `weft build` alive forever.
|
|
32
|
+
const pool = createPool({ uri: url, connectionLimit: max })
|
|
33
|
+
const handle: DbHandle = {
|
|
34
|
+
dialect: 'mysql',
|
|
35
|
+
exempt: new Set(),
|
|
36
|
+
async migrate(list: Migration[]) {
|
|
37
|
+
const c = await pool.getConnection()
|
|
38
|
+
try {
|
|
39
|
+
await c.query('select get_lock(?, 60)', [LOCK])
|
|
40
|
+
await c.query('create table if not exists __weft_migrations (tag varchar(255) primary key, at timestamp not null default current_timestamp)')
|
|
41
|
+
const [rows] = await c.query('select tag from __weft_migrations')
|
|
42
|
+
const done = new Set((rows as { tag: string }[]).map(r => r.tag))
|
|
43
|
+
for (const m of list) {
|
|
44
|
+
if (done.has(m.tag)) continue
|
|
45
|
+
for (const s of statements(m.sql)) await c.query(s)
|
|
46
|
+
await c.query('insert into __weft_migrations (tag) values (?)', [m.tag])
|
|
47
|
+
}
|
|
48
|
+
} finally {
|
|
49
|
+
await c.query('select release_lock(?)', [LOCK]).catch(() => {})
|
|
50
|
+
c.release()
|
|
51
|
+
}
|
|
52
|
+
},
|
|
53
|
+
changes: (tables, o) => import('./changes-mysql').then(m => m.mysqlChanges({ ...o, pool, tables })),
|
|
54
|
+
}
|
|
55
|
+
const ready = gate(handle, migrate)
|
|
56
|
+
// All Drizzle sends goes through these three, so holding them holds it; `handle.migrate` keeps the plain pool.
|
|
57
|
+
const held = new Proxy(pool, {
|
|
58
|
+
get(t, key) {
|
|
59
|
+
const v = Reflect.get(t, key)
|
|
60
|
+
if (typeof v !== 'function') return v
|
|
61
|
+
if (key !== 'query' && key !== 'execute' && key !== 'getConnection') return v.bind(t)
|
|
62
|
+
return async (...a: unknown[]) => (await ready(), v.apply(t, a))
|
|
63
|
+
},
|
|
64
|
+
})
|
|
65
|
+
const db = drizzle(held, { schema, mode: 'default', logger: { logQuery: sql => touch(handle, sql) } })
|
|
66
|
+
return register(db, handle)
|
|
67
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
// Postgres for `@weft-js/start` (research 28 §2): Drizzle on postgres.js, pure JS on Node, Bun and Deno. The client
|
|
2
|
+
// connects on the first statement, so `weft build` and a page with no queries need no database. Migrations hold an
|
|
3
|
+
// advisory lock, so instances starting together apply each file once.
|
|
4
|
+
|
|
5
|
+
import postgres from 'postgres'
|
|
6
|
+
import { drizzle, type PostgresJsDatabase } from 'drizzle-orm/postgres-js'
|
|
7
|
+
import { StartError } from '../data'
|
|
8
|
+
import { gate, register, statements, touch, type DbHandle, type Migration } from './core'
|
|
9
|
+
|
|
10
|
+
/** `createDb` on Postgres: the schema, the URL, and whether to migrate on first use. */
|
|
11
|
+
export interface PostgresOptions<S extends Record<string, unknown>> {
|
|
12
|
+
schema: S
|
|
13
|
+
/** `postgres://user:pass@host:5432/db`. */
|
|
14
|
+
url: string
|
|
15
|
+
/** Apply `drizzle/` before the first statement a request runs (default true). */
|
|
16
|
+
migrate?: boolean
|
|
17
|
+
/** Pool size (default 10). */
|
|
18
|
+
max?: number
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** Drizzle on postgres.js, with `$weft` for the framework's own needs. */
|
|
22
|
+
export type PostgresDb<S extends Record<string, unknown>> = PostgresJsDatabase<S> & { $weft: DbHandle }
|
|
23
|
+
|
|
24
|
+
const LOCK = 7_301_986
|
|
25
|
+
|
|
26
|
+
/** A pooled Drizzle client on Postgres, with this app's migrations and write tracking. */
|
|
27
|
+
export function createDb<S extends Record<string, unknown>>({ schema, url, migrate = true, max = 10 }: PostgresOptions<S>): PostgresDb<S> {
|
|
28
|
+
if (!url && process.env.WEFT_BUILD) url = 'postgres://localhost/build' // a build loads this module and never connects
|
|
29
|
+
if (!/^postgres(ql)?:\/\//.test(url ?? ''))
|
|
30
|
+
throw new StartError('Q013', `\`@weft-js/start/db/postgres\` needs a \`postgres://\` URL, not \`${(url ?? '').replace(/\/\/.*@/, '//…@')}\``, 'set `DATABASE_URL`, or `createDb` from `@weft-js/start/db/sqlite` for a file')
|
|
31
|
+
// Idle connections close, so a stopped server can exit.
|
|
32
|
+
const client = postgres(url, { max, idle_timeout: 20, onnotice: () => {} })
|
|
33
|
+
const handle: DbHandle = {
|
|
34
|
+
dialect: 'postgresql',
|
|
35
|
+
exempt: new Set(),
|
|
36
|
+
async migrate(list: Migration[]) {
|
|
37
|
+
await client.begin(async tx => {
|
|
38
|
+
await tx.unsafe(`select pg_advisory_xact_lock(${LOCK})`)
|
|
39
|
+
await tx.unsafe('create table if not exists __weft_migrations (tag text primary key, at timestamptz not null default now())')
|
|
40
|
+
const done = new Set((await tx.unsafe('select tag from __weft_migrations')).map(r => r.tag))
|
|
41
|
+
for (const m of list) {
|
|
42
|
+
if (done.has(m.tag)) continue
|
|
43
|
+
for (const s of statements(m.sql)) await tx.unsafe(s)
|
|
44
|
+
await tx.unsafe('insert into __weft_migrations (tag) values ($1)', [m.tag])
|
|
45
|
+
}
|
|
46
|
+
})
|
|
47
|
+
},
|
|
48
|
+
changes: (tables, o) => import('./changes-postgres').then(m => m.postgresChanges({ ...o, url, client, tables })),
|
|
49
|
+
}
|
|
50
|
+
const ready = gate(handle, migrate)
|
|
51
|
+
// All Drizzle sends goes through `unsafe` and `begin`, so holding those holds it; `handle.migrate` keeps the plain client.
|
|
52
|
+
const held = new Proxy(client, {
|
|
53
|
+
get(t, key) {
|
|
54
|
+
if (key === 'begin') return async (...a: Parameters<typeof t.begin>) => (await ready(), t.begin(...a))
|
|
55
|
+
if (key !== 'unsafe') return Reflect.get(t, key)
|
|
56
|
+
return (...a: Parameters<typeof t.unsafe>) => {
|
|
57
|
+
const after = <T>(send: () => PromiseLike<T>) => Promise.resolve(ready()).then(send)
|
|
58
|
+
// oxlint-disable-next-line unicorn/no-thenable -- postgres.js's query is a lazy thenable, and Drizzle awaits it as one
|
|
59
|
+
return { values: () => after(() => t.unsafe(...a).values()), then: <A, B>(ok: (rows: unknown) => A, no: (e: unknown) => B) => after(() => t.unsafe(...a)).then(ok, no) }
|
|
60
|
+
}
|
|
61
|
+
},
|
|
62
|
+
})
|
|
63
|
+
const db = drizzle(held, { schema, logger: { logQuery: sql => touch(handle, sql) } })
|
|
64
|
+
return register(db, handle)
|
|
65
|
+
}
|