@defrex/autobuild-postgres-store 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -0
- package/package.json +37 -0
- package/src/auth-schema.ts +238 -0
- package/src/bin.ts +17 -0
- package/src/blobs.test.ts +121 -0
- package/src/config.test.ts +174 -0
- package/src/config.ts +152 -0
- package/src/env.test.ts +48 -0
- package/src/env.ts +31 -0
- package/src/event-wait-poll-ms.test.ts +23 -0
- package/src/index.ts +41 -0
- package/src/migrate.test.ts +640 -0
- package/src/s3.ts +45 -0
- package/src/schema.ts +1099 -0
- package/src/store.live.test.ts +453 -0
- package/src/store.ts +1176 -0
- package/src/testing/concurrent-worker.ts +42 -0
- package/src/tickets.live.test.ts +49 -0
- package/src/tickets.ts +276 -0
- package/src/vercel.ts +53 -0
|
@@ -0,0 +1,640 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test'
|
|
2
|
+
import { SQL } from 'bun'
|
|
3
|
+
import { CONTRACT_T0, MemoryBlobStore } from '@defrex/autobuild/plugin-sdk'
|
|
4
|
+
|
|
5
|
+
import {
|
|
6
|
+
AUTH_SCHEMA_CHECKSUM,
|
|
7
|
+
AUTH_SCHEMA_V1_CHECKSUM,
|
|
8
|
+
AUTH_SCHEMA_V1_DDL,
|
|
9
|
+
AUTH_SCHEMA_V2_CHECKSUM,
|
|
10
|
+
AUTH_SCHEMA_V2_DDL,
|
|
11
|
+
AUTH_SCHEMA_VERSION,
|
|
12
|
+
} from './auth-schema'
|
|
13
|
+
|
|
14
|
+
type Row = Record<string, unknown>
|
|
15
|
+
import {
|
|
16
|
+
MIGRATE_COMMAND,
|
|
17
|
+
SCHEMA_CHECKSUM,
|
|
18
|
+
SCHEMA_DDL,
|
|
19
|
+
SCHEMA_V1_CHECKSUM,
|
|
20
|
+
SCHEMA_V1_DDL,
|
|
21
|
+
SCHEMA_V2_CHECKSUM,
|
|
22
|
+
SCHEMA_V2_DDL,
|
|
23
|
+
SCHEMA_V3_CHECKSUM,
|
|
24
|
+
SCHEMA_V3_DDL,
|
|
25
|
+
SCHEMA_V4_CHECKSUM,
|
|
26
|
+
SCHEMA_V4_DDL,
|
|
27
|
+
SCHEMA_V5_CHECKSUM,
|
|
28
|
+
SCHEMA_V5_DDL,
|
|
29
|
+
SCHEMA_VERSION,
|
|
30
|
+
migratePostgres,
|
|
31
|
+
} from './schema'
|
|
32
|
+
import { openPostgresBuildStore } from './store'
|
|
33
|
+
|
|
34
|
+
const testUrl = process.env.AB_POSTGRES_TEST_URL?.trim()
|
|
35
|
+
|
|
36
|
+
async function schemaHarness(): Promise<{ url: string; cleanup: () => Promise<void> }> {
|
|
37
|
+
if (!testUrl) throw new Error('AB_POSTGRES_TEST_URL is required')
|
|
38
|
+
const schema = `ab_migrate_${crypto.randomUUID().replaceAll('-', '')}`
|
|
39
|
+
const admin = new SQL(testUrl)
|
|
40
|
+
await admin.unsafe(`CREATE SCHEMA ${schema}`)
|
|
41
|
+
await admin.close()
|
|
42
|
+
const scoped = new URL(testUrl)
|
|
43
|
+
scoped.searchParams.set('options', `-csearch_path=${schema}`)
|
|
44
|
+
return {
|
|
45
|
+
url: scoped.toString(),
|
|
46
|
+
cleanup: async () => {
|
|
47
|
+
const sql = new SQL(testUrl)
|
|
48
|
+
try {
|
|
49
|
+
await sql.unsafe(`DROP SCHEMA ${schema} CASCADE`)
|
|
50
|
+
} finally {
|
|
51
|
+
await sql.close()
|
|
52
|
+
}
|
|
53
|
+
},
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
if (testUrl) {
|
|
58
|
+
describe('PostgreSQL schema migration', () => {
|
|
59
|
+
test('is repeatable and serializes concurrent initializers', async () => {
|
|
60
|
+
const harness = await schemaHarness()
|
|
61
|
+
try {
|
|
62
|
+
await Promise.all([migratePostgres(harness.url), migratePostgres(harness.url)])
|
|
63
|
+
await migratePostgres(harness.url)
|
|
64
|
+
const store = await openPostgresBuildStore(harness.url, new MemoryBlobStore())
|
|
65
|
+
await store.close()
|
|
66
|
+
} finally {
|
|
67
|
+
await harness.cleanup()
|
|
68
|
+
}
|
|
69
|
+
})
|
|
70
|
+
|
|
71
|
+
for (const scenario of [
|
|
72
|
+
{
|
|
73
|
+
name: 'older-with-a-foreign-checksum',
|
|
74
|
+
version: SCHEMA_VERSION - 1,
|
|
75
|
+
checksum: SCHEMA_CHECKSUM,
|
|
76
|
+
},
|
|
77
|
+
{ name: 'newer', version: SCHEMA_VERSION + 1, checksum: SCHEMA_CHECKSUM },
|
|
78
|
+
{ name: 'checksum-mismatched', version: SCHEMA_VERSION, checksum: 'wrong' },
|
|
79
|
+
]) {
|
|
80
|
+
test(`rejects an ${scenario.name} marker`, async () => {
|
|
81
|
+
const harness = await schemaHarness()
|
|
82
|
+
const sql = new SQL(harness.url)
|
|
83
|
+
try {
|
|
84
|
+
await sql`CREATE TABLE ab_schema_migrations (
|
|
85
|
+
singleton boolean PRIMARY KEY, version integer NOT NULL,
|
|
86
|
+
checksum text NOT NULL, applied_at timestamptz NOT NULL
|
|
87
|
+
)`
|
|
88
|
+
await sql`INSERT INTO ab_schema_migrations VALUES
|
|
89
|
+
(true, ${scenario.version}, ${scenario.checksum}, ${new Date().toISOString()})`
|
|
90
|
+
const error = await openPostgresBuildStore(harness.url, new MemoryBlobStore()).catch(
|
|
91
|
+
(caught: unknown) => caught,
|
|
92
|
+
)
|
|
93
|
+
expect(error).toBeInstanceOf(Error)
|
|
94
|
+
expect((error as Error).message).toContain(MIGRATE_COMMAND)
|
|
95
|
+
await expect(migratePostgres(harness.url)).rejects.toThrow('incompatible')
|
|
96
|
+
} finally {
|
|
97
|
+
await sql.close()
|
|
98
|
+
await harness.cleanup()
|
|
99
|
+
}
|
|
100
|
+
})
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
test('upgrades a genuine v1 database in place, preserving prior rows and enabling streams', async () => {
|
|
104
|
+
const harness = await schemaHarness()
|
|
105
|
+
const sql = new SQL(harness.url)
|
|
106
|
+
try {
|
|
107
|
+
// Create a real v1 database: v1 DDL, v1 marker, plus a build, an
|
|
108
|
+
// event, and an artifact written before streams existed.
|
|
109
|
+
await sql.unsafe(SCHEMA_V1_DDL)
|
|
110
|
+
await sql`INSERT INTO ab_schema_migrations VALUES
|
|
111
|
+
(true, 1, ${SCHEMA_V1_CHECKSUM}, ${new Date().toISOString()})`
|
|
112
|
+
await sql`INSERT INTO builds (slug, repo, created_at, updated_at)
|
|
113
|
+
VALUES ('v1-build', 'acme/v1', ${CONTRACT_T0}, ${CONTRACT_T0})`
|
|
114
|
+
await sql`INSERT INTO events (build, seq, ts, actor, type, payload)
|
|
115
|
+
VALUES ('v1-build', 1, ${CONTRACT_T0}, '{"kind":"dispatcher"}', 'build.created',
|
|
116
|
+
'{"ticket":{"source":"linear","id":"TICK-1"},"repo":"acme/v1","baseBranch":"main"}')`
|
|
117
|
+
await sql`INSERT INTO artifacts (build, kind, revision, blob_ref, metadata, created_at)
|
|
118
|
+
VALUES ('v1-build', 'plan', 0, 'deadbeef', '{}', ${CONTRACT_T0})`
|
|
119
|
+
|
|
120
|
+
await migratePostgres(harness.url)
|
|
121
|
+
|
|
122
|
+
const marker = await sql`SELECT version, checksum FROM ab_schema_migrations`
|
|
123
|
+
expect(Number(marker[0]?.version)).toBe(SCHEMA_VERSION)
|
|
124
|
+
expect(marker[0]?.checksum).toBe(SCHEMA_CHECKSUM)
|
|
125
|
+
|
|
126
|
+
// Prior rows survive untouched.
|
|
127
|
+
const builds = await sql`SELECT slug FROM builds`
|
|
128
|
+
expect(builds.map((row: Row) => row.slug)).toEqual(['v1-build'])
|
|
129
|
+
const events = await sql`SELECT seq FROM events WHERE build = 'v1-build'`
|
|
130
|
+
expect(events).toHaveLength(1)
|
|
131
|
+
|
|
132
|
+
// The stream tables work end to end through the migrated store.
|
|
133
|
+
const store = await openPostgresBuildStore(harness.url, new MemoryBlobStore())
|
|
134
|
+
try {
|
|
135
|
+
const stream = await store.createStream({ kind: 'build', build: 'v1-build' }, 'migrated')
|
|
136
|
+
await store.appendStreamParts(stream.id, [
|
|
137
|
+
{ type: 'start', messageId: 'm' },
|
|
138
|
+
{ type: 'text-start', id: 't' },
|
|
139
|
+
{ type: 'text-delta', id: 't', delta: 'survived' },
|
|
140
|
+
{ type: 'text-end', id: 't' },
|
|
141
|
+
])
|
|
142
|
+
const closed = await store.closeStream(stream.id, 'completed')
|
|
143
|
+
expect(closed.status).toBe('closed')
|
|
144
|
+
const read = await store.readStream(stream.id)
|
|
145
|
+
expect(read.chunks).toHaveLength(1)
|
|
146
|
+
expect(read.status).toBe('closed')
|
|
147
|
+
expect(read.outcome).toBe('completed')
|
|
148
|
+
} finally {
|
|
149
|
+
await store.close()
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// The upgrade is idempotent.
|
|
153
|
+
await migratePostgres(harness.url)
|
|
154
|
+
} finally {
|
|
155
|
+
await sql.close()
|
|
156
|
+
await harness.cleanup()
|
|
157
|
+
}
|
|
158
|
+
})
|
|
159
|
+
|
|
160
|
+
test('upgrades a genuine v2 database in place: session tables, the streams.session column, and widened CHECKs, preserving prior rows', async () => {
|
|
161
|
+
const harness = await schemaHarness()
|
|
162
|
+
const sql = new SQL(harness.url)
|
|
163
|
+
try {
|
|
164
|
+
// Create a real v2 database: v2 DDL, v2 marker, plus a build and a
|
|
165
|
+
// closed build-scoped stream written before sessions existed.
|
|
166
|
+
await sql.unsafe(SCHEMA_V2_DDL)
|
|
167
|
+
// The genuine v2 marker is version 2 literally: SCHEMA_VERSION moves
|
|
168
|
+
// on with every schema revision, and a v2 checksum under any other
|
|
169
|
+
// version is (correctly) rejected as incompatible.
|
|
170
|
+
await sql`INSERT INTO ab_schema_migrations VALUES
|
|
171
|
+
(true, 2, ${SCHEMA_V2_CHECKSUM}, ${new Date().toISOString()})`
|
|
172
|
+
await sql`INSERT INTO builds (slug, repo, created_at, updated_at)
|
|
173
|
+
VALUES ('v2-build', 'acme/v2', ${CONTRACT_T0}, ${CONTRACT_T0})`
|
|
174
|
+
await sql`INSERT INTO events (build, seq, ts, actor, type, payload)
|
|
175
|
+
VALUES ('v2-build', 1, ${CONTRACT_T0}, '{"kind":"dispatcher"}', 'build.created',
|
|
176
|
+
'{"ticket":{"source":"linear","id":"TICK-1"},"repo":"acme/v2","baseBranch":"main"}')`
|
|
177
|
+
await sql`INSERT INTO streams (id, scope_kind, build, label, format, status, created_at)
|
|
178
|
+
VALUES ('st_v2', 'build', 'v2-build', 'before', 'ai-ui-message-stream/v1', 'open', ${CONTRACT_T0})`
|
|
179
|
+
|
|
180
|
+
await migratePostgres(harness.url)
|
|
181
|
+
|
|
182
|
+
const marker = await sql`SELECT version, checksum FROM ab_schema_migrations`
|
|
183
|
+
expect(Number(marker[0]?.version)).toBe(SCHEMA_VERSION)
|
|
184
|
+
expect(marker[0]?.checksum).toBe(SCHEMA_CHECKSUM)
|
|
185
|
+
|
|
186
|
+
// Prior rows survive untouched.
|
|
187
|
+
const events = await sql`SELECT seq FROM events WHERE build = 'v2-build'`
|
|
188
|
+
expect(events).toHaveLength(1)
|
|
189
|
+
const oldStream = await sql`SELECT scope_kind, session FROM streams WHERE id = 'st_v2'`
|
|
190
|
+
expect(oldStream[0]?.scope_kind).toBe('build')
|
|
191
|
+
expect(oldStream[0]?.session).toBeNull()
|
|
192
|
+
|
|
193
|
+
// The migrated store works end to end, including sessions and
|
|
194
|
+
// session-scoped streams.
|
|
195
|
+
const store = await openPostgresBuildStore(harness.url, new MemoryBlobStore())
|
|
196
|
+
try {
|
|
197
|
+
const session = await store.createSession({ repo: 'acme/v2', operator: 'op' })
|
|
198
|
+
await store.appendSessionEvent(session.id, {
|
|
199
|
+
actor: { kind: 'human', user: 'op' },
|
|
200
|
+
type: 'message.posted',
|
|
201
|
+
payload: { text: 'hello' },
|
|
202
|
+
})
|
|
203
|
+
const stream = await store.createStream(
|
|
204
|
+
{ kind: 'session', session: session.id },
|
|
205
|
+
'migrated turn',
|
|
206
|
+
)
|
|
207
|
+
await store.appendStreamParts(stream.id, [{ type: 'text-delta', id: 't', delta: 'x' }])
|
|
208
|
+
const closed = await store.closeStream(stream.id, 'completed')
|
|
209
|
+
expect(closed.status).toBe('closed')
|
|
210
|
+
expect(
|
|
211
|
+
(await store.getSessionArtifact(session.id, `stream:${stream.id}`))?.meta.revision,
|
|
212
|
+
).toBe(0)
|
|
213
|
+
} finally {
|
|
214
|
+
await store.close()
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// The upgrade is idempotent.
|
|
218
|
+
await migratePostgres(harness.url)
|
|
219
|
+
} finally {
|
|
220
|
+
await sql.close()
|
|
221
|
+
await harness.cleanup()
|
|
222
|
+
}
|
|
223
|
+
})
|
|
224
|
+
|
|
225
|
+
test('upgrades a genuine v3 database in place: the sessions.creation_seq backfill, sequence continuity, and preserving prior rows', async () => {
|
|
226
|
+
const harness = await schemaHarness()
|
|
227
|
+
const sql = new SQL(harness.url)
|
|
228
|
+
try {
|
|
229
|
+
// Create a real v3 database: v3 DDL, v3 marker, plus two
|
|
230
|
+
// same-millisecond sessions whose id order is reversed from insertion
|
|
231
|
+
// order. Legacy ties are genuinely unorderable, so the pinned
|
|
232
|
+
// insertion order is not asserted — the backfill assigns a distinct,
|
|
233
|
+
// deterministic (created_at, id)-ordered counter per row and
|
|
234
|
+
// otherwise leaves the rows untouched.
|
|
235
|
+
await sql.unsafe(SCHEMA_V3_DDL)
|
|
236
|
+
// The genuine v3 marker is version 3 literally: SCHEMA_VERSION moves
|
|
237
|
+
// on with every schema revision, and a v3 checksum under any other
|
|
238
|
+
// version is (correctly) rejected as incompatible. The genuine v4
|
|
239
|
+
// fixture below covers the streams.creation_seq backfill; on the
|
|
240
|
+
// next schema revision, add a genuine v5 fixture and retire or
|
|
241
|
+
// re-pin this one.
|
|
242
|
+
await sql`INSERT INTO ab_schema_migrations VALUES
|
|
243
|
+
(true, 3, ${SCHEMA_V3_CHECKSUM}, ${new Date().toISOString()})`
|
|
244
|
+
for (const id of ['os_legacy-2', 'os_legacy-1']) {
|
|
245
|
+
await sql`INSERT INTO sessions (id, repo, operator, created_at, updated_at)
|
|
246
|
+
VALUES (${id}, 'acme/v3', 'op', ${CONTRACT_T0}, ${CONTRACT_T0})`
|
|
247
|
+
await sql`INSERT INTO session_events (session, seq, ts, actor, type, payload)
|
|
248
|
+
VALUES (${id}, 1, ${CONTRACT_T0}, '{"kind":"human","user":"op"}', 'session.created', '{}')`
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
await migratePostgres(harness.url)
|
|
252
|
+
|
|
253
|
+
const marker = await sql`SELECT version, checksum FROM ab_schema_migrations`
|
|
254
|
+
expect(Number(marker[0]?.version)).toBe(SCHEMA_VERSION)
|
|
255
|
+
expect(marker[0]?.checksum).toBe(SCHEMA_CHECKSUM)
|
|
256
|
+
|
|
257
|
+
// Both rows survived with distinct backfilled counters, ordered by
|
|
258
|
+
// (created_at, id) — the lexicographically smaller id lands first
|
|
259
|
+
// despite being inserted second.
|
|
260
|
+
const backfilled = await sql`SELECT id, creation_seq FROM sessions ORDER BY creation_seq`
|
|
261
|
+
expect(backfilled.map((row: Row) => [row.id, Number(row.creation_seq)])).toEqual([
|
|
262
|
+
['os_legacy-1', 1],
|
|
263
|
+
['os_legacy-2', 2],
|
|
264
|
+
])
|
|
265
|
+
|
|
266
|
+
// The migrated store works end to end: the sequence continues above
|
|
267
|
+
// the backfill, so a same-millisecond post-migration creation gets a
|
|
268
|
+
// fresh counter and sorts after both migrated rows.
|
|
269
|
+
const store = await openPostgresBuildStore(harness.url, new MemoryBlobStore(), {
|
|
270
|
+
clock: () => new Date(CONTRACT_T0),
|
|
271
|
+
})
|
|
272
|
+
try {
|
|
273
|
+
const created = await store.createSession({ repo: 'acme/v3', operator: 'op' })
|
|
274
|
+
expect(created.createdAt).toBe(CONTRACT_T0)
|
|
275
|
+
const seqs = await sql`SELECT id, creation_seq FROM sessions ORDER BY creation_seq`
|
|
276
|
+
expect(seqs.map((row: Row) => row.id)).toEqual(['os_legacy-1', 'os_legacy-2', created.id])
|
|
277
|
+
expect((await store.listSessions('acme/v3')).map((s) => s.id)).toEqual([
|
|
278
|
+
'os_legacy-1',
|
|
279
|
+
'os_legacy-2',
|
|
280
|
+
created.id,
|
|
281
|
+
])
|
|
282
|
+
await store.appendSessionEvent(created.id, {
|
|
283
|
+
actor: { kind: 'human', user: 'op' },
|
|
284
|
+
type: 'message.posted',
|
|
285
|
+
payload: { text: 'hello' },
|
|
286
|
+
})
|
|
287
|
+
} finally {
|
|
288
|
+
await store.close()
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// The upgrade is idempotent.
|
|
292
|
+
await migratePostgres(harness.url)
|
|
293
|
+
} finally {
|
|
294
|
+
await sql.close()
|
|
295
|
+
await harness.cleanup()
|
|
296
|
+
}
|
|
297
|
+
})
|
|
298
|
+
|
|
299
|
+
test('upgrades a genuine v4 database in place: the streams.creation_seq backfill, sequence continuity, and preserving prior rows', async () => {
|
|
300
|
+
const harness = await schemaHarness()
|
|
301
|
+
const sql = new SQL(harness.url)
|
|
302
|
+
try {
|
|
303
|
+
// Create a real v4 database: v4 DDL, v4 marker, plus two
|
|
304
|
+
// same-millisecond build-scoped streams whose id order is reversed
|
|
305
|
+
// from insertion order. Legacy ties are genuinely unorderable, so
|
|
306
|
+
// the pinned insertion order is not asserted — the backfill assigns
|
|
307
|
+
// a distinct, deterministic (created_at, id)-ordered counter per row
|
|
308
|
+
// and otherwise leaves the rows untouched.
|
|
309
|
+
await sql.unsafe(SCHEMA_V4_DDL)
|
|
310
|
+
// The genuine v4 marker is version 4 literally: SCHEMA_VERSION moves
|
|
311
|
+
// on with every schema revision, and a v4 checksum under any other
|
|
312
|
+
// version is (correctly) rejected as incompatible.
|
|
313
|
+
await sql`INSERT INTO ab_schema_migrations VALUES
|
|
314
|
+
(true, 4, ${SCHEMA_V4_CHECKSUM}, ${new Date().toISOString()})`
|
|
315
|
+
await sql`INSERT INTO builds (slug, repo, created_at, updated_at)
|
|
316
|
+
VALUES ('v4-build', 'acme/v4', ${CONTRACT_T0}, ${CONTRACT_T0})`
|
|
317
|
+
for (const id of ['st_legacy-2', 'st_legacy-1']) {
|
|
318
|
+
await sql`INSERT INTO streams (id, scope_kind, build, label, format, status, created_at)
|
|
319
|
+
VALUES (${id}, 'build', 'v4-build', 'before', 'ai-ui-message-stream/v1', 'open', ${CONTRACT_T0})`
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
await migratePostgres(harness.url)
|
|
323
|
+
|
|
324
|
+
const marker = await sql`SELECT version, checksum FROM ab_schema_migrations`
|
|
325
|
+
expect(Number(marker[0]?.version)).toBe(SCHEMA_VERSION)
|
|
326
|
+
expect(marker[0]?.checksum).toBe(SCHEMA_CHECKSUM)
|
|
327
|
+
|
|
328
|
+
// Both rows survived with distinct backfilled counters, ordered by
|
|
329
|
+
// (created_at, id) — the lexicographically smaller id lands first
|
|
330
|
+
// despite being inserted second.
|
|
331
|
+
const backfilled = await sql`SELECT id, creation_seq FROM streams ORDER BY creation_seq`
|
|
332
|
+
expect(backfilled.map((row: Row) => [row.id, Number(row.creation_seq)])).toEqual([
|
|
333
|
+
['st_legacy-1', 1],
|
|
334
|
+
['st_legacy-2', 2],
|
|
335
|
+
])
|
|
336
|
+
|
|
337
|
+
// The migrated store works end to end: the sequence continues above
|
|
338
|
+
// the backfill, so a same-millisecond post-migration creation gets a
|
|
339
|
+
// fresh counter and sorts after both migrated rows.
|
|
340
|
+
const store = await openPostgresBuildStore(harness.url, new MemoryBlobStore(), {
|
|
341
|
+
clock: () => new Date(CONTRACT_T0),
|
|
342
|
+
})
|
|
343
|
+
try {
|
|
344
|
+
const created = await store.createStream({ kind: 'build', build: 'v4-build' }, 'after')
|
|
345
|
+
expect(created.createdAt).toBe(CONTRACT_T0)
|
|
346
|
+
const seqs = await sql`SELECT id, creation_seq FROM streams ORDER BY creation_seq`
|
|
347
|
+
expect(seqs.map((row: Row) => row.id)).toEqual(['st_legacy-1', 'st_legacy-2', created.id])
|
|
348
|
+
expect(
|
|
349
|
+
(await store.listStreams({ kind: 'build', build: 'v4-build' })).map((s) => s.id),
|
|
350
|
+
).toEqual(['st_legacy-1', 'st_legacy-2', created.id])
|
|
351
|
+
await store.appendStreamParts(created.id, [{ type: 'text-delta', id: 't', delta: 'x' }])
|
|
352
|
+
} finally {
|
|
353
|
+
await store.close()
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
// The upgrade is idempotent.
|
|
357
|
+
await migratePostgres(harness.url)
|
|
358
|
+
} finally {
|
|
359
|
+
await sql.close()
|
|
360
|
+
await harness.cleanup()
|
|
361
|
+
}
|
|
362
|
+
})
|
|
363
|
+
|
|
364
|
+
test('upgrades a genuine v5 database in place: the streams.creation_seq backfill the hosted database is missing, sequence continuity, and preserving prior rows', async () => {
|
|
365
|
+
const harness = await schemaHarness()
|
|
366
|
+
const sql = new SQL(harness.url)
|
|
367
|
+
try {
|
|
368
|
+
// Create a real v5 database — the shape the hosted service deployed
|
|
369
|
+
// before the tiebreak landed without a version bump: v5 DDL, v5
|
|
370
|
+
// marker, plus two
|
|
371
|
+
// same-millisecond build-scoped streams whose id order is reversed
|
|
372
|
+
// from insertion order. Legacy ties are genuinely unorderable, so
|
|
373
|
+
// the pinned insertion order is not asserted — the backfill assigns
|
|
374
|
+
// a distinct, deterministic (created_at, id)-ordered counter per row
|
|
375
|
+
// and otherwise leaves the rows untouched.
|
|
376
|
+
await sql.unsafe(SCHEMA_V5_DDL)
|
|
377
|
+
// The genuine v5 marker is version 5 literally: SCHEMA_VERSION moves
|
|
378
|
+
// on with every schema revision, and a v5 checksum under any other
|
|
379
|
+
// version is (correctly) rejected as incompatible.
|
|
380
|
+
await sql`INSERT INTO ab_schema_migrations VALUES
|
|
381
|
+
(true, 5, ${SCHEMA_V5_CHECKSUM}, ${new Date().toISOString()})`
|
|
382
|
+
await sql`INSERT INTO builds (slug, repo, created_at, updated_at)
|
|
383
|
+
VALUES ('v5-build', 'acme/v5', ${CONTRACT_T0}, ${CONTRACT_T0})`
|
|
384
|
+
for (const id of ['st_legacy-2', 'st_legacy-1']) {
|
|
385
|
+
await sql`INSERT INTO streams (id, scope_kind, build, label, format, status, created_at)
|
|
386
|
+
VALUES (${id}, 'build', 'v5-build', 'before', 'ai-ui-message-stream/v1', 'open', ${CONTRACT_T0})`
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
await migratePostgres(harness.url)
|
|
390
|
+
|
|
391
|
+
const marker = await sql`SELECT version, checksum FROM ab_schema_migrations`
|
|
392
|
+
expect(Number(marker[0]?.version)).toBe(SCHEMA_VERSION)
|
|
393
|
+
expect(marker[0]?.checksum).toBe(SCHEMA_CHECKSUM)
|
|
394
|
+
|
|
395
|
+
// Both rows survived with distinct backfilled counters, ordered by
|
|
396
|
+
// (created_at, id) — the lexicographically smaller id lands first
|
|
397
|
+
// despite being inserted second.
|
|
398
|
+
const backfilled = await sql`SELECT id, creation_seq FROM streams ORDER BY creation_seq`
|
|
399
|
+
expect(backfilled.map((row: Row) => [row.id, Number(row.creation_seq)])).toEqual([
|
|
400
|
+
['st_legacy-1', 1],
|
|
401
|
+
['st_legacy-2', 2],
|
|
402
|
+
])
|
|
403
|
+
|
|
404
|
+
// The migrated store works end to end: the sequence continues above
|
|
405
|
+
// the backfill, so a same-millisecond post-migration creation gets a
|
|
406
|
+
// fresh counter and sorts after both migrated rows.
|
|
407
|
+
const store = await openPostgresBuildStore(harness.url, new MemoryBlobStore(), {
|
|
408
|
+
clock: () => new Date(CONTRACT_T0),
|
|
409
|
+
})
|
|
410
|
+
try {
|
|
411
|
+
const created = await store.createStream({ kind: 'build', build: 'v5-build' }, 'after')
|
|
412
|
+
expect(created.createdAt).toBe(CONTRACT_T0)
|
|
413
|
+
const seqs = await sql`SELECT id, creation_seq FROM streams ORDER BY creation_seq`
|
|
414
|
+
expect(seqs.map((row: Row) => row.id)).toEqual(['st_legacy-1', 'st_legacy-2', created.id])
|
|
415
|
+
expect(
|
|
416
|
+
(await store.listStreams({ kind: 'build', build: 'v5-build' })).map((s) => s.id),
|
|
417
|
+
).toEqual(['st_legacy-1', 'st_legacy-2', created.id])
|
|
418
|
+
await store.appendStreamParts(created.id, [{ type: 'text-delta', id: 't', delta: 'x' }])
|
|
419
|
+
} finally {
|
|
420
|
+
await store.close()
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
// The upgrade is idempotent.
|
|
424
|
+
await migratePostgres(harness.url)
|
|
425
|
+
} finally {
|
|
426
|
+
await sql.close()
|
|
427
|
+
await harness.cleanup()
|
|
428
|
+
}
|
|
429
|
+
})
|
|
430
|
+
|
|
431
|
+
test('upgrades a genuine v4 database in place: the builds.repo_origin column, preserving prior rows', async () => {
|
|
432
|
+
const harness = await schemaHarness()
|
|
433
|
+
const sql = new SQL(harness.url)
|
|
434
|
+
try {
|
|
435
|
+
// Create a real v4 database: v4 DDL, v4 marker, plus a build written
|
|
436
|
+
// before repoOrigin was persisted — legacy rows legitimately have no
|
|
437
|
+
// origin, so the column must backfill as NULL ("field absent").
|
|
438
|
+
await sql.unsafe(SCHEMA_V4_DDL)
|
|
439
|
+
// The genuine v4 marker is version 4 literally: SCHEMA_VERSION moves
|
|
440
|
+
// on with every schema revision, and a v4 checksum under any other
|
|
441
|
+
// version is (correctly) rejected as incompatible.
|
|
442
|
+
await sql`INSERT INTO ab_schema_migrations VALUES
|
|
443
|
+
(true, 4, ${SCHEMA_V4_CHECKSUM}, ${new Date().toISOString()})`
|
|
444
|
+
await sql`INSERT INTO builds (slug, repo, created_at, updated_at)
|
|
445
|
+
VALUES ('v4-build', 'acme/v4', ${CONTRACT_T0}, ${CONTRACT_T0})`
|
|
446
|
+
|
|
447
|
+
await migratePostgres(harness.url)
|
|
448
|
+
|
|
449
|
+
const marker = await sql`SELECT version, checksum FROM ab_schema_migrations`
|
|
450
|
+
expect(Number(marker[0]?.version)).toBe(SCHEMA_VERSION)
|
|
451
|
+
expect(marker[0]?.checksum).toBe(SCHEMA_CHECKSUM)
|
|
452
|
+
|
|
453
|
+
// The legacy row survives with repo_origin NULL — the read path
|
|
454
|
+
// treats NULL as "field absent".
|
|
455
|
+
const legacy = await sql`SELECT repo_origin FROM builds WHERE slug = 'v4-build'`
|
|
456
|
+
expect(legacy[0]?.repo_origin).toBeNull()
|
|
457
|
+
|
|
458
|
+
// The migrated store persists repoOrigin both ways: with an origin
|
|
459
|
+
// the value round-trips; without one the field stays absent.
|
|
460
|
+
const store = await openPostgresBuildStore(harness.url, new MemoryBlobStore())
|
|
461
|
+
try {
|
|
462
|
+
const origin = 'https://github.com/acme/v4.git'
|
|
463
|
+
const withOrigin = await store.createBuild({
|
|
464
|
+
slug: 'v4-origin-set',
|
|
465
|
+
repo: 'acme/v4',
|
|
466
|
+
repoOrigin: origin,
|
|
467
|
+
})
|
|
468
|
+
expect(withOrigin.repoOrigin).toBe(origin)
|
|
469
|
+
expect((await store.getBuild('v4-origin-set'))?.repoOrigin).toBe(origin)
|
|
470
|
+
expect(
|
|
471
|
+
(await store.listBuilds()).find((build) => build.slug === 'v4-origin-set')?.repoOrigin,
|
|
472
|
+
).toBe(origin)
|
|
473
|
+
|
|
474
|
+
await store.createBuild({ slug: 'v4-origin-absent', repo: 'acme/v4' })
|
|
475
|
+
expect((await store.getBuild('v4-origin-absent'))?.repoOrigin).toBeUndefined()
|
|
476
|
+
} finally {
|
|
477
|
+
await store.close()
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
// The upgrade is idempotent.
|
|
481
|
+
await migratePostgres(harness.url)
|
|
482
|
+
} finally {
|
|
483
|
+
await sql.close()
|
|
484
|
+
await harness.cleanup()
|
|
485
|
+
}
|
|
486
|
+
})
|
|
487
|
+
|
|
488
|
+
test('refuses a current marker when a required table is missing', async () => {
|
|
489
|
+
const harness = await schemaHarness()
|
|
490
|
+
const sql = new SQL(harness.url)
|
|
491
|
+
try {
|
|
492
|
+
await migratePostgres(harness.url)
|
|
493
|
+
await sql`DROP TABLE events`
|
|
494
|
+
const error = await openPostgresBuildStore(harness.url, new MemoryBlobStore()).catch(
|
|
495
|
+
(caught: unknown) => caught,
|
|
496
|
+
)
|
|
497
|
+
expect((error as Error).message).toContain('table events is missing or mismatched')
|
|
498
|
+
expect((error as Error).message).toContain(MIGRATE_COMMAND)
|
|
499
|
+
|
|
500
|
+
await migratePostgres(harness.url)
|
|
501
|
+
const repaired = await openPostgresBuildStore(harness.url, new MemoryBlobStore())
|
|
502
|
+
await repaired.close()
|
|
503
|
+
} finally {
|
|
504
|
+
await sql.close()
|
|
505
|
+
await harness.cleanup()
|
|
506
|
+
}
|
|
507
|
+
})
|
|
508
|
+
|
|
509
|
+
test('refuses a current marker when a required column is altered', async () => {
|
|
510
|
+
const harness = await schemaHarness()
|
|
511
|
+
const sql = new SQL(harness.url)
|
|
512
|
+
try {
|
|
513
|
+
await migratePostgres(harness.url)
|
|
514
|
+
await sql`ALTER TABLE events ALTER COLUMN type TYPE varchar(100)`
|
|
515
|
+
const error = await openPostgresBuildStore(harness.url, new MemoryBlobStore()).catch(
|
|
516
|
+
(caught: unknown) => caught,
|
|
517
|
+
)
|
|
518
|
+
expect((error as Error).message).toContain('table events is missing or mismatched')
|
|
519
|
+
expect((error as Error).message).toContain(MIGRATE_COMMAND)
|
|
520
|
+
await expect(migratePostgres(harness.url)).rejects.toThrow('table events')
|
|
521
|
+
} finally {
|
|
522
|
+
await sql.close()
|
|
523
|
+
await harness.cleanup()
|
|
524
|
+
}
|
|
525
|
+
})
|
|
526
|
+
|
|
527
|
+
test('upgrades a genuine v1 auth database in place: the four MCP-plugin tables, preserving prior rows', async () => {
|
|
528
|
+
const harness = await schemaHarness()
|
|
529
|
+
const sql = new SQL(harness.url)
|
|
530
|
+
try {
|
|
531
|
+
// Create a real v1 auth database: v1 DDL, a v1 marker for both the
|
|
532
|
+
// build-store and auth markers (migratePostgres validates both), plus
|
|
533
|
+
// a user written before the MCP tables existed.
|
|
534
|
+
await sql.unsafe(SCHEMA_V1_DDL)
|
|
535
|
+
await sql.unsafe(AUTH_SCHEMA_V1_DDL)
|
|
536
|
+
await sql`INSERT INTO ab_schema_migrations VALUES
|
|
537
|
+
(true, 1, ${SCHEMA_V1_CHECKSUM}, ${new Date().toISOString()})`
|
|
538
|
+
await sql`INSERT INTO ab_auth_schema_migrations VALUES
|
|
539
|
+
(true, 1, ${AUTH_SCHEMA_V1_CHECKSUM}, ${new Date().toISOString()})`
|
|
540
|
+
await sql`INSERT INTO "user" (id, name, email, "emailVerified", "createdAt", "updatedAt")
|
|
541
|
+
VALUES ('u1', 'Ada', 'ada@example.com', true, ${CONTRACT_T0}, ${CONTRACT_T0})`
|
|
542
|
+
|
|
543
|
+
await migratePostgres(harness.url)
|
|
544
|
+
|
|
545
|
+
const marker = await sql`SELECT version, checksum FROM ab_auth_schema_migrations`
|
|
546
|
+
expect(Number(marker[0]?.version)).toBe(AUTH_SCHEMA_VERSION)
|
|
547
|
+
expect(marker[0]?.checksum).toBe(AUTH_SCHEMA_CHECKSUM)
|
|
548
|
+
|
|
549
|
+
// Prior rows survive untouched.
|
|
550
|
+
const users = await sql`SELECT id FROM "user"`
|
|
551
|
+
expect(users.map((row: Row) => row.id)).toEqual(['u1'])
|
|
552
|
+
|
|
553
|
+
// The new tables work: a dynamically registered client row inserts
|
|
554
|
+
// and reads back with the exact camelCase columns the plugin writes,
|
|
555
|
+
// including the v3 authenticationScheme column.
|
|
556
|
+
await sql`INSERT INTO "oauthApplication"
|
|
557
|
+
(id, name, "clientId", "clientSecret", "redirectUrls", type, disabled,
|
|
558
|
+
"createdAt", "updatedAt", "authenticationScheme")
|
|
559
|
+
VALUES ('c1', 'e2e-mcp-client', 'client-1', 'secret', 'https://claude.ai', 'web',
|
|
560
|
+
false, ${CONTRACT_T0}, ${CONTRACT_T0}, 'none')`
|
|
561
|
+
const clients =
|
|
562
|
+
await sql`SELECT name, "authenticationScheme" FROM "oauthApplication" WHERE "clientId" = 'client-1'`
|
|
563
|
+
expect(clients.map((row: Row) => row.name)).toEqual(['e2e-mcp-client'])
|
|
564
|
+
expect(clients[0]?.authenticationScheme).toBe('none')
|
|
565
|
+
|
|
566
|
+
// The upgrade is idempotent.
|
|
567
|
+
await migratePostgres(harness.url)
|
|
568
|
+
} finally {
|
|
569
|
+
await sql.close()
|
|
570
|
+
await harness.cleanup()
|
|
571
|
+
}
|
|
572
|
+
})
|
|
573
|
+
|
|
574
|
+
test('upgrades a genuine v2 auth database in place: the oauthApplication.authenticationScheme column, preserving prior rows', async () => {
|
|
575
|
+
const harness = await schemaHarness()
|
|
576
|
+
const sql = new SQL(harness.url)
|
|
577
|
+
try {
|
|
578
|
+
// Create a real v2 auth database: v2 DDL, a current build-store
|
|
579
|
+
// marker (migratePostgres validates both), a v2 auth marker, plus a
|
|
580
|
+
// legacy client row written before authenticationScheme existed —
|
|
581
|
+
// legacy rows legitimately lack the field, so it backfills as NULL
|
|
582
|
+
// ("field absent", the builds.repo_origin precedent).
|
|
583
|
+
await sql.unsafe(SCHEMA_DDL)
|
|
584
|
+
await sql.unsafe(AUTH_SCHEMA_V2_DDL)
|
|
585
|
+
await sql`INSERT INTO ab_schema_migrations VALUES
|
|
586
|
+
(true, ${SCHEMA_VERSION}, ${SCHEMA_CHECKSUM}, ${new Date().toISOString()})`
|
|
587
|
+
await sql`INSERT INTO ab_auth_schema_migrations VALUES
|
|
588
|
+
(true, 2, ${AUTH_SCHEMA_V2_CHECKSUM}, ${new Date().toISOString()})`
|
|
589
|
+
await sql`INSERT INTO "oauthApplication"
|
|
590
|
+
(id, name, "clientId", "clientSecret", "redirectUrls", type, disabled,
|
|
591
|
+
"createdAt", "updatedAt")
|
|
592
|
+
VALUES ('legacy', 'legacy-client', 'legacy-id', 'secret', 'https://claude.ai', 'web',
|
|
593
|
+
false, ${CONTRACT_T0}, ${CONTRACT_T0})`
|
|
594
|
+
|
|
595
|
+
await migratePostgres(harness.url)
|
|
596
|
+
|
|
597
|
+
const marker = await sql`SELECT version, checksum FROM ab_auth_schema_migrations`
|
|
598
|
+
expect(Number(marker[0]?.version)).toBe(AUTH_SCHEMA_VERSION)
|
|
599
|
+
expect(marker[0]?.checksum).toBe(AUTH_SCHEMA_CHECKSUM)
|
|
600
|
+
|
|
601
|
+
// The legacy row survives with authenticationScheme NULL.
|
|
602
|
+
const legacy =
|
|
603
|
+
await sql`SELECT "authenticationScheme" FROM "oauthApplication" WHERE id = 'legacy'`
|
|
604
|
+
expect(legacy[0]?.authenticationScheme).toBeNull()
|
|
605
|
+
|
|
606
|
+
// A row carrying the column round-trips.
|
|
607
|
+
await sql`INSERT INTO "oauthApplication"
|
|
608
|
+
(id, name, "clientId", "clientSecret", "redirectUrls", type, disabled,
|
|
609
|
+
"createdAt", "updatedAt", "authenticationScheme")
|
|
610
|
+
VALUES ('c2', 'e2e-mcp-client', 'client-2', 'secret', 'https://claude.ai', 'web',
|
|
611
|
+
false, ${CONTRACT_T0}, ${CONTRACT_T0}, 'none')`
|
|
612
|
+
const clients =
|
|
613
|
+
await sql`SELECT "authenticationScheme" FROM "oauthApplication" WHERE id = 'c2'`
|
|
614
|
+
expect(clients[0]?.authenticationScheme).toBe('none')
|
|
615
|
+
|
|
616
|
+
// The upgrade is idempotent.
|
|
617
|
+
await migratePostgres(harness.url)
|
|
618
|
+
} finally {
|
|
619
|
+
await sql.close()
|
|
620
|
+
await harness.cleanup()
|
|
621
|
+
}
|
|
622
|
+
})
|
|
623
|
+
|
|
624
|
+
test('opening an uninitialized database names the migration step', async () => {
|
|
625
|
+
const harness = await schemaHarness()
|
|
626
|
+
try {
|
|
627
|
+
const error = await openPostgresBuildStore(harness.url, new MemoryBlobStore()).catch(
|
|
628
|
+
(caught: unknown) => caught,
|
|
629
|
+
)
|
|
630
|
+
expect((error as Error).message).toContain(MIGRATE_COMMAND)
|
|
631
|
+
} finally {
|
|
632
|
+
await harness.cleanup()
|
|
633
|
+
}
|
|
634
|
+
})
|
|
635
|
+
})
|
|
636
|
+
} else {
|
|
637
|
+
describe('PostgreSQL migration live tests', () => {
|
|
638
|
+
test.skip('set AB_POSTGRES_TEST_URL to run schema migration tests', () => {})
|
|
639
|
+
})
|
|
640
|
+
}
|
package/src/s3.ts
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { GetObjectCommand, NoSuchKey, PutObjectCommand, type S3Client } from '@aws-sdk/client-s3'
|
|
2
|
+
import type { BlobStore } from '@defrex/autobuild/plugin-sdk'
|
|
3
|
+
|
|
4
|
+
export interface S3BlobStoreOptions {
|
|
5
|
+
client: Pick<S3Client, 'send'>
|
|
6
|
+
bucket: string
|
|
7
|
+
prefix?: string
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export function blobPath(hash: string, prefix = ''): string {
|
|
11
|
+
const clean = prefix.split('/').filter(Boolean).join('/')
|
|
12
|
+
return clean ? `${clean}/${hash}` : hash
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export class S3BlobStore implements BlobStore {
|
|
16
|
+
constructor(private readonly options: S3BlobStoreOptions) {}
|
|
17
|
+
|
|
18
|
+
async put(hash: string, bytes: Uint8Array): Promise<void> {
|
|
19
|
+
await this.options.client.send(
|
|
20
|
+
new PutObjectCommand({
|
|
21
|
+
Bucket: this.options.bucket,
|
|
22
|
+
Key: blobPath(hash, this.options.prefix),
|
|
23
|
+
Body: bytes,
|
|
24
|
+
}),
|
|
25
|
+
)
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
async get(hash: string): Promise<Uint8Array | null> {
|
|
29
|
+
try {
|
|
30
|
+
const result = await this.options.client.send(
|
|
31
|
+
new GetObjectCommand({
|
|
32
|
+
Bucket: this.options.bucket,
|
|
33
|
+
Key: blobPath(hash, this.options.prefix),
|
|
34
|
+
}),
|
|
35
|
+
)
|
|
36
|
+
if (!result.Body) return new Uint8Array()
|
|
37
|
+
return await result.Body.transformToByteArray()
|
|
38
|
+
} catch (error) {
|
|
39
|
+
const status = (error as { $metadata?: { httpStatusCode?: number } }).$metadata
|
|
40
|
+
?.httpStatusCode
|
|
41
|
+
if (error instanceof NoSuchKey || status === 404) return null
|
|
42
|
+
throw error
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|