@open-pencil/cloud 0.16.0 → 0.17.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/listen.mjs +236 -117
- package/dist/listen.mjs.map +1 -1
- package/dist/runtime-node.d.mts +2937 -703
- package/dist/runtime-node.d.mts.map +1 -1
- package/dist/server.d.ts +3985 -941
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +833 -376
- package/dist/server.js.map +1 -1
- package/dist/server.mjs +665 -271
- package/dist/server.mjs.map +1 -1
- package/package.json +4 -2
- package/src/admin/api/routes.ts +44 -33
- package/src/runtime/cloudflare/environment.ts +16 -2
- package/src/runtime/cloudflare/relay/index.ts +128 -0
- package/src/runtime/cloudflare/relay/sockets.ts +112 -0
- package/src/runtime/cloudflare/worker.ts +56 -19
- package/src/runtime/node/bootstrap.ts +9 -4
- package/src/runtime/node/server.ts +40 -8
- package/src/runtime/posthog/errors.ts +48 -0
- package/src/runtime/s3/objects.ts +26 -0
- package/src/runtime/site/server.ts +5 -3
- package/src/server/api/router.ts +3 -2
- package/src/server/api/types.ts +7 -0
- package/src/server/app.ts +4 -0
- package/src/server/cleanup/objects.ts +10 -2
- package/src/server/cleanup/revisions.ts +8 -2
- package/src/server/collaboration/index.ts +3 -0
- package/src/server/collaboration/relay/connection.ts +195 -0
- package/src/server/collaboration/relay/hub.ts +199 -187
- package/src/server/collaboration/relay/index.ts +3 -1
- package/src/server/collaboration/relay/log.ts +76 -0
- package/src/server/collaboration/relay/persistence.ts +147 -22
- package/src/server/collaboration/relay/resume.ts +28 -0
- package/src/server/collaboration/routes.ts +27 -16
- package/src/server/config/deployment.ts +11 -0
- package/src/server/config/schema.ts +24 -2
- package/src/server/db/migrate.ts +3 -1
- package/src/server/db/migrations/028_collaboration_updates.ts +84 -0
- package/src/server/db/tables.ts +18 -4
- package/src/server/documents/routes.ts +106 -80
- package/src/server/documents/versions/routes.ts +36 -25
- package/src/server/documents/versions/service.ts +9 -0
- package/src/server/errors/index.ts +56 -0
- package/src/server/index.ts +11 -0
- package/src/server/internal/routes.ts +3 -8
- package/src/server/objects/types.ts +12 -0
- package/src/server/pictures/routes.ts +4 -2
- package/src/server/rate-limit/index.ts +4 -1
- package/src/server/rate-limit/middleware.ts +99 -11
- package/src/server/sharing/document-routes.ts +30 -22
- package/src/server/sharing/routes.ts +122 -83
- package/src/server/validation.ts +58 -2
- package/src/server/workspaces/routes.ts +63 -45
- package/src/runtime/cloudflare/relay.ts +0 -136
|
@@ -1,48 +1,173 @@
|
|
|
1
1
|
import type { CloudDatabase } from '#cloud/server/db'
|
|
2
|
-
import type {
|
|
2
|
+
import type { ObjectStore } from '#cloud/server/objects'
|
|
3
|
+
import { sha256 } from '@noble/hashes/sha2.js'
|
|
4
|
+
import { fromUint8Array } from 'js-base64'
|
|
5
|
+
import type { Kysely, Transaction } from 'kysely'
|
|
3
6
|
|
|
4
7
|
export type CollaborationRoomIdentity = {
|
|
5
8
|
documentId: string
|
|
6
9
|
roomEpoch: number
|
|
7
10
|
}
|
|
8
11
|
|
|
9
|
-
/**
|
|
12
|
+
/** A room as the store keeps it: its last snapshot and the updates logged after it. */
|
|
13
|
+
export type SavedRoom = {
|
|
14
|
+
snapshot: Uint8Array | null
|
|
15
|
+
/** The updates after the snapshot, in the order they were logged. */
|
|
16
|
+
updates: Uint8Array[]
|
|
17
|
+
/** The sequence of the last update the snapshot or the log holds; 0 for a new room. */
|
|
18
|
+
sequence: number
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* The relay's saved copy of each room's document, by document and collaboration epoch: a snapshot
|
|
23
|
+
* and an append-only log of the updates after it. Each room has one writer, the relay that holds
|
|
24
|
+
* it, so sequences only grow.
|
|
25
|
+
*/
|
|
10
26
|
export type CollaborationStateStore = {
|
|
11
|
-
load(room: CollaborationRoomIdentity): Promise<
|
|
12
|
-
|
|
27
|
+
load(room: CollaborationRoomIdentity): Promise<SavedRoom>
|
|
28
|
+
/**
|
|
29
|
+
* Logs one update; fails when the sequence is already taken. Writes nothing once a restore moved
|
|
30
|
+
* the document past the room's epoch, since that room never opens again.
|
|
31
|
+
*/
|
|
32
|
+
append(room: CollaborationRoomIdentity, sequence: number, update: Uint8Array): Promise<void>
|
|
33
|
+
/**
|
|
34
|
+
* Replaces the snapshot with `state`, which holds every update through `sequence`, and drops
|
|
35
|
+
* those updates; running it again leaves the same room. Like `append`, it changes no room of an
|
|
36
|
+
* epoch a restore replaced.
|
|
37
|
+
*/
|
|
38
|
+
compact(room: CollaborationRoomIdentity, sequence: number, state: Uint8Array): Promise<void>
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Whether the document is still at the room's epoch, holding its row so a restore, which advances
|
|
43
|
+
* the epoch and deletes the replaced rooms' rows, waits for this write or this write sees it.
|
|
44
|
+
*/
|
|
45
|
+
async function holdsEpoch(
|
|
46
|
+
transaction: Transaction<CloudDatabase>,
|
|
47
|
+
room: CollaborationRoomIdentity
|
|
48
|
+
): Promise<boolean> {
|
|
49
|
+
const document = await transaction
|
|
50
|
+
.selectFrom('document')
|
|
51
|
+
.select('collaborationEpoch')
|
|
52
|
+
.where('id', '=', room.documentId)
|
|
53
|
+
.forShare()
|
|
54
|
+
.executeTakeFirst()
|
|
55
|
+
return document?.collaborationEpoch === room.roomEpoch
|
|
13
56
|
}
|
|
14
57
|
|
|
58
|
+
/** A snapshot is a Yjs update, as the store hands it back. */
|
|
59
|
+
const SNAPSHOT_CONTENT_TYPE = 'application/octet-stream'
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Where a room's snapshots go. Each compaction writes a new file and points the room at it; the one
|
|
63
|
+
* it replaced is no longer referenced, and the object sweep removes it as it does a deleted
|
|
64
|
+
* revision's file.
|
|
65
|
+
*/
|
|
66
|
+
function snapshotKey(room: CollaborationRoomIdentity): string {
|
|
67
|
+
return `collaboration/${room.documentId}/${room.roomEpoch}/${crypto.randomUUID()}.yjs`
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* A store keeping each room's snapshot as a file in the object store and the updates logged after
|
|
72
|
+
* it in the database, so the database holds only what changed since the last snapshot.
|
|
73
|
+
*/
|
|
15
74
|
export function createCollaborationStateStore(
|
|
16
|
-
database: Kysely<CloudDatabase
|
|
75
|
+
database: Kysely<CloudDatabase>,
|
|
76
|
+
objects: Pick<ObjectStore, 'write' | 'read'>
|
|
17
77
|
): CollaborationStateStore {
|
|
18
78
|
return {
|
|
19
79
|
async load(room) {
|
|
20
|
-
const
|
|
21
|
-
.selectFrom('
|
|
22
|
-
.
|
|
80
|
+
const snapshot = await database
|
|
81
|
+
.selectFrom('documentRoomSnapshot')
|
|
82
|
+
.innerJoin('storageObject', 'storageObject.id', 'documentRoomSnapshot.snapshotObjectId')
|
|
83
|
+
.select(['storageObject.objectKey', 'documentRoomSnapshot.throughSequence'])
|
|
84
|
+
.where('documentRoomSnapshot.documentId', '=', room.documentId)
|
|
85
|
+
.where('documentRoomSnapshot.roomEpoch', '=', room.roomEpoch)
|
|
86
|
+
.executeTakeFirst()
|
|
87
|
+
const state = snapshot ? await objects.read(snapshot.objectKey) : null
|
|
88
|
+
if (snapshot && !state) {
|
|
89
|
+
throw new Error(`The snapshot of room ${room.documentId}:${room.roomEpoch} is missing`)
|
|
90
|
+
}
|
|
91
|
+
const through = snapshot?.throughSequence ?? 0
|
|
92
|
+
const updates = await database
|
|
93
|
+
.selectFrom('documentRoomUpdate')
|
|
94
|
+
.select(['sequence', 'data'])
|
|
23
95
|
.where('documentId', '=', room.documentId)
|
|
24
96
|
.where('roomEpoch', '=', room.roomEpoch)
|
|
25
|
-
.
|
|
26
|
-
|
|
97
|
+
.where('sequence', '>', through)
|
|
98
|
+
.orderBy('sequence')
|
|
99
|
+
.execute()
|
|
100
|
+
return {
|
|
101
|
+
snapshot: state,
|
|
102
|
+
updates: updates.map((row) => new Uint8Array(row.data)),
|
|
103
|
+
sequence: updates.at(-1)?.sequence ?? through
|
|
104
|
+
}
|
|
27
105
|
},
|
|
28
106
|
|
|
29
|
-
async
|
|
107
|
+
async append(room, sequence, update) {
|
|
108
|
+
await database.transaction().execute(async (transaction) => {
|
|
109
|
+
if (!(await holdsEpoch(transaction, room))) return
|
|
110
|
+
await transaction
|
|
111
|
+
.insertInto('documentRoomUpdate')
|
|
112
|
+
.values({
|
|
113
|
+
documentId: room.documentId,
|
|
114
|
+
roomEpoch: room.roomEpoch,
|
|
115
|
+
sequence,
|
|
116
|
+
data: update
|
|
117
|
+
})
|
|
118
|
+
.execute()
|
|
119
|
+
})
|
|
120
|
+
},
|
|
121
|
+
|
|
122
|
+
async compact(room, sequence, state) {
|
|
123
|
+
// The file is recorded before it is written and referenced only once the room points at it,
|
|
124
|
+
// so a compaction that fails anywhere leaves a file nothing refers to, which the object sweep
|
|
125
|
+
// removes; none is ever deleted while a room might still point at it.
|
|
126
|
+
const snapshotObjectId = crypto.randomUUID()
|
|
127
|
+
const objectKey = snapshotKey(room)
|
|
128
|
+
const checksum = fromUint8Array(sha256(state))
|
|
30
129
|
await database
|
|
31
|
-
.insertInto('
|
|
130
|
+
.insertInto('storageObject')
|
|
32
131
|
.values({
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
132
|
+
id: snapshotObjectId,
|
|
133
|
+
objectKey,
|
|
134
|
+
checksum,
|
|
135
|
+
byteSize: state.byteLength,
|
|
136
|
+
contentType: SNAPSHOT_CONTENT_TYPE
|
|
37
137
|
})
|
|
38
|
-
.
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
138
|
+
.execute()
|
|
139
|
+
await objects.write({
|
|
140
|
+
key: objectKey,
|
|
141
|
+
bytes: state,
|
|
142
|
+
contentType: SNAPSHOT_CONTENT_TYPE,
|
|
143
|
+
checksum
|
|
144
|
+
})
|
|
145
|
+
await database.transaction().execute(async (transaction) => {
|
|
146
|
+
if (!(await holdsEpoch(transaction, room))) return
|
|
147
|
+
await transaction
|
|
148
|
+
.insertInto('documentRoomSnapshot')
|
|
149
|
+
.values({
|
|
150
|
+
documentId: room.documentId,
|
|
151
|
+
roomEpoch: room.roomEpoch,
|
|
152
|
+
snapshotObjectId,
|
|
153
|
+
throughSequence: sequence,
|
|
42
154
|
updatedAt: new Date()
|
|
43
155
|
})
|
|
44
|
-
|
|
45
|
-
|
|
156
|
+
.onConflict((conflict) =>
|
|
157
|
+
conflict.columns(['documentId', 'roomEpoch']).doUpdateSet({
|
|
158
|
+
snapshotObjectId,
|
|
159
|
+
throughSequence: sequence,
|
|
160
|
+
updatedAt: new Date()
|
|
161
|
+
})
|
|
162
|
+
)
|
|
163
|
+
.execute()
|
|
164
|
+
await transaction
|
|
165
|
+
.deleteFrom('documentRoomUpdate')
|
|
166
|
+
.where('documentId', '=', room.documentId)
|
|
167
|
+
.where('roomEpoch', '=', room.roomEpoch)
|
|
168
|
+
.where('sequence', '<=', sequence)
|
|
169
|
+
.execute()
|
|
170
|
+
})
|
|
46
171
|
}
|
|
47
172
|
}
|
|
48
173
|
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { collaborationPrincipalSchema } from '#cloud/contract'
|
|
2
|
+
import * as v from 'valibot'
|
|
3
|
+
|
|
4
|
+
const peerStateSchema = v.object({
|
|
5
|
+
peerId: v.pipe(v.string(), v.uuid()),
|
|
6
|
+
authorization: v.object({
|
|
7
|
+
roomId: v.string(),
|
|
8
|
+
documentId: v.pipe(v.string(), v.uuid()),
|
|
9
|
+
roomEpoch: v.pipe(v.number(), v.integer(), v.minValue(0)),
|
|
10
|
+
permission: v.picklist(['view', 'edit']),
|
|
11
|
+
principal: collaborationPrincipalSchema,
|
|
12
|
+
expiresAt: v.number()
|
|
13
|
+
}),
|
|
14
|
+
presenceClients: v.array(v.pipe(v.number(), v.integer()))
|
|
15
|
+
})
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* What a runtime keeps with each admitted socket so the relay can resume its peer after the
|
|
19
|
+
* runtime set the relay aside, as a hibernating Durable Object does: who it is, what its ticket
|
|
20
|
+
* allows, and the presence clients it speaks for.
|
|
21
|
+
*/
|
|
22
|
+
export type RelayPeerState = v.InferOutput<typeof peerStateSchema>
|
|
23
|
+
|
|
24
|
+
/** A peer's kept state, or null when what the runtime kept is not one. */
|
|
25
|
+
export function readRelayPeerState(value: unknown): RelayPeerState | null {
|
|
26
|
+
const parsed = v.safeParse(peerStateSchema, value)
|
|
27
|
+
return parsed.success ? parsed.output : null
|
|
28
|
+
}
|
|
@@ -3,9 +3,13 @@ import type { CloudActor, CloudSessionResolver } from '#cloud/server/auth'
|
|
|
3
3
|
import type { CollaborationTicketService } from '#cloud/server/collaboration/service'
|
|
4
4
|
import type { CloudDatabase } from '#cloud/server/db'
|
|
5
5
|
import { DocumentNotFoundError } from '#cloud/server/documents'
|
|
6
|
-
import {
|
|
6
|
+
import {
|
|
7
|
+
CLOUD_RATE_LIMITS,
|
|
8
|
+
routeParamResource,
|
|
9
|
+
createResourceRateLimiter
|
|
10
|
+
} from '#cloud/server/rate-limit'
|
|
7
11
|
import { DocumentShareInvalidError, DocumentShareSignInRequiredError } from '#cloud/server/sharing'
|
|
8
|
-
import { validatedJSON } from '#cloud/server/validation'
|
|
12
|
+
import { documentParams, shareParams, validatedJSON } from '#cloud/server/validation'
|
|
9
13
|
import { Hono, type Context } from 'hono'
|
|
10
14
|
import type { Kysely } from 'kysely'
|
|
11
15
|
|
|
@@ -29,26 +33,32 @@ export function createCollaborationRoutes(
|
|
|
29
33
|
if (rateLimit) {
|
|
30
34
|
router.use(
|
|
31
35
|
'/documents/:documentId/collaboration-ticket',
|
|
32
|
-
|
|
36
|
+
createResourceRateLimiter(
|
|
33
37
|
rateLimit.database,
|
|
34
38
|
rateLimit.secret,
|
|
35
39
|
CLOUD_RATE_LIMITS.collaborationTicket,
|
|
36
|
-
|
|
37
|
-
(context) => context.req.param('documentId') ?? 'unknown-document'
|
|
40
|
+
routeParamResource('documentId')
|
|
38
41
|
)
|
|
39
42
|
)
|
|
40
43
|
}
|
|
41
|
-
return router.post(
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
44
|
+
return router.post(
|
|
45
|
+
'/documents/:documentId/collaboration-ticket',
|
|
46
|
+
documentParams,
|
|
47
|
+
async (context) => {
|
|
48
|
+
try {
|
|
49
|
+
return context.json({
|
|
50
|
+
ticket: await service.issueUserTicket(
|
|
51
|
+
context.get('actor'),
|
|
52
|
+
context.req.valid('param').documentId
|
|
53
|
+
)
|
|
54
|
+
})
|
|
55
|
+
} catch (error) {
|
|
56
|
+
const response = notFound(context, error)
|
|
57
|
+
if (response) return response
|
|
58
|
+
throw error
|
|
59
|
+
}
|
|
50
60
|
}
|
|
51
|
-
|
|
61
|
+
)
|
|
52
62
|
}
|
|
53
63
|
|
|
54
64
|
export function createPublicCollaborationRoutes(
|
|
@@ -57,12 +67,13 @@ export function createPublicCollaborationRoutes(
|
|
|
57
67
|
) {
|
|
58
68
|
return new Hono().post(
|
|
59
69
|
'/shares/:shareId/collaboration-ticket',
|
|
70
|
+
shareParams,
|
|
60
71
|
validatedJSON(parseResolveDocumentShare),
|
|
61
72
|
async (context) => {
|
|
62
73
|
try {
|
|
63
74
|
return context.json({
|
|
64
75
|
ticket: await service.issueShareTicket(
|
|
65
|
-
context.req.
|
|
76
|
+
context.req.valid('param').shareId,
|
|
66
77
|
context.req.valid('json'),
|
|
67
78
|
resolveSession ? ((await resolveSession(context.req.raw)) ?? undefined) : undefined
|
|
68
79
|
)
|
|
@@ -99,6 +99,16 @@ function serviceConfig(config: CloudDeploymentConfig, environment: CloudEnvironm
|
|
|
99
99
|
}
|
|
100
100
|
}
|
|
101
101
|
|
|
102
|
+
function errorTrackingConfig(config: CloudDeploymentConfig, environment: CloudEnvironment) {
|
|
103
|
+
const tracking = config.error_tracking
|
|
104
|
+
if (!tracking) return undefined
|
|
105
|
+
return {
|
|
106
|
+
provider: tracking.provider,
|
|
107
|
+
host: tracking.host,
|
|
108
|
+
projectAPIKey: configuredText(environment, tracking.project_api_key)
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
102
112
|
/** Hard limits, whatever a plan allows; each lives in the section it limits. */
|
|
103
113
|
function technicalLimits(config: CloudDeploymentConfig) {
|
|
104
114
|
return {
|
|
@@ -166,6 +176,7 @@ export function parseCloudDeploymentConfig(
|
|
|
166
176
|
...emailConfig(config, environment),
|
|
167
177
|
...workerConfig(config),
|
|
168
178
|
...serviceConfig(config, environment),
|
|
179
|
+
errorTracking: errorTrackingConfig(config, environment),
|
|
169
180
|
technicalLimits: technicalLimits(config)
|
|
170
181
|
})
|
|
171
182
|
}
|
|
@@ -146,7 +146,15 @@ const rawCloudServerConfigSchema = v.object({
|
|
|
146
146
|
),
|
|
147
147
|
billingPortalURL: v.optional(httpURLSchema),
|
|
148
148
|
/** The bearer token services such as billing present to the internal API. */
|
|
149
|
-
internalServiceToken: v.optional(v.pipe(v.string(), v.minLength(32)))
|
|
149
|
+
internalServiceToken: v.optional(v.pipe(v.string(), v.minLength(32))),
|
|
150
|
+
/** Where server errors are reported; without it the server reports nothing. */
|
|
151
|
+
errorTracking: v.optional(
|
|
152
|
+
v.object({
|
|
153
|
+
provider: v.literal('posthog'),
|
|
154
|
+
host: httpURLSchema,
|
|
155
|
+
projectAPIKey: v.pipe(v.string(), v.trim(), v.minLength(1))
|
|
156
|
+
})
|
|
157
|
+
)
|
|
150
158
|
})
|
|
151
159
|
|
|
152
160
|
export type CloudServerConfig = v.InferOutput<typeof rawCloudServerConfigSchema>
|
|
@@ -263,6 +271,9 @@ export function parseCloudServerConfig(input: unknown): CloudServerConfig {
|
|
|
263
271
|
return config
|
|
264
272
|
}
|
|
265
273
|
|
|
274
|
+
/** PostHog's US Cloud ingestion host. */
|
|
275
|
+
const POSTHOG_DEFAULT_HOST = 'https://us.i.posthog.com'
|
|
276
|
+
|
|
266
277
|
const DEFAULT_SECRET_NAMES = {
|
|
267
278
|
databaseURL: 'DATABASE_URL',
|
|
268
279
|
authSecret: 'BETTER_AUTH_SECRET',
|
|
@@ -276,7 +287,8 @@ const DEFAULT_SECRET_NAMES = {
|
|
|
276
287
|
smtpPassword: 'OPENPENCIL_CLOUD_SMTP_PASSWORD',
|
|
277
288
|
turnstileSecretKey: 'TURNSTILE_SECRET_KEY',
|
|
278
289
|
aiSigningKey: 'OPENPENCIL_CLOUD_AI_SIGNING_KEY',
|
|
279
|
-
internalServiceToken: 'OPENPENCIL_CLOUD_SERVICE_TOKEN'
|
|
290
|
+
internalServiceToken: 'OPENPENCIL_CLOUD_SERVICE_TOKEN',
|
|
291
|
+
posthogProjectAPIKey: 'POSTHOG_PROJECT_API_KEY'
|
|
280
292
|
} as const
|
|
281
293
|
|
|
282
294
|
const urlSchema = v.pipe(v.string(), v.url())
|
|
@@ -503,6 +515,16 @@ export const cloudDeploymentConfigSchema = v.strictObject({
|
|
|
503
515
|
billing: v.optional(v.strictObject({ portal_url: urlSchema })),
|
|
504
516
|
internal_api: v.optional(
|
|
505
517
|
v.strictObject({ service_token: defaultReference(DEFAULT_SECRET_NAMES.internalServiceToken) })
|
|
518
|
+
),
|
|
519
|
+
/** Optional: reports server errors to PostHog error tracking. Leave it out to send nothing. */
|
|
520
|
+
error_tracking: v.optional(
|
|
521
|
+
v.strictObject({
|
|
522
|
+
provider: v.literal('posthog'),
|
|
523
|
+
host: v.optional(urlSchema, POSTHOG_DEFAULT_HOST),
|
|
524
|
+
project_api_key: v.optional(configurableTextSchema, {
|
|
525
|
+
from_env: DEFAULT_SECRET_NAMES.posthogProjectAPIKey
|
|
526
|
+
})
|
|
527
|
+
})
|
|
506
528
|
)
|
|
507
529
|
})
|
|
508
530
|
|
package/src/server/db/migrate.ts
CHANGED
|
@@ -28,6 +28,7 @@ import * as accountActivity from './migrations/024_account_activity'
|
|
|
28
28
|
import * as tableNames from './migrations/025_table_names'
|
|
29
29
|
import * as userPicture from './migrations/026_user_picture'
|
|
30
30
|
import * as documentKind from './migrations/027_document_kind'
|
|
31
|
+
import * as collaborationUpdates from './migrations/028_collaboration_updates'
|
|
31
32
|
import type { CloudDatabase } from './tables'
|
|
32
33
|
|
|
33
34
|
const migrations: Record<string, Migration> = {
|
|
@@ -57,7 +58,8 @@ const migrations: Record<string, Migration> = {
|
|
|
57
58
|
'024_account_activity': accountActivity,
|
|
58
59
|
'025_table_names': tableNames,
|
|
59
60
|
'026_user_picture': userPicture,
|
|
60
|
-
'027_document_kind': documentKind
|
|
61
|
+
'027_document_kind': documentKind,
|
|
62
|
+
'028_collaboration_updates': collaborationUpdates
|
|
61
63
|
}
|
|
62
64
|
|
|
63
65
|
class CloudMigrationProvider implements MigrationProvider {
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import type { CloudDatabase } from '#cloud/server/db/tables'
|
|
2
|
+
import type { Kysely } from 'kysely'
|
|
3
|
+
|
|
4
|
+
/** The renamed table's constraints, whose names spelled its old name. */
|
|
5
|
+
const CONSTRAINTS = [
|
|
6
|
+
['documentCollaborationStatePrimary', 'documentRoomSnapshotPrimary'],
|
|
7
|
+
['documentCollaborationStateDocumentIdFkey', 'documentRoomSnapshotDocumentIdFkey']
|
|
8
|
+
] as const
|
|
9
|
+
|
|
10
|
+
async function renameConstraints(database: Kysely<CloudDatabase>, direction: 'up' | 'down') {
|
|
11
|
+
for (const [old, current] of CONSTRAINTS) {
|
|
12
|
+
const [from, to] = direction === 'up' ? [old, current] : [current, old]
|
|
13
|
+
await database.schema.alterTable('documentRoomSnapshot').renameConstraint(from, to).execute()
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/** Moves every document with a saved room to a new epoch, and drops the saved rooms. */
|
|
18
|
+
export async function startRoomsOver(database: Kysely<CloudDatabase>): Promise<void> {
|
|
19
|
+
await database
|
|
20
|
+
.updateTable('document')
|
|
21
|
+
.set((expression) => ({ collaborationEpoch: expression('collaborationEpoch', '+', 1) }))
|
|
22
|
+
.where('id', 'in', database.selectFrom('documentRoomSnapshot').select('documentId'))
|
|
23
|
+
.execute()
|
|
24
|
+
await database.deleteFrom('documentRoomSnapshot').execute()
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* A room's edits as an append-only log beside a snapshot kept in the object store, so a save
|
|
29
|
+
* writes what changed instead of the whole room, and the room's bulk lives with the documents'
|
|
30
|
+
* files rather than in the database. The snapshot row names its stored file and the last logged
|
|
31
|
+
* update it holds; compaction points it at a new file and drops the updates it covers.
|
|
32
|
+
*
|
|
33
|
+
* Rooms saved before held their snapshots in the row itself. Those rows go, and each such
|
|
34
|
+
* document moves to a new collaboration epoch, so its next room starts from its last saved
|
|
35
|
+
* revision and no tab's local copy of the earlier room joins it.
|
|
36
|
+
*/
|
|
37
|
+
export async function up(database: Kysely<CloudDatabase>): Promise<void> {
|
|
38
|
+
await database.schema
|
|
39
|
+
.alterTable('documentCollaborationState')
|
|
40
|
+
.renameTo('documentRoomSnapshot')
|
|
41
|
+
.execute()
|
|
42
|
+
await renameConstraints(database, 'up')
|
|
43
|
+
await startRoomsOver(database)
|
|
44
|
+
await database.schema
|
|
45
|
+
.alterTable('documentRoomSnapshot')
|
|
46
|
+
.dropColumn('state')
|
|
47
|
+
.dropColumn('version')
|
|
48
|
+
.addColumn('snapshotObjectId', 'uuid', (column) =>
|
|
49
|
+
column.notNull().references('storageObject.id')
|
|
50
|
+
)
|
|
51
|
+
.addColumn('throughSequence', 'integer', (column) => column.notNull())
|
|
52
|
+
.execute()
|
|
53
|
+
await database.schema
|
|
54
|
+
.createTable('documentRoomUpdate')
|
|
55
|
+
.addColumn('documentId', 'uuid', (column) =>
|
|
56
|
+
column.notNull().references('document.id').onDelete('cascade')
|
|
57
|
+
)
|
|
58
|
+
.addColumn('roomEpoch', 'integer', (column) => column.notNull())
|
|
59
|
+
.addColumn('sequence', 'integer', (column) => column.notNull())
|
|
60
|
+
.addColumn('data', 'bytea', (column) => column.notNull())
|
|
61
|
+
.addColumn('createdAt', 'timestamptz', (column) =>
|
|
62
|
+
column.notNull().defaultTo(database.fn('now'))
|
|
63
|
+
)
|
|
64
|
+
.addPrimaryKeyConstraint('documentRoomUpdatePrimary', ['documentId', 'roomEpoch', 'sequence'])
|
|
65
|
+
.execute()
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** Back to snapshots in rows; the rooms saved meanwhile start over, as they did going up. */
|
|
69
|
+
export async function down(database: Kysely<CloudDatabase>): Promise<void> {
|
|
70
|
+
await database.schema.dropTable('documentRoomUpdate').execute()
|
|
71
|
+
await database.deleteFrom('documentRoomSnapshot').execute()
|
|
72
|
+
await database.schema
|
|
73
|
+
.alterTable('documentRoomSnapshot')
|
|
74
|
+
.dropColumn('snapshotObjectId')
|
|
75
|
+
.dropColumn('throughSequence')
|
|
76
|
+
.addColumn('state', 'bytea', (column) => column.notNull())
|
|
77
|
+
.addColumn('version', 'integer', (column) => column.notNull().defaultTo(1))
|
|
78
|
+
.execute()
|
|
79
|
+
await renameConstraints(database, 'down')
|
|
80
|
+
await database.schema
|
|
81
|
+
.alterTable('documentRoomSnapshot')
|
|
82
|
+
.renameTo('documentCollaborationState')
|
|
83
|
+
.execute()
|
|
84
|
+
}
|
package/src/server/db/tables.ts
CHANGED
|
@@ -233,15 +233,28 @@ export interface UserPictureTable {
|
|
|
233
233
|
createdAt: TimestampColumn
|
|
234
234
|
}
|
|
235
235
|
|
|
236
|
-
|
|
236
|
+
/** A collaboration room's latest snapshot, a stored file, by document and epoch. */
|
|
237
|
+
export interface DocumentRoomSnapshotTable {
|
|
237
238
|
documentId: string
|
|
238
239
|
roomEpoch: number
|
|
239
|
-
|
|
240
|
-
|
|
240
|
+
/** The stored file holding the room's snapshot. */
|
|
241
|
+
snapshotObjectId: string
|
|
242
|
+
/** The last logged update the snapshot holds; later ones are in `documentRoomUpdate`. */
|
|
243
|
+
throughSequence: number
|
|
241
244
|
createdAt: TimestampColumn
|
|
242
245
|
updatedAt: TimestampColumn
|
|
243
246
|
}
|
|
244
247
|
|
|
248
|
+
/** An update a room logged after its snapshot, one per save. */
|
|
249
|
+
export interface DocumentRoomUpdateTable {
|
|
250
|
+
documentId: string
|
|
251
|
+
roomEpoch: number
|
|
252
|
+
sequence: number
|
|
253
|
+
/** One Yjs update, as the relay merged the changes of one save. */
|
|
254
|
+
data: Uint8Array
|
|
255
|
+
createdAt: TimestampColumn
|
|
256
|
+
}
|
|
257
|
+
|
|
245
258
|
export interface InvitationContinuationTable {
|
|
246
259
|
id: string
|
|
247
260
|
invitationId: string
|
|
@@ -363,7 +376,8 @@ export interface CloudDatabase {
|
|
|
363
376
|
documentShare: DocumentShareTable
|
|
364
377
|
documentGrant: DocumentGrantTable
|
|
365
378
|
documentInvitation: DocumentInvitationTable
|
|
366
|
-
|
|
379
|
+
documentRoomSnapshot: DocumentRoomSnapshotTable
|
|
380
|
+
documentRoomUpdate: DocumentRoomUpdateTable
|
|
367
381
|
invitationContinuation: InvitationContinuationTable
|
|
368
382
|
workspaceEntitlement: WorkspaceEntitlementTable
|
|
369
383
|
workspaceStorageUsage: WorkspaceStorageUsageTable
|