@open-pencil/cloud 0.17.0-beta.2 → 0.17.0-beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (223) hide show
  1. package/README.md +1 -1
  2. package/dist/bin/{ai-relay.mjs → services.mjs} +8 -8
  3. package/dist/bin/services.mjs.map +1 -0
  4. package/dist/client.d.ts +236 -46
  5. package/dist/client.d.ts.map +1 -1
  6. package/dist/client.js +649 -400
  7. package/dist/client.js.map +1 -1
  8. package/dist/contract.d.ts +2 -2
  9. package/dist/contract.js +2 -2
  10. package/dist/contract2.js +680 -443
  11. package/dist/contract2.js.map +1 -1
  12. package/dist/email.d.ts +9 -1
  13. package/dist/email.d.ts.map +1 -1
  14. package/dist/email.js +99 -58
  15. package/dist/email.js.map +1 -1
  16. package/dist/index.d.ts +469 -65
  17. package/dist/index.d.ts.map +1 -1
  18. package/dist/listen.mjs +505 -787
  19. package/dist/listen.mjs.map +1 -1
  20. package/dist/runtime-node.d.mts +3920 -2524
  21. package/dist/runtime-node.d.mts.map +1 -1
  22. package/dist/runtime-node.mjs +6 -5
  23. package/dist/runtime-node.mjs.map +1 -1
  24. package/dist/server.d.ts +3586 -1897
  25. package/dist/server.d.ts.map +1 -1
  26. package/dist/server.js +5216 -4272
  27. package/dist/server.js.map +1 -1
  28. package/dist/server.mjs +5179 -3623
  29. package/dist/server.mjs.map +1 -1
  30. package/dist/server2.mjs +101 -79
  31. package/dist/server2.mjs.map +1 -1
  32. package/package.json +7 -6
  33. package/src/admin/api/routes.ts +59 -0
  34. package/src/admin/audit/service.ts +25 -6
  35. package/src/admin/documents/service.ts +361 -0
  36. package/src/admin/errors.ts +9 -1
  37. package/src/admin/index.ts +6 -0
  38. package/src/admin/operations/service.ts +7 -0
  39. package/src/admin/reports/service.ts +147 -0
  40. package/src/admin/schema.ts +6 -0
  41. package/src/client/api.ts +36 -21
  42. package/src/client/auth/navigation.ts +6 -5
  43. package/src/client/index.ts +7 -7
  44. package/src/client/portal/account.ts +25 -0
  45. package/src/client/portal/administration.ts +56 -0
  46. package/src/client/{ai.ts → services.ts} +44 -27
  47. package/src/contract/account.ts +58 -2
  48. package/src/contract/admin/index.ts +67 -3
  49. package/src/contract/collaboration.ts +6 -1
  50. package/src/contract/discovery.ts +22 -4
  51. package/src/contract/documents.ts +27 -0
  52. package/src/contract/entitlements.ts +2 -2
  53. package/src/contract/index.ts +63 -37
  54. package/src/contract/relay.ts +6 -0
  55. package/src/contract/reports.ts +42 -0
  56. package/src/contract/services.ts +233 -0
  57. package/src/contract/sharing.ts +3 -9
  58. package/src/email/render.ts +26 -0
  59. package/src/email/schema.ts +11 -1
  60. package/src/email/templates/brand.ts +1 -1
  61. package/src/runtime/cloudflare/environment.ts +22 -11
  62. package/src/runtime/cloudflare/index.ts +1 -1
  63. package/src/runtime/cloudflare/logging.ts +15 -0
  64. package/src/runtime/cloudflare/maintenance.ts +31 -0
  65. package/src/runtime/cloudflare/relay/index.ts +62 -14
  66. package/src/runtime/cloudflare/worker.ts +39 -13
  67. package/src/runtime/node/application.ts +5 -3
  68. package/src/runtime/node/bootstrap.ts +15 -4
  69. package/src/runtime/node/cleanup.ts +16 -3
  70. package/src/runtime/node/entitlements/main.ts +14 -11
  71. package/src/runtime/node/index.ts +1 -1
  72. package/src/runtime/node/migrate.ts +5 -1
  73. package/src/runtime/node/relay.ts +8 -2
  74. package/src/runtime/node/request-id.ts +13 -0
  75. package/src/runtime/node/server.ts +39 -15
  76. package/src/runtime/node/{ai-relay → services}/main.ts +6 -6
  77. package/src/runtime/node/{ai-relay → services}/server.ts +18 -18
  78. package/src/runtime/postgres/database.ts +20 -1
  79. package/src/runtime/posthog/errors.ts +15 -31
  80. package/src/runtime/s3/objects.ts +24 -1
  81. package/src/runtime/site/server.ts +32 -5
  82. package/src/server/account/actions.ts +5 -0
  83. package/src/server/account/deletion.ts +302 -0
  84. package/src/server/account/routes.ts +47 -3
  85. package/src/server/account/terms.ts +42 -0
  86. package/src/server/api/router.ts +9 -5
  87. package/src/server/api/types.ts +2 -0
  88. package/src/server/app.ts +90 -16
  89. package/src/server/auth/access-control.ts +63 -0
  90. package/src/server/auth/adapter.ts +11 -0
  91. package/src/server/auth/endpoints.ts +5 -1
  92. package/src/server/auth/factory.ts +30 -3
  93. package/src/server/auth/index.ts +10 -1
  94. package/src/server/cleanup/configured.ts +12 -5
  95. package/src/server/cleanup/documents.ts +165 -71
  96. package/src/server/cleanup/revisions.ts +16 -5
  97. package/src/server/cleanup/uploads.ts +8 -2
  98. package/src/server/cleanup/worker.ts +5 -2
  99. package/src/server/collaboration/relay/connection.ts +4 -3
  100. package/src/server/collaboration/relay/documents.ts +13 -10
  101. package/src/server/collaboration/relay/hub.ts +97 -25
  102. package/src/server/collaboration/relay/log.ts +5 -2
  103. package/src/server/collaboration/service.ts +27 -31
  104. package/src/server/config/deployment.ts +24 -4
  105. package/src/server/config/public.ts +5 -4
  106. package/src/server/config/schema.ts +82 -16
  107. package/src/server/db/expressions.ts +15 -1
  108. package/src/server/db/migrate.ts +34 -116
  109. package/src/server/db/migrations/001_accounts.ts +79 -0
  110. package/src/server/db/migrations/002_workspaces.ts +48 -0
  111. package/src/server/db/migrations/003_storage.ts +44 -0
  112. package/src/server/db/migrations/004_documents.ts +193 -0
  113. package/src/server/db/migrations/005_sharing.ts +95 -0
  114. package/src/server/db/migrations/006_operations.ts +110 -0
  115. package/src/server/db/migrations/007_trash.ts +33 -0
  116. package/src/server/db/migrations/008_terms.ts +18 -0
  117. package/src/server/db/migrations/schema.ts +71 -0
  118. package/src/server/db/tables.ts +52 -3
  119. package/src/server/documents/access.ts +66 -55
  120. package/src/server/documents/errors.ts +7 -0
  121. package/src/server/documents/images.ts +2 -1
  122. package/src/server/documents/limits.ts +9 -9
  123. package/src/server/documents/purge.ts +75 -0
  124. package/src/server/documents/repository.ts +2 -0
  125. package/src/server/documents/routes.ts +26 -0
  126. package/src/server/documents/service.ts +18 -4
  127. package/src/server/documents/summary.ts +1 -0
  128. package/src/server/documents/trash.ts +203 -0
  129. package/src/server/documents/versions/checkpoint.ts +2 -1
  130. package/src/server/documents/versions/history.ts +11 -7
  131. package/src/server/documents/versions/service.ts +13 -4
  132. package/src/server/errors/index.ts +59 -16
  133. package/src/server/errors/posthog.ts +80 -0
  134. package/src/server/index.ts +46 -17
  135. package/src/server/internal/index.ts +3 -3
  136. package/src/server/internal/routes.ts +7 -10
  137. package/src/server/internal/schema.ts +1 -1
  138. package/src/server/internal/service.ts +17 -23
  139. package/src/server/invitations/budget.ts +66 -0
  140. package/src/server/invitations/index.ts +1 -0
  141. package/src/server/logging.ts +76 -0
  142. package/src/server/pictures/service.ts +2 -1
  143. package/src/server/policy/database.ts +39 -25
  144. package/src/server/policy/index.ts +2 -1
  145. package/src/server/policy/keys.ts +5 -2
  146. package/src/server/policy/policy.ts +1 -0
  147. package/src/server/policy/schema.ts +2 -2
  148. package/src/server/policy/service.ts +12 -9
  149. package/src/server/policy/static.ts +5 -5
  150. package/src/server/policy/workspace.ts +41 -0
  151. package/src/server/quota/reconcile.ts +2 -0
  152. package/src/server/quota/storage.ts +69 -31
  153. package/src/server/rate-limit/cleanup.ts +17 -5
  154. package/src/server/rate-limit/middleware.ts +2 -2
  155. package/src/server/reports/index.ts +8 -0
  156. package/src/server/reports/routes.ts +37 -0
  157. package/src/server/reports/service.ts +123 -0
  158. package/src/server/services/errors.ts +7 -0
  159. package/src/server/services/index.ts +8 -0
  160. package/src/server/{ai → services}/keys.ts +11 -11
  161. package/src/server/services/routes.ts +47 -0
  162. package/src/server/{ai/service.ts → services/tokens.ts} +23 -20
  163. package/src/server/sharing/capabilities/service.ts +68 -22
  164. package/src/server/sharing/grants/service.ts +11 -18
  165. package/src/server/sharing/http.ts +4 -0
  166. package/src/server/sharing/invitations/service.ts +21 -11
  167. package/src/server/sharing/policy.ts +9 -14
  168. package/src/server/sharing/routes.ts +0 -16
  169. package/src/server/sharing/service.ts +1 -17
  170. package/src/server/validation.ts +1 -0
  171. package/src/server/workspaces/index.ts +1 -0
  172. package/src/server/workspaces/members.ts +89 -38
  173. package/src/server/workspaces/repository.ts +2 -1
  174. package/src/server/workspaces/routes.ts +4 -0
  175. package/src/{ai-relay → services}/app.ts +48 -44
  176. package/src/{ai-relay → services}/config.ts +25 -19
  177. package/src/services/db/index.ts +10 -0
  178. package/src/services/db/migrate.ts +37 -0
  179. package/src/{ai-relay/db/migrations/001_usage.ts → services/db/migrations/001_baseline.ts} +18 -7
  180. package/src/services/db/schema.ts +22 -0
  181. package/src/services/index.ts +18 -0
  182. package/src/{ai-relay → services}/metering.ts +2 -2
  183. package/src/{ai-relay → services}/usage.ts +43 -19
  184. package/dist/bin/ai-relay.mjs.map +0 -1
  185. package/src/ai-relay/db/index.ts +0 -10
  186. package/src/ai-relay/db/migrate.ts +0 -32
  187. package/src/ai-relay/db/schema.ts +0 -20
  188. package/src/ai-relay/index.ts +0 -18
  189. package/src/contract/ai.ts +0 -150
  190. package/src/server/ai/errors.ts +0 -7
  191. package/src/server/ai/index.ts +0 -4
  192. package/src/server/ai/routes.ts +0 -43
  193. package/src/server/auth/workspace-access.ts +0 -23
  194. package/src/server/db/migrations/001_foundation.ts +0 -164
  195. package/src/server/db/migrations/002_upload_cleanup_claims.ts +0 -34
  196. package/src/server/db/migrations/003_document_cleanup_claims.ts +0 -23
  197. package/src/server/db/migrations/004_document_sharing.ts +0 -82
  198. package/src/server/db/migrations/005_document_collaboration_epoch.ts +0 -13
  199. package/src/server/db/migrations/006_invitation_continuation.ts +0 -27
  200. package/src/server/db/migrations/007_collaboration_state.ts +0 -25
  201. package/src/server/db/migrations/008_storage_reservations.ts +0 -37
  202. package/src/server/db/migrations/009_workspace_entitlements.ts +0 -22
  203. package/src/server/db/migrations/010_upload_finalization.ts +0 -22
  204. package/src/server/db/migrations/011_upload_finalization_lease.ts +0 -12
  205. package/src/server/db/migrations/012_transactional_email.ts +0 -48
  206. package/src/server/db/migrations/013_admin_enrollment.ts +0 -54
  207. package/src/server/db/migrations/014_enrollment_email_kinds.ts +0 -30
  208. package/src/server/db/migrations/015_cloud_rate_limit.ts +0 -22
  209. package/src/server/db/migrations/016_auth_schema.ts +0 -16
  210. package/src/server/db/migrations/017_authentication_email_kinds.ts +0 -32
  211. package/src/server/db/migrations/018_authentication_mfa.ts +0 -78
  212. package/src/server/db/migrations/019_better_auth_account_issuer.ts +0 -42
  213. package/src/server/db/migrations/020_workspace_invitations.ts +0 -57
  214. package/src/server/db/migrations/021_document_versions.ts +0 -66
  215. package/src/server/db/migrations/022_entitlement_external_version.ts +0 -17
  216. package/src/server/db/migrations/023_document_images.ts +0 -61
  217. package/src/server/db/migrations/024_account_activity.ts +0 -66
  218. package/src/server/db/migrations/025_table_names.ts +0 -58
  219. package/src/server/db/migrations/026_user_picture.ts +0 -30
  220. package/src/server/db/migrations/027_document_kind.ts +0 -19
  221. package/src/server/db/migrations/028_collaboration_updates.ts +0 -84
  222. package/src/server/db/migrations/helpers.ts +0 -78
  223. /package/dist/bin/{ai-relay.d.mts → services.d.mts} +0 -0
@@ -8,6 +8,7 @@ import {
8
8
  relayServerMessages,
9
9
  type RelayServerFrame
10
10
  } from '#cloud/contract'
11
+ import { logCloud, loggedError } from '#cloud/server/logging'
11
12
  import * as Y from 'yjs'
12
13
 
13
14
  import type { RelayAuthorization } from './authorize'
@@ -36,9 +37,14 @@ export type CollaborationRelayOptions = {
36
37
  documentId?: string
37
38
  /**
38
39
  * The most one socket message may carry. An editor's larger frames arrive as fragments joined
39
- * up to `MAX_COLLAB_MESSAGE_BYTES`; anyone else's frame must fit in one message.
40
+ * up to `maximumRoomBytes`; anyone else's frame must fit in one message.
40
41
  */
41
42
  maximumMessageBytes: number
43
+ /**
44
+ * The most a room's document may hold, and so the most one editor's update may carry:
45
+ * `MAX_COLLAB_MESSAGE_BYTES` unless the runtime holds rooms in less memory.
46
+ */
47
+ maximumRoomBytes?: number
42
48
  maximumConnectionsPerRoom: number
43
49
  /** The participant limit policy sets for a document, or null for none. */
44
50
  maximumParticipants?: (documentId: string) => Promise<number | null>
@@ -67,8 +73,13 @@ const VIEWER_ACTIONS = new Set<string>([COLLAB_ACTIONS.syncStep1, COLLAB_ACTIONS
67
73
 
68
74
  type Peer = RelayPeer
69
75
 
70
- /** A room's document and the log its saves go to, loaded when a message first needs them. */
71
- type RoomContent = { document: Y.Doc; log: RoomLog }
76
+ /**
77
+ * A room's document and the log its saves go to, loaded when a message first needs them, with an
78
+ * upper bound on the document's size: what was saved plus every update since. The bound is
79
+ * measured again only once it reaches `maximumRoomBytes` after something was deleted, since only
80
+ * a deletion can leave the document smaller than the bound.
81
+ */
82
+ type RoomContent = { document: Y.Doc; log: RoomLog; bytes: number; deletedSinceMeasure: boolean }
72
83
 
73
84
  type Room = {
74
85
  id: string
@@ -85,6 +96,18 @@ type Room = {
85
96
 
86
97
  export type CollaborationRelay = ReturnType<typeof createCollaborationRelay>
87
98
 
99
+ /** Whether `update` adds nothing and deletes only content `document` already holds. */
100
+ function deletesOnlyKnown(document: Y.Doc, update: Uint8Array): boolean {
101
+ const { from, to } = Y.parseUpdateMetaV2(update)
102
+ if (![...to].every(([client, clock]) => from.get(client) === clock)) return false
103
+ const { ds } = Y.decodeUpdateV2(update)
104
+ for (const [client, deletions] of ds.clients) {
105
+ const held = Y.getState(document.store, client)
106
+ if (deletions.some((deletion) => deletion.clock + deletion.len > held)) return false
107
+ }
108
+ return true
109
+ }
110
+
88
111
  /**
89
112
  * A runtime-neutral relay for collaboration rooms. It forwards room messages between peers and
90
113
  * joins every room as a peer itself: it holds the room's document, answers sync requests from
@@ -103,8 +126,14 @@ export function createCollaborationRelay(options: CollaborationRelayOptions) {
103
126
  updates: options.compactAfterUpdates ?? 500,
104
127
  bytes: options.compactAfterBytes ?? 4 * 1024 * 1024
105
128
  }
129
+ const maximumRoomBytes = Math.min(
130
+ options.maximumRoomBytes ?? MAX_COLLAB_MESSAGE_BYTES,
131
+ MAX_COLLAB_MESSAGE_BYTES
132
+ )
106
133
  const authTimeoutMs = options.authTimeoutMs ?? 10_000
107
- const report = options.onError ?? ((message, error) => console.error(`[Cloud] ${message}`, error))
134
+ const report =
135
+ options.onError ??
136
+ ((message, error) => logCloud('error', message, { source: 'relay', error: loggedError(error) }))
108
137
 
109
138
  function send(socket: RelaySocket, frame: RelayServerFrame) {
110
139
  for (const message of relayServerMessages(frame)) socket.send(message)
@@ -148,8 +177,14 @@ export function createCollaborationRelay(options: CollaborationRelayOptions) {
148
177
  function contentOf(room: Room): Promise<RoomContent> {
149
178
  if (room.content) return Promise.resolve(room.content)
150
179
  room.loading ??= openRoomLog(options.store, room.identity, logLimits).then(
151
- (content) => {
152
- content.document.on('update', () => schedulePersist(room))
180
+ ({ document, log, savedBytes }) => {
181
+ document.on('update', () => schedulePersist(room))
182
+ // The saved log may hold deletions, so the first time the bound reaches the limit, it is
183
+ // measured.
184
+ const content: RoomContent = { document, log, bytes: savedBytes, deletedSinceMeasure: true }
185
+ document.on('afterTransaction', (transaction: Y.Transaction) => {
186
+ if (transaction.deleteSet.clients.size > 0) content.deletedSinceMeasure = true
187
+ })
153
188
  room.content = content
154
189
  return content
155
190
  },
@@ -173,7 +208,7 @@ export function createCollaborationRelay(options: CollaborationRelayOptions) {
173
208
  await log.save({ closing })
174
209
  return true
175
210
  } catch (error) {
176
- report('Saving a collaboration room failed:', error)
211
+ report('Saving a collaboration room failed', error)
177
212
  // The room holds edits the store doesn't have; save again without waiting for another edit.
178
213
  // A closing room retries through `saveAndClose`.
179
214
  if (!closing && rooms.get(room.id) === room) {
@@ -192,21 +227,23 @@ export function createCollaborationRelay(options: CollaborationRelayOptions) {
192
227
  * V1 updates; only what peers exchange is V2.
193
228
  */
194
229
  function applyToRoom(
195
- document: Y.Doc,
230
+ content: RoomContent,
196
231
  peer: Peer,
197
232
  namespace: string,
198
233
  payload: Uint8Array
199
234
  ): boolean {
235
+ const { document } = content
200
236
  try {
201
237
  const update = decodeRoomUpdate(payload, {
202
- limit: MAX_COLLAB_MESSAGE_BYTES,
238
+ limit: maximumRoomBytes,
203
239
  compressed: namespace === COLLAB_ACTIONS.syncReply
204
240
  })
205
- if (!update) {
241
+ if (!update || !fits(content, update)) {
206
242
  peer.socket.close(RELAY_CLOSE.messageTooLarge, 'Collaboration update is too large')
207
243
  return false
208
244
  }
209
245
  Y.applyUpdateV2(document, update, peer.id)
246
+ content.bytes += update.byteLength
210
247
  return true
211
248
  } catch {
212
249
  peer.socket.close(RELAY_CLOSE.protocol, 'Malformed document update')
@@ -228,8 +265,24 @@ export function createCollaborationRelay(options: CollaborationRelayOptions) {
228
265
  sendAll(deliverable(recipients.flatMap((peer) => (peer ? [peer] : []))), frame)
229
266
  }
230
267
 
268
+ /**
269
+ * Whether the room's document stays within its limit after `update`. An update that only
270
+ * deletes what the room holds shrinks it, so it always fits; one that deletes what the room has
271
+ * not received waits in the document, so it counts like any other. Otherwise the bound grows by
272
+ * the update; past the limit, the document is measured only if a deletion may have shrunk it
273
+ * since the last measure, so edits near the limit never encode the whole document each time.
274
+ */
275
+ function fits(content: RoomContent, update: Uint8Array): boolean {
276
+ if (deletesOnlyKnown(content.document, update)) return true
277
+ if (content.bytes + update.byteLength <= maximumRoomBytes) return true
278
+ if (!content.deletedSinceMeasure) return false
279
+ content.bytes = Y.encodeStateAsUpdateV2(content.document).byteLength
280
+ content.deletedSinceMeasure = false
281
+ return content.bytes + update.byteLength <= maximumRoomBytes
282
+ }
283
+
231
284
  function handleServerMessage(
232
- document: Y.Doc,
285
+ content: RoomContent,
233
286
  peer: Peer,
234
287
  namespace: string,
235
288
  payload: Uint8Array
@@ -237,7 +290,7 @@ export function createCollaborationRelay(options: CollaborationRelayOptions) {
237
290
  if (namespace === COLLAB_ACTIONS.syncStep1) {
238
291
  let reply: Uint8Array
239
292
  try {
240
- reply = encodeSyncReply(document, payload)
293
+ reply = encodeSyncReply(content.document, payload)
241
294
  } catch {
242
295
  peer.socket.close(RELAY_CLOSE.protocol, 'Malformed state vector')
243
296
  return
@@ -249,7 +302,7 @@ export function createCollaborationRelay(options: CollaborationRelayOptions) {
249
302
  payload: reply
250
303
  })
251
304
  } else if (namespace === COLLAB_ACTIONS.syncReply) {
252
- applyToRoom(document, peer, namespace, payload)
305
+ applyToRoom(content, peer, namespace, payload)
253
306
  }
254
307
  }
255
308
 
@@ -262,7 +315,7 @@ export function createCollaborationRelay(options: CollaborationRelayOptions) {
262
315
 
263
316
  function handleMessage(
264
317
  room: Room,
265
- document: Y.Doc | null,
318
+ content: RoomContent | null,
266
319
  peer: Peer,
267
320
  namespace: string,
268
321
  target: string | null,
@@ -288,12 +341,12 @@ export function createCollaborationRelay(options: CollaborationRelayOptions) {
288
341
  }
289
342
 
290
343
  if (target === RELAY_SERVER_PEER) {
291
- if (document) handleServerMessage(document, peer, namespace, payload)
344
+ if (content) handleServerMessage(content, peer, namespace, payload)
292
345
  return
293
346
  }
294
347
  if (
295
348
  (namespace === COLLAB_ACTIONS.yjsUpdate || namespace === COLLAB_ACTIONS.syncReply) &&
296
- !(document && applyToRoom(document, peer, namespace, payload))
349
+ !(content && applyToRoom(content, peer, namespace, payload))
297
350
  ) {
298
351
  return
299
352
  }
@@ -334,18 +387,18 @@ export function createCollaborationRelay(options: CollaborationRelayOptions) {
334
387
  if (!current(peer)) return undefined
335
388
  const { namespace, target, payload } = frame
336
389
  if (room.content || !needsDocument(namespace, target)) {
337
- handleMessage(room, room.content?.document ?? null, peer, namespace, target, payload)
390
+ handleMessage(room, room.content, peer, namespace, target, payload)
338
391
  return undefined
339
392
  }
340
393
  return contentOf(room).then(
341
- ({ document }) => {
342
- // The peer may have left while the document loaded.
343
- if (room.peers.get(peer.id) !== peer) return undefined
344
- handleMessage(room, document, peer, namespace, target, payload)
394
+ (content) => {
395
+ // The peer may have left, or its ticket lapsed, while the document loaded.
396
+ if (room.peers.get(peer.id) !== peer || !current(peer)) return undefined
397
+ handleMessage(room, content, peer, namespace, target, payload)
345
398
  return undefined
346
399
  },
347
400
  (error: unknown) => {
348
- report('Loading a collaboration room failed:', error)
401
+ report('Loading a collaboration room failed', error)
349
402
  peer.socket.close(RELAY_CLOSE.protocol, 'Could not open the collaboration room')
350
403
  }
351
404
  )
@@ -355,13 +408,15 @@ export function createCollaborationRelay(options: CollaborationRelayOptions) {
355
408
  socket: RelaySocket,
356
409
  authorization: RelayAuthorization
357
410
  ): Promise<Peer | null> {
358
- const limit = await options.maximumParticipants?.(authorization.documentId)
359
411
  let room: Room
412
+ let limit: number | null | undefined
360
413
  let epoch: number | null | undefined
361
- // The room's last peer may have left while it opened, or a restore ended it; a closed room
362
- // takes no newcomers. The epoch is read once the room is open, so a restore cannot slip between.
414
+ // The room's last peer may have left while it opened, or a restore or takedown ended it; a
415
+ // closed room takes no newcomers. The limit and epoch are read once the room is open, so
416
+ // neither a restore nor a takedown can slip between.
363
417
  do {
364
418
  room = await openRoom(authorization)
419
+ limit = await options.maximumParticipants?.(authorization.documentId)
365
420
  epoch = await options.currentRoomEpoch?.(authorization.documentId)
366
421
  } while (rooms.get(room.id) !== room)
367
422
  if (epoch != null && authorization.roomEpoch < epoch) {
@@ -442,6 +497,7 @@ export function createCollaborationRelay(options: CollaborationRelayOptions) {
442
497
  authSecret: options.authSecret,
443
498
  documentId: options.documentId,
444
499
  maximumMessageBytes: options.maximumMessageBytes,
500
+ maximumUpdateBytes: maximumRoomBytes,
445
501
  authTimeoutMs,
446
502
  admit,
447
503
  refreshed: remember,
@@ -499,6 +555,22 @@ export function createCollaborationRelay(options: CollaborationRelayOptions) {
499
555
  }
500
556
  },
501
557
 
558
+ /**
559
+ * Ends a taken-down document's rooms for everyone in them, saving what each room holds, so
560
+ * a restore opens them again with every edit. Nobody gets back in meanwhile: tickets are
561
+ * issued only for documents people can reach.
562
+ */
563
+ closeDocument(documentId: string) {
564
+ for (const room of rooms.values()) {
565
+ if (room.identity.documentId !== documentId) continue
566
+ for (const peer of room.peers.values()) {
567
+ peer.socket.close(RELAY_CLOSE.unauthorized, 'The document was taken down')
568
+ }
569
+ room.peers.clear()
570
+ void leaveRoom(room)
571
+ }
572
+ },
573
+
502
574
  /** Tells everyone in a document's rooms that its version history changed. */
503
575
  announceVersions(documentId: string) {
504
576
  const frame: RelayServerFrame = {
@@ -22,7 +22,7 @@ export async function openRoomLog(
22
22
  store: CollaborationStateStore,
23
23
  room: CollaborationRoomIdentity,
24
24
  limits: RoomLogLimits
25
- ): Promise<{ document: Y.Doc; log: RoomLog }> {
25
+ ): Promise<{ document: Y.Doc; log: RoomLog; savedBytes: number }> {
26
26
  const saved = await store.load(room)
27
27
  const document = new Y.Doc()
28
28
  if (saved.snapshot) Y.applyUpdate(document, saved.snapshot)
@@ -72,5 +72,8 @@ export async function openRoomLog(
72
72
  return run
73
73
  }
74
74
  }
75
- return { document, log }
75
+ const savedBytes =
76
+ (saved.snapshot?.byteLength ?? 0) +
77
+ saved.updates.reduce((total, update) => total + update.byteLength, 0)
78
+ return { document, log, savedBytes }
76
79
  }
@@ -11,6 +11,7 @@ import { DocumentNotFoundError } from '#cloud/server/documents'
11
11
  import { resolveDocumentAccess } from '#cloud/server/documents/access'
12
12
  import { CLOUD_FEATURE_KEYS } from '#cloud/server/policy/keys'
13
13
  import type { CloudPolicy } from '#cloud/server/policy/policy'
14
+ import { workspacePolicyContext } from '#cloud/server/policy/workspace'
14
15
  import type { DocumentSharingService } from '#cloud/server/sharing'
15
16
  import { base64url, SignJWT } from 'jose'
16
17
  import type { Kysely } from 'kysely'
@@ -74,13 +75,21 @@ export async function signCollaborationTicket({
74
75
  .setExpirationTime(expiresAtSeconds)
75
76
  .setSubject(principal.kind === 'user' ? principal.userId : principal.guestId)
76
77
  .sign(new TextEncoder().encode(authSecret))
77
- return {
78
+ const ticket = {
78
79
  token,
79
- provider: relayURL ? ('relay' as const) : ('trystero' as const),
80
- ...(relayURL ? { serverURL: withQuery(relayURL, { [RELAY_DOCUMENT_PARAM]: documentId }) } : {}),
80
+ provider: 'trystero' as const,
81
81
  ...claims,
82
82
  roomKey,
83
- expiresAt: new Date(expiresAtSeconds * 1000).toISOString()
83
+ expiresAt: new Date(expiresAtSeconds * 1000).toISOString(),
84
+ relayHeartbeat: false
85
+ }
86
+ if (!relayURL) return ticket
87
+ // This deployment's relay answers the heartbeat, which editors then use to notice a lost socket.
88
+ return {
89
+ ...ticket,
90
+ provider: 'relay' as const,
91
+ serverURL: withQuery(relayURL, { [RELAY_DOCUMENT_PARAM]: documentId }),
92
+ relayHeartbeat: true
84
93
  }
85
94
  }
86
95
 
@@ -106,20 +115,12 @@ export function createCollaborationTicketService({
106
115
  const access = await resolveDocumentAccess(database, actor.userId, documentId)
107
116
  if (!access) throw new DocumentNotFoundError()
108
117
  if (policy) {
109
- const workspace = await database
110
- .selectFrom('document')
111
- .select('workspaceId')
112
- .where('id', '=', documentId)
113
- .executeTakeFirstOrThrow()
114
- if (
115
- !(await policy.boolean(CLOUD_FEATURE_KEYS.collaboration, false, {
116
- targetingKey: workspace.workspaceId,
117
- actorId: actor.userId,
118
- workspaceId: workspace.workspaceId,
119
- documentId,
120
- deploymentMode
121
- }))
122
- ) {
118
+ const context = await workspacePolicyContext(
119
+ database,
120
+ { documentId },
121
+ { actorId: actor.userId, deploymentMode }
122
+ )
123
+ if (!context || !(await policy.boolean(CLOUD_FEATURE_KEYS.collaboration, false, context))) {
123
124
  throw new DocumentNotFoundError()
124
125
  }
125
126
  }
@@ -128,6 +129,7 @@ export function createCollaborationTicketService({
128
129
  .select('collaborationEpoch')
129
130
  .where('id', '=', documentId)
130
131
  .where('deletedAt', 'is', null)
132
+ .where('disabledAt', 'is', null)
131
133
  .executeTakeFirst()
132
134
  if (!document) throw new DocumentNotFoundError()
133
135
  return signCollaborationTicket({
@@ -147,19 +149,12 @@ export function createCollaborationTicketService({
147
149
  ): Promise<CollaborationTicket> {
148
150
  const resolved = await sharing.resolveShare(shareId, input, actor)
149
151
  if (policy) {
150
- const workspace = await database
151
- .selectFrom('document')
152
- .select('workspaceId')
153
- .where('id', '=', resolved.documentId)
154
- .executeTakeFirstOrThrow()
155
- const context = {
156
- targetingKey: workspace.workspaceId,
157
- ...(actor ? { actorId: actor.userId } : {}),
158
- workspaceId: workspace.workspaceId,
159
- documentId: resolved.documentId,
160
- deploymentMode
161
- }
162
- if (!(await policy.boolean(CLOUD_FEATURE_KEYS.collaboration, false, context))) {
152
+ const context = await workspacePolicyContext(
153
+ database,
154
+ { documentId: resolved.documentId },
155
+ { actorId: actor?.userId, deploymentMode }
156
+ )
157
+ if (!context || !(await policy.boolean(CLOUD_FEATURE_KEYS.collaboration, false, context))) {
163
158
  throw new DocumentNotFoundError()
164
159
  }
165
160
  if (!actor && !(await policy.boolean(CLOUD_FEATURE_KEYS.guestPresence, false, context))) {
@@ -171,6 +166,7 @@ export function createCollaborationTicketService({
171
166
  .select('collaborationEpoch')
172
167
  .where('id', '=', resolved.documentId)
173
168
  .where('deletedAt', 'is', null)
169
+ .where('disabledAt', 'is', null)
174
170
  .executeTakeFirst()
175
171
  if (!document) throw new DocumentNotFoundError()
176
172
  return signCollaborationTicket({
@@ -83,15 +83,20 @@ function workerConfig(config: CloudDeploymentConfig) {
83
83
  cleanupBatchSize: cleanup?.batch_size,
84
84
  cleanupIntervalMs: cleanup?.interval_ms,
85
85
  cleanupLeaseDurationMs: cleanup?.lease_ms,
86
- documentRetentionMs: config.documents.retention_days * 86_400_000
86
+ documentRetentionMs: config.documents.retention_days * 86_400_000,
87
+ documentPurgeGraceMs: config.documents.purge_grace_days * 86_400_000
87
88
  }
88
89
  }
89
90
 
90
91
  function serviceConfig(config: CloudDeploymentConfig, environment: CloudEnvironment) {
91
92
  return {
92
- aiEndpoint: config.ai?.endpoint,
93
- aiSigningKey: config.ai ? requiredSecret(environment, config.ai.signing_key) : undefined,
94
- aiTokenLifetimeSeconds: config.ai ? config.ai.token_lifetime_minutes * 60 : undefined,
93
+ servicesEndpoint: config.services?.endpoint,
94
+ servicesSigningKey: config.services
95
+ ? requiredSecret(environment, config.services.signing_key)
96
+ : undefined,
97
+ servicesTokenLifetimeSeconds: config.services
98
+ ? config.services.token_lifetime_minutes * 60
99
+ : undefined,
95
100
  billingPortalURL: config.billing?.portal_url,
96
101
  internalServiceToken: config.internal_api
97
102
  ? requiredSecret(environment, config.internal_api.service_token)
@@ -99,6 +104,19 @@ function serviceConfig(config: CloudDeploymentConfig, environment: CloudEnvironm
99
104
  }
100
105
  }
101
106
 
107
+ function legalConfig(config: CloudDeploymentConfig) {
108
+ const legal = config.legal
109
+ if (!legal) return undefined
110
+ return {
111
+ termsURL: legal.terms_url,
112
+ privacyURL: legal.privacy_url,
113
+ termsVersion: legal.terms_version,
114
+ requireAcceptance: legal.require_acceptance,
115
+ abuseEmail: legal.abuse_email,
116
+ minimumAge: legal.minimum_age
117
+ }
118
+ }
119
+
102
120
  function errorTrackingConfig(config: CloudDeploymentConfig, environment: CloudEnvironment) {
103
121
  const tracking = config.error_tracking
104
122
  if (!tracking) return undefined
@@ -171,11 +189,13 @@ export function parseCloudDeploymentConfig(
171
189
  s3ChecksumVerification: config.object_storage.checksum_verification,
172
190
  s3ServerSideEncryption: config.object_storage.server_side_encryption,
173
191
  s3KmsKeyId: config.object_storage.kms_key_id,
192
+ s3DeletedPrefix: config.object_storage.deleted_prefix,
174
193
  relayURL: config.collaboration.relay_url,
175
194
  staticEntitlements: staticEntitlements(config),
176
195
  ...emailConfig(config, environment),
177
196
  ...workerConfig(config),
178
197
  ...serviceConfig(config, environment),
198
+ legal: legalConfig(config),
179
199
  errorTracking: errorTrackingConfig(config, environment),
180
200
  technicalLimits: technicalLimits(config)
181
201
  })
@@ -1,5 +1,5 @@
1
1
  import {
2
- CLOUD_AI_CONTRACT_VERSION,
2
+ CLOUD_SERVICES_CONTRACT_VERSION,
3
3
  CLOUD_PROTOCOL_VERSION,
4
4
  parseCloudDiscovery,
5
5
  type CloudDiscovery
@@ -58,9 +58,10 @@ export function cloudDiscoveryFromConfig(config: CloudServerConfig): CloudDiscov
58
58
  listing: config.listingMinimumEditorVersion
59
59
  ? { minimumEditorVersion: config.listingMinimumEditorVersion }
60
60
  : undefined,
61
- ai: config.aiEndpoint
62
- ? { endpoint: config.aiEndpoint, contractVersion: CLOUD_AI_CONTRACT_VERSION }
61
+ services: config.servicesEndpoint
62
+ ? { endpoint: config.servicesEndpoint, contractVersion: CLOUD_SERVICES_CONTRACT_VERSION }
63
63
  : undefined,
64
- billing: config.billingPortalURL ? { portalURL: config.billingPortalURL } : undefined
64
+ billing: config.billingPortalURL ? { portalURL: config.billingPortalURL } : undefined,
65
+ legal: config.legal
65
66
  })
66
67
  }
@@ -31,6 +31,16 @@ const websocketURLSchema = v.pipe(
31
31
 
32
32
  const optionalTextSchema = v.optional(v.pipe(v.string(), v.trim(), v.minLength(1)))
33
33
 
34
+ /**
35
+ * A prefix deleted files move under: one lowercase path segment and a slash, which no object key
36
+ * the server makes starts with (they start with `documents/`).
37
+ */
38
+ const deletedPrefixSchema = v.pipe(
39
+ v.string(),
40
+ v.regex(/^[a-z0-9][a-z0-9-]*\/$/),
41
+ v.check((prefix) => prefix !== 'documents/', 'deleted_prefix cannot be documents/')
42
+ )
43
+
34
44
  const rawCloudServerConfigSchema = v.object({
35
45
  deployment: v.picklist(['official', 'self-hosted']),
36
46
  publicURL: httpURLSchema,
@@ -111,6 +121,8 @@ const rawCloudServerConfigSchema = v.object({
111
121
  s3ChecksumVerification: v.optional(v.picklist(['native', 'metadata']), 'native'),
112
122
  s3ServerSideEncryption: v.optional(v.picklist(['AES256', 'aws:kms'])),
113
123
  s3KmsKeyId: optionalTextSchema,
124
+ /** Where a deleted file is kept until the bucket's lifecycle rule expires it; see `deleted_prefix`. */
125
+ s3DeletedPrefix: v.optional(deletedPrefixSchema),
114
126
  smtpHost: optionalTextSchema,
115
127
  smtpPort: v.optional(v.pipe(v.number(), v.integer(), v.minValue(1), v.maxValue(65_535))),
116
128
  smtpSecure: v.optional(v.boolean()),
@@ -136,17 +148,40 @@ const rawCloudServerConfigSchema = v.object({
136
148
  v.pipe(v.number(), v.integer(), v.minValue(0)),
137
149
  30 * 24 * 60 * 60_000
138
150
  ),
139
- /** The AI service editors reach with tokens this server signs; see `src/contract/ai.ts`. */
140
- aiEndpoint: v.optional(httpURLSchema),
141
- /** The PKCS #8 PEM of the P-256 key that signs AI service tokens. */
142
- aiSigningKey: optionalTextSchema,
143
- aiTokenLifetimeSeconds: v.optional(
151
+ /**
152
+ * How long a document deleted forever keeps its files and rows, out of everyone's sight, so a
153
+ * server administrator can still bring it back; 0 destroys it at once.
154
+ */
155
+ documentPurgeGraceMs: v.optional(
156
+ v.pipe(v.number(), v.integer(), v.minValue(0)),
157
+ 7 * 24 * 60 * 60_000
158
+ ),
159
+ /** The OpenPencil services editors reach with tokens this server signs; see `src/contract/services.ts`. */
160
+ servicesEndpoint: v.optional(httpURLSchema),
161
+ /** The PKCS #8 PEM of the P-256 key that signs service tokens. */
162
+ servicesSigningKey: optionalTextSchema,
163
+ servicesTokenLifetimeSeconds: v.optional(
144
164
  v.pipe(v.number(), v.integer(), v.minValue(60), v.maxValue(60 * 60)),
145
165
  10 * 60
146
166
  ),
147
167
  billingPortalURL: v.optional(httpURLSchema),
148
168
  /** The bearer token services such as billing present to the internal API. */
149
169
  internalServiceToken: v.optional(v.pipe(v.string(), v.minLength(32))),
170
+ /**
171
+ * The operator's own terms, privacy policy, abuse contact, and minimum age; without it the
172
+ * server shows none. With `requireAcceptance`, an account uses the server only once it accepted
173
+ * `termsVersion`.
174
+ */
175
+ legal: v.optional(
176
+ v.object({
177
+ termsURL: v.optional(httpURLSchema),
178
+ privacyURL: v.optional(httpURLSchema),
179
+ termsVersion: optionalTextSchema,
180
+ requireAcceptance: v.boolean(),
181
+ abuseEmail: v.optional(v.pipe(v.string(), v.email())),
182
+ minimumAge: v.optional(v.pipe(v.number(), v.integer(), v.minValue(13), v.maxValue(21)))
183
+ })
184
+ ),
150
185
  /** Where server errors are reported; without it the server reports nothing. */
151
186
  errorTracking: v.optional(
152
187
  v.object({
@@ -189,6 +224,9 @@ function validateAuthenticationConfig(config: CloudServerConfig): void {
189
224
  if (config.deploymentAdminMFARequired && !config.totpEnabled && !config.passkeysEnabled) {
190
225
  throw new CloudConfigError('Deployment administrator MFA requires TOTP or passkeys')
191
226
  }
227
+ if (config.deployment === 'official' && !config.deploymentAdminMFARequired) {
228
+ throw new CloudConfigError('An official deployment requires MFA for deployment administrators')
229
+ }
192
230
  if (config.recoveryCodesEnabled && !config.totpEnabled) {
193
231
  throw new CloudConfigError('MFA recovery codes require TOTP')
194
232
  }
@@ -230,7 +268,7 @@ function validateEmailConfig(config: CloudServerConfig): void {
230
268
  }
231
269
 
232
270
  function validateServiceConfig(config: CloudServerConfig): void {
233
- requireTogether(config, 'AI service', ['aiEndpoint', 'aiSigningKey'])
271
+ requireTogether(config, 'Services', ['servicesEndpoint', 'servicesSigningKey'])
234
272
  if (config.internalServiceToken && config.staticEntitlements) {
235
273
  throw new CloudConfigError(
236
274
  'The internal API writes database entitlements, which static entitlements would hide'
@@ -238,10 +276,17 @@ function validateServiceConfig(config: CloudServerConfig): void {
238
276
  }
239
277
  if (
240
278
  config.deployment === 'official' &&
241
- config.aiEndpoint &&
242
- new URL(config.aiEndpoint).protocol !== 'https:'
279
+ config.servicesEndpoint &&
280
+ new URL(config.servicesEndpoint).protocol !== 'https:'
243
281
  ) {
244
- throw new CloudConfigError('Official AI service endpoint must use HTTPS')
282
+ throw new CloudConfigError('Official services endpoint must use HTTPS')
283
+ }
284
+ }
285
+
286
+ function validateLegalConfig(config: CloudServerConfig): void {
287
+ const legal = config.legal
288
+ if (legal?.requireAcceptance && !(legal.termsURL && legal.termsVersion)) {
289
+ throw new CloudConfigError('[legal] require_acceptance needs terms_url and terms_version')
245
290
  }
246
291
  }
247
292
 
@@ -268,11 +313,12 @@ export function parseCloudServerConfig(input: unknown): CloudServerConfig {
268
313
  validateEmailConfig(config)
269
314
  validateServiceConfig(config)
270
315
  validateObjectStorageConfig(config)
316
+ validateLegalConfig(config)
271
317
  return config
272
318
  }
273
319
 
274
- /** PostHog's US Cloud ingestion host. */
275
- const POSTHOG_DEFAULT_HOST = 'https://us.i.posthog.com'
320
+ /** PostHog's EU Cloud ingestion host, where OpenPencil's own project is. */
321
+ const POSTHOG_DEFAULT_HOST = 'https://eu.i.posthog.com'
276
322
 
277
323
  const DEFAULT_SECRET_NAMES = {
278
324
  databaseURL: 'DATABASE_URL',
@@ -286,7 +332,7 @@ const DEFAULT_SECRET_NAMES = {
286
332
  smtpUser: 'OPENPENCIL_CLOUD_SMTP_USER',
287
333
  smtpPassword: 'OPENPENCIL_CLOUD_SMTP_PASSWORD',
288
334
  turnstileSecretKey: 'TURNSTILE_SECRET_KEY',
289
- aiSigningKey: 'OPENPENCIL_CLOUD_AI_SIGNING_KEY',
335
+ servicesSigningKey: 'OPENPENCIL_CLOUD_SERVICES_SIGNING_KEY',
290
336
  internalServiceToken: 'OPENPENCIL_CLOUD_SERVICE_TOKEN',
291
337
  posthogProjectAPIKey: 'POSTHOG_PROJECT_API_KEY'
292
338
  } as const
@@ -457,6 +503,11 @@ export const cloudDeploymentConfigSchema = v.strictObject({
457
503
  checksum_verification: v.optional(v.picklist(['native', 'metadata']), 'native'),
458
504
  server_side_encryption: v.optional(v.picklist(['AES256', 'aws:kms'])),
459
505
  kms_key_id: v.optional(textSchema),
506
+ /**
507
+ * Keeps each file cleanup deletes under this prefix, such as `deleted/`, so a database restored
508
+ * to an earlier time still finds it; a lifecycle rule on the prefix expires them.
509
+ */
510
+ deleted_prefix: v.optional(deletedPrefixSchema),
460
511
  maximum_upload_bytes: v.optional(positiveIntegerSchema)
461
512
  }),
462
513
  collaboration: v.optional(
@@ -469,9 +520,10 @@ export const cloudDeploymentConfigSchema = v.strictObject({
469
520
  ),
470
521
  documents: v.optional(
471
522
  v.strictObject({
472
- retention_days: v.optional(v.pipe(v.number(), v.integer(), v.minValue(0)), 30)
523
+ retention_days: v.optional(v.pipe(v.number(), v.integer(), v.minValue(0)), 30),
524
+ purge_grace_days: v.optional(v.pipe(v.number(), v.integer(), v.minValue(0)), 7)
473
525
  }),
474
- { retention_days: 30 }
526
+ { retention_days: 30, purge_grace_days: 7 }
475
527
  ),
476
528
  email: emailSchema,
477
529
  workers: v.optional(
@@ -502,10 +554,10 @@ export const cloudDeploymentConfigSchema = v.strictObject({
502
554
  v.strictObject({ source: v.literal('database') })
503
555
  ])
504
556
  ),
505
- ai: v.optional(
557
+ services: v.optional(
506
558
  v.strictObject({
507
559
  endpoint: urlSchema,
508
- signing_key: defaultReference(DEFAULT_SECRET_NAMES.aiSigningKey),
560
+ signing_key: defaultReference(DEFAULT_SECRET_NAMES.servicesSigningKey),
509
561
  token_lifetime_minutes: v.optional(
510
562
  v.pipe(v.number(), v.integer(), v.minValue(1), v.maxValue(60)),
511
563
  10
@@ -516,6 +568,20 @@ export const cloudDeploymentConfigSchema = v.strictObject({
516
568
  internal_api: v.optional(
517
569
  v.strictObject({ service_token: defaultReference(DEFAULT_SECRET_NAMES.internalServiceToken) })
518
570
  ),
571
+ /**
572
+ * Optional: the operator's terms and privacy policy, linked at sign-up and in the portal, and
573
+ * with `require_acceptance`, accepted once per `terms_version` before an account uses the server.
574
+ */
575
+ legal: v.optional(
576
+ v.strictObject({
577
+ terms_url: v.optional(urlSchema),
578
+ privacy_url: v.optional(urlSchema),
579
+ terms_version: v.optional(textSchema),
580
+ require_acceptance: v.optional(v.boolean(), false),
581
+ abuse_email: v.optional(v.pipe(v.string(), v.email())),
582
+ minimum_age: v.optional(v.pipe(v.number(), v.integer(), v.minValue(13), v.maxValue(21)))
583
+ })
584
+ ),
519
585
  /** Optional: reports server errors to PostHog error tracking. Leave it out to send nothing. */
520
586
  error_tracking: v.optional(
521
587
  v.strictObject({