@open-pencil/cloud 0.17.0-beta.1 → 0.17.0-beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (235) hide show
  1. package/README.md +1 -1
  2. package/dist/bin/{ai-relay.mjs → services.mjs} +8 -8
  3. package/dist/bin/services.mjs.map +1 -0
  4. package/dist/client.d.ts +236 -46
  5. package/dist/client.d.ts.map +1 -1
  6. package/dist/client.js +649 -400
  7. package/dist/client.js.map +1 -1
  8. package/dist/contract.d.ts +2 -2
  9. package/dist/contract.js +2 -2
  10. package/dist/contract2.js +680 -443
  11. package/dist/contract2.js.map +1 -1
  12. package/dist/email.d.ts +9 -1
  13. package/dist/email.d.ts.map +1 -1
  14. package/dist/email.js +99 -58
  15. package/dist/email.js.map +1 -1
  16. package/dist/index.d.ts +468 -64
  17. package/dist/index.d.ts.map +1 -1
  18. package/dist/listen.mjs +575 -738
  19. package/dist/listen.mjs.map +1 -1
  20. package/dist/runtime-node.d.mts +6564 -2934
  21. package/dist/runtime-node.d.mts.map +1 -1
  22. package/dist/runtime-node.mjs +6 -5
  23. package/dist/runtime-node.mjs.map +1 -1
  24. package/dist/server.d.ts +8931 -4198
  25. package/dist/server.d.ts.map +1 -1
  26. package/dist/server.js +5260 -3859
  27. package/dist/server.js.map +1 -1
  28. package/dist/server.mjs +5686 -3736
  29. package/dist/server.mjs.map +1 -1
  30. package/dist/server2.mjs +101 -79
  31. package/dist/server2.mjs.map +1 -1
  32. package/package.json +9 -6
  33. package/src/admin/api/routes.ts +103 -33
  34. package/src/admin/audit/service.ts +25 -6
  35. package/src/admin/documents/service.ts +361 -0
  36. package/src/admin/errors.ts +9 -1
  37. package/src/admin/index.ts +6 -0
  38. package/src/admin/operations/service.ts +7 -0
  39. package/src/admin/reports/service.ts +147 -0
  40. package/src/admin/schema.ts +6 -0
  41. package/src/client/api.ts +36 -21
  42. package/src/client/auth/navigation.ts +6 -5
  43. package/src/client/index.ts +7 -7
  44. package/src/client/portal/account.ts +25 -0
  45. package/src/client/portal/administration.ts +56 -0
  46. package/src/client/{ai.ts → services.ts} +44 -27
  47. package/src/contract/account.ts +58 -2
  48. package/src/contract/admin/index.ts +67 -3
  49. package/src/contract/collaboration.ts +6 -1
  50. package/src/contract/discovery.ts +22 -4
  51. package/src/contract/documents.ts +27 -0
  52. package/src/contract/entitlements.ts +2 -2
  53. package/src/contract/index.ts +63 -37
  54. package/src/contract/relay.ts +6 -0
  55. package/src/contract/reports.ts +42 -0
  56. package/src/contract/services.ts +233 -0
  57. package/src/contract/sharing.ts +3 -9
  58. package/src/email/render.ts +26 -0
  59. package/src/email/schema.ts +11 -1
  60. package/src/email/templates/brand.ts +1 -1
  61. package/src/runtime/cloudflare/environment.ts +31 -6
  62. package/src/runtime/cloudflare/index.ts +1 -1
  63. package/src/runtime/cloudflare/logging.ts +15 -0
  64. package/src/runtime/cloudflare/maintenance.ts +31 -0
  65. package/src/runtime/cloudflare/relay/index.ts +176 -0
  66. package/src/runtime/cloudflare/relay/sockets.ts +112 -0
  67. package/src/runtime/cloudflare/worker.ts +89 -26
  68. package/src/runtime/node/application.ts +5 -3
  69. package/src/runtime/node/bootstrap.ts +20 -4
  70. package/src/runtime/node/cleanup.ts +16 -3
  71. package/src/runtime/node/entitlements/main.ts +14 -11
  72. package/src/runtime/node/index.ts +1 -1
  73. package/src/runtime/node/migrate.ts +5 -1
  74. package/src/runtime/node/relay.ts +8 -2
  75. package/src/runtime/node/request-id.ts +13 -0
  76. package/src/runtime/node/server.ts +71 -15
  77. package/src/runtime/node/{ai-relay → services}/main.ts +6 -6
  78. package/src/runtime/node/{ai-relay → services}/server.ts +18 -18
  79. package/src/runtime/postgres/database.ts +20 -1
  80. package/src/runtime/posthog/errors.ts +32 -0
  81. package/src/runtime/s3/objects.ts +50 -1
  82. package/src/runtime/site/server.ts +33 -4
  83. package/src/server/account/actions.ts +5 -0
  84. package/src/server/account/deletion.ts +302 -0
  85. package/src/server/account/routes.ts +47 -3
  86. package/src/server/account/terms.ts +42 -0
  87. package/src/server/api/router.ts +12 -7
  88. package/src/server/api/types.ts +9 -0
  89. package/src/server/app.ts +93 -15
  90. package/src/server/auth/access-control.ts +63 -0
  91. package/src/server/auth/adapter.ts +11 -0
  92. package/src/server/auth/endpoints.ts +5 -1
  93. package/src/server/auth/factory.ts +30 -3
  94. package/src/server/auth/index.ts +10 -1
  95. package/src/server/cleanup/configured.ts +12 -5
  96. package/src/server/cleanup/documents.ts +165 -71
  97. package/src/server/cleanup/objects.ts +10 -2
  98. package/src/server/cleanup/revisions.ts +24 -7
  99. package/src/server/cleanup/uploads.ts +8 -2
  100. package/src/server/cleanup/worker.ts +5 -2
  101. package/src/server/collaboration/index.ts +3 -0
  102. package/src/server/collaboration/relay/connection.ts +196 -0
  103. package/src/server/collaboration/relay/documents.ts +13 -10
  104. package/src/server/collaboration/relay/hub.ts +279 -195
  105. package/src/server/collaboration/relay/index.ts +3 -1
  106. package/src/server/collaboration/relay/log.ts +79 -0
  107. package/src/server/collaboration/relay/persistence.ts +147 -22
  108. package/src/server/collaboration/relay/resume.ts +28 -0
  109. package/src/server/collaboration/routes.ts +27 -16
  110. package/src/server/collaboration/service.ts +27 -31
  111. package/src/server/config/deployment.ts +35 -4
  112. package/src/server/config/public.ts +5 -4
  113. package/src/server/config/schema.ts +104 -16
  114. package/src/server/db/expressions.ts +15 -1
  115. package/src/server/db/migrate.ts +34 -114
  116. package/src/server/db/migrations/001_accounts.ts +79 -0
  117. package/src/server/db/migrations/002_workspaces.ts +48 -0
  118. package/src/server/db/migrations/003_storage.ts +44 -0
  119. package/src/server/db/migrations/004_documents.ts +193 -0
  120. package/src/server/db/migrations/005_sharing.ts +95 -0
  121. package/src/server/db/migrations/006_operations.ts +110 -0
  122. package/src/server/db/migrations/007_trash.ts +33 -0
  123. package/src/server/db/migrations/008_terms.ts +18 -0
  124. package/src/server/db/migrations/schema.ts +71 -0
  125. package/src/server/db/tables.ts +70 -7
  126. package/src/server/documents/access.ts +66 -55
  127. package/src/server/documents/errors.ts +7 -0
  128. package/src/server/documents/images.ts +2 -1
  129. package/src/server/documents/limits.ts +9 -9
  130. package/src/server/documents/purge.ts +75 -0
  131. package/src/server/documents/repository.ts +2 -0
  132. package/src/server/documents/routes.ts +132 -80
  133. package/src/server/documents/service.ts +18 -4
  134. package/src/server/documents/summary.ts +1 -0
  135. package/src/server/documents/trash.ts +203 -0
  136. package/src/server/documents/versions/checkpoint.ts +2 -1
  137. package/src/server/documents/versions/history.ts +11 -7
  138. package/src/server/documents/versions/routes.ts +36 -25
  139. package/src/server/documents/versions/service.ts +22 -4
  140. package/src/server/errors/index.ts +99 -0
  141. package/src/server/errors/posthog.ts +80 -0
  142. package/src/server/index.ts +56 -16
  143. package/src/server/internal/index.ts +3 -3
  144. package/src/server/internal/routes.ts +7 -15
  145. package/src/server/internal/schema.ts +1 -1
  146. package/src/server/internal/service.ts +17 -23
  147. package/src/server/invitations/budget.ts +66 -0
  148. package/src/server/invitations/index.ts +1 -0
  149. package/src/server/logging.ts +76 -0
  150. package/src/server/objects/types.ts +12 -0
  151. package/src/server/pictures/routes.ts +4 -2
  152. package/src/server/pictures/service.ts +2 -1
  153. package/src/server/policy/database.ts +39 -25
  154. package/src/server/policy/index.ts +2 -1
  155. package/src/server/policy/keys.ts +5 -2
  156. package/src/server/policy/policy.ts +1 -0
  157. package/src/server/policy/schema.ts +2 -2
  158. package/src/server/policy/service.ts +12 -9
  159. package/src/server/policy/static.ts +5 -5
  160. package/src/server/policy/workspace.ts +41 -0
  161. package/src/server/quota/reconcile.ts +2 -0
  162. package/src/server/quota/storage.ts +69 -31
  163. package/src/server/rate-limit/cleanup.ts +17 -5
  164. package/src/server/rate-limit/index.ts +4 -1
  165. package/src/server/rate-limit/middleware.ts +100 -12
  166. package/src/server/reports/index.ts +8 -0
  167. package/src/server/reports/routes.ts +37 -0
  168. package/src/server/reports/service.ts +123 -0
  169. package/src/server/services/errors.ts +7 -0
  170. package/src/server/services/index.ts +8 -0
  171. package/src/server/{ai → services}/keys.ts +11 -11
  172. package/src/server/services/routes.ts +47 -0
  173. package/src/server/{ai/service.ts → services/tokens.ts} +23 -20
  174. package/src/server/sharing/capabilities/service.ts +68 -22
  175. package/src/server/sharing/document-routes.ts +30 -22
  176. package/src/server/sharing/grants/service.ts +11 -18
  177. package/src/server/sharing/http.ts +4 -0
  178. package/src/server/sharing/invitations/service.ts +21 -11
  179. package/src/server/sharing/policy.ts +9 -14
  180. package/src/server/sharing/routes.ts +108 -85
  181. package/src/server/sharing/service.ts +1 -17
  182. package/src/server/validation.ts +59 -2
  183. package/src/server/workspaces/index.ts +1 -0
  184. package/src/server/workspaces/members.ts +89 -38
  185. package/src/server/workspaces/repository.ts +2 -1
  186. package/src/server/workspaces/routes.ts +67 -45
  187. package/src/{ai-relay → services}/app.ts +48 -44
  188. package/src/{ai-relay → services}/config.ts +25 -19
  189. package/src/services/db/index.ts +10 -0
  190. package/src/services/db/migrate.ts +37 -0
  191. package/src/{ai-relay/db/migrations/001_usage.ts → services/db/migrations/001_baseline.ts} +18 -7
  192. package/src/services/db/schema.ts +22 -0
  193. package/src/services/index.ts +18 -0
  194. package/src/{ai-relay → services}/metering.ts +2 -2
  195. package/src/{ai-relay → services}/usage.ts +43 -19
  196. package/dist/bin/ai-relay.mjs.map +0 -1
  197. package/src/ai-relay/db/index.ts +0 -10
  198. package/src/ai-relay/db/migrate.ts +0 -32
  199. package/src/ai-relay/db/schema.ts +0 -20
  200. package/src/ai-relay/index.ts +0 -18
  201. package/src/contract/ai.ts +0 -150
  202. package/src/runtime/cloudflare/relay.ts +0 -136
  203. package/src/server/ai/errors.ts +0 -7
  204. package/src/server/ai/index.ts +0 -4
  205. package/src/server/ai/routes.ts +0 -43
  206. package/src/server/auth/workspace-access.ts +0 -23
  207. package/src/server/db/migrations/001_foundation.ts +0 -164
  208. package/src/server/db/migrations/002_upload_cleanup_claims.ts +0 -34
  209. package/src/server/db/migrations/003_document_cleanup_claims.ts +0 -23
  210. package/src/server/db/migrations/004_document_sharing.ts +0 -82
  211. package/src/server/db/migrations/005_document_collaboration_epoch.ts +0 -13
  212. package/src/server/db/migrations/006_invitation_continuation.ts +0 -27
  213. package/src/server/db/migrations/007_collaboration_state.ts +0 -25
  214. package/src/server/db/migrations/008_storage_reservations.ts +0 -37
  215. package/src/server/db/migrations/009_workspace_entitlements.ts +0 -22
  216. package/src/server/db/migrations/010_upload_finalization.ts +0 -22
  217. package/src/server/db/migrations/011_upload_finalization_lease.ts +0 -12
  218. package/src/server/db/migrations/012_transactional_email.ts +0 -48
  219. package/src/server/db/migrations/013_admin_enrollment.ts +0 -54
  220. package/src/server/db/migrations/014_enrollment_email_kinds.ts +0 -30
  221. package/src/server/db/migrations/015_cloud_rate_limit.ts +0 -22
  222. package/src/server/db/migrations/016_auth_schema.ts +0 -16
  223. package/src/server/db/migrations/017_authentication_email_kinds.ts +0 -32
  224. package/src/server/db/migrations/018_authentication_mfa.ts +0 -78
  225. package/src/server/db/migrations/019_better_auth_account_issuer.ts +0 -42
  226. package/src/server/db/migrations/020_workspace_invitations.ts +0 -57
  227. package/src/server/db/migrations/021_document_versions.ts +0 -66
  228. package/src/server/db/migrations/022_entitlement_external_version.ts +0 -17
  229. package/src/server/db/migrations/023_document_images.ts +0 -61
  230. package/src/server/db/migrations/024_account_activity.ts +0 -66
  231. package/src/server/db/migrations/025_table_names.ts +0 -58
  232. package/src/server/db/migrations/026_user_picture.ts +0 -30
  233. package/src/server/db/migrations/027_document_kind.ts +0 -19
  234. package/src/server/db/migrations/helpers.ts +0 -78
  235. /package/dist/bin/{ai-relay.d.mts → services.d.mts} +0 -0
@@ -1,8 +1,6 @@
1
1
  import {
2
2
  COLLAB_ACTIONS,
3
- createRelayClientAssembler,
4
3
  encodeSyncReply,
5
- decodeRelayClientFrame,
6
4
  decodeRoomUpdate,
7
5
  MAX_COLLAB_MESSAGE_BYTES,
8
6
  RELAY_CLOSE,
@@ -10,23 +8,24 @@ import {
10
8
  relayServerMessages,
11
9
  type RelayServerFrame
12
10
  } from '#cloud/contract'
11
+ import { logCloud, loggedError } from '#cloud/server/logging'
13
12
  import * as Y from 'yjs'
14
13
 
15
- import { authorizeRelayTicket, principalKey, type RelayAuthorization } from './authorize'
14
+ import type { RelayAuthorization } from './authorize'
16
15
  import { stampAwareness } from './awareness'
16
+ import {
17
+ openRelayConnection,
18
+ type ConnectionRelay,
19
+ type RelayConnection,
20
+ type RelayPeer,
21
+ type RelayRoomFrame,
22
+ type RelaySocket
23
+ } from './connection'
24
+ import { openRoomLog, type RoomLog } from './log'
17
25
  import type { CollaborationStateStore } from './persistence'
26
+ import type { RelayPeerState } from './resume'
18
27
 
19
- /** One client's socket, as a runtime adapter hands it to the relay. */
20
- export type RelaySocket = {
21
- send(data: Uint8Array): void
22
- close(code: number, reason: string): void
23
- }
24
-
25
- /** What the adapter reports back about that socket. */
26
- export type RelayConnection = {
27
- receive(data: Uint8Array): void
28
- closed(): void
29
- }
28
+ export type { RelayConnection, RelaySocket } from './connection'
30
29
 
31
30
  export type CollaborationRelayOptions = {
32
31
  authSecret: string
@@ -38,9 +37,14 @@ export type CollaborationRelayOptions = {
38
37
  documentId?: string
39
38
  /**
40
39
  * The most one socket message may carry. An editor's larger frames arrive as fragments joined
41
- * up to `MAX_COLLAB_MESSAGE_BYTES`; anyone else's frame must fit in one message.
40
+ * up to `maximumRoomBytes`; anyone else's frame must fit in one message.
42
41
  */
43
42
  maximumMessageBytes: number
43
+ /**
44
+ * The most a room's document may hold, and so the most one editor's update may carry:
45
+ * `MAX_COLLAB_MESSAGE_BYTES` unless the runtime holds rooms in less memory.
46
+ */
47
+ maximumRoomBytes?: number
44
48
  maximumConnectionsPerRoom: number
45
49
  /** The participant limit policy sets for a document, or null for none. */
46
50
  maximumParticipants?: (documentId: string) => Promise<number | null>
@@ -51,6 +55,12 @@ export type CollaborationRelayOptions = {
51
55
  currentRoomEpoch?: (documentId: string) => Promise<number | null>
52
56
  /** How long after a change the room's document is saved. */
53
57
  persistDelayMs?: number
58
+ /**
59
+ * How many logged updates, or how many of their bytes, a room saves before it writes a fresh
60
+ * snapshot in their place; a closing room always does.
61
+ */
62
+ compactAfterUpdates?: number
63
+ compactAfterBytes?: number
54
64
  /** How long a room whose last save failed waits before saving again. */
55
65
  persistRetryMs?: number
56
66
  /** How long a socket may stay open without presenting a ticket. */
@@ -61,24 +71,43 @@ export type CollaborationRelayOptions = {
61
71
  /** Messages a viewer may send; their document writes never leave the relay. */
62
72
  const VIEWER_ACTIONS = new Set<string>([COLLAB_ACTIONS.syncStep1, COLLAB_ACTIONS.awareness])
63
73
 
64
- type Peer = {
65
- id: string
66
- socket: RelaySocket
67
- authorization: RelayAuthorization
68
- presenceClients: Set<number>
69
- expiry?: ReturnType<typeof setTimeout>
70
- }
74
+ type Peer = RelayPeer
75
+
76
+ /**
77
+ * A room's document and the log its saves go to, loaded when a message first needs them, with an
78
+ * upper bound on the document's size: what was saved plus every update since. The bound is
79
+ * measured again only once it reaches `maximumRoomBytes` after something was deleted, since only
80
+ * a deletion can leave the document smaller than the bound.
81
+ */
82
+ type RoomContent = { document: Y.Doc; log: RoomLog; bytes: number; deletedSinceMeasure: boolean }
71
83
 
72
84
  type Room = {
73
85
  id: string
74
86
  identity: { documentId: string; roomEpoch: number }
75
- document: Y.Doc
76
87
  peers: Map<string, Peer>
77
88
  persistTimer: ReturnType<typeof setTimeout> | null
89
+ /**
90
+ * Null until a message needs the document: peers resumed after the runtime woke the relay
91
+ * exchange presence without it, so a wake for presence alone loads nothing from the store.
92
+ */
93
+ content: RoomContent | null
94
+ loading: Promise<RoomContent> | null
78
95
  }
79
96
 
80
97
  export type CollaborationRelay = ReturnType<typeof createCollaborationRelay>
81
98
 
99
+ /** Whether `update` adds nothing and deletes only content `document` already holds. */
100
+ function deletesOnlyKnown(document: Y.Doc, update: Uint8Array): boolean {
101
+ const { from, to } = Y.parseUpdateMetaV2(update)
102
+ if (![...to].every(([client, clock]) => from.get(client) === clock)) return false
103
+ const { ds } = Y.decodeUpdateV2(update)
104
+ for (const [client, deletions] of ds.clients) {
105
+ const held = Y.getState(document.store, client)
106
+ if (deletions.some((deletion) => deletion.clock + deletion.len > held)) return false
107
+ }
108
+ return true
109
+ }
110
+
82
111
  /**
83
112
  * A runtime-neutral relay for collaboration rooms. It forwards room messages between peers and
84
113
  * joins every room as a peer itself: it holds the room's document, answers sync requests from
@@ -93,8 +122,18 @@ export function createCollaborationRelay(options: CollaborationRelayOptions) {
93
122
  let shuttingDown = false
94
123
  const persistDelayMs = options.persistDelayMs ?? 2_000
95
124
  const persistRetryMs = options.persistRetryMs ?? 30_000
125
+ const logLimits = {
126
+ updates: options.compactAfterUpdates ?? 500,
127
+ bytes: options.compactAfterBytes ?? 4 * 1024 * 1024
128
+ }
129
+ const maximumRoomBytes = Math.min(
130
+ options.maximumRoomBytes ?? MAX_COLLAB_MESSAGE_BYTES,
131
+ MAX_COLLAB_MESSAGE_BYTES
132
+ )
96
133
  const authTimeoutMs = options.authTimeoutMs ?? 10_000
97
- const report = options.onError ?? ((message, error) => console.error(`[Cloud] ${message}`, error))
134
+ const report =
135
+ options.onError ??
136
+ ((message, error) => logCloud('error', message, { source: 'relay', error: loggedError(error) }))
98
137
 
99
138
  function send(socket: RelaySocket, frame: RelayServerFrame) {
100
139
  for (const message of relayServerMessages(frame)) socket.send(message)
@@ -115,20 +154,11 @@ export function createCollaborationRelay(options: CollaborationRelayOptions) {
115
154
  // A room reopened while its last save is in flight must load what that save writes, or,
116
155
  // when the save failed, take the room it kept.
117
156
  await closing.get(authorization.roomId)
118
- const kept = rooms.get(authorization.roomId)
119
- if (kept) return kept
120
- const document = new Y.Doc()
121
- const state = await options.store.load(identity)
122
- if (state) Y.applyUpdate(document, state)
123
- const room: Room = {
124
- id: authorization.roomId,
125
- identity,
126
- document,
127
- peers: new Map(),
128
- persistTimer: null
129
- }
130
- document.on('update', () => schedulePersist(room))
131
- rooms.set(room.id, room)
157
+ // A room resumed without its document loads it in place; a new one joins the rooms once
158
+ // loaded, so a restore while it loads finds nothing to end and the epoch check turns it away.
159
+ const room = rooms.get(authorization.roomId) ?? newRoom(authorization.roomId, identity)
160
+ await contentOf(room)
161
+ if (!rooms.has(room.id)) rooms.set(room.id, room)
132
162
  return room
133
163
  })()
134
164
  opening.set(authorization.roomId, created)
@@ -139,15 +169,51 @@ export function createCollaborationRelay(options: CollaborationRelayOptions) {
139
169
  }
140
170
  }
141
171
 
142
- /** Saves the room's document; false when the store refused it. */
143
- async function persist(room: Room): Promise<boolean> {
172
+ function newRoom(id: string, identity: Room['identity']): Room {
173
+ return { id, identity, peers: new Map(), persistTimer: null, content: null, loading: null }
174
+ }
175
+
176
+ /** The room's document and log, loaded from the store the first time something needs them. */
177
+ function contentOf(room: Room): Promise<RoomContent> {
178
+ if (room.content) return Promise.resolve(room.content)
179
+ room.loading ??= openRoomLog(options.store, room.identity, logLimits).then(
180
+ ({ document, log, savedBytes }) => {
181
+ document.on('update', () => schedulePersist(room))
182
+ // The saved log may hold deletions, so the first time the bound reaches the limit, it is
183
+ // measured.
184
+ const content: RoomContent = { document, log, bytes: savedBytes, deletedSinceMeasure: true }
185
+ document.on('afterTransaction', (transaction: Y.Transaction) => {
186
+ if (transaction.deleteSet.clients.size > 0) content.deletedSinceMeasure = true
187
+ })
188
+ room.content = content
189
+ return content
190
+ },
191
+ (error: unknown) => {
192
+ room.loading = null
193
+ throw error
194
+ }
195
+ )
196
+ return room.loading
197
+ }
198
+
199
+ /** Saves what changed in the room since its last save; false when the store refused it. */
200
+ async function persist(room: Room, closing = false): Promise<boolean> {
144
201
  if (room.persistTimer) clearTimeout(room.persistTimer)
145
202
  room.persistTimer = null
203
+ // A room nobody needed the document of since it was resumed has nothing new to save, but
204
+ // a closing one loads it anyway, to fold what its earlier wakes logged into a snapshot.
205
+ if (!room.content && !closing) return true
146
206
  try {
147
- await options.store.store(room.identity, Y.encodeStateAsUpdate(room.document))
207
+ const { log } = await contentOf(room)
208
+ await log.save({ closing })
148
209
  return true
149
210
  } catch (error) {
150
- report('Saving a collaboration room failed:', error)
211
+ report('Saving a collaboration room failed', error)
212
+ // The room holds edits the store doesn't have; save again without waiting for another edit.
213
+ // A closing room retries through `saveAndClose`.
214
+ if (!closing && rooms.get(room.id) === room) {
215
+ room.persistTimer ??= setTimeout(() => void persist(room), persistRetryMs)
216
+ }
151
217
  return false
152
218
  }
153
219
  }
@@ -160,17 +226,24 @@ export function createCollaborationRelay(options: CollaborationRelayOptions) {
160
226
  * Applies a peer's V2 update to the room, inflated first for a sync reply. Rooms are saved as
161
227
  * V1 updates; only what peers exchange is V2.
162
228
  */
163
- function applyToRoom(room: Room, peer: Peer, namespace: string, payload: Uint8Array): boolean {
229
+ function applyToRoom(
230
+ content: RoomContent,
231
+ peer: Peer,
232
+ namespace: string,
233
+ payload: Uint8Array
234
+ ): boolean {
235
+ const { document } = content
164
236
  try {
165
237
  const update = decodeRoomUpdate(payload, {
166
- limit: MAX_COLLAB_MESSAGE_BYTES,
238
+ limit: maximumRoomBytes,
167
239
  compressed: namespace === COLLAB_ACTIONS.syncReply
168
240
  })
169
- if (!update) {
241
+ if (!update || !fits(content, update)) {
170
242
  peer.socket.close(RELAY_CLOSE.messageTooLarge, 'Collaboration update is too large')
171
243
  return false
172
244
  }
173
- Y.applyUpdateV2(room.document, update, peer.id)
245
+ Y.applyUpdateV2(document, update, peer.id)
246
+ content.bytes += update.byteLength
174
247
  return true
175
248
  } catch {
176
249
  peer.socket.close(RELAY_CLOSE.protocol, 'Malformed document update')
@@ -189,17 +262,35 @@ export function createCollaborationRelay(options: CollaborationRelayOptions) {
189
262
  const recipients = target
190
263
  ? [room.peers.get(target)]
191
264
  : [...room.peers.values()].filter((peer) => peer !== from)
192
- sendAll(
193
- recipients.flatMap((peer) => (peer ? [peer.socket] : [])),
194
- frame
195
- )
265
+ sendAll(deliverable(recipients.flatMap((peer) => (peer ? [peer] : []))), frame)
266
+ }
267
+
268
+ /**
269
+ * Whether the room's document stays within its limit after `update`. An update that only
270
+ * deletes what the room holds shrinks it, so it always fits; one that deletes what the room has
271
+ * not received waits in the document, so it counts like any other. Otherwise the bound grows by
272
+ * the update; past the limit, the document is measured only if a deletion may have shrunk it
273
+ * since the last measure, so edits near the limit never encode the whole document each time.
274
+ */
275
+ function fits(content: RoomContent, update: Uint8Array): boolean {
276
+ if (deletesOnlyKnown(content.document, update)) return true
277
+ if (content.bytes + update.byteLength <= maximumRoomBytes) return true
278
+ if (!content.deletedSinceMeasure) return false
279
+ content.bytes = Y.encodeStateAsUpdateV2(content.document).byteLength
280
+ content.deletedSinceMeasure = false
281
+ return content.bytes + update.byteLength <= maximumRoomBytes
196
282
  }
197
283
 
198
- function handleServerMessage(room: Room, peer: Peer, namespace: string, payload: Uint8Array) {
284
+ function handleServerMessage(
285
+ content: RoomContent,
286
+ peer: Peer,
287
+ namespace: string,
288
+ payload: Uint8Array
289
+ ) {
199
290
  if (namespace === COLLAB_ACTIONS.syncStep1) {
200
291
  let reply: Uint8Array
201
292
  try {
202
- reply = encodeSyncReply(room.document, payload)
293
+ reply = encodeSyncReply(content.document, payload)
203
294
  } catch {
204
295
  peer.socket.close(RELAY_CLOSE.protocol, 'Malformed state vector')
205
296
  return
@@ -211,12 +302,20 @@ export function createCollaborationRelay(options: CollaborationRelayOptions) {
211
302
  payload: reply
212
303
  })
213
304
  } else if (namespace === COLLAB_ACTIONS.syncReply) {
214
- applyToRoom(room, peer, namespace, payload)
305
+ applyToRoom(content, peer, namespace, payload)
215
306
  }
216
307
  }
217
308
 
309
+ /** Whether handling a message reads or writes the room's document. */
310
+ function needsDocument(namespace: string, target: string | null): boolean {
311
+ if (namespace === COLLAB_ACTIONS.yjsUpdate || namespace === COLLAB_ACTIONS.syncReply)
312
+ return true
313
+ return target === RELAY_SERVER_PEER && namespace === COLLAB_ACTIONS.syncStep1
314
+ }
315
+
218
316
  function handleMessage(
219
317
  room: Room,
318
+ content: RoomContent | null,
220
319
  peer: Peer,
221
320
  namespace: string,
222
321
  target: string | null,
@@ -234,29 +333,74 @@ export function createCollaborationRelay(options: CollaborationRelayOptions) {
234
333
  (other) => other !== peer && other.presenceClients.has(clientId)
235
334
  )
236
335
  )
336
+ const known = peer.presenceClients.size
237
337
  for (const clientId of stamped.claimed) peer.presenceClients.add(clientId)
338
+ if (peer.presenceClients.size !== known) remember(peer)
238
339
  if (!stamped.update) return
239
340
  forwarded = stamped.update
240
341
  }
241
342
 
242
343
  if (target === RELAY_SERVER_PEER) {
243
- handleServerMessage(room, peer, namespace, payload)
344
+ if (content) handleServerMessage(content, peer, namespace, payload)
244
345
  return
245
346
  }
246
347
  if (
247
348
  (namespace === COLLAB_ACTIONS.yjsUpdate || namespace === COLLAB_ACTIONS.syncReply) &&
248
- !applyToRoom(room, peer, namespace, payload)
349
+ !(content && applyToRoom(content, peer, namespace, payload))
249
350
  ) {
250
351
  return
251
352
  }
252
353
  forward(room, peer, namespace, target, forwarded)
253
354
  }
254
355
 
255
- function scheduleExpiry(peer: Peer) {
256
- clearTimeout(peer.expiry)
257
- peer.expiry = setTimeout(
258
- () => peer.socket.close(RELAY_CLOSE.expired, 'Collaboration ticket expired'),
259
- Math.max(0, peer.authorization.expiresAt - Date.now())
356
+ /**
357
+ * Whether the peer's ticket still holds, closing its socket when it lapsed. It is checked when
358
+ * the relay hears from the peer and before it sends the peer anything, never by a timer, so a
359
+ * runtime can set an idle relay aside while sockets stay open.
360
+ */
361
+ function current(peer: Peer): boolean {
362
+ if (Date.now() < peer.authorization.expiresAt) return true
363
+ peer.socket.close(RELAY_CLOSE.expired, 'Collaboration ticket expired')
364
+ return false
365
+ }
366
+
367
+ /** The sockets of peers whose tickets still hold. */
368
+ function deliverable(peers: Iterable<Peer>): RelaySocket[] {
369
+ return [...peers].filter(current).map((peer) => peer.socket)
370
+ }
371
+
372
+ function remember(peer: Peer) {
373
+ peer.socket.remember?.({
374
+ peerId: peer.id,
375
+ authorization: peer.authorization,
376
+ presenceClients: [...peer.presenceClients]
377
+ })
378
+ }
379
+
380
+ /** Handles a room message from an admitted peer; a promise while the room's document loads. */
381
+ function deliver(peer: Peer, frame: RelayRoomFrame): Promise<void> | undefined {
382
+ const room = rooms.get(peer.authorization.roomId)
383
+ if (!room) {
384
+ peer.socket.close(RELAY_CLOSE.unauthorized, 'Expected a collaboration ticket')
385
+ return undefined
386
+ }
387
+ if (!current(peer)) return undefined
388
+ const { namespace, target, payload } = frame
389
+ if (room.content || !needsDocument(namespace, target)) {
390
+ handleMessage(room, room.content, peer, namespace, target, payload)
391
+ return undefined
392
+ }
393
+ return contentOf(room).then(
394
+ (content) => {
395
+ // The peer may have left, or its ticket lapsed, while the document loaded.
396
+ if (room.peers.get(peer.id) !== peer || !current(peer)) return undefined
397
+ handleMessage(room, content, peer, namespace, target, payload)
398
+ return undefined
399
+ },
400
+ (error: unknown) => {
401
+ report('Loading a collaboration room failed', error)
402
+ peer.socket.close(RELAY_CLOSE.protocol, 'Could not open the collaboration room')
403
+ }
260
404
  )
261
405
  }
262
406
 
@@ -264,13 +408,15 @@ export function createCollaborationRelay(options: CollaborationRelayOptions) {
264
408
  socket: RelaySocket,
265
409
  authorization: RelayAuthorization
266
410
  ): Promise<Peer | null> {
267
- const limit = await options.maximumParticipants?.(authorization.documentId)
268
411
  let room: Room
412
+ let limit: number | null | undefined
269
413
  let epoch: number | null | undefined
270
- // The room's last peer may have left while it opened, or a restore ended it; a closed room
271
- // takes no newcomers. The epoch is read once the room is open, so a restore cannot slip between.
414
+ // The room's last peer may have left while it opened, or a restore or takedown ended it; a
415
+ // closed room takes no newcomers. The limit and epoch are read once the room is open, so
416
+ // neither a restore nor a takedown can slip between.
272
417
  do {
273
418
  room = await openRoom(authorization)
419
+ limit = await options.maximumParticipants?.(authorization.documentId)
274
420
  epoch = await options.currentRoomEpoch?.(authorization.documentId)
275
421
  } while (rooms.get(room.id) !== room)
276
422
  if (epoch != null && authorization.roomEpoch < epoch) {
@@ -290,29 +436,30 @@ export function createCollaborationRelay(options: CollaborationRelayOptions) {
290
436
  authorization,
291
437
  presenceClients: new Set()
292
438
  }
293
- scheduleExpiry(peer)
439
+ remember(peer)
294
440
  send(socket, {
295
441
  type: 'welcome',
296
442
  peerId: peer.id,
297
443
  peers: [RELAY_SERVER_PEER, ...room.peers.keys()]
298
444
  })
299
- for (const other of room.peers.values()) send(other.socket, { type: 'join', peerId: peer.id })
445
+ sendAll(deliverable(room.peers.values()), { type: 'join', peerId: peer.id })
300
446
  room.peers.set(peer.id, peer)
301
447
  // Ask for what the newcomer has that the room lacks, such as edits made offline.
302
448
  if (authorization.permission === 'edit') {
449
+ const { document } = await contentOf(room)
303
450
  send(socket, {
304
451
  type: 'message',
305
452
  namespace: COLLAB_ACTIONS.syncStep1,
306
453
  from: RELAY_SERVER_PEER,
307
- payload: Y.encodeStateVector(room.document)
454
+ payload: Y.encodeStateVector(document)
308
455
  })
309
456
  }
310
457
  return peer
311
458
  }
312
459
 
313
460
  async function saveAndClose(room: Room) {
314
- if ((await persist(room)) || shuttingDown) {
315
- room.document.destroy()
461
+ if ((await persist(room, true)) || shuttingDown) {
462
+ room.content?.document.destroy()
316
463
  return
317
464
  }
318
465
  // The room holds edits the store doesn't have: keep it for whoever joins next, and save
@@ -336,137 +483,58 @@ export function createCollaborationRelay(options: CollaborationRelayOptions) {
336
483
  }
337
484
 
338
485
  function depart(peer: Peer) {
339
- clearTimeout(peer.expiry)
340
486
  const room = rooms.get(peer.authorization.roomId)
341
487
  if (!room?.peers.delete(peer.id)) return
342
- for (const other of room.peers.values()) send(other.socket, { type: 'leave', peerId: peer.id })
488
+ sendAll(deliverable(room.peers.values()), { type: 'leave', peerId: peer.id })
343
489
  void leaveRoom(room)
344
490
  }
345
491
 
492
+ /**
493
+ * A socket's connection: a new one, which presents a ticket before it joins, or one resumed with
494
+ * the peer it already is.
495
+ */
496
+ const connectionRelay: ConnectionRelay = {
497
+ authSecret: options.authSecret,
498
+ documentId: options.documentId,
499
+ maximumMessageBytes: options.maximumMessageBytes,
500
+ maximumUpdateBytes: maximumRoomBytes,
501
+ authTimeoutMs,
502
+ admit,
503
+ refreshed: remember,
504
+ deliver,
505
+ depart,
506
+ report
507
+ }
508
+
346
509
  return {
347
510
  connect(socket: RelaySocket): RelayConnection {
348
- let peer: Peer | null = null
349
- /** Read through an object, since it changes across awaits and between messages. */
350
- const state = { closed: false, authenticating: false }
351
- const isClosed = () => state.closed
352
- const queued: Uint8Array[] = []
353
- let queuedBytes = 0
354
- // Only an editor's writes can be large: a viewer's frames, and a peer's before it is
355
- // admitted, fit in one message, so nobody else can make the relay hold more.
356
- const assembler = createRelayClientAssembler(() =>
357
- peer?.authorization.permission === 'edit'
358
- ? MAX_COLLAB_MESSAGE_BYTES
359
- : options.maximumMessageBytes
360
- )
361
- const authTimer = setTimeout(
362
- () => socket.close(RELAY_CLOSE.unauthorized, 'Expected a collaboration ticket'),
363
- authTimeoutMs
364
- )
365
-
366
- async function authenticate(token: string) {
367
- let authorization: RelayAuthorization
368
- try {
369
- authorization = await authorizeRelayTicket(token, options.authSecret)
370
- } catch {
371
- socket.close(RELAY_CLOSE.unauthorized, 'Invalid collaboration ticket')
372
- return
373
- }
374
- if (options.documentId !== undefined && authorization.documentId !== options.documentId) {
375
- socket.close(RELAY_CLOSE.unauthorized, 'Ticket is for a different document')
376
- return
377
- }
378
- if (state.closed) return
379
- if (peer) {
380
- // A refreshed ticket keeps the peer in its room as the same person; a lowered
381
- // permission applies from now on.
382
- const current = peer.authorization
383
- if (
384
- authorization.roomId !== current.roomId ||
385
- principalKey(authorization.principal) !== principalKey(current.principal)
386
- ) {
387
- socket.close(RELAY_CLOSE.unauthorized, 'Ticket is for a different room or person')
388
- return
389
- }
390
- peer.authorization = authorization
391
- scheduleExpiry(peer)
392
- return
393
- }
394
- clearTimeout(authTimer)
395
- const admitted = await admit(socket, authorization)
396
- // The socket may have closed while the room loaded.
397
- if (admitted && isClosed()) depart(admitted)
398
- else peer = admitted
399
- }
400
-
401
- function process(data: Uint8Array) {
402
- if (data.byteLength > options.maximumMessageBytes) {
403
- socket.close(RELAY_CLOSE.messageTooLarge, 'Collaboration message is too large')
404
- return
405
- }
406
- const assembled = assembler.push(data)
407
- if (assembled.type === 'partial') return
408
- if (assembled.type === 'too-large') {
409
- socket.close(RELAY_CLOSE.messageTooLarge, 'Collaboration message is too large')
410
- return
411
- }
412
- const frame = assembled.type === 'frame' ? decodeRelayClientFrame(assembled.data) : null
413
- if (!frame) {
414
- socket.close(RELAY_CLOSE.protocol, 'Malformed collaboration frame')
415
- return
416
- }
417
- if (frame.type === 'auth') {
418
- state.authenticating = true
419
- void authenticate(frame.token)
420
- .catch((error: unknown) => {
421
- report('Admitting a collaboration peer failed:', error)
422
- socket.close(RELAY_CLOSE.protocol, 'Could not open the collaboration room')
423
- })
424
- .finally(() => {
425
- state.authenticating = false
426
- flush()
427
- })
428
- return
429
- }
430
- const room = peer && rooms.get(peer.authorization.roomId)
431
- if (!peer || !room) {
432
- socket.close(RELAY_CLOSE.unauthorized, 'Expected a collaboration ticket')
433
- return
434
- }
435
- handleMessage(room, peer, frame.namespace, frame.target, frame.payload)
436
- }
511
+ return openRelayConnection(socket, null, connectionRelay)
512
+ },
437
513
 
438
- function flush() {
439
- while (!state.authenticating && !state.closed && queued.length > 0) {
440
- const next = queued.shift()
441
- if (!next) continue
442
- queuedBytes -= next.byteLength
443
- process(next)
444
- }
514
+ /**
515
+ * Takes back a socket whose peer was admitted before the runtime set the relay aside, as the
516
+ * peer it was, without loading the room's document until a message needs it. A peer whose
517
+ * ticket lapsed meanwhile is closed.
518
+ */
519
+ resume(socket: RelaySocket, saved: RelayPeerState): RelayConnection {
520
+ const { authorization } = saved
521
+ const identity = { documentId: authorization.documentId, roomEpoch: authorization.roomEpoch }
522
+ let room = rooms.get(authorization.roomId)
523
+ if (!room) {
524
+ room = newRoom(authorization.roomId, identity)
525
+ rooms.set(room.id, room)
445
526
  }
446
-
447
- return {
448
- receive(data) {
449
- if (state.closed) return
450
- queuedBytes += data.byteLength
451
- // Before admission a peer has sent its ticket and has nothing else to say, so what
452
- // waits for the room to open is held to one message's worth.
453
- if (!peer && queuedBytes > options.maximumMessageBytes) {
454
- queued.length = 0
455
- queuedBytes = 0
456
- socket.close(RELAY_CLOSE.messageTooLarge, 'Too much sent before joining the room')
457
- return
458
- }
459
- queued.push(data)
460
- flush()
461
- },
462
- closed() {
463
- state.closed = true
464
- queued.length = 0
465
- queuedBytes = 0
466
- clearTimeout(authTimer)
467
- if (peer) depart(peer)
468
- }
527
+ const peer: Peer = {
528
+ id: saved.peerId,
529
+ socket,
530
+ authorization,
531
+ presenceClients: new Set(saved.presenceClients)
469
532
  }
533
+ room.peers.set(peer.id, peer)
534
+ const connection = openRelayConnection(socket, peer, connectionRelay)
535
+ // After the runtime holds the connection, which its socket's close reports to.
536
+ queueMicrotask(() => current(peer))
537
+ return connection
470
538
  },
471
539
 
472
540
  /**
@@ -480,11 +548,26 @@ export function createCollaborationRelay(options: CollaborationRelayOptions) {
480
548
  rooms.delete(room.id)
481
549
  if (room.persistTimer) clearTimeout(room.persistTimer)
482
550
  for (const peer of room.peers.values()) {
483
- clearTimeout(peer.expiry)
484
551
  peer.socket.close(RELAY_CLOSE.restored, 'The document was restored')
485
552
  }
486
553
  room.peers.clear()
487
- room.document.destroy()
554
+ room.content?.document.destroy()
555
+ }
556
+ },
557
+
558
+ /**
559
+ * Ends a taken-down document's rooms for everyone in them, saving what each room holds, so
560
+ * a restore opens them again with every edit. Nobody gets back in meanwhile: tickets are
561
+ * issued only for documents people can reach.
562
+ */
563
+ closeDocument(documentId: string) {
564
+ for (const room of rooms.values()) {
565
+ if (room.identity.documentId !== documentId) continue
566
+ for (const peer of room.peers.values()) {
567
+ peer.socket.close(RELAY_CLOSE.unauthorized, 'The document was taken down')
568
+ }
569
+ room.peers.clear()
570
+ void leaveRoom(room)
488
571
  }
489
572
  },
490
573
 
@@ -497,9 +580,11 @@ export function createCollaborationRelay(options: CollaborationRelayOptions) {
497
580
  payload: new Uint8Array()
498
581
  }
499
582
  sendAll(
500
- [...rooms.values()]
501
- .filter((room) => room.identity.documentId === documentId)
502
- .flatMap((room) => [...room.peers.values()].map((peer) => peer.socket)),
583
+ deliverable(
584
+ [...rooms.values()]
585
+ .filter((room) => room.identity.documentId === documentId)
586
+ .flatMap((room) => [...room.peers.values()])
587
+ ),
503
588
  frame
504
589
  )
505
590
  },
@@ -510,7 +595,6 @@ export function createCollaborationRelay(options: CollaborationRelayOptions) {
510
595
  await Promise.all(
511
596
  [...rooms.values()].map(async (room) => {
512
597
  for (const peer of room.peers.values()) {
513
- clearTimeout(peer.expiry)
514
598
  peer.socket.close(1001, 'Server shutting down')
515
599
  }
516
600
  room.peers.clear()