@sixb/server 0.2.1 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. package/README.md +3 -3
  2. package/dist/artifacts/frame.d.ts.map +1 -1
  3. package/dist/auth/scope.d.ts +3 -0
  4. package/dist/auth/scope.d.ts.map +1 -1
  5. package/dist/auth/shared-access.d.ts +23 -0
  6. package/dist/auth/shared-access.d.ts.map +1 -1
  7. package/dist/connectors/http.d.ts +6 -1
  8. package/dist/connectors/http.d.ts.map +1 -1
  9. package/dist/index.js +2876 -667
  10. package/dist/index.js.map +43 -36
  11. package/dist/live/awareness.d.ts +48 -0
  12. package/dist/live/awareness.d.ts.map +1 -0
  13. package/dist/live/room.d.ts +198 -0
  14. package/dist/live/room.d.ts.map +1 -0
  15. package/dist/live/rooms.d.ts +67 -0
  16. package/dist/live/rooms.d.ts.map +1 -0
  17. package/dist/registerRoutes.d.ts.map +1 -1
  18. package/dist/routes/actions.d.ts +3 -3
  19. package/dist/routes/agent-approvals.d.ts +21 -11
  20. package/dist/routes/agent-approvals.d.ts.map +1 -1
  21. package/dist/routes/agent-children.d.ts +4 -4
  22. package/dist/routes/agent-computers.d.ts +146 -1
  23. package/dist/routes/agent-computers.d.ts.map +1 -1
  24. package/dist/routes/agent-runners.d.ts.map +1 -1
  25. package/dist/routes/agents.d.ts +152 -60
  26. package/dist/routes/agents.d.ts.map +1 -1
  27. package/dist/routes/artifact-frames.d.ts +2 -1
  28. package/dist/routes/artifact-frames.d.ts.map +1 -1
  29. package/dist/routes/artifacts.d.ts +1713 -178
  30. package/dist/routes/artifacts.d.ts.map +1 -1
  31. package/dist/routes/auth.d.ts +21 -21
  32. package/dist/routes/connector-connection-runs.d.ts +124 -5
  33. package/dist/routes/connector-connection-runs.d.ts.map +1 -1
  34. package/dist/routes/connector-connections.d.ts +9 -1
  35. package/dist/routes/connector-connections.d.ts.map +1 -1
  36. package/dist/routes/datasets.d.ts +5 -5
  37. package/dist/routes/events.d.ts +4 -4
  38. package/dist/routes/lake.d.ts +91 -0
  39. package/dist/routes/lake.d.ts.map +1 -0
  40. package/dist/routes/links.d.ts +6 -6
  41. package/dist/routes/logs.d.ts +2 -2
  42. package/dist/routes/model-generations.d.ts +2 -2
  43. package/dist/routes/objects.d.ts +6 -6
  44. package/dist/routes/objects.d.ts.map +1 -1
  45. package/dist/routes/observed-agent-threads.d.ts +21 -11
  46. package/dist/routes/observed-agent-threads.d.ts.map +1 -1
  47. package/dist/routes/ontology.d.ts +6 -6
  48. package/dist/routes/pipelines.d.ts +13 -9
  49. package/dist/routes/pipelines.d.ts.map +1 -1
  50. package/dist/routes/projections.d.ts +27 -27
  51. package/dist/routes/share-grants.d.ts +9 -9
  52. package/dist/routes/shared-access.d.ts +57 -2
  53. package/dist/routes/shared-access.d.ts.map +1 -1
  54. package/dist/routes/status.d.ts +11 -15
  55. package/dist/routes/status.d.ts.map +1 -1
  56. package/dist/routes/syncs.d.ts +9 -6
  57. package/dist/routes/syncs.d.ts.map +1 -1
  58. package/dist/routes/telemetry.d.ts +7 -7
  59. package/dist/routes/webhooks.d.ts.map +1 -1
  60. package/dist/routes/workflows.d.ts +14 -14
  61. package/dist/routes/ws/artifacts.d.ts +57 -0
  62. package/dist/routes/ws/artifacts.d.ts.map +1 -0
  63. package/dist/routes/ws/events.d.ts +8 -7
  64. package/dist/routes/ws/events.d.ts.map +1 -1
  65. package/dist/routes/ws/index.d.ts +2 -1
  66. package/dist/routes/ws/index.d.ts.map +1 -1
  67. package/dist/routes/ws/logs.d.ts +10 -10
  68. package/dist/routes/ws/shared-sockets.d.ts +52 -0
  69. package/dist/routes/ws/shared-sockets.d.ts.map +1 -0
  70. package/dist/schemas/actions.d.ts +7 -7
  71. package/dist/schemas/activity.d.ts +16 -16
  72. package/dist/schemas/agent-approvals.d.ts +27 -27
  73. package/dist/schemas/agents.d.ts +642 -196
  74. package/dist/schemas/agents.d.ts.map +1 -1
  75. package/dist/schemas/ai-accounting.d.ts +41 -41
  76. package/dist/schemas/ai-accounting.d.ts.map +1 -1
  77. package/dist/schemas/artifacts.d.ts +2106 -127
  78. package/dist/schemas/artifacts.d.ts.map +1 -1
  79. package/dist/schemas/auth.d.ts +63 -63
  80. package/dist/schemas/connectors.d.ts +180 -20
  81. package/dist/schemas/connectors.d.ts.map +1 -1
  82. package/dist/schemas/datasets.d.ts +44 -44
  83. package/dist/schemas/event-payloads.d.ts +2 -2
  84. package/dist/schemas/events.d.ts +26 -26
  85. package/dist/schemas/events.d.ts.map +1 -1
  86. package/dist/schemas/files.d.ts +12 -0
  87. package/dist/schemas/files.d.ts.map +1 -1
  88. package/dist/schemas/lake.d.ts +251 -0
  89. package/dist/schemas/lake.d.ts.map +1 -0
  90. package/dist/schemas/lineage.d.ts +56 -56
  91. package/dist/schemas/logs.d.ts +18 -18
  92. package/dist/schemas/models.d.ts +4 -4
  93. package/dist/schemas/objects.d.ts.map +1 -1
  94. package/dist/schemas/ontology.d.ts +14 -14
  95. package/dist/schemas/pipelines.d.ts +86 -64
  96. package/dist/schemas/pipelines.d.ts.map +1 -1
  97. package/dist/schemas/projections.d.ts +160 -160
  98. package/dist/schemas/projections.d.ts.map +1 -1
  99. package/dist/schemas/share-grants.d.ts +30 -30
  100. package/dist/schemas/shared-access.d.ts +17 -4
  101. package/dist/schemas/shared-access.d.ts.map +1 -1
  102. package/dist/schemas/status.d.ts +59 -79
  103. package/dist/schemas/status.d.ts.map +1 -1
  104. package/dist/schemas/syncs.d.ts +38 -22
  105. package/dist/schemas/syncs.d.ts.map +1 -1
  106. package/dist/schemas/telemetry.d.ts +26 -26
  107. package/dist/schemas/workflows.d.ts +71 -71
  108. package/dist/server.d.ts +13 -0
  109. package/dist/server.d.ts.map +1 -1
  110. package/dist/utils/ws.d.ts +3 -0
  111. package/dist/utils/ws.d.ts.map +1 -1
  112. package/package.json +11 -7
  113. package/src/artifacts/frame.ts +4 -13
  114. package/src/auth/scope.ts +10 -1
  115. package/src/auth/shared-access.ts +108 -12
  116. package/src/connectors/http.ts +35 -1
  117. package/src/live/README.md +136 -0
  118. package/src/live/awareness.ts +141 -0
  119. package/src/live/room.ts +933 -0
  120. package/src/live/rooms.ts +288 -0
  121. package/src/registerRoutes.ts +2 -0
  122. package/src/routes/agent-computers.ts +208 -14
  123. package/src/routes/agent-runners.ts +1 -0
  124. package/src/routes/agents.ts +52 -7
  125. package/src/routes/artifact-frames.ts +40 -12
  126. package/src/routes/artifacts.ts +713 -28
  127. package/src/routes/connector-connection-runs.ts +7 -12
  128. package/src/routes/connector-connections.ts +10 -7
  129. package/src/routes/lake.ts +73 -0
  130. package/src/routes/objects.ts +3 -9
  131. package/src/routes/pipelines.ts +1 -0
  132. package/src/routes/shared-access.ts +53 -1
  133. package/src/routes/status.ts +9 -4
  134. package/src/routes/syncs.ts +1 -0
  135. package/src/routes/webhooks.ts +44 -3
  136. package/src/routes/ws/artifacts.ts +138 -0
  137. package/src/routes/ws/events.ts +7 -2
  138. package/src/routes/ws/index.ts +13 -2
  139. package/src/routes/ws/shared-sockets.ts +152 -0
  140. package/src/schemas/agents.ts +71 -1
  141. package/src/schemas/ai-accounting.ts +1 -0
  142. package/src/schemas/artifacts.ts +255 -18
  143. package/src/schemas/connectors.ts +17 -2
  144. package/src/schemas/files.ts +12 -0
  145. package/src/schemas/lake.ts +53 -0
  146. package/src/schemas/objects.ts +1 -0
  147. package/src/schemas/pipelines.ts +2 -0
  148. package/src/schemas/projections.ts +2 -2
  149. package/src/schemas/shared-access.ts +16 -0
  150. package/src/schemas/status.ts +16 -22
  151. package/src/schemas/syncs.ts +2 -0
  152. package/src/server.ts +35 -4
  153. package/src/utils/ws.ts +7 -0
@@ -0,0 +1,933 @@
1
+ import { ArtifactRequestError, type BlobStorage, type Storage } from "@sixb/core"
2
+ import {
3
+ AGENT_AWARENESS_FIELD,
4
+ type ArtifactAgentPresence,
5
+ type ArtifactLiveAccess,
6
+ type ArtifactLiveSaveState,
7
+ type ArtifactLiveServerMessage,
8
+ type ArtifactLiveUser,
9
+ decodeLiveData,
10
+ documentFiles,
11
+ encodeLiveData,
12
+ USER_AWARENESS_FIELD,
13
+ } from "@sixb/core/artifacts/live"
14
+ import type { Broker, BrokerRecord } from "@sixb/core/broker"
15
+ import {
16
+ AGENT_PRESENCE_MS,
17
+ artifactLiveStream,
18
+ artifactPresenceStream,
19
+ loadArtifactDocument,
20
+ mergeIntoDocument,
21
+ parseArtifactLiveRecord,
22
+ readDocumentState,
23
+ readLiveFiles,
24
+ sameArtifactFiles,
25
+ storeLiveFiles,
26
+ writeLiveArtifact,
27
+ } from "@sixb/core/internal/artifacts/live"
28
+ import { reportArtifactLiveWriteFailure } from "@sixb/core/internal/error-reporting"
29
+ import type { Sixb } from "@sixb/core/internal/request-execution"
30
+ import type { ArtifactFile } from "@sixb/core/storage"
31
+ import * as decoding from "lib0/decoding"
32
+ import * as encoding from "lib0/encoding"
33
+ import * as awarenessProtocol from "y-protocols/awareness"
34
+ import * as syncProtocol from "y-protocols/sync"
35
+ import * as Y from "yjs"
36
+ import { MAX_AWARENESS_CLIENTS, vouchedAwareness } from "./awareness"
37
+
38
+ // One artifact held live on this server: its Yjs document as storage keeps it, the connections
39
+ // that edit or watch it, the other servers that hold it too, and the write-back of what they
40
+ // change. See README.md.
41
+
42
+ /** After the last change, how long the document waits before it writes itself back. */
43
+ export const SAVE_IDLE_MS = 1_500
44
+ /** How long changes made without pause wait, at most, to be written back. */
45
+ export const SAVE_MAX_MS = 10_000
46
+ /** How long an editor's right to change the artifact holds before it is read again. */
47
+ export const ACCESS_RECHECK_MS = 2_000
48
+ /** The first wait before a failed write-back is tried again; it doubles, up to the next. */
49
+ const RETRY_MS = 2_000
50
+ const RETRY_MAX_MS = 30_000
51
+ /** How old another server's awareness update may be for a server joining late to apply it. */
52
+ const AWARENESS_REPLAY_MS = awarenessProtocol.outdatedTimeout
53
+ /**
54
+ * How long an edit heard from another server waits before this one writes it back too, in case
55
+ * the server that heard it never does: long enough that it normally has.
56
+ */
57
+ export const HEARD_SAVE_MS = 3 * SAVE_MAX_MS
58
+ /** How often every member's right to read and change the artifact is read again. */
59
+ export const MEMBERS_RECHECK_MS = 60_000
60
+ /** How often, at most, this server shares its people's awareness with the others. */
61
+ export const AWARENESS_SHARE_MS = 100
62
+
63
+ /** A connection, as a room sees it: who it is, through its SDK, and how to reach it. */
64
+ export interface LiveConnection {
65
+ readonly id: string
66
+ /** The connection's SDK, as the person who opened it: every write goes through it. */
67
+ readonly sixb: Sixb
68
+ /** Who the connection is when its SDK names nobody: a shared link's guest. */
69
+ readonly user?: () => Promise<ArtifactLiveUser>
70
+ send(message: ArtifactLiveServerMessage): void
71
+ }
72
+
73
+ export interface LiveRoomHost {
74
+ readonly projectId: string
75
+ readonly storage: Pick<Storage, "artifacts" | "auth">
76
+ readonly blobStorage: Pick<BlobStorage, "open" | "put">
77
+ readonly broker: Broker
78
+ /** This server, among those that hold the artifact live. */
79
+ readonly instance: string
80
+ /** What reports to the project's `onError`: changes that could never be stored. */
81
+ readonly errorReporterHost: object
82
+ }
83
+
84
+ interface Member {
85
+ readonly connection: LiveConnection
86
+ /** Who the connection is, in every awareness state it sets. */
87
+ user: ArtifactLiveUser
88
+ access: ArtifactLiveAccess
89
+ /** When the right to edit was last read. */
90
+ checkedAt: number
91
+ /** The awareness clients the connection set. */
92
+ readonly clients: Set<number>
93
+ }
94
+
95
+ /** The open version the document last took in: what a write-back is based on. */
96
+ interface Synced {
97
+ readonly revision: number
98
+ readonly title: string
99
+ readonly files: readonly ArtifactFile[]
100
+ }
101
+
102
+ /** The origin of what the server takes from storage: written already, never written back. */
103
+ const MERGED = Symbol("merged")
104
+ /**
105
+ * The origin of what another server holding the artifact heard from its editors: shown here,
106
+ * never shared again, and written back by that server, or by this one should that one never do.
107
+ */
108
+ const REMOTE = Symbol("remote")
109
+ /** The origin of a connection's awareness taken back as it leaves: shared, so others see it go. */
110
+ const LEFT = Symbol("left")
111
+
112
+ export class LiveRoom {
113
+ readonly doc: Y.Doc
114
+ readonly awareness: awarenessProtocol.Awareness
115
+ private readonly members = new Map<string, Member>()
116
+ private synced: Synced
117
+ /**
118
+ * What the document held when it was last written back, as a Yjs snapshot, its insertions and
119
+ * its deletions: a client forgets the changes it covers.
120
+ */
121
+ private savedSnapshot: Uint8Array
122
+ private dirty = false
123
+ /** What another server heard is not known to be written yet. */
124
+ private unwritten = false
125
+ private author: LiveConnection | undefined
126
+ /** The last connection that joined to edit: who writes back what another server heard. */
127
+ private lastEditor: LiveConnection | undefined
128
+ /**
129
+ * The document as it is, again: a client's update is tried on it first, so one that would leave
130
+ * the document unreadable is refused before anyone gets it.
131
+ */
132
+ private shadow: Y.Doc
133
+ private state: ArtifactLiveSaveState = "saved"
134
+ private error: string | undefined
135
+ /** The last write-back's failure, which a loss of the changes reports. */
136
+ private failure: unknown
137
+ private attempts = 0
138
+ private idle: ReturnType<typeof setTimeout> | undefined
139
+ private deadline: ReturnType<typeof setTimeout> | undefined
140
+ private retry: ReturnType<typeof setTimeout> | undefined
141
+ private heardSave: ReturnType<typeof setTimeout> | undefined
142
+ private recheck: ReturnType<typeof setInterval> | undefined
143
+ /** The awareness clients whose state waits to be shared with the other servers. */
144
+ private readonly unshared = new Set<number>()
145
+ private sharing: ReturnType<typeof setTimeout> | undefined
146
+ private queue: Promise<void> = Promise.resolve()
147
+ private readonly agents = new Map<
148
+ string,
149
+ { readonly boards: Set<string>; expiry: ReturnType<typeof setTimeout> }
150
+ >()
151
+ /** The awareness clients other servers speak for. */
152
+ private readonly remoteClients = new Set<number>()
153
+ private unsubscribe: (() => void) | undefined
154
+ private closed = false
155
+
156
+ private constructor(
157
+ private readonly host: LiveRoomHost,
158
+ readonly artifactId: string,
159
+ doc: Y.Doc,
160
+ synced: Synced,
161
+ private readonly onClose: (room: LiveRoom) => void
162
+ ) {
163
+ this.synced = synced
164
+ this.savedSnapshot = Y.encodeSnapshot(Y.snapshot(doc))
165
+ this.doc = doc
166
+ this.shadow = copyOf(doc)
167
+ this.awareness = new awarenessProtocol.Awareness(this.doc)
168
+ // The server is no one: it is here only for the Agent, while a turn changes boards.
169
+ this.awareness.setLocalState(null)
170
+
171
+ this.doc.on("update", (update: Uint8Array, origin: unknown) => this.updated(update, origin))
172
+ this.awareness.on("update", (changes: AwarenessChanges, origin: unknown) =>
173
+ this.presenceChanged(changes, origin)
174
+ )
175
+ }
176
+
177
+ /**
178
+ * The room of an artifact, its document loaded as storage keeps it; a reason instead for an
179
+ * artifact that is gone or is not edited live. Whoever opens it must already have been allowed
180
+ * to read it.
181
+ */
182
+ static async open(
183
+ host: LiveRoomHost,
184
+ artifactId: string,
185
+ onClose: (room: LiveRoom) => void
186
+ ): Promise<LiveRoom | "not_found" | "not_live"> {
187
+ const storage = requireArtifacts(host)
188
+ const artifact = await storage.get({ projectId: host.projectId, id: artifactId })
189
+ if (!artifact) return "not_found"
190
+ if (artifact.kind !== "canvas") return "not_live"
191
+
192
+ const version = await storage.getVersion({
193
+ projectId: host.projectId,
194
+ artifactId,
195
+ version: artifact.latestVersion,
196
+ })
197
+ if (!version) return "not_found"
198
+
199
+ const doc = await loadArtifactDocument(host.blobStorage, artifact, version.files)
200
+ const room = new LiveRoom(
201
+ host,
202
+ artifactId,
203
+ doc,
204
+ { revision: version.revision, title: version.title, files: version.files },
205
+ onClose
206
+ )
207
+ // What editors on other servers changed since the last write-back is on the live stream. A
208
+ // room that cannot follow it is no room: it stops, timers and all.
209
+ try {
210
+ await room.follow()
211
+ } catch (error) {
212
+ room.close()
213
+ throw error
214
+ }
215
+ return room
216
+ }
217
+
218
+ get isClosed(): boolean {
219
+ return this.closed
220
+ }
221
+
222
+ get revision(): number {
223
+ return this.synced.revision
224
+ }
225
+
226
+ /** Adds a connection, which gets the document and who is here; sync is the client's to start. */
227
+ join(connection: LiveConnection, access: ArtifactLiveAccess, user: ArtifactLiveUser): void {
228
+ if (access === "edit") this.lastEditor = connection
229
+ this.recheck ??= setInterval(() => this.enqueue(() => this.recheckAccess()), MEMBERS_RECHECK_MS)
230
+ this.members.set(connection.id, {
231
+ connection,
232
+ user,
233
+ access,
234
+ checkedAt: Date.now(),
235
+ clients: new Set(),
236
+ })
237
+ connection.send({
238
+ type: "joined",
239
+ artifactId: this.artifactId,
240
+ access,
241
+ revision: this.synced.revision,
242
+ user,
243
+ })
244
+
245
+ const encoder = encoding.createEncoder()
246
+ syncProtocol.writeSyncStep1(encoder, this.doc)
247
+ this.sendSync(connection, encoding.toUint8Array(encoder))
248
+
249
+ const clients = [...this.awareness.getStates().keys()]
250
+ if (clients.length > 0) {
251
+ connection.send({
252
+ type: "awareness",
253
+ artifactId: this.artifactId,
254
+ data: encodeLiveData(awarenessProtocol.encodeAwarenessUpdate(this.awareness, clients)),
255
+ })
256
+ }
257
+ connection.send(this.status())
258
+ }
259
+
260
+ /**
261
+ * Removes a connection and what it said of itself. The last editor to leave writes the document
262
+ * back at once, and the last connection closes the room once nothing waits to be written: a
263
+ * client that comes back loads the same history from storage.
264
+ */
265
+ leave(connectionId: string): void {
266
+ const member = this.members.get(connectionId)
267
+ if (!member) return
268
+
269
+ this.members.delete(connectionId)
270
+ awarenessProtocol.removeAwarenessStates(this.awareness, [...member.clients], LEFT)
271
+ if (![...this.members.values()].some((each) => each.access === "edit")) this.save()
272
+ this.closeWhenDone()
273
+ }
274
+
275
+ /**
276
+ * A y-protocols sync message from a connection. One that watches may ask for the document, never
277
+ * change it; one that edits is read again at most every {@link ACCESS_RECHECK_MS}.
278
+ */
279
+ async sync(connectionId: string, bytes: Uint8Array): Promise<void> {
280
+ const member = this.members.get(connectionId)
281
+ if (!member) return
282
+
283
+ const type = decoding.readVarUint(decoding.createDecoder(bytes))
284
+ if (type !== syncProtocol.messageYjsSyncStep1 && !(await this.mayEdit(member))) {
285
+ member.connection.send({
286
+ type: "error",
287
+ artifactId: this.artifactId,
288
+ message: "[SixbServer] You watch this artifact: your changes are not kept.",
289
+ })
290
+ return
291
+ }
292
+
293
+ // A change is tried on the shadow first: one that leaves the document unreadable is refused.
294
+ if (type !== syncProtocol.messageYjsSyncStep1 && !this.readable(bytes)) {
295
+ member.connection.send({
296
+ type: "error",
297
+ artifactId: this.artifactId,
298
+ message:
299
+ "[SixbServer] That change leaves the artifact unreadable: it is not kept. Load the artifact again.",
300
+ })
301
+ return
302
+ }
303
+
304
+ const encoder = encoding.createEncoder()
305
+ syncProtocol.readSyncMessage(decoding.createDecoder(bytes), encoder, this.doc, member)
306
+ if (encoding.length(encoder) > 0)
307
+ this.sendSync(member.connection, encoding.toUint8Array(encoder))
308
+
309
+ // Answering step 1, the server asks for what the client holds that it does not: a client
310
+ // that edited offline, or before a reconnect, sends it back.
311
+ if (type === syncProtocol.messageYjsSyncStep1 && member.access === "edit") {
312
+ const ask = encoding.createEncoder()
313
+ syncProtocol.writeSyncStep1(ask, this.doc)
314
+ this.sendSync(member.connection, encoding.toUint8Array(ask))
315
+ }
316
+ }
317
+
318
+ /**
319
+ * An awareness update from a connection: what it points at, under the name the server knows it
320
+ * by, for the clients it may speak for.
321
+ */
322
+ presence(connectionId: string, data: string): void {
323
+ const member = this.members.get(connectionId)
324
+ const bytes = decodeLiveData(data)
325
+ if (!member || !bytes) return
326
+
327
+ const vouched = vouchedAwareness(bytes, {
328
+ user: member.user,
329
+ mayUse: (client) => this.mayUse(member, client),
330
+ })
331
+ if (vouched) awarenessProtocol.applyAwarenessUpdate(this.awareness, vouched, member)
332
+ }
333
+
334
+ /**
335
+ * The connection is someone else now, as the server read it: each awareness state it set takes
336
+ * the new `user`, shared as its own updates are.
337
+ */
338
+ rename(connectionId: string, user: ArtifactLiveUser): void {
339
+ const member = this.members.get(connectionId)
340
+ if (!member) return
341
+ member.user = user
342
+
343
+ const states = this.awareness.getStates()
344
+ const clients = [...member.clients].filter((client) => states.get(client))
345
+ if (clients.length === 0) return
346
+
347
+ const encoder = encoding.createEncoder()
348
+ encoding.writeVarUint(encoder, clients.length)
349
+ for (const client of clients) {
350
+ const clock = this.awareness.meta.get(client)?.clock ?? 0
351
+ encoding.writeVarUint(encoder, client)
352
+ encoding.writeVarUint(encoder, clock + 1)
353
+ encoding.writeVarString(
354
+ encoder,
355
+ JSON.stringify({ ...states.get(client), [USER_AWARENESS_FIELD]: user })
356
+ )
357
+ }
358
+ awarenessProtocol.applyAwarenessUpdate(this.awareness, encoding.toUint8Array(encoder), member)
359
+ }
360
+
361
+ /**
362
+ * The artifact was written at `revision`, by this room or anyone else. Every write is made into
363
+ * the stored document, which the room takes in: what its editors changed meanwhile stays.
364
+ */
365
+ written(revision: number): void {
366
+ // Whatever changed, who may read or change the artifact may have too.
367
+ this.enqueue(() => this.recheckAccess())
368
+ if (revision <= this.synced.revision) return
369
+ this.enqueue(() => this.reconcile(revision))
370
+ }
371
+
372
+ /** Closes the room if nobody holds it and nothing waits to be written: a join gave up. */
373
+ closeIfUnheld(): void {
374
+ this.closeWhenDone()
375
+ }
376
+
377
+ /** Writes the document back now, if it changed. */
378
+ save(): void {
379
+ this.clearTimers()
380
+ this.enqueue(() => this.flush())
381
+ }
382
+
383
+ /** Settles once what the room has queued, a write-back or a merge, is done. */
384
+ settled(): Promise<void> {
385
+ return this.queue
386
+ }
387
+
388
+ /**
389
+ * Stops the room. What it still holds unwritten is the caller's to save first: changes it could
390
+ * not write are reported lost.
391
+ */
392
+ close(): void {
393
+ if (this.closed) return
394
+ const lost = this.dirty
395
+ this.closed = true
396
+ if (lost) this.lose()
397
+ this.clearTimers()
398
+ if (this.recheck) clearInterval(this.recheck)
399
+ if (this.sharing) clearTimeout(this.sharing)
400
+ for (const agent of this.agents.values()) clearTimeout(agent.expiry)
401
+ this.unsubscribe?.()
402
+ this.awareness.destroy()
403
+ this.doc.destroy()
404
+ this.shadow.destroy()
405
+ this.onClose(this)
406
+ }
407
+
408
+ // ── The document ────────────────────────────────────────────────────
409
+
410
+ private updated(update: Uint8Array, origin: unknown): void {
411
+ // The shadow takes every change; one it was tried on already changes nothing.
412
+ Y.applyUpdate(this.shadow, update)
413
+
414
+ const message = encoding.createEncoder()
415
+ syncProtocol.writeUpdate(message, update)
416
+ const data = encoding.toUint8Array(message)
417
+ for (const member of this.members.values()) {
418
+ if (member !== origin) this.sendSync(member.connection, data)
419
+ }
420
+
421
+ // What an editor here changed reaches the other servers at once, and waits to be written back
422
+ // from here. What another server heard is written back there, and here too, later, should that
423
+ // server never do it; what came from storage is written already.
424
+ if (origin === REMOTE) {
425
+ this.heard()
426
+ return
427
+ }
428
+ if (!(origin instanceof Object && this.isMember(origin))) return
429
+ this.changed(origin)
430
+ this.share("update", update)
431
+ }
432
+
433
+ private heard(): void {
434
+ this.unwritten = true
435
+ this.heardSave ??= setTimeout(() => {
436
+ this.heardSave = undefined
437
+ this.save()
438
+ }, HEARD_SAVE_MS)
439
+ }
440
+
441
+ private changed(member: Member): void {
442
+ this.dirty = true
443
+ this.author = member.connection
444
+ this.setState("unsaved")
445
+
446
+ if (this.idle) clearTimeout(this.idle)
447
+ this.idle = setTimeout(() => this.save(), SAVE_IDLE_MS)
448
+ this.deadline ??= setTimeout(() => this.save(), SAVE_MAX_MS)
449
+ }
450
+
451
+ /**
452
+ * Writes the document back into the open version, as the last editor: a normal write, which
453
+ * bumps the revision and tells every reader. The write carries the document itself, which
454
+ * storage merges into its own, so whatever was written meanwhile stays, there and back. A
455
+ * failure leaves the changes waiting, says so, and tries again.
456
+ */
457
+ private async flush(): Promise<void> {
458
+ if (this.closed || !(this.dirty || this.unwritten)) return this.closeWhenDone()
459
+
460
+ // What another server heard is written as whoever edits here, or last edited.
461
+ const author =
462
+ this.author ??
463
+ [...this.members.values()].find((each) => each.access === "edit")?.connection ??
464
+ this.lastEditor
465
+ if (!author) {
466
+ if (this.unwritten && !this.dirty) {
467
+ console.warn(
468
+ `[SixbServer] Nobody here may write back what another server heard of '${this.artifactId}'; that server writes it.`
469
+ )
470
+ this.unwritten = false
471
+ }
472
+ return this.closeWhenDone()
473
+ }
474
+
475
+ const wasDirty = this.dirty
476
+ this.dirty = false
477
+ this.unwritten = false
478
+ if (wasDirty) this.setState("saving")
479
+
480
+ try {
481
+ const update = Y.encodeStateAsUpdate(this.doc)
482
+ const written = Y.encodeSnapshot(Y.snapshot(this.doc))
483
+ const snapshot = documentFiles(this.doc)
484
+ const files = await storeLiveFiles(this.host.blobStorage, snapshot, this.synced.files)
485
+ if (sameArtifactFiles(files, this.synced.files)) {
486
+ this.saved(this.synced, written)
487
+ return this.closeWhenDone()
488
+ }
489
+
490
+ // The document holds no title: one given since stays.
491
+ const published = await writeLiveArtifact(author.sixb.artifacts, this.artifactId, {
492
+ title: this.synced.title,
493
+ baseTitle: this.synced.title,
494
+ files,
495
+ baseRevision: this.synced.revision,
496
+ update,
497
+ })
498
+
499
+ // Merged with a write made meanwhile: the stored document holds it too.
500
+ const stored = published.artifact.document
501
+ if (published.merged && stored) {
502
+ Y.applyUpdate(this.doc, await readDocumentState(this.host.blobStorage, stored.file), MERGED)
503
+ }
504
+ this.saved(
505
+ {
506
+ revision: published.version.revision,
507
+ title: published.version.title,
508
+ files: published.version.files,
509
+ },
510
+ written
511
+ )
512
+ } catch (error) {
513
+ this.failed(author, error)
514
+ }
515
+ this.closeWhenDone()
516
+ }
517
+
518
+ private saved(synced: Synced, written: Uint8Array): void {
519
+ this.synced = synced
520
+ this.savedSnapshot = written
521
+ this.attempts = 0
522
+ this.error = undefined
523
+ this.failure = undefined
524
+ this.setState(this.dirty ? "unsaved" : "saved", true)
525
+ if (this.dirty && !this.idle) this.idle = setTimeout(() => this.save(), SAVE_IDLE_MS)
526
+ }
527
+
528
+ /**
529
+ * Keeps the changes and says why they were not written. A refusal of what the document holds
530
+ * waits for the next change; one of who wrote it is tried as another editor. Either is lost once
531
+ * nobody is left to write it. Anything else is tried again later, for as long as the server
532
+ * runs: the changes are lost only if it stops first.
533
+ */
534
+ private failed(author: LiveConnection, error: unknown): void {
535
+ this.dirty = true
536
+ this.attempts += 1
537
+ this.error = error instanceof Error ? error.message : String(error)
538
+ this.failure = error
539
+ this.setState("failed", true)
540
+
541
+ const code = error instanceof ArtifactRequestError ? error.code : undefined
542
+ if (code === "artifact_not_found") {
543
+ for (const member of [...this.members.values()]) {
544
+ member.connection.send({ type: "left", artifactId: this.artifactId, reason: "not_found" })
545
+ }
546
+ this.members.clear()
547
+ // Deleted: there is nothing left to write the changes into.
548
+ this.dirty = false
549
+ this.close()
550
+ return
551
+ }
552
+
553
+ if (code === "not_shared" || code === "read_only" || code === "not_allowed") {
554
+ const member = this.members.get(author.id)
555
+ if (member) this.demote(member)
556
+ this.author = [...this.members.values()].find((each) => each.access === "edit")?.connection
557
+ if (this.author) this.scheduleRetry(0)
558
+ else if (this.members.size === 0) this.lose()
559
+ return
560
+ }
561
+
562
+ if (code === "invalid_artifact") {
563
+ if (this.members.size === 0) this.lose()
564
+ return
565
+ }
566
+
567
+ if (this.members.size === 0) {
568
+ console.warn(
569
+ `[SixbServer] Could not write back the live changes to artifact '${this.artifactId}' (attempt ${this.attempts}); trying again:`,
570
+ error
571
+ )
572
+ }
573
+ this.scheduleRetry(Math.min(RETRY_MS * 2 ** (this.attempts - 1), RETRY_MAX_MS))
574
+ }
575
+
576
+ /** The changes will never be stored: the project's `onError` hears of it, and they are dropped. */
577
+ private lose(): void {
578
+ reportArtifactLiveWriteFailure(this.host.errorReporterHost, this.failure, {
579
+ projectId: this.host.projectId,
580
+ artifactId: this.artifactId,
581
+ revision: this.synced.revision,
582
+ attempts: this.attempts,
583
+ })
584
+ this.dirty = false
585
+ this.closeWhenDone()
586
+ }
587
+
588
+ private scheduleRetry(delayMs: number): void {
589
+ if (this.retry) clearTimeout(this.retry)
590
+ this.retry = setTimeout(() => {
591
+ this.retry = undefined
592
+ this.save()
593
+ }, delayMs)
594
+ }
595
+
596
+ /**
597
+ * Takes in what someone else wrote: the stored document, which shares this one's history. One
598
+ * a write left behind is brought up to the content first, as every server that loads it does. An
599
+ * artifact that keeps none takes the change from the files the room last took in.
600
+ */
601
+ private async reconcile(revision: number): Promise<void> {
602
+ if (this.closed || revision <= this.synced.revision) return
603
+
604
+ const storage = requireArtifacts(this.host)
605
+ const artifact = await storage.get({ projectId: this.host.projectId, id: this.artifactId })
606
+ const version =
607
+ artifact &&
608
+ (await storage.getVersion({
609
+ projectId: this.host.projectId,
610
+ artifactId: this.artifactId,
611
+ version: artifact.latestVersion,
612
+ }))
613
+ if (!version) {
614
+ for (const member of this.members.values()) {
615
+ member.connection.send({ type: "left", artifactId: this.artifactId, reason: "not_found" })
616
+ }
617
+ this.members.clear()
618
+ this.close()
619
+ return
620
+ }
621
+ if (version.revision <= this.synced.revision) return
622
+
623
+ const { blobStorage } = this.host
624
+ if (artifact.document) {
625
+ const stored = await loadArtifactDocument(blobStorage, artifact, version.files)
626
+ Y.applyUpdate(this.doc, Y.encodeStateAsUpdate(stored), MERGED)
627
+ stored.destroy()
628
+ } else {
629
+ const [base, next] = await Promise.all([
630
+ readLiveFiles(blobStorage, this.synced.files),
631
+ readLiveFiles(blobStorage, version.files),
632
+ ])
633
+ mergeIntoDocument(this.doc, base, next, MERGED)
634
+ }
635
+ this.synced = { revision: version.revision, title: version.title, files: version.files }
636
+ // Nothing here waits to be written: the document is all stored now.
637
+ if (!this.dirty) this.savedSnapshot = Y.encodeSnapshot(Y.snapshot(this.doc))
638
+ this.setState(this.dirty ? this.state : "saved", true)
639
+ }
640
+
641
+ private enqueue(task: () => Promise<void>): void {
642
+ this.queue = this.queue.then(task).catch((error: unknown) => {
643
+ console.error(`[SixbServer] Live artifact '${this.artifactId}' failed:`, error)
644
+ })
645
+ }
646
+
647
+ /** Closes the room once nobody holds it and nothing waits to be written. */
648
+ private closeWhenDone(): void {
649
+ if (this.members.size === 0 && !this.dirty && !this.unwritten && !this.retry) this.close()
650
+ }
651
+
652
+ private clearTimers(): void {
653
+ for (const timer of [this.idle, this.deadline, this.retry, this.heardSave]) {
654
+ if (timer) clearTimeout(timer)
655
+ }
656
+ this.idle = undefined
657
+ this.deadline = undefined
658
+ this.retry = undefined
659
+ this.heardSave = undefined
660
+ }
661
+
662
+ /**
663
+ * Whether the document stays readable with a client's sync message: tried on the shadow, which
664
+ * is made again from the document when it does not.
665
+ */
666
+ private readable(bytes: Uint8Array): boolean {
667
+ try {
668
+ syncProtocol.readSyncMessage(
669
+ decoding.createDecoder(bytes),
670
+ encoding.createEncoder(),
671
+ this.shadow,
672
+ null
673
+ )
674
+ documentFiles(this.shadow)
675
+ return true
676
+ } catch {
677
+ this.shadow.destroy()
678
+ this.shadow = copyOf(this.doc)
679
+ return false
680
+ }
681
+ }
682
+
683
+ // ── Who may edit ────────────────────────────────────────────────────
684
+
685
+ private async mayEdit(member: Member): Promise<boolean> {
686
+ if (member.access !== "edit") return false
687
+ if (Date.now() - member.checkedAt < ACCESS_RECHECK_MS) return true
688
+
689
+ const access = await member.connection.sixb.artifacts.access(this.artifactId)
690
+ member.checkedAt = Date.now()
691
+ if (access === "owner" || access === "update") return true
692
+
693
+ if (access === "read") this.demote(member)
694
+ else {
695
+ this.members.delete(member.connection.id)
696
+ member.connection.send({ type: "left", artifactId: this.artifactId, reason: "not_shared" })
697
+ this.closeWhenDone()
698
+ }
699
+ return false
700
+ }
701
+
702
+ private demote(member: Member): void {
703
+ if (member.access === "watch") return
704
+ member.access = "watch"
705
+ member.connection.send({ type: "access", artifactId: this.artifactId, access: "watch" })
706
+ }
707
+
708
+ /**
709
+ * Reads every member's right to the artifact again, editors and watchers alike: who may no
710
+ * longer read it leaves, who may no longer change it watches, and who may now change it edits.
711
+ */
712
+ private async recheckAccess(): Promise<void> {
713
+ for (const member of [...this.members.values()]) {
714
+ if (this.closed) return
715
+ const access = await member.connection.sixb.artifacts.access(this.artifactId)
716
+ if (this.members.get(member.connection.id) !== member) continue
717
+ member.checkedAt = Date.now()
718
+
719
+ if (access === null || access === "none") {
720
+ this.members.delete(member.connection.id)
721
+ awarenessProtocol.removeAwarenessStates(this.awareness, [...member.clients], LEFT)
722
+ member.connection.send({ type: "left", artifactId: this.artifactId, reason: "not_shared" })
723
+ } else if (access === "read") {
724
+ this.demote(member)
725
+ } else if (member.access === "watch") {
726
+ member.access = "edit"
727
+ this.lastEditor = member.connection
728
+ member.connection.send({ type: "access", artifactId: this.artifactId, access: "edit" })
729
+ }
730
+ }
731
+ this.closeWhenDone()
732
+ }
733
+
734
+ private isMember(value: object): value is Member {
735
+ return [...this.members.values()].includes(value as Member)
736
+ }
737
+
738
+ /**
739
+ * Whether a connection may speak for an awareness client: one it already speaks for, or a new
740
+ * one nobody else does, the server's own included, up to {@link MAX_AWARENESS_CLIENTS}.
741
+ */
742
+ private mayUse(member: Member, client: number): boolean {
743
+ if (member.clients.has(client)) return true
744
+ if (client === this.doc.clientID || this.remoteClients.has(client)) return false
745
+ if (member.clients.size >= MAX_AWARENESS_CLIENTS) return false
746
+ return ![...this.members.values()].some((each) => each.clients.has(client))
747
+ }
748
+
749
+ // ── Presence ────────────────────────────────────────────────────────
750
+
751
+ private presenceChanged(changes: AwarenessChanges, origin: unknown): void {
752
+ const speaker =
753
+ origin === REMOTE
754
+ ? this.remoteClients
755
+ : origin instanceof Object && this.isMember(origin)
756
+ ? origin.clients
757
+ : null
758
+ for (const client of changes.added) speaker?.add(client)
759
+ for (const client of changes.removed) {
760
+ speaker?.delete(client)
761
+ this.remoteClients.delete(client)
762
+ }
763
+
764
+ const changed = [...changes.added, ...changes.updated, ...changes.removed]
765
+ const data = encodeLiveData(awarenessProtocol.encodeAwarenessUpdate(this.awareness, changed))
766
+ for (const member of this.members.values()) {
767
+ member.connection.send({ type: "awareness", artifactId: this.artifactId, data })
768
+ }
769
+
770
+ // People reach the other servers holding the artifact; the Agent's entry each server sets.
771
+ // People reach the other servers holding the artifact, a few times a second at most, each
772
+ // as they are by then; the Agent's entry each server sets.
773
+ if (origin === REMOTE) return
774
+ for (const client of changed) if (client !== this.doc.clientID) this.unshared.add(client)
775
+ if (this.unshared.size === 0 || this.sharing) return
776
+ this.sharing = setTimeout(() => {
777
+ this.sharing = undefined
778
+ const clients = [...this.unshared]
779
+ this.unshared.clear()
780
+ this.share("awareness", awarenessProtocol.encodeAwarenessUpdate(this.awareness, clients))
781
+ }, AWARENESS_SHARE_MS)
782
+ }
783
+
784
+ /** Tells the other servers holding the artifact what an editor here changed or pointed at. */
785
+ private share(name: "update" | "awareness", update: Uint8Array): void {
786
+ if (this.closed) return
787
+ // The room made sure of the streams when it started to follow them.
788
+ const payload = { instance: this.host.instance, update: encodeLiveData(update) }
789
+ const stream = name === "update" ? artifactLiveStream : artifactPresenceStream
790
+ this.host.broker
791
+ .append({
792
+ projectId: this.host.projectId,
793
+ streamId: stream(this.artifactId).id,
794
+ records: [{ name, payload }],
795
+ })
796
+ .catch((error: unknown) => {
797
+ // The document's own update still reaches storage with the next write-back.
798
+ console.warn(`[SixbServer] Could not share a live ${name} of '${this.artifactId}':`, error)
799
+ })
800
+ }
801
+
802
+ /**
803
+ * Reads the artifact's live stream: other servers' editors' updates and awareness, and the
804
+ * Agent's presence. From its start, so the room takes in what was changed elsewhere since the
805
+ * last write-back; an update it already holds changes nothing.
806
+ */
807
+ private async follow(): Promise<void> {
808
+ const { broker, projectId } = this.host
809
+ const unsubscribes: (() => void)[] = []
810
+ this.unsubscribe = () => {
811
+ for (const unsubscribe of unsubscribes) unsubscribe()
812
+ }
813
+ for (const stream of [
814
+ artifactLiveStream(this.artifactId),
815
+ artifactPresenceStream(this.artifactId),
816
+ ]) {
817
+ await broker.ensureStream({ projectId, stream })
818
+ unsubscribes.push(
819
+ await broker.subscribe({ projectId, streamId: stream.id, from: "earliest" }, (records) => {
820
+ for (const record of records) this.heardRecord(record)
821
+ })
822
+ )
823
+ }
824
+ }
825
+
826
+ private heardRecord(record: BrokerRecord): void {
827
+ if (this.closed) return
828
+ const live = parseArtifactLiveRecord(record)
829
+ if (!live) return
830
+ const age = Date.now() - Date.parse(record.publishedAt)
831
+
832
+ switch (live.name) {
833
+ case "update": {
834
+ const update = decodeLiveData(live.payload.update)
835
+ if (live.payload.instance === this.host.instance || !update) return
836
+ try {
837
+ Y.applyUpdate(this.doc, update, REMOTE)
838
+ } catch (error) {
839
+ console.warn(
840
+ `[SixbServer] Skipped an unreadable live update of '${this.artifactId}':`,
841
+ error
842
+ )
843
+ }
844
+ return
845
+ }
846
+ case "awareness": {
847
+ const update = decodeLiveData(live.payload.update)
848
+ if (live.payload.instance === this.host.instance || !update || age > AWARENESS_REPLAY_MS) {
849
+ return
850
+ }
851
+ awarenessProtocol.applyAwarenessUpdate(this.awareness, update, REMOTE)
852
+ return
853
+ }
854
+ case "agent": {
855
+ if (age > AGENT_PRESENCE_MS) return
856
+ const agent = this.agents.get(live.payload.runId)
857
+ if (agent) clearTimeout(agent.expiry)
858
+ const boards = new Set([...(agent?.boards ?? []), ...live.payload.boards])
859
+ const expiry = setTimeout(
860
+ () => this.agentEnded(live.payload.runId),
861
+ AGENT_PRESENCE_MS - age
862
+ )
863
+ this.agents.set(live.payload.runId, { boards, expiry })
864
+ this.showAgents()
865
+ return
866
+ }
867
+ case "agent-ended":
868
+ this.agentEnded(live.payload.runId)
869
+ return
870
+ }
871
+ }
872
+
873
+ private agentEnded(runId: string): void {
874
+ const agent = this.agents.get(runId)
875
+ if (!agent) return
876
+ clearTimeout(agent.expiry)
877
+ this.agents.delete(runId)
878
+ this.showAgents()
879
+ }
880
+
881
+ /** The server's awareness entry: each Agent turn at work on the canvas, and its boards. */
882
+ private showAgents(): void {
883
+ if (this.closed) return
884
+ const turns: ArtifactAgentPresence[] = [...this.agents].map(([runId, agent]) => ({
885
+ runId,
886
+ boards: [...agent.boards],
887
+ }))
888
+ this.awareness.setLocalState(turns.length === 0 ? null : { [AGENT_AWARENESS_FIELD]: turns })
889
+ }
890
+
891
+ // ── Messages ────────────────────────────────────────────────────────
892
+
893
+ private sendSync(connection: LiveConnection, bytes: Uint8Array): void {
894
+ connection.send({ type: "sync", artifactId: this.artifactId, data: encodeLiveData(bytes) })
895
+ }
896
+
897
+ private status(): ArtifactLiveServerMessage {
898
+ return {
899
+ type: "status",
900
+ artifactId: this.artifactId,
901
+ state: this.state,
902
+ revision: this.synced.revision,
903
+ snapshot: encodeLiveData(this.savedSnapshot),
904
+ ...(this.state === "failed" && this.error ? { error: this.error } : {}),
905
+ }
906
+ }
907
+
908
+ private setState(state: ArtifactLiveSaveState, announce = false): void {
909
+ if (state === this.state && !announce) return
910
+ this.state = state
911
+ const status = this.status()
912
+ for (const member of this.members.values()) member.connection.send(status)
913
+ }
914
+ }
915
+
916
+ interface AwarenessChanges {
917
+ readonly added: readonly number[]
918
+ readonly updated: readonly number[]
919
+ readonly removed: readonly number[]
920
+ }
921
+
922
+ /** Another document holding the same history. */
923
+ function copyOf(doc: Y.Doc): Y.Doc {
924
+ const copy = new Y.Doc()
925
+ Y.applyUpdate(copy, Y.encodeStateAsUpdate(doc))
926
+ return copy
927
+ }
928
+
929
+ function requireArtifacts(host: LiveRoomHost) {
930
+ const storage = host.storage.artifacts
931
+ if (!storage) throw new Error("[SixbServer] Live artifacts need storage that holds artifacts.")
932
+ return storage
933
+ }