@sixb/server 0.2.1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/artifacts/frame.d.ts.map +1 -1
- package/dist/auth/scope.d.ts +3 -0
- package/dist/auth/scope.d.ts.map +1 -1
- package/dist/auth/shared-access.d.ts +23 -0
- package/dist/auth/shared-access.d.ts.map +1 -1
- package/dist/connectors/http.d.ts +6 -1
- package/dist/connectors/http.d.ts.map +1 -1
- package/dist/index.js +2876 -667
- package/dist/index.js.map +43 -36
- package/dist/live/awareness.d.ts +48 -0
- package/dist/live/awareness.d.ts.map +1 -0
- package/dist/live/room.d.ts +198 -0
- package/dist/live/room.d.ts.map +1 -0
- package/dist/live/rooms.d.ts +67 -0
- package/dist/live/rooms.d.ts.map +1 -0
- package/dist/registerRoutes.d.ts.map +1 -1
- package/dist/routes/actions.d.ts +3 -3
- package/dist/routes/agent-approvals.d.ts +21 -11
- package/dist/routes/agent-approvals.d.ts.map +1 -1
- package/dist/routes/agent-children.d.ts +4 -4
- package/dist/routes/agent-computers.d.ts +146 -1
- package/dist/routes/agent-computers.d.ts.map +1 -1
- package/dist/routes/agent-runners.d.ts.map +1 -1
- package/dist/routes/agents.d.ts +152 -60
- package/dist/routes/agents.d.ts.map +1 -1
- package/dist/routes/artifact-frames.d.ts +2 -1
- package/dist/routes/artifact-frames.d.ts.map +1 -1
- package/dist/routes/artifacts.d.ts +1713 -178
- package/dist/routes/artifacts.d.ts.map +1 -1
- package/dist/routes/auth.d.ts +21 -21
- package/dist/routes/connector-connection-runs.d.ts +124 -5
- package/dist/routes/connector-connection-runs.d.ts.map +1 -1
- package/dist/routes/connector-connections.d.ts +9 -1
- package/dist/routes/connector-connections.d.ts.map +1 -1
- package/dist/routes/datasets.d.ts +5 -5
- package/dist/routes/events.d.ts +4 -4
- package/dist/routes/lake.d.ts +91 -0
- package/dist/routes/lake.d.ts.map +1 -0
- package/dist/routes/links.d.ts +6 -6
- package/dist/routes/logs.d.ts +2 -2
- package/dist/routes/model-generations.d.ts +2 -2
- package/dist/routes/objects.d.ts +6 -6
- package/dist/routes/objects.d.ts.map +1 -1
- package/dist/routes/observed-agent-threads.d.ts +21 -11
- package/dist/routes/observed-agent-threads.d.ts.map +1 -1
- package/dist/routes/ontology.d.ts +6 -6
- package/dist/routes/pipelines.d.ts +13 -9
- package/dist/routes/pipelines.d.ts.map +1 -1
- package/dist/routes/projections.d.ts +27 -27
- package/dist/routes/share-grants.d.ts +9 -9
- package/dist/routes/shared-access.d.ts +57 -2
- package/dist/routes/shared-access.d.ts.map +1 -1
- package/dist/routes/status.d.ts +11 -15
- package/dist/routes/status.d.ts.map +1 -1
- package/dist/routes/syncs.d.ts +9 -6
- package/dist/routes/syncs.d.ts.map +1 -1
- package/dist/routes/telemetry.d.ts +7 -7
- package/dist/routes/webhooks.d.ts.map +1 -1
- package/dist/routes/workflows.d.ts +14 -14
- package/dist/routes/ws/artifacts.d.ts +57 -0
- package/dist/routes/ws/artifacts.d.ts.map +1 -0
- package/dist/routes/ws/events.d.ts +8 -7
- package/dist/routes/ws/events.d.ts.map +1 -1
- package/dist/routes/ws/index.d.ts +2 -1
- package/dist/routes/ws/index.d.ts.map +1 -1
- package/dist/routes/ws/logs.d.ts +10 -10
- package/dist/routes/ws/shared-sockets.d.ts +52 -0
- package/dist/routes/ws/shared-sockets.d.ts.map +1 -0
- package/dist/schemas/actions.d.ts +7 -7
- package/dist/schemas/activity.d.ts +16 -16
- package/dist/schemas/agent-approvals.d.ts +27 -27
- package/dist/schemas/agents.d.ts +642 -196
- package/dist/schemas/agents.d.ts.map +1 -1
- package/dist/schemas/ai-accounting.d.ts +41 -41
- package/dist/schemas/ai-accounting.d.ts.map +1 -1
- package/dist/schemas/artifacts.d.ts +2106 -127
- package/dist/schemas/artifacts.d.ts.map +1 -1
- package/dist/schemas/auth.d.ts +63 -63
- package/dist/schemas/connectors.d.ts +180 -20
- package/dist/schemas/connectors.d.ts.map +1 -1
- package/dist/schemas/datasets.d.ts +44 -44
- package/dist/schemas/event-payloads.d.ts +2 -2
- package/dist/schemas/events.d.ts +26 -26
- package/dist/schemas/events.d.ts.map +1 -1
- package/dist/schemas/files.d.ts +12 -0
- package/dist/schemas/files.d.ts.map +1 -1
- package/dist/schemas/lake.d.ts +251 -0
- package/dist/schemas/lake.d.ts.map +1 -0
- package/dist/schemas/lineage.d.ts +56 -56
- package/dist/schemas/logs.d.ts +18 -18
- package/dist/schemas/models.d.ts +4 -4
- package/dist/schemas/objects.d.ts.map +1 -1
- package/dist/schemas/ontology.d.ts +14 -14
- package/dist/schemas/pipelines.d.ts +86 -64
- package/dist/schemas/pipelines.d.ts.map +1 -1
- package/dist/schemas/projections.d.ts +160 -160
- package/dist/schemas/projections.d.ts.map +1 -1
- package/dist/schemas/share-grants.d.ts +30 -30
- package/dist/schemas/shared-access.d.ts +17 -4
- package/dist/schemas/shared-access.d.ts.map +1 -1
- package/dist/schemas/status.d.ts +59 -79
- package/dist/schemas/status.d.ts.map +1 -1
- package/dist/schemas/syncs.d.ts +38 -22
- package/dist/schemas/syncs.d.ts.map +1 -1
- package/dist/schemas/telemetry.d.ts +26 -26
- package/dist/schemas/workflows.d.ts +71 -71
- package/dist/server.d.ts +13 -0
- package/dist/server.d.ts.map +1 -1
- package/dist/utils/ws.d.ts +3 -0
- package/dist/utils/ws.d.ts.map +1 -1
- package/package.json +11 -7
- package/src/artifacts/frame.ts +4 -13
- package/src/auth/scope.ts +10 -1
- package/src/auth/shared-access.ts +108 -12
- package/src/connectors/http.ts +35 -1
- package/src/live/README.md +136 -0
- package/src/live/awareness.ts +141 -0
- package/src/live/room.ts +933 -0
- package/src/live/rooms.ts +288 -0
- package/src/registerRoutes.ts +2 -0
- package/src/routes/agent-computers.ts +208 -14
- package/src/routes/agent-runners.ts +1 -0
- package/src/routes/agents.ts +52 -7
- package/src/routes/artifact-frames.ts +40 -12
- package/src/routes/artifacts.ts +713 -28
- package/src/routes/connector-connection-runs.ts +7 -12
- package/src/routes/connector-connections.ts +10 -7
- package/src/routes/lake.ts +73 -0
- package/src/routes/objects.ts +3 -9
- package/src/routes/pipelines.ts +1 -0
- package/src/routes/shared-access.ts +53 -1
- package/src/routes/status.ts +9 -4
- package/src/routes/syncs.ts +1 -0
- package/src/routes/webhooks.ts +44 -3
- package/src/routes/ws/artifacts.ts +138 -0
- package/src/routes/ws/events.ts +7 -2
- package/src/routes/ws/index.ts +13 -2
- package/src/routes/ws/shared-sockets.ts +152 -0
- package/src/schemas/agents.ts +71 -1
- package/src/schemas/ai-accounting.ts +1 -0
- package/src/schemas/artifacts.ts +255 -18
- package/src/schemas/connectors.ts +17 -2
- package/src/schemas/files.ts +12 -0
- package/src/schemas/lake.ts +53 -0
- package/src/schemas/objects.ts +1 -0
- package/src/schemas/pipelines.ts +2 -0
- package/src/schemas/projections.ts +2 -2
- package/src/schemas/shared-access.ts +16 -0
- package/src/schemas/status.ts +16 -22
- package/src/schemas/syncs.ts +2 -0
- package/src/server.ts +35 -4
- package/src/utils/ws.ts +7 -0
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
import type { SixbHostView } from "@sixb/core"
|
|
2
|
+
import {
|
|
3
|
+
type ArtifactLiveClientMessage,
|
|
4
|
+
type ArtifactLiveServerMessage,
|
|
5
|
+
type ArtifactLiveUser,
|
|
6
|
+
decodeLiveData,
|
|
7
|
+
} from "@sixb/core/artifacts/live"
|
|
8
|
+
import { liveUserOf } from "./awareness"
|
|
9
|
+
import { type LiveConnection, LiveRoom, type LiveRoomHost } from "./room"
|
|
10
|
+
|
|
11
|
+
/** How many artifacts one connection may hold live at once. */
|
|
12
|
+
export const MAX_LIVE_ARTIFACTS_PER_CONNECTION = 16
|
|
13
|
+
/** The largest sync message a connection may send in parts: four boards' worth of HTML. */
|
|
14
|
+
export const MAX_LIVE_SYNC_BYTES = 8 * 1024 * 1024
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* The artifacts this server holds live, one room each, opened by the first connection that joins
|
|
18
|
+
* and closed once the last leaves and its changes are written back. Every room learns of the
|
|
19
|
+
* artifact's writes from the project's events, wherever they were made.
|
|
20
|
+
*/
|
|
21
|
+
export class LiveRooms {
|
|
22
|
+
private readonly rooms = new Map<string, Promise<LiveRoom | "not_found" | "not_live">>()
|
|
23
|
+
/**
|
|
24
|
+
* The artifacts each connection joined, or is joining: the slot is taken before the join reads
|
|
25
|
+
* anything, so joins at once count against the cap, and each is its own ticket, so a join that
|
|
26
|
+
* resumes once the connection left, or joined again, knows it is stale.
|
|
27
|
+
*/
|
|
28
|
+
private readonly joined = new Map<string, Map<string, symbol>>()
|
|
29
|
+
/** Who each connection is, read once. */
|
|
30
|
+
private readonly users = new Map<string, Promise<ArtifactLiveUser>>()
|
|
31
|
+
/** The parts of a sync message a connection is sending, by connection and artifact. */
|
|
32
|
+
private readonly parts = new Map<string, { chunks: Uint8Array[]; bytes: number }>()
|
|
33
|
+
private events: Promise<() => void> | undefined
|
|
34
|
+
private closed = false
|
|
35
|
+
|
|
36
|
+
constructor(private readonly host: LiveRoomHost & { readonly events: SixbHostView["events"] }) {}
|
|
37
|
+
|
|
38
|
+
/** Handles a message from a connection. */
|
|
39
|
+
async receive(connection: LiveConnection, message: ArtifactLiveClientMessage): Promise<void> {
|
|
40
|
+
switch (message.type) {
|
|
41
|
+
case "join":
|
|
42
|
+
return this.join(connection, message.artifactId)
|
|
43
|
+
case "leave":
|
|
44
|
+
return this.leave(connection, message.artifactId)
|
|
45
|
+
case "sync": {
|
|
46
|
+
const room = await this.held(connection, message.artifactId)
|
|
47
|
+
const bytes = room && this.assemble(connection, message)
|
|
48
|
+
if (bytes) await room.sync(connection.id, bytes)
|
|
49
|
+
return
|
|
50
|
+
}
|
|
51
|
+
case "awareness": {
|
|
52
|
+
const room = await this.held(connection, message.artifactId)
|
|
53
|
+
room?.presence(connection.id, message.data)
|
|
54
|
+
return
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** Leaves every artifact a connection holds: it closed. */
|
|
60
|
+
async disconnect(connection: LiveConnection): Promise<void> {
|
|
61
|
+
for (const artifactId of [...(this.joined.get(connection.id)?.keys() ?? [])]) {
|
|
62
|
+
await this.leave(connection, artifactId)
|
|
63
|
+
}
|
|
64
|
+
this.joined.delete(connection.id)
|
|
65
|
+
this.users.delete(connection.id)
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Writes back what every room holds unwritten, then stops them and the events feed: the server
|
|
70
|
+
* stops.
|
|
71
|
+
*/
|
|
72
|
+
async close(): Promise<void> {
|
|
73
|
+
this.closed = true
|
|
74
|
+
for (const opening of this.rooms.values()) {
|
|
75
|
+
const room = await opening.catch(() => null)
|
|
76
|
+
if (!(room instanceof LiveRoom)) continue
|
|
77
|
+
room.save()
|
|
78
|
+
await room.settled()
|
|
79
|
+
room.close()
|
|
80
|
+
}
|
|
81
|
+
this.rooms.clear()
|
|
82
|
+
const unsubscribe = await this.events?.catch(() => undefined)
|
|
83
|
+
unsubscribe?.()
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** Settles once every room's queued work is done. */
|
|
87
|
+
async settled(): Promise<void> {
|
|
88
|
+
for (const opening of [...this.rooms.values()]) {
|
|
89
|
+
const room = await opening.catch(() => null)
|
|
90
|
+
if (room instanceof LiveRoom) await room.settled()
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Joins a connection to an artifact it may read: to edit it when it may change it, to watch it
|
|
96
|
+
* otherwise. One it may not read, or that is not edited live, is refused with why.
|
|
97
|
+
*/
|
|
98
|
+
private async join(connection: LiveConnection, artifactId: string): Promise<void> {
|
|
99
|
+
const send = (message: ArtifactLiveServerMessage) => connection.send(message)
|
|
100
|
+
const held = this.joined.get(connection.id) ?? new Map<string, symbol>()
|
|
101
|
+
if (held.has(artifactId)) return
|
|
102
|
+
if (held.size >= MAX_LIVE_ARTIFACTS_PER_CONNECTION) {
|
|
103
|
+
send({
|
|
104
|
+
type: "error",
|
|
105
|
+
artifactId,
|
|
106
|
+
message: `[SixbServer] A connection holds at most ${MAX_LIVE_ARTIFACTS_PER_CONNECTION} artifacts live: leave one first.`,
|
|
107
|
+
})
|
|
108
|
+
return
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// Taken now, before anything is read: what follows waits, and other joins run meanwhile.
|
|
112
|
+
const ticket = Symbol(artifactId)
|
|
113
|
+
held.set(artifactId, ticket)
|
|
114
|
+
this.joined.set(connection.id, held)
|
|
115
|
+
const current = () => this.joined.get(connection.id)?.get(artifactId) === ticket
|
|
116
|
+
const release = () => {
|
|
117
|
+
if (current()) this.joined.get(connection.id)?.delete(artifactId)
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
try {
|
|
121
|
+
const access = await connection.sixb.artifacts.access(artifactId)
|
|
122
|
+
if (access === null || access === "none") {
|
|
123
|
+
release()
|
|
124
|
+
send({ type: "left", artifactId, reason: access === null ? "not_found" : "not_shared" })
|
|
125
|
+
return
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const user = await this.userOf(connection)
|
|
129
|
+
const room = await this.open(artifactId)
|
|
130
|
+
if (!(room instanceof LiveRoom)) {
|
|
131
|
+
release()
|
|
132
|
+
send({ type: "left", artifactId, reason: room })
|
|
133
|
+
return
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// The connection left, or closed, while this read: it is not here, and a room this join
|
|
137
|
+
// opened for it closes again. Otherwise it joins now, before the room can close.
|
|
138
|
+
if (!current()) {
|
|
139
|
+
room.closeIfUnheld()
|
|
140
|
+
return
|
|
141
|
+
}
|
|
142
|
+
room.join(connection, access === "read" ? "watch" : "edit", user)
|
|
143
|
+
} catch (error) {
|
|
144
|
+
release()
|
|
145
|
+
throw error
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
private userOf(connection: LiveConnection): Promise<ArtifactLiveUser> {
|
|
150
|
+
// A guest is read at each join: the name they give after joining shows from their next one.
|
|
151
|
+
if (connection.user) return connection.user()
|
|
152
|
+
|
|
153
|
+
const known = this.users.get(connection.id)
|
|
154
|
+
if (known) return known
|
|
155
|
+
|
|
156
|
+
const user = liveUserOf(
|
|
157
|
+
this.host.storage,
|
|
158
|
+
this.host.projectId,
|
|
159
|
+
connection.sixb.execution.requestedBy
|
|
160
|
+
)
|
|
161
|
+
this.users.set(connection.id, user)
|
|
162
|
+
return user
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Reads again who a connection is, and shows it so in every artifact it holds: a shared link's
|
|
167
|
+
* guest gave their name, which the server reads from their session, never from the client.
|
|
168
|
+
*/
|
|
169
|
+
async refreshUser(connection: LiveConnection): Promise<void> {
|
|
170
|
+
this.users.delete(connection.id)
|
|
171
|
+
const user = await this.userOf(connection)
|
|
172
|
+
|
|
173
|
+
for (const artifactId of this.joined.get(connection.id)?.keys() ?? []) {
|
|
174
|
+
const room = await this.rooms.get(artifactId)
|
|
175
|
+
if (room instanceof LiveRoom) room.rename(connection.id, user)
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** Leaves an artifact, as asked: the connection is told it left, once it did. */
|
|
180
|
+
private async leave(connection: LiveConnection, artifactId: string): Promise<void> {
|
|
181
|
+
this.parts.delete(partsKey(connection, artifactId))
|
|
182
|
+
const held = this.joined.get(connection.id)
|
|
183
|
+
if (!held?.delete(artifactId)) return
|
|
184
|
+
|
|
185
|
+
const room = await this.rooms.get(artifactId)
|
|
186
|
+
if (room instanceof LiveRoom) room.leave(connection.id)
|
|
187
|
+
connection.send({ type: "left", artifactId, reason: "left" })
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* The bytes of a whole sync message: this one's, or, on its last part, every part's. Null while
|
|
192
|
+
* parts are still to come, and for a message that is not base64 or grew too large.
|
|
193
|
+
*/
|
|
194
|
+
private assemble(
|
|
195
|
+
connection: LiveConnection,
|
|
196
|
+
message: Extract<ArtifactLiveClientMessage, { type: "sync" }>
|
|
197
|
+
): Uint8Array | null {
|
|
198
|
+
const key = partsKey(connection, message.artifactId)
|
|
199
|
+
const bytes = decodeLiveData(message.data)
|
|
200
|
+
const parts = this.parts.get(key)
|
|
201
|
+
const size = (parts?.bytes ?? 0) + (bytes?.length ?? 0)
|
|
202
|
+
|
|
203
|
+
if (!bytes || size > MAX_LIVE_SYNC_BYTES) {
|
|
204
|
+
this.parts.delete(key)
|
|
205
|
+
connection.send({
|
|
206
|
+
type: "error",
|
|
207
|
+
artifactId: message.artifactId,
|
|
208
|
+
message: bytes
|
|
209
|
+
? `[SixbServer] A sync message holds at most ${MAX_LIVE_SYNC_BYTES} bytes.`
|
|
210
|
+
: "[SixbServer] A sync message's data is base64.",
|
|
211
|
+
})
|
|
212
|
+
return null
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
if (message.more) {
|
|
216
|
+
this.parts.set(key, { chunks: [...(parts?.chunks ?? []), bytes], bytes: size })
|
|
217
|
+
return null
|
|
218
|
+
}
|
|
219
|
+
if (!parts) return bytes
|
|
220
|
+
|
|
221
|
+
this.parts.delete(key)
|
|
222
|
+
const whole = new Uint8Array(size)
|
|
223
|
+
let offset = 0
|
|
224
|
+
for (const chunk of [...parts.chunks, bytes]) {
|
|
225
|
+
whole.set(chunk, offset)
|
|
226
|
+
offset += chunk.length
|
|
227
|
+
}
|
|
228
|
+
return whole
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/** The room of an artifact the connection joined. */
|
|
232
|
+
private async held(connection: LiveConnection, artifactId: string): Promise<LiveRoom | null> {
|
|
233
|
+
if (!this.joined.get(connection.id)?.has(artifactId)) {
|
|
234
|
+
connection.send({
|
|
235
|
+
type: "error",
|
|
236
|
+
artifactId,
|
|
237
|
+
message: "[SixbServer] Join the artifact before sending it changes.",
|
|
238
|
+
})
|
|
239
|
+
return null
|
|
240
|
+
}
|
|
241
|
+
const room = await this.rooms.get(artifactId)
|
|
242
|
+
return room instanceof LiveRoom ? room : null
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/** The artifact's room, opened once however many join at once; a refusal is not kept. */
|
|
246
|
+
private async open(artifactId: string): Promise<LiveRoom | "not_found" | "not_live"> {
|
|
247
|
+
const existing = await this.rooms.get(artifactId)
|
|
248
|
+
if (existing instanceof LiveRoom && !existing.isClosed) return existing
|
|
249
|
+
|
|
250
|
+
this.followEvents()
|
|
251
|
+
const opening = LiveRoom.open(this.host, artifactId, () => {
|
|
252
|
+
// A room reopened since holds the entry now.
|
|
253
|
+
if (this.rooms.get(artifactId) === opening) this.rooms.delete(artifactId)
|
|
254
|
+
})
|
|
255
|
+
this.rooms.set(artifactId, opening)
|
|
256
|
+
const room = await opening.catch((error: unknown) => {
|
|
257
|
+
this.rooms.delete(artifactId)
|
|
258
|
+
throw error
|
|
259
|
+
})
|
|
260
|
+
if (!(room instanceof LiveRoom)) this.rooms.delete(artifactId)
|
|
261
|
+
return room
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/** Every write to an artifact a room holds reaches it, whichever server or worker made it. */
|
|
265
|
+
private followEvents(): void {
|
|
266
|
+
if (this.events || this.closed) return
|
|
267
|
+
const following = this.host.events.subscribe(
|
|
268
|
+
{ from: "latest", types: ["artifact.updated"] },
|
|
269
|
+
async (events) => {
|
|
270
|
+
for (const event of events) {
|
|
271
|
+
if (event.type !== "artifact.updated") continue
|
|
272
|
+
const room = await this.rooms.get(event.payload.artifactId)
|
|
273
|
+
if (room instanceof LiveRoom) room.written(event.payload.revision)
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
)
|
|
277
|
+
// A feed that could not start is asked for again by the next room that opens.
|
|
278
|
+
this.events = following
|
|
279
|
+
following.catch((error: unknown) => {
|
|
280
|
+
if (this.events === following) this.events = undefined
|
|
281
|
+
console.error("[SixbServer] Could not follow the artifacts' writes:", error)
|
|
282
|
+
})
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
function partsKey(connection: LiveConnection, artifactId: string): string {
|
|
287
|
+
return `${connection.id}\n${artifactId}`
|
|
288
|
+
}
|
package/src/registerRoutes.ts
CHANGED
|
@@ -24,6 +24,7 @@ import { registerDatasetRoutes } from "./routes/datasets"
|
|
|
24
24
|
import { registerEventRoutes } from "./routes/events"
|
|
25
25
|
import { registerFileDownloadRoutes } from "./routes/file-downloads"
|
|
26
26
|
import { registerFileRoutes } from "./routes/files"
|
|
27
|
+
import { registerLakeRoutes } from "./routes/lake"
|
|
27
28
|
import { registerLinkRoutes } from "./routes/links"
|
|
28
29
|
import { registerLogRoutes } from "./routes/logs"
|
|
29
30
|
import { registerModelCallRoutes } from "./routes/model-calls"
|
|
@@ -62,6 +63,7 @@ export function registerHttpRoutes(app: Elysia, host: SixbHostView, options: Htt
|
|
|
62
63
|
registerConnectorConnectionRoutes(app, host)
|
|
63
64
|
registerConnectorConnectionRunRoutes(app, host, options.connectorConnections)
|
|
64
65
|
registerDatasetRoutes(app, host)
|
|
66
|
+
registerLakeRoutes(app, host)
|
|
65
67
|
registerSyncRoutes(app, host)
|
|
66
68
|
registerPipelineRoutes(app, host)
|
|
67
69
|
registerWorkflowRoutes(app, host)
|
|
@@ -1,8 +1,14 @@
|
|
|
1
|
-
import
|
|
1
|
+
import {
|
|
2
|
+
AgentRequestError,
|
|
3
|
+
principalsEqual,
|
|
4
|
+
type SixbHostView,
|
|
5
|
+
stableJsonStringify,
|
|
6
|
+
} from "@sixb/core"
|
|
2
7
|
import { sameComputerPlace } from "@sixb/core/internal/agent-run-storage-provider"
|
|
3
8
|
import type { AgentRunSessionHost } from "@sixb/core/internal/agents"
|
|
4
9
|
import { publishAgentComputerActivity } from "@sixb/core/internal/agents/streams"
|
|
5
10
|
import type {
|
|
11
|
+
AgentComputerBackgroundCommand,
|
|
6
12
|
AgentComputerSessionRecord,
|
|
7
13
|
AgentStorage,
|
|
8
14
|
AgentThreadComputer,
|
|
@@ -12,9 +18,14 @@ import type {
|
|
|
12
18
|
import type { Elysia } from "elysia"
|
|
13
19
|
import { z } from "zod"
|
|
14
20
|
import { requestActor, requireRequestSixb } from "../auth/scope"
|
|
15
|
-
import {
|
|
21
|
+
import {
|
|
22
|
+
SIXB_SESSION_MUTATION_SECURITY_REQUIREMENT,
|
|
23
|
+
SIXB_SESSION_SECURITY_REQUIREMENT,
|
|
24
|
+
} from "../openapi/security"
|
|
16
25
|
import { OPENAPI_TAGS } from "../openapi/tags"
|
|
17
26
|
import {
|
|
27
|
+
AgentBackgroundCommandParamsSchema,
|
|
28
|
+
AgentComputerListResponseSchema,
|
|
18
29
|
AgentComputerSessionParamsSchema,
|
|
19
30
|
AgentRequestErrorResponseSchema,
|
|
20
31
|
AgentThreadComputerInputSchema,
|
|
@@ -23,10 +34,17 @@ import {
|
|
|
23
34
|
HoldAgentComputerSessionBodySchema,
|
|
24
35
|
HoldAgentComputerSessionResponseSchema,
|
|
25
36
|
MoveAgentThreadToComputerBodySchema,
|
|
37
|
+
StopAgentBackgroundCommandBodySchema,
|
|
38
|
+
StopAgentBackgroundCommandResponseSchema,
|
|
26
39
|
} from "../schemas/agents"
|
|
27
40
|
import { ErrorResponseSchema } from "../schemas/common"
|
|
28
41
|
import { unconfiguredStorageResponse } from "../utils/http"
|
|
29
|
-
import {
|
|
42
|
+
import {
|
|
43
|
+
handleAgentRouteError,
|
|
44
|
+
principalKey,
|
|
45
|
+
sameBackgroundCommand,
|
|
46
|
+
serializeThreadRead,
|
|
47
|
+
} from "./agents"
|
|
30
48
|
|
|
31
49
|
// A computer's thread runs on one of its owner's computers, in one folder there: only `sixb` open
|
|
32
50
|
// in that folder claims its turns. Each open `sixb` holds a session, a lease it renews, so the
|
|
@@ -35,8 +53,11 @@ import { handleAgentRouteError, principalKey, serializeThreadRead } from "./agen
|
|
|
35
53
|
|
|
36
54
|
/** How long a session stays live without a renewal; `sixb` renews it three times as often. */
|
|
37
55
|
const SESSION_MS = 60_000
|
|
38
|
-
/**
|
|
39
|
-
|
|
56
|
+
/**
|
|
57
|
+
* How many sessions one person holds at once, a folder connected on a computer each. A computer
|
|
58
|
+
* keeps every folder chosen on it connected, so this only bounds abuse.
|
|
59
|
+
*/
|
|
60
|
+
const MAX_SESSIONS = 200
|
|
40
61
|
|
|
41
62
|
type ComputerClaim =
|
|
42
63
|
| { readonly computer: AgentThreadComputer }
|
|
@@ -104,6 +125,40 @@ export async function endLapsedComputerSessions(host: SixbHostView): Promise<voi
|
|
|
104
125
|
|
|
105
126
|
export function registerAgentComputerRoutes(app: Elysia, host: SixbHostView) {
|
|
106
127
|
return app
|
|
128
|
+
.get(
|
|
129
|
+
"/api/agent-computers",
|
|
130
|
+
async (context) => {
|
|
131
|
+
const { set } = context
|
|
132
|
+
const sixb = requireRequestSixb(context)
|
|
133
|
+
try {
|
|
134
|
+
const agents = host.storage.agents
|
|
135
|
+
if (!agents) return unconfiguredStorageResponse(set, "Agent storage")
|
|
136
|
+
|
|
137
|
+
const live = await agents.computers.listLive({
|
|
138
|
+
projectId: host.id,
|
|
139
|
+
ownerPrincipal: requestActor(sixb),
|
|
140
|
+
now: new Date(),
|
|
141
|
+
})
|
|
142
|
+
return AgentComputerListResponseSchema.parse({ computers: computersOf(live) })
|
|
143
|
+
} catch (error) {
|
|
144
|
+
return handleAgentRouteError(error, set)
|
|
145
|
+
}
|
|
146
|
+
},
|
|
147
|
+
{
|
|
148
|
+
response: {
|
|
149
|
+
200: AgentComputerListResponseSchema,
|
|
150
|
+
501: ErrorResponseSchema,
|
|
151
|
+
},
|
|
152
|
+
detail: {
|
|
153
|
+
summary: "List your computers with sixb open now",
|
|
154
|
+
description:
|
|
155
|
+
"Each of your computers with a live session, and the folders `sixb` is open in there: a thread created to run in one of them runs there now.",
|
|
156
|
+
tags: [OPENAPI_TAGS.agentThreads.name],
|
|
157
|
+
operationId: "listAgentComputers",
|
|
158
|
+
security: SIXB_SESSION_SECURITY_REQUIREMENT,
|
|
159
|
+
},
|
|
160
|
+
}
|
|
161
|
+
)
|
|
107
162
|
.post(
|
|
108
163
|
"/api/agent-threads/:threadId/computer",
|
|
109
164
|
async (context) => {
|
|
@@ -183,7 +238,8 @@ export function registerAgentComputerRoutes(app: Elysia, host: SixbHostView) {
|
|
|
183
238
|
const agents = host.storage.agents
|
|
184
239
|
if (!agents) return unconfiguredStorageResponse(set, "Agent storage")
|
|
185
240
|
|
|
186
|
-
const { computer } =
|
|
241
|
+
const { computer, backgroundCommands = [] } =
|
|
242
|
+
HoldAgentComputerSessionBodySchema.parse(body)
|
|
187
243
|
const ownerPrincipal = requestActor(sixb)
|
|
188
244
|
const now = new Date()
|
|
189
245
|
const before = await agents.computers.listLive({
|
|
@@ -197,7 +253,7 @@ export function registerAgentComputerRoutes(app: Elysia, host: SixbHostView) {
|
|
|
197
253
|
) {
|
|
198
254
|
set.status = 429
|
|
199
255
|
return {
|
|
200
|
-
error: `[SixbServer] You have ${MAX_SESSIONS}
|
|
256
|
+
error: `[SixbServer] You have ${MAX_SESSIONS} folders connected: forget one to connect another.`,
|
|
201
257
|
}
|
|
202
258
|
}
|
|
203
259
|
const expiresAt = new Date(now.getTime() + SESSION_MS)
|
|
@@ -207,13 +263,29 @@ export function registerAgentComputerRoutes(app: Elysia, host: SixbHostView) {
|
|
|
207
263
|
ownerPrincipal,
|
|
208
264
|
computer,
|
|
209
265
|
expiresAt,
|
|
266
|
+
...(backgroundCommands.length > 0 ? { backgroundCommands } : {}),
|
|
210
267
|
})
|
|
211
|
-
|
|
268
|
+
|
|
269
|
+
// The web follows the folder coming online, and its commands starting and ending.
|
|
270
|
+
const held = before.find((session) => session.id === params.sessionId)
|
|
271
|
+
const reported = !sameCommands(held?.backgroundCommands ?? [], backgroundCommands)
|
|
272
|
+
if (
|
|
273
|
+
reported ||
|
|
274
|
+
!before.some((session) => sameComputerPlace(session.computer, computer))
|
|
275
|
+
) {
|
|
212
276
|
await announce(host, ownerPrincipal, computer, true)
|
|
213
277
|
}
|
|
278
|
+
|
|
279
|
+
// What was asked to stop, as long as it runs: a lost answer leaves it for the next one.
|
|
280
|
+
const stop = (held?.stopRequests ?? []).filter((ref) =>
|
|
281
|
+
backgroundCommands.some(
|
|
282
|
+
(command) => command.status === "running" && sameBackgroundCommand(command, ref)
|
|
283
|
+
)
|
|
284
|
+
)
|
|
214
285
|
return HoldAgentComputerSessionResponseSchema.parse({
|
|
215
286
|
expiresAt: expiresAt.toISOString(),
|
|
216
287
|
waiting: await waitingTurns(host, agents, ownerPrincipal, computer),
|
|
288
|
+
...(stop.length > 0 ? { stop } : {}),
|
|
217
289
|
})
|
|
218
290
|
} catch (error) {
|
|
219
291
|
return handleAgentRouteError(error, set)
|
|
@@ -231,13 +303,100 @@ export function registerAgentComputerRoutes(app: Elysia, host: SixbHostView) {
|
|
|
231
303
|
detail: {
|
|
232
304
|
summary: "Open or renew a computer session",
|
|
233
305
|
description:
|
|
234
|
-
"`sixb` holds one while it is open: your threads on this computer, in this folder, read as online, and you can continue them from anywhere. It lapses a minute after its last renewal.
|
|
306
|
+
"`sixb` holds one while it is open: your threads on this computer, in this folder, read as online, and you can continue them from anywhere. It lapses a minute after its last renewal. It reports the background commands it runs for your threads; the answer lists the turns waiting for it, and the commands someone asked to stop.",
|
|
235
307
|
tags: [OPENAPI_TAGS.agentThreads.name],
|
|
236
308
|
operationId: "holdAgentComputerSession",
|
|
237
309
|
security: SIXB_SESSION_MUTATION_SECURITY_REQUIREMENT,
|
|
238
310
|
},
|
|
239
311
|
}
|
|
240
312
|
)
|
|
313
|
+
.post(
|
|
314
|
+
"/api/agent-threads/:threadId/background-commands/:commandId/stop",
|
|
315
|
+
async (context) => {
|
|
316
|
+
const { params, body, set } = context
|
|
317
|
+
const sixb = requireRequestSixb(context)
|
|
318
|
+
try {
|
|
319
|
+
const agents = host.storage.agents
|
|
320
|
+
if (!agents) return unconfiguredStorageResponse(set, "Agent storage")
|
|
321
|
+
|
|
322
|
+
const { startedAt } = StopAgentBackgroundCommandBodySchema.parse(body)
|
|
323
|
+
const thread = await sixb.agent.threads.getById(params.threadId)
|
|
324
|
+
if (!thread) {
|
|
325
|
+
throw new AgentRequestError("thread_not_found", "[SixbServer] Agent thread not found.")
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
// It runs on its owner's computer, under their account: as moving the thread, theirs to
|
|
329
|
+
// stop. Anyone else asks the agent, whose turn its requester can stop.
|
|
330
|
+
const ownerPrincipal = requestActor(sixb)
|
|
331
|
+
if (!principalsEqual(thread.ownerPrincipal, ownerPrincipal)) {
|
|
332
|
+
throw new AgentRequestError(
|
|
333
|
+
"not_thread_owner",
|
|
334
|
+
`[SixbServer] Only the owner of Agent thread '${thread.id}' stops its background commands, which run on their computer.`
|
|
335
|
+
)
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
// Whichever of their sixb runs it: one started in a folder runs on there once the thread
|
|
339
|
+
// moved to another, or to the cloud.
|
|
340
|
+
const ref = { threadId: thread.id, id: params.commandId, startedAt }
|
|
341
|
+
const live = await agents.computers.listLive({
|
|
342
|
+
projectId: host.id,
|
|
343
|
+
ownerPrincipal,
|
|
344
|
+
now: new Date(),
|
|
345
|
+
})
|
|
346
|
+
const running = live
|
|
347
|
+
.flatMap((session) =>
|
|
348
|
+
(session.backgroundCommands ?? []).map((command) => ({ session, command }))
|
|
349
|
+
)
|
|
350
|
+
.find(
|
|
351
|
+
({ command }) => command.status === "running" && sameBackgroundCommand(command, ref)
|
|
352
|
+
)
|
|
353
|
+
// Its sixb may close between the two: the stop then has no session to wait in.
|
|
354
|
+
const asked =
|
|
355
|
+
running &&
|
|
356
|
+
(await agents.computers.requestStop({
|
|
357
|
+
projectId: host.id,
|
|
358
|
+
id: running.session.id,
|
|
359
|
+
ownerPrincipal,
|
|
360
|
+
command: ref,
|
|
361
|
+
}))
|
|
362
|
+
if (!running || !asked) {
|
|
363
|
+
set.status = 404
|
|
364
|
+
return {
|
|
365
|
+
error: `[SixbServer] Background command '${params.commandId}' does not run on an open sixb: it ended, or sixb closed.`,
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
// Its sixb hears it and renews at once: the stop is in the answer.
|
|
370
|
+
await announce(host, ownerPrincipal, running.session.computer, true)
|
|
371
|
+
|
|
372
|
+
set.status = 202
|
|
373
|
+
return StopAgentBackgroundCommandResponseSchema.parse({
|
|
374
|
+
command: { ...running.command, stopping: true },
|
|
375
|
+
})
|
|
376
|
+
} catch (error) {
|
|
377
|
+
return handleAgentRouteError(error, set)
|
|
378
|
+
}
|
|
379
|
+
},
|
|
380
|
+
{
|
|
381
|
+
params: AgentBackgroundCommandParamsSchema,
|
|
382
|
+
body: StopAgentBackgroundCommandBodySchema,
|
|
383
|
+
response: {
|
|
384
|
+
202: StopAgentBackgroundCommandResponseSchema,
|
|
385
|
+
400: AgentRequestErrorResponseSchema,
|
|
386
|
+
403: AgentRequestErrorResponseSchema,
|
|
387
|
+
404: AgentRequestErrorResponseSchema,
|
|
388
|
+
501: ErrorResponseSchema,
|
|
389
|
+
},
|
|
390
|
+
detail: {
|
|
391
|
+
summary: "Stop a background command on your computer",
|
|
392
|
+
description:
|
|
393
|
+
"A command the agent ran in the background on your computer keeps running after its turn. `sixb` there stops it, and everything it started, at its next renewal, which this asks for at once. The thread's owner alone stops it.",
|
|
394
|
+
tags: [OPENAPI_TAGS.agentThreads.name],
|
|
395
|
+
operationId: "stopAgentBackgroundCommand",
|
|
396
|
+
security: SIXB_SESSION_MUTATION_SECURITY_REQUIREMENT,
|
|
397
|
+
},
|
|
398
|
+
}
|
|
399
|
+
)
|
|
241
400
|
.delete(
|
|
242
401
|
"/api/agent-computers/sessions/:sessionId",
|
|
243
402
|
async (context) => {
|
|
@@ -276,6 +435,34 @@ export function registerAgentComputerRoutes(app: Elysia, host: SixbHostView) {
|
|
|
276
435
|
)
|
|
277
436
|
}
|
|
278
437
|
|
|
438
|
+
/** Whether two reports of a session's background commands say the same. */
|
|
439
|
+
function sameCommands(
|
|
440
|
+
before: readonly AgentComputerBackgroundCommand[],
|
|
441
|
+
now: readonly AgentComputerBackgroundCommand[]
|
|
442
|
+
): boolean {
|
|
443
|
+
return stableJsonStringify(before) === stableJsonStringify(now)
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
/** Live sessions as computers, each with its folders: sorted, so a list reads the same each time. */
|
|
447
|
+
function computersOf(
|
|
448
|
+
sessions: readonly AgentComputerSessionRecord[]
|
|
449
|
+
): { id: string; name: string; folders: string[] }[] {
|
|
450
|
+
const byId = new Map<string, { id: string; name: string; folders: Set<string> }>()
|
|
451
|
+
for (const { computer } of sessions) {
|
|
452
|
+
const known = byId.get(computer.id)
|
|
453
|
+
if (known) known.folders.add(computer.folder)
|
|
454
|
+
else
|
|
455
|
+
byId.set(computer.id, {
|
|
456
|
+
id: computer.id,
|
|
457
|
+
name: computer.name,
|
|
458
|
+
folders: new Set([computer.folder]),
|
|
459
|
+
})
|
|
460
|
+
}
|
|
461
|
+
return [...byId.values()]
|
|
462
|
+
.map(({ id, name, folders }) => ({ id, name, folders: [...folders].sort() }))
|
|
463
|
+
.sort((a, b) => a.name.localeCompare(b.name) || a.id.localeCompare(b.id))
|
|
464
|
+
}
|
|
465
|
+
|
|
279
466
|
/** The owner's queued turns that wait for this computer, in this folder. */
|
|
280
467
|
async function waitingTurns(
|
|
281
468
|
host: SixbHostView,
|
|
@@ -301,7 +488,10 @@ async function waitingTurns(
|
|
|
301
488
|
.map((run) => run.id)
|
|
302
489
|
}
|
|
303
490
|
|
|
304
|
-
/**
|
|
491
|
+
/**
|
|
492
|
+
* For each folder these sessions held, say it went offline when no other session holds it, and
|
|
493
|
+
* that its background commands stopped when the session that ended ran some.
|
|
494
|
+
*/
|
|
305
495
|
async function announceOffline(
|
|
306
496
|
host: SixbHostView,
|
|
307
497
|
agents: AgentStorage,
|
|
@@ -320,11 +510,15 @@ async function announceOffline(
|
|
|
320
510
|
const { ownerPrincipal } = first
|
|
321
511
|
const live = await agents.computers.listLive({ projectId: host.id, ownerPrincipal, now })
|
|
322
512
|
const announced: AgentThreadComputer[] = []
|
|
323
|
-
for (const { computer } of [first, ...rest]) {
|
|
324
|
-
|
|
325
|
-
|
|
513
|
+
for (const { computer, backgroundCommands = [] } of [first, ...rest]) {
|
|
514
|
+
if (announced.some((other) => sameComputerPlace(other, computer))) continue
|
|
515
|
+
|
|
516
|
+
// Another sixb still holds the folder: the web reads it again for the commands that stopped.
|
|
517
|
+
const online = live.some((other) => sameComputerPlace(other.computer, computer))
|
|
518
|
+
if (online && backgroundCommands.length === 0) continue
|
|
519
|
+
|
|
326
520
|
announced.push(computer)
|
|
327
|
-
await announce(host, ownerPrincipal, computer,
|
|
521
|
+
await announce(host, ownerPrincipal, computer, online)
|
|
328
522
|
}
|
|
329
523
|
}
|
|
330
524
|
}
|