@sixb/cli 0.2.1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -22
- package/dist/commands/access-token-utils.d.ts.map +1 -1
- package/dist/commands/agent.d.ts +4 -1
- package/dist/commands/agent.d.ts.map +1 -1
- package/dist/commands/api.d.ts.map +1 -1
- package/dist/commands/check.d.ts.map +1 -1
- package/dist/commands/connect.d.ts +13 -0
- package/dist/commands/connect.d.ts.map +1 -0
- package/dist/commands/db-migrate.d.ts.map +1 -1
- package/dist/commands/db-prepare.d.ts.map +1 -1
- package/dist/commands/deploy.d.ts.map +1 -1
- package/dist/commands/desktop.d.ts.map +1 -1
- package/dist/commands/dev-runner.d.ts.map +1 -1
- package/dist/commands/env.d.ts +6 -1
- package/dist/commands/env.d.ts.map +1 -1
- package/dist/commands/lake-cleanup.d.ts +2 -12
- package/dist/commands/lake-cleanup.d.ts.map +1 -1
- package/dist/commands/lake-rejected.d.ts +10 -0
- package/dist/commands/lake-rejected.d.ts.map +1 -0
- package/dist/commands/orchestrator.d.ts.map +1 -1
- package/dist/commands/scheduler.d.ts +2 -5
- package/dist/commands/scheduler.d.ts.map +1 -1
- package/dist/commands/worker-group.d.ts.map +1 -1
- package/dist/lib/command-line.d.ts +22 -7
- package/dist/lib/command-line.d.ts.map +1 -1
- package/dist/{tui → lib}/connection.d.ts +11 -1
- package/dist/{tui → lib}/connection.d.ts.map +1 -1
- package/dist/lib/deploy-release.d.ts +7 -1
- package/dist/lib/deploy-release.d.ts.map +1 -1
- package/dist/lib/desktop.d.ts +4 -0
- package/dist/lib/desktop.d.ts.map +1 -1
- package/dist/lib/dev-supervisor.d.ts.map +1 -1
- package/dist/lib/duration.d.ts +3 -0
- package/dist/lib/duration.d.ts.map +1 -0
- package/dist/lib/loadSixb.d.ts.map +1 -1
- package/dist/lib/production-roles.d.ts +0 -4
- package/dist/lib/production-roles.d.ts.map +1 -1
- package/dist/lib/profiles.d.ts +5 -0
- package/dist/lib/profiles.d.ts.map +1 -1
- package/dist/lib/runtime.d.ts +7 -9
- package/dist/lib/runtime.d.ts.map +1 -1
- package/dist/lib/worker-concurrency.d.ts.map +1 -1
- package/dist/lib/worker-registry.d.ts +4 -1
- package/dist/lib/worker-registry.d.ts.map +1 -1
- package/dist/runner/allowed-folders.d.ts +17 -0
- package/dist/runner/allowed-folders.d.ts.map +1 -0
- package/dist/runner/computer-connection.d.ts +70 -0
- package/dist/runner/computer-connection.d.ts.map +1 -0
- package/dist/runner/computer.d.ts.map +1 -0
- package/dist/runner/desktop-runner-main.d.ts +2 -0
- package/dist/runner/desktop-runner-main.d.ts.map +1 -0
- package/dist/runner/desktop-runner.d.ts +43 -0
- package/dist/runner/desktop-runner.d.ts.map +1 -0
- package/dist/runner/keep-awake.d.ts +11 -0
- package/dist/runner/keep-awake.d.ts.map +1 -0
- package/dist/runner/local-runner.d.ts +131 -0
- package/dist/runner/local-runner.d.ts.map +1 -0
- package/dist/tui/app.d.ts +2 -2
- package/dist/tui/app.d.ts.map +1 -1
- package/dist/tui/format.d.ts +29 -0
- package/dist/tui/format.d.ts.map +1 -1
- package/dist/tui/index.d.ts.map +1 -1
- package/dist/tui/settings.d.ts +5 -8
- package/dist/tui/settings.d.ts.map +1 -1
- package/dist/ui/deploy-screens.d.ts +4 -2
- package/dist/ui/deploy-screens.d.ts.map +1 -1
- package/dist/ui/env-screens.d.ts +14 -1
- package/dist/ui/env-screens.d.ts.map +1 -1
- package/dist/ui/screens.d.ts +5 -11
- package/dist/ui/screens.d.ts.map +1 -1
- package/package.json +21 -17
- package/src/commands/access-token-utils.ts +2 -20
- package/src/commands/agent.ts +122 -39
- package/src/commands/api.ts +7 -0
- package/src/commands/check.ts +75 -1
- package/src/commands/connect.ts +197 -0
- package/src/commands/db-migrate.ts +1 -0
- package/src/commands/db-prepare.ts +1 -0
- package/src/commands/deploy.ts +15 -3
- package/src/commands/desktop.ts +2 -1
- package/src/commands/dev-runner.ts +5 -0
- package/src/commands/env.ts +99 -20
- package/src/commands/lake-cleanup.ts +25 -48
- package/src/commands/lake-rejected.ts +36 -0
- package/src/commands/orchestrator.ts +2 -7
- package/src/commands/scheduler.ts +11 -61
- package/src/commands/worker-group.ts +6 -3
- package/src/index.tsx +23 -4
- package/src/lib/command-line.ts +47 -15
- package/src/{tui → lib}/connection.ts +64 -17
- package/src/lib/deploy-release.ts +42 -2
- package/src/lib/desktop.ts +8 -1
- package/src/lib/dev-supervisor.ts +7 -2
- package/src/lib/duration.ts +11 -0
- package/src/lib/loadSixb.ts +3 -4
- package/src/lib/production-roles.ts +0 -1
- package/src/lib/profiles.ts +15 -1
- package/src/lib/runtime.ts +63 -43
- package/src/lib/shareable-providers.ts +2 -2
- package/src/lib/storage-migration.ts +1 -0
- package/src/lib/worker-concurrency.ts +1 -0
- package/src/lib/worker-registry.ts +36 -1
- package/src/runner/allowed-folders.ts +126 -0
- package/src/runner/computer-connection.ts +184 -0
- package/src/runner/desktop-runner-main.ts +9 -0
- package/src/runner/desktop-runner.ts +217 -0
- package/src/runner/keep-awake.ts +50 -0
- package/src/runner/local-runner.ts +427 -0
- package/src/tui/app.tsx +168 -12
- package/src/tui/format.ts +213 -7
- package/src/tui/index.tsx +14 -13
- package/src/tui/settings.ts +12 -44
- package/src/ui/deploy-screens.ts +8 -1
- package/src/ui/env-screens.ts +39 -5
- package/src/ui/screens.ts +21 -20
- package/dist/tui/computer.d.ts.map +0 -1
- package/dist/tui/runner.d.ts +0 -75
- package/dist/tui/runner.d.ts.map +0 -1
- package/src/tui/runner.ts +0 -256
- /package/dist/{tui → runner}/computer.d.ts +0 -0
- /package/src/{tui → runner}/computer.ts +0 -0
package/src/lib/runtime.ts
CHANGED
|
@@ -7,15 +7,17 @@ import {
|
|
|
7
7
|
getProjectionDispatchDescriptors,
|
|
8
8
|
ProjectionRunDispatcher,
|
|
9
9
|
} from "@sixb/core/internal/projections"
|
|
10
|
-
import { setApiPublicOrigin } from "@sixb/core/internal/runtime"
|
|
10
|
+
import { getDatasetIngestRuntime, setApiPublicOrigin } from "@sixb/core/internal/runtime"
|
|
11
11
|
import { checkPipelineMarkings, getAppliedMarkings } from "@sixb/core/internal/security"
|
|
12
12
|
import { SyncRunDispatcher } from "@sixb/core/internal/syncs"
|
|
13
13
|
import type { Worker } from "@sixb/core/internal/workers"
|
|
14
14
|
import { WorkflowRunDispatcher } from "@sixb/core/internal/workflows"
|
|
15
15
|
import { assertLakeDatasetDefinitionsCompatible } from "@sixb/core/lake-storage"
|
|
16
|
+
import { LakeWorker } from "@sixb/lake-worker"
|
|
16
17
|
import {
|
|
17
18
|
type CompileRoutesDiagnostic,
|
|
18
19
|
compileRoutesWithDiagnostics,
|
|
20
|
+
createMaintenancePasses,
|
|
19
21
|
OrchestratorWorker,
|
|
20
22
|
} from "@sixb/orchestrator"
|
|
21
23
|
import { PipelineWorker } from "@sixb/pipeline-worker"
|
|
@@ -27,7 +29,7 @@ import type { LoadedSixbHost } from "./loadSixb"
|
|
|
27
29
|
import { configuredOrigin } from "./public-origin"
|
|
28
30
|
import { stopQuietly } from "./role-lifecycle"
|
|
29
31
|
import type { WorkerConcurrency } from "./worker-registry"
|
|
30
|
-
import { agentRuntimeRequired, agentWorkerRequired } from "./worker-registry"
|
|
32
|
+
import { agentRuntimeRequired, agentWorkerRequired, lakeWorkerRequired } from "./worker-registry"
|
|
31
33
|
|
|
32
34
|
export function waitForWorkerFailure(worker: Worker | null | undefined): Promise<never> {
|
|
33
35
|
return new Promise<never>((_resolve, reject) => {
|
|
@@ -42,6 +44,7 @@ async function closeProvider(provider: { close?(): void | Promise<void> }): Prom
|
|
|
42
44
|
export interface RunningSixbRuntime {
|
|
43
45
|
readonly rulesWorker: RulesWorker | null
|
|
44
46
|
readonly syncWorker: SyncWorker | null
|
|
47
|
+
readonly lakeWorker: LakeWorker | null
|
|
45
48
|
readonly agentWorker: AgentWorker | null
|
|
46
49
|
readonly projectionWorker: ProjectionWorker | null
|
|
47
50
|
readonly pipelineWorker: PipelineWorker | null
|
|
@@ -76,10 +79,6 @@ export interface RunningRulesRuntime {
|
|
|
76
79
|
stop(): Promise<void>
|
|
77
80
|
}
|
|
78
81
|
|
|
79
|
-
export interface RunningSchedulerRuntime {
|
|
80
|
-
stop(): Promise<void>
|
|
81
|
-
}
|
|
82
|
-
|
|
83
82
|
export interface RunningOrchestratorRuntime {
|
|
84
83
|
readonly orchestratorWorker: OrchestratorWorker | null
|
|
85
84
|
readonly warnings: readonly string[]
|
|
@@ -100,6 +99,7 @@ export async function migrateRuntimeStorage(sixb: LoadedSixbHost): Promise<void>
|
|
|
100
99
|
await migrateStorage(sixb.storage, {
|
|
101
100
|
projectId: sixb.id,
|
|
102
101
|
ontology: sixb.definitions.ontology,
|
|
102
|
+
security: sixb.definitions.security,
|
|
103
103
|
})
|
|
104
104
|
}
|
|
105
105
|
|
|
@@ -160,18 +160,6 @@ export async function startRulesRuntime(sixb: LoadedSixbHost): Promise<RunningRu
|
|
|
160
160
|
}
|
|
161
161
|
}
|
|
162
162
|
|
|
163
|
-
export async function startSchedulerRuntime(
|
|
164
|
-
sixb: LoadedSixbHost
|
|
165
|
-
): Promise<RunningSchedulerRuntime> {
|
|
166
|
-
await sixb.scheduler.start()
|
|
167
|
-
|
|
168
|
-
return {
|
|
169
|
-
async stop() {
|
|
170
|
-
await stopQuietly(() => sixb.scheduler.stop())
|
|
171
|
-
},
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
|
|
175
163
|
export async function startOrchestratorRuntime(
|
|
176
164
|
sixb: LoadedSixbHost
|
|
177
165
|
): Promise<RunningOrchestratorRuntime> {
|
|
@@ -184,42 +172,69 @@ export async function startOrchestratorRuntime(
|
|
|
184
172
|
workflows: sixb.definitions.workflows.list(),
|
|
185
173
|
})
|
|
186
174
|
const warnings = diagnostics.map(formatRouteDiagnosticWarning)
|
|
187
|
-
|
|
175
|
+
const dispatchers = {
|
|
176
|
+
syncs: new SyncRunDispatcher(sixb),
|
|
177
|
+
pipelines: new PipelineRunDispatcher(sixb),
|
|
178
|
+
projections: new ProjectionRunDispatcher(sixb),
|
|
179
|
+
workflows: new WorkflowRunDispatcher(sixb),
|
|
180
|
+
}
|
|
188
181
|
|
|
182
|
+
const { projectionRuns, pipelineRuns, datasetIngest } = sixb.storage
|
|
183
|
+
if (projections.length > 0 && !projectionRuns) {
|
|
184
|
+
throw new Error("[SixbCLI] Projection dispatch requires storage.projectionRuns.")
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
let orchestratorWorker: OrchestratorWorker | null = null
|
|
189
188
|
if (routes.size > 0) {
|
|
190
|
-
const projectionRuns = sixb.storage.projectionRuns
|
|
191
|
-
if (projections.length > 0 && !projectionRuns) {
|
|
192
|
-
throw new Error("[SixbCLI] Projection dispatch requires storage.projectionRuns.")
|
|
193
|
-
}
|
|
194
|
-
const projectionReconciliation =
|
|
195
|
-
projections.length > 0 && projectionRuns
|
|
196
|
-
? { lakeStorage: sixb.lakeStorage, projectionRuns }
|
|
197
|
-
: undefined
|
|
198
189
|
orchestratorWorker = new OrchestratorWorker({
|
|
199
190
|
projectId: sixb.id,
|
|
200
191
|
events: sixb.events,
|
|
201
192
|
queues: sixb.queues,
|
|
202
193
|
routes,
|
|
203
|
-
dispatchers
|
|
204
|
-
|
|
205
|
-
pipelines: new PipelineRunDispatcher(sixb),
|
|
206
|
-
projections: new ProjectionRunDispatcher(sixb),
|
|
207
|
-
workflows: new WorkflowRunDispatcher(sixb),
|
|
208
|
-
},
|
|
209
|
-
...(projectionReconciliation ? { projectionReconciliation } : {}),
|
|
194
|
+
dispatchers,
|
|
195
|
+
reporter: sixb,
|
|
210
196
|
})
|
|
211
197
|
await orchestratorWorker.start()
|
|
212
198
|
}
|
|
213
199
|
|
|
200
|
+
// Runs with or without routes: every project commits events, and cron emits what routes consume.
|
|
201
|
+
const maintenance = sixb.startMaintenance(
|
|
202
|
+
createMaintenancePasses({
|
|
203
|
+
projectId: sixb.id,
|
|
204
|
+
routes,
|
|
205
|
+
dispatchers,
|
|
206
|
+
lakeStorage: sixb.lakeStorage,
|
|
207
|
+
...(projectionRuns ? { projectionRuns } : {}),
|
|
208
|
+
...(pipelineRuns ? { pipelineRuns } : {}),
|
|
209
|
+
...(datasetIngest && getDatasetIngestRuntime(sixb).enabled
|
|
210
|
+
? { ingest: { inbox: datasetIngest, queue: sixb.queues.lake, reporter: sixb } }
|
|
211
|
+
: {}),
|
|
212
|
+
...(hasRecoverableRuns(sixb)
|
|
213
|
+
? { runs: { storage: sixb.storage, queues: sixb.queues, events: sixb.events } }
|
|
214
|
+
: {}),
|
|
215
|
+
})
|
|
216
|
+
)
|
|
217
|
+
|
|
214
218
|
return {
|
|
215
219
|
orchestratorWorker,
|
|
216
220
|
warnings,
|
|
217
221
|
async stop() {
|
|
222
|
+
await stopQuietly(() => maintenance.stop())
|
|
218
223
|
await stopQuietly(() => orchestratorWorker?.stop() ?? Promise.resolve())
|
|
219
224
|
},
|
|
220
225
|
}
|
|
221
226
|
}
|
|
222
227
|
|
|
228
|
+
/** A project with none of these has no run a worker can lose, and pays for no look at them. */
|
|
229
|
+
function hasRecoverableRuns({ definitions }: LoadedSixbHost): boolean {
|
|
230
|
+
return [
|
|
231
|
+
definitions.syncs,
|
|
232
|
+
definitions.pipelines,
|
|
233
|
+
definitions.workflows,
|
|
234
|
+
definitions.projections,
|
|
235
|
+
].some((registry) => registry.list().length > 0)
|
|
236
|
+
}
|
|
237
|
+
|
|
223
238
|
/**
|
|
224
239
|
* Starts the background runtimes owned by the CLI host.
|
|
225
240
|
*
|
|
@@ -229,32 +244,32 @@ export async function startOrchestratorRuntime(
|
|
|
229
244
|
* 3. PipelineWorker (claims from queues)
|
|
230
245
|
* 4. WorkflowWorker (claims from queues)
|
|
231
246
|
* 5. SyncWorker (claims from queues)
|
|
232
|
-
* 6.
|
|
233
|
-
* 7.
|
|
247
|
+
* 6. LakeWorker (claims from queues)
|
|
248
|
+
* 7. Orchestrator (subscribes to events, enqueues jobs, runs cron and the maintenance loop)
|
|
234
249
|
* 8. API server, when this process serves it
|
|
235
250
|
* 9. AgentWorker (claims from queues; its turns call the API)
|
|
236
251
|
*
|
|
237
252
|
* Shutdown order (producers before consumers):
|
|
238
253
|
* 1. AgentWorker (ends its turns while the API still answers them)
|
|
239
254
|
* 2. API server
|
|
240
|
-
* 3.
|
|
241
|
-
* 4.
|
|
255
|
+
* 3. Orchestrator (stops cron, drains pending dispatches)
|
|
256
|
+
* 4. LakeWorker
|
|
242
257
|
* 5. SyncWorker
|
|
243
258
|
* 6. WorkflowWorker
|
|
244
259
|
* 7. PipelineWorker
|
|
245
260
|
* 8. ProjectionWorker
|
|
246
261
|
* 9. RulesWorker (drains pending evaluations)
|
|
247
|
-
*
|
|
262
|
+
* 10. Runtime providers (connectors, broker)
|
|
248
263
|
*/
|
|
249
264
|
export async function startSixbRuntime(
|
|
250
265
|
sixb: LoadedSixbHost,
|
|
251
266
|
options: StartSixbRuntimeOptions = {}
|
|
252
267
|
): Promise<RunningSixbRuntime> {
|
|
253
268
|
let rulesRuntime: RunningRulesRuntime | null = null
|
|
254
|
-
let schedulerRuntime: RunningSchedulerRuntime | null = null
|
|
255
269
|
let orchestratorRuntime: RunningOrchestratorRuntime | null = null
|
|
256
270
|
let rulesWorker: RulesWorker | null = null
|
|
257
271
|
let syncWorker: SyncWorker | null = null
|
|
272
|
+
let lakeWorker: LakeWorker | null = null
|
|
258
273
|
let agentWorker: AgentWorker | null = null
|
|
259
274
|
let pipelineWorker: PipelineWorker | null = null
|
|
260
275
|
let workflowWorker: WorkflowWorker | null = null
|
|
@@ -265,8 +280,8 @@ export async function startSixbRuntime(
|
|
|
265
280
|
async function stop() {
|
|
266
281
|
await stopQuietly(() => agentWorker?.stop() ?? Promise.resolve())
|
|
267
282
|
await stopQuietly(() => options.api?.stop() ?? Promise.resolve())
|
|
268
|
-
await stopQuietly(() => schedulerRuntime?.stop() ?? Promise.resolve())
|
|
269
283
|
await stopQuietly(() => orchestratorRuntime?.stop() ?? Promise.resolve())
|
|
284
|
+
await stopQuietly(() => lakeWorker?.stop() ?? Promise.resolve())
|
|
270
285
|
await stopQuietly(() => syncWorker?.stop() ?? Promise.resolve())
|
|
271
286
|
await stopQuietly(() => workflowWorker?.stop() ?? Promise.resolve())
|
|
272
287
|
await stopQuietly(() => pipelineWorker?.stop() ?? Promise.resolve())
|
|
@@ -324,11 +339,14 @@ export async function startSixbRuntime(
|
|
|
324
339
|
await syncWorker.start()
|
|
325
340
|
}
|
|
326
341
|
|
|
342
|
+
if (lakeWorkerRequired(sixb)) {
|
|
343
|
+
lakeWorker = new LakeWorker(sixb, { concurrency: options.workerConcurrency?.lake })
|
|
344
|
+
await lakeWorker.start()
|
|
345
|
+
}
|
|
346
|
+
|
|
327
347
|
orchestratorRuntime = await startOrchestratorRuntime(sixb)
|
|
328
348
|
orchestratorWorker = orchestratorRuntime.orchestratorWorker
|
|
329
349
|
warnings.push(...orchestratorRuntime.warnings)
|
|
330
|
-
|
|
331
|
-
schedulerRuntime = await startSchedulerRuntime(sixb)
|
|
332
350
|
}
|
|
333
351
|
|
|
334
352
|
await options.api?.start()
|
|
@@ -342,6 +360,7 @@ export async function startSixbRuntime(
|
|
|
342
360
|
const workers: Worker[] = []
|
|
343
361
|
if (rulesWorker) workers.push(rulesWorker)
|
|
344
362
|
if (syncWorker) workers.push(syncWorker)
|
|
363
|
+
if (lakeWorker) workers.push(lakeWorker)
|
|
345
364
|
if (agentWorker) workers.push(agentWorker)
|
|
346
365
|
if (projectionWorker) workers.push(projectionWorker)
|
|
347
366
|
if (pipelineWorker) workers.push(pipelineWorker)
|
|
@@ -353,6 +372,7 @@ export async function startSixbRuntime(
|
|
|
353
372
|
return {
|
|
354
373
|
rulesWorker,
|
|
355
374
|
syncWorker,
|
|
375
|
+
lakeWorker,
|
|
356
376
|
agentWorker,
|
|
357
377
|
projectionWorker,
|
|
358
378
|
pipelineWorker,
|
|
@@ -14,12 +14,12 @@ const SLOTS = [
|
|
|
14
14
|
{
|
|
15
15
|
name: "queues",
|
|
16
16
|
get: (sixb: LoadedSixbHost): Broker | Queues => sixb.queues,
|
|
17
|
-
replacements: "@sixb/bullmq",
|
|
17
|
+
replacements: "@sixb/queues-bullmq",
|
|
18
18
|
},
|
|
19
19
|
{
|
|
20
20
|
name: "broker",
|
|
21
21
|
get: (sixb: LoadedSixbHost): Broker | Queues => sixb.broker,
|
|
22
|
-
replacements: "@sixb/redis or @sixb/nats",
|
|
22
|
+
replacements: "@sixb/broker-redis or @sixb/broker-nats",
|
|
23
23
|
},
|
|
24
24
|
] as const
|
|
25
25
|
|
|
@@ -97,6 +97,7 @@ async function runMigration(sixb: LoadedSixbHost, role: StorageSchemaRole) {
|
|
|
97
97
|
return await migrateStorage(sixb.storage, {
|
|
98
98
|
projectId: sixb.id,
|
|
99
99
|
ontology: sixb.definitions.ontology,
|
|
100
|
+
security: sixb.definitions.security,
|
|
100
101
|
})
|
|
101
102
|
} catch (error) {
|
|
102
103
|
throw isBusyDatabaseError(error) ? concurrentMigrationError(error, role) : error
|
|
@@ -7,6 +7,7 @@ const ENVIRONMENT_VARIABLES = {
|
|
|
7
7
|
pipeline: "SIXB_PIPELINE_WORKER_CONCURRENCY",
|
|
8
8
|
projection: "SIXB_PROJECTION_WORKER_CONCURRENCY",
|
|
9
9
|
workflow: "SIXB_WORKFLOW_WORKER_CONCURRENCY",
|
|
10
|
+
lake: "SIXB_LAKE_WORKER_CONCURRENCY",
|
|
10
11
|
} as const satisfies Record<WorkerType, string>
|
|
11
12
|
|
|
12
13
|
/** Resolve the scalar concurrency accepted by `sixb worker <type>`. */
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { AgentWorker } from "@sixb/agent-worker"
|
|
2
|
+
import { getDatasetIngestRuntime, getLakeRetentionRuntime } from "@sixb/core/internal/runtime"
|
|
2
3
|
import type { Worker } from "@sixb/core/internal/workers"
|
|
4
|
+
import { LakeWorker } from "@sixb/lake-worker"
|
|
3
5
|
import { PipelineWorker } from "@sixb/pipeline-worker"
|
|
4
6
|
import { ProjectionWorker } from "@sixb/projection-worker"
|
|
5
7
|
import { SyncWorker } from "@sixb/sync-worker"
|
|
@@ -8,7 +10,7 @@ import { SixbCliError } from "./errors"
|
|
|
8
10
|
import type { LoadedSixbHost } from "./loadSixb"
|
|
9
11
|
import { configuredOrigin } from "./public-origin"
|
|
10
12
|
|
|
11
|
-
export const WORKER_TYPES = ["sync", "agent", "pipeline", "projection", "workflow"] as const
|
|
13
|
+
export const WORKER_TYPES = ["sync", "agent", "pipeline", "projection", "workflow", "lake"] as const
|
|
12
14
|
|
|
13
15
|
export type WorkerType = (typeof WORKER_TYPES)[number]
|
|
14
16
|
|
|
@@ -74,6 +76,10 @@ const workerFactories: Record<WorkerType, WorkerFactory> = {
|
|
|
74
76
|
create: (sixb, options) =>
|
|
75
77
|
new WorkflowWorker(sixb, { concurrency: options.workerConcurrency?.workflow }),
|
|
76
78
|
},
|
|
79
|
+
lake: {
|
|
80
|
+
create: (sixb, options) =>
|
|
81
|
+
new LakeWorker(sixb, { concurrency: options.workerConcurrency?.lake }),
|
|
82
|
+
},
|
|
77
83
|
}
|
|
78
84
|
|
|
79
85
|
export function createWorkerForType(
|
|
@@ -197,6 +203,10 @@ export function resolveRegisteredWorkerTypes(sixb: LoadedSixbHost): readonly Wor
|
|
|
197
203
|
workerTypes.push("workflow")
|
|
198
204
|
}
|
|
199
205
|
|
|
206
|
+
if (lakeWorkerRequired(sixb)) {
|
|
207
|
+
workerTypes.push("lake")
|
|
208
|
+
}
|
|
209
|
+
|
|
200
210
|
return workerTypes
|
|
201
211
|
}
|
|
202
212
|
|
|
@@ -220,6 +230,31 @@ function resolveAgentApiBaseUrl(value: string | undefined): string {
|
|
|
220
230
|
return apiBaseUrl
|
|
221
231
|
}
|
|
222
232
|
|
|
233
|
+
export function lakeWorkerRequired(sixb: LoadedSixbHost): boolean {
|
|
234
|
+
return getLakeRetentionRuntime(sixb).automatic || getDatasetIngestRuntime(sixb).enabled
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/** Requests stay queued while no worker claims them, which looks like an idle system. */
|
|
238
|
+
export function omittedWorkerWarnings(
|
|
239
|
+
sixb: LoadedSixbHost,
|
|
240
|
+
workerTypes: readonly string[]
|
|
241
|
+
): readonly string[] {
|
|
242
|
+
if (workerTypes.includes("lake")) return []
|
|
243
|
+
|
|
244
|
+
return [
|
|
245
|
+
...(getLakeRetentionRuntime(sixb).automatic
|
|
246
|
+
? [
|
|
247
|
+
"Lake cleanup needs a lake worker and this group has none: run `sixb worker lake`, or the lake keeps growing.",
|
|
248
|
+
]
|
|
249
|
+
: []),
|
|
250
|
+
...(getDatasetIngestRuntime(sixb).enabled
|
|
251
|
+
? [
|
|
252
|
+
"Ingested changes need a lake worker and this group has none: run `sixb worker lake`, or they wait and never reach their dataset.",
|
|
253
|
+
]
|
|
254
|
+
: []),
|
|
255
|
+
]
|
|
256
|
+
}
|
|
257
|
+
|
|
223
258
|
/** Explicit model calls can enqueue accounting recovery without a model catalog. */
|
|
224
259
|
export function agentWorkerRequired(sixb: LoadedSixbHost): boolean {
|
|
225
260
|
const { storage } = sixb
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto"
|
|
2
|
+
import { watch } from "node:fs"
|
|
3
|
+
import { mkdir, readFile, rename, writeFile } from "node:fs/promises"
|
|
4
|
+
import { basename, dirname, join } from "node:path"
|
|
5
|
+
import type { Connection } from "../lib/connection"
|
|
6
|
+
import { resolveConfigPath } from "../lib/profiles"
|
|
7
|
+
|
|
8
|
+
// The folders chosen on this computer, for each project. A conversation's folder is only text on
|
|
9
|
+
// the server, so a stolen web sign-in could name any folder: nothing sent from elsewhere runs in one
|
|
10
|
+
// that isn't listed here for that project, and choosing a folder for one project says nothing about
|
|
11
|
+
// another. Kept beside the CLI's sign-in, where the agent's own commands can't reach it.
|
|
12
|
+
|
|
13
|
+
const ALLOWED_FOLDERS_FILE = () => join(dirname(resolveConfigPath()), "allowed-folders.json")
|
|
14
|
+
|
|
15
|
+
/** The project a list is for: `connect` names it. */
|
|
16
|
+
export type AllowedFor = Pick<Connection, "apiUrl" | "project">
|
|
17
|
+
|
|
18
|
+
/** Its address as well as its id: a project reports its own id, and two can share one. */
|
|
19
|
+
function keyOf({ apiUrl, project }: AllowedFor): string {
|
|
20
|
+
return `${project.id} at ${apiUrl}`
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
type StoredFolders = Readonly<Record<string, readonly string[]>>
|
|
24
|
+
|
|
25
|
+
async function readStored(): Promise<StoredFolders> {
|
|
26
|
+
const text = await readFile(ALLOWED_FOLDERS_FILE(), "utf8").catch((error: unknown) => {
|
|
27
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return "{}"
|
|
28
|
+
throw error
|
|
29
|
+
})
|
|
30
|
+
const stored: unknown = JSON.parse(text)
|
|
31
|
+
if (typeof stored !== "object" || stored === null || Array.isArray(stored)) {
|
|
32
|
+
throw new Error("[SixbCLI] allowed-folders.json doesn't list folders by project.")
|
|
33
|
+
}
|
|
34
|
+
return Object.fromEntries(
|
|
35
|
+
Object.entries(stored).map(([key, folders]) => [
|
|
36
|
+
key,
|
|
37
|
+
Array.isArray(folders)
|
|
38
|
+
? folders.filter((folder): folder is string => typeof folder === "string")
|
|
39
|
+
: [],
|
|
40
|
+
])
|
|
41
|
+
)
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** The folders chosen on this computer for `project`. Throws when the list can't be read. */
|
|
45
|
+
export async function readAllowedFolders(project: AllowedFor): Promise<readonly string[]> {
|
|
46
|
+
return (await readStored())[keyOf(project)] ?? []
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Whether `folder` was chosen here for `project`. A list that can't be read allows nothing. */
|
|
50
|
+
export async function isFolderAllowed(project: AllowedFor, folder: string): Promise<boolean> {
|
|
51
|
+
return readAllowedFolders(project).then(
|
|
52
|
+
(folders) => folders.includes(folder),
|
|
53
|
+
() => false
|
|
54
|
+
)
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
let lastChange: Promise<unknown> = Promise.resolve()
|
|
58
|
+
|
|
59
|
+
/** One at a time here, so that a quick off and on can't undo each other. */
|
|
60
|
+
function oneAtATime(write: () => Promise<void>): Promise<void> {
|
|
61
|
+
const change = lastChange.then(write)
|
|
62
|
+
lastChange = change.catch(() => {})
|
|
63
|
+
return change
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Allow or forget folders for `project`. A folder forgotten stays forgotten. */
|
|
67
|
+
export function setFoldersAllowed(
|
|
68
|
+
project: AllowedFor,
|
|
69
|
+
changed: readonly string[],
|
|
70
|
+
allowed: boolean
|
|
71
|
+
): Promise<void> {
|
|
72
|
+
return oneAtATime(() => writeChange(project, changed, allowed))
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Reads the file first, as another `sixb` may have written it, and refuses one it can't read rather
|
|
77
|
+
* than lose the choices in it. Written whole, so no reader finds half.
|
|
78
|
+
*/
|
|
79
|
+
async function writeChange(
|
|
80
|
+
project: AllowedFor,
|
|
81
|
+
changed: readonly string[],
|
|
82
|
+
allowed: boolean
|
|
83
|
+
): Promise<void> {
|
|
84
|
+
const stored = await readStored()
|
|
85
|
+
const key = keyOf(project)
|
|
86
|
+
const others = (stored[key] ?? []).filter((folder) => !changed.includes(folder))
|
|
87
|
+
await writeStored({ ...stored, [key]: allowed ? [...others, ...changed] : others })
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
async function writeStored(stored: StoredFolders): Promise<void> {
|
|
91
|
+
const path = ALLOWED_FOLDERS_FILE()
|
|
92
|
+
const written = `${path}.${randomUUID()}`
|
|
93
|
+
await mkdir(dirname(path), { recursive: true, mode: 0o700 })
|
|
94
|
+
await writeFile(written, `${JSON.stringify(stored, null, 2)}\n`, { mode: 0o600 })
|
|
95
|
+
await rename(written, path)
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Forget `folder` for every project at `apiUrl`. Needs no sign-in, as a list on this computer doesn't:
|
|
100
|
+
* someone signed out can still take a folder back, and it stays taken back.
|
|
101
|
+
*/
|
|
102
|
+
export function forgetFolderAt(apiUrl: string, folder: string): Promise<void> {
|
|
103
|
+
return oneAtATime(async () => {
|
|
104
|
+
const stored = await readStored()
|
|
105
|
+
const suffix = ` at ${apiUrl}`
|
|
106
|
+
await writeStored(
|
|
107
|
+
Object.fromEntries(
|
|
108
|
+
Object.entries(stored).map(([key, folders]) => [
|
|
109
|
+
key,
|
|
110
|
+
key.endsWith(suffix) ? folders.filter((known) => known !== folder) : folders,
|
|
111
|
+
])
|
|
112
|
+
)
|
|
113
|
+
)
|
|
114
|
+
})
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** Call `changed` whenever the list may have changed. Returns the way to stop. */
|
|
118
|
+
export async function watchAllowedFolders(changed: () => void): Promise<() => void> {
|
|
119
|
+
const path = ALLOWED_FOLDERS_FILE()
|
|
120
|
+
await mkdir(dirname(path), { recursive: true, mode: 0o700 })
|
|
121
|
+
// The directory: a write replaces the file, which a watch on the file itself would lose.
|
|
122
|
+
const watcher = watch(dirname(path), (_event, name) => {
|
|
123
|
+
if (name === basename(path)) changed()
|
|
124
|
+
})
|
|
125
|
+
return () => watcher.close()
|
|
126
|
+
}
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
import { stat } from "node:fs/promises"
|
|
2
|
+
import { createAgentActivitySocket, getAgentThread, isSixbApiError } from "@sixb/client"
|
|
3
|
+
import type { Connection } from "../lib/connection"
|
|
4
|
+
import { errorMessage } from "../lib/errors"
|
|
5
|
+
import { readAllowedFolders, watchAllowedFolders } from "./allowed-folders"
|
|
6
|
+
import { LocalRunner } from "./local-runner"
|
|
7
|
+
|
|
8
|
+
/** How often the allowed folders are checked for being there still. */
|
|
9
|
+
const FOLDER_RECHECK_MS = 20_000
|
|
10
|
+
/** Exit status when the person's sign-in ended: whatever restarts this should know. */
|
|
11
|
+
export const EXIT_SIGNED_OUT = 3
|
|
12
|
+
|
|
13
|
+
/** What a connected computer reports, one line each with `sixb connect --json`. */
|
|
14
|
+
export type ComputerEvent =
|
|
15
|
+
| { readonly type: "folders"; readonly folders: readonly string[] }
|
|
16
|
+
| {
|
|
17
|
+
readonly type: "turn.started"
|
|
18
|
+
readonly folder: string
|
|
19
|
+
readonly threadId: string
|
|
20
|
+
readonly runId: string
|
|
21
|
+
}
|
|
22
|
+
| {
|
|
23
|
+
readonly type: "turn.ended"
|
|
24
|
+
readonly folder: string
|
|
25
|
+
readonly threadId: string
|
|
26
|
+
readonly runId: string
|
|
27
|
+
/** False when it could not be claimed or failed before it ended on its own. */
|
|
28
|
+
readonly ran: boolean
|
|
29
|
+
}
|
|
30
|
+
| { readonly type: "session.lost"; readonly folder: string; readonly message: string }
|
|
31
|
+
| { readonly type: "session.back"; readonly folder: string }
|
|
32
|
+
| { readonly type: "signed-out" }
|
|
33
|
+
| { readonly type: "error"; readonly message: string }
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* This computer, connected to a project: one session per folder allowed for it, and the turns sent
|
|
37
|
+
* to their conversations run in their folders. The list of folders is read again whenever it changes, so
|
|
38
|
+
* choosing one needs no restart.
|
|
39
|
+
*/
|
|
40
|
+
export class ComputerConnection {
|
|
41
|
+
private readonly runners = new Map<string, LocalRunner>()
|
|
42
|
+
private socket: { close(): void } | null = null
|
|
43
|
+
private unwatch: (() => void) | null = null
|
|
44
|
+
private recheck: ReturnType<typeof setInterval> | null = null
|
|
45
|
+
private reconciled: Promise<void> = Promise.resolve()
|
|
46
|
+
private shown: readonly string[] | null = null
|
|
47
|
+
private stopped = false
|
|
48
|
+
private signedOut = false
|
|
49
|
+
|
|
50
|
+
constructor(
|
|
51
|
+
private readonly connection: Connection,
|
|
52
|
+
private readonly onEvent: (event: ComputerEvent) => void,
|
|
53
|
+
private readonly keepsAwake: boolean,
|
|
54
|
+
/** What the agent's commands also never read, beyond what `sixb` keeps from them. */
|
|
55
|
+
private readonly alsoDenied: readonly string[] = []
|
|
56
|
+
) {}
|
|
57
|
+
|
|
58
|
+
async start(): Promise<void> {
|
|
59
|
+
this.unwatch = await watchAllowedFolders(() => void this.reconcile())
|
|
60
|
+
await this.reconcile()
|
|
61
|
+
// A folder can go or come back with no change to the list: a deleted one, an unmounted volume.
|
|
62
|
+
this.recheck = setInterval(() => void this.reconcile(), FOLDER_RECHECK_MS)
|
|
63
|
+
this.socket = createAgentActivitySocket({
|
|
64
|
+
client: this.connection.client,
|
|
65
|
+
onActivity: (event) => {
|
|
66
|
+
if (event.runsOn !== "computer") return
|
|
67
|
+
if (event.cancelRequested) {
|
|
68
|
+
for (const runner of this.runners.values()) runner.hearStop(event.runId)
|
|
69
|
+
return
|
|
70
|
+
}
|
|
71
|
+
// Queued turns only reach here by this event for speed: each renewal lists them too.
|
|
72
|
+
if (event.status !== "queued") return
|
|
73
|
+
void this.route(event.threadId, event.runId).catch((error: unknown) =>
|
|
74
|
+
this.onEvent({ type: "error", message: errorMessage(error) })
|
|
75
|
+
)
|
|
76
|
+
},
|
|
77
|
+
})
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Stop every turn running here, each ending interrupted, and close every session. */
|
|
81
|
+
async stop(): Promise<void> {
|
|
82
|
+
this.stopped = true
|
|
83
|
+
this.socket?.close()
|
|
84
|
+
this.unwatch?.()
|
|
85
|
+
if (this.recheck) clearInterval(this.recheck)
|
|
86
|
+
// Not after `reconciled`: a stalled call there mustn't keep a turn from hearing its stop.
|
|
87
|
+
await Promise.allSettled([...this.runners.values()].map((runner) => runner.stop()))
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** One at a time, so two quick changes can't leave the older read in charge. */
|
|
91
|
+
private reconcile(): Promise<void> {
|
|
92
|
+
this.reconciled = this.reconciled.then(() => this.applyAllowedFolders())
|
|
93
|
+
return this.reconciled
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
private async applyAllowedFolders(): Promise<void> {
|
|
97
|
+
if (this.stopped) return
|
|
98
|
+
const allowed = await this.readAllowed()
|
|
99
|
+
const folders = await this.existing(allowed)
|
|
100
|
+
if (this.stopped) return
|
|
101
|
+
|
|
102
|
+
for (const folder of folders) this.runnerFor(folder).openSession()
|
|
103
|
+
for (const [folder, runner] of this.runners) {
|
|
104
|
+
if (folders.includes(folder)) continue
|
|
105
|
+
// Not awaited: a stalled call there mustn't keep the next change from being applied. Taken back,
|
|
106
|
+
// nothing more runs there. Only gone for now, a volume coming back mustn't cost a turn.
|
|
107
|
+
void (allowed.includes(folder) ? runner.closeSession() : runner.stop())
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
if (folders.join("\n") !== this.shown?.join("\n")) {
|
|
111
|
+
this.shown = folders
|
|
112
|
+
this.onEvent({ type: "folders", folders })
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** A list that can't be read allows nothing, and says so. */
|
|
117
|
+
private async readAllowed(): Promise<readonly string[]> {
|
|
118
|
+
try {
|
|
119
|
+
return await readAllowedFolders(this.connection)
|
|
120
|
+
} catch (error) {
|
|
121
|
+
this.onEvent({ type: "error", message: errorMessage(error) })
|
|
122
|
+
return []
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** A folder that's gone reads as offline, rather than as one whose turns all fail. */
|
|
127
|
+
private async existing(folders: readonly string[]): Promise<readonly string[]> {
|
|
128
|
+
const found = await Promise.all(
|
|
129
|
+
folders.map((folder) =>
|
|
130
|
+
stat(folder).then(
|
|
131
|
+
(entry) => entry.isDirectory(),
|
|
132
|
+
() => false
|
|
133
|
+
)
|
|
134
|
+
)
|
|
135
|
+
)
|
|
136
|
+
return folders.filter((_folder, index) => found[index])
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
private runnerFor(folder: string): LocalRunner {
|
|
140
|
+
const known = this.runners.get(folder)
|
|
141
|
+
if (known) return known
|
|
142
|
+
|
|
143
|
+
const runner = new LocalRunner(
|
|
144
|
+
this.connection,
|
|
145
|
+
folder,
|
|
146
|
+
(error) => this.onEvent({ type: "error", message: errorMessage(error) }),
|
|
147
|
+
{
|
|
148
|
+
onSessionChange: (failure) => this.onSessionChange(folder, failure),
|
|
149
|
+
alsoDenied: this.alsoDenied,
|
|
150
|
+
}
|
|
151
|
+
)
|
|
152
|
+
runner.keepsAwake = this.keepsAwake
|
|
153
|
+
runner.onElsewhere((run, ended) => {
|
|
154
|
+
const turn = { folder, threadId: run.threadId, runId: run.id }
|
|
155
|
+
this.onEvent({ type: "turn.started", ...turn })
|
|
156
|
+
void ended.then((ran) => this.onEvent({ type: "turn.ended", ...turn, ran }))
|
|
157
|
+
})
|
|
158
|
+
this.runners.set(folder, runner)
|
|
159
|
+
return runner
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
private onSessionChange(folder: string, failure: unknown | null): void {
|
|
163
|
+
if (failure === null) {
|
|
164
|
+
this.onEvent({ type: "session.back", folder })
|
|
165
|
+
} else if (isSixbApiError(failure) && failure.status === 401) {
|
|
166
|
+
// Every folder's session ends with the sign-in: once is enough to say so.
|
|
167
|
+
if (this.signedOut) return
|
|
168
|
+
this.signedOut = true
|
|
169
|
+
this.onEvent({ type: "signed-out" })
|
|
170
|
+
} else {
|
|
171
|
+
this.onEvent({ type: "session.lost", folder, message: errorMessage(failure) })
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/** Each folder's runner would read the thread to learn it isn't theirs: read it once. */
|
|
176
|
+
private async route(threadId: string, runId: string): Promise<void> {
|
|
177
|
+
const { data: thread } = await getAgentThread({
|
|
178
|
+
client: this.connection.client,
|
|
179
|
+
path: { threadId },
|
|
180
|
+
})
|
|
181
|
+
const folder = thread?.computer?.folder
|
|
182
|
+
if (thread && folder) await this.runners.get(folder)?.offer(thread, runId)
|
|
183
|
+
}
|
|
184
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
// What `@sixb/desktop` compiles into a desktop app: `<binary> <api-url> <client-name> [window-data]`.
|
|
2
|
+
import { runDesktopRunner } from "./desktop-runner"
|
|
3
|
+
|
|
4
|
+
const [apiUrl, clientName, windowData] = process.argv.slice(2)
|
|
5
|
+
if (!apiUrl || !clientName) {
|
|
6
|
+
console.error("[SixbCLI] Usage: <binary> <api-url> <client-name> [window-data]")
|
|
7
|
+
process.exit(2)
|
|
8
|
+
}
|
|
9
|
+
await runDesktopRunner({ apiUrl, clientName, ...(windowData ? { windowData } : {}) })
|