@sixb/cli 0.2.2 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/README.md +25 -22
  2. package/dist/commands/access-token-utils.d.ts.map +1 -1
  3. package/dist/commands/api.d.ts.map +1 -1
  4. package/dist/commands/check.d.ts.map +1 -1
  5. package/dist/commands/deploy.d.ts.map +1 -1
  6. package/dist/commands/dev-runner.d.ts.map +1 -1
  7. package/dist/commands/env.d.ts +6 -1
  8. package/dist/commands/env.d.ts.map +1 -1
  9. package/dist/commands/lake-cleanup.d.ts +2 -12
  10. package/dist/commands/lake-cleanup.d.ts.map +1 -1
  11. package/dist/commands/lake-rejected.d.ts +10 -0
  12. package/dist/commands/lake-rejected.d.ts.map +1 -0
  13. package/dist/commands/orchestrator.d.ts.map +1 -1
  14. package/dist/commands/scheduler.d.ts +2 -5
  15. package/dist/commands/scheduler.d.ts.map +1 -1
  16. package/dist/commands/worker-group.d.ts.map +1 -1
  17. package/dist/lib/command-line.d.ts +12 -7
  18. package/dist/lib/command-line.d.ts.map +1 -1
  19. package/dist/lib/deploy-release.d.ts +7 -1
  20. package/dist/lib/deploy-release.d.ts.map +1 -1
  21. package/dist/lib/duration.d.ts +3 -0
  22. package/dist/lib/duration.d.ts.map +1 -0
  23. package/dist/lib/loadSixb.d.ts.map +1 -1
  24. package/dist/lib/production-roles.d.ts +0 -4
  25. package/dist/lib/production-roles.d.ts.map +1 -1
  26. package/dist/lib/runtime.d.ts +7 -9
  27. package/dist/lib/runtime.d.ts.map +1 -1
  28. package/dist/lib/worker-concurrency.d.ts.map +1 -1
  29. package/dist/lib/worker-registry.d.ts +4 -1
  30. package/dist/lib/worker-registry.d.ts.map +1 -1
  31. package/dist/tui/format.d.ts.map +1 -1
  32. package/dist/ui/deploy-screens.d.ts +4 -2
  33. package/dist/ui/deploy-screens.d.ts.map +1 -1
  34. package/dist/ui/env-screens.d.ts +14 -1
  35. package/dist/ui/env-screens.d.ts.map +1 -1
  36. package/dist/ui/screens.d.ts +5 -11
  37. package/dist/ui/screens.d.ts.map +1 -1
  38. package/package.json +20 -16
  39. package/src/commands/access-token-utils.ts +2 -20
  40. package/src/commands/api.ts +7 -0
  41. package/src/commands/check.ts +75 -1
  42. package/src/commands/deploy.ts +15 -3
  43. package/src/commands/dev-runner.ts +5 -0
  44. package/src/commands/env.ts +99 -20
  45. package/src/commands/lake-cleanup.ts +25 -48
  46. package/src/commands/lake-rejected.ts +36 -0
  47. package/src/commands/orchestrator.ts +2 -7
  48. package/src/commands/scheduler.ts +11 -61
  49. package/src/commands/worker-group.ts +6 -3
  50. package/src/index.tsx +10 -4
  51. package/src/lib/command-line.ts +25 -12
  52. package/src/lib/deploy-release.ts +42 -2
  53. package/src/lib/duration.ts +11 -0
  54. package/src/lib/loadSixb.ts +3 -4
  55. package/src/lib/production-roles.ts +0 -1
  56. package/src/lib/runtime.ts +62 -43
  57. package/src/lib/shareable-providers.ts +2 -2
  58. package/src/lib/worker-concurrency.ts +1 -0
  59. package/src/lib/worker-registry.ts +36 -1
  60. package/src/tui/format.ts +10 -1
  61. package/src/ui/deploy-screens.ts +8 -1
  62. package/src/ui/env-screens.ts +39 -5
  63. package/src/ui/screens.ts +21 -20
@@ -0,0 +1,36 @@
1
+ import { loadSixbFromEntry } from "../lib/loadSixb"
2
+ import { resolveRuntimeEntry } from "../lib/production"
3
+ import { stopSixbProviders } from "../lib/runtime"
4
+ import { withTask } from "../ui"
5
+
6
+ /** A week of refusals is rarely this many; past it, ask for one dataset. */
7
+ const LIMIT = 500
8
+
9
+ export interface LakeRejectedOptions {
10
+ entry?: string
11
+ dataset?: string
12
+ }
13
+
14
+ /**
15
+ * Prints, as JSON, the deliveries refused after `ingest` answered: what each held, whole, and why,
16
+ * so that someone can fix them and send them again. They stay for a week.
17
+ */
18
+ export async function runLakeRejected(options: LakeRejectedOptions = {}) {
19
+ const entry = await resolveRuntimeEntry({ entry: options.entry })
20
+ const sixb = await withTask("lake rejected", "Loading project", () => loadSixbFromEntry(entry))
21
+
22
+ try {
23
+ const inbox = sixb.storage.datasetIngest
24
+ if (!inbox) throw new Error("[SixbCLI] Configured storage keeps no ingested changes.")
25
+
26
+ const rejected = await inbox.listRejected({
27
+ projectId: sixb.id,
28
+ limit: LIMIT,
29
+ ...(options.dataset === undefined ? {} : { datasetId: options.dataset }),
30
+ })
31
+
32
+ process.stdout.write(`${JSON.stringify(rejected, null, 2)}\n`)
33
+ } finally {
34
+ await stopSixbProviders(sixb)
35
+ }
36
+ }
@@ -33,19 +33,14 @@ export async function runOrchestrator(options: OrchestratorOptions = {}) {
33
33
 
34
34
  runtime = await startOrchestratorRuntime(sixb)
35
35
 
36
- const warnings = [...runtime.warnings]
37
- if (runtime.orchestratorWorker === null) {
38
- warnings.push("No orchestrator routes are registered; the orchestrator process is idle.")
39
- }
40
-
41
36
  app.stop()
42
37
  await renderStatic(
43
38
  roleView({
44
39
  command: "orchestrator",
45
40
  name: sixb.id,
46
- items: [{ label: "role", value: "event-to-queue dispatcher" }],
41
+ items: [{ label: "role", value: "event dispatch, cron and upkeep" }],
47
42
  storage: migration.summary,
48
- warnings,
43
+ warnings: runtime.warnings,
49
44
  })
50
45
  )
51
46
 
@@ -1,63 +1,13 @@
1
- import type { LoadedSixbHost } from "../lib/loadSixb"
2
- import { loadProductionSixb } from "../lib/production"
3
- import { runUntilSignal, stopQuietly } from "../lib/role-lifecycle"
4
- import {
5
- type RunningSchedulerRuntime,
6
- startSchedulerRuntime,
7
- stopSixbProviders,
8
- } from "../lib/runtime"
9
- import { migrateStorageForRole } from "../lib/storage-migration"
10
- import { printStopping, renderCliError, renderStatic, roleView, startTask } from "../ui"
11
-
12
- export interface SchedulerOptions {
13
- entry?: string
14
- noMigrate?: boolean
15
- }
16
-
17
- export async function runScheduler(options: SchedulerOptions = {}) {
18
- process.env.NODE_ENV = "production"
19
-
20
- const loaded = await loadProductionSixb({ entry: options.entry, role: "scheduler" })
21
- const app = startTask("scheduler", "Starting scheduler")
22
-
23
- let sixb: LoadedSixbHost | null = loaded.sixb
24
- let runtime: RunningSchedulerRuntime | null = null
25
-
26
- try {
27
- const migration = await migrateStorageForRole(sixb, {
28
- role: "scheduler",
29
- noMigrate: options.noMigrate,
30
- onStart: () => app.step("Migrating storage"),
31
- })
32
-
33
- runtime = await startSchedulerRuntime(sixb)
34
-
35
- app.stop()
36
- await renderStatic(
37
- roleView({
38
- command: "scheduler",
39
- name: sixb.id,
40
- items: [{ label: "role", value: "schedule event producer" }],
41
- storage: migration.summary,
42
- })
43
- )
44
-
45
- await runUntilSignal(async () => {
46
- app.stop()
47
- printStopping("scheduler")
48
- await stopQuietly(() => runtime?.stop() ?? Promise.resolve())
49
- if (sixb) {
50
- await stopSixbProviders(sixb)
51
- }
52
- sixb = null
1
+ import { SixbCliError } from "../lib/errors"
2
+ import { renderCliError } from "../ui"
3
+
4
+ /** The command stays only to say where cron went: a process manager left on it must see a failure. */
5
+ export async function runScheduler(): Promise<void> {
6
+ await renderCliError(
7
+ new SixbCliError("[SixbCLI] `sixb scheduler` no longer runs: `sixb orchestrator` runs cron.", {
8
+ remediation:
9
+ "Remove this process from your deployment and keep one `sixb orchestrator` running.",
53
10
  })
54
- } catch (error) {
55
- app.stop()
56
- await stopQuietly(() => runtime?.stop() ?? Promise.resolve())
57
- if (sixb) {
58
- await stopSixbProviders(sixb)
59
- }
60
- await renderCliError(error)
61
- process.exit(1)
62
- }
11
+ )
12
+ process.exit(1)
63
13
  }
@@ -13,6 +13,7 @@ import {
13
13
  agentRuntimeRequired,
14
14
  assertWorkerInputs,
15
15
  createWorkerForType,
16
+ omittedWorkerWarnings,
16
17
  type QueueWorkerProcess,
17
18
  resolveRegisteredWorkerTypes,
18
19
  resolveWorkerTypeToStart,
@@ -102,10 +103,12 @@ export async function runWorkerGroup(options: WorkerGroupOptions = {}) {
102
103
  )
103
104
  await Promise.all(workers.map((worker) => worker.start()))
104
105
 
105
- const warnings =
106
- workerTypes.length === 0
106
+ const warnings = [
107
+ ...(workerTypes.length === 0
107
108
  ? ["No queue worker types are registered; the worker group process is idle."]
108
- : []
109
+ : []),
110
+ ...omittedWorkerWarnings(host, workerTypes),
111
+ ]
109
112
 
110
113
  app.stop()
111
114
  await renderStatic(
package/src/index.tsx CHANGED
@@ -315,7 +315,7 @@ async function main(): Promise<void> {
315
315
 
316
316
  case "scheduler": {
317
317
  const { runScheduler } = await import("./commands/scheduler")
318
- await runScheduler({ entry: getFlag("entry"), noMigrate: hasFlag("no-migrate") })
318
+ await runScheduler()
319
319
  break
320
320
  }
321
321
 
@@ -376,13 +376,19 @@ async function main(): Promise<void> {
376
376
  break
377
377
  }
378
378
 
379
+ case "lake:rejected": {
380
+ const { runLakeRejected } = await import("./commands/lake-rejected")
381
+ await runLakeRejected({ entry: getFlag("entry"), dataset: getFlag("dataset") })
382
+ break
383
+ }
384
+
379
385
  case "lake:cleanup": {
380
386
  const { runLakeCleanup } = await import("./commands/lake-cleanup")
381
387
  await runLakeCleanup({
382
388
  entry: getFlag("entry"),
383
389
  dryRun: hasFlag("dry-run"),
384
- expireOlderThan: getFlag("expire-older-than"),
385
- deleteOlderThan: getFlag("delete-older-than"),
390
+ keepVersionsFor: getFlag("keep-versions-for"),
391
+ deleteFilesAfter: getFlag("delete-files-after"),
386
392
  })
387
393
  break
388
394
  }
@@ -392,7 +398,7 @@ async function main(): Promise<void> {
392
398
  const dev = getFlag("dev")
393
399
  const prod = getFlag("prod")
394
400
  await runEnvInit({
395
- project: positionals[0] ?? "",
401
+ ...(positionals[0] === undefined ? {} : { project: positionals[0] }),
396
402
  ...(dev === undefined ? {} : { dev }),
397
403
  ...(prod === undefined ? {} : { prod }),
398
404
  })
@@ -186,14 +186,20 @@ export const CLI_OPTION_DEFINITIONS = {
186
186
  summary: "Show what would change without changing it",
187
187
  kind: "boolean",
188
188
  },
189
- "expire-older-than": {
190
- syntax: "--expire-older-than <interval>",
191
- summary: "Lake snapshot expiration window",
189
+ dataset: {
190
+ syntax: "--dataset <id>",
191
+ summary: "Show one dataset's rejected deliveries (default: every dataset)",
192
192
  kind: "string",
193
193
  },
194
- "delete-older-than": {
195
- syntax: "--delete-older-than <interval>",
196
- summary: "Lake file deletion window",
194
+ "keep-versions-for": {
195
+ syntax: "--keep-versions-for <duration>",
196
+ summary:
197
+ "How long a replaced lake version stays readable (default: `lakeMaintenance`, at least 1h)",
198
+ kind: "string",
199
+ },
200
+ "delete-files-after": {
201
+ syntax: "--delete-files-after <duration>",
202
+ summary: "How long unread lake files wait before deletion (default: `lakeMaintenance`)",
197
203
  kind: "string",
198
204
  },
199
205
  } as const satisfies Record<string, { syntax: string; summary: string; kind: OptionKind }>
@@ -233,6 +239,7 @@ export type LocalCommandId =
233
239
  | "db:prepare"
234
240
  | "lake:check"
235
241
  | "lake:cleanup"
242
+ | "lake:rejected"
236
243
  | "deploy"
237
244
  | "deploy:status"
238
245
  | "deploy:logs"
@@ -300,6 +307,7 @@ const INSTANCE_SUMMARIES = {
300
307
  workflows: "Discover and start workflows",
301
308
  "workflow-runs": "Inspect workflow execution history",
302
309
  users: "List and search project members",
310
+ artifacts: "Read, resolve and reply to an artifact's comments",
303
311
  models: "List the project's models and their prices",
304
312
  datasets: "Read datasets, their versions and rows",
305
313
  syncs: "List the syncs you may run",
@@ -489,10 +497,10 @@ const commandTree: readonly CommandNode[] = [
489
497
  },
490
498
  ],
491
499
  },
492
- command("scheduler", "Start the production scheduler", {
500
+ command("scheduler", "Removed: the orchestrator runs cron schedules", {
493
501
  options: ["entry", "no-migrate"],
494
502
  }),
495
- command("orchestrator", "Start the production orchestrator, which queues work for events", {
503
+ command("orchestrator", "Start the production orchestrator: cron, event dispatch and upkeep", {
496
504
  options: ["entry", "no-migrate"],
497
505
  }),
498
506
  command("rules", "Start the production rules runtime", { options: ["entry", "no-migrate"] }),
@@ -576,9 +584,14 @@ const commandTree: readonly CommandNode[] = [
576
584
  }),
577
585
  command("lake:cleanup", "Clean up old lake snapshots and files", {
578
586
  path: ["lake", "cleanup"],
579
- options: ["entry", "dry-run", "expire-older-than", "delete-older-than"],
587
+ options: ["entry", "dry-run", "keep-versions-for", "delete-files-after"],
580
588
  rootHelpLabel: "lake cleanup",
581
589
  }),
590
+ command("lake:rejected", "Show ingested deliveries refused after they were accepted", {
591
+ path: ["lake", "rejected"],
592
+ options: ["entry", "dataset"],
593
+ rootHelpLabel: "lake rejected",
594
+ }),
582
595
  ],
583
596
  },
584
597
  {
@@ -650,9 +663,8 @@ const commandTree: readonly CommandNode[] = [
650
663
  children: [
651
664
  command("env:init", "Move a project's values to its source and write sixb.env.ts", {
652
665
  path: ["env", "init"],
653
- usage: "sixb env init <project> [--dev <file>] [--prod <file>]",
666
+ usage: "sixb env init [doppler-project] [--dev <file>] [--prod <file>]",
654
667
  options: ["dev", "prod"],
655
- minimumPositionals: 1,
656
668
  maximumPositionals: 1,
657
669
  }),
658
670
  command("env:pull", "Write .env from the project's source", {
@@ -743,8 +755,9 @@ const ROOT_HELP_GROUP_COMMANDS: readonly {
743
755
  { title: "Deploy", names: ["deploy", "desktop"] },
744
756
  {
745
757
  title: "Run in production",
746
- names: ["api", "app", "scheduler", "orchestrator", "rules", "worker", "worker-group"],
758
+ names: ["api", "app", "orchestrator", "rules", "worker", "worker-group"],
747
759
  },
760
+ { title: "Removed", names: ["scheduler"] },
748
761
  { title: "Storage", names: ["db", "lake"] },
749
762
  {
750
763
  title: "Connect to an instance",
@@ -1,4 +1,5 @@
1
1
  import type {
2
+ DeployCheck,
2
3
  DeployCommand,
3
4
  DeployConfig,
4
5
  DeployEnv,
@@ -28,7 +29,6 @@ const ROLE_SERVICES = {
28
29
  api: "api",
29
30
  app: "app",
30
31
  orchestrator: "orchestrator",
31
- scheduler: "scheduler",
32
32
  rules: "rules",
33
33
  "worker-group": "workers",
34
34
  // A deployment co-hosts its workers in one `worker-group`; `workers.process.instances` scales it.
@@ -44,7 +44,6 @@ const SERVICE_ROLES: { readonly [Service in DeployServiceName]: RoleOf<Service>
44
44
  api: "api",
45
45
  app: "app",
46
46
  orchestrator: "orchestrator",
47
- scheduler: "scheduler",
48
47
  rules: "rules",
49
48
  workers: "worker-group",
50
49
  }
@@ -168,6 +167,47 @@ function sixbService(
168
167
  }
169
168
  }
170
169
 
170
+ /**
171
+ * What this config leaves the lake's background work without: its retention, and the changes
172
+ * `ingest` stores. Only the project knows whether it has datasets, and this reads the config alone,
173
+ * so it warns and never stops a deploy.
174
+ */
175
+ export function lakeWorkChecks(config: DeployConfig): readonly DeployCheck[] {
176
+ const { orchestrator, workers } = config.services ?? {}
177
+ const types = typeof workers === "object" ? workers.types : undefined
178
+ const lakeWorkerOmitted = workers === false || (Array.isArray(types) && !types.includes("lake"))
179
+
180
+ const missing = [
181
+ ...(orchestrator === false ? ["the orchestrator requests it"] : []),
182
+ ...(lakeWorkerOmitted ? ["a `lake` worker runs it"] : []),
183
+ ]
184
+
185
+ return [
186
+ ...(missing.length > 0
187
+ ? [
188
+ {
189
+ id: "lake-retention",
190
+ label: "Lake retention",
191
+ status: "warning" as const,
192
+ detail: `Not deployed: ${missing.join(", and ")}.`,
193
+ remedy: "Deploy them, or a project with datasets keeps every replaced version.",
194
+ },
195
+ ]
196
+ : []),
197
+ ...(lakeWorkerOmitted
198
+ ? [
199
+ {
200
+ id: "dataset-ingestion",
201
+ label: "Dataset ingestion",
202
+ status: "warning" as const,
203
+ detail: "Not deployed: a `lake` worker writes what `ingest` stores.",
204
+ remedy: "Deploy one, or ingested changes wait and never reach their dataset.",
205
+ },
206
+ ]
207
+ : []),
208
+ ]
209
+ }
210
+
171
211
  function workerGroupArgs(options: DeployWorkersConfig): string[] {
172
212
  const types = options.types === undefined || options.types === "all" ? [] : options.types
173
213
  const args = ["worker-group", ...types]
@@ -0,0 +1,11 @@
1
+ import { parseDuration } from "@sixb/core/internal/runtime"
2
+
3
+ /** A duration flag such as 30m, 12h, 7d, 4w or 1y, in milliseconds. */
4
+ export function parseDurationMs(value: string, flag: string): number {
5
+ const durationMs = parseDuration(value)
6
+ if (durationMs === null) {
7
+ throw new Error(`[SixbCLI] Use ${flag} like 30d, 12h, 4w, or 1y.`)
8
+ }
9
+
10
+ return durationMs
11
+ }
@@ -17,13 +17,14 @@ const REQUIRED_HOST_PROPERTIES = [
17
17
  "blobStorage",
18
18
  "queues",
19
19
  "auth",
20
- "scheduler",
21
20
  ] as const
22
21
 
23
22
  const REQUIRED_LIFECYCLE_METHODS = [
24
23
  "withScope",
24
+ "startMaintenance",
25
25
  "startOntologyMaintenance",
26
26
  "getOntologyOperationalStatus",
27
+ "prepareReadiness",
27
28
  "checkReadiness",
28
29
  "closeLogger",
29
30
  "closeConnectors",
@@ -103,9 +104,7 @@ function isSixbHost(value: unknown): value is LoadedSixbHost {
103
104
  isRecord(value.logging) &&
104
105
  hasMethods(value.logging, ["startExecution", "read", "tail"]) &&
105
106
  isRecord(value.blobStorage) &&
106
- hasMethods(value.blobStorage, ["put", "open", "stat"]) &&
107
- isRecord(value.scheduler) &&
108
- hasMethods(value.scheduler, ["start", "stop"])
107
+ hasMethods(value.blobStorage, ["put", "open", "stat"])
109
108
  )
110
109
  }
111
110
 
@@ -7,7 +7,6 @@
7
7
  const PRODUCTION_ROLES = {
8
8
  api: { onEventPlane: true, usesStorageSchema: true },
9
9
  rules: { onEventPlane: true, usesStorageSchema: true },
10
- scheduler: { onEventPlane: true, usesStorageSchema: true },
11
10
  orchestrator: { onEventPlane: true, usesStorageSchema: true },
12
11
  worker: { onEventPlane: true, usesStorageSchema: true },
13
12
  "worker-group": { onEventPlane: true, usesStorageSchema: true },
@@ -7,15 +7,17 @@ import {
7
7
  getProjectionDispatchDescriptors,
8
8
  ProjectionRunDispatcher,
9
9
  } from "@sixb/core/internal/projections"
10
- import { setApiPublicOrigin } from "@sixb/core/internal/runtime"
10
+ import { getDatasetIngestRuntime, setApiPublicOrigin } from "@sixb/core/internal/runtime"
11
11
  import { checkPipelineMarkings, getAppliedMarkings } from "@sixb/core/internal/security"
12
12
  import { SyncRunDispatcher } from "@sixb/core/internal/syncs"
13
13
  import type { Worker } from "@sixb/core/internal/workers"
14
14
  import { WorkflowRunDispatcher } from "@sixb/core/internal/workflows"
15
15
  import { assertLakeDatasetDefinitionsCompatible } from "@sixb/core/lake-storage"
16
+ import { LakeWorker } from "@sixb/lake-worker"
16
17
  import {
17
18
  type CompileRoutesDiagnostic,
18
19
  compileRoutesWithDiagnostics,
20
+ createMaintenancePasses,
19
21
  OrchestratorWorker,
20
22
  } from "@sixb/orchestrator"
21
23
  import { PipelineWorker } from "@sixb/pipeline-worker"
@@ -27,7 +29,7 @@ import type { LoadedSixbHost } from "./loadSixb"
27
29
  import { configuredOrigin } from "./public-origin"
28
30
  import { stopQuietly } from "./role-lifecycle"
29
31
  import type { WorkerConcurrency } from "./worker-registry"
30
- import { agentRuntimeRequired, agentWorkerRequired } from "./worker-registry"
32
+ import { agentRuntimeRequired, agentWorkerRequired, lakeWorkerRequired } from "./worker-registry"
31
33
 
32
34
  export function waitForWorkerFailure(worker: Worker | null | undefined): Promise<never> {
33
35
  return new Promise<never>((_resolve, reject) => {
@@ -42,6 +44,7 @@ async function closeProvider(provider: { close?(): void | Promise<void> }): Prom
42
44
  export interface RunningSixbRuntime {
43
45
  readonly rulesWorker: RulesWorker | null
44
46
  readonly syncWorker: SyncWorker | null
47
+ readonly lakeWorker: LakeWorker | null
45
48
  readonly agentWorker: AgentWorker | null
46
49
  readonly projectionWorker: ProjectionWorker | null
47
50
  readonly pipelineWorker: PipelineWorker | null
@@ -76,10 +79,6 @@ export interface RunningRulesRuntime {
76
79
  stop(): Promise<void>
77
80
  }
78
81
 
79
- export interface RunningSchedulerRuntime {
80
- stop(): Promise<void>
81
- }
82
-
83
82
  export interface RunningOrchestratorRuntime {
84
83
  readonly orchestratorWorker: OrchestratorWorker | null
85
84
  readonly warnings: readonly string[]
@@ -161,18 +160,6 @@ export async function startRulesRuntime(sixb: LoadedSixbHost): Promise<RunningRu
161
160
  }
162
161
  }
163
162
 
164
- export async function startSchedulerRuntime(
165
- sixb: LoadedSixbHost
166
- ): Promise<RunningSchedulerRuntime> {
167
- await sixb.scheduler.start()
168
-
169
- return {
170
- async stop() {
171
- await stopQuietly(() => sixb.scheduler.stop())
172
- },
173
- }
174
- }
175
-
176
163
  export async function startOrchestratorRuntime(
177
164
  sixb: LoadedSixbHost
178
165
  ): Promise<RunningOrchestratorRuntime> {
@@ -185,42 +172,69 @@ export async function startOrchestratorRuntime(
185
172
  workflows: sixb.definitions.workflows.list(),
186
173
  })
187
174
  const warnings = diagnostics.map(formatRouteDiagnosticWarning)
188
- let orchestratorWorker: OrchestratorWorker | null = null
175
+ const dispatchers = {
176
+ syncs: new SyncRunDispatcher(sixb),
177
+ pipelines: new PipelineRunDispatcher(sixb),
178
+ projections: new ProjectionRunDispatcher(sixb),
179
+ workflows: new WorkflowRunDispatcher(sixb),
180
+ }
189
181
 
182
+ const { projectionRuns, pipelineRuns, datasetIngest } = sixb.storage
183
+ if (projections.length > 0 && !projectionRuns) {
184
+ throw new Error("[SixbCLI] Projection dispatch requires storage.projectionRuns.")
185
+ }
186
+
187
+ let orchestratorWorker: OrchestratorWorker | null = null
190
188
  if (routes.size > 0) {
191
- const projectionRuns = sixb.storage.projectionRuns
192
- if (projections.length > 0 && !projectionRuns) {
193
- throw new Error("[SixbCLI] Projection dispatch requires storage.projectionRuns.")
194
- }
195
- const projectionReconciliation =
196
- projections.length > 0 && projectionRuns
197
- ? { lakeStorage: sixb.lakeStorage, projectionRuns }
198
- : undefined
199
189
  orchestratorWorker = new OrchestratorWorker({
200
190
  projectId: sixb.id,
201
191
  events: sixb.events,
202
192
  queues: sixb.queues,
203
193
  routes,
204
- dispatchers: {
205
- syncs: new SyncRunDispatcher(sixb),
206
- pipelines: new PipelineRunDispatcher(sixb),
207
- projections: new ProjectionRunDispatcher(sixb),
208
- workflows: new WorkflowRunDispatcher(sixb),
209
- },
210
- ...(projectionReconciliation ? { projectionReconciliation } : {}),
194
+ dispatchers,
195
+ reporter: sixb,
211
196
  })
212
197
  await orchestratorWorker.start()
213
198
  }
214
199
 
200
+ // Runs with or without routes: every project commits events, and cron emits what routes consume.
201
+ const maintenance = sixb.startMaintenance(
202
+ createMaintenancePasses({
203
+ projectId: sixb.id,
204
+ routes,
205
+ dispatchers,
206
+ lakeStorage: sixb.lakeStorage,
207
+ ...(projectionRuns ? { projectionRuns } : {}),
208
+ ...(pipelineRuns ? { pipelineRuns } : {}),
209
+ ...(datasetIngest && getDatasetIngestRuntime(sixb).enabled
210
+ ? { ingest: { inbox: datasetIngest, queue: sixb.queues.lake, reporter: sixb } }
211
+ : {}),
212
+ ...(hasRecoverableRuns(sixb)
213
+ ? { runs: { storage: sixb.storage, queues: sixb.queues, events: sixb.events } }
214
+ : {}),
215
+ })
216
+ )
217
+
215
218
  return {
216
219
  orchestratorWorker,
217
220
  warnings,
218
221
  async stop() {
222
+ await stopQuietly(() => maintenance.stop())
219
223
  await stopQuietly(() => orchestratorWorker?.stop() ?? Promise.resolve())
220
224
  },
221
225
  }
222
226
  }
223
227
 
228
+ /** A project with none of these has no run a worker can lose, and pays for no look at them. */
229
+ function hasRecoverableRuns({ definitions }: LoadedSixbHost): boolean {
230
+ return [
231
+ definitions.syncs,
232
+ definitions.pipelines,
233
+ definitions.workflows,
234
+ definitions.projections,
235
+ ].some((registry) => registry.list().length > 0)
236
+ }
237
+
224
238
  /**
225
239
  * Starts the background runtimes owned by the CLI host.
226
240
  *
@@ -230,32 +244,32 @@ export async function startOrchestratorRuntime(
230
244
  * 3. PipelineWorker (claims from queues)
231
245
  * 4. WorkflowWorker (claims from queues)
232
246
  * 5. SyncWorker (claims from queues)
233
- * 6. Orchestrator (subscribes to events and enqueues jobs)
234
- * 7. Scheduler (emits schedule.triggered)
247
+ * 6. LakeWorker (claims from queues)
248
+ * 7. Orchestrator (subscribes to events, enqueues jobs, runs cron and the maintenance loop)
235
249
  * 8. API server, when this process serves it
236
250
  * 9. AgentWorker (claims from queues; its turns call the API)
237
251
  *
238
252
  * Shutdown order (producers before consumers):
239
253
  * 1. AgentWorker (ends its turns while the API still answers them)
240
254
  * 2. API server
241
- * 3. Scheduler
242
- * 4. Orchestrator (drains pending dispatches)
255
+ * 3. Orchestrator (stops cron, drains pending dispatches)
256
+ * 4. LakeWorker
243
257
  * 5. SyncWorker
244
258
  * 6. WorkflowWorker
245
259
  * 7. PipelineWorker
246
260
  * 8. ProjectionWorker
247
261
  * 9. RulesWorker (drains pending evaluations)
248
- * 10. Runtime providers (connectors, broker)
262
+ * 10. Runtime providers (connectors, broker)
249
263
  */
250
264
  export async function startSixbRuntime(
251
265
  sixb: LoadedSixbHost,
252
266
  options: StartSixbRuntimeOptions = {}
253
267
  ): Promise<RunningSixbRuntime> {
254
268
  let rulesRuntime: RunningRulesRuntime | null = null
255
- let schedulerRuntime: RunningSchedulerRuntime | null = null
256
269
  let orchestratorRuntime: RunningOrchestratorRuntime | null = null
257
270
  let rulesWorker: RulesWorker | null = null
258
271
  let syncWorker: SyncWorker | null = null
272
+ let lakeWorker: LakeWorker | null = null
259
273
  let agentWorker: AgentWorker | null = null
260
274
  let pipelineWorker: PipelineWorker | null = null
261
275
  let workflowWorker: WorkflowWorker | null = null
@@ -266,8 +280,8 @@ export async function startSixbRuntime(
266
280
  async function stop() {
267
281
  await stopQuietly(() => agentWorker?.stop() ?? Promise.resolve())
268
282
  await stopQuietly(() => options.api?.stop() ?? Promise.resolve())
269
- await stopQuietly(() => schedulerRuntime?.stop() ?? Promise.resolve())
270
283
  await stopQuietly(() => orchestratorRuntime?.stop() ?? Promise.resolve())
284
+ await stopQuietly(() => lakeWorker?.stop() ?? Promise.resolve())
271
285
  await stopQuietly(() => syncWorker?.stop() ?? Promise.resolve())
272
286
  await stopQuietly(() => workflowWorker?.stop() ?? Promise.resolve())
273
287
  await stopQuietly(() => pipelineWorker?.stop() ?? Promise.resolve())
@@ -325,11 +339,14 @@ export async function startSixbRuntime(
325
339
  await syncWorker.start()
326
340
  }
327
341
 
342
+ if (lakeWorkerRequired(sixb)) {
343
+ lakeWorker = new LakeWorker(sixb, { concurrency: options.workerConcurrency?.lake })
344
+ await lakeWorker.start()
345
+ }
346
+
328
347
  orchestratorRuntime = await startOrchestratorRuntime(sixb)
329
348
  orchestratorWorker = orchestratorRuntime.orchestratorWorker
330
349
  warnings.push(...orchestratorRuntime.warnings)
331
-
332
- schedulerRuntime = await startSchedulerRuntime(sixb)
333
350
  }
334
351
 
335
352
  await options.api?.start()
@@ -343,6 +360,7 @@ export async function startSixbRuntime(
343
360
  const workers: Worker[] = []
344
361
  if (rulesWorker) workers.push(rulesWorker)
345
362
  if (syncWorker) workers.push(syncWorker)
363
+ if (lakeWorker) workers.push(lakeWorker)
346
364
  if (agentWorker) workers.push(agentWorker)
347
365
  if (projectionWorker) workers.push(projectionWorker)
348
366
  if (pipelineWorker) workers.push(pipelineWorker)
@@ -354,6 +372,7 @@ export async function startSixbRuntime(
354
372
  return {
355
373
  rulesWorker,
356
374
  syncWorker,
375
+ lakeWorker,
357
376
  agentWorker,
358
377
  projectionWorker,
359
378
  pipelineWorker,
@@ -14,12 +14,12 @@ const SLOTS = [
14
14
  {
15
15
  name: "queues",
16
16
  get: (sixb: LoadedSixbHost): Broker | Queues => sixb.queues,
17
- replacements: "@sixb/bullmq",
17
+ replacements: "@sixb/queues-bullmq",
18
18
  },
19
19
  {
20
20
  name: "broker",
21
21
  get: (sixb: LoadedSixbHost): Broker | Queues => sixb.broker,
22
- replacements: "@sixb/redis or @sixb/nats",
22
+ replacements: "@sixb/broker-redis or @sixb/broker-nats",
23
23
  },
24
24
  ] as const
25
25
 
@@ -7,6 +7,7 @@ const ENVIRONMENT_VARIABLES = {
7
7
  pipeline: "SIXB_PIPELINE_WORKER_CONCURRENCY",
8
8
  projection: "SIXB_PROJECTION_WORKER_CONCURRENCY",
9
9
  workflow: "SIXB_WORKFLOW_WORKER_CONCURRENCY",
10
+ lake: "SIXB_LAKE_WORKER_CONCURRENCY",
10
11
  } as const satisfies Record<WorkerType, string>
11
12
 
12
13
  /** Resolve the scalar concurrency accepted by `sixb worker <type>`. */