@sixb/cli 0.2.2 → 0.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/README.md +25 -22
  2. package/dist/commands/access-token-utils.d.ts.map +1 -1
  3. package/dist/commands/agent.d.ts +5 -0
  4. package/dist/commands/agent.d.ts.map +1 -1
  5. package/dist/commands/api.d.ts.map +1 -1
  6. package/dist/commands/check.d.ts.map +1 -1
  7. package/dist/commands/deploy.d.ts.map +1 -1
  8. package/dist/commands/dev-runner.d.ts.map +1 -1
  9. package/dist/commands/env.d.ts +6 -1
  10. package/dist/commands/env.d.ts.map +1 -1
  11. package/dist/commands/lake-cleanup.d.ts +2 -12
  12. package/dist/commands/lake-cleanup.d.ts.map +1 -1
  13. package/dist/commands/lake-rejected.d.ts +10 -0
  14. package/dist/commands/lake-rejected.d.ts.map +1 -0
  15. package/dist/commands/orchestrator.d.ts.map +1 -1
  16. package/dist/commands/scheduler.d.ts +2 -5
  17. package/dist/commands/scheduler.d.ts.map +1 -1
  18. package/dist/commands/worker-group.d.ts.map +1 -1
  19. package/dist/lib/command-line.d.ts +12 -7
  20. package/dist/lib/command-line.d.ts.map +1 -1
  21. package/dist/lib/deploy-release.d.ts +7 -1
  22. package/dist/lib/deploy-release.d.ts.map +1 -1
  23. package/dist/lib/duration.d.ts +3 -0
  24. package/dist/lib/duration.d.ts.map +1 -0
  25. package/dist/lib/loadSixb.d.ts.map +1 -1
  26. package/dist/lib/production-roles.d.ts +0 -4
  27. package/dist/lib/production-roles.d.ts.map +1 -1
  28. package/dist/lib/runtime.d.ts +7 -9
  29. package/dist/lib/runtime.d.ts.map +1 -1
  30. package/dist/lib/worker-concurrency.d.ts.map +1 -1
  31. package/dist/lib/worker-registry.d.ts +4 -1
  32. package/dist/lib/worker-registry.d.ts.map +1 -1
  33. package/dist/runner/local-runner.d.ts +1 -1
  34. package/dist/tui/app.d.ts.map +1 -1
  35. package/dist/tui/format.d.ts +20 -0
  36. package/dist/tui/format.d.ts.map +1 -1
  37. package/dist/tui/transcript.d.ts.map +1 -1
  38. package/dist/ui/deploy-screens.d.ts +4 -2
  39. package/dist/ui/deploy-screens.d.ts.map +1 -1
  40. package/dist/ui/env-screens.d.ts +14 -1
  41. package/dist/ui/env-screens.d.ts.map +1 -1
  42. package/dist/ui/screens.d.ts +5 -11
  43. package/dist/ui/screens.d.ts.map +1 -1
  44. package/package.json +20 -16
  45. package/src/commands/access-token-utils.ts +2 -20
  46. package/src/commands/agent.ts +54 -8
  47. package/src/commands/api.ts +7 -0
  48. package/src/commands/check.ts +75 -1
  49. package/src/commands/deploy.ts +15 -3
  50. package/src/commands/dev-runner.ts +5 -0
  51. package/src/commands/env.ts +99 -20
  52. package/src/commands/lake-cleanup.ts +25 -48
  53. package/src/commands/lake-rejected.ts +36 -0
  54. package/src/commands/orchestrator.ts +2 -7
  55. package/src/commands/scheduler.ts +11 -61
  56. package/src/commands/worker-group.ts +6 -3
  57. package/src/index.tsx +10 -4
  58. package/src/lib/command-line.ts +25 -12
  59. package/src/lib/deploy-release.ts +42 -2
  60. package/src/lib/duration.ts +11 -0
  61. package/src/lib/loadSixb.ts +3 -4
  62. package/src/lib/production-roles.ts +0 -1
  63. package/src/lib/runtime.ts +62 -43
  64. package/src/lib/shareable-providers.ts +2 -2
  65. package/src/lib/worker-concurrency.ts +1 -0
  66. package/src/lib/worker-registry.ts +36 -1
  67. package/src/runner/local-runner.ts +1 -1
  68. package/src/tui/app.tsx +2 -1
  69. package/src/tui/format.ts +190 -3
  70. package/src/tui/transcript.tsx +8 -1
  71. package/src/ui/deploy-screens.ts +8 -1
  72. package/src/ui/env-screens.ts +39 -5
  73. package/src/ui/screens.ts +21 -20
@@ -23,6 +23,7 @@ import { errorMessage, SixbCliError } from "../lib/errors"
23
23
  import { promptHidden } from "../lib/hidden-prompt"
24
24
  import { renderStatic } from "../ui"
25
25
  import {
26
+ type EnvInitConfig,
26
27
  type EnvStatusCell,
27
28
  envInitView,
28
29
  envKeptView,
@@ -34,6 +35,7 @@ import { canRunLive } from "../ui/live/can-run-live"
34
35
 
35
36
  const ENV_NAMES: readonly EnvName[] = ["dev", "prod"]
36
37
  const DOPPLER_PACKAGE = "@sixb/env-doppler"
38
+ const SOURCE_EXAMPLE = `Add one, such as \`source: doppler({ project: "shop" })\` from ${DOPPLER_PACKAGE}, or \`source: azureKeyVault({ vaults: { dev: "shop-dev", prod: "shop-prod" } })\` from @sixb/env-azure-key-vault.`
37
39
 
38
40
  // ─── pull ────────────────────────────────────────────────────────────────────
39
41
 
@@ -264,16 +266,25 @@ export async function runEnvStatus(options: { readonly cwd?: string } = {}): Pro
264
266
 
265
267
  // ─── init ────────────────────────────────────────────────────────────────────
266
268
 
269
+ /**
270
+ * Moves a project's values to its source. Given a Doppler project, it also writes `sixb.env.ts`.
271
+ * Without one, it uploads to the source `sixb.env.ts` already names, whatever it is, and fills in
272
+ * its empty `vars: {}`.
273
+ */
267
274
  export async function runEnvInit(options: {
268
- readonly project: string
275
+ readonly project?: string
269
276
  readonly dev?: string
270
277
  readonly prod?: string
271
278
  readonly cwd?: string
272
279
  }): Promise<void> {
273
280
  const cwd = options.cwd ?? process.cwd()
274
281
  const configPath = join(cwd, ENV_CONFIG_FILE)
275
- if (await Bun.file(configPath).exists()) {
276
- throw new SixbCliError(`[SixbEnv] ${ENV_CONFIG_FILE} already exists.`)
282
+ const { project } = options
283
+ const configExists = await Bun.file(configPath).exists()
284
+ if (project !== undefined && configExists) {
285
+ throw new SixbCliError(`[SixbEnv] ${ENV_CONFIG_FILE} already exists.`, {
286
+ remediation: `To upload to the source it names, leave out the project: ${INIT_USAGE}`,
287
+ })
277
288
  }
278
289
 
279
290
  const files = [
@@ -283,11 +294,13 @@ export async function runEnvInit(options: {
283
294
  const given = files.flatMap(([env, file]) => (file === undefined ? [] : [{ env, file }]))
284
295
  if (given.length === 0) {
285
296
  throw new SixbCliError("[SixbEnv] Say where the current values are.", {
286
- remediation: "sixb env init <project> --dev .env --prod .env.prod",
297
+ remediation: project === undefined ? INIT_USAGE : `sixb env init ${project} ${INIT_FILES}`,
287
298
  })
288
299
  }
289
300
 
290
- const source = await loadDopplerSource(cwd, options.project)
301
+ const existing = configExists ? await loadEnvConfig(cwd) : null
302
+ const source =
303
+ project === undefined ? sourceToFill(existing) : await loadDopplerSource(cwd, project)
291
304
  const uploads = await Promise.all(
292
305
  given.map(async ({ env, file }) => {
293
306
  const path = resolve(cwd, file)
@@ -298,33 +311,102 @@ export async function runEnvInit(options: {
298
311
  })
299
312
  )
300
313
 
301
- // Check every environment before writing to any, so a conflict changes nothing. Running this
302
- // again after a failed upload finds the same values, which aren't a conflict.
314
+ // Check every environment before writing to any, so a conflict changes nothing. A value the
315
+ // source already has isn't sent again, so running this twice, or after an upload that didn't
316
+ // finish, changes nothing it doesn't need to.
317
+ const sends: { readonly env: EnvName; readonly values: Record<string, string> }[] = []
303
318
  for (const { env, values } of uploads) {
304
- const existing = await source.read(env)
319
+ const current = await source.read(env)
305
320
  const conflicts = Object.keys(values).filter(
306
- (name) => name in existing && existing[name] !== values[name]
321
+ (name) => name in current && current[name] !== values[name]
307
322
  )
308
323
  if (conflicts.length > 0) {
309
324
  throw new SixbCliError(
310
- `[SixbEnv] ${options.project} already has other values in ${env} for ${conflicts.sort().join(", ")}.`,
325
+ `[SixbEnv] ${project ?? "The source"} already has other values in ${env} for ${conflicts.sort().join(", ")}.`,
311
326
  {
312
327
  remediation:
313
328
  "Change them in your source's dashboard, or edit the file, and run this again.",
314
329
  }
315
330
  )
316
331
  }
332
+ sends.push({
333
+ env,
334
+ values: Object.fromEntries(Object.entries(values).filter(([name]) => !(name in current))),
335
+ })
317
336
  }
318
- for (const { env, values } of uploads) await source.write(env, values)
337
+ for (const { env, values } of sends) await source.write(env, values)
319
338
 
320
339
  const vars = declaredVars(uploads)
321
- await writeFile(configPath, renderEnvConfig(options.project, vars))
340
+ let config: EnvInitConfig
341
+ if (project !== undefined) {
342
+ await writeFile(configPath, renderEnvConfig(project, vars))
343
+ config = { kind: "written", names: Object.keys(vars).length }
344
+ } else {
345
+ config = await fillVars(configPath, existing?.vars ?? {}, vars)
346
+ }
322
347
  await renderStatic(
323
348
  envInitView({
324
- uploaded: uploads.map(({ env, values }) => ({ env, count: Object.keys(values).length })),
325
- names: Object.keys(vars).length,
349
+ uploaded: uploads.map(({ env, values }, index) => {
350
+ const sent = Object.keys(sends[index]?.values ?? {}).length
351
+ return { env, count: sent, already: Object.keys(values).length - sent }
352
+ }),
353
+ config,
354
+ })
355
+ )
356
+ }
357
+
358
+ const INIT_FILES = "--dev .env --prod .env.prod"
359
+ const INIT_USAGE = `sixb env init ${INIT_FILES}`
360
+
361
+ function sourceToFill(config: EnvConfig | null): EnvSource {
362
+ if (!config) {
363
+ throw new SixbCliError(`[SixbEnv] There's no ${ENV_CONFIG_FILE} naming a source yet.`, {
364
+ remediation: `Write one with its source and \`vars: {}\`, then run this again. For Doppler, \`sixb env init <project> ${INIT_FILES}\` writes it for you.`,
365
+ })
366
+ }
367
+ if (!config.source) {
368
+ throw new SixbCliError(`[SixbEnv] ${ENV_CONFIG_FILE} has no \`source\` to upload to.`, {
369
+ remediation: SOURCE_EXAMPLE,
326
370
  })
371
+ }
372
+ return config.source
373
+ }
374
+
375
+ /**
376
+ * Declares the uploaded names in an existing `sixb.env.ts`. Only an empty `vars: {}` is replaced:
377
+ * anything else is the author's code, so the rules are printed for them to add instead.
378
+ */
379
+ async function fillVars(
380
+ path: string,
381
+ declared: Readonly<Record<string, EnvVarRule>>,
382
+ guessed: Readonly<Record<string, EnvVarRule>>
383
+ ): Promise<EnvInitConfig> {
384
+ const undeclared = Object.fromEntries(
385
+ Object.entries(guessed).filter(([name]) => !(name in declared))
386
+ )
387
+ const names = Object.keys(undeclared).length
388
+ if (names === 0) return { kind: "unchanged" }
389
+
390
+ const text = await readFile(path, "utf8")
391
+ const empty = [...text.matchAll(EMPTY_VARS)]
392
+ if (Object.keys(declared).length > 0 || empty.length !== 1) {
393
+ return { kind: "paste", names, lines: varLines(undeclared, "") }
394
+ }
395
+ await writeFile(
396
+ path,
397
+ text.replace(EMPTY_VARS, (_match, indent: string) => renderVars(undeclared, indent))
327
398
  )
399
+ return { kind: "written", names }
400
+ }
401
+
402
+ const EMPTY_VARS = /^([ \t]*)vars:\s*\{\s*\}/gm
403
+
404
+ function renderVars(vars: Readonly<Record<string, EnvVarRule>>, indent: string): string {
405
+ return [`${indent}vars: {`, ...varLines(vars, `${indent} `), `${indent}}`].join("\n")
406
+ }
407
+
408
+ function varLines(vars: Readonly<Record<string, EnvVarRule>>, indent: string): string[] {
409
+ return Object.entries(vars).map(([name, rule]) => `${indent}${name}: "${rule}",`)
328
410
  }
329
411
 
330
412
  /**
@@ -366,16 +448,13 @@ function uploadableValues(
366
448
  }
367
449
 
368
450
  function renderEnvConfig(project: string, vars: Readonly<Record<string, EnvVarRule>>): string {
369
- const lines = Object.entries(vars).map(([name, rule]) => ` ${name}: "${rule}",`)
370
451
  return [
371
452
  'import { defineEnv } from "@sixb/core/env"',
372
453
  `import { doppler } from "${DOPPLER_PACKAGE}"`,
373
454
  "",
374
455
  "export default defineEnv({",
375
456
  ` source: doppler({ project: ${JSON.stringify(project)} }),`,
376
- " vars: {",
377
- ...lines,
378
- " },",
457
+ `${renderVars(vars, " ")},`,
379
458
  "})",
380
459
  "",
381
460
  ].join("\n")
@@ -410,12 +489,12 @@ async function requireSource(
410
489
  const config = await loadEnvConfig(cwd)
411
490
  if (!config) {
412
491
  throw new SixbCliError(`[SixbEnv] No ${ENV_CONFIG_FILE} in ${cwd}.`, {
413
- remediation: "sixb env init <project> --dev .env --prod .env.prod",
492
+ remediation: `sixb env init <project> ${INIT_FILES}`,
414
493
  })
415
494
  }
416
495
  if (!config.source) {
417
496
  throw new SixbCliError(`[SixbEnv] ${ENV_CONFIG_FILE} has no \`source\`.`, {
418
- remediation: `Add one, such as \`source: doppler({ project: "shop" })\` from ${DOPPLER_PACKAGE}.`,
497
+ remediation: SOURCE_EXAMPLE,
419
498
  })
420
499
  }
421
500
  return { config, source: config.source }
@@ -1,79 +1,56 @@
1
+ import { getLakeRetentionRuntime } from "@sixb/core/internal/runtime"
2
+ import { parseDurationMs } from "../lib/duration"
1
3
  import { loadSixbFromEntry } from "../lib/loadSixb"
2
4
  import { resolveRuntimeEntry } from "../lib/production"
3
5
  import { stopSixbProviders } from "../lib/runtime"
4
6
  import { lakeCleanupView, renderStatic, withTask } from "../ui"
5
7
 
6
- const DEFAULT_RETENTION = "7 days"
7
-
8
8
  export interface LakeCleanupOptions {
9
9
  entry?: string
10
10
  dryRun?: boolean
11
- expireOlderThan?: string
12
- deleteOlderThan?: string
13
- }
14
-
15
- interface LakeMaintenanceOptions {
16
- readonly dryRun?: boolean
17
- readonly expireOlderThan?: string
18
- readonly deleteOlderThan?: string
19
- readonly protect?: readonly { readonly datasetId: string; readonly versionId: string }[]
20
- }
21
-
22
- export interface LakeMaintenanceReport {
23
- readonly dryRun: boolean
24
- readonly expireOlderThan: string
25
- readonly deleteOlderThan: string
26
- readonly snapshots: number
27
- /** Old snapshots kept because a version they hold is still needed, when the provider says. */
28
- readonly keptSnapshots?: number
29
- readonly oldFiles: number
30
- readonly orphanedFiles: number
31
- }
32
-
33
- interface LakeMaintenanceProvider {
34
- runMaintenance(options?: LakeMaintenanceOptions): Promise<LakeMaintenanceReport>
11
+ keepVersionsFor?: string
12
+ deleteFilesAfter?: string
35
13
  }
36
14
 
37
15
  export async function runLakeCleanup(options: LakeCleanupOptions = {}) {
16
+ const keepVersions = durationFlag(options.keepVersionsFor, "--keep-versions-for")
17
+ const deleteFiles = durationFlag(options.deleteFilesAfter, "--delete-files-after")
18
+
38
19
  const entry = await resolveRuntimeEntry({ entry: options.entry })
39
20
  const sixb = await withTask("lake cleanup", "Loading project", () => loadSixbFromEntry(entry))
40
21
 
41
22
  try {
42
- const lakeStorage = sixb.lakeStorage
43
- if (!isLakeMaintenanceProvider(lakeStorage)) {
44
- throw new Error("Configured lake storage does not support maintenance cleanup.")
23
+ if (sixb.lakeStorage.maintenance === undefined) {
24
+ throw new Error("[SixbCLI] Configured lake storage does not support retention cleanup.")
45
25
  }
46
26
 
47
- const expireOlderThan = options.expireOlderThan ?? DEFAULT_RETENTION
48
- const deleteOlderThan = options.deleteOlderThan ?? expireOlderThan
27
+ const retention = getLakeRetentionRuntime(sixb)
28
+ const { settings } = retention
29
+ const keepVersionsFor = options.keepVersionsFor ?? settings.keepVersionsFor
30
+ const deleteFilesAfter = options.deleteFilesAfter ?? settings.deleteFilesAfter
49
31
 
50
- // Objects hold the rows of the version each projection applied, and their markings are read
51
- // from it: the lake keeps those versions too. Without them, cleanup does not run.
52
- const applied = await sixb.storage.ontology.sources.listActive({ projectId: sixb.projectId })
32
+ // The same passes as automatic retention, without its time budget: they keep the versions
33
+ // projections applied, and run until none reports more. A dry run measures one pass.
53
34
  const report = await withTask(
54
35
  "lake cleanup",
55
36
  options.dryRun ? "Measuring" : "Cleaning up",
56
37
  () =>
57
- lakeStorage.runMaintenance({
38
+ retention.apply({
58
39
  dryRun: options.dryRun ?? false,
59
- expireOlderThan,
60
- deleteOlderThan,
61
- protect: applied.map(({ datasetVersion }) => ({
62
- datasetId: datasetVersion.datasetId,
63
- versionId: datasetVersion.versionId,
64
- })),
40
+ keepVersionsForMs: keepVersions ?? settings.keepVersionsForMs,
41
+ deleteFilesAfterMs: deleteFiles ?? settings.deleteFilesAfterMs,
65
42
  })
66
43
  )
67
- await renderStatic(lakeCleanupView({ projectId: sixb.id, report }))
44
+
45
+ await renderStatic(
46
+ lakeCleanupView({ projectId: sixb.id, keepVersionsFor, deleteFilesAfter, report })
47
+ )
68
48
  } finally {
69
49
  await stopSixbProviders(sixb)
70
50
  }
71
51
  }
72
52
 
73
- function isLakeMaintenanceProvider(value: unknown): value is LakeMaintenanceProvider {
74
- return (
75
- typeof value === "object" &&
76
- value !== null &&
77
- typeof (value as { runMaintenance?: unknown }).runMaintenance === "function"
78
- )
53
+ /** Read before anything loads, so that a typo fails at once. */
54
+ function durationFlag(value: string | undefined, flag: string): number | null {
55
+ return value === undefined ? null : parseDurationMs(value, flag)
79
56
  }
@@ -0,0 +1,36 @@
1
+ import { loadSixbFromEntry } from "../lib/loadSixb"
2
+ import { resolveRuntimeEntry } from "../lib/production"
3
+ import { stopSixbProviders } from "../lib/runtime"
4
+ import { withTask } from "../ui"
5
+
6
+ /** A week of refusals is rarely this many; past it, ask for one dataset. */
7
+ const LIMIT = 500
8
+
9
+ export interface LakeRejectedOptions {
10
+ entry?: string
11
+ dataset?: string
12
+ }
13
+
14
+ /**
15
+ * Prints, as JSON, the deliveries refused after `ingest` answered: what each held, whole, and why,
16
+ * so that someone can fix them and send them again. They stay for a week.
17
+ */
18
+ export async function runLakeRejected(options: LakeRejectedOptions = {}) {
19
+ const entry = await resolveRuntimeEntry({ entry: options.entry })
20
+ const sixb = await withTask("lake rejected", "Loading project", () => loadSixbFromEntry(entry))
21
+
22
+ try {
23
+ const inbox = sixb.storage.datasetIngest
24
+ if (!inbox) throw new Error("[SixbCLI] Configured storage keeps no ingested changes.")
25
+
26
+ const rejected = await inbox.listRejected({
27
+ projectId: sixb.id,
28
+ limit: LIMIT,
29
+ ...(options.dataset === undefined ? {} : { datasetId: options.dataset }),
30
+ })
31
+
32
+ process.stdout.write(`${JSON.stringify(rejected, null, 2)}\n`)
33
+ } finally {
34
+ await stopSixbProviders(sixb)
35
+ }
36
+ }
@@ -33,19 +33,14 @@ export async function runOrchestrator(options: OrchestratorOptions = {}) {
33
33
 
34
34
  runtime = await startOrchestratorRuntime(sixb)
35
35
 
36
- const warnings = [...runtime.warnings]
37
- if (runtime.orchestratorWorker === null) {
38
- warnings.push("No orchestrator routes are registered; the orchestrator process is idle.")
39
- }
40
-
41
36
  app.stop()
42
37
  await renderStatic(
43
38
  roleView({
44
39
  command: "orchestrator",
45
40
  name: sixb.id,
46
- items: [{ label: "role", value: "event-to-queue dispatcher" }],
41
+ items: [{ label: "role", value: "event dispatch, cron and upkeep" }],
47
42
  storage: migration.summary,
48
- warnings,
43
+ warnings: runtime.warnings,
49
44
  })
50
45
  )
51
46
 
@@ -1,63 +1,13 @@
1
- import type { LoadedSixbHost } from "../lib/loadSixb"
2
- import { loadProductionSixb } from "../lib/production"
3
- import { runUntilSignal, stopQuietly } from "../lib/role-lifecycle"
4
- import {
5
- type RunningSchedulerRuntime,
6
- startSchedulerRuntime,
7
- stopSixbProviders,
8
- } from "../lib/runtime"
9
- import { migrateStorageForRole } from "../lib/storage-migration"
10
- import { printStopping, renderCliError, renderStatic, roleView, startTask } from "../ui"
11
-
12
- export interface SchedulerOptions {
13
- entry?: string
14
- noMigrate?: boolean
15
- }
16
-
17
- export async function runScheduler(options: SchedulerOptions = {}) {
18
- process.env.NODE_ENV = "production"
19
-
20
- const loaded = await loadProductionSixb({ entry: options.entry, role: "scheduler" })
21
- const app = startTask("scheduler", "Starting scheduler")
22
-
23
- let sixb: LoadedSixbHost | null = loaded.sixb
24
- let runtime: RunningSchedulerRuntime | null = null
25
-
26
- try {
27
- const migration = await migrateStorageForRole(sixb, {
28
- role: "scheduler",
29
- noMigrate: options.noMigrate,
30
- onStart: () => app.step("Migrating storage"),
31
- })
32
-
33
- runtime = await startSchedulerRuntime(sixb)
34
-
35
- app.stop()
36
- await renderStatic(
37
- roleView({
38
- command: "scheduler",
39
- name: sixb.id,
40
- items: [{ label: "role", value: "schedule event producer" }],
41
- storage: migration.summary,
42
- })
43
- )
44
-
45
- await runUntilSignal(async () => {
46
- app.stop()
47
- printStopping("scheduler")
48
- await stopQuietly(() => runtime?.stop() ?? Promise.resolve())
49
- if (sixb) {
50
- await stopSixbProviders(sixb)
51
- }
52
- sixb = null
1
+ import { SixbCliError } from "../lib/errors"
2
+ import { renderCliError } from "../ui"
3
+
4
+ /** The command stays only to say where cron went: a process manager left on it must see a failure. */
5
+ export async function runScheduler(): Promise<void> {
6
+ await renderCliError(
7
+ new SixbCliError("[SixbCLI] `sixb scheduler` no longer runs: `sixb orchestrator` runs cron.", {
8
+ remediation:
9
+ "Remove this process from your deployment and keep one `sixb orchestrator` running.",
53
10
  })
54
- } catch (error) {
55
- app.stop()
56
- await stopQuietly(() => runtime?.stop() ?? Promise.resolve())
57
- if (sixb) {
58
- await stopSixbProviders(sixb)
59
- }
60
- await renderCliError(error)
61
- process.exit(1)
62
- }
11
+ )
12
+ process.exit(1)
63
13
  }
@@ -13,6 +13,7 @@ import {
13
13
  agentRuntimeRequired,
14
14
  assertWorkerInputs,
15
15
  createWorkerForType,
16
+ omittedWorkerWarnings,
16
17
  type QueueWorkerProcess,
17
18
  resolveRegisteredWorkerTypes,
18
19
  resolveWorkerTypeToStart,
@@ -102,10 +103,12 @@ export async function runWorkerGroup(options: WorkerGroupOptions = {}) {
102
103
  )
103
104
  await Promise.all(workers.map((worker) => worker.start()))
104
105
 
105
- const warnings =
106
- workerTypes.length === 0
106
+ const warnings = [
107
+ ...(workerTypes.length === 0
107
108
  ? ["No queue worker types are registered; the worker group process is idle."]
108
- : []
109
+ : []),
110
+ ...omittedWorkerWarnings(host, workerTypes),
111
+ ]
109
112
 
110
113
  app.stop()
111
114
  await renderStatic(
package/src/index.tsx CHANGED
@@ -315,7 +315,7 @@ async function main(): Promise<void> {
315
315
 
316
316
  case "scheduler": {
317
317
  const { runScheduler } = await import("./commands/scheduler")
318
- await runScheduler({ entry: getFlag("entry"), noMigrate: hasFlag("no-migrate") })
318
+ await runScheduler()
319
319
  break
320
320
  }
321
321
 
@@ -376,13 +376,19 @@ async function main(): Promise<void> {
376
376
  break
377
377
  }
378
378
 
379
+ case "lake:rejected": {
380
+ const { runLakeRejected } = await import("./commands/lake-rejected")
381
+ await runLakeRejected({ entry: getFlag("entry"), dataset: getFlag("dataset") })
382
+ break
383
+ }
384
+
379
385
  case "lake:cleanup": {
380
386
  const { runLakeCleanup } = await import("./commands/lake-cleanup")
381
387
  await runLakeCleanup({
382
388
  entry: getFlag("entry"),
383
389
  dryRun: hasFlag("dry-run"),
384
- expireOlderThan: getFlag("expire-older-than"),
385
- deleteOlderThan: getFlag("delete-older-than"),
390
+ keepVersionsFor: getFlag("keep-versions-for"),
391
+ deleteFilesAfter: getFlag("delete-files-after"),
386
392
  })
387
393
  break
388
394
  }
@@ -392,7 +398,7 @@ async function main(): Promise<void> {
392
398
  const dev = getFlag("dev")
393
399
  const prod = getFlag("prod")
394
400
  await runEnvInit({
395
- project: positionals[0] ?? "",
401
+ ...(positionals[0] === undefined ? {} : { project: positionals[0] }),
396
402
  ...(dev === undefined ? {} : { dev }),
397
403
  ...(prod === undefined ? {} : { prod }),
398
404
  })
@@ -186,14 +186,20 @@ export const CLI_OPTION_DEFINITIONS = {
186
186
  summary: "Show what would change without changing it",
187
187
  kind: "boolean",
188
188
  },
189
- "expire-older-than": {
190
- syntax: "--expire-older-than <interval>",
191
- summary: "Lake snapshot expiration window",
189
+ dataset: {
190
+ syntax: "--dataset <id>",
191
+ summary: "Show one dataset's rejected deliveries (default: every dataset)",
192
192
  kind: "string",
193
193
  },
194
- "delete-older-than": {
195
- syntax: "--delete-older-than <interval>",
196
- summary: "Lake file deletion window",
194
+ "keep-versions-for": {
195
+ syntax: "--keep-versions-for <duration>",
196
+ summary:
197
+ "How long a replaced lake version stays readable (default: `lakeMaintenance`, at least 1h)",
198
+ kind: "string",
199
+ },
200
+ "delete-files-after": {
201
+ syntax: "--delete-files-after <duration>",
202
+ summary: "How long unread lake files wait before deletion (default: `lakeMaintenance`)",
197
203
  kind: "string",
198
204
  },
199
205
  } as const satisfies Record<string, { syntax: string; summary: string; kind: OptionKind }>
@@ -233,6 +239,7 @@ export type LocalCommandId =
233
239
  | "db:prepare"
234
240
  | "lake:check"
235
241
  | "lake:cleanup"
242
+ | "lake:rejected"
236
243
  | "deploy"
237
244
  | "deploy:status"
238
245
  | "deploy:logs"
@@ -300,6 +307,7 @@ const INSTANCE_SUMMARIES = {
300
307
  workflows: "Discover and start workflows",
301
308
  "workflow-runs": "Inspect workflow execution history",
302
309
  users: "List and search project members",
310
+ artifacts: "Read, resolve and reply to an artifact's comments",
303
311
  models: "List the project's models and their prices",
304
312
  datasets: "Read datasets, their versions and rows",
305
313
  syncs: "List the syncs you may run",
@@ -489,10 +497,10 @@ const commandTree: readonly CommandNode[] = [
489
497
  },
490
498
  ],
491
499
  },
492
- command("scheduler", "Start the production scheduler", {
500
+ command("scheduler", "Removed: the orchestrator runs cron schedules", {
493
501
  options: ["entry", "no-migrate"],
494
502
  }),
495
- command("orchestrator", "Start the production orchestrator, which queues work for events", {
503
+ command("orchestrator", "Start the production orchestrator: cron, event dispatch and upkeep", {
496
504
  options: ["entry", "no-migrate"],
497
505
  }),
498
506
  command("rules", "Start the production rules runtime", { options: ["entry", "no-migrate"] }),
@@ -576,9 +584,14 @@ const commandTree: readonly CommandNode[] = [
576
584
  }),
577
585
  command("lake:cleanup", "Clean up old lake snapshots and files", {
578
586
  path: ["lake", "cleanup"],
579
- options: ["entry", "dry-run", "expire-older-than", "delete-older-than"],
587
+ options: ["entry", "dry-run", "keep-versions-for", "delete-files-after"],
580
588
  rootHelpLabel: "lake cleanup",
581
589
  }),
590
+ command("lake:rejected", "Show ingested deliveries refused after they were accepted", {
591
+ path: ["lake", "rejected"],
592
+ options: ["entry", "dataset"],
593
+ rootHelpLabel: "lake rejected",
594
+ }),
582
595
  ],
583
596
  },
584
597
  {
@@ -650,9 +663,8 @@ const commandTree: readonly CommandNode[] = [
650
663
  children: [
651
664
  command("env:init", "Move a project's values to its source and write sixb.env.ts", {
652
665
  path: ["env", "init"],
653
- usage: "sixb env init <project> [--dev <file>] [--prod <file>]",
666
+ usage: "sixb env init [doppler-project] [--dev <file>] [--prod <file>]",
654
667
  options: ["dev", "prod"],
655
- minimumPositionals: 1,
656
668
  maximumPositionals: 1,
657
669
  }),
658
670
  command("env:pull", "Write .env from the project's source", {
@@ -743,8 +755,9 @@ const ROOT_HELP_GROUP_COMMANDS: readonly {
743
755
  { title: "Deploy", names: ["deploy", "desktop"] },
744
756
  {
745
757
  title: "Run in production",
746
- names: ["api", "app", "scheduler", "orchestrator", "rules", "worker", "worker-group"],
758
+ names: ["api", "app", "orchestrator", "rules", "worker", "worker-group"],
747
759
  },
760
+ { title: "Removed", names: ["scheduler"] },
748
761
  { title: "Storage", names: ["db", "lake"] },
749
762
  {
750
763
  title: "Connect to an instance",
@@ -1,4 +1,5 @@
1
1
  import type {
2
+ DeployCheck,
2
3
  DeployCommand,
3
4
  DeployConfig,
4
5
  DeployEnv,
@@ -28,7 +29,6 @@ const ROLE_SERVICES = {
28
29
  api: "api",
29
30
  app: "app",
30
31
  orchestrator: "orchestrator",
31
- scheduler: "scheduler",
32
32
  rules: "rules",
33
33
  "worker-group": "workers",
34
34
  // A deployment co-hosts its workers in one `worker-group`; `workers.process.instances` scales it.
@@ -44,7 +44,6 @@ const SERVICE_ROLES: { readonly [Service in DeployServiceName]: RoleOf<Service>
44
44
  api: "api",
45
45
  app: "app",
46
46
  orchestrator: "orchestrator",
47
- scheduler: "scheduler",
48
47
  rules: "rules",
49
48
  workers: "worker-group",
50
49
  }
@@ -168,6 +167,47 @@ function sixbService(
168
167
  }
169
168
  }
170
169
 
170
+ /**
171
+ * What this config leaves the lake's background work without: its retention, and the changes
172
+ * `ingest` stores. Only the project knows whether it has datasets, and this reads the config alone,
173
+ * so it warns and never stops a deploy.
174
+ */
175
+ export function lakeWorkChecks(config: DeployConfig): readonly DeployCheck[] {
176
+ const { orchestrator, workers } = config.services ?? {}
177
+ const types = typeof workers === "object" ? workers.types : undefined
178
+ const lakeWorkerOmitted = workers === false || (Array.isArray(types) && !types.includes("lake"))
179
+
180
+ const missing = [
181
+ ...(orchestrator === false ? ["the orchestrator requests it"] : []),
182
+ ...(lakeWorkerOmitted ? ["a `lake` worker runs it"] : []),
183
+ ]
184
+
185
+ return [
186
+ ...(missing.length > 0
187
+ ? [
188
+ {
189
+ id: "lake-retention",
190
+ label: "Lake retention",
191
+ status: "warning" as const,
192
+ detail: `Not deployed: ${missing.join(", and ")}.`,
193
+ remedy: "Deploy them, or a project with datasets keeps every replaced version.",
194
+ },
195
+ ]
196
+ : []),
197
+ ...(lakeWorkerOmitted
198
+ ? [
199
+ {
200
+ id: "dataset-ingestion",
201
+ label: "Dataset ingestion",
202
+ status: "warning" as const,
203
+ detail: "Not deployed: a `lake` worker writes what `ingest` stores.",
204
+ remedy: "Deploy one, or ingested changes wait and never reach their dataset.",
205
+ },
206
+ ]
207
+ : []),
208
+ ]
209
+ }
210
+
171
211
  function workerGroupArgs(options: DeployWorkersConfig): string[] {
172
212
  const types = options.types === undefined || options.types === "all" ? [] : options.types
173
213
  const args = ["worker-group", ...types]