@sixb/cli 0.2.2 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -22
- package/dist/commands/access-token-utils.d.ts.map +1 -1
- package/dist/commands/agent.d.ts +5 -0
- package/dist/commands/agent.d.ts.map +1 -1
- package/dist/commands/api.d.ts.map +1 -1
- package/dist/commands/check.d.ts.map +1 -1
- package/dist/commands/deploy.d.ts.map +1 -1
- package/dist/commands/dev-runner.d.ts.map +1 -1
- package/dist/commands/env.d.ts +6 -1
- package/dist/commands/env.d.ts.map +1 -1
- package/dist/commands/lake-cleanup.d.ts +2 -12
- package/dist/commands/lake-cleanup.d.ts.map +1 -1
- package/dist/commands/lake-rejected.d.ts +10 -0
- package/dist/commands/lake-rejected.d.ts.map +1 -0
- package/dist/commands/orchestrator.d.ts.map +1 -1
- package/dist/commands/scheduler.d.ts +2 -5
- package/dist/commands/scheduler.d.ts.map +1 -1
- package/dist/commands/worker-group.d.ts.map +1 -1
- package/dist/lib/command-line.d.ts +12 -7
- package/dist/lib/command-line.d.ts.map +1 -1
- package/dist/lib/deploy-release.d.ts +7 -1
- package/dist/lib/deploy-release.d.ts.map +1 -1
- package/dist/lib/duration.d.ts +3 -0
- package/dist/lib/duration.d.ts.map +1 -0
- package/dist/lib/loadSixb.d.ts.map +1 -1
- package/dist/lib/production-roles.d.ts +0 -4
- package/dist/lib/production-roles.d.ts.map +1 -1
- package/dist/lib/runtime.d.ts +7 -9
- package/dist/lib/runtime.d.ts.map +1 -1
- package/dist/lib/worker-concurrency.d.ts.map +1 -1
- package/dist/lib/worker-registry.d.ts +4 -1
- package/dist/lib/worker-registry.d.ts.map +1 -1
- package/dist/runner/local-runner.d.ts +1 -1
- package/dist/tui/app.d.ts.map +1 -1
- package/dist/tui/format.d.ts +20 -0
- package/dist/tui/format.d.ts.map +1 -1
- package/dist/tui/transcript.d.ts.map +1 -1
- package/dist/ui/deploy-screens.d.ts +4 -2
- package/dist/ui/deploy-screens.d.ts.map +1 -1
- package/dist/ui/env-screens.d.ts +14 -1
- package/dist/ui/env-screens.d.ts.map +1 -1
- package/dist/ui/screens.d.ts +5 -11
- package/dist/ui/screens.d.ts.map +1 -1
- package/package.json +20 -16
- package/src/commands/access-token-utils.ts +2 -20
- package/src/commands/agent.ts +54 -8
- package/src/commands/api.ts +7 -0
- package/src/commands/check.ts +75 -1
- package/src/commands/deploy.ts +15 -3
- package/src/commands/dev-runner.ts +5 -0
- package/src/commands/env.ts +99 -20
- package/src/commands/lake-cleanup.ts +25 -48
- package/src/commands/lake-rejected.ts +36 -0
- package/src/commands/orchestrator.ts +2 -7
- package/src/commands/scheduler.ts +11 -61
- package/src/commands/worker-group.ts +6 -3
- package/src/index.tsx +10 -4
- package/src/lib/command-line.ts +25 -12
- package/src/lib/deploy-release.ts +42 -2
- package/src/lib/duration.ts +11 -0
- package/src/lib/loadSixb.ts +3 -4
- package/src/lib/production-roles.ts +0 -1
- package/src/lib/runtime.ts +62 -43
- package/src/lib/shareable-providers.ts +2 -2
- package/src/lib/worker-concurrency.ts +1 -0
- package/src/lib/worker-registry.ts +36 -1
- package/src/runner/local-runner.ts +1 -1
- package/src/tui/app.tsx +2 -1
- package/src/tui/format.ts +190 -3
- package/src/tui/transcript.tsx +8 -1
- package/src/ui/deploy-screens.ts +8 -1
- package/src/ui/env-screens.ts +39 -5
- package/src/ui/screens.ts +21 -20
package/src/commands/env.ts
CHANGED
|
@@ -23,6 +23,7 @@ import { errorMessage, SixbCliError } from "../lib/errors"
|
|
|
23
23
|
import { promptHidden } from "../lib/hidden-prompt"
|
|
24
24
|
import { renderStatic } from "../ui"
|
|
25
25
|
import {
|
|
26
|
+
type EnvInitConfig,
|
|
26
27
|
type EnvStatusCell,
|
|
27
28
|
envInitView,
|
|
28
29
|
envKeptView,
|
|
@@ -34,6 +35,7 @@ import { canRunLive } from "../ui/live/can-run-live"
|
|
|
34
35
|
|
|
35
36
|
const ENV_NAMES: readonly EnvName[] = ["dev", "prod"]
|
|
36
37
|
const DOPPLER_PACKAGE = "@sixb/env-doppler"
|
|
38
|
+
const SOURCE_EXAMPLE = `Add one, such as \`source: doppler({ project: "shop" })\` from ${DOPPLER_PACKAGE}, or \`source: azureKeyVault({ vaults: { dev: "shop-dev", prod: "shop-prod" } })\` from @sixb/env-azure-key-vault.`
|
|
37
39
|
|
|
38
40
|
// ─── pull ────────────────────────────────────────────────────────────────────
|
|
39
41
|
|
|
@@ -264,16 +266,25 @@ export async function runEnvStatus(options: { readonly cwd?: string } = {}): Pro
|
|
|
264
266
|
|
|
265
267
|
// ─── init ────────────────────────────────────────────────────────────────────
|
|
266
268
|
|
|
269
|
+
/**
|
|
270
|
+
* Moves a project's values to its source. Given a Doppler project, it also writes `sixb.env.ts`.
|
|
271
|
+
* Without one, it uploads to the source `sixb.env.ts` already names, whatever it is, and fills in
|
|
272
|
+
* its empty `vars: {}`.
|
|
273
|
+
*/
|
|
267
274
|
export async function runEnvInit(options: {
|
|
268
|
-
readonly project
|
|
275
|
+
readonly project?: string
|
|
269
276
|
readonly dev?: string
|
|
270
277
|
readonly prod?: string
|
|
271
278
|
readonly cwd?: string
|
|
272
279
|
}): Promise<void> {
|
|
273
280
|
const cwd = options.cwd ?? process.cwd()
|
|
274
281
|
const configPath = join(cwd, ENV_CONFIG_FILE)
|
|
275
|
-
|
|
276
|
-
|
|
282
|
+
const { project } = options
|
|
283
|
+
const configExists = await Bun.file(configPath).exists()
|
|
284
|
+
if (project !== undefined && configExists) {
|
|
285
|
+
throw new SixbCliError(`[SixbEnv] ${ENV_CONFIG_FILE} already exists.`, {
|
|
286
|
+
remediation: `To upload to the source it names, leave out the project: ${INIT_USAGE}`,
|
|
287
|
+
})
|
|
277
288
|
}
|
|
278
289
|
|
|
279
290
|
const files = [
|
|
@@ -283,11 +294,13 @@ export async function runEnvInit(options: {
|
|
|
283
294
|
const given = files.flatMap(([env, file]) => (file === undefined ? [] : [{ env, file }]))
|
|
284
295
|
if (given.length === 0) {
|
|
285
296
|
throw new SixbCliError("[SixbEnv] Say where the current values are.", {
|
|
286
|
-
remediation:
|
|
297
|
+
remediation: project === undefined ? INIT_USAGE : `sixb env init ${project} ${INIT_FILES}`,
|
|
287
298
|
})
|
|
288
299
|
}
|
|
289
300
|
|
|
290
|
-
const
|
|
301
|
+
const existing = configExists ? await loadEnvConfig(cwd) : null
|
|
302
|
+
const source =
|
|
303
|
+
project === undefined ? sourceToFill(existing) : await loadDopplerSource(cwd, project)
|
|
291
304
|
const uploads = await Promise.all(
|
|
292
305
|
given.map(async ({ env, file }) => {
|
|
293
306
|
const path = resolve(cwd, file)
|
|
@@ -298,33 +311,102 @@ export async function runEnvInit(options: {
|
|
|
298
311
|
})
|
|
299
312
|
)
|
|
300
313
|
|
|
301
|
-
// Check every environment before writing to any, so a conflict changes nothing.
|
|
302
|
-
//
|
|
314
|
+
// Check every environment before writing to any, so a conflict changes nothing. A value the
|
|
315
|
+
// source already has isn't sent again, so running this twice, or after an upload that didn't
|
|
316
|
+
// finish, changes nothing it doesn't need to.
|
|
317
|
+
const sends: { readonly env: EnvName; readonly values: Record<string, string> }[] = []
|
|
303
318
|
for (const { env, values } of uploads) {
|
|
304
|
-
const
|
|
319
|
+
const current = await source.read(env)
|
|
305
320
|
const conflicts = Object.keys(values).filter(
|
|
306
|
-
(name) => name in
|
|
321
|
+
(name) => name in current && current[name] !== values[name]
|
|
307
322
|
)
|
|
308
323
|
if (conflicts.length > 0) {
|
|
309
324
|
throw new SixbCliError(
|
|
310
|
-
`[SixbEnv] ${
|
|
325
|
+
`[SixbEnv] ${project ?? "The source"} already has other values in ${env} for ${conflicts.sort().join(", ")}.`,
|
|
311
326
|
{
|
|
312
327
|
remediation:
|
|
313
328
|
"Change them in your source's dashboard, or edit the file, and run this again.",
|
|
314
329
|
}
|
|
315
330
|
)
|
|
316
331
|
}
|
|
332
|
+
sends.push({
|
|
333
|
+
env,
|
|
334
|
+
values: Object.fromEntries(Object.entries(values).filter(([name]) => !(name in current))),
|
|
335
|
+
})
|
|
317
336
|
}
|
|
318
|
-
for (const { env, values } of
|
|
337
|
+
for (const { env, values } of sends) await source.write(env, values)
|
|
319
338
|
|
|
320
339
|
const vars = declaredVars(uploads)
|
|
321
|
-
|
|
340
|
+
let config: EnvInitConfig
|
|
341
|
+
if (project !== undefined) {
|
|
342
|
+
await writeFile(configPath, renderEnvConfig(project, vars))
|
|
343
|
+
config = { kind: "written", names: Object.keys(vars).length }
|
|
344
|
+
} else {
|
|
345
|
+
config = await fillVars(configPath, existing?.vars ?? {}, vars)
|
|
346
|
+
}
|
|
322
347
|
await renderStatic(
|
|
323
348
|
envInitView({
|
|
324
|
-
uploaded: uploads.map(({ env, values }) =>
|
|
325
|
-
|
|
349
|
+
uploaded: uploads.map(({ env, values }, index) => {
|
|
350
|
+
const sent = Object.keys(sends[index]?.values ?? {}).length
|
|
351
|
+
return { env, count: sent, already: Object.keys(values).length - sent }
|
|
352
|
+
}),
|
|
353
|
+
config,
|
|
354
|
+
})
|
|
355
|
+
)
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
const INIT_FILES = "--dev .env --prod .env.prod"
|
|
359
|
+
const INIT_USAGE = `sixb env init ${INIT_FILES}`
|
|
360
|
+
|
|
361
|
+
function sourceToFill(config: EnvConfig | null): EnvSource {
|
|
362
|
+
if (!config) {
|
|
363
|
+
throw new SixbCliError(`[SixbEnv] There's no ${ENV_CONFIG_FILE} naming a source yet.`, {
|
|
364
|
+
remediation: `Write one with its source and \`vars: {}\`, then run this again. For Doppler, \`sixb env init <project> ${INIT_FILES}\` writes it for you.`,
|
|
365
|
+
})
|
|
366
|
+
}
|
|
367
|
+
if (!config.source) {
|
|
368
|
+
throw new SixbCliError(`[SixbEnv] ${ENV_CONFIG_FILE} has no \`source\` to upload to.`, {
|
|
369
|
+
remediation: SOURCE_EXAMPLE,
|
|
326
370
|
})
|
|
371
|
+
}
|
|
372
|
+
return config.source
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
/**
|
|
376
|
+
* Declares the uploaded names in an existing `sixb.env.ts`. Only an empty `vars: {}` is replaced:
|
|
377
|
+
* anything else is the author's code, so the rules are printed for them to add instead.
|
|
378
|
+
*/
|
|
379
|
+
async function fillVars(
|
|
380
|
+
path: string,
|
|
381
|
+
declared: Readonly<Record<string, EnvVarRule>>,
|
|
382
|
+
guessed: Readonly<Record<string, EnvVarRule>>
|
|
383
|
+
): Promise<EnvInitConfig> {
|
|
384
|
+
const undeclared = Object.fromEntries(
|
|
385
|
+
Object.entries(guessed).filter(([name]) => !(name in declared))
|
|
386
|
+
)
|
|
387
|
+
const names = Object.keys(undeclared).length
|
|
388
|
+
if (names === 0) return { kind: "unchanged" }
|
|
389
|
+
|
|
390
|
+
const text = await readFile(path, "utf8")
|
|
391
|
+
const empty = [...text.matchAll(EMPTY_VARS)]
|
|
392
|
+
if (Object.keys(declared).length > 0 || empty.length !== 1) {
|
|
393
|
+
return { kind: "paste", names, lines: varLines(undeclared, "") }
|
|
394
|
+
}
|
|
395
|
+
await writeFile(
|
|
396
|
+
path,
|
|
397
|
+
text.replace(EMPTY_VARS, (_match, indent: string) => renderVars(undeclared, indent))
|
|
327
398
|
)
|
|
399
|
+
return { kind: "written", names }
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
const EMPTY_VARS = /^([ \t]*)vars:\s*\{\s*\}/gm
|
|
403
|
+
|
|
404
|
+
function renderVars(vars: Readonly<Record<string, EnvVarRule>>, indent: string): string {
|
|
405
|
+
return [`${indent}vars: {`, ...varLines(vars, `${indent} `), `${indent}}`].join("\n")
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
function varLines(vars: Readonly<Record<string, EnvVarRule>>, indent: string): string[] {
|
|
409
|
+
return Object.entries(vars).map(([name, rule]) => `${indent}${name}: "${rule}",`)
|
|
328
410
|
}
|
|
329
411
|
|
|
330
412
|
/**
|
|
@@ -366,16 +448,13 @@ function uploadableValues(
|
|
|
366
448
|
}
|
|
367
449
|
|
|
368
450
|
function renderEnvConfig(project: string, vars: Readonly<Record<string, EnvVarRule>>): string {
|
|
369
|
-
const lines = Object.entries(vars).map(([name, rule]) => ` ${name}: "${rule}",`)
|
|
370
451
|
return [
|
|
371
452
|
'import { defineEnv } from "@sixb/core/env"',
|
|
372
453
|
`import { doppler } from "${DOPPLER_PACKAGE}"`,
|
|
373
454
|
"",
|
|
374
455
|
"export default defineEnv({",
|
|
375
456
|
` source: doppler({ project: ${JSON.stringify(project)} }),`,
|
|
376
|
-
|
|
377
|
-
...lines,
|
|
378
|
-
" },",
|
|
457
|
+
`${renderVars(vars, " ")},`,
|
|
379
458
|
"})",
|
|
380
459
|
"",
|
|
381
460
|
].join("\n")
|
|
@@ -410,12 +489,12 @@ async function requireSource(
|
|
|
410
489
|
const config = await loadEnvConfig(cwd)
|
|
411
490
|
if (!config) {
|
|
412
491
|
throw new SixbCliError(`[SixbEnv] No ${ENV_CONFIG_FILE} in ${cwd}.`, {
|
|
413
|
-
remediation:
|
|
492
|
+
remediation: `sixb env init <project> ${INIT_FILES}`,
|
|
414
493
|
})
|
|
415
494
|
}
|
|
416
495
|
if (!config.source) {
|
|
417
496
|
throw new SixbCliError(`[SixbEnv] ${ENV_CONFIG_FILE} has no \`source\`.`, {
|
|
418
|
-
remediation:
|
|
497
|
+
remediation: SOURCE_EXAMPLE,
|
|
419
498
|
})
|
|
420
499
|
}
|
|
421
500
|
return { config, source: config.source }
|
|
@@ -1,79 +1,56 @@
|
|
|
1
|
+
import { getLakeRetentionRuntime } from "@sixb/core/internal/runtime"
|
|
2
|
+
import { parseDurationMs } from "../lib/duration"
|
|
1
3
|
import { loadSixbFromEntry } from "../lib/loadSixb"
|
|
2
4
|
import { resolveRuntimeEntry } from "../lib/production"
|
|
3
5
|
import { stopSixbProviders } from "../lib/runtime"
|
|
4
6
|
import { lakeCleanupView, renderStatic, withTask } from "../ui"
|
|
5
7
|
|
|
6
|
-
const DEFAULT_RETENTION = "7 days"
|
|
7
|
-
|
|
8
8
|
export interface LakeCleanupOptions {
|
|
9
9
|
entry?: string
|
|
10
10
|
dryRun?: boolean
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
interface LakeMaintenanceOptions {
|
|
16
|
-
readonly dryRun?: boolean
|
|
17
|
-
readonly expireOlderThan?: string
|
|
18
|
-
readonly deleteOlderThan?: string
|
|
19
|
-
readonly protect?: readonly { readonly datasetId: string; readonly versionId: string }[]
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
export interface LakeMaintenanceReport {
|
|
23
|
-
readonly dryRun: boolean
|
|
24
|
-
readonly expireOlderThan: string
|
|
25
|
-
readonly deleteOlderThan: string
|
|
26
|
-
readonly snapshots: number
|
|
27
|
-
/** Old snapshots kept because a version they hold is still needed, when the provider says. */
|
|
28
|
-
readonly keptSnapshots?: number
|
|
29
|
-
readonly oldFiles: number
|
|
30
|
-
readonly orphanedFiles: number
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
interface LakeMaintenanceProvider {
|
|
34
|
-
runMaintenance(options?: LakeMaintenanceOptions): Promise<LakeMaintenanceReport>
|
|
11
|
+
keepVersionsFor?: string
|
|
12
|
+
deleteFilesAfter?: string
|
|
35
13
|
}
|
|
36
14
|
|
|
37
15
|
export async function runLakeCleanup(options: LakeCleanupOptions = {}) {
|
|
16
|
+
const keepVersions = durationFlag(options.keepVersionsFor, "--keep-versions-for")
|
|
17
|
+
const deleteFiles = durationFlag(options.deleteFilesAfter, "--delete-files-after")
|
|
18
|
+
|
|
38
19
|
const entry = await resolveRuntimeEntry({ entry: options.entry })
|
|
39
20
|
const sixb = await withTask("lake cleanup", "Loading project", () => loadSixbFromEntry(entry))
|
|
40
21
|
|
|
41
22
|
try {
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
throw new Error("Configured lake storage does not support maintenance cleanup.")
|
|
23
|
+
if (sixb.lakeStorage.maintenance === undefined) {
|
|
24
|
+
throw new Error("[SixbCLI] Configured lake storage does not support retention cleanup.")
|
|
45
25
|
}
|
|
46
26
|
|
|
47
|
-
const
|
|
48
|
-
const
|
|
27
|
+
const retention = getLakeRetentionRuntime(sixb)
|
|
28
|
+
const { settings } = retention
|
|
29
|
+
const keepVersionsFor = options.keepVersionsFor ?? settings.keepVersionsFor
|
|
30
|
+
const deleteFilesAfter = options.deleteFilesAfter ?? settings.deleteFilesAfter
|
|
49
31
|
|
|
50
|
-
//
|
|
51
|
-
//
|
|
52
|
-
const applied = await sixb.storage.ontology.sources.listActive({ projectId: sixb.projectId })
|
|
32
|
+
// The same passes as automatic retention, without its time budget: they keep the versions
|
|
33
|
+
// projections applied, and run until none reports more. A dry run measures one pass.
|
|
53
34
|
const report = await withTask(
|
|
54
35
|
"lake cleanup",
|
|
55
36
|
options.dryRun ? "Measuring" : "Cleaning up",
|
|
56
37
|
() =>
|
|
57
|
-
|
|
38
|
+
retention.apply({
|
|
58
39
|
dryRun: options.dryRun ?? false,
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
protect: applied.map(({ datasetVersion }) => ({
|
|
62
|
-
datasetId: datasetVersion.datasetId,
|
|
63
|
-
versionId: datasetVersion.versionId,
|
|
64
|
-
})),
|
|
40
|
+
keepVersionsForMs: keepVersions ?? settings.keepVersionsForMs,
|
|
41
|
+
deleteFilesAfterMs: deleteFiles ?? settings.deleteFilesAfterMs,
|
|
65
42
|
})
|
|
66
43
|
)
|
|
67
|
-
|
|
44
|
+
|
|
45
|
+
await renderStatic(
|
|
46
|
+
lakeCleanupView({ projectId: sixb.id, keepVersionsFor, deleteFilesAfter, report })
|
|
47
|
+
)
|
|
68
48
|
} finally {
|
|
69
49
|
await stopSixbProviders(sixb)
|
|
70
50
|
}
|
|
71
51
|
}
|
|
72
52
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
value !== null &&
|
|
77
|
-
typeof (value as { runMaintenance?: unknown }).runMaintenance === "function"
|
|
78
|
-
)
|
|
53
|
+
/** Read before anything loads, so that a typo fails at once. */
|
|
54
|
+
function durationFlag(value: string | undefined, flag: string): number | null {
|
|
55
|
+
return value === undefined ? null : parseDurationMs(value, flag)
|
|
79
56
|
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { loadSixbFromEntry } from "../lib/loadSixb"
|
|
2
|
+
import { resolveRuntimeEntry } from "../lib/production"
|
|
3
|
+
import { stopSixbProviders } from "../lib/runtime"
|
|
4
|
+
import { withTask } from "../ui"
|
|
5
|
+
|
|
6
|
+
/** A week of refusals is rarely this many; past it, ask for one dataset. */
|
|
7
|
+
const LIMIT = 500
|
|
8
|
+
|
|
9
|
+
export interface LakeRejectedOptions {
|
|
10
|
+
entry?: string
|
|
11
|
+
dataset?: string
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Prints, as JSON, the deliveries refused after `ingest` answered: what each held, whole, and why,
|
|
16
|
+
* so that someone can fix them and send them again. They stay for a week.
|
|
17
|
+
*/
|
|
18
|
+
export async function runLakeRejected(options: LakeRejectedOptions = {}) {
|
|
19
|
+
const entry = await resolveRuntimeEntry({ entry: options.entry })
|
|
20
|
+
const sixb = await withTask("lake rejected", "Loading project", () => loadSixbFromEntry(entry))
|
|
21
|
+
|
|
22
|
+
try {
|
|
23
|
+
const inbox = sixb.storage.datasetIngest
|
|
24
|
+
if (!inbox) throw new Error("[SixbCLI] Configured storage keeps no ingested changes.")
|
|
25
|
+
|
|
26
|
+
const rejected = await inbox.listRejected({
|
|
27
|
+
projectId: sixb.id,
|
|
28
|
+
limit: LIMIT,
|
|
29
|
+
...(options.dataset === undefined ? {} : { datasetId: options.dataset }),
|
|
30
|
+
})
|
|
31
|
+
|
|
32
|
+
process.stdout.write(`${JSON.stringify(rejected, null, 2)}\n`)
|
|
33
|
+
} finally {
|
|
34
|
+
await stopSixbProviders(sixb)
|
|
35
|
+
}
|
|
36
|
+
}
|
|
@@ -33,19 +33,14 @@ export async function runOrchestrator(options: OrchestratorOptions = {}) {
|
|
|
33
33
|
|
|
34
34
|
runtime = await startOrchestratorRuntime(sixb)
|
|
35
35
|
|
|
36
|
-
const warnings = [...runtime.warnings]
|
|
37
|
-
if (runtime.orchestratorWorker === null) {
|
|
38
|
-
warnings.push("No orchestrator routes are registered; the orchestrator process is idle.")
|
|
39
|
-
}
|
|
40
|
-
|
|
41
36
|
app.stop()
|
|
42
37
|
await renderStatic(
|
|
43
38
|
roleView({
|
|
44
39
|
command: "orchestrator",
|
|
45
40
|
name: sixb.id,
|
|
46
|
-
items: [{ label: "role", value: "event
|
|
41
|
+
items: [{ label: "role", value: "event dispatch, cron and upkeep" }],
|
|
47
42
|
storage: migration.summary,
|
|
48
|
-
warnings,
|
|
43
|
+
warnings: runtime.warnings,
|
|
49
44
|
})
|
|
50
45
|
)
|
|
51
46
|
|
|
@@ -1,63 +1,13 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
import { printStopping, renderCliError, renderStatic, roleView, startTask } from "../ui"
|
|
11
|
-
|
|
12
|
-
export interface SchedulerOptions {
|
|
13
|
-
entry?: string
|
|
14
|
-
noMigrate?: boolean
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
export async function runScheduler(options: SchedulerOptions = {}) {
|
|
18
|
-
process.env.NODE_ENV = "production"
|
|
19
|
-
|
|
20
|
-
const loaded = await loadProductionSixb({ entry: options.entry, role: "scheduler" })
|
|
21
|
-
const app = startTask("scheduler", "Starting scheduler")
|
|
22
|
-
|
|
23
|
-
let sixb: LoadedSixbHost | null = loaded.sixb
|
|
24
|
-
let runtime: RunningSchedulerRuntime | null = null
|
|
25
|
-
|
|
26
|
-
try {
|
|
27
|
-
const migration = await migrateStorageForRole(sixb, {
|
|
28
|
-
role: "scheduler",
|
|
29
|
-
noMigrate: options.noMigrate,
|
|
30
|
-
onStart: () => app.step("Migrating storage"),
|
|
31
|
-
})
|
|
32
|
-
|
|
33
|
-
runtime = await startSchedulerRuntime(sixb)
|
|
34
|
-
|
|
35
|
-
app.stop()
|
|
36
|
-
await renderStatic(
|
|
37
|
-
roleView({
|
|
38
|
-
command: "scheduler",
|
|
39
|
-
name: sixb.id,
|
|
40
|
-
items: [{ label: "role", value: "schedule event producer" }],
|
|
41
|
-
storage: migration.summary,
|
|
42
|
-
})
|
|
43
|
-
)
|
|
44
|
-
|
|
45
|
-
await runUntilSignal(async () => {
|
|
46
|
-
app.stop()
|
|
47
|
-
printStopping("scheduler")
|
|
48
|
-
await stopQuietly(() => runtime?.stop() ?? Promise.resolve())
|
|
49
|
-
if (sixb) {
|
|
50
|
-
await stopSixbProviders(sixb)
|
|
51
|
-
}
|
|
52
|
-
sixb = null
|
|
1
|
+
import { SixbCliError } from "../lib/errors"
|
|
2
|
+
import { renderCliError } from "../ui"
|
|
3
|
+
|
|
4
|
+
/** The command stays only to say where cron went: a process manager left on it must see a failure. */
|
|
5
|
+
export async function runScheduler(): Promise<void> {
|
|
6
|
+
await renderCliError(
|
|
7
|
+
new SixbCliError("[SixbCLI] `sixb scheduler` no longer runs: `sixb orchestrator` runs cron.", {
|
|
8
|
+
remediation:
|
|
9
|
+
"Remove this process from your deployment and keep one `sixb orchestrator` running.",
|
|
53
10
|
})
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
await stopQuietly(() => runtime?.stop() ?? Promise.resolve())
|
|
57
|
-
if (sixb) {
|
|
58
|
-
await stopSixbProviders(sixb)
|
|
59
|
-
}
|
|
60
|
-
await renderCliError(error)
|
|
61
|
-
process.exit(1)
|
|
62
|
-
}
|
|
11
|
+
)
|
|
12
|
+
process.exit(1)
|
|
63
13
|
}
|
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
agentRuntimeRequired,
|
|
14
14
|
assertWorkerInputs,
|
|
15
15
|
createWorkerForType,
|
|
16
|
+
omittedWorkerWarnings,
|
|
16
17
|
type QueueWorkerProcess,
|
|
17
18
|
resolveRegisteredWorkerTypes,
|
|
18
19
|
resolveWorkerTypeToStart,
|
|
@@ -102,10 +103,12 @@ export async function runWorkerGroup(options: WorkerGroupOptions = {}) {
|
|
|
102
103
|
)
|
|
103
104
|
await Promise.all(workers.map((worker) => worker.start()))
|
|
104
105
|
|
|
105
|
-
const warnings =
|
|
106
|
-
workerTypes.length === 0
|
|
106
|
+
const warnings = [
|
|
107
|
+
...(workerTypes.length === 0
|
|
107
108
|
? ["No queue worker types are registered; the worker group process is idle."]
|
|
108
|
-
: []
|
|
109
|
+
: []),
|
|
110
|
+
...omittedWorkerWarnings(host, workerTypes),
|
|
111
|
+
]
|
|
109
112
|
|
|
110
113
|
app.stop()
|
|
111
114
|
await renderStatic(
|
package/src/index.tsx
CHANGED
|
@@ -315,7 +315,7 @@ async function main(): Promise<void> {
|
|
|
315
315
|
|
|
316
316
|
case "scheduler": {
|
|
317
317
|
const { runScheduler } = await import("./commands/scheduler")
|
|
318
|
-
await runScheduler(
|
|
318
|
+
await runScheduler()
|
|
319
319
|
break
|
|
320
320
|
}
|
|
321
321
|
|
|
@@ -376,13 +376,19 @@ async function main(): Promise<void> {
|
|
|
376
376
|
break
|
|
377
377
|
}
|
|
378
378
|
|
|
379
|
+
case "lake:rejected": {
|
|
380
|
+
const { runLakeRejected } = await import("./commands/lake-rejected")
|
|
381
|
+
await runLakeRejected({ entry: getFlag("entry"), dataset: getFlag("dataset") })
|
|
382
|
+
break
|
|
383
|
+
}
|
|
384
|
+
|
|
379
385
|
case "lake:cleanup": {
|
|
380
386
|
const { runLakeCleanup } = await import("./commands/lake-cleanup")
|
|
381
387
|
await runLakeCleanup({
|
|
382
388
|
entry: getFlag("entry"),
|
|
383
389
|
dryRun: hasFlag("dry-run"),
|
|
384
|
-
|
|
385
|
-
|
|
390
|
+
keepVersionsFor: getFlag("keep-versions-for"),
|
|
391
|
+
deleteFilesAfter: getFlag("delete-files-after"),
|
|
386
392
|
})
|
|
387
393
|
break
|
|
388
394
|
}
|
|
@@ -392,7 +398,7 @@ async function main(): Promise<void> {
|
|
|
392
398
|
const dev = getFlag("dev")
|
|
393
399
|
const prod = getFlag("prod")
|
|
394
400
|
await runEnvInit({
|
|
395
|
-
project: positionals[0]
|
|
401
|
+
...(positionals[0] === undefined ? {} : { project: positionals[0] }),
|
|
396
402
|
...(dev === undefined ? {} : { dev }),
|
|
397
403
|
...(prod === undefined ? {} : { prod }),
|
|
398
404
|
})
|
package/src/lib/command-line.ts
CHANGED
|
@@ -186,14 +186,20 @@ export const CLI_OPTION_DEFINITIONS = {
|
|
|
186
186
|
summary: "Show what would change without changing it",
|
|
187
187
|
kind: "boolean",
|
|
188
188
|
},
|
|
189
|
-
|
|
190
|
-
syntax: "--
|
|
191
|
-
summary: "
|
|
189
|
+
dataset: {
|
|
190
|
+
syntax: "--dataset <id>",
|
|
191
|
+
summary: "Show one dataset's rejected deliveries (default: every dataset)",
|
|
192
192
|
kind: "string",
|
|
193
193
|
},
|
|
194
|
-
"
|
|
195
|
-
syntax: "--
|
|
196
|
-
summary:
|
|
194
|
+
"keep-versions-for": {
|
|
195
|
+
syntax: "--keep-versions-for <duration>",
|
|
196
|
+
summary:
|
|
197
|
+
"How long a replaced lake version stays readable (default: `lakeMaintenance`, at least 1h)",
|
|
198
|
+
kind: "string",
|
|
199
|
+
},
|
|
200
|
+
"delete-files-after": {
|
|
201
|
+
syntax: "--delete-files-after <duration>",
|
|
202
|
+
summary: "How long unread lake files wait before deletion (default: `lakeMaintenance`)",
|
|
197
203
|
kind: "string",
|
|
198
204
|
},
|
|
199
205
|
} as const satisfies Record<string, { syntax: string; summary: string; kind: OptionKind }>
|
|
@@ -233,6 +239,7 @@ export type LocalCommandId =
|
|
|
233
239
|
| "db:prepare"
|
|
234
240
|
| "lake:check"
|
|
235
241
|
| "lake:cleanup"
|
|
242
|
+
| "lake:rejected"
|
|
236
243
|
| "deploy"
|
|
237
244
|
| "deploy:status"
|
|
238
245
|
| "deploy:logs"
|
|
@@ -300,6 +307,7 @@ const INSTANCE_SUMMARIES = {
|
|
|
300
307
|
workflows: "Discover and start workflows",
|
|
301
308
|
"workflow-runs": "Inspect workflow execution history",
|
|
302
309
|
users: "List and search project members",
|
|
310
|
+
artifacts: "Read, resolve and reply to an artifact's comments",
|
|
303
311
|
models: "List the project's models and their prices",
|
|
304
312
|
datasets: "Read datasets, their versions and rows",
|
|
305
313
|
syncs: "List the syncs you may run",
|
|
@@ -489,10 +497,10 @@ const commandTree: readonly CommandNode[] = [
|
|
|
489
497
|
},
|
|
490
498
|
],
|
|
491
499
|
},
|
|
492
|
-
command("scheduler", "
|
|
500
|
+
command("scheduler", "Removed: the orchestrator runs cron schedules", {
|
|
493
501
|
options: ["entry", "no-migrate"],
|
|
494
502
|
}),
|
|
495
|
-
command("orchestrator", "Start the production orchestrator,
|
|
503
|
+
command("orchestrator", "Start the production orchestrator: cron, event dispatch and upkeep", {
|
|
496
504
|
options: ["entry", "no-migrate"],
|
|
497
505
|
}),
|
|
498
506
|
command("rules", "Start the production rules runtime", { options: ["entry", "no-migrate"] }),
|
|
@@ -576,9 +584,14 @@ const commandTree: readonly CommandNode[] = [
|
|
|
576
584
|
}),
|
|
577
585
|
command("lake:cleanup", "Clean up old lake snapshots and files", {
|
|
578
586
|
path: ["lake", "cleanup"],
|
|
579
|
-
options: ["entry", "dry-run", "
|
|
587
|
+
options: ["entry", "dry-run", "keep-versions-for", "delete-files-after"],
|
|
580
588
|
rootHelpLabel: "lake cleanup",
|
|
581
589
|
}),
|
|
590
|
+
command("lake:rejected", "Show ingested deliveries refused after they were accepted", {
|
|
591
|
+
path: ["lake", "rejected"],
|
|
592
|
+
options: ["entry", "dataset"],
|
|
593
|
+
rootHelpLabel: "lake rejected",
|
|
594
|
+
}),
|
|
582
595
|
],
|
|
583
596
|
},
|
|
584
597
|
{
|
|
@@ -650,9 +663,8 @@ const commandTree: readonly CommandNode[] = [
|
|
|
650
663
|
children: [
|
|
651
664
|
command("env:init", "Move a project's values to its source and write sixb.env.ts", {
|
|
652
665
|
path: ["env", "init"],
|
|
653
|
-
usage: "sixb env init
|
|
666
|
+
usage: "sixb env init [doppler-project] [--dev <file>] [--prod <file>]",
|
|
654
667
|
options: ["dev", "prod"],
|
|
655
|
-
minimumPositionals: 1,
|
|
656
668
|
maximumPositionals: 1,
|
|
657
669
|
}),
|
|
658
670
|
command("env:pull", "Write .env from the project's source", {
|
|
@@ -743,8 +755,9 @@ const ROOT_HELP_GROUP_COMMANDS: readonly {
|
|
|
743
755
|
{ title: "Deploy", names: ["deploy", "desktop"] },
|
|
744
756
|
{
|
|
745
757
|
title: "Run in production",
|
|
746
|
-
names: ["api", "app", "
|
|
758
|
+
names: ["api", "app", "orchestrator", "rules", "worker", "worker-group"],
|
|
747
759
|
},
|
|
760
|
+
{ title: "Removed", names: ["scheduler"] },
|
|
748
761
|
{ title: "Storage", names: ["db", "lake"] },
|
|
749
762
|
{
|
|
750
763
|
title: "Connect to an instance",
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type {
|
|
2
|
+
DeployCheck,
|
|
2
3
|
DeployCommand,
|
|
3
4
|
DeployConfig,
|
|
4
5
|
DeployEnv,
|
|
@@ -28,7 +29,6 @@ const ROLE_SERVICES = {
|
|
|
28
29
|
api: "api",
|
|
29
30
|
app: "app",
|
|
30
31
|
orchestrator: "orchestrator",
|
|
31
|
-
scheduler: "scheduler",
|
|
32
32
|
rules: "rules",
|
|
33
33
|
"worker-group": "workers",
|
|
34
34
|
// A deployment co-hosts its workers in one `worker-group`; `workers.process.instances` scales it.
|
|
@@ -44,7 +44,6 @@ const SERVICE_ROLES: { readonly [Service in DeployServiceName]: RoleOf<Service>
|
|
|
44
44
|
api: "api",
|
|
45
45
|
app: "app",
|
|
46
46
|
orchestrator: "orchestrator",
|
|
47
|
-
scheduler: "scheduler",
|
|
48
47
|
rules: "rules",
|
|
49
48
|
workers: "worker-group",
|
|
50
49
|
}
|
|
@@ -168,6 +167,47 @@ function sixbService(
|
|
|
168
167
|
}
|
|
169
168
|
}
|
|
170
169
|
|
|
170
|
+
/**
|
|
171
|
+
* What this config leaves the lake's background work without: its retention, and the changes
|
|
172
|
+
* `ingest` stores. Only the project knows whether it has datasets, and this reads the config alone,
|
|
173
|
+
* so it warns and never stops a deploy.
|
|
174
|
+
*/
|
|
175
|
+
export function lakeWorkChecks(config: DeployConfig): readonly DeployCheck[] {
|
|
176
|
+
const { orchestrator, workers } = config.services ?? {}
|
|
177
|
+
const types = typeof workers === "object" ? workers.types : undefined
|
|
178
|
+
const lakeWorkerOmitted = workers === false || (Array.isArray(types) && !types.includes("lake"))
|
|
179
|
+
|
|
180
|
+
const missing = [
|
|
181
|
+
...(orchestrator === false ? ["the orchestrator requests it"] : []),
|
|
182
|
+
...(lakeWorkerOmitted ? ["a `lake` worker runs it"] : []),
|
|
183
|
+
]
|
|
184
|
+
|
|
185
|
+
return [
|
|
186
|
+
...(missing.length > 0
|
|
187
|
+
? [
|
|
188
|
+
{
|
|
189
|
+
id: "lake-retention",
|
|
190
|
+
label: "Lake retention",
|
|
191
|
+
status: "warning" as const,
|
|
192
|
+
detail: `Not deployed: ${missing.join(", and ")}.`,
|
|
193
|
+
remedy: "Deploy them, or a project with datasets keeps every replaced version.",
|
|
194
|
+
},
|
|
195
|
+
]
|
|
196
|
+
: []),
|
|
197
|
+
...(lakeWorkerOmitted
|
|
198
|
+
? [
|
|
199
|
+
{
|
|
200
|
+
id: "dataset-ingestion",
|
|
201
|
+
label: "Dataset ingestion",
|
|
202
|
+
status: "warning" as const,
|
|
203
|
+
detail: "Not deployed: a `lake` worker writes what `ingest` stores.",
|
|
204
|
+
remedy: "Deploy one, or ingested changes wait and never reach their dataset.",
|
|
205
|
+
},
|
|
206
|
+
]
|
|
207
|
+
: []),
|
|
208
|
+
]
|
|
209
|
+
}
|
|
210
|
+
|
|
171
211
|
function workerGroupArgs(options: DeployWorkersConfig): string[] {
|
|
172
212
|
const types = options.types === undefined || options.types === "all" ? [] : options.types
|
|
173
213
|
const args = ["worker-group", ...types]
|