@sixb/cli 0.2.1 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/README.md +25 -22
  2. package/dist/commands/access-token-utils.d.ts.map +1 -1
  3. package/dist/commands/agent.d.ts +4 -1
  4. package/dist/commands/agent.d.ts.map +1 -1
  5. package/dist/commands/api.d.ts.map +1 -1
  6. package/dist/commands/check.d.ts.map +1 -1
  7. package/dist/commands/connect.d.ts +13 -0
  8. package/dist/commands/connect.d.ts.map +1 -0
  9. package/dist/commands/db-migrate.d.ts.map +1 -1
  10. package/dist/commands/db-prepare.d.ts.map +1 -1
  11. package/dist/commands/deploy.d.ts.map +1 -1
  12. package/dist/commands/desktop.d.ts.map +1 -1
  13. package/dist/commands/dev-runner.d.ts.map +1 -1
  14. package/dist/commands/env.d.ts +6 -1
  15. package/dist/commands/env.d.ts.map +1 -1
  16. package/dist/commands/lake-cleanup.d.ts +2 -12
  17. package/dist/commands/lake-cleanup.d.ts.map +1 -1
  18. package/dist/commands/lake-rejected.d.ts +10 -0
  19. package/dist/commands/lake-rejected.d.ts.map +1 -0
  20. package/dist/commands/orchestrator.d.ts.map +1 -1
  21. package/dist/commands/scheduler.d.ts +2 -5
  22. package/dist/commands/scheduler.d.ts.map +1 -1
  23. package/dist/commands/worker-group.d.ts.map +1 -1
  24. package/dist/lib/command-line.d.ts +22 -7
  25. package/dist/lib/command-line.d.ts.map +1 -1
  26. package/dist/{tui → lib}/connection.d.ts +11 -1
  27. package/dist/{tui → lib}/connection.d.ts.map +1 -1
  28. package/dist/lib/deploy-release.d.ts +7 -1
  29. package/dist/lib/deploy-release.d.ts.map +1 -1
  30. package/dist/lib/desktop.d.ts +4 -0
  31. package/dist/lib/desktop.d.ts.map +1 -1
  32. package/dist/lib/dev-supervisor.d.ts.map +1 -1
  33. package/dist/lib/duration.d.ts +3 -0
  34. package/dist/lib/duration.d.ts.map +1 -0
  35. package/dist/lib/loadSixb.d.ts.map +1 -1
  36. package/dist/lib/production-roles.d.ts +0 -4
  37. package/dist/lib/production-roles.d.ts.map +1 -1
  38. package/dist/lib/profiles.d.ts +5 -0
  39. package/dist/lib/profiles.d.ts.map +1 -1
  40. package/dist/lib/runtime.d.ts +7 -9
  41. package/dist/lib/runtime.d.ts.map +1 -1
  42. package/dist/lib/worker-concurrency.d.ts.map +1 -1
  43. package/dist/lib/worker-registry.d.ts +4 -1
  44. package/dist/lib/worker-registry.d.ts.map +1 -1
  45. package/dist/runner/allowed-folders.d.ts +17 -0
  46. package/dist/runner/allowed-folders.d.ts.map +1 -0
  47. package/dist/runner/computer-connection.d.ts +70 -0
  48. package/dist/runner/computer-connection.d.ts.map +1 -0
  49. package/dist/runner/computer.d.ts.map +1 -0
  50. package/dist/runner/desktop-runner-main.d.ts +2 -0
  51. package/dist/runner/desktop-runner-main.d.ts.map +1 -0
  52. package/dist/runner/desktop-runner.d.ts +43 -0
  53. package/dist/runner/desktop-runner.d.ts.map +1 -0
  54. package/dist/runner/keep-awake.d.ts +11 -0
  55. package/dist/runner/keep-awake.d.ts.map +1 -0
  56. package/dist/runner/local-runner.d.ts +131 -0
  57. package/dist/runner/local-runner.d.ts.map +1 -0
  58. package/dist/tui/app.d.ts +2 -2
  59. package/dist/tui/app.d.ts.map +1 -1
  60. package/dist/tui/format.d.ts +29 -0
  61. package/dist/tui/format.d.ts.map +1 -1
  62. package/dist/tui/index.d.ts.map +1 -1
  63. package/dist/tui/settings.d.ts +5 -8
  64. package/dist/tui/settings.d.ts.map +1 -1
  65. package/dist/ui/deploy-screens.d.ts +4 -2
  66. package/dist/ui/deploy-screens.d.ts.map +1 -1
  67. package/dist/ui/env-screens.d.ts +14 -1
  68. package/dist/ui/env-screens.d.ts.map +1 -1
  69. package/dist/ui/screens.d.ts +5 -11
  70. package/dist/ui/screens.d.ts.map +1 -1
  71. package/package.json +21 -17
  72. package/src/commands/access-token-utils.ts +2 -20
  73. package/src/commands/agent.ts +122 -39
  74. package/src/commands/api.ts +7 -0
  75. package/src/commands/check.ts +75 -1
  76. package/src/commands/connect.ts +197 -0
  77. package/src/commands/db-migrate.ts +1 -0
  78. package/src/commands/db-prepare.ts +1 -0
  79. package/src/commands/deploy.ts +15 -3
  80. package/src/commands/desktop.ts +2 -1
  81. package/src/commands/dev-runner.ts +5 -0
  82. package/src/commands/env.ts +99 -20
  83. package/src/commands/lake-cleanup.ts +25 -48
  84. package/src/commands/lake-rejected.ts +36 -0
  85. package/src/commands/orchestrator.ts +2 -7
  86. package/src/commands/scheduler.ts +11 -61
  87. package/src/commands/worker-group.ts +6 -3
  88. package/src/index.tsx +23 -4
  89. package/src/lib/command-line.ts +47 -15
  90. package/src/{tui → lib}/connection.ts +64 -17
  91. package/src/lib/deploy-release.ts +42 -2
  92. package/src/lib/desktop.ts +8 -1
  93. package/src/lib/dev-supervisor.ts +7 -2
  94. package/src/lib/duration.ts +11 -0
  95. package/src/lib/loadSixb.ts +3 -4
  96. package/src/lib/production-roles.ts +0 -1
  97. package/src/lib/profiles.ts +15 -1
  98. package/src/lib/runtime.ts +63 -43
  99. package/src/lib/shareable-providers.ts +2 -2
  100. package/src/lib/storage-migration.ts +1 -0
  101. package/src/lib/worker-concurrency.ts +1 -0
  102. package/src/lib/worker-registry.ts +36 -1
  103. package/src/runner/allowed-folders.ts +126 -0
  104. package/src/runner/computer-connection.ts +184 -0
  105. package/src/runner/desktop-runner-main.ts +9 -0
  106. package/src/runner/desktop-runner.ts +217 -0
  107. package/src/runner/keep-awake.ts +50 -0
  108. package/src/runner/local-runner.ts +427 -0
  109. package/src/tui/app.tsx +168 -12
  110. package/src/tui/format.ts +213 -7
  111. package/src/tui/index.tsx +14 -13
  112. package/src/tui/settings.ts +12 -44
  113. package/src/ui/deploy-screens.ts +8 -1
  114. package/src/ui/env-screens.ts +39 -5
  115. package/src/ui/screens.ts +21 -20
  116. package/dist/tui/computer.d.ts.map +0 -1
  117. package/dist/tui/runner.d.ts +0 -75
  118. package/dist/tui/runner.d.ts.map +0 -1
  119. package/src/tui/runner.ts +0 -256
  120. /package/dist/{tui → runner}/computer.d.ts +0 -0
  121. /package/src/{tui → runner}/computer.ts +0 -0
@@ -0,0 +1,197 @@
1
+ import { realpath, stat } from "node:fs/promises"
2
+ import { homedir } from "node:os"
3
+ import { resolve } from "node:path"
4
+ import { createInterface } from "node:readline/promises"
5
+ import { CliError } from "@sixb/cli-core"
6
+ import { type Connection, connect } from "../lib/connection"
7
+ import { readAllowedFolders, setFoldersAllowed } from "../runner/allowed-folders"
8
+ import { thisComputer } from "../runner/computer"
9
+ import {
10
+ ComputerConnection,
11
+ type ComputerEvent,
12
+ EXIT_SIGNED_OUT,
13
+ } from "../runner/computer-connection"
14
+ import { stopInTime, stopOnHangUp } from "../runner/local-runner"
15
+ import { shortPath } from "../tui/format"
16
+ import { loadSettings } from "../tui/settings"
17
+ import { glyph, tone } from "../ui/theme"
18
+
19
+ /**
20
+ * `sixb connect [folder...]`: keep this computer available without `sixb` open. Turns sent from
21
+ * elsewhere to conversations in the allowed folders run here until this stops. It stays in the
22
+ * foreground: whatever should keep it running starts it, and sees what it prints.
23
+ */
24
+ export async function runConnect(options: {
25
+ readonly folders: readonly string[]
26
+ readonly yes: boolean
27
+ readonly json: boolean
28
+ readonly apiUrl?: string
29
+ readonly profile?: string
30
+ }): Promise<void> {
31
+ const connection = await connect(options)
32
+ if ("reason" in connection) {
33
+ throw connection.reason === "unreachable"
34
+ ? refusal(`[SixbCLI] Sixb isn't answering at ${connection.apiUrl}.`)
35
+ : refusal(
36
+ `[SixbCLI] Sign in first: sixb login ${connection.apiUrl || "<api-url>"}.`,
37
+ EXIT_SIGNED_OUT
38
+ )
39
+ }
40
+
41
+ await allowFolders(connection, options)
42
+
43
+ const settings = await loadSettings()
44
+ const report = options.json ? printJson : printLine
45
+ let finish: (code: number) => void = () => {}
46
+ const finished = new Promise<number>((resolve) => {
47
+ finish = resolve
48
+ })
49
+ const computer = new ComputerConnection(
50
+ connection,
51
+ (event) => {
52
+ if (event.type === "signed-out") finish(EXIT_SIGNED_OUT)
53
+ report(event)
54
+ },
55
+ settings.keepAwake
56
+ )
57
+
58
+ const unhook = stopOnHangUp(
59
+ () => computer,
60
+ (signal) => finish(signal === "SIGHUP" ? 129 : 143)
61
+ )
62
+ // Ctrl+C is how this is meant to stop; a second one doesn't wait for a turn to record its end.
63
+ process.once("SIGINT", () => {
64
+ process.once("SIGINT", () => process.exit(130))
65
+ void stopInTime(computer).finally(() => finish(0))
66
+ })
67
+
68
+ if (!options.json) {
69
+ const { name } = await thisComputer()
70
+ say(
71
+ `${tone.blue(glyph.mark)} ${tone.strong(name)} ${tone.faint(`connected to ${connection.project.id}. ctrl+c to stop.`)}`
72
+ )
73
+ }
74
+ await computer.start()
75
+
76
+ const code = await finished
77
+ unhook()
78
+ if (code === EXIT_SIGNED_OUT) await stopInTime(computer)
79
+ process.exit(code)
80
+ }
81
+
82
+ /**
83
+ * Add what was named to the folders allowed for this project, asking first, or offer this one if
84
+ * there are none.
85
+ */
86
+ async function allowFolders(
87
+ connection: Connection,
88
+ options: {
89
+ readonly folders: readonly string[]
90
+ readonly yes: boolean
91
+ readonly json: boolean
92
+ }
93
+ ): Promise<void> {
94
+ const project = connection.project.id
95
+ const allowed = await readAllowedFolders(connection).catch((error: unknown) => {
96
+ throw refusal(
97
+ `[SixbCLI] Could not read your allowed folders. ${error instanceof Error ? error.message : ""}`
98
+ )
99
+ })
100
+ const named = [...new Set(await Promise.all(options.folders.map(realFolder)))]
101
+ const interactive = process.stdin.isTTY && !options.json
102
+ const newlyNamed = named.filter((folder) => !allowed.includes(folder))
103
+
104
+ if (newlyNamed.length > 0 && !options.yes && !interactive) {
105
+ throw refusal(
106
+ `[SixbCLI] ${newlyNamed.join(", ")} isn't allowed for ${project} yet: run sixb connect in a terminal to choose it, or pass --yes.`
107
+ )
108
+ }
109
+ const chosen: string[] = options.yes ? [...newlyNamed] : [...(await askEach(newlyNamed, project))]
110
+ // Nothing named and nothing allowed: this folder is the one to offer, to a person, never by --yes.
111
+ if (named.length === 0 && allowed.length === 0 && interactive) {
112
+ chosen.push(...(await askEach([await realFolder(".")], project)))
113
+ }
114
+ if (chosen.length > 0) await setFoldersAllowed(connection, chosen, true)
115
+ if (allowed.length + chosen.length === 0) {
116
+ throw refusal(
117
+ `[SixbCLI] No folders are allowed for ${project}, so nothing would run here. Choose one: sixb connect <folder>.`
118
+ )
119
+ }
120
+ }
121
+
122
+ async function askEach(folders: readonly string[], project: string): Promise<readonly string[]> {
123
+ if (folders.length === 0) return []
124
+ const prompt = createInterface({ input: process.stdin, output: process.stderr })
125
+ const chosen: string[] = []
126
+ try {
127
+ say(
128
+ tone.muted(
129
+ `Messages sent to ${project} from the web and your other devices will run in a folder you allow, on this computer, as you. The agent's commands run there without asking.`
130
+ )
131
+ )
132
+ for (const folder of folders) {
133
+ const answer = await prompt.question(
134
+ `Allow ${tone.strong(shown(folder))} for ${project}? [y/N] `
135
+ )
136
+ if (answer.trim().toLowerCase() === "y") chosen.push(folder)
137
+ }
138
+ } finally {
139
+ prompt.close()
140
+ }
141
+ return chosen
142
+ }
143
+
144
+ /** As the project keeps it, and the local sandbox compares it: its real path. */
145
+ async function realFolder(folder: string): Promise<string> {
146
+ const path = resolve(folder)
147
+ const entry = await stat(path).catch(() => null)
148
+ if (!entry?.isDirectory()) throw refusal(`[SixbCLI] ${path} isn't a folder.`)
149
+ return realpath(path)
150
+ }
151
+
152
+ function refusal(message: string, exitCode = 1): CliError {
153
+ return new CliError({ code: "connect", message }, exitCode)
154
+ }
155
+
156
+ function shown(folder: string): string {
157
+ return shortPath(folder, homedir(), 72) || folder
158
+ }
159
+
160
+ function say(line: string): void {
161
+ process.stderr.write(`${line}\n`)
162
+ }
163
+
164
+ function printJson(event: ComputerEvent): void {
165
+ process.stdout.write(`${JSON.stringify(event)}\n`)
166
+ }
167
+
168
+ function printLine(event: ComputerEvent): void {
169
+ switch (event.type) {
170
+ case "folders":
171
+ say(
172
+ event.folders.length > 0
173
+ ? ` ${tone.muted("Running messages from the web in")} ${event.folders.map(shown).join(", ")}`
174
+ : tone.warn(" No allowed folders exist now, so nothing runs here.")
175
+ )
176
+ return
177
+ case "turn.started":
178
+ say(` ${tone.blue(glyph.live)} ${shown(event.folder)} ${tone.faint(event.threadId)}`)
179
+ return
180
+ case "turn.ended":
181
+ say(
182
+ ` ${event.ran ? tone.ok(glyph.ok) : tone.fail(glyph.fail)} ${shown(event.folder)} ${tone.faint(event.threadId)}`
183
+ )
184
+ return
185
+ case "session.lost":
186
+ say(` ${tone.warn(glyph.warn)} ${shown(event.folder)} ${tone.muted(event.message)}`)
187
+ return
188
+ case "session.back":
189
+ say(` ${tone.ok(glyph.ok)} ${shown(event.folder)} ${tone.muted("back online")}`)
190
+ return
191
+ case "signed-out":
192
+ say(tone.fail(" Your sign-in ended. Sign in again with sixb login, then reconnect."))
193
+ return
194
+ case "error":
195
+ say(` ${tone.fail(glyph.fail)} ${tone.muted(event.message)}`)
196
+ }
197
+ }
@@ -17,6 +17,7 @@ export async function runDbMigrate(options: DbMigrateOptions = {}) {
17
17
  migrateStorage(sixb.storage, {
18
18
  projectId: sixb.id,
19
19
  ontology: sixb.definitions.ontology,
20
+ security: sixb.definitions.security,
20
21
  })
21
22
  )
22
23
  await renderStatic(
@@ -14,6 +14,7 @@ export async function runDbPrepare(options: { entry?: string } = {}): Promise<vo
14
14
  const result = await prepareObjectQueries({
15
15
  projectId: sixb.id,
16
16
  ontology: sixb.definitions.ontology,
17
+ security: sixb.definitions.security,
17
18
  storage: sixb.storage,
18
19
  })
19
20
  console.info(
@@ -10,7 +10,7 @@ import type {
10
10
  DeployRelease,
11
11
  } from "@sixb/core/deploy"
12
12
  import { loadDeployConfig } from "../lib/deploy-config"
13
- import { buildDeployRelease } from "../lib/deploy-release"
13
+ import { buildDeployRelease, lakeWorkChecks } from "../lib/deploy-release"
14
14
  import {
15
15
  commitHasProjectFile,
16
16
  deployer,
@@ -68,7 +68,13 @@ export async function runDeploy(options: DeployOptions = {}): Promise<void> {
68
68
  envFromSource,
69
69
  })
70
70
  if (options.json) return writeJson(release)
71
- await renderStatic(deployReleaseView({ release, configPath: relative(cwd, path) || path }))
71
+ await renderStatic(
72
+ deployReleaseView({
73
+ release,
74
+ configPath: relative(cwd, path) || path,
75
+ warnings: configWarnings(config),
76
+ })
77
+ )
72
78
  return
73
79
  }
74
80
 
@@ -118,6 +124,7 @@ export async function runDeploy(options: DeployOptions = {}): Promise<void> {
118
124
  release,
119
125
  seconds: (Date.now() - deployStartedAt) / 1000,
120
126
  keptEnv: progress.state.keptEnv,
127
+ warnings: configWarnings(config),
121
128
  })
122
129
  )
123
130
  }
@@ -130,9 +137,10 @@ export async function runDeployCheck(options: { readonly json?: boolean; readonl
130
137
  envFromSource: Boolean((await loadEnvConfig(cwd))?.source),
131
138
  })
132
139
  const projectPath = await projectPathOf(cwd)
133
- const checks = await withTask("deploy check", `Checking ${location(config)}`, () =>
140
+ const targetChecks = await withTask("deploy check", `Checking ${location(config)}`, () =>
134
141
  config.target.check(release, { name: config.name, projectPath })
135
142
  )
143
+ const checks = [...targetChecks, ...lakeWorkChecks(config)]
136
144
  if (options.json) writeJson(checks)
137
145
  else
138
146
  await renderStatic(deployChecksView({ name: config.name, location: location(config), checks }))
@@ -300,6 +308,10 @@ async function publicKeys(value: string): Promise<string[]> {
300
308
  return [value]
301
309
  }
302
310
 
311
+ function configWarnings(config: DeployConfig): readonly string[] {
312
+ return lakeWorkChecks(config).map((check) => `${check.label}: ${check.detail} ${check.remedy}`)
313
+ }
314
+
303
315
  function blocksDeploy(check: DeployCheck): boolean {
304
316
  return check.status === "fixable" || check.status === "manual"
305
317
  }
@@ -1,7 +1,7 @@
1
1
  import { join } from "node:path"
2
2
  import { DEPLOY_CONFIG_FILE, loadDeployConfig } from "../lib/deploy-config"
3
3
  import { deployPublicOrigins } from "../lib/deploy-release"
4
- import { loadDesktop } from "../lib/desktop"
4
+ import { DESKTOP_RUNNER_ENTRY, loadDesktop } from "../lib/desktop"
5
5
  import { SixbCliError } from "../lib/errors"
6
6
  import { configuredOrigin } from "../lib/public-origin"
7
7
  import { keyValueResultView, renderStatic } from "../ui"
@@ -20,6 +20,7 @@ export async function runDesktopBuild(options: DesktopBuildOptions = {}): Promis
20
20
  projectRoot,
21
21
  appUrl: origins.app,
22
22
  apiOrigin: origins.api,
23
+ runnerEntry: DESKTOP_RUNNER_ENTRY,
23
24
  })
24
25
 
25
26
  await renderStatic(
@@ -113,6 +113,10 @@ export async function runDevRuntime(options: DevOptions = {}) {
113
113
  port: topology.apiPort,
114
114
  hostname: topology.apiHost,
115
115
  quiet: true,
116
+ // The orchestrator this process co-hosts owns it.
117
+ ontologyMaintenance: false,
118
+ // So does the lake worker.
119
+ ingestFlush: false,
116
120
  browser: {
117
121
  publicOrigin: topology.apiPublicOrigin,
118
122
  allowedOrigins: topology.allowedBrowserOrigins,
@@ -156,6 +160,7 @@ export async function runDevRuntime(options: DevOptions = {}) {
156
160
  { type: "pipeline", worker: runtime.pipelineWorker },
157
161
  { type: "workflow", worker: runtime.workflowWorker },
158
162
  { type: "sync", worker: runtime.syncWorker },
163
+ { type: "lake", worker: runtime.lakeWorker },
159
164
  ].flatMap(({ type, worker }) => (worker ? [{ type, concurrency: worker.concurrency }] : [])),
160
165
  warnings: devApp.warning ? [devApp.warning, ...runtime.warnings] : runtime.warnings,
161
166
  }
@@ -23,6 +23,7 @@ import { errorMessage, SixbCliError } from "../lib/errors"
23
23
  import { promptHidden } from "../lib/hidden-prompt"
24
24
  import { renderStatic } from "../ui"
25
25
  import {
26
+ type EnvInitConfig,
26
27
  type EnvStatusCell,
27
28
  envInitView,
28
29
  envKeptView,
@@ -34,6 +35,7 @@ import { canRunLive } from "../ui/live/can-run-live"
34
35
 
35
36
  const ENV_NAMES: readonly EnvName[] = ["dev", "prod"]
36
37
  const DOPPLER_PACKAGE = "@sixb/env-doppler"
38
+ const SOURCE_EXAMPLE = `Add one, such as \`source: doppler({ project: "shop" })\` from ${DOPPLER_PACKAGE}, or \`source: azureKeyVault({ vaults: { dev: "shop-dev", prod: "shop-prod" } })\` from @sixb/env-azure-key-vault.`
37
39
 
38
40
  // ─── pull ────────────────────────────────────────────────────────────────────
39
41
 
@@ -264,16 +266,25 @@ export async function runEnvStatus(options: { readonly cwd?: string } = {}): Pro
264
266
 
265
267
  // ─── init ────────────────────────────────────────────────────────────────────
266
268
 
269
+ /**
270
+ * Moves a project's values to its source. Given a Doppler project, it also writes `sixb.env.ts`.
271
+ * Without one, it uploads to the source `sixb.env.ts` already names, whatever it is, and fills in
272
+ * its empty `vars: {}`.
273
+ */
267
274
  export async function runEnvInit(options: {
268
- readonly project: string
275
+ readonly project?: string
269
276
  readonly dev?: string
270
277
  readonly prod?: string
271
278
  readonly cwd?: string
272
279
  }): Promise<void> {
273
280
  const cwd = options.cwd ?? process.cwd()
274
281
  const configPath = join(cwd, ENV_CONFIG_FILE)
275
- if (await Bun.file(configPath).exists()) {
276
- throw new SixbCliError(`[SixbEnv] ${ENV_CONFIG_FILE} already exists.`)
282
+ const { project } = options
283
+ const configExists = await Bun.file(configPath).exists()
284
+ if (project !== undefined && configExists) {
285
+ throw new SixbCliError(`[SixbEnv] ${ENV_CONFIG_FILE} already exists.`, {
286
+ remediation: `To upload to the source it names, leave out the project: ${INIT_USAGE}`,
287
+ })
277
288
  }
278
289
 
279
290
  const files = [
@@ -283,11 +294,13 @@ export async function runEnvInit(options: {
283
294
  const given = files.flatMap(([env, file]) => (file === undefined ? [] : [{ env, file }]))
284
295
  if (given.length === 0) {
285
296
  throw new SixbCliError("[SixbEnv] Say where the current values are.", {
286
- remediation: "sixb env init <project> --dev .env --prod .env.prod",
297
+ remediation: project === undefined ? INIT_USAGE : `sixb env init ${project} ${INIT_FILES}`,
287
298
  })
288
299
  }
289
300
 
290
- const source = await loadDopplerSource(cwd, options.project)
301
+ const existing = configExists ? await loadEnvConfig(cwd) : null
302
+ const source =
303
+ project === undefined ? sourceToFill(existing) : await loadDopplerSource(cwd, project)
291
304
  const uploads = await Promise.all(
292
305
  given.map(async ({ env, file }) => {
293
306
  const path = resolve(cwd, file)
@@ -298,33 +311,102 @@ export async function runEnvInit(options: {
298
311
  })
299
312
  )
300
313
 
301
- // Check every environment before writing to any, so a conflict changes nothing. Running this
302
- // again after a failed upload finds the same values, which aren't a conflict.
314
+ // Check every environment before writing to any, so a conflict changes nothing. A value the
315
+ // source already has isn't sent again, so running this twice, or after an upload that didn't
316
+ // finish, changes nothing it doesn't need to.
317
+ const sends: { readonly env: EnvName; readonly values: Record<string, string> }[] = []
303
318
  for (const { env, values } of uploads) {
304
- const existing = await source.read(env)
319
+ const current = await source.read(env)
305
320
  const conflicts = Object.keys(values).filter(
306
- (name) => name in existing && existing[name] !== values[name]
321
+ (name) => name in current && current[name] !== values[name]
307
322
  )
308
323
  if (conflicts.length > 0) {
309
324
  throw new SixbCliError(
310
- `[SixbEnv] ${options.project} already has other values in ${env} for ${conflicts.sort().join(", ")}.`,
325
+ `[SixbEnv] ${project ?? "The source"} already has other values in ${env} for ${conflicts.sort().join(", ")}.`,
311
326
  {
312
327
  remediation:
313
328
  "Change them in your source's dashboard, or edit the file, and run this again.",
314
329
  }
315
330
  )
316
331
  }
332
+ sends.push({
333
+ env,
334
+ values: Object.fromEntries(Object.entries(values).filter(([name]) => !(name in current))),
335
+ })
317
336
  }
318
- for (const { env, values } of uploads) await source.write(env, values)
337
+ for (const { env, values } of sends) await source.write(env, values)
319
338
 
320
339
  const vars = declaredVars(uploads)
321
- await writeFile(configPath, renderEnvConfig(options.project, vars))
340
+ let config: EnvInitConfig
341
+ if (project !== undefined) {
342
+ await writeFile(configPath, renderEnvConfig(project, vars))
343
+ config = { kind: "written", names: Object.keys(vars).length }
344
+ } else {
345
+ config = await fillVars(configPath, existing?.vars ?? {}, vars)
346
+ }
322
347
  await renderStatic(
323
348
  envInitView({
324
- uploaded: uploads.map(({ env, values }) => ({ env, count: Object.keys(values).length })),
325
- names: Object.keys(vars).length,
349
+ uploaded: uploads.map(({ env, values }, index) => {
350
+ const sent = Object.keys(sends[index]?.values ?? {}).length
351
+ return { env, count: sent, already: Object.keys(values).length - sent }
352
+ }),
353
+ config,
354
+ })
355
+ )
356
+ }
357
+
358
+ const INIT_FILES = "--dev .env --prod .env.prod"
359
+ const INIT_USAGE = `sixb env init ${INIT_FILES}`
360
+
361
+ function sourceToFill(config: EnvConfig | null): EnvSource {
362
+ if (!config) {
363
+ throw new SixbCliError(`[SixbEnv] There's no ${ENV_CONFIG_FILE} naming a source yet.`, {
364
+ remediation: `Write one with its source and \`vars: {}\`, then run this again. For Doppler, \`sixb env init <project> ${INIT_FILES}\` writes it for you.`,
365
+ })
366
+ }
367
+ if (!config.source) {
368
+ throw new SixbCliError(`[SixbEnv] ${ENV_CONFIG_FILE} has no \`source\` to upload to.`, {
369
+ remediation: SOURCE_EXAMPLE,
326
370
  })
371
+ }
372
+ return config.source
373
+ }
374
+
375
+ /**
376
+ * Declares the uploaded names in an existing `sixb.env.ts`. Only an empty `vars: {}` is replaced:
377
+ * anything else is the author's code, so the rules are printed for them to add instead.
378
+ */
379
+ async function fillVars(
380
+ path: string,
381
+ declared: Readonly<Record<string, EnvVarRule>>,
382
+ guessed: Readonly<Record<string, EnvVarRule>>
383
+ ): Promise<EnvInitConfig> {
384
+ const undeclared = Object.fromEntries(
385
+ Object.entries(guessed).filter(([name]) => !(name in declared))
386
+ )
387
+ const names = Object.keys(undeclared).length
388
+ if (names === 0) return { kind: "unchanged" }
389
+
390
+ const text = await readFile(path, "utf8")
391
+ const empty = [...text.matchAll(EMPTY_VARS)]
392
+ if (Object.keys(declared).length > 0 || empty.length !== 1) {
393
+ return { kind: "paste", names, lines: varLines(undeclared, "") }
394
+ }
395
+ await writeFile(
396
+ path,
397
+ text.replace(EMPTY_VARS, (_match, indent: string) => renderVars(undeclared, indent))
327
398
  )
399
+ return { kind: "written", names }
400
+ }
401
+
402
+ const EMPTY_VARS = /^([ \t]*)vars:\s*\{\s*\}/gm
403
+
404
+ function renderVars(vars: Readonly<Record<string, EnvVarRule>>, indent: string): string {
405
+ return [`${indent}vars: {`, ...varLines(vars, `${indent} `), `${indent}}`].join("\n")
406
+ }
407
+
408
+ function varLines(vars: Readonly<Record<string, EnvVarRule>>, indent: string): string[] {
409
+ return Object.entries(vars).map(([name, rule]) => `${indent}${name}: "${rule}",`)
328
410
  }
329
411
 
330
412
  /**
@@ -366,16 +448,13 @@ function uploadableValues(
366
448
  }
367
449
 
368
450
  function renderEnvConfig(project: string, vars: Readonly<Record<string, EnvVarRule>>): string {
369
- const lines = Object.entries(vars).map(([name, rule]) => ` ${name}: "${rule}",`)
370
451
  return [
371
452
  'import { defineEnv } from "@sixb/core/env"',
372
453
  `import { doppler } from "${DOPPLER_PACKAGE}"`,
373
454
  "",
374
455
  "export default defineEnv({",
375
456
  ` source: doppler({ project: ${JSON.stringify(project)} }),`,
376
- " vars: {",
377
- ...lines,
378
- " },",
457
+ `${renderVars(vars, " ")},`,
379
458
  "})",
380
459
  "",
381
460
  ].join("\n")
@@ -410,12 +489,12 @@ async function requireSource(
410
489
  const config = await loadEnvConfig(cwd)
411
490
  if (!config) {
412
491
  throw new SixbCliError(`[SixbEnv] No ${ENV_CONFIG_FILE} in ${cwd}.`, {
413
- remediation: "sixb env init <project> --dev .env --prod .env.prod",
492
+ remediation: `sixb env init <project> ${INIT_FILES}`,
414
493
  })
415
494
  }
416
495
  if (!config.source) {
417
496
  throw new SixbCliError(`[SixbEnv] ${ENV_CONFIG_FILE} has no \`source\`.`, {
418
- remediation: `Add one, such as \`source: doppler({ project: "shop" })\` from ${DOPPLER_PACKAGE}.`,
497
+ remediation: SOURCE_EXAMPLE,
419
498
  })
420
499
  }
421
500
  return { config, source: config.source }
@@ -1,79 +1,56 @@
1
+ import { getLakeRetentionRuntime } from "@sixb/core/internal/runtime"
2
+ import { parseDurationMs } from "../lib/duration"
1
3
  import { loadSixbFromEntry } from "../lib/loadSixb"
2
4
  import { resolveRuntimeEntry } from "../lib/production"
3
5
  import { stopSixbProviders } from "../lib/runtime"
4
6
  import { lakeCleanupView, renderStatic, withTask } from "../ui"
5
7
 
6
- const DEFAULT_RETENTION = "7 days"
7
-
8
8
  export interface LakeCleanupOptions {
9
9
  entry?: string
10
10
  dryRun?: boolean
11
- expireOlderThan?: string
12
- deleteOlderThan?: string
13
- }
14
-
15
- interface LakeMaintenanceOptions {
16
- readonly dryRun?: boolean
17
- readonly expireOlderThan?: string
18
- readonly deleteOlderThan?: string
19
- readonly protect?: readonly { readonly datasetId: string; readonly versionId: string }[]
20
- }
21
-
22
- export interface LakeMaintenanceReport {
23
- readonly dryRun: boolean
24
- readonly expireOlderThan: string
25
- readonly deleteOlderThan: string
26
- readonly snapshots: number
27
- /** Old snapshots kept because a version they hold is still needed, when the provider says. */
28
- readonly keptSnapshots?: number
29
- readonly oldFiles: number
30
- readonly orphanedFiles: number
31
- }
32
-
33
- interface LakeMaintenanceProvider {
34
- runMaintenance(options?: LakeMaintenanceOptions): Promise<LakeMaintenanceReport>
11
+ keepVersionsFor?: string
12
+ deleteFilesAfter?: string
35
13
  }
36
14
 
37
15
  export async function runLakeCleanup(options: LakeCleanupOptions = {}) {
16
+ const keepVersions = durationFlag(options.keepVersionsFor, "--keep-versions-for")
17
+ const deleteFiles = durationFlag(options.deleteFilesAfter, "--delete-files-after")
18
+
38
19
  const entry = await resolveRuntimeEntry({ entry: options.entry })
39
20
  const sixb = await withTask("lake cleanup", "Loading project", () => loadSixbFromEntry(entry))
40
21
 
41
22
  try {
42
- const lakeStorage = sixb.lakeStorage
43
- if (!isLakeMaintenanceProvider(lakeStorage)) {
44
- throw new Error("Configured lake storage does not support maintenance cleanup.")
23
+ if (sixb.lakeStorage.maintenance === undefined) {
24
+ throw new Error("[SixbCLI] Configured lake storage does not support retention cleanup.")
45
25
  }
46
26
 
47
- const expireOlderThan = options.expireOlderThan ?? DEFAULT_RETENTION
48
- const deleteOlderThan = options.deleteOlderThan ?? expireOlderThan
27
+ const retention = getLakeRetentionRuntime(sixb)
28
+ const { settings } = retention
29
+ const keepVersionsFor = options.keepVersionsFor ?? settings.keepVersionsFor
30
+ const deleteFilesAfter = options.deleteFilesAfter ?? settings.deleteFilesAfter
49
31
 
50
- // Objects hold the rows of the version each projection applied, and their markings are read
51
- // from it: the lake keeps those versions too. Without them, cleanup does not run.
52
- const applied = await sixb.storage.ontology.sources.listActive({ projectId: sixb.projectId })
32
+ // The same passes as automatic retention, without its time budget: they keep the versions
33
+ // projections applied, and run until none reports more. A dry run measures one pass.
53
34
  const report = await withTask(
54
35
  "lake cleanup",
55
36
  options.dryRun ? "Measuring" : "Cleaning up",
56
37
  () =>
57
- lakeStorage.runMaintenance({
38
+ retention.apply({
58
39
  dryRun: options.dryRun ?? false,
59
- expireOlderThan,
60
- deleteOlderThan,
61
- protect: applied.map(({ datasetVersion }) => ({
62
- datasetId: datasetVersion.datasetId,
63
- versionId: datasetVersion.versionId,
64
- })),
40
+ keepVersionsForMs: keepVersions ?? settings.keepVersionsForMs,
41
+ deleteFilesAfterMs: deleteFiles ?? settings.deleteFilesAfterMs,
65
42
  })
66
43
  )
67
- await renderStatic(lakeCleanupView({ projectId: sixb.id, report }))
44
+
45
+ await renderStatic(
46
+ lakeCleanupView({ projectId: sixb.id, keepVersionsFor, deleteFilesAfter, report })
47
+ )
68
48
  } finally {
69
49
  await stopSixbProviders(sixb)
70
50
  }
71
51
  }
72
52
 
73
- function isLakeMaintenanceProvider(value: unknown): value is LakeMaintenanceProvider {
74
- return (
75
- typeof value === "object" &&
76
- value !== null &&
77
- typeof (value as { runMaintenance?: unknown }).runMaintenance === "function"
78
- )
53
+ /** Read before anything loads, so that a typo fails at once. */
54
+ function durationFlag(value: string | undefined, flag: string): number | null {
55
+ return value === undefined ? null : parseDurationMs(value, flag)
79
56
  }
@@ -0,0 +1,36 @@
1
+ import { loadSixbFromEntry } from "../lib/loadSixb"
2
+ import { resolveRuntimeEntry } from "../lib/production"
3
+ import { stopSixbProviders } from "../lib/runtime"
4
+ import { withTask } from "../ui"
5
+
6
+ /** A week of refusals is rarely this many; past it, ask for one dataset. */
7
+ const LIMIT = 500
8
+
9
+ export interface LakeRejectedOptions {
10
+ entry?: string
11
+ dataset?: string
12
+ }
13
+
14
+ /**
15
+ * Prints, as JSON, the deliveries refused after `ingest` answered: what each held, whole, and why,
16
+ * so that someone can fix them and send them again. They stay for a week.
17
+ */
18
+ export async function runLakeRejected(options: LakeRejectedOptions = {}) {
19
+ const entry = await resolveRuntimeEntry({ entry: options.entry })
20
+ const sixb = await withTask("lake rejected", "Loading project", () => loadSixbFromEntry(entry))
21
+
22
+ try {
23
+ const inbox = sixb.storage.datasetIngest
24
+ if (!inbox) throw new Error("[SixbCLI] Configured storage keeps no ingested changes.")
25
+
26
+ const rejected = await inbox.listRejected({
27
+ projectId: sixb.id,
28
+ limit: LIMIT,
29
+ ...(options.dataset === undefined ? {} : { datasetId: options.dataset }),
30
+ })
31
+
32
+ process.stdout.write(`${JSON.stringify(rejected, null, 2)}\n`)
33
+ } finally {
34
+ await stopSixbProviders(sixb)
35
+ }
36
+ }
@@ -33,19 +33,14 @@ export async function runOrchestrator(options: OrchestratorOptions = {}) {
33
33
 
34
34
  runtime = await startOrchestratorRuntime(sixb)
35
35
 
36
- const warnings = [...runtime.warnings]
37
- if (runtime.orchestratorWorker === null) {
38
- warnings.push("No orchestrator routes are registered; the orchestrator process is idle.")
39
- }
40
-
41
36
  app.stop()
42
37
  await renderStatic(
43
38
  roleView({
44
39
  command: "orchestrator",
45
40
  name: sixb.id,
46
- items: [{ label: "role", value: "event-to-queue dispatcher" }],
41
+ items: [{ label: "role", value: "event dispatch, cron and upkeep" }],
47
42
  storage: migration.summary,
48
- warnings,
43
+ warnings: runtime.warnings,
49
44
  })
50
45
  )
51
46