@sixb/cli 0.2.1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -22
- package/dist/commands/access-token-utils.d.ts.map +1 -1
- package/dist/commands/agent.d.ts +4 -1
- package/dist/commands/agent.d.ts.map +1 -1
- package/dist/commands/api.d.ts.map +1 -1
- package/dist/commands/check.d.ts.map +1 -1
- package/dist/commands/connect.d.ts +13 -0
- package/dist/commands/connect.d.ts.map +1 -0
- package/dist/commands/db-migrate.d.ts.map +1 -1
- package/dist/commands/db-prepare.d.ts.map +1 -1
- package/dist/commands/deploy.d.ts.map +1 -1
- package/dist/commands/desktop.d.ts.map +1 -1
- package/dist/commands/dev-runner.d.ts.map +1 -1
- package/dist/commands/env.d.ts +6 -1
- package/dist/commands/env.d.ts.map +1 -1
- package/dist/commands/lake-cleanup.d.ts +2 -12
- package/dist/commands/lake-cleanup.d.ts.map +1 -1
- package/dist/commands/lake-rejected.d.ts +10 -0
- package/dist/commands/lake-rejected.d.ts.map +1 -0
- package/dist/commands/orchestrator.d.ts.map +1 -1
- package/dist/commands/scheduler.d.ts +2 -5
- package/dist/commands/scheduler.d.ts.map +1 -1
- package/dist/commands/worker-group.d.ts.map +1 -1
- package/dist/lib/command-line.d.ts +22 -7
- package/dist/lib/command-line.d.ts.map +1 -1
- package/dist/{tui → lib}/connection.d.ts +11 -1
- package/dist/{tui → lib}/connection.d.ts.map +1 -1
- package/dist/lib/deploy-release.d.ts +7 -1
- package/dist/lib/deploy-release.d.ts.map +1 -1
- package/dist/lib/desktop.d.ts +4 -0
- package/dist/lib/desktop.d.ts.map +1 -1
- package/dist/lib/dev-supervisor.d.ts.map +1 -1
- package/dist/lib/duration.d.ts +3 -0
- package/dist/lib/duration.d.ts.map +1 -0
- package/dist/lib/loadSixb.d.ts.map +1 -1
- package/dist/lib/production-roles.d.ts +0 -4
- package/dist/lib/production-roles.d.ts.map +1 -1
- package/dist/lib/profiles.d.ts +5 -0
- package/dist/lib/profiles.d.ts.map +1 -1
- package/dist/lib/runtime.d.ts +7 -9
- package/dist/lib/runtime.d.ts.map +1 -1
- package/dist/lib/worker-concurrency.d.ts.map +1 -1
- package/dist/lib/worker-registry.d.ts +4 -1
- package/dist/lib/worker-registry.d.ts.map +1 -1
- package/dist/runner/allowed-folders.d.ts +17 -0
- package/dist/runner/allowed-folders.d.ts.map +1 -0
- package/dist/runner/computer-connection.d.ts +70 -0
- package/dist/runner/computer-connection.d.ts.map +1 -0
- package/dist/runner/computer.d.ts.map +1 -0
- package/dist/runner/desktop-runner-main.d.ts +2 -0
- package/dist/runner/desktop-runner-main.d.ts.map +1 -0
- package/dist/runner/desktop-runner.d.ts +43 -0
- package/dist/runner/desktop-runner.d.ts.map +1 -0
- package/dist/runner/keep-awake.d.ts +11 -0
- package/dist/runner/keep-awake.d.ts.map +1 -0
- package/dist/runner/local-runner.d.ts +131 -0
- package/dist/runner/local-runner.d.ts.map +1 -0
- package/dist/tui/app.d.ts +2 -2
- package/dist/tui/app.d.ts.map +1 -1
- package/dist/tui/format.d.ts +29 -0
- package/dist/tui/format.d.ts.map +1 -1
- package/dist/tui/index.d.ts.map +1 -1
- package/dist/tui/settings.d.ts +5 -8
- package/dist/tui/settings.d.ts.map +1 -1
- package/dist/ui/deploy-screens.d.ts +4 -2
- package/dist/ui/deploy-screens.d.ts.map +1 -1
- package/dist/ui/env-screens.d.ts +14 -1
- package/dist/ui/env-screens.d.ts.map +1 -1
- package/dist/ui/screens.d.ts +5 -11
- package/dist/ui/screens.d.ts.map +1 -1
- package/package.json +21 -17
- package/src/commands/access-token-utils.ts +2 -20
- package/src/commands/agent.ts +122 -39
- package/src/commands/api.ts +7 -0
- package/src/commands/check.ts +75 -1
- package/src/commands/connect.ts +197 -0
- package/src/commands/db-migrate.ts +1 -0
- package/src/commands/db-prepare.ts +1 -0
- package/src/commands/deploy.ts +15 -3
- package/src/commands/desktop.ts +2 -1
- package/src/commands/dev-runner.ts +5 -0
- package/src/commands/env.ts +99 -20
- package/src/commands/lake-cleanup.ts +25 -48
- package/src/commands/lake-rejected.ts +36 -0
- package/src/commands/orchestrator.ts +2 -7
- package/src/commands/scheduler.ts +11 -61
- package/src/commands/worker-group.ts +6 -3
- package/src/index.tsx +23 -4
- package/src/lib/command-line.ts +47 -15
- package/src/{tui → lib}/connection.ts +64 -17
- package/src/lib/deploy-release.ts +42 -2
- package/src/lib/desktop.ts +8 -1
- package/src/lib/dev-supervisor.ts +7 -2
- package/src/lib/duration.ts +11 -0
- package/src/lib/loadSixb.ts +3 -4
- package/src/lib/production-roles.ts +0 -1
- package/src/lib/profiles.ts +15 -1
- package/src/lib/runtime.ts +63 -43
- package/src/lib/shareable-providers.ts +2 -2
- package/src/lib/storage-migration.ts +1 -0
- package/src/lib/worker-concurrency.ts +1 -0
- package/src/lib/worker-registry.ts +36 -1
- package/src/runner/allowed-folders.ts +126 -0
- package/src/runner/computer-connection.ts +184 -0
- package/src/runner/desktop-runner-main.ts +9 -0
- package/src/runner/desktop-runner.ts +217 -0
- package/src/runner/keep-awake.ts +50 -0
- package/src/runner/local-runner.ts +427 -0
- package/src/tui/app.tsx +168 -12
- package/src/tui/format.ts +213 -7
- package/src/tui/index.tsx +14 -13
- package/src/tui/settings.ts +12 -44
- package/src/ui/deploy-screens.ts +8 -1
- package/src/ui/env-screens.ts +39 -5
- package/src/ui/screens.ts +21 -20
- package/dist/tui/computer.d.ts.map +0 -1
- package/dist/tui/runner.d.ts +0 -75
- package/dist/tui/runner.d.ts.map +0 -1
- package/src/tui/runner.ts +0 -256
- /package/dist/{tui → runner}/computer.d.ts +0 -0
- /package/src/{tui → runner}/computer.ts +0 -0
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
import { realpath, stat } from "node:fs/promises"
|
|
2
|
+
import { homedir } from "node:os"
|
|
3
|
+
import { resolve } from "node:path"
|
|
4
|
+
import { createInterface } from "node:readline/promises"
|
|
5
|
+
import { CliError } from "@sixb/cli-core"
|
|
6
|
+
import { type Connection, connect } from "../lib/connection"
|
|
7
|
+
import { readAllowedFolders, setFoldersAllowed } from "../runner/allowed-folders"
|
|
8
|
+
import { thisComputer } from "../runner/computer"
|
|
9
|
+
import {
|
|
10
|
+
ComputerConnection,
|
|
11
|
+
type ComputerEvent,
|
|
12
|
+
EXIT_SIGNED_OUT,
|
|
13
|
+
} from "../runner/computer-connection"
|
|
14
|
+
import { stopInTime, stopOnHangUp } from "../runner/local-runner"
|
|
15
|
+
import { shortPath } from "../tui/format"
|
|
16
|
+
import { loadSettings } from "../tui/settings"
|
|
17
|
+
import { glyph, tone } from "../ui/theme"
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* `sixb connect [folder...]`: keep this computer available without `sixb` open. Turns sent from
|
|
21
|
+
* elsewhere to conversations in the allowed folders run here until this stops. It stays in the
|
|
22
|
+
* foreground: whatever should keep it running starts it, and sees what it prints.
|
|
23
|
+
*/
|
|
24
|
+
export async function runConnect(options: {
|
|
25
|
+
readonly folders: readonly string[]
|
|
26
|
+
readonly yes: boolean
|
|
27
|
+
readonly json: boolean
|
|
28
|
+
readonly apiUrl?: string
|
|
29
|
+
readonly profile?: string
|
|
30
|
+
}): Promise<void> {
|
|
31
|
+
const connection = await connect(options)
|
|
32
|
+
if ("reason" in connection) {
|
|
33
|
+
throw connection.reason === "unreachable"
|
|
34
|
+
? refusal(`[SixbCLI] Sixb isn't answering at ${connection.apiUrl}.`)
|
|
35
|
+
: refusal(
|
|
36
|
+
`[SixbCLI] Sign in first: sixb login ${connection.apiUrl || "<api-url>"}.`,
|
|
37
|
+
EXIT_SIGNED_OUT
|
|
38
|
+
)
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
await allowFolders(connection, options)
|
|
42
|
+
|
|
43
|
+
const settings = await loadSettings()
|
|
44
|
+
const report = options.json ? printJson : printLine
|
|
45
|
+
let finish: (code: number) => void = () => {}
|
|
46
|
+
const finished = new Promise<number>((resolve) => {
|
|
47
|
+
finish = resolve
|
|
48
|
+
})
|
|
49
|
+
const computer = new ComputerConnection(
|
|
50
|
+
connection,
|
|
51
|
+
(event) => {
|
|
52
|
+
if (event.type === "signed-out") finish(EXIT_SIGNED_OUT)
|
|
53
|
+
report(event)
|
|
54
|
+
},
|
|
55
|
+
settings.keepAwake
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
const unhook = stopOnHangUp(
|
|
59
|
+
() => computer,
|
|
60
|
+
(signal) => finish(signal === "SIGHUP" ? 129 : 143)
|
|
61
|
+
)
|
|
62
|
+
// Ctrl+C is how this is meant to stop; a second one doesn't wait for a turn to record its end.
|
|
63
|
+
process.once("SIGINT", () => {
|
|
64
|
+
process.once("SIGINT", () => process.exit(130))
|
|
65
|
+
void stopInTime(computer).finally(() => finish(0))
|
|
66
|
+
})
|
|
67
|
+
|
|
68
|
+
if (!options.json) {
|
|
69
|
+
const { name } = await thisComputer()
|
|
70
|
+
say(
|
|
71
|
+
`${tone.blue(glyph.mark)} ${tone.strong(name)} ${tone.faint(`connected to ${connection.project.id}. ctrl+c to stop.`)}`
|
|
72
|
+
)
|
|
73
|
+
}
|
|
74
|
+
await computer.start()
|
|
75
|
+
|
|
76
|
+
const code = await finished
|
|
77
|
+
unhook()
|
|
78
|
+
if (code === EXIT_SIGNED_OUT) await stopInTime(computer)
|
|
79
|
+
process.exit(code)
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Add what was named to the folders allowed for this project, asking first, or offer this one if
|
|
84
|
+
* there are none.
|
|
85
|
+
*/
|
|
86
|
+
async function allowFolders(
|
|
87
|
+
connection: Connection,
|
|
88
|
+
options: {
|
|
89
|
+
readonly folders: readonly string[]
|
|
90
|
+
readonly yes: boolean
|
|
91
|
+
readonly json: boolean
|
|
92
|
+
}
|
|
93
|
+
): Promise<void> {
|
|
94
|
+
const project = connection.project.id
|
|
95
|
+
const allowed = await readAllowedFolders(connection).catch((error: unknown) => {
|
|
96
|
+
throw refusal(
|
|
97
|
+
`[SixbCLI] Could not read your allowed folders. ${error instanceof Error ? error.message : ""}`
|
|
98
|
+
)
|
|
99
|
+
})
|
|
100
|
+
const named = [...new Set(await Promise.all(options.folders.map(realFolder)))]
|
|
101
|
+
const interactive = process.stdin.isTTY && !options.json
|
|
102
|
+
const newlyNamed = named.filter((folder) => !allowed.includes(folder))
|
|
103
|
+
|
|
104
|
+
if (newlyNamed.length > 0 && !options.yes && !interactive) {
|
|
105
|
+
throw refusal(
|
|
106
|
+
`[SixbCLI] ${newlyNamed.join(", ")} isn't allowed for ${project} yet: run sixb connect in a terminal to choose it, or pass --yes.`
|
|
107
|
+
)
|
|
108
|
+
}
|
|
109
|
+
const chosen: string[] = options.yes ? [...newlyNamed] : [...(await askEach(newlyNamed, project))]
|
|
110
|
+
// Nothing named and nothing allowed: this folder is the one to offer, to a person, never by --yes.
|
|
111
|
+
if (named.length === 0 && allowed.length === 0 && interactive) {
|
|
112
|
+
chosen.push(...(await askEach([await realFolder(".")], project)))
|
|
113
|
+
}
|
|
114
|
+
if (chosen.length > 0) await setFoldersAllowed(connection, chosen, true)
|
|
115
|
+
if (allowed.length + chosen.length === 0) {
|
|
116
|
+
throw refusal(
|
|
117
|
+
`[SixbCLI] No folders are allowed for ${project}, so nothing would run here. Choose one: sixb connect <folder>.`
|
|
118
|
+
)
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
async function askEach(folders: readonly string[], project: string): Promise<readonly string[]> {
|
|
123
|
+
if (folders.length === 0) return []
|
|
124
|
+
const prompt = createInterface({ input: process.stdin, output: process.stderr })
|
|
125
|
+
const chosen: string[] = []
|
|
126
|
+
try {
|
|
127
|
+
say(
|
|
128
|
+
tone.muted(
|
|
129
|
+
`Messages sent to ${project} from the web and your other devices will run in a folder you allow, on this computer, as you. The agent's commands run there without asking.`
|
|
130
|
+
)
|
|
131
|
+
)
|
|
132
|
+
for (const folder of folders) {
|
|
133
|
+
const answer = await prompt.question(
|
|
134
|
+
`Allow ${tone.strong(shown(folder))} for ${project}? [y/N] `
|
|
135
|
+
)
|
|
136
|
+
if (answer.trim().toLowerCase() === "y") chosen.push(folder)
|
|
137
|
+
}
|
|
138
|
+
} finally {
|
|
139
|
+
prompt.close()
|
|
140
|
+
}
|
|
141
|
+
return chosen
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** As the project keeps it, and the local sandbox compares it: its real path. */
|
|
145
|
+
async function realFolder(folder: string): Promise<string> {
|
|
146
|
+
const path = resolve(folder)
|
|
147
|
+
const entry = await stat(path).catch(() => null)
|
|
148
|
+
if (!entry?.isDirectory()) throw refusal(`[SixbCLI] ${path} isn't a folder.`)
|
|
149
|
+
return realpath(path)
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function refusal(message: string, exitCode = 1): CliError {
|
|
153
|
+
return new CliError({ code: "connect", message }, exitCode)
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function shown(folder: string): string {
|
|
157
|
+
return shortPath(folder, homedir(), 72) || folder
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function say(line: string): void {
|
|
161
|
+
process.stderr.write(`${line}\n`)
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function printJson(event: ComputerEvent): void {
|
|
165
|
+
process.stdout.write(`${JSON.stringify(event)}\n`)
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function printLine(event: ComputerEvent): void {
|
|
169
|
+
switch (event.type) {
|
|
170
|
+
case "folders":
|
|
171
|
+
say(
|
|
172
|
+
event.folders.length > 0
|
|
173
|
+
? ` ${tone.muted("Running messages from the web in")} ${event.folders.map(shown).join(", ")}`
|
|
174
|
+
: tone.warn(" No allowed folders exist now, so nothing runs here.")
|
|
175
|
+
)
|
|
176
|
+
return
|
|
177
|
+
case "turn.started":
|
|
178
|
+
say(` ${tone.blue(glyph.live)} ${shown(event.folder)} ${tone.faint(event.threadId)}`)
|
|
179
|
+
return
|
|
180
|
+
case "turn.ended":
|
|
181
|
+
say(
|
|
182
|
+
` ${event.ran ? tone.ok(glyph.ok) : tone.fail(glyph.fail)} ${shown(event.folder)} ${tone.faint(event.threadId)}`
|
|
183
|
+
)
|
|
184
|
+
return
|
|
185
|
+
case "session.lost":
|
|
186
|
+
say(` ${tone.warn(glyph.warn)} ${shown(event.folder)} ${tone.muted(event.message)}`)
|
|
187
|
+
return
|
|
188
|
+
case "session.back":
|
|
189
|
+
say(` ${tone.ok(glyph.ok)} ${shown(event.folder)} ${tone.muted("back online")}`)
|
|
190
|
+
return
|
|
191
|
+
case "signed-out":
|
|
192
|
+
say(tone.fail(" Your sign-in ended. Sign in again with sixb login, then reconnect."))
|
|
193
|
+
return
|
|
194
|
+
case "error":
|
|
195
|
+
say(` ${tone.fail(glyph.fail)} ${tone.muted(event.message)}`)
|
|
196
|
+
}
|
|
197
|
+
}
|
|
@@ -14,6 +14,7 @@ export async function runDbPrepare(options: { entry?: string } = {}): Promise<vo
|
|
|
14
14
|
const result = await prepareObjectQueries({
|
|
15
15
|
projectId: sixb.id,
|
|
16
16
|
ontology: sixb.definitions.ontology,
|
|
17
|
+
security: sixb.definitions.security,
|
|
17
18
|
storage: sixb.storage,
|
|
18
19
|
})
|
|
19
20
|
console.info(
|
package/src/commands/deploy.ts
CHANGED
|
@@ -10,7 +10,7 @@ import type {
|
|
|
10
10
|
DeployRelease,
|
|
11
11
|
} from "@sixb/core/deploy"
|
|
12
12
|
import { loadDeployConfig } from "../lib/deploy-config"
|
|
13
|
-
import { buildDeployRelease } from "../lib/deploy-release"
|
|
13
|
+
import { buildDeployRelease, lakeWorkChecks } from "../lib/deploy-release"
|
|
14
14
|
import {
|
|
15
15
|
commitHasProjectFile,
|
|
16
16
|
deployer,
|
|
@@ -68,7 +68,13 @@ export async function runDeploy(options: DeployOptions = {}): Promise<void> {
|
|
|
68
68
|
envFromSource,
|
|
69
69
|
})
|
|
70
70
|
if (options.json) return writeJson(release)
|
|
71
|
-
await renderStatic(
|
|
71
|
+
await renderStatic(
|
|
72
|
+
deployReleaseView({
|
|
73
|
+
release,
|
|
74
|
+
configPath: relative(cwd, path) || path,
|
|
75
|
+
warnings: configWarnings(config),
|
|
76
|
+
})
|
|
77
|
+
)
|
|
72
78
|
return
|
|
73
79
|
}
|
|
74
80
|
|
|
@@ -118,6 +124,7 @@ export async function runDeploy(options: DeployOptions = {}): Promise<void> {
|
|
|
118
124
|
release,
|
|
119
125
|
seconds: (Date.now() - deployStartedAt) / 1000,
|
|
120
126
|
keptEnv: progress.state.keptEnv,
|
|
127
|
+
warnings: configWarnings(config),
|
|
121
128
|
})
|
|
122
129
|
)
|
|
123
130
|
}
|
|
@@ -130,9 +137,10 @@ export async function runDeployCheck(options: { readonly json?: boolean; readonl
|
|
|
130
137
|
envFromSource: Boolean((await loadEnvConfig(cwd))?.source),
|
|
131
138
|
})
|
|
132
139
|
const projectPath = await projectPathOf(cwd)
|
|
133
|
-
const
|
|
140
|
+
const targetChecks = await withTask("deploy check", `Checking ${location(config)}`, () =>
|
|
134
141
|
config.target.check(release, { name: config.name, projectPath })
|
|
135
142
|
)
|
|
143
|
+
const checks = [...targetChecks, ...lakeWorkChecks(config)]
|
|
136
144
|
if (options.json) writeJson(checks)
|
|
137
145
|
else
|
|
138
146
|
await renderStatic(deployChecksView({ name: config.name, location: location(config), checks }))
|
|
@@ -300,6 +308,10 @@ async function publicKeys(value: string): Promise<string[]> {
|
|
|
300
308
|
return [value]
|
|
301
309
|
}
|
|
302
310
|
|
|
311
|
+
function configWarnings(config: DeployConfig): readonly string[] {
|
|
312
|
+
return lakeWorkChecks(config).map((check) => `${check.label}: ${check.detail} ${check.remedy}`)
|
|
313
|
+
}
|
|
314
|
+
|
|
303
315
|
function blocksDeploy(check: DeployCheck): boolean {
|
|
304
316
|
return check.status === "fixable" || check.status === "manual"
|
|
305
317
|
}
|
package/src/commands/desktop.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { join } from "node:path"
|
|
2
2
|
import { DEPLOY_CONFIG_FILE, loadDeployConfig } from "../lib/deploy-config"
|
|
3
3
|
import { deployPublicOrigins } from "../lib/deploy-release"
|
|
4
|
-
import { loadDesktop } from "../lib/desktop"
|
|
4
|
+
import { DESKTOP_RUNNER_ENTRY, loadDesktop } from "../lib/desktop"
|
|
5
5
|
import { SixbCliError } from "../lib/errors"
|
|
6
6
|
import { configuredOrigin } from "../lib/public-origin"
|
|
7
7
|
import { keyValueResultView, renderStatic } from "../ui"
|
|
@@ -20,6 +20,7 @@ export async function runDesktopBuild(options: DesktopBuildOptions = {}): Promis
|
|
|
20
20
|
projectRoot,
|
|
21
21
|
appUrl: origins.app,
|
|
22
22
|
apiOrigin: origins.api,
|
|
23
|
+
runnerEntry: DESKTOP_RUNNER_ENTRY,
|
|
23
24
|
})
|
|
24
25
|
|
|
25
26
|
await renderStatic(
|
|
@@ -113,6 +113,10 @@ export async function runDevRuntime(options: DevOptions = {}) {
|
|
|
113
113
|
port: topology.apiPort,
|
|
114
114
|
hostname: topology.apiHost,
|
|
115
115
|
quiet: true,
|
|
116
|
+
// The orchestrator this process co-hosts owns it.
|
|
117
|
+
ontologyMaintenance: false,
|
|
118
|
+
// So does the lake worker.
|
|
119
|
+
ingestFlush: false,
|
|
116
120
|
browser: {
|
|
117
121
|
publicOrigin: topology.apiPublicOrigin,
|
|
118
122
|
allowedOrigins: topology.allowedBrowserOrigins,
|
|
@@ -156,6 +160,7 @@ export async function runDevRuntime(options: DevOptions = {}) {
|
|
|
156
160
|
{ type: "pipeline", worker: runtime.pipelineWorker },
|
|
157
161
|
{ type: "workflow", worker: runtime.workflowWorker },
|
|
158
162
|
{ type: "sync", worker: runtime.syncWorker },
|
|
163
|
+
{ type: "lake", worker: runtime.lakeWorker },
|
|
159
164
|
].flatMap(({ type, worker }) => (worker ? [{ type, concurrency: worker.concurrency }] : [])),
|
|
160
165
|
warnings: devApp.warning ? [devApp.warning, ...runtime.warnings] : runtime.warnings,
|
|
161
166
|
}
|
package/src/commands/env.ts
CHANGED
|
@@ -23,6 +23,7 @@ import { errorMessage, SixbCliError } from "../lib/errors"
|
|
|
23
23
|
import { promptHidden } from "../lib/hidden-prompt"
|
|
24
24
|
import { renderStatic } from "../ui"
|
|
25
25
|
import {
|
|
26
|
+
type EnvInitConfig,
|
|
26
27
|
type EnvStatusCell,
|
|
27
28
|
envInitView,
|
|
28
29
|
envKeptView,
|
|
@@ -34,6 +35,7 @@ import { canRunLive } from "../ui/live/can-run-live"
|
|
|
34
35
|
|
|
35
36
|
const ENV_NAMES: readonly EnvName[] = ["dev", "prod"]
|
|
36
37
|
const DOPPLER_PACKAGE = "@sixb/env-doppler"
|
|
38
|
+
const SOURCE_EXAMPLE = `Add one, such as \`source: doppler({ project: "shop" })\` from ${DOPPLER_PACKAGE}, or \`source: azureKeyVault({ vaults: { dev: "shop-dev", prod: "shop-prod" } })\` from @sixb/env-azure-key-vault.`
|
|
37
39
|
|
|
38
40
|
// ─── pull ────────────────────────────────────────────────────────────────────
|
|
39
41
|
|
|
@@ -264,16 +266,25 @@ export async function runEnvStatus(options: { readonly cwd?: string } = {}): Pro
|
|
|
264
266
|
|
|
265
267
|
// ─── init ────────────────────────────────────────────────────────────────────
|
|
266
268
|
|
|
269
|
+
/**
|
|
270
|
+
* Moves a project's values to its source. Given a Doppler project, it also writes `sixb.env.ts`.
|
|
271
|
+
* Without one, it uploads to the source `sixb.env.ts` already names, whatever it is, and fills in
|
|
272
|
+
* its empty `vars: {}`.
|
|
273
|
+
*/
|
|
267
274
|
export async function runEnvInit(options: {
|
|
268
|
-
readonly project
|
|
275
|
+
readonly project?: string
|
|
269
276
|
readonly dev?: string
|
|
270
277
|
readonly prod?: string
|
|
271
278
|
readonly cwd?: string
|
|
272
279
|
}): Promise<void> {
|
|
273
280
|
const cwd = options.cwd ?? process.cwd()
|
|
274
281
|
const configPath = join(cwd, ENV_CONFIG_FILE)
|
|
275
|
-
|
|
276
|
-
|
|
282
|
+
const { project } = options
|
|
283
|
+
const configExists = await Bun.file(configPath).exists()
|
|
284
|
+
if (project !== undefined && configExists) {
|
|
285
|
+
throw new SixbCliError(`[SixbEnv] ${ENV_CONFIG_FILE} already exists.`, {
|
|
286
|
+
remediation: `To upload to the source it names, leave out the project: ${INIT_USAGE}`,
|
|
287
|
+
})
|
|
277
288
|
}
|
|
278
289
|
|
|
279
290
|
const files = [
|
|
@@ -283,11 +294,13 @@ export async function runEnvInit(options: {
|
|
|
283
294
|
const given = files.flatMap(([env, file]) => (file === undefined ? [] : [{ env, file }]))
|
|
284
295
|
if (given.length === 0) {
|
|
285
296
|
throw new SixbCliError("[SixbEnv] Say where the current values are.", {
|
|
286
|
-
remediation:
|
|
297
|
+
remediation: project === undefined ? INIT_USAGE : `sixb env init ${project} ${INIT_FILES}`,
|
|
287
298
|
})
|
|
288
299
|
}
|
|
289
300
|
|
|
290
|
-
const
|
|
301
|
+
const existing = configExists ? await loadEnvConfig(cwd) : null
|
|
302
|
+
const source =
|
|
303
|
+
project === undefined ? sourceToFill(existing) : await loadDopplerSource(cwd, project)
|
|
291
304
|
const uploads = await Promise.all(
|
|
292
305
|
given.map(async ({ env, file }) => {
|
|
293
306
|
const path = resolve(cwd, file)
|
|
@@ -298,33 +311,102 @@ export async function runEnvInit(options: {
|
|
|
298
311
|
})
|
|
299
312
|
)
|
|
300
313
|
|
|
301
|
-
// Check every environment before writing to any, so a conflict changes nothing.
|
|
302
|
-
//
|
|
314
|
+
// Check every environment before writing to any, so a conflict changes nothing. A value the
|
|
315
|
+
// source already has isn't sent again, so running this twice, or after an upload that didn't
|
|
316
|
+
// finish, changes nothing it doesn't need to.
|
|
317
|
+
const sends: { readonly env: EnvName; readonly values: Record<string, string> }[] = []
|
|
303
318
|
for (const { env, values } of uploads) {
|
|
304
|
-
const
|
|
319
|
+
const current = await source.read(env)
|
|
305
320
|
const conflicts = Object.keys(values).filter(
|
|
306
|
-
(name) => name in
|
|
321
|
+
(name) => name in current && current[name] !== values[name]
|
|
307
322
|
)
|
|
308
323
|
if (conflicts.length > 0) {
|
|
309
324
|
throw new SixbCliError(
|
|
310
|
-
`[SixbEnv] ${
|
|
325
|
+
`[SixbEnv] ${project ?? "The source"} already has other values in ${env} for ${conflicts.sort().join(", ")}.`,
|
|
311
326
|
{
|
|
312
327
|
remediation:
|
|
313
328
|
"Change them in your source's dashboard, or edit the file, and run this again.",
|
|
314
329
|
}
|
|
315
330
|
)
|
|
316
331
|
}
|
|
332
|
+
sends.push({
|
|
333
|
+
env,
|
|
334
|
+
values: Object.fromEntries(Object.entries(values).filter(([name]) => !(name in current))),
|
|
335
|
+
})
|
|
317
336
|
}
|
|
318
|
-
for (const { env, values } of
|
|
337
|
+
for (const { env, values } of sends) await source.write(env, values)
|
|
319
338
|
|
|
320
339
|
const vars = declaredVars(uploads)
|
|
321
|
-
|
|
340
|
+
let config: EnvInitConfig
|
|
341
|
+
if (project !== undefined) {
|
|
342
|
+
await writeFile(configPath, renderEnvConfig(project, vars))
|
|
343
|
+
config = { kind: "written", names: Object.keys(vars).length }
|
|
344
|
+
} else {
|
|
345
|
+
config = await fillVars(configPath, existing?.vars ?? {}, vars)
|
|
346
|
+
}
|
|
322
347
|
await renderStatic(
|
|
323
348
|
envInitView({
|
|
324
|
-
uploaded: uploads.map(({ env, values }) =>
|
|
325
|
-
|
|
349
|
+
uploaded: uploads.map(({ env, values }, index) => {
|
|
350
|
+
const sent = Object.keys(sends[index]?.values ?? {}).length
|
|
351
|
+
return { env, count: sent, already: Object.keys(values).length - sent }
|
|
352
|
+
}),
|
|
353
|
+
config,
|
|
354
|
+
})
|
|
355
|
+
)
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
const INIT_FILES = "--dev .env --prod .env.prod"
|
|
359
|
+
const INIT_USAGE = `sixb env init ${INIT_FILES}`
|
|
360
|
+
|
|
361
|
+
function sourceToFill(config: EnvConfig | null): EnvSource {
|
|
362
|
+
if (!config) {
|
|
363
|
+
throw new SixbCliError(`[SixbEnv] There's no ${ENV_CONFIG_FILE} naming a source yet.`, {
|
|
364
|
+
remediation: `Write one with its source and \`vars: {}\`, then run this again. For Doppler, \`sixb env init <project> ${INIT_FILES}\` writes it for you.`,
|
|
365
|
+
})
|
|
366
|
+
}
|
|
367
|
+
if (!config.source) {
|
|
368
|
+
throw new SixbCliError(`[SixbEnv] ${ENV_CONFIG_FILE} has no \`source\` to upload to.`, {
|
|
369
|
+
remediation: SOURCE_EXAMPLE,
|
|
326
370
|
})
|
|
371
|
+
}
|
|
372
|
+
return config.source
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
/**
|
|
376
|
+
* Declares the uploaded names in an existing `sixb.env.ts`. Only an empty `vars: {}` is replaced:
|
|
377
|
+
* anything else is the author's code, so the rules are printed for them to add instead.
|
|
378
|
+
*/
|
|
379
|
+
async function fillVars(
|
|
380
|
+
path: string,
|
|
381
|
+
declared: Readonly<Record<string, EnvVarRule>>,
|
|
382
|
+
guessed: Readonly<Record<string, EnvVarRule>>
|
|
383
|
+
): Promise<EnvInitConfig> {
|
|
384
|
+
const undeclared = Object.fromEntries(
|
|
385
|
+
Object.entries(guessed).filter(([name]) => !(name in declared))
|
|
386
|
+
)
|
|
387
|
+
const names = Object.keys(undeclared).length
|
|
388
|
+
if (names === 0) return { kind: "unchanged" }
|
|
389
|
+
|
|
390
|
+
const text = await readFile(path, "utf8")
|
|
391
|
+
const empty = [...text.matchAll(EMPTY_VARS)]
|
|
392
|
+
if (Object.keys(declared).length > 0 || empty.length !== 1) {
|
|
393
|
+
return { kind: "paste", names, lines: varLines(undeclared, "") }
|
|
394
|
+
}
|
|
395
|
+
await writeFile(
|
|
396
|
+
path,
|
|
397
|
+
text.replace(EMPTY_VARS, (_match, indent: string) => renderVars(undeclared, indent))
|
|
327
398
|
)
|
|
399
|
+
return { kind: "written", names }
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
const EMPTY_VARS = /^([ \t]*)vars:\s*\{\s*\}/gm
|
|
403
|
+
|
|
404
|
+
function renderVars(vars: Readonly<Record<string, EnvVarRule>>, indent: string): string {
|
|
405
|
+
return [`${indent}vars: {`, ...varLines(vars, `${indent} `), `${indent}}`].join("\n")
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
function varLines(vars: Readonly<Record<string, EnvVarRule>>, indent: string): string[] {
|
|
409
|
+
return Object.entries(vars).map(([name, rule]) => `${indent}${name}: "${rule}",`)
|
|
328
410
|
}
|
|
329
411
|
|
|
330
412
|
/**
|
|
@@ -366,16 +448,13 @@ function uploadableValues(
|
|
|
366
448
|
}
|
|
367
449
|
|
|
368
450
|
function renderEnvConfig(project: string, vars: Readonly<Record<string, EnvVarRule>>): string {
|
|
369
|
-
const lines = Object.entries(vars).map(([name, rule]) => ` ${name}: "${rule}",`)
|
|
370
451
|
return [
|
|
371
452
|
'import { defineEnv } from "@sixb/core/env"',
|
|
372
453
|
`import { doppler } from "${DOPPLER_PACKAGE}"`,
|
|
373
454
|
"",
|
|
374
455
|
"export default defineEnv({",
|
|
375
456
|
` source: doppler({ project: ${JSON.stringify(project)} }),`,
|
|
376
|
-
|
|
377
|
-
...lines,
|
|
378
|
-
" },",
|
|
457
|
+
`${renderVars(vars, " ")},`,
|
|
379
458
|
"})",
|
|
380
459
|
"",
|
|
381
460
|
].join("\n")
|
|
@@ -410,12 +489,12 @@ async function requireSource(
|
|
|
410
489
|
const config = await loadEnvConfig(cwd)
|
|
411
490
|
if (!config) {
|
|
412
491
|
throw new SixbCliError(`[SixbEnv] No ${ENV_CONFIG_FILE} in ${cwd}.`, {
|
|
413
|
-
remediation:
|
|
492
|
+
remediation: `sixb env init <project> ${INIT_FILES}`,
|
|
414
493
|
})
|
|
415
494
|
}
|
|
416
495
|
if (!config.source) {
|
|
417
496
|
throw new SixbCliError(`[SixbEnv] ${ENV_CONFIG_FILE} has no \`source\`.`, {
|
|
418
|
-
remediation:
|
|
497
|
+
remediation: SOURCE_EXAMPLE,
|
|
419
498
|
})
|
|
420
499
|
}
|
|
421
500
|
return { config, source: config.source }
|
|
@@ -1,79 +1,56 @@
|
|
|
1
|
+
import { getLakeRetentionRuntime } from "@sixb/core/internal/runtime"
|
|
2
|
+
import { parseDurationMs } from "../lib/duration"
|
|
1
3
|
import { loadSixbFromEntry } from "../lib/loadSixb"
|
|
2
4
|
import { resolveRuntimeEntry } from "../lib/production"
|
|
3
5
|
import { stopSixbProviders } from "../lib/runtime"
|
|
4
6
|
import { lakeCleanupView, renderStatic, withTask } from "../ui"
|
|
5
7
|
|
|
6
|
-
const DEFAULT_RETENTION = "7 days"
|
|
7
|
-
|
|
8
8
|
export interface LakeCleanupOptions {
|
|
9
9
|
entry?: string
|
|
10
10
|
dryRun?: boolean
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
interface LakeMaintenanceOptions {
|
|
16
|
-
readonly dryRun?: boolean
|
|
17
|
-
readonly expireOlderThan?: string
|
|
18
|
-
readonly deleteOlderThan?: string
|
|
19
|
-
readonly protect?: readonly { readonly datasetId: string; readonly versionId: string }[]
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
export interface LakeMaintenanceReport {
|
|
23
|
-
readonly dryRun: boolean
|
|
24
|
-
readonly expireOlderThan: string
|
|
25
|
-
readonly deleteOlderThan: string
|
|
26
|
-
readonly snapshots: number
|
|
27
|
-
/** Old snapshots kept because a version they hold is still needed, when the provider says. */
|
|
28
|
-
readonly keptSnapshots?: number
|
|
29
|
-
readonly oldFiles: number
|
|
30
|
-
readonly orphanedFiles: number
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
interface LakeMaintenanceProvider {
|
|
34
|
-
runMaintenance(options?: LakeMaintenanceOptions): Promise<LakeMaintenanceReport>
|
|
11
|
+
keepVersionsFor?: string
|
|
12
|
+
deleteFilesAfter?: string
|
|
35
13
|
}
|
|
36
14
|
|
|
37
15
|
export async function runLakeCleanup(options: LakeCleanupOptions = {}) {
|
|
16
|
+
const keepVersions = durationFlag(options.keepVersionsFor, "--keep-versions-for")
|
|
17
|
+
const deleteFiles = durationFlag(options.deleteFilesAfter, "--delete-files-after")
|
|
18
|
+
|
|
38
19
|
const entry = await resolveRuntimeEntry({ entry: options.entry })
|
|
39
20
|
const sixb = await withTask("lake cleanup", "Loading project", () => loadSixbFromEntry(entry))
|
|
40
21
|
|
|
41
22
|
try {
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
throw new Error("Configured lake storage does not support maintenance cleanup.")
|
|
23
|
+
if (sixb.lakeStorage.maintenance === undefined) {
|
|
24
|
+
throw new Error("[SixbCLI] Configured lake storage does not support retention cleanup.")
|
|
45
25
|
}
|
|
46
26
|
|
|
47
|
-
const
|
|
48
|
-
const
|
|
27
|
+
const retention = getLakeRetentionRuntime(sixb)
|
|
28
|
+
const { settings } = retention
|
|
29
|
+
const keepVersionsFor = options.keepVersionsFor ?? settings.keepVersionsFor
|
|
30
|
+
const deleteFilesAfter = options.deleteFilesAfter ?? settings.deleteFilesAfter
|
|
49
31
|
|
|
50
|
-
//
|
|
51
|
-
//
|
|
52
|
-
const applied = await sixb.storage.ontology.sources.listActive({ projectId: sixb.projectId })
|
|
32
|
+
// The same passes as automatic retention, without its time budget: they keep the versions
|
|
33
|
+
// projections applied, and run until none reports more. A dry run measures one pass.
|
|
53
34
|
const report = await withTask(
|
|
54
35
|
"lake cleanup",
|
|
55
36
|
options.dryRun ? "Measuring" : "Cleaning up",
|
|
56
37
|
() =>
|
|
57
|
-
|
|
38
|
+
retention.apply({
|
|
58
39
|
dryRun: options.dryRun ?? false,
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
protect: applied.map(({ datasetVersion }) => ({
|
|
62
|
-
datasetId: datasetVersion.datasetId,
|
|
63
|
-
versionId: datasetVersion.versionId,
|
|
64
|
-
})),
|
|
40
|
+
keepVersionsForMs: keepVersions ?? settings.keepVersionsForMs,
|
|
41
|
+
deleteFilesAfterMs: deleteFiles ?? settings.deleteFilesAfterMs,
|
|
65
42
|
})
|
|
66
43
|
)
|
|
67
|
-
|
|
44
|
+
|
|
45
|
+
await renderStatic(
|
|
46
|
+
lakeCleanupView({ projectId: sixb.id, keepVersionsFor, deleteFilesAfter, report })
|
|
47
|
+
)
|
|
68
48
|
} finally {
|
|
69
49
|
await stopSixbProviders(sixb)
|
|
70
50
|
}
|
|
71
51
|
}
|
|
72
52
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
value !== null &&
|
|
77
|
-
typeof (value as { runMaintenance?: unknown }).runMaintenance === "function"
|
|
78
|
-
)
|
|
53
|
+
/** Read before anything loads, so that a typo fails at once. */
|
|
54
|
+
function durationFlag(value: string | undefined, flag: string): number | null {
|
|
55
|
+
return value === undefined ? null : parseDurationMs(value, flag)
|
|
79
56
|
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { loadSixbFromEntry } from "../lib/loadSixb"
|
|
2
|
+
import { resolveRuntimeEntry } from "../lib/production"
|
|
3
|
+
import { stopSixbProviders } from "../lib/runtime"
|
|
4
|
+
import { withTask } from "../ui"
|
|
5
|
+
|
|
6
|
+
/** A week of refusals is rarely this many; past it, ask for one dataset. */
|
|
7
|
+
const LIMIT = 500
|
|
8
|
+
|
|
9
|
+
export interface LakeRejectedOptions {
|
|
10
|
+
entry?: string
|
|
11
|
+
dataset?: string
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Prints, as JSON, the deliveries refused after `ingest` answered: what each held, whole, and why,
|
|
16
|
+
* so that someone can fix them and send them again. They stay for a week.
|
|
17
|
+
*/
|
|
18
|
+
export async function runLakeRejected(options: LakeRejectedOptions = {}) {
|
|
19
|
+
const entry = await resolveRuntimeEntry({ entry: options.entry })
|
|
20
|
+
const sixb = await withTask("lake rejected", "Loading project", () => loadSixbFromEntry(entry))
|
|
21
|
+
|
|
22
|
+
try {
|
|
23
|
+
const inbox = sixb.storage.datasetIngest
|
|
24
|
+
if (!inbox) throw new Error("[SixbCLI] Configured storage keeps no ingested changes.")
|
|
25
|
+
|
|
26
|
+
const rejected = await inbox.listRejected({
|
|
27
|
+
projectId: sixb.id,
|
|
28
|
+
limit: LIMIT,
|
|
29
|
+
...(options.dataset === undefined ? {} : { datasetId: options.dataset }),
|
|
30
|
+
})
|
|
31
|
+
|
|
32
|
+
process.stdout.write(`${JSON.stringify(rejected, null, 2)}\n`)
|
|
33
|
+
} finally {
|
|
34
|
+
await stopSixbProviders(sixb)
|
|
35
|
+
}
|
|
36
|
+
}
|
|
@@ -33,19 +33,14 @@ export async function runOrchestrator(options: OrchestratorOptions = {}) {
|
|
|
33
33
|
|
|
34
34
|
runtime = await startOrchestratorRuntime(sixb)
|
|
35
35
|
|
|
36
|
-
const warnings = [...runtime.warnings]
|
|
37
|
-
if (runtime.orchestratorWorker === null) {
|
|
38
|
-
warnings.push("No orchestrator routes are registered; the orchestrator process is idle.")
|
|
39
|
-
}
|
|
40
|
-
|
|
41
36
|
app.stop()
|
|
42
37
|
await renderStatic(
|
|
43
38
|
roleView({
|
|
44
39
|
command: "orchestrator",
|
|
45
40
|
name: sixb.id,
|
|
46
|
-
items: [{ label: "role", value: "event
|
|
41
|
+
items: [{ label: "role", value: "event dispatch, cron and upkeep" }],
|
|
47
42
|
storage: migration.summary,
|
|
48
|
-
warnings,
|
|
43
|
+
warnings: runtime.warnings,
|
|
49
44
|
})
|
|
50
45
|
)
|
|
51
46
|
|