@hyperfixation/cli 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.d.ts +2 -2
- package/dist/cli.js +45 -12
- package/dist/cloud-steps/template.d.ts +2 -7
- package/dist/cloud-steps/template.js +12 -12
- package/dist/doctor.d.ts +22 -1
- package/dist/doctor.js +141 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/new.d.ts +24 -0
- package/dist/new.js +38 -2
- package/dist/restore-check.d.ts +33 -4
- package/dist/restore-check.js +71 -16
- package/dist/roles.d.ts +1 -0
- package/dist/roles.js +1 -1
- package/dist/state.d.ts +8 -0
- package/dist/state.js +1 -0
- package/dist/version.d.ts +8 -0
- package/dist/version.js +13 -0
- package/package.json +6 -6
package/dist/cli.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
export declare const COMMANDS: readonly ["new", "migrate", "bootstrap", "status-token", "check", "gen", "dev", "up", "deploy", "doctor", "restore-check"];
|
|
1
|
+
export declare const COMMANDS: readonly ["new", "migrate", "bootstrap", "status-token", "check", "gen", "dev", "up", "deploy", "doctor", "restore-check", "version"];
|
|
2
2
|
export type Command = (typeof COMMANDS)[number];
|
|
3
|
-
export declare const USAGE = "hf \u2014 the hyperfixation CLI\n\n hf new <name> provision the app in the cloud: fetch the template, push a private\n repo, register a backup, Sentry, Langfuse and DNS, create the\n database and its roles, create the Coolify application and its\n environment, deploy, and print what is left to do by hand.\n Resumable \u2014 a rerun repeats only what did not finish\n --budget-usd <amount> the app's monthly LLM budget (required; no default)\n --email <address> the bootstrap admin's address (required)\n --from <specifier> template to fetch (default: gh:grahamlutz/hyperfixation-template)\n --into <dir> where to create <name> (default: the working directory)\n\n hf new <name> --local copy the template into ./<name>, substitute its placeholders, and\n
|
|
3
|
+
export declare const USAGE = "hf \u2014 the hyperfixation CLI\n\n hf new <name> provision the app in the cloud: fetch the template, push a private\n repo, register a backup, Sentry, Langfuse and DNS, create the\n database and its roles, create the Coolify application and its\n environment, deploy, and print what is left to do by hand.\n Resumable \u2014 a rerun repeats only what did not finish\n --budget-usd <amount> the app's monthly LLM budget (required; no default)\n --email <address> the bootstrap admin's address (required)\n --from <specifier> template to fetch (default: gh:grahamlutz/hyperfixation-template)\n --into <dir> where to create <name> (default: the working directory)\n\n hf new <name> --local copy the template into ./<name>, substitute its placeholders, and\n write both bootstrap answers into its .env\n --budget-usd <amount> the app's monthly LLM budget (required; no default)\n --email <address> the bootstrap admin's address (required)\n --from <dir> template checkout (default: the sibling hyperfixation-template)\n --into <dir> where to create <name> (default: the working directory)\n\n hf up install, infra, migrate, bootstrap, status tokens, then hf dev \u2014\n the whole local loop after hf new, safe to rerun; seeds a $10\n budget unless HF_BOOTSTRAP_BUDGET_USD is set in .env\n\n hf migrate create the application role, then run the app's migrate.ts\n --skip-roles the cloud path, where the roles already exist\n\n hf bootstrap grant the app its one bootstrap admin, and seed hf_app_state\n --email <address> the address to promote; otherwise HF_BOOTSTRAP_EMAIL\n --budget-usd <amount> the app's starting budget; otherwise HF_BOOTSTRAP_BUDGET_USD\n\n hf status-token provision /api/status's read and write tokens\n --read only the read token; refuses if it is already set\n --write only the write token; refuses if it is already set\n --rotate replace a token that is already set\n (no flags: fills in whichever of the two is unset)\n\n hf check declared env, pending migrations, and E001-E006\n\n hf deploy <name> set SOURCE_COMMIT to a commit, deploy it, and wait until /api/status\n reports it. Coolify's push auto-deploy is off, so this is what\n publishes a merge to main\n --sha <sha> the commit to deploy (default: main's, via git ls-remote)\n\n hf doctor [name] every deployed app in the state cache, or one: /api/status under its\n read token, the deployed version against main, E006 as the app role,\n the box's connection slots, the worker's advisory lock, the last\n restore check, and open core-bump PRs. Exits 1 on any finding\n\n hf gen [generator] the app's turbo generators\n\n hf dev docker compose up, then pnpm dev under HF_BUILD_SHA=dev-<timestamp>\n --no-compose leave the dev infrastructure alone\n --compose-only bring the infrastructure up and stop\n\n hf restore-check <name> restore the newest hf_<name> dump beside the live database and\n compare row counts. An append-only table the live side has moved\n on from reads ok (drift +N); exits 1 on any mismatch, and on a\n dump older than 24 h\n --backup-dir <dir> where the dumps are (default: Coolify's on the box)\n --from-s3 read the dump from object storage (not implemented)\n --strict compare every table exactly; no table may drift\n\n hf version the @hyperfixation/cli version behind this hf, also as --version, -v\n\nEvery command but `new`, `deploy`, `doctor` and `restore-check` runs against the app at or above the working directory, or --dir.\n";
|
|
4
4
|
export interface Io {
|
|
5
5
|
out(line: string): void;
|
|
6
6
|
err(line: string): void;
|
package/dist/cli.js
CHANGED
|
@@ -12,6 +12,7 @@ import { formatRestoreCheck, restoreCheckApp } from "./restore-check.js";
|
|
|
12
12
|
import { statusTokenApp } from "./status-token.js";
|
|
13
13
|
import { requireTemplateSource } from "./template-source.js";
|
|
14
14
|
import { DEV_BUDGET_USD, upApp } from "./up.js";
|
|
15
|
+
import { cliVersion } from "./version.js";
|
|
15
16
|
export const COMMANDS = [
|
|
16
17
|
"new",
|
|
17
18
|
"migrate",
|
|
@@ -24,6 +25,7 @@ export const COMMANDS = [
|
|
|
24
25
|
"deploy",
|
|
25
26
|
"doctor",
|
|
26
27
|
"restore-check",
|
|
28
|
+
"version",
|
|
27
29
|
];
|
|
28
30
|
export const USAGE = `hf — the hyperfixation CLI
|
|
29
31
|
|
|
@@ -38,10 +40,11 @@ export const USAGE = `hf — the hyperfixation CLI
|
|
|
38
40
|
--into <dir> where to create <name> (default: the working directory)
|
|
39
41
|
|
|
40
42
|
hf new <name> --local copy the template into ./<name>, substitute its placeholders, and
|
|
41
|
-
|
|
43
|
+
write both bootstrap answers into its .env
|
|
44
|
+
--budget-usd <amount> the app's monthly LLM budget (required; no default)
|
|
45
|
+
--email <address> the bootstrap admin's address (required)
|
|
42
46
|
--from <dir> template checkout (default: the sibling hyperfixation-template)
|
|
43
47
|
--into <dir> where to create <name> (default: the working directory)
|
|
44
|
-
--email <address> the bootstrap admin's address; skips the prompt
|
|
45
48
|
|
|
46
49
|
hf up install, infra, migrate, bootstrap, status tokens, then hf dev —
|
|
47
50
|
the whole local loop after hf new, safe to rerun; seeds a $10
|
|
@@ -69,7 +72,8 @@ export const USAGE = `hf — the hyperfixation CLI
|
|
|
69
72
|
|
|
70
73
|
hf doctor [name] every deployed app in the state cache, or one: /api/status under its
|
|
71
74
|
read token, the deployed version against main, E006 as the app role,
|
|
72
|
-
the
|
|
75
|
+
the box's connection slots, the worker's advisory lock, the last
|
|
76
|
+
restore check, and open core-bump PRs. Exits 1 on any finding
|
|
73
77
|
|
|
74
78
|
hf gen [generator] the app's turbo generators
|
|
75
79
|
|
|
@@ -78,9 +82,14 @@ export const USAGE = `hf — the hyperfixation CLI
|
|
|
78
82
|
--compose-only bring the infrastructure up and stop
|
|
79
83
|
|
|
80
84
|
hf restore-check <name> restore the newest hf_<name> dump beside the live database and
|
|
81
|
-
compare row counts
|
|
85
|
+
compare row counts. An append-only table the live side has moved
|
|
86
|
+
on from reads ok (drift +N); exits 1 on any mismatch, and on a
|
|
87
|
+
dump older than 24 h
|
|
82
88
|
--backup-dir <dir> where the dumps are (default: Coolify's on the box)
|
|
83
89
|
--from-s3 read the dump from object storage (not implemented)
|
|
90
|
+
--strict compare every table exactly; no table may drift
|
|
91
|
+
|
|
92
|
+
hf version the @hyperfixation/cli version behind this hf, also as --version, -v
|
|
84
93
|
|
|
85
94
|
Every command but \`new\`, \`deploy\`, \`doctor\` and \`restore-check\` runs against the app at or above the working directory, or --dir.
|
|
86
95
|
`;
|
|
@@ -101,6 +110,10 @@ export async function main(argv, io = consoleIo) {
|
|
|
101
110
|
io.out(USAGE);
|
|
102
111
|
return command === undefined ? 1 : 0;
|
|
103
112
|
}
|
|
113
|
+
if (command === "--version" || command === "-v") {
|
|
114
|
+
io.out(await cliVersion());
|
|
115
|
+
return 0;
|
|
116
|
+
}
|
|
104
117
|
if (!COMMANDS.includes(command)) {
|
|
105
118
|
io.err(`unknown command ${JSON.stringify(command)}`);
|
|
106
119
|
io.err(USAGE);
|
|
@@ -138,6 +151,9 @@ async function dispatch(command, argv, io) {
|
|
|
138
151
|
return await commandDoctor(argv, io);
|
|
139
152
|
case "restore-check":
|
|
140
153
|
return await commandRestoreCheck(argv, io);
|
|
154
|
+
case "version":
|
|
155
|
+
io.out(await cliVersion());
|
|
156
|
+
return 0;
|
|
141
157
|
}
|
|
142
158
|
}
|
|
143
159
|
async function commandNew(argv, io) {
|
|
@@ -160,6 +176,15 @@ async function commandNew(argv, io) {
|
|
|
160
176
|
if (!values.local) {
|
|
161
177
|
return await commandNewCloud(name, values, io);
|
|
162
178
|
}
|
|
179
|
+
// Refused here rather than prompted for: `hf up` reads both out of the app's `.env`, so a
|
|
180
|
+
// local app started without them is the same half-answered app the cloud path has refused
|
|
181
|
+
// since Phase 3 — one that either cannot spend or cannot stop.
|
|
182
|
+
const missing = missingBootstrapFlags(values);
|
|
183
|
+
if (missing.length > 0) {
|
|
184
|
+
io.err(`hf new ${name} --local needs ${missing.join(" and ")}: hf up reads both from the app's ` +
|
|
185
|
+
".env and neither has a default.");
|
|
186
|
+
return 1;
|
|
187
|
+
}
|
|
163
188
|
const from = values.from ?? (await requireTemplateSource());
|
|
164
189
|
const result = await newApp({
|
|
165
190
|
name,
|
|
@@ -167,29 +192,35 @@ async function commandNew(argv, io) {
|
|
|
167
192
|
into: values.into,
|
|
168
193
|
local: values.local,
|
|
169
194
|
email: values.email,
|
|
195
|
+
budgetUsd: values["budget-usd"],
|
|
170
196
|
});
|
|
171
197
|
io.out(`created ${result.dir} from ${from}`);
|
|
172
198
|
io.out(` app ${result.appName}, database ${result.databaseName}`);
|
|
173
199
|
io.out(` ${result.substituted.length} file(s) substituted` +
|
|
174
200
|
(result.wroteEnv ? ", .env written from .env.example" : "") +
|
|
175
|
-
(result.wroteBootstrapEmail ? ", HF_BOOTSTRAP_EMAIL set" : "")
|
|
201
|
+
(result.wroteBootstrapEmail ? ", HF_BOOTSTRAP_EMAIL set" : "") +
|
|
202
|
+
(result.wroteBootstrapBudget ? ", HF_BOOTSTRAP_BUDGET_USD set" : ""));
|
|
176
203
|
io.out("");
|
|
177
204
|
io.out(`next: cd ${result.given} && hf up`);
|
|
178
205
|
return 0;
|
|
179
206
|
}
|
|
180
207
|
/**
|
|
181
|
-
* The
|
|
208
|
+
* The two answers `hf new` cannot invent, named in the order the usage lists them.
|
|
182
209
|
*
|
|
183
|
-
*
|
|
184
|
-
*
|
|
185
|
-
*
|
|
186
|
-
*
|
|
210
|
+
* `--email` designates the one admin an app is ever granted without an admin behind it, and a
|
|
211
|
+
* budget nobody chose is an app that either cannot spend or cannot stop — `hf bootstrap` has
|
|
212
|
+
* refused an unset one since Phase 1, and this is the same rule one command earlier, where the
|
|
213
|
+
* answer costs nothing yet.
|
|
187
214
|
*/
|
|
188
|
-
|
|
189
|
-
|
|
215
|
+
function missingBootstrapFlags(values) {
|
|
216
|
+
return [
|
|
190
217
|
...(values["budget-usd"] === undefined ? ["--budget-usd <amount>"] : []),
|
|
191
218
|
...(values.email === undefined ? ["--email <address>"] : []),
|
|
192
219
|
];
|
|
220
|
+
}
|
|
221
|
+
/** The cloud half: both inputs are refused up front, with nowhere else to carry them. */
|
|
222
|
+
async function commandNewCloud(name, values, io) {
|
|
223
|
+
const missing = missingBootstrapFlags(values);
|
|
193
224
|
if (missing.length > 0) {
|
|
194
225
|
io.err(`hf new ${name} needs ${missing.join(" and ")}: a cloud app has no prompt and no .env to ` +
|
|
195
226
|
"carry either. Pass --local for Phase 1's local copy.");
|
|
@@ -342,6 +373,7 @@ async function commandRestoreCheck(argv, io) {
|
|
|
342
373
|
options: {
|
|
343
374
|
"backup-dir": { type: "string" },
|
|
344
375
|
"from-s3": { type: "boolean", default: false },
|
|
376
|
+
strict: { type: "boolean", default: false },
|
|
345
377
|
},
|
|
346
378
|
allowPositionals: true,
|
|
347
379
|
});
|
|
@@ -354,6 +386,7 @@ async function commandRestoreCheck(argv, io) {
|
|
|
354
386
|
app: name,
|
|
355
387
|
backupDir: values["backup-dir"],
|
|
356
388
|
fromS3: values["from-s3"],
|
|
389
|
+
strict: values.strict,
|
|
357
390
|
});
|
|
358
391
|
for (const line of formatRestoreCheck(result))
|
|
359
392
|
io.out(line);
|
|
@@ -1,12 +1,7 @@
|
|
|
1
1
|
import type { Step } from "../new-cloud.js";
|
|
2
|
+
import { templateTempDir } from "../new.js";
|
|
2
3
|
import { type CloudStepContext } from "./context.js";
|
|
3
|
-
|
|
4
|
-
* Where the fetch lands before it becomes the app.
|
|
5
|
-
*
|
|
6
|
-
* Beside the target rather than under `os.tmpdir()`, so the rename is a rename and not a second
|
|
7
|
-
* copy across filesystems, and dot-prefixed so a half-fetched tree does not look like an app.
|
|
8
|
-
*/
|
|
9
|
-
export declare function templateTempDir(dir: string): string;
|
|
4
|
+
export { templateTempDir };
|
|
10
5
|
/**
|
|
11
6
|
* The app's files: giget's fetch of the template, substituted, renamed into place.
|
|
12
7
|
*
|
|
@@ -1,16 +1,8 @@
|
|
|
1
1
|
import { readFile, rename, rm } from "node:fs/promises";
|
|
2
2
|
import path from "node:path";
|
|
3
|
-
import { substituteTree, TEMPLATE_MARKER, TemplateError } from "../new.js";
|
|
3
|
+
import { scratchInTheWay, substituteTree, targetInTheWay, templateTempDir, TEMPLATE_MARKER, TemplateError, } from "../new.js";
|
|
4
4
|
import { exists } from "./context.js";
|
|
5
|
-
|
|
6
|
-
* Where the fetch lands before it becomes the app.
|
|
7
|
-
*
|
|
8
|
-
* Beside the target rather than under `os.tmpdir()`, so the rename is a rename and not a second
|
|
9
|
-
* copy across filesystems, and dot-prefixed so a half-fetched tree does not look like an app.
|
|
10
|
-
*/
|
|
11
|
-
export function templateTempDir(dir) {
|
|
12
|
-
return path.join(path.dirname(dir), `.${path.basename(dir)}.hf-new`);
|
|
13
|
-
}
|
|
5
|
+
export { templateTempDir };
|
|
14
6
|
/**
|
|
15
7
|
* The app's files: giget's fetch of the template, substituted, renamed into place.
|
|
16
8
|
*
|
|
@@ -32,7 +24,15 @@ export const templateStep = {
|
|
|
32
24
|
return;
|
|
33
25
|
}
|
|
34
26
|
const temp = templateTempDir(dir);
|
|
35
|
-
await
|
|
27
|
+
if (await exists(temp)) {
|
|
28
|
+
// Cleared only on a resume, by the same rule the target directory gets: `fetchStartedAt`
|
|
29
|
+
// says a run of *this* app fetched into that path, and without it the directory is someone
|
|
30
|
+
// else's however much its name looks like ours.
|
|
31
|
+
if (context.state.state.templateFetchStartedAt === undefined)
|
|
32
|
+
throw scratchInTheWay(temp);
|
|
33
|
+
await rm(temp, { recursive: true, force: true });
|
|
34
|
+
}
|
|
35
|
+
await context.state.patch({ templateFetchStartedAt: new Date(context.now()).toISOString() });
|
|
36
36
|
const fetched = await context.fetchTemplate(context.from, temp);
|
|
37
37
|
await substituteTree(fetched, names);
|
|
38
38
|
// The marker is what `assertTemplateSource` looks for: an app is never a template twice.
|
|
@@ -66,7 +66,7 @@ async function adoptOrRefuse(context) {
|
|
|
66
66
|
}
|
|
67
67
|
const substituted = !(await exists(path.join(dir, TEMPLATE_MARKER))) && (await packageName(dir)) === names.appName;
|
|
68
68
|
if (!substituted) {
|
|
69
|
-
throw
|
|
69
|
+
throw targetInTheWay(dir);
|
|
70
70
|
}
|
|
71
71
|
context.io.out(`${names.given}: adopting the app directory already at ${dir}`);
|
|
72
72
|
}
|
package/dist/doctor.d.ts
CHANGED
|
@@ -1,15 +1,23 @@
|
|
|
1
1
|
import { type OperatorConfig } from "./config.js";
|
|
2
|
+
import { type Database } from "./database.js";
|
|
2
3
|
import type { FetchLike } from "./providers/http.js";
|
|
3
4
|
import { type Runner } from "./runner.js";
|
|
4
5
|
/** A restore check older than this is a warning: E5 is meant to run weekly, not once. */
|
|
5
6
|
export declare const RESTORE_CHECK_MAX_AGE_DAYS = 7;
|
|
7
|
+
/** The `CONNECTION LIMIT` every application role is created with, in `provisionRoles`. */
|
|
8
|
+
export declare const APPLICATION_ROLE_CONNECTION_LIMIT = 25;
|
|
9
|
+
/** Past this share of `max_connections`, the next app to deploy is the one that cannot connect. */
|
|
10
|
+
export declare const CONNECTIONS_WARN_FRACTION = 0.8;
|
|
6
11
|
/** The branch prefix Phase 4's core bumps open their pull requests on. */
|
|
7
12
|
export declare const CORE_BUMP_BRANCH_PREFIX = "core-bump/";
|
|
8
13
|
export type Severity = "ok" | "warn" | "fail";
|
|
9
14
|
export interface DoctorFinding {
|
|
10
15
|
/** The app as the state cache names it. */
|
|
11
16
|
app: string;
|
|
12
|
-
/**
|
|
17
|
+
/**
|
|
18
|
+
* `state`, `status`, `runs`, `version`, `budget`, `E006`, `connections`, `lock`,
|
|
19
|
+
* `restore-check`, `core-bump`.
|
|
20
|
+
*/
|
|
13
21
|
check: string;
|
|
14
22
|
severity: Severity;
|
|
15
23
|
message: string;
|
|
@@ -39,6 +47,11 @@ export interface DoctorOptions {
|
|
|
39
47
|
now?: () => Date;
|
|
40
48
|
/** How E006 is read. Defaults to the tunnel to `HF_SSH_HOST` as `postgres`. */
|
|
41
49
|
privileges?: PrivilegeCheck;
|
|
50
|
+
/**
|
|
51
|
+
* Where the connection counts and the worker locks are read: the whole cluster, as the admin
|
|
52
|
+
* E006 already goes in as. Defaults to the same tunnel to `HF_SSH_HOST`.
|
|
53
|
+
*/
|
|
54
|
+
database?: () => Promise<Database>;
|
|
42
55
|
env?: NodeJS.ProcessEnv;
|
|
43
56
|
}
|
|
44
57
|
/**
|
|
@@ -70,3 +83,11 @@ export declare function tunnelPrivilegeCheck(runner: Runner, options?: {
|
|
|
70
83
|
* privilege query is a second thing to keep in step with the grants the migrator makes.
|
|
71
84
|
*/
|
|
72
85
|
export declare function checkAppRolePrivileges(adminUrl: string, role: string): Promise<void>;
|
|
86
|
+
/**
|
|
87
|
+
* The worker's advisory lock in one app's database: exactly one, under the key the worker takes.
|
|
88
|
+
*
|
|
89
|
+
* `pg_try_advisory_lock(bigint)` splits its key across `classid` and `objid`, so the key is
|
|
90
|
+
* reassembled rather than compared whole — and masked rather than only shifted, because
|
|
91
|
+
* `hashtext` answers `int4` and a negative hash widens to a bigint of sign bits.
|
|
92
|
+
*/
|
|
93
|
+
export declare function workerLockSql(appName: string): string;
|
package/dist/doctor.js
CHANGED
|
@@ -6,10 +6,15 @@ import { DEFAULT_PG_ADMIN_USER, loadOperatorConfig, pgAdminUser, postgresContain
|
|
|
6
6
|
import { openDatabase } from "./database.js";
|
|
7
7
|
import { deriveNames } from "./names.js";
|
|
8
8
|
import { GithubClient } from "./providers/github.js";
|
|
9
|
+
import { quoteLiteral } from "./roles.js";
|
|
9
10
|
import { createSshRunner } from "./runner.js";
|
|
10
11
|
import { openAppState, stateDir } from "./state.js";
|
|
11
12
|
/** A restore check older than this is a warning: E5 is meant to run weekly, not once. */
|
|
12
13
|
export const RESTORE_CHECK_MAX_AGE_DAYS = 7;
|
|
14
|
+
/** The `CONNECTION LIMIT` every application role is created with, in `provisionRoles`. */
|
|
15
|
+
export const APPLICATION_ROLE_CONNECTION_LIMIT = 25;
|
|
16
|
+
/** Past this share of `max_connections`, the next app to deploy is the one that cannot connect. */
|
|
17
|
+
export const CONNECTIONS_WARN_FRACTION = 0.8;
|
|
13
18
|
/** The branch prefix Phase 4's core bumps open their pull requests on. */
|
|
14
19
|
export const CORE_BUMP_BRANCH_PREFIX = "core-bump/";
|
|
15
20
|
/**
|
|
@@ -27,6 +32,7 @@ export async function doctor(options = {}) {
|
|
|
27
32
|
const config = options.config ?? (await loadOperatorConfig({ env }));
|
|
28
33
|
const required = requireOperatorConfig(config, ["HF_BASE_DOMAIN", "HF_GITHUB_TOKEN"], { env });
|
|
29
34
|
const dir = options.stateDir ?? stateDir(env);
|
|
35
|
+
const cluster = lazyDatabase(options.database ?? defaultDatabase(config, env));
|
|
30
36
|
const context = {
|
|
31
37
|
dir,
|
|
32
38
|
env,
|
|
@@ -35,11 +41,20 @@ export async function doctor(options = {}) {
|
|
|
35
41
|
fetch: options.fetch ?? ((input, init) => globalThis.fetch(input, init)),
|
|
36
42
|
now: options.now ?? (() => new Date()),
|
|
37
43
|
privileges: options.privileges ?? defaultPrivilegeCheck(config, env),
|
|
44
|
+
database: cluster.get,
|
|
45
|
+
// One snapshot for the whole run: the counts are the box's, not any one app's, and an app
|
|
46
|
+
// whose line is read a second later has not moved the cluster.
|
|
47
|
+
backends: once(async () => await readBackends(await cluster.get())),
|
|
38
48
|
};
|
|
39
49
|
const names = options.name === undefined ? await stateNames(dir) : [options.name];
|
|
40
50
|
const findings = [];
|
|
41
|
-
|
|
42
|
-
|
|
51
|
+
try {
|
|
52
|
+
for (const name of names)
|
|
53
|
+
findings.push(...(await doctorApp(context, name)));
|
|
54
|
+
}
|
|
55
|
+
finally {
|
|
56
|
+
await cluster.close();
|
|
57
|
+
}
|
|
43
58
|
return { findings, ok: findings.every((finding) => finding.severity === "ok") };
|
|
44
59
|
}
|
|
45
60
|
const MARKER = { ok: "OK ", warn: "WARN", fail: "FAIL" };
|
|
@@ -109,6 +124,30 @@ function defaultPrivilegeCheck(config, env) {
|
|
|
109
124
|
adminUser: pgAdminUser(config),
|
|
110
125
|
});
|
|
111
126
|
}
|
|
127
|
+
function defaultDatabase(config, env) {
|
|
128
|
+
const { HF_SSH_HOST } = requireOperatorConfig(config, ["HF_SSH_HOST"], { env });
|
|
129
|
+
const runner = createSshRunner({ host: HF_SSH_HOST });
|
|
130
|
+
return async () => await openDatabase(runner, {
|
|
131
|
+
admin: { user: pgAdminUser(config) },
|
|
132
|
+
containers: postgresContainers(config),
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
/** One cluster connection for the whole run: opened when a check first needs it, closed once. */
|
|
136
|
+
function lazyDatabase(open) {
|
|
137
|
+
let pending;
|
|
138
|
+
return {
|
|
139
|
+
get: () => (pending ??= open()),
|
|
140
|
+
close: async () => {
|
|
141
|
+
// A run whose every app failed before the first query never opened one, and an open that
|
|
142
|
+
// failed is already a finding.
|
|
143
|
+
await pending?.then(async (db) => await db.close(), () => undefined);
|
|
144
|
+
},
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
function once(read) {
|
|
148
|
+
let pending;
|
|
149
|
+
return () => (pending ??= read());
|
|
150
|
+
}
|
|
112
151
|
async function doctorApp(context, name) {
|
|
113
152
|
const findings = [];
|
|
114
153
|
const add = (check, severity, message) => {
|
|
@@ -148,6 +187,13 @@ async function doctorApp(context, name) {
|
|
|
148
187
|
budgetFinding(report.budget.previous, "previous", add);
|
|
149
188
|
}
|
|
150
189
|
await privilegeFindings(context, name, add);
|
|
190
|
+
// A name `deriveNames` refuses has already failed E006 on that same message; the cluster checks
|
|
191
|
+
// have no names to run under and say nothing more.
|
|
192
|
+
const names = tryNames(name);
|
|
193
|
+
if (names !== undefined) {
|
|
194
|
+
await connectionFindings(context, names.applicationRole, add);
|
|
195
|
+
await lockFindings(context, names, add);
|
|
196
|
+
}
|
|
151
197
|
restoreCheckFindings(context, state, add);
|
|
152
198
|
if (repo !== undefined)
|
|
153
199
|
await bumpFindings(context, repo, add);
|
|
@@ -294,6 +340,99 @@ async function privilegeFindings(context, name, add) {
|
|
|
294
340
|
add("E006", "fail", flatten(error.message));
|
|
295
341
|
}
|
|
296
342
|
}
|
|
343
|
+
/** Every backend belonging to an app role — `hf_<app>` and its `_migrator` and `_ro`. */
|
|
344
|
+
const BACKENDS_SQL = "SELECT usename, count(*) FROM pg_stat_activity WHERE usename LIKE 'hf\\_%' GROUP BY usename";
|
|
345
|
+
async function readBackends(db) {
|
|
346
|
+
const setting = (await db.query("SHOW max_connections")).rows[0]?.[0];
|
|
347
|
+
const byRole = new Map();
|
|
348
|
+
for (const row of (await db.query(BACKENDS_SQL)).rows) {
|
|
349
|
+
if (row[0] !== undefined)
|
|
350
|
+
byRole.set(row[0], Number(row[1]));
|
|
351
|
+
}
|
|
352
|
+
let total = 0;
|
|
353
|
+
for (const count of byRole.values())
|
|
354
|
+
total += count;
|
|
355
|
+
return { max: setting === undefined ? undefined : numeric(Number(setting)), total, byRole };
|
|
356
|
+
}
|
|
357
|
+
/**
|
|
358
|
+
* What the box's connection slots are spent on, and what this app has of them.
|
|
359
|
+
*
|
|
360
|
+
* Box-wide rather than per-app because that is where it runs out: every app on the box draws on
|
|
361
|
+
* one `max_connections`, and the app that then cannot connect is whichever one deploys next.
|
|
362
|
+
*/
|
|
363
|
+
async function connectionFindings(context, role, add) {
|
|
364
|
+
let backends;
|
|
365
|
+
try {
|
|
366
|
+
backends = await context.backends();
|
|
367
|
+
}
|
|
368
|
+
catch (error) {
|
|
369
|
+
add("connections", "fail", flatten(error.message));
|
|
370
|
+
return;
|
|
371
|
+
}
|
|
372
|
+
const limit = String(APPLICATION_ROLE_CONNECTION_LIMIT);
|
|
373
|
+
const line = `${role} ${String(backends.byRole.get(role) ?? 0)}/${limit}, box ` +
|
|
374
|
+
`${String(backends.total)}/${backends.max === undefined ? UNKNOWN : String(backends.max)} ` +
|
|
375
|
+
"on hf_ roles";
|
|
376
|
+
const crowded = backends.max !== undefined && backends.total > backends.max * CONNECTIONS_WARN_FRACTION;
|
|
377
|
+
add("connections", crowded ? "warn" : "ok", crowded
|
|
378
|
+
? `${line} — over ${String(CONNECTIONS_WARN_FRACTION * 100)}% of max_connections`
|
|
379
|
+
: line);
|
|
380
|
+
}
|
|
381
|
+
/**
|
|
382
|
+
* The worker's advisory lock in one app's database: exactly one, under the key the worker takes.
|
|
383
|
+
*
|
|
384
|
+
* `pg_try_advisory_lock(bigint)` splits its key across `classid` and `objid`, so the key is
|
|
385
|
+
* reassembled rather than compared whole — and masked rather than only shifted, because
|
|
386
|
+
* `hashtext` answers `int4` and a negative hash widens to a bigint of sign bits.
|
|
387
|
+
*/
|
|
388
|
+
export function workerLockSql(appName) {
|
|
389
|
+
return (`WITH k AS (SELECT hashtext('hf-worker:' || ${quoteLiteral(appName)})::bigint AS value) ` +
|
|
390
|
+
"SELECT count(l.pid), count(l.pid) FILTER (WHERE " +
|
|
391
|
+
"l.classid = ((k.value >> 32) & 4294967295)::oid AND " +
|
|
392
|
+
"l.objid = (k.value & 4294967295)::oid) " +
|
|
393
|
+
"FROM k LEFT JOIN pg_locks AS l ON l.locktype = 'advisory' AND " +
|
|
394
|
+
"l.database = (SELECT oid FROM pg_database WHERE datname = current_database())");
|
|
395
|
+
}
|
|
396
|
+
async function lockFindings(context, names, add) {
|
|
397
|
+
let row;
|
|
398
|
+
try {
|
|
399
|
+
const db = await context.database();
|
|
400
|
+
row = (await db.query(workerLockSql(names.appName), { database: names.databaseName })).rows[0];
|
|
401
|
+
}
|
|
402
|
+
catch (error) {
|
|
403
|
+
add("lock", "fail", flatten(error.message));
|
|
404
|
+
return;
|
|
405
|
+
}
|
|
406
|
+
const key = `hf-worker:${names.appName}`;
|
|
407
|
+
const held = Number(row?.[0]);
|
|
408
|
+
const matching = Number(row?.[1]);
|
|
409
|
+
if (!Number.isFinite(held)) {
|
|
410
|
+
add("lock", "fail", `pg_locks in ${names.databaseName} answered no count`);
|
|
411
|
+
return;
|
|
412
|
+
}
|
|
413
|
+
if (held === 0) {
|
|
414
|
+
add("lock", "fail", `no advisory lock in ${names.databaseName}: no worker holds ${key}`);
|
|
415
|
+
return;
|
|
416
|
+
}
|
|
417
|
+
if (held !== 1) {
|
|
418
|
+
add("lock", "fail", `${String(held)} advisory locks in ${names.databaseName}; one worker per app holds one`);
|
|
419
|
+
return;
|
|
420
|
+
}
|
|
421
|
+
if (matching !== 1) {
|
|
422
|
+
add("lock", "fail", `the one advisory lock in ${names.databaseName} is not hashtext('${key}'): ` +
|
|
423
|
+
"something other than the worker holds it");
|
|
424
|
+
return;
|
|
425
|
+
}
|
|
426
|
+
add("lock", "ok", `one worker holds ${key} in ${names.databaseName}`);
|
|
427
|
+
}
|
|
428
|
+
function tryNames(name) {
|
|
429
|
+
try {
|
|
430
|
+
return deriveNames(name);
|
|
431
|
+
}
|
|
432
|
+
catch {
|
|
433
|
+
return undefined;
|
|
434
|
+
}
|
|
435
|
+
}
|
|
297
436
|
function restoreCheckFindings(context, state, add) {
|
|
298
437
|
const last = state.lastRestoreCheckAt;
|
|
299
438
|
if (last === undefined) {
|
package/dist/index.d.ts
CHANGED
|
@@ -17,5 +17,5 @@ export { run, CommandFailed, type RunOptions } from "./spawn.js";
|
|
|
17
17
|
export { createLocalRunner, createSshRunner, shellQuote, sshExecArgv, sshTunnelArgv, RunnerError, DEFAULT_TUNNEL_READY_TIMEOUT_MS, type ExecOptions, type ExecResult, type LocalRunner, type LocalRunnerOptions, type Runner, type SshRunnerOptions, type Tunnel, } from "./runner.js";
|
|
18
18
|
export { findPostgresContainer, openDatabase, openDatabaseUrl, redactPasswords, DatabaseTransportError, DEFAULT_POSTGRES_PORT, type AdminCredentials, type Database, type DatabaseTransport, type OpenDatabaseOptions, type QueryOptions, type QueryResult, } from "./database.js";
|
|
19
19
|
export { createLocalDirectoryBackupSource, createS3BackupSource, BackupSourceError, COOLIFY_BACKUP_DIR, type BackupDump, type BackupSource, type BackupSourceKind, type LocalDirectoryBackupSourceOptions, } from "./backup-source.js";
|
|
20
|
-
export { formatRestoreCheck, pgRestoreArgv, pgRestoreInContainerArgv, restoreCheck, restoreCheckApp, RestoreCheckError, SCRATCH_SUFFIX, STALE_DUMP_HOURS, type RestoreCheckAppOptions, type RestoreCheckOptions, type RestoreCheckResult, type RestoreCheckRow, type RestoreVerdict, } from "./restore-check.js";
|
|
20
|
+
export { APPEND_ONLY_TABLES, formatRestoreCheck, pgRestoreArgv, pgRestoreInContainerArgv, restoreCheck, restoreCheckApp, RestoreCheckError, SCRATCH_SUFFIX, STALE_DUMP_HOURS, type RestoreCheckAppOptions, type RestoreCheckOptions, type RestoreCheckResult, type RestoreCheckRow, type RestoreVerdict, } from "./restore-check.js";
|
|
21
21
|
export { provisionDatabase, ProvisionDatabaseError, REQUIRED_EXTENSIONS, type ProvisionDatabaseOptions, type ProvisionDatabaseResult, } from "./provision-database.js";
|
package/dist/index.js
CHANGED
|
@@ -17,5 +17,5 @@ export { run, CommandFailed } from "./spawn.js";
|
|
|
17
17
|
export { createLocalRunner, createSshRunner, shellQuote, sshExecArgv, sshTunnelArgv, RunnerError, DEFAULT_TUNNEL_READY_TIMEOUT_MS, } from "./runner.js";
|
|
18
18
|
export { findPostgresContainer, openDatabase, openDatabaseUrl, redactPasswords, DatabaseTransportError, DEFAULT_POSTGRES_PORT, } from "./database.js";
|
|
19
19
|
export { createLocalDirectoryBackupSource, createS3BackupSource, BackupSourceError, COOLIFY_BACKUP_DIR, } from "./backup-source.js";
|
|
20
|
-
export { formatRestoreCheck, pgRestoreArgv, pgRestoreInContainerArgv, restoreCheck, restoreCheckApp, RestoreCheckError, SCRATCH_SUFFIX, STALE_DUMP_HOURS, } from "./restore-check.js";
|
|
20
|
+
export { APPEND_ONLY_TABLES, formatRestoreCheck, pgRestoreArgv, pgRestoreInContainerArgv, restoreCheck, restoreCheckApp, RestoreCheckError, SCRATCH_SUFFIX, STALE_DUMP_HOURS, } from "./restore-check.js";
|
|
21
21
|
export { provisionDatabase, ProvisionDatabaseError, REQUIRED_EXTENSIONS, } from "./provision-database.js";
|
package/dist/new.d.ts
CHANGED
|
@@ -14,6 +14,23 @@ export declare const EXCLUDED_ENTRIES: readonly string[];
|
|
|
14
14
|
export declare class TemplateError extends Error {
|
|
15
15
|
constructor(message: string);
|
|
16
16
|
}
|
|
17
|
+
/**
|
|
18
|
+
* Where a cloud `hf new` fetches the template before it becomes the app.
|
|
19
|
+
*
|
|
20
|
+
* Beside the target rather than under `os.tmpdir()`, so the rename is a rename and not a second
|
|
21
|
+
* copy across filesystems, and dot-prefixed so a half-fetched tree does not look like an app.
|
|
22
|
+
*/
|
|
23
|
+
export declare function templateTempDir(dir: string): string;
|
|
24
|
+
/**
|
|
25
|
+
* The two refusals a directory in the way earns, worded once because both halves of `hf new`
|
|
26
|
+
* raise them.
|
|
27
|
+
*
|
|
28
|
+
* Each names the absolute path and the single move that clears it: "already exists" alone leaves
|
|
29
|
+
* the operator to work out which of the app directory and the dot-prefixed scratch beside it is
|
|
30
|
+
* meant, and they are one keystroke apart.
|
|
31
|
+
*/
|
|
32
|
+
export declare function targetInTheWay(dir: string): TemplateError;
|
|
33
|
+
export declare function scratchInTheWay(scratch: string): TemplateError;
|
|
17
34
|
export interface NewAppOptions {
|
|
18
35
|
/** The name as typed; becomes the directory and, underscored, both placeholders. */
|
|
19
36
|
name: string;
|
|
@@ -29,6 +46,11 @@ export interface NewAppOptions {
|
|
|
29
46
|
local: boolean;
|
|
30
47
|
/** The bootstrap admin's address, written to `.env` as `HF_BOOTSTRAP_EMAIL`. Skips the prompt. */
|
|
31
48
|
email?: string;
|
|
49
|
+
/**
|
|
50
|
+
* The app's monthly LLM budget, written to `.env` as `HF_BOOTSTRAP_BUDGET_USD`. `hf up` seeds
|
|
51
|
+
* the app state from there instead of falling back to its dev default.
|
|
52
|
+
*/
|
|
53
|
+
budgetUsd?: string;
|
|
32
54
|
/** Overrides the real interactive prompt; for tests and other callers with their own stdin. */
|
|
33
55
|
promptEmail?: () => Promise<string>;
|
|
34
56
|
}
|
|
@@ -41,6 +63,8 @@ export interface NewAppResult extends AppNames {
|
|
|
41
63
|
wroteEnv: boolean;
|
|
42
64
|
/** True when `HF_BOOTSTRAP_EMAIL` was written to `.env`, from `--email` or the prompt. */
|
|
43
65
|
wroteBootstrapEmail: boolean;
|
|
66
|
+
/** True when `HF_BOOTSTRAP_BUDGET_USD` was written to `.env`, from `--budget-usd`. */
|
|
67
|
+
wroteBootstrapBudget: boolean;
|
|
44
68
|
}
|
|
45
69
|
/**
|
|
46
70
|
* Copies the template checkout and substitutes the two placeholders across it.
|
package/dist/new.js
CHANGED
|
@@ -46,6 +46,31 @@ export class TemplateError extends Error {
|
|
|
46
46
|
this.name = "TemplateError";
|
|
47
47
|
}
|
|
48
48
|
}
|
|
49
|
+
/**
|
|
50
|
+
* Where a cloud `hf new` fetches the template before it becomes the app.
|
|
51
|
+
*
|
|
52
|
+
* Beside the target rather than under `os.tmpdir()`, so the rename is a rename and not a second
|
|
53
|
+
* copy across filesystems, and dot-prefixed so a half-fetched tree does not look like an app.
|
|
54
|
+
*/
|
|
55
|
+
export function templateTempDir(dir) {
|
|
56
|
+
return path.join(path.dirname(dir), `.${path.basename(dir)}.hf-new`);
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* The two refusals a directory in the way earns, worded once because both halves of `hf new`
|
|
60
|
+
* raise them.
|
|
61
|
+
*
|
|
62
|
+
* Each names the absolute path and the single move that clears it: "already exists" alone leaves
|
|
63
|
+
* the operator to work out which of the app directory and the dot-prefixed scratch beside it is
|
|
64
|
+
* meant, and they are one keystroke apart.
|
|
65
|
+
*/
|
|
66
|
+
export function targetInTheWay(dir) {
|
|
67
|
+
return new TemplateError(`${dir} already exists; hf new will not write into it. Move it away (or delete it) and rerun — ` +
|
|
68
|
+
"the template is fetched into a directory of its own.");
|
|
69
|
+
}
|
|
70
|
+
export function scratchInTheWay(scratch) {
|
|
71
|
+
return new TemplateError(`${scratch} is a leftover hf new scratch directory and hf has no record of creating it. ` +
|
|
72
|
+
"Remove it and rerun; the template is fetched into it fresh.");
|
|
73
|
+
}
|
|
49
74
|
/**
|
|
50
75
|
* Copies the template checkout and substitutes the two placeholders across it.
|
|
51
76
|
*
|
|
@@ -64,7 +89,13 @@ export async function newApp(options) {
|
|
|
64
89
|
await assertTemplateSource(source);
|
|
65
90
|
const dir = path.resolve(options.into ?? process.cwd(), names.given);
|
|
66
91
|
if (await exists(dir)) {
|
|
67
|
-
throw
|
|
92
|
+
throw targetInTheWay(dir);
|
|
93
|
+
}
|
|
94
|
+
// A scratch directory here is an interrupted cloud run for this same name: copying an app over
|
|
95
|
+
// the top of it would leave that run's rerun to judge a directory neither flow made.
|
|
96
|
+
const scratch = templateTempDir(dir);
|
|
97
|
+
if (await exists(scratch)) {
|
|
98
|
+
throw scratchInTheWay(scratch);
|
|
68
99
|
}
|
|
69
100
|
await cp(source, dir, {
|
|
70
101
|
recursive: true,
|
|
@@ -75,6 +106,7 @@ export async function newApp(options) {
|
|
|
75
106
|
const example = path.join(dir, ".env.example");
|
|
76
107
|
const wroteEnv = await exists(example);
|
|
77
108
|
let wroteBootstrapEmail = false;
|
|
109
|
+
let wroteBootstrapBudget = false;
|
|
78
110
|
if (wroteEnv) {
|
|
79
111
|
let contents = await readFile(example, "utf8");
|
|
80
112
|
const email = options.email ?? (await (options.promptEmail ?? promptForBootstrapEmail)());
|
|
@@ -82,9 +114,13 @@ export async function newApp(options) {
|
|
|
82
114
|
contents = `${contents.trimEnd()}\nHF_BOOTSTRAP_EMAIL=${email}\n`;
|
|
83
115
|
wroteBootstrapEmail = true;
|
|
84
116
|
}
|
|
117
|
+
if (options.budgetUsd !== undefined && options.budgetUsd !== "") {
|
|
118
|
+
contents = `${contents.trimEnd()}\nHF_BOOTSTRAP_BUDGET_USD=${options.budgetUsd}\n`;
|
|
119
|
+
wroteBootstrapBudget = true;
|
|
120
|
+
}
|
|
85
121
|
await writeFile(path.join(dir, ".env"), contents);
|
|
86
122
|
}
|
|
87
|
-
return { ...names, dir, substituted, wroteEnv, wroteBootstrapEmail };
|
|
123
|
+
return { ...names, dir, substituted, wroteEnv, wroteBootstrapEmail, wroteBootstrapBudget };
|
|
88
124
|
}
|
|
89
125
|
/** The one-time prompt: the address `hf up` later hands `hf bootstrap` via `.env`. */
|
|
90
126
|
async function promptForBootstrapEmail() {
|
package/dist/restore-check.d.ts
CHANGED
|
@@ -4,8 +4,21 @@ import { type Runner } from "./runner.js";
|
|
|
4
4
|
import { type AppStateStore } from "./state.js";
|
|
5
5
|
/** Appended to `hf_<app>` for the database the dump is restored into and then dropped. */
|
|
6
6
|
export declare const SCRATCH_SUFFIX = "_restore_check";
|
|
7
|
-
/** Older than this and the dump gets a
|
|
8
|
-
export declare const STALE_DUMP_HOURS =
|
|
7
|
+
/** Older than this and the dump gets a WARN line, which — as in `hf doctor` — exits 1. */
|
|
8
|
+
export declare const STALE_DUMP_HOURS = 24;
|
|
9
|
+
/**
|
|
10
|
+
* Tables a live row is only ever added to, so a live count above the dump's is the app working,
|
|
11
|
+
* not lost data.
|
|
12
|
+
*
|
|
13
|
+
* X1 found this the hard way: against a 1.3-hour-old dump of a running app, 9 of 24 tables
|
|
14
|
+
* "mismatched" purely from churn since the dump, and the same check against a fresh dump matched
|
|
15
|
+
* all 24. A table earns a place here only when no code path deletes from it and none updates it
|
|
16
|
+
* in a way that lowers its count — checked against `packages/db/src/schema` and every statement
|
|
17
|
+
* in `core`, `workflows`, `auth` and `admin`. Anything else, including a table whose rows merely
|
|
18
|
+
* look permanent, stays exact: a false `ok` here hides exactly the data loss this command exists
|
|
19
|
+
* to catch.
|
|
20
|
+
*/
|
|
21
|
+
export declare const APPEND_ONLY_TABLES: readonly string[];
|
|
9
22
|
export type RestoreVerdict = "ok" | "mismatch" | "live only" | "restored only";
|
|
10
23
|
export interface RestoreCheckRow {
|
|
11
24
|
table: string;
|
|
@@ -13,6 +26,14 @@ export interface RestoreCheckRow {
|
|
|
13
26
|
live?: number;
|
|
14
27
|
/** Absent when the table is not in the restored dump. */
|
|
15
28
|
restored?: number;
|
|
29
|
+
/**
|
|
30
|
+
* Rows the live side gained since the dump, on an `APPEND_ONLY_TABLES` table.
|
|
31
|
+
*
|
|
32
|
+
* Drift is not its own verdict because it is not its own outcome: the restore held everything
|
|
33
|
+
* the dump had, which is `ok`. The column prints it as `ok (drift +N)` so the operator can see
|
|
34
|
+
* why two counts differ without having to decide whether it mattered.
|
|
35
|
+
*/
|
|
36
|
+
drift?: number;
|
|
16
37
|
verdict: RestoreVerdict;
|
|
17
38
|
}
|
|
18
39
|
export interface RestoreCheckResult {
|
|
@@ -21,11 +42,15 @@ export interface RestoreCheckResult {
|
|
|
21
42
|
scratchDatabase: string;
|
|
22
43
|
dump: BackupDump;
|
|
23
44
|
dumpAgeHours: number;
|
|
24
|
-
/** The dump is older than `STALE_DUMP_HOURS
|
|
45
|
+
/** The dump is older than `STALE_DUMP_HOURS`; on its own enough to exit 1. */
|
|
25
46
|
dumpStale: boolean;
|
|
47
|
+
/** Exact matching was asked for, so no table was allowed to drift. */
|
|
48
|
+
strict: boolean;
|
|
26
49
|
/** One row per table, `hf_*` or carrying `normalized_name`, sorted by name. */
|
|
27
50
|
rows: readonly RestoreCheckRow[];
|
|
28
|
-
/** Every row's verdict is `ok
|
|
51
|
+
/** Every row's verdict is `ok`, drift included. What `lastRestoreCheckAt` is written on. */
|
|
52
|
+
matched: boolean;
|
|
53
|
+
/** `matched` and the dump is not stale. The command's exit code is `ok ? 0 : 1`. */
|
|
29
54
|
ok: boolean;
|
|
30
55
|
}
|
|
31
56
|
export declare class RestoreCheckError extends Error {
|
|
@@ -57,6 +82,8 @@ export interface RestoreCheckOptions {
|
|
|
57
82
|
restoreAdminUrl?: string;
|
|
58
83
|
/** The `pg_restore` binary inside the container. */
|
|
59
84
|
pgRestorePath?: string;
|
|
85
|
+
/** Compare every table exactly, `APPEND_ONLY_TABLES` included. */
|
|
86
|
+
strict?: boolean;
|
|
60
87
|
now?: Date;
|
|
61
88
|
}
|
|
62
89
|
/**
|
|
@@ -104,6 +131,8 @@ export interface RestoreCheckAppOptions {
|
|
|
104
131
|
backupDir?: string;
|
|
105
132
|
/** Read the dump from Hetzner object storage instead. Not implemented; see `backup-source`. */
|
|
106
133
|
fromS3?: boolean;
|
|
134
|
+
/** Compare every table exactly, `APPEND_ONLY_TABLES` included. */
|
|
135
|
+
strict?: boolean;
|
|
107
136
|
env?: NodeJS.ProcessEnv;
|
|
108
137
|
}
|
|
109
138
|
/**
|
package/dist/restore-check.js
CHANGED
|
@@ -10,8 +10,40 @@ import { openAppState } from "./state.js";
|
|
|
10
10
|
export const SCRATCH_SUFFIX = "_restore_check";
|
|
11
11
|
/** Postgres truncates an identifier past this, which would collide with the live database. */
|
|
12
12
|
const MAX_IDENTIFIER_BYTES = 63;
|
|
13
|
-
/** Older than this and the dump gets a
|
|
14
|
-
export const STALE_DUMP_HOURS =
|
|
13
|
+
/** Older than this and the dump gets a WARN line, which — as in `hf doctor` — exits 1. */
|
|
14
|
+
export const STALE_DUMP_HOURS = 24;
|
|
15
|
+
/**
|
|
16
|
+
* Tables a live row is only ever added to, so a live count above the dump's is the app working,
|
|
17
|
+
* not lost data.
|
|
18
|
+
*
|
|
19
|
+
* X1 found this the hard way: against a 1.3-hour-old dump of a running app, 9 of 24 tables
|
|
20
|
+
* "mismatched" purely from churn since the dump, and the same check against a fresh dump matched
|
|
21
|
+
* all 24. A table earns a place here only when no code path deletes from it and none updates it
|
|
22
|
+
* in a way that lowers its count — checked against `packages/db/src/schema` and every statement
|
|
23
|
+
* in `core`, `workflows`, `auth` and `admin`. Anything else, including a table whose rows merely
|
|
24
|
+
* look permanent, stays exact: a false `ok` here hides exactly the data loss this command exists
|
|
25
|
+
* to catch.
|
|
26
|
+
*/
|
|
27
|
+
export const APPEND_ONLY_TABLES = [
|
|
28
|
+
// Insert-only ledgers: nothing but `INSERT` touches either.
|
|
29
|
+
"hf_audit",
|
|
30
|
+
"hf_activity",
|
|
31
|
+
// Ledger rows are inserted `started` and then `UPDATE`d to a terminal status — including
|
|
32
|
+
// `reconcile()`'s sweep to `abandoned`/`uncertain`, which is still an update.
|
|
33
|
+
"hf_llm_call",
|
|
34
|
+
"hf_action_log",
|
|
35
|
+
// Inserted pending and decided by `UPDATE`; `reconcile()` step (5) expires a stale one the
|
|
36
|
+
// same way. The delete guard exists precisely so an approval outlives the record it names.
|
|
37
|
+
"hf_approval",
|
|
38
|
+
// Inserted, or upserted on `(run_id, key, spec_name)` by a replayed step; never deleted.
|
|
39
|
+
"hf_score",
|
|
40
|
+
// `INSERT` at the start of a source run, `UPDATE` at its end.
|
|
41
|
+
"hf_source_run",
|
|
42
|
+
// `INSERT` at `runs.start`; every later write is an `UPDATE` of status, attempt or the
|
|
43
|
+
// fencing token. Runs are never purged — there is no retention sweep.
|
|
44
|
+
"hf_run",
|
|
45
|
+
];
|
|
46
|
+
const APPEND_ONLY = new Set(APPEND_ONLY_TABLES);
|
|
15
47
|
export class RestoreCheckError extends Error {
|
|
16
48
|
constructor(message) {
|
|
17
49
|
super(message);
|
|
@@ -72,22 +104,27 @@ export async function restoreCheck(options) {
|
|
|
72
104
|
`TO ${quoteIdent(names.migratorRole)}`);
|
|
73
105
|
await scratch.query(`GRANT CREATE, USAGE ON SCHEMA public TO ${quoteIdent(names.migratorRole)}`);
|
|
74
106
|
await runRestore(options, scratchDatabase, names.migratorRole, dump.path);
|
|
75
|
-
rows = compare(await countTables(db, names.databaseName), await countTables(scratch));
|
|
107
|
+
rows = compare(await countTables(db, names.databaseName), await countTables(scratch), options.strict ?? false);
|
|
76
108
|
}
|
|
77
109
|
finally {
|
|
78
110
|
await scratch.close();
|
|
79
111
|
}
|
|
80
|
-
const
|
|
81
|
-
|
|
112
|
+
const dumpStale = dumpAgeHours > STALE_DUMP_HOURS;
|
|
113
|
+
const matched = rows.every((row) => row.verdict === "ok");
|
|
114
|
+
// A stale dump keeps the exit code but not the timestamp: the restore itself was proved,
|
|
115
|
+
// and it is `hf doctor` that decides how long a proof stays good.
|
|
116
|
+
if (matched)
|
|
82
117
|
await options.state.patch({ lastRestoreCheckAt: now.toISOString() });
|
|
83
118
|
return {
|
|
84
119
|
databaseName: names.databaseName,
|
|
85
120
|
scratchDatabase,
|
|
86
121
|
dump,
|
|
87
122
|
dumpAgeHours,
|
|
88
|
-
dumpStale
|
|
123
|
+
dumpStale,
|
|
124
|
+
strict: options.strict ?? false,
|
|
89
125
|
rows,
|
|
90
|
-
|
|
126
|
+
matched,
|
|
127
|
+
ok: matched && !dumpStale,
|
|
91
128
|
};
|
|
92
129
|
}
|
|
93
130
|
finally {
|
|
@@ -203,23 +240,35 @@ async function countTables(db, database) {
|
|
|
203
240
|
* A table on one side only is its own verdict rather than a crash or a zero: an app migration
|
|
204
241
|
* between the backup and the check is the ordinary reason for it, and reading it as a count of
|
|
205
242
|
* zero would make an added table look like lost data.
|
|
243
|
+
*
|
|
244
|
+
* An `APPEND_ONLY_TABLES` table the live side is *ahead* on is `ok` with a `drift`, unless
|
|
245
|
+
* `strict`. A restored count above the live one is still a `mismatch` there: the dump cannot
|
|
246
|
+
* hold rows an append-only live table has since lost unless something did lose them.
|
|
206
247
|
*/
|
|
207
|
-
function compare(live, restored) {
|
|
248
|
+
function compare(live, restored, strict) {
|
|
208
249
|
const tables = [...new Set([...live.keys(), ...restored.keys()])].sort();
|
|
209
250
|
return tables.map((table) => {
|
|
210
251
|
const liveCount = live.get(table);
|
|
211
252
|
const restoredCount = restored.get(table);
|
|
253
|
+
const drifted = !strict &&
|
|
254
|
+
APPEND_ONLY.has(table) &&
|
|
255
|
+
liveCount !== undefined &&
|
|
256
|
+
restoredCount !== undefined &&
|
|
257
|
+
restoredCount < liveCount;
|
|
212
258
|
const verdict = liveCount === undefined
|
|
213
259
|
? "restored only"
|
|
214
260
|
: restoredCount === undefined
|
|
215
261
|
? "live only"
|
|
216
|
-
: liveCount === restoredCount
|
|
262
|
+
: liveCount === restoredCount || drifted
|
|
217
263
|
? "ok"
|
|
218
264
|
: "mismatch";
|
|
219
265
|
return {
|
|
220
266
|
table,
|
|
221
267
|
...(liveCount === undefined ? {} : { live: liveCount }),
|
|
222
268
|
...(restoredCount === undefined ? {} : { restored: restoredCount }),
|
|
269
|
+
...(drifted && liveCount !== undefined && restoredCount !== undefined
|
|
270
|
+
? { drift: liveCount - restoredCount }
|
|
271
|
+
: {}),
|
|
223
272
|
verdict,
|
|
224
273
|
};
|
|
225
274
|
});
|
|
@@ -227,24 +276,29 @@ function compare(live, restored) {
|
|
|
227
276
|
/** The table `hf restore-check` prints, and the two lines around it. */
|
|
228
277
|
export function formatRestoreCheck(result) {
|
|
229
278
|
const age = `${result.dumpAgeHours.toFixed(1)} h old`;
|
|
230
|
-
const lines = [
|
|
279
|
+
const lines = [
|
|
280
|
+
`${result.databaseName}: ${result.dump.path}, ${age}${result.strict ? ", strict" : ""}`,
|
|
281
|
+
];
|
|
231
282
|
if (result.dumpStale) {
|
|
232
|
-
lines.push(`
|
|
283
|
+
lines.push(`WARN: the dump is ${age} — over ${String(STALE_DUMP_HOURS)} h; ` +
|
|
284
|
+
"this compares against stale data");
|
|
233
285
|
}
|
|
234
286
|
const header = ["table", "live", "restored", "verdict"];
|
|
235
287
|
const cells = result.rows.map((row) => [
|
|
236
288
|
row.table,
|
|
237
289
|
row.live === undefined ? "—" : String(row.live),
|
|
238
290
|
row.restored === undefined ? "—" : String(row.restored),
|
|
239
|
-
row.verdict
|
|
291
|
+
row.drift === undefined ? row.verdict : `${row.verdict} (drift +${String(row.drift)})`,
|
|
240
292
|
]);
|
|
241
293
|
const widths = header.map((name, column) => Math.max(name.length, ...cells.map((row) => row[column]?.length ?? 0)));
|
|
242
294
|
const line = (row) => row.map((cell, column) => cell.padEnd(widths[column] ?? 0)).join(" ").trimEnd();
|
|
243
295
|
lines.push(line(header), ...cells.map(line));
|
|
244
|
-
const
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
296
|
+
const drifted = result.rows.filter((row) => row.drift !== undefined).length;
|
|
297
|
+
const failed = result.rows.filter((row) => row.verdict !== "ok").length;
|
|
298
|
+
const drift = drifted === 0 ? "" : `, ${String(drifted)} with drift since the dump`;
|
|
299
|
+
lines.push(result.matched
|
|
300
|
+
? `${String(result.rows.length)} table(s) matched${drift}`
|
|
301
|
+
: `${String(failed)} of ${String(result.rows.length)} table(s) did not match${drift}`);
|
|
248
302
|
return lines;
|
|
249
303
|
}
|
|
250
304
|
/**
|
|
@@ -273,6 +327,7 @@ export async function restoreCheckApp(options) {
|
|
|
273
327
|
database: db,
|
|
274
328
|
container: await restoreContainer(runner, db, containers),
|
|
275
329
|
adminUser: admin.user,
|
|
330
|
+
strict: options.strict,
|
|
276
331
|
});
|
|
277
332
|
}
|
|
278
333
|
finally {
|
package/dist/roles.d.ts
CHANGED
package/dist/roles.js
CHANGED
|
@@ -46,6 +46,6 @@ export function credentialsOf(connectionString) {
|
|
|
46
46
|
const url = new URL(connectionString);
|
|
47
47
|
return { user: decodeURIComponent(url.username), password: decodeURIComponent(url.password) };
|
|
48
48
|
}
|
|
49
|
-
function quoteLiteral(value) {
|
|
49
|
+
export function quoteLiteral(value) {
|
|
50
50
|
return `'${value.replace(/'/g, "''")}'`;
|
|
51
51
|
}
|
package/dist/state.d.ts
CHANGED
|
@@ -59,6 +59,14 @@ export interface AppState {
|
|
|
59
59
|
* earlier `hf new --local` looks exactly like a lost run.
|
|
60
60
|
*/
|
|
61
61
|
templateStartedAt?: string;
|
|
62
|
+
/**
|
|
63
|
+
* ISO 8601, written just before the `template` step fetches into the scratch directory beside
|
|
64
|
+
* the app's path.
|
|
65
|
+
*
|
|
66
|
+
* The same proof one step earlier: a scratch directory the state does not vouch for is refused
|
|
67
|
+
* rather than deleted, and a fetch that died half way is this run's own to clear and redo.
|
|
68
|
+
*/
|
|
69
|
+
templateFetchStartedAt?: string;
|
|
62
70
|
/** `owner/name` of the app's GitHub repository. */
|
|
63
71
|
repo?: string;
|
|
64
72
|
coolify?: CoolifyState;
|
package/dist/state.js
CHANGED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This package's own version, read from its `package.json` at runtime.
|
|
3
|
+
*
|
|
4
|
+
* Not a constant the build stamps in: the nine packages are one fixed version group, and a
|
|
5
|
+
* release rewrites `package.json` alone — a baked-in string would be a second copy that is wrong
|
|
6
|
+
* from the next release onwards. `../package.json` resolves the same from `src/` and from `dist/`.
|
|
7
|
+
*/
|
|
8
|
+
export declare function cliVersion(): Promise<string>;
|
package/dist/version.js
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { fileURLToPath } from "node:url";
|
|
3
|
+
/**
|
|
4
|
+
* This package's own version, read from its `package.json` at runtime.
|
|
5
|
+
*
|
|
6
|
+
* Not a constant the build stamps in: the nine packages are one fixed version group, and a
|
|
7
|
+
* release rewrites `package.json` alone — a baked-in string would be a second copy that is wrong
|
|
8
|
+
* from the next release onwards. `../package.json` resolves the same from `src/` and from `dist/`.
|
|
9
|
+
*/
|
|
10
|
+
export async function cliVersion() {
|
|
11
|
+
const manifest = JSON.parse(await readFile(fileURLToPath(new URL("../package.json", import.meta.url)), "utf8"));
|
|
12
|
+
return typeof manifest.version === "string" ? manifest.version : "unknown";
|
|
13
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hyperfixation/cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.8",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"description": "The hf binary and its Turborepo generator templates",
|
|
6
6
|
"repository": {
|
|
@@ -29,15 +29,15 @@
|
|
|
29
29
|
"!dist/test-support/**"
|
|
30
30
|
],
|
|
31
31
|
"dependencies": {
|
|
32
|
-
"@hyperfixation/auth": "0.1.
|
|
33
|
-
"@hyperfixation/core": "0.1.
|
|
34
|
-
"@hyperfixation/db": "0.1.
|
|
32
|
+
"@hyperfixation/auth": "0.1.8",
|
|
33
|
+
"@hyperfixation/core": "0.1.8",
|
|
34
|
+
"@hyperfixation/db": "0.1.8",
|
|
35
35
|
"giget": "3.3.1",
|
|
36
36
|
"pg": "^8.23.0"
|
|
37
37
|
},
|
|
38
38
|
"devDependencies": {
|
|
39
|
-
"@hyperfixation/eslint-config": "0.1.
|
|
40
|
-
"@hyperfixation/testing": "0.1.
|
|
39
|
+
"@hyperfixation/eslint-config": "0.1.8",
|
|
40
|
+
"@hyperfixation/testing": "0.1.8",
|
|
41
41
|
"@microsoft/api-extractor": "^7.59.1",
|
|
42
42
|
"@types/pg": "^8.23.1",
|
|
43
43
|
"eslint": "^10.10.0",
|