@hyperfixation/cli 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app.d.ts +15 -2
- package/dist/app.js +4 -2
- package/dist/backup-source.d.ts +47 -0
- package/dist/backup-source.js +107 -0
- package/dist/bootstrap.d.ts +2 -0
- package/dist/bootstrap.js +1 -1
- package/dist/checklist.d.ts +25 -0
- package/dist/checklist.js +32 -0
- package/dist/cli.d.ts +2 -2
- package/dist/cli.js +95 -2
- package/dist/cloud-steps/backup.d.ts +17 -0
- package/dist/cloud-steps/backup.js +40 -0
- package/dist/cloud-steps/context.d.ts +120 -0
- package/dist/cloud-steps/context.js +88 -0
- package/dist/cloud-steps/coolify.d.ts +74 -0
- package/dist/cloud-steps/coolify.js +300 -0
- package/dist/cloud-steps/database.d.ts +12 -0
- package/dist/cloud-steps/database.js +25 -0
- package/dist/cloud-steps/deploy.d.ts +18 -0
- package/dist/cloud-steps/deploy.js +110 -0
- package/dist/cloud-steps/dns.d.ts +11 -0
- package/dist/cloud-steps/dns.js +53 -0
- package/dist/cloud-steps/index.d.ts +21 -0
- package/dist/cloud-steps/index.js +30 -0
- package/dist/cloud-steps/install.d.ts +12 -0
- package/dist/cloud-steps/install.js +53 -0
- package/dist/cloud-steps/langfuse.d.ts +12 -0
- package/dist/cloud-steps/langfuse.js +35 -0
- package/dist/cloud-steps/repo.d.ts +20 -0
- package/dist/cloud-steps/repo.js +163 -0
- package/dist/cloud-steps/sentry.d.ts +13 -0
- package/dist/cloud-steps/sentry.js +55 -0
- package/dist/cloud-steps/template.d.ts +22 -0
- package/dist/cloud-steps/template.js +68 -0
- package/dist/config.d.ts +53 -0
- package/dist/config.js +155 -0
- package/dist/database.d.ts +65 -0
- package/dist/database.js +142 -0
- package/dist/doctor.d.ts +71 -0
- package/dist/doctor.js +310 -0
- package/dist/index.d.ts +6 -1
- package/dist/index.js +5 -0
- package/dist/migrate.d.ts +11 -0
- package/dist/migrate.js +26 -2
- package/dist/new-cloud.d.ts +126 -0
- package/dist/new-cloud.js +210 -0
- package/dist/new.d.ts +2 -0
- package/dist/new.js +2 -1
- package/dist/providers/cloudflare.d.ts +49 -0
- package/dist/providers/cloudflare.js +27 -0
- package/dist/providers/coolify.d.ts +148 -0
- package/dist/providers/coolify.js +87 -0
- package/dist/providers/github.d.ts +117 -0
- package/dist/providers/github.js +98 -0
- package/dist/providers/http.d.ts +41 -0
- package/dist/providers/http.js +56 -0
- package/dist/providers/langfuse.d.ts +41 -0
- package/dist/providers/langfuse.js +29 -0
- package/dist/providers/sentry.d.ts +31 -0
- package/dist/providers/sentry.js +27 -0
- package/dist/provision-database.d.ts +42 -0
- package/dist/provision-database.js +107 -0
- package/dist/restore-check.d.ts +91 -0
- package/dist/restore-check.js +257 -0
- package/dist/runner.d.ts +65 -0
- package/dist/runner.js +199 -0
- package/dist/secret-file.d.ts +30 -0
- package/dist/secret-file.js +69 -0
- package/dist/state.d.ts +124 -0
- package/dist/state.js +217 -0
- package/dist/status-token.d.ts +2 -0
- package/dist/status-token.js +1 -1
- package/dist/template-source.d.ts +23 -0
- package/dist/template-source.js +23 -0
- package/package.json +10 -7
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
import { quoteIdent } from "@hyperfixation/db";
|
|
2
|
+
import { createLocalDirectoryBackupSource, createS3BackupSource, } from "./backup-source.js";
|
|
3
|
+
import { loadOperatorConfig, requireOperatorConfig } from "./config.js";
|
|
4
|
+
import { openDatabase, openDatabaseUrl, redactPasswords, DEFAULT_POSTGRES_PORT, } from "./database.js";
|
|
5
|
+
import { deriveNames } from "./names.js";
|
|
6
|
+
import { REQUIRED_EXTENSIONS } from "./provision-database.js";
|
|
7
|
+
import { createSshRunner } from "./runner.js";
|
|
8
|
+
import { openAppState } from "./state.js";
|
|
9
|
+
/** Appended to `hf_<app>` for the database the dump is restored into and then dropped. */
|
|
10
|
+
export const SCRATCH_SUFFIX = "_restore_check";
|
|
11
|
+
/** Postgres truncates an identifier past this, which would collide with the live database. */
|
|
12
|
+
const MAX_IDENTIFIER_BYTES = 63;
|
|
13
|
+
/** Older than this and the dump gets a warning line; it never changes the exit code. */
|
|
14
|
+
export const STALE_DUMP_HOURS = 36;
|
|
15
|
+
/** The box's Coolify Postgres superuser — the role the whole check runs as. */
|
|
16
|
+
const CLUSTER_ADMIN_USER = "postgres";
|
|
17
|
+
const CLUSTER_ADMIN_DATABASE = "postgres";
|
|
18
|
+
export class RestoreCheckError extends Error {
|
|
19
|
+
constructor(message) {
|
|
20
|
+
super(message);
|
|
21
|
+
this.name = "RestoreCheckError";
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Restores the newest dump of `hf_<app>` beside the live database and compares row counts.
|
|
26
|
+
*
|
|
27
|
+
* The scratch database is dropped in a `finally`: it holds a full copy of the app's data, so
|
|
28
|
+
* leaving one behind on a failure would double the disk the app uses until someone noticed.
|
|
29
|
+
*
|
|
30
|
+
* `lastRestoreCheckAt` is written only by a run that counted both sides and found them equal.
|
|
31
|
+
* `hf doctor` warns on a stale timestamp, so a check that died halfway — or one that found a
|
|
32
|
+
* mismatch — has to leave the warning standing until a check actually passes.
|
|
33
|
+
*/
|
|
34
|
+
export async function restoreCheck(options) {
|
|
35
|
+
// Refuses a name carrying a quote, a semicolon or a glob before it reaches SQL, `find` or
|
|
36
|
+
// `pg_restore`'s argv.
|
|
37
|
+
const names = deriveNames(options.app);
|
|
38
|
+
const scratchDatabase = `${names.databaseName}${SCRATCH_SUFFIX}`;
|
|
39
|
+
if (Buffer.byteLength(scratchDatabase) > MAX_IDENTIFIER_BYTES) {
|
|
40
|
+
throw new RestoreCheckError(`${scratchDatabase} is longer than Postgres's ${String(MAX_IDENTIFIER_BYTES)}-byte ` +
|
|
41
|
+
`identifier limit, and a truncated name would collide with another database`);
|
|
42
|
+
}
|
|
43
|
+
const dump = await options.source.newest(names.databaseName);
|
|
44
|
+
if (dump === undefined) {
|
|
45
|
+
throw new RestoreCheckError(`no ${names.databaseName} dump in the ${options.source.kind} backup source: nothing to check`);
|
|
46
|
+
}
|
|
47
|
+
const db = typeof options.database === "string" ? openDatabaseUrl(options.database) : options.database;
|
|
48
|
+
const clusterUrl = db.adminUrl();
|
|
49
|
+
if (clusterUrl === undefined) {
|
|
50
|
+
throw new RestoreCheckError(`a restore cannot be run over the ${db.kind} transport: pg_restore needs an address. ` +
|
|
51
|
+
"Publish the Coolify Postgres port on the box's loopback so the tunnel works.");
|
|
52
|
+
}
|
|
53
|
+
const restoreTarget = urlOnto(options.restoreAdminUrl ?? clusterUrl, scratchDatabase);
|
|
54
|
+
const now = options.now ?? new Date();
|
|
55
|
+
const dumpAgeHours = (now.getTime() - dump.takenAt.getTime()) / 3_600_000;
|
|
56
|
+
try {
|
|
57
|
+
// A scratch database left by a killed run is the one thing in the way of this one.
|
|
58
|
+
await db.query(`DROP DATABASE IF EXISTS ${quoteIdent(scratchDatabase)} WITH (FORCE)`);
|
|
59
|
+
await db.query(`CREATE DATABASE ${quoteIdent(scratchDatabase)}`);
|
|
60
|
+
try {
|
|
61
|
+
// Its own connection, so that it can be closed before the drop: `WITH (FORCE)` terminates
|
|
62
|
+
// the backends it finds, and a `pg` client whose backend was killed under it raises an
|
|
63
|
+
// error event nothing is listening for.
|
|
64
|
+
const scratch = openDatabaseUrl(urlOnto(clusterUrl, scratchDatabase));
|
|
65
|
+
let rows;
|
|
66
|
+
try {
|
|
67
|
+
for (const extension of REQUIRED_EXTENSIONS) {
|
|
68
|
+
await scratch.query(`CREATE EXTENSION IF NOT EXISTS ${quoteIdent(extension)}`);
|
|
69
|
+
}
|
|
70
|
+
// What `provisionRoles` grants the migrator in the live database. Without them the
|
|
71
|
+
// restore runs as a role that may not create anything: the extensions have to be created
|
|
72
|
+
// by the admin (pgvector needs a superuser) but everything the dump carries is the
|
|
73
|
+
// migrator's, which is the ownership the live database has.
|
|
74
|
+
await db.query(`GRANT CONNECT, CREATE ON DATABASE ${quoteIdent(scratchDatabase)} ` +
|
|
75
|
+
`TO ${quoteIdent(names.migratorRole)}`);
|
|
76
|
+
await scratch.query(`GRANT CREATE, USAGE ON SCHEMA public TO ${quoteIdent(names.migratorRole)}`);
|
|
77
|
+
await runRestore(options, restoreTarget, names.migratorRole, dump.path);
|
|
78
|
+
rows = compare(await countTables(db, names.databaseName), await countTables(scratch));
|
|
79
|
+
}
|
|
80
|
+
finally {
|
|
81
|
+
await scratch.close();
|
|
82
|
+
}
|
|
83
|
+
const ok = rows.every((row) => row.verdict === "ok");
|
|
84
|
+
if (ok)
|
|
85
|
+
await options.state.patch({ lastRestoreCheckAt: now.toISOString() });
|
|
86
|
+
return {
|
|
87
|
+
databaseName: names.databaseName,
|
|
88
|
+
scratchDatabase,
|
|
89
|
+
dump,
|
|
90
|
+
dumpAgeHours,
|
|
91
|
+
dumpStale: dumpAgeHours > STALE_DUMP_HOURS,
|
|
92
|
+
rows,
|
|
93
|
+
ok,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
finally {
|
|
97
|
+
await db.query(`DROP DATABASE IF EXISTS ${quoteIdent(scratchDatabase)} WITH (FORCE)`);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
finally {
|
|
101
|
+
if (typeof options.database === "string")
|
|
102
|
+
await db.close();
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
/** The argv `restoreCheck` runs — the array the test asserts against. */
|
|
106
|
+
export function pgRestoreArgv(options) {
|
|
107
|
+
return [
|
|
108
|
+
options.pgRestorePath ?? "pg_restore",
|
|
109
|
+
"--no-owner",
|
|
110
|
+
// Every object comes out owned by the migrator, as in the live database. `--no-comments`
|
|
111
|
+
// because a dump's `COMMENT ON EXTENSION` belongs to the admin that created the extension,
|
|
112
|
+
// and a comment nobody may set is not a restore failure.
|
|
113
|
+
"--no-comments",
|
|
114
|
+
`--role=${options.role}`,
|
|
115
|
+
"--dbname",
|
|
116
|
+
options.url,
|
|
117
|
+
options.file,
|
|
118
|
+
];
|
|
119
|
+
}
|
|
120
|
+
async function runRestore(options, url, role, file) {
|
|
121
|
+
const argv = pgRestoreArgv({ url, role, file, pgRestorePath: options.pgRestorePath });
|
|
122
|
+
const result = await options.runner.exec(argv);
|
|
123
|
+
if (result.code !== 0) {
|
|
124
|
+
// `url` carries the cluster's admin password, and so does anything pg_restore echoed of it.
|
|
125
|
+
throw new RestoreCheckError(`pg_restore exited ${String(result.code)} restoring ${file}: ` +
|
|
126
|
+
redactPasswords(result.stderr.trim()));
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Row counts for every table the app's data lives in: `hf_*`, plus every table carrying
|
|
131
|
+
* `normalized_name`, which is how `@hyperfixation/db` spells a record table the app declared.
|
|
132
|
+
*/
|
|
133
|
+
async function countTables(db, database) {
|
|
134
|
+
const { rows: tables } = await db.query(`SELECT c.relname
|
|
135
|
+
FROM pg_class c
|
|
136
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
137
|
+
WHERE n.nspname = 'public'
|
|
138
|
+
AND c.relkind IN ('r', 'p')
|
|
139
|
+
AND (c.relname LIKE 'hf\\_%'
|
|
140
|
+
OR EXISTS (SELECT 1
|
|
141
|
+
FROM pg_attribute a
|
|
142
|
+
WHERE a.attrelid = c.oid
|
|
143
|
+
AND a.attname = 'normalized_name'
|
|
144
|
+
AND a.attnum > 0
|
|
145
|
+
AND NOT a.attisdropped))
|
|
146
|
+
ORDER BY c.relname`, { database });
|
|
147
|
+
const names = tables.map((row) => row[0]).filter((name) => name !== undefined);
|
|
148
|
+
const counts = new Map();
|
|
149
|
+
if (names.length === 0)
|
|
150
|
+
return counts;
|
|
151
|
+
const { rows } = await db.query(names
|
|
152
|
+
.map((name) => `SELECT ${quoteLiteral(name)} AS relname, count(*) AS rows FROM public.${quoteIdent(name)}`)
|
|
153
|
+
.join(" UNION ALL "), { database });
|
|
154
|
+
for (const [name, count] of rows) {
|
|
155
|
+
if (name !== undefined && count !== undefined)
|
|
156
|
+
counts.set(name, Number(count));
|
|
157
|
+
}
|
|
158
|
+
return counts;
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* One row per table either side has.
|
|
162
|
+
*
|
|
163
|
+
* A table on one side only is its own verdict rather than a crash or a zero: an app migration
|
|
164
|
+
* between the backup and the check is the ordinary reason for it, and reading it as a count of
|
|
165
|
+
* zero would make an added table look like lost data.
|
|
166
|
+
*/
|
|
167
|
+
function compare(live, restored) {
|
|
168
|
+
const tables = [...new Set([...live.keys(), ...restored.keys()])].sort();
|
|
169
|
+
return tables.map((table) => {
|
|
170
|
+
const liveCount = live.get(table);
|
|
171
|
+
const restoredCount = restored.get(table);
|
|
172
|
+
const verdict = liveCount === undefined
|
|
173
|
+
? "restored only"
|
|
174
|
+
: restoredCount === undefined
|
|
175
|
+
? "live only"
|
|
176
|
+
: liveCount === restoredCount
|
|
177
|
+
? "ok"
|
|
178
|
+
: "mismatch";
|
|
179
|
+
return {
|
|
180
|
+
table,
|
|
181
|
+
...(liveCount === undefined ? {} : { live: liveCount }),
|
|
182
|
+
...(restoredCount === undefined ? {} : { restored: restoredCount }),
|
|
183
|
+
verdict,
|
|
184
|
+
};
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
/** The table `hf restore-check` prints, and the two lines around it. */
|
|
188
|
+
export function formatRestoreCheck(result) {
|
|
189
|
+
const age = `${result.dumpAgeHours.toFixed(1)} h old`;
|
|
190
|
+
const lines = [`${result.databaseName}: ${result.dump.path}, ${age}`];
|
|
191
|
+
if (result.dumpStale) {
|
|
192
|
+
lines.push(`WARNING: the dump is ${age} — over ${String(STALE_DUMP_HOURS)} h`);
|
|
193
|
+
}
|
|
194
|
+
const header = ["table", "live", "restored", "verdict"];
|
|
195
|
+
const cells = result.rows.map((row) => [
|
|
196
|
+
row.table,
|
|
197
|
+
row.live === undefined ? "—" : String(row.live),
|
|
198
|
+
row.restored === undefined ? "—" : String(row.restored),
|
|
199
|
+
row.verdict,
|
|
200
|
+
]);
|
|
201
|
+
const widths = header.map((name, column) => Math.max(name.length, ...cells.map((row) => row[column]?.length ?? 0)));
|
|
202
|
+
const line = (row) => row.map((cell, column) => cell.padEnd(widths[column] ?? 0)).join(" ").trimEnd();
|
|
203
|
+
lines.push(line(header), ...cells.map(line));
|
|
204
|
+
const mismatched = result.rows.filter((row) => row.verdict !== "ok").length;
|
|
205
|
+
lines.push(result.ok
|
|
206
|
+
? `${String(result.rows.length)} table(s) matched`
|
|
207
|
+
: `${String(mismatched)} of ${String(result.rows.length)} table(s) did not match`);
|
|
208
|
+
return lines;
|
|
209
|
+
}
|
|
210
|
+
/**
|
|
211
|
+
* `hf restore-check <name>`: the operator config, an `ssh` runner onto the box, and the check.
|
|
212
|
+
*
|
|
213
|
+
* The cluster admin password comes from libpq's own `PGPASSWORD` until E3 records Coolify's — the
|
|
214
|
+
* operator config has no key for it, and inventing one before the box has been looked at is
|
|
215
|
+
* exactly what risk 3 warns against.
|
|
216
|
+
*/
|
|
217
|
+
export async function restoreCheckApp(options) {
|
|
218
|
+
const env = options.env ?? process.env;
|
|
219
|
+
const config = await loadOperatorConfig({ env });
|
|
220
|
+
const { HF_SSH_HOST } = requireOperatorConfig(config, ["HF_SSH_HOST"], { env });
|
|
221
|
+
const runner = createSshRunner({ host: HF_SSH_HOST });
|
|
222
|
+
const admin = { user: CLUSTER_ADMIN_USER, password: env.PGPASSWORD };
|
|
223
|
+
const db = await openDatabase(runner, { admin });
|
|
224
|
+
try {
|
|
225
|
+
return await restoreCheck({
|
|
226
|
+
app: options.app,
|
|
227
|
+
state: await openAppState(options.app, { env }),
|
|
228
|
+
source: options.fromS3 === true
|
|
229
|
+
? createS3BackupSource()
|
|
230
|
+
: createLocalDirectoryBackupSource({ runner, directory: options.backupDir }),
|
|
231
|
+
runner,
|
|
232
|
+
database: db,
|
|
233
|
+
restoreAdminUrl: boxAdminUrl(admin),
|
|
234
|
+
});
|
|
235
|
+
}
|
|
236
|
+
finally {
|
|
237
|
+
await db.close();
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
/** The cluster as the box itself sees it, where `pg_restore` runs. */
|
|
241
|
+
function boxAdminUrl(admin) {
|
|
242
|
+
const url = new URL("postgresql://127.0.0.1");
|
|
243
|
+
url.port = String(DEFAULT_POSTGRES_PORT);
|
|
244
|
+
url.username = encodeURIComponent(admin.user);
|
|
245
|
+
if (admin.password !== undefined)
|
|
246
|
+
url.password = encodeURIComponent(admin.password);
|
|
247
|
+
url.pathname = `/${CLUSTER_ADMIN_DATABASE}`;
|
|
248
|
+
return url.toString();
|
|
249
|
+
}
|
|
250
|
+
function urlOnto(connectionString, database) {
|
|
251
|
+
const url = new URL(connectionString);
|
|
252
|
+
url.pathname = `/${encodeURIComponent(database)}`;
|
|
253
|
+
return url.toString();
|
|
254
|
+
}
|
|
255
|
+
function quoteLiteral(value) {
|
|
256
|
+
return `'${value.replaceAll("'", "''")}'`;
|
|
257
|
+
}
|
package/dist/runner.d.ts
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
export interface ExecOptions {
|
|
2
|
+
/** Written to the command's stdin and then closed. SQL goes here, never into argv. */
|
|
3
|
+
input?: string;
|
|
4
|
+
}
|
|
5
|
+
export interface ExecResult {
|
|
6
|
+
/** `null` when the command was killed by a signal or by `timeoutMs`. */
|
|
7
|
+
code: number | null;
|
|
8
|
+
stdout: string;
|
|
9
|
+
stderr: string;
|
|
10
|
+
}
|
|
11
|
+
export interface Tunnel {
|
|
12
|
+
/** On 127.0.0.1, already accepting connections. */
|
|
13
|
+
localPort: number;
|
|
14
|
+
close(): Promise<void>;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Somewhere commands run: the Coolify box over `ssh`, or this machine in a test.
|
|
18
|
+
*
|
|
19
|
+
* `exec` takes an argument array, never a string. Everything it is asked to run carries an app
|
|
20
|
+
* name, a container name or a role name that came from a flag or an API, and the one shape that
|
|
21
|
+
* cannot be talked into a second command is a vector the shell never sees as one token.
|
|
22
|
+
*/
|
|
23
|
+
export interface Runner {
|
|
24
|
+
exec(command: readonly string[], options?: ExecOptions): Promise<ExecResult>;
|
|
25
|
+
/** Forwards a local port to `127.0.0.1:<remotePort>` on the far side. */
|
|
26
|
+
tunnel(remotePort: number): Promise<Tunnel>;
|
|
27
|
+
}
|
|
28
|
+
export declare class RunnerError extends Error {
|
|
29
|
+
constructor(message: string, options?: {
|
|
30
|
+
cause?: unknown;
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
/** The argv `exec` runs, `ssh` excluded — the array the test asserts against. */
|
|
34
|
+
export declare function sshExecArgv(host: string, command: readonly string[]): string[];
|
|
35
|
+
export declare function sshTunnelArgv(host: string, localPort: number, remotePort: number): string[];
|
|
36
|
+
/** Single-quotes one word for a POSIX remote shell. */
|
|
37
|
+
export declare function shellQuote(command: readonly string[]): string;
|
|
38
|
+
export interface SshRunnerOptions {
|
|
39
|
+
/** `HF_SSH_HOST`. */
|
|
40
|
+
host: string;
|
|
41
|
+
/** The `ssh` binary; overridden only by tests that assert the argv. */
|
|
42
|
+
sshPath?: string;
|
|
43
|
+
/** How long to wait for a forwarded port to accept a connection. */
|
|
44
|
+
tunnelReadyTimeoutMs?: number;
|
|
45
|
+
}
|
|
46
|
+
export declare const DEFAULT_TUNNEL_READY_TIMEOUT_MS = 10000;
|
|
47
|
+
/** A `Runner` that reaches the box over `ssh`. */
|
|
48
|
+
export declare function createSshRunner(options: SshRunnerOptions): Runner;
|
|
49
|
+
export interface LocalRunnerOptions {
|
|
50
|
+
/** What `tunnel()` reports; the local Postgres a test already has. */
|
|
51
|
+
tunnelPort?: number;
|
|
52
|
+
}
|
|
53
|
+
export interface LocalRunner extends Runner {
|
|
54
|
+
/** Every argv `exec` was asked for, in order. */
|
|
55
|
+
readonly commands: readonly (readonly string[])[];
|
|
56
|
+
/** Every remote port `tunnel` was asked for, in order. */
|
|
57
|
+
readonly tunnels: readonly number[];
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* A `Runner` that runs on this machine and records what it was asked to run.
|
|
61
|
+
*
|
|
62
|
+
* `tunnel()` forwards nothing — it names a port the test already has — so provisioning can be
|
|
63
|
+
* exercised end to end against the test cluster without an `ssh` anywhere in the suite.
|
|
64
|
+
*/
|
|
65
|
+
export declare function createLocalRunner(options?: LocalRunnerOptions): LocalRunner;
|
package/dist/runner.js
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { createServer, connect as connectTcp } from "node:net";
|
|
3
|
+
export class RunnerError extends Error {
|
|
4
|
+
constructor(message, options) {
|
|
5
|
+
super(message, options);
|
|
6
|
+
this.name = "RunnerError";
|
|
7
|
+
}
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* What `HF_SSH_HOST` may be: `[user@]host`, and nothing that `ssh` would read as an option.
|
|
11
|
+
*
|
|
12
|
+
* A destination beginning `-o` is `ssh`'s own `-oProxyCommand=…`, which runs whatever it says
|
|
13
|
+
* on *this* machine. The config file is the operator's, but it is also the one place an env
|
|
14
|
+
* override reaches, so the destination is checked rather than trusted.
|
|
15
|
+
*/
|
|
16
|
+
const SSH_HOST = /^[A-Za-z0-9._-]+(@[A-Za-z0-9._-]+)?$/;
|
|
17
|
+
/**
|
|
18
|
+
* Options every `ssh` invocation carries.
|
|
19
|
+
*
|
|
20
|
+
* `BatchMode=yes` so a missing key fails instead of prompting a non-interactive run for a
|
|
21
|
+
* password; `ControlMaster=no` with `ControlPath=none` so a multiplexed session left over from
|
|
22
|
+
* the operator's own `ssh` cannot silently carry a tunnel that outlives this process, and so a
|
|
23
|
+
* broken master socket cannot wedge provisioning.
|
|
24
|
+
*/
|
|
25
|
+
const SSH_OPTIONS = [
|
|
26
|
+
"-o",
|
|
27
|
+
"BatchMode=yes",
|
|
28
|
+
"-o",
|
|
29
|
+
"ControlMaster=no",
|
|
30
|
+
"-o",
|
|
31
|
+
"ControlPath=none",
|
|
32
|
+
];
|
|
33
|
+
/** The argv `exec` runs, `ssh` excluded — the array the test asserts against. */
|
|
34
|
+
export function sshExecArgv(host, command) {
|
|
35
|
+
assertHost(host);
|
|
36
|
+
// `ssh` hands the remote end one string and the login shell splits it, so the array has to be
|
|
37
|
+
// re-quoted for that shell; nothing else in this file ever builds a shell word.
|
|
38
|
+
return ["-T", ...SSH_OPTIONS, host, shellQuote(command)];
|
|
39
|
+
}
|
|
40
|
+
export function sshTunnelArgv(host, localPort, remotePort) {
|
|
41
|
+
assertHost(host);
|
|
42
|
+
return [
|
|
43
|
+
"-N",
|
|
44
|
+
"-T",
|
|
45
|
+
...SSH_OPTIONS,
|
|
46
|
+
"-L",
|
|
47
|
+
`${String(localPort)}:127.0.0.1:${String(remotePort)}`,
|
|
48
|
+
host,
|
|
49
|
+
];
|
|
50
|
+
}
|
|
51
|
+
/** Single-quotes one word for a POSIX remote shell. */
|
|
52
|
+
export function shellQuote(command) {
|
|
53
|
+
return command.map((word) => `'${word.replaceAll("'", `'\\''`)}'`).join(" ");
|
|
54
|
+
}
|
|
55
|
+
export const DEFAULT_TUNNEL_READY_TIMEOUT_MS = 10_000;
|
|
56
|
+
/** A `Runner` that reaches the box over `ssh`. */
|
|
57
|
+
export function createSshRunner(options) {
|
|
58
|
+
const ssh = options.sshPath ?? "ssh";
|
|
59
|
+
assertHost(options.host);
|
|
60
|
+
return {
|
|
61
|
+
exec: async (command, execOptions) => await spawnCollecting(ssh, sshExecArgv(options.host, command), execOptions),
|
|
62
|
+
tunnel: async (remotePort) => {
|
|
63
|
+
const localPort = await freeLocalPort();
|
|
64
|
+
const child = spawn(ssh, sshTunnelArgv(options.host, localPort, remotePort), {
|
|
65
|
+
stdio: ["ignore", "ignore", "pipe"],
|
|
66
|
+
});
|
|
67
|
+
let stderr = "";
|
|
68
|
+
child.stderr?.on("data", (chunk) => {
|
|
69
|
+
stderr += chunk.toString("utf8");
|
|
70
|
+
});
|
|
71
|
+
const exited = new Promise((resolve) => child.once("exit", () => resolve()));
|
|
72
|
+
try {
|
|
73
|
+
await waitForPort(localPort, options.tunnelReadyTimeoutMs ?? DEFAULT_TUNNEL_READY_TIMEOUT_MS, child);
|
|
74
|
+
}
|
|
75
|
+
catch (cause) {
|
|
76
|
+
child.kill("SIGTERM");
|
|
77
|
+
await exited;
|
|
78
|
+
throw new RunnerError(`ssh -L ${String(localPort)}:127.0.0.1:${String(remotePort)} never became ready` +
|
|
79
|
+
(stderr === "" ? "" : `: ${stderr.trim()}`), { cause });
|
|
80
|
+
}
|
|
81
|
+
return {
|
|
82
|
+
localPort,
|
|
83
|
+
close: async () => {
|
|
84
|
+
child.kill("SIGTERM");
|
|
85
|
+
await exited;
|
|
86
|
+
},
|
|
87
|
+
};
|
|
88
|
+
},
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* A `Runner` that runs on this machine and records what it was asked to run.
|
|
93
|
+
*
|
|
94
|
+
* `tunnel()` forwards nothing — it names a port the test already has — so provisioning can be
|
|
95
|
+
* exercised end to end against the test cluster without an `ssh` anywhere in the suite.
|
|
96
|
+
*/
|
|
97
|
+
export function createLocalRunner(options = {}) {
|
|
98
|
+
const commands = [];
|
|
99
|
+
const tunnels = [];
|
|
100
|
+
return {
|
|
101
|
+
get commands() {
|
|
102
|
+
return commands;
|
|
103
|
+
},
|
|
104
|
+
get tunnels() {
|
|
105
|
+
return tunnels;
|
|
106
|
+
},
|
|
107
|
+
exec: async (command, execOptions) => {
|
|
108
|
+
commands.push([...command]);
|
|
109
|
+
const [bin, ...args] = command;
|
|
110
|
+
if (bin === undefined)
|
|
111
|
+
throw new RunnerError("exec was given an empty command");
|
|
112
|
+
return await spawnCollecting(bin, args, execOptions);
|
|
113
|
+
},
|
|
114
|
+
tunnel: async (remotePort) => {
|
|
115
|
+
tunnels.push(remotePort);
|
|
116
|
+
const localPort = options.tunnelPort;
|
|
117
|
+
if (localPort === undefined) {
|
|
118
|
+
throw new RunnerError("this local Runner was not given a tunnelPort");
|
|
119
|
+
}
|
|
120
|
+
return { localPort, close: async () => undefined };
|
|
121
|
+
},
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
async function spawnCollecting(bin, args, options = {}) {
|
|
125
|
+
const child = spawn(bin, [...args], { stdio: ["pipe", "pipe", "pipe"] });
|
|
126
|
+
let stdout = "";
|
|
127
|
+
let stderr = "";
|
|
128
|
+
child.stdout?.on("data", (chunk) => {
|
|
129
|
+
stdout += chunk.toString("utf8");
|
|
130
|
+
});
|
|
131
|
+
child.stderr?.on("data", (chunk) => {
|
|
132
|
+
stderr += chunk.toString("utf8");
|
|
133
|
+
});
|
|
134
|
+
// A child that exits before reading its stdin (`true`, a refused ssh) makes the write fail with
|
|
135
|
+
// EPIPE; the exit code already says what happened, so it must not surface as an unhandled error.
|
|
136
|
+
child.stdin?.on("error", () => { });
|
|
137
|
+
child.stdin?.end(options.input ?? "");
|
|
138
|
+
const code = await new Promise((resolve, reject) => {
|
|
139
|
+
child.once("error", reject);
|
|
140
|
+
child.once("close", (exitCode) => resolve(exitCode));
|
|
141
|
+
});
|
|
142
|
+
return { code, stdout, stderr };
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* A port nothing is listening on, by binding one and letting go.
|
|
146
|
+
*
|
|
147
|
+
* Inherently a race — something else on the machine may take it in between — so the ready check
|
|
148
|
+
* below is what actually decides whether the forward came up.
|
|
149
|
+
*/
|
|
150
|
+
async function freeLocalPort() {
|
|
151
|
+
return await new Promise((resolve, reject) => {
|
|
152
|
+
const server = createServer();
|
|
153
|
+
server.once("error", reject);
|
|
154
|
+
server.listen(0, "127.0.0.1", () => {
|
|
155
|
+
const address = server.address();
|
|
156
|
+
if (address === null || typeof address === "string") {
|
|
157
|
+
server.close();
|
|
158
|
+
reject(new RunnerError("could not take a local port for the tunnel"));
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
const { port } = address;
|
|
162
|
+
server.close(() => resolve(port));
|
|
163
|
+
});
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
async function waitForPort(port, timeoutMs, child) {
|
|
167
|
+
const deadline = Date.now() + timeoutMs;
|
|
168
|
+
for (;;) {
|
|
169
|
+
if (child.exitCode !== null || child.signalCode !== null) {
|
|
170
|
+
throw new RunnerError(`ssh exited before the forward on ${String(port)} was ready`);
|
|
171
|
+
}
|
|
172
|
+
if (await canConnect(port))
|
|
173
|
+
return;
|
|
174
|
+
if (Date.now() >= deadline) {
|
|
175
|
+
throw new RunnerError(`nothing accepted on 127.0.0.1:${String(port)} within ${String(timeoutMs)}ms`);
|
|
176
|
+
}
|
|
177
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
async function canConnect(port) {
|
|
181
|
+
return await new Promise((resolve) => {
|
|
182
|
+
let socket;
|
|
183
|
+
const done = (ok) => {
|
|
184
|
+
socket.removeAllListeners();
|
|
185
|
+
socket.destroy();
|
|
186
|
+
resolve(ok);
|
|
187
|
+
};
|
|
188
|
+
socket = connectTcp({ port, host: "127.0.0.1" });
|
|
189
|
+
socket.setTimeout(1_000);
|
|
190
|
+
socket.once("connect", () => done(true));
|
|
191
|
+
socket.once("error", () => done(false));
|
|
192
|
+
socket.once("timeout", () => done(false));
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
function assertHost(host) {
|
|
196
|
+
if (!SSH_HOST.test(host)) {
|
|
197
|
+
throw new RunnerError(`HF_SSH_HOST must match ${SSH_HOST.source}, got ${JSON.stringify(host)}`);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/** Owner read/write and nothing else — the only mode the operator's files are read at. */
|
|
2
|
+
export declare const SECRET_MODE = 384;
|
|
3
|
+
/**
|
|
4
|
+
* A file that holds secrets was readable or writable by someone other than its owner.
|
|
5
|
+
*
|
|
6
|
+
* Thrown *after* the file has been tightened to 0600: the operator's next run works, and this
|
|
7
|
+
* run stops loudly enough that they can decide whether anything in it needs rotating. Nothing
|
|
8
|
+
* of the file's contents reaches the message.
|
|
9
|
+
*/
|
|
10
|
+
export declare class InsecureFileMode extends Error {
|
|
11
|
+
readonly file: string;
|
|
12
|
+
/** The permission bits found, before they were tightened. */
|
|
13
|
+
readonly found: number;
|
|
14
|
+
constructor(file: string, found: number);
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Reads a file that holds secrets, or `undefined` when it does not exist.
|
|
18
|
+
*
|
|
19
|
+
* Refuses a file any other user can reach, having first tightened it — see `InsecureFileMode`.
|
|
20
|
+
*/
|
|
21
|
+
export declare function readSecretFile(file: string): Promise<string | undefined>;
|
|
22
|
+
/**
|
|
23
|
+
* Writes a file that holds secrets, at 0600, atomically.
|
|
24
|
+
*
|
|
25
|
+
* Temp file then `rename`, because the state cache is written between provisioning steps: a
|
|
26
|
+
* crash partway through a write would otherwise leave a half-written JSON file that the next
|
|
27
|
+
* run refuses, and the passwords and tokens it held are not recoverable from anywhere else.
|
|
28
|
+
* The temp file is created 0600 too, so the secrets are never briefly world-readable.
|
|
29
|
+
*/
|
|
30
|
+
export declare function writeSecretFile(file: string, contents: string): Promise<void>;
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { randomBytes } from "node:crypto";
|
|
2
|
+
import { chmod, mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
/** Owner read/write and nothing else — the only mode the operator's files are read at. */
|
|
5
|
+
export const SECRET_MODE = 0o600;
|
|
6
|
+
/** The mode a secret directory is created at, so a new file cannot be listed by anyone else. */
|
|
7
|
+
const SECRET_DIR_MODE = 0o700;
|
|
8
|
+
/**
|
|
9
|
+
* A file that holds secrets was readable or writable by someone other than its owner.
|
|
10
|
+
*
|
|
11
|
+
* Thrown *after* the file has been tightened to 0600: the operator's next run works, and this
|
|
12
|
+
* run stops loudly enough that they can decide whether anything in it needs rotating. Nothing
|
|
13
|
+
* of the file's contents reaches the message.
|
|
14
|
+
*/
|
|
15
|
+
export class InsecureFileMode extends Error {
|
|
16
|
+
file;
|
|
17
|
+
/** The permission bits found, before they were tightened. */
|
|
18
|
+
found;
|
|
19
|
+
constructor(file, found) {
|
|
20
|
+
super(`${file} was mode ${found.toString(8).padStart(4, "0")}, not 0600: other users on this ` +
|
|
21
|
+
`machine could read it. It has been tightened to 0600 — rerun, and rotate anything it ` +
|
|
22
|
+
`holds that may have been read.`);
|
|
23
|
+
this.name = "InsecureFileMode";
|
|
24
|
+
this.file = file;
|
|
25
|
+
this.found = found;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Reads a file that holds secrets, or `undefined` when it does not exist.
|
|
30
|
+
*
|
|
31
|
+
* Refuses a file any other user can reach, having first tightened it — see `InsecureFileMode`.
|
|
32
|
+
*/
|
|
33
|
+
export async function readSecretFile(file) {
|
|
34
|
+
let mode;
|
|
35
|
+
try {
|
|
36
|
+
mode = (await stat(file)).mode & 0o777;
|
|
37
|
+
}
|
|
38
|
+
catch (cause) {
|
|
39
|
+
if (cause.code === "ENOENT")
|
|
40
|
+
return undefined;
|
|
41
|
+
throw cause;
|
|
42
|
+
}
|
|
43
|
+
if ((mode & 0o077) !== 0) {
|
|
44
|
+
await chmod(file, SECRET_MODE);
|
|
45
|
+
throw new InsecureFileMode(file, mode);
|
|
46
|
+
}
|
|
47
|
+
return await readFile(file, "utf8");
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Writes a file that holds secrets, at 0600, atomically.
|
|
51
|
+
*
|
|
52
|
+
* Temp file then `rename`, because the state cache is written between provisioning steps: a
|
|
53
|
+
* crash partway through a write would otherwise leave a half-written JSON file that the next
|
|
54
|
+
* run refuses, and the passwords and tokens it held are not recoverable from anywhere else.
|
|
55
|
+
* The temp file is created 0600 too, so the secrets are never briefly world-readable.
|
|
56
|
+
*/
|
|
57
|
+
export async function writeSecretFile(file, contents) {
|
|
58
|
+
await mkdir(path.dirname(file), { recursive: true, mode: SECRET_DIR_MODE });
|
|
59
|
+
const temp = `${file}.${randomBytes(6).toString("hex")}.tmp`;
|
|
60
|
+
try {
|
|
61
|
+
await writeFile(temp, contents, { mode: SECRET_MODE });
|
|
62
|
+
await chmod(temp, SECRET_MODE);
|
|
63
|
+
await rename(temp, file);
|
|
64
|
+
}
|
|
65
|
+
catch (cause) {
|
|
66
|
+
await rm(temp, { force: true });
|
|
67
|
+
throw cause;
|
|
68
|
+
}
|
|
69
|
+
}
|