@hyperfixation/cli 0.1.4 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,17 +1,19 @@
1
1
  import type { Step } from "../new-cloud.js";
2
2
  import type { CloudStepContext } from "./context.js";
3
3
  /**
4
- * A daily dump of the app's database, and a checklist line about the one thing the API cannot
5
- * answer.
4
+ * A daily dump of the app's database, off the box.
6
5
  *
7
- * Every other step detects its own previous work by name; this one cannot. Coolify documents the
8
- * backups list as "Content is very complex. Will be implemented later.", so there is no response
9
- * to read a schedule out of, and a POST is the only way to find out anything at all. So the step
10
- * registers the schedule and says out loud that a duplicate from an earlier run is possible —
11
- * being told to look is better than a silent second dump, and better than a step that never runs
12
- * because it cannot prove it is needed.
6
+ * Coolify accepts `save_s3: true` with no `s3_storage_uuid`, runs the backup, logs `S3 storage
7
+ * configuration is missing or has been deleted (S3 storage ID: null). S3 backup has been disabled`
8
+ * and keeps the only copy on the same disk as the database which is not a backup. So the storage
9
+ * is resolved first: `HF_COOLIFY_S3_STORAGE_UUID`, or the one usable storage on the box when there
10
+ * is exactly one. None or several and the step does not guess: it registers the schedule
11
+ * `save_s3: false` and says so in a warning and the closing checklist, because a schedule that is
12
+ * honestly local-only beats one that looks remote in the UI and is not.
13
13
  *
14
- * The state record is written by the runner once this resolves, so a failed POST leaves the step
15
- * unrecorded and the next run registers it instead.
14
+ * A rerun reconciles rather than duplicates: the schedules are listed and the one for this app's
15
+ * database is PATCHed. Coolify documents that list as "Content is very complex. Will be implemented
16
+ * later.", so the response is narrowed by hand and anything unreadable falls back to the old
17
+ * behaviour — register, and say out loud that a duplicate is possible.
16
18
  */
17
19
  export declare const backupStep: Step<CloudStepContext>;
@@ -1,18 +1,21 @@
1
1
  import { requireOperatorConfig } from "../config.js";
2
2
  import { CoolifyClient } from "../providers/coolify.js";
3
+ import { ProviderError } from "../providers/http.js";
3
4
  /**
4
- * A daily dump of the app's database, and a checklist line about the one thing the API cannot
5
- * answer.
5
+ * A daily dump of the app's database, off the box.
6
6
  *
7
- * Every other step detects its own previous work by name; this one cannot. Coolify documents the
8
- * backups list as "Content is very complex. Will be implemented later.", so there is no response
9
- * to read a schedule out of, and a POST is the only way to find out anything at all. So the step
10
- * registers the schedule and says out loud that a duplicate from an earlier run is possible —
11
- * being told to look is better than a silent second dump, and better than a step that never runs
12
- * because it cannot prove it is needed.
7
+ * Coolify accepts `save_s3: true` with no `s3_storage_uuid`, runs the backup, logs `S3 storage
8
+ * configuration is missing or has been deleted (S3 storage ID: null). S3 backup has been disabled`
9
+ * and keeps the only copy on the same disk as the database which is not a backup. So the storage
10
+ * is resolved first: `HF_COOLIFY_S3_STORAGE_UUID`, or the one usable storage on the box when there
11
+ * is exactly one. None or several and the step does not guess: it registers the schedule
12
+ * `save_s3: false` and says so in a warning and the closing checklist, because a schedule that is
13
+ * honestly local-only beats one that looks remote in the UI and is not.
13
14
  *
14
- * The state record is written by the runner once this resolves, so a failed POST leaves the step
15
- * unrecorded and the next run registers it instead.
15
+ * A rerun reconciles rather than duplicates: the schedules are listed and the one for this app's
16
+ * database is PATCHed. Coolify documents that list as "Content is very complex. Will be implemented
17
+ * later.", so the response is narrowed by hand and anything unreadable falls back to the old
18
+ * behaviour — register, and say out loud that a duplicate is possible.
16
19
  */
17
20
  export const backupStep = {
18
21
  name: "backup",
@@ -24,17 +27,90 @@ export const backupStep = {
24
27
  token: required.HF_COOLIFY_TOKEN,
25
28
  fetch: context.fetch,
26
29
  });
27
- await coolify.createDatabaseBackup(required.HF_COOLIFY_POSTGRES_UUID, {
30
+ const storage = await resolveStorage(context, coolify);
31
+ const schedule = {
28
32
  frequency: "daily",
29
33
  enabled: true,
30
34
  // This app's database alone: `dump_all` would put every app on the cluster in one dump, and
31
35
  // E5 restores one database at a time.
32
36
  databases_to_backup: names.databaseName,
33
37
  dump_all: false,
34
- backup_now: false,
35
- });
36
- context.io.out(`${names.given}: registered a daily backup of ${names.databaseName}`);
37
- context.checklist.push(`check Coolify for an existing backup schedule for ${names.databaseName} — the API cannot ` +
38
- `list schedules, so an earlier run may have left a second one`);
38
+ ...(storage === undefined
39
+ ? { save_s3: false }
40
+ : { save_s3: true, s3_storage_uuid: storage }),
41
+ };
42
+ const where = storage === undefined ? "on the box only" : "to S3";
43
+ const database = required.HF_COOLIFY_POSTGRES_UUID;
44
+ const existing = await existingSchedule(coolify, database, names.databaseName);
45
+ if (typeof existing === "object") {
46
+ await coolify.updateDatabaseBackup(database, existing.uuid, schedule);
47
+ context.io.out(`${names.given}: updated the daily backup of ${names.databaseName} ${where}, ` +
48
+ `the schedule an earlier run registered`);
49
+ return;
50
+ }
51
+ await coolify.createDatabaseBackup(database, { ...schedule, backup_now: false });
52
+ context.io.out(`${names.given}: registered a daily backup of ${names.databaseName} ${where}`);
53
+ if (existing === "unreadable") {
54
+ context.checklist.push(`check Coolify for a second backup schedule for ${names.databaseName} — its backups list ` +
55
+ `could not be read, so an earlier run may have left one`);
56
+ }
39
57
  },
40
58
  };
59
+ /**
60
+ * The S3 storage the dump goes to, or `undefined` with the reason said out loud.
61
+ *
62
+ * The config key wins outright and is not checked against the list: an operator who named a uuid
63
+ * has answered the question, and a `GET` that disagrees would only be a second opinion about a
64
+ * storage Coolify itself will validate.
65
+ */
66
+ async function resolveStorage(context, coolify) {
67
+ const configured = context.config.HF_COOLIFY_S3_STORAGE_UUID;
68
+ if (configured !== undefined && configured !== "")
69
+ return configured;
70
+ const usable = (await coolify.listS3Storages()).filter((storage) => storage.is_usable === true);
71
+ if (usable.length === 1)
72
+ return usable[0].uuid;
73
+ const note = localOnlyNote(context.names.databaseName, usable);
74
+ context.io.out(`WARNING: ${context.names.given}: ${note}`);
75
+ context.checklist.push(note);
76
+ return undefined;
77
+ }
78
+ /** What is left to the operator when the step will not pick a storage for them. */
79
+ function localOnlyNote(databaseName, usable) {
80
+ const problem = usable.length === 0
81
+ ? "Coolify has no usable S3 storage"
82
+ : `Coolify has ${String(usable.length)} usable S3 storages (${usable
83
+ .map((storage) => `${storage.name} (${storage.uuid})`)
84
+ .join(", ")}) and hf will not choose between them`;
85
+ return (`${databaseName}'s daily backup is local-only — every copy sits on the same box as the ` +
86
+ `database, so losing the box loses the data. ${problem}. Set HF_COOLIFY_S3_STORAGE_UUID to ` +
87
+ `the storage to upload to (Coolify → Storages → the S3 storage → uuid in the URL), adding ` +
88
+ `one there first if there is none, and re-run hf new.`);
89
+ }
90
+ /**
91
+ * This app's existing schedule, `"none"` for a list that has none, and `"unreadable"` for a list
92
+ * this cannot narrow — a shape upstream does not document, or a box that refuses the request.
93
+ */
94
+ async function existingSchedule(coolify, databaseUuid, databaseName) {
95
+ let listed;
96
+ try {
97
+ listed = await coolify.listDatabaseBackups(databaseUuid);
98
+ }
99
+ catch (error) {
100
+ if (!(error instanceof ProviderError))
101
+ throw error;
102
+ return "unreadable";
103
+ }
104
+ if (!Array.isArray(listed))
105
+ return "unreadable";
106
+ for (const entry of listed) {
107
+ if (typeof entry !== "object" || entry === null)
108
+ return "unreadable";
109
+ const { uuid, databases_to_backup } = entry;
110
+ if (typeof uuid !== "string" || typeof databases_to_backup !== "string")
111
+ return "unreadable";
112
+ if (databases_to_backup === databaseName)
113
+ return { uuid };
114
+ }
115
+ return "none";
116
+ }
package/dist/config.d.ts CHANGED
@@ -3,7 +3,7 @@
3
3
  * overrides it. One flat list of names, no nesting: these are pasted in from account pages, and
4
4
  * a shape is one more thing to get wrong.
5
5
  */
6
- export declare const CONFIG_KEYS: readonly ["HF_COOLIFY_URL", "HF_COOLIFY_TOKEN", "HF_COOLIFY_SERVER_UUID", "HF_COOLIFY_GITHUB_APP_UUID", "HF_COOLIFY_POSTGRES_UUID", "HF_DB_HOST_INTERNAL", "HF_DB_CONTAINER", "HF_PG_ADMIN_USER", "HF_SSH_HOST", "HF_CLOUDFLARE_TOKEN", "HF_CLOUDFLARE_ZONE_ID", "HF_BASE_DOMAIN", "HF_GITHUB_TOKEN", "HF_GITHUB_OWNER", "HF_GITHUB_APP_SLUGS", "HF_SENTRY_TOKEN", "HF_SENTRY_ORG", "HF_LANGFUSE_URL", "HF_LANGFUSE_ORG_KEY", "HF_LANGFUSE_PUBLIC_KEY", "HF_LANGFUSE_SECRET_KEY", "HF_BOX_IP", "HF_SMTP_URL", "HF_EMAIL_FROM", "HF_ANTHROPIC_API_KEY", "HF_OPENAI_API_KEY"];
6
+ export declare const CONFIG_KEYS: readonly ["HF_COOLIFY_URL", "HF_COOLIFY_TOKEN", "HF_COOLIFY_SERVER_UUID", "HF_COOLIFY_GITHUB_APP_UUID", "HF_COOLIFY_POSTGRES_UUID", "HF_COOLIFY_S3_STORAGE_UUID", "HF_DB_HOST_INTERNAL", "HF_DB_CONTAINER", "HF_PG_ADMIN_USER", "HF_SSH_HOST", "HF_CLOUDFLARE_TOKEN", "HF_CLOUDFLARE_ZONE_ID", "HF_BASE_DOMAIN", "HF_GITHUB_TOKEN", "HF_GITHUB_OWNER", "HF_GITHUB_APP_SLUGS", "HF_SENTRY_TOKEN", "HF_SENTRY_ORG", "HF_LANGFUSE_URL", "HF_LANGFUSE_ORG_KEY", "HF_LANGFUSE_PUBLIC_KEY", "HF_LANGFUSE_SECRET_KEY", "HF_BOX_IP", "HF_SMTP_URL", "HF_EMAIL_FROM", "HF_ANTHROPIC_API_KEY", "HF_OPENAI_API_KEY"];
7
7
  export type ConfigKey = (typeof CONFIG_KEYS)[number];
8
8
  /** What the operator has configured. Every key is optional until a command asks for it. */
9
9
  export type OperatorConfig = Partial<Record<ConfigKey, string>>;
package/dist/config.js CHANGED
@@ -12,6 +12,11 @@ export const CONFIG_KEYS = [
12
12
  "HF_COOLIFY_SERVER_UUID",
13
13
  "HF_COOLIFY_GITHUB_APP_UUID",
14
14
  "HF_COOLIFY_POSTGRES_UUID",
15
+ // Optional: the Coolify S3 storage the daily dump is uploaded to. Unset, the backup step asks
16
+ // `GET /s3-storages` and takes the one usable storage when there is exactly one; this is how a
17
+ // box with several — or with one the step should not pick — is told which, rather than guessed
18
+ // at. With neither the schedule is registered local-only and the step warns.
19
+ "HF_COOLIFY_S3_STORAGE_UUID",
15
20
  // The Postgres container's hostname on the docker network, as the app's containers see it.
16
21
  // Configurable because Coolify's API document reports no such field: its own compose generator
17
22
  // names the container after the database's uuid, so `HF_COOLIFY_POSTGRES_UUID` is the default a
@@ -29,6 +29,14 @@ export interface Database {
29
29
  host: string;
30
30
  port: number;
31
31
  };
32
+ /**
33
+ * The Postgres container this cluster runs in, when opening it had to discover one.
34
+ *
35
+ * What `hf restore-check` runs `pg_restore` inside: the box host has no Postgres client tools,
36
+ * only the container does. `undefined` when the box's own loopback answered and no container
37
+ * was ever looked for.
38
+ */
39
+ readonly container?: string;
32
40
  /** A libpq URL onto `databaseName`, or `undefined` when the transport has no address. */
33
41
  adminUrl(databaseName?: string): string | undefined;
34
42
  query(sql: string, options?: QueryOptions): Promise<QueryResult>;
@@ -93,3 +101,15 @@ export declare const DEFAULT_POSTGRES_PORT = 5432;
93
101
  export declare function openDatabase(runner: Runner, options: OpenDatabaseOptions): Promise<Database>;
94
102
  /** The cluster at a URL this process can already dial — a test's Postgres, or a live tunnel. */
95
103
  export declare function openDatabaseUrl(adminUrl: string): Database;
104
+ /**
105
+ * The first of `containers` that exists, and its own address, asked of the box.
106
+ *
107
+ * The `coolify` network by name, because a Coolify service also sits on a per-service network
108
+ * that only its own stack is on; the first address is the fallback for a box that names its
109
+ * network something else. Every candidate that failed is reported, because which name a database
110
+ * got is a fact about how it was created and the operator is the one who knows it.
111
+ */
112
+ export declare function findPostgresContainer(runner: Runner, containers: readonly string[]): Promise<{
113
+ container: string;
114
+ address: string;
115
+ }>;
package/dist/database.js CHANGED
@@ -61,8 +61,8 @@ export async function openDatabase(runner, options) {
61
61
  throw new DatabaseTransportError("tunnel", `could not reach Postgres on ${TUNNEL_LOOPBACK}:${String(remotePort)} on the box, and no ` +
62
62
  "container was named to discover an address on the docker network", { cause: loopback.failure });
63
63
  }
64
- const found = await containerAddress(runner, candidates);
65
- const direct = await tryTunnel(runner, options.admin, remotePort, found.address);
64
+ const found = await findPostgresContainer(runner, candidates);
65
+ const direct = await tryTunnel(runner, options.admin, remotePort, found.address, found.container);
66
66
  if ("database" in direct)
67
67
  return direct.database;
68
68
  if (options.dockerExec !== true) {
@@ -74,16 +74,13 @@ export async function openDatabase(runner, options) {
74
74
  }
75
75
  /** The cluster at a URL this process can already dial — a test's Postgres, or a live tunnel. */
76
76
  export function openDatabaseUrl(adminUrl) {
77
- return tunnelDatabase(adminUrl, undefined, undefined);
77
+ return tunnelDatabase(adminUrl, undefined, undefined, undefined);
78
78
  }
79
- async function tryTunnel(runner, admin, remotePort, remoteHost) {
79
+ async function tryTunnel(runner, admin, remotePort, remoteHost, container) {
80
80
  let tunnel;
81
81
  try {
82
82
  tunnel = await runner.tunnel(remotePort, remoteHost);
83
- const database = tunnelDatabase(adminUrlOf(admin, tunnel.localPort), tunnel, {
84
- host: remoteHost,
85
- port: remotePort,
86
- });
83
+ const database = tunnelDatabase(adminUrlOf(admin, tunnel.localPort), tunnel, { host: remoteHost, port: remotePort }, container);
87
84
  await database.query("SELECT 1");
88
85
  return { database };
89
86
  }
@@ -100,7 +97,7 @@ async function tryTunnel(runner, admin, remotePort, remoteHost) {
100
97
  * network something else. Every candidate that failed is reported, because which name a database
101
98
  * got is a fact about how it was created and the operator is the one who knows it.
102
99
  */
103
- async function containerAddress(runner, containers) {
100
+ export async function findPostgresContainer(runner, containers) {
104
101
  const problems = [];
105
102
  for (const container of containers) {
106
103
  const command = ["docker", "inspect", "-f", DOCKER_NETWORKS_FORMAT, container];
@@ -135,7 +132,7 @@ function coolifyAddress(stdout) {
135
132
  .filter((entry) => isIPv4(entry.address));
136
133
  return (networks.find((entry) => entry.network === COOLIFY_NETWORK) ?? networks[0])?.address;
137
134
  }
138
- function tunnelDatabase(adminUrl, tunnel, boxAddress) {
135
+ function tunnelDatabase(adminUrl, tunnel, boxAddress, container) {
139
136
  const clients = new Map();
140
137
  const clientFor = async (databaseName) => {
141
138
  const url = withDatabase(adminUrl, databaseName);
@@ -150,6 +147,7 @@ function tunnelDatabase(adminUrl, tunnel, boxAddress) {
150
147
  return {
151
148
  kind: "tunnel",
152
149
  boxAddress,
150
+ container,
153
151
  adminUrl: (databaseName) => withDatabase(adminUrl, databaseName),
154
152
  query: async (sql, queryOptions) => {
155
153
  const client = await clientFor(queryOptions?.database);
@@ -175,6 +173,7 @@ function tunnelDatabase(adminUrl, tunnel, boxAddress) {
175
173
  function dockerExecDatabase(runner, container, admin) {
176
174
  return {
177
175
  kind: "docker-exec",
176
+ container,
178
177
  adminUrl: () => undefined,
179
178
  query: async (sql, queryOptions) => {
180
179
  // `-f -`: the statement goes down stdin, so it never appears in the box's process list
package/dist/index.d.ts CHANGED
@@ -15,7 +15,7 @@ export { generate, GENERATOR_BIN, GENERATOR_CONFIG, NoGenerators, type GenerateO
15
15
  export { dev, devBuildSha, DEV_COMPOSE_FILE, type DevOptions, type DevResult } from "./dev.js";
16
16
  export { run, CommandFailed, type RunOptions } from "./spawn.js";
17
17
  export { createLocalRunner, createSshRunner, shellQuote, sshExecArgv, sshTunnelArgv, RunnerError, DEFAULT_TUNNEL_READY_TIMEOUT_MS, type ExecOptions, type ExecResult, type LocalRunner, type LocalRunnerOptions, type Runner, type SshRunnerOptions, type Tunnel, } from "./runner.js";
18
- export { openDatabase, openDatabaseUrl, redactPasswords, DatabaseTransportError, DEFAULT_POSTGRES_PORT, type AdminCredentials, type Database, type DatabaseTransport, type OpenDatabaseOptions, type QueryOptions, type QueryResult, } from "./database.js";
18
+ export { findPostgresContainer, openDatabase, openDatabaseUrl, redactPasswords, DatabaseTransportError, DEFAULT_POSTGRES_PORT, type AdminCredentials, type Database, type DatabaseTransport, type OpenDatabaseOptions, type QueryOptions, type QueryResult, } from "./database.js";
19
19
  export { createLocalDirectoryBackupSource, createS3BackupSource, BackupSourceError, COOLIFY_BACKUP_DIR, type BackupDump, type BackupSource, type BackupSourceKind, type LocalDirectoryBackupSourceOptions, } from "./backup-source.js";
20
- export { formatRestoreCheck, pgRestoreArgv, restoreCheck, restoreCheckApp, RestoreCheckError, SCRATCH_SUFFIX, STALE_DUMP_HOURS, type RestoreCheckAppOptions, type RestoreCheckOptions, type RestoreCheckResult, type RestoreCheckRow, type RestoreVerdict, } from "./restore-check.js";
20
+ export { formatRestoreCheck, pgRestoreArgv, pgRestoreInContainerArgv, restoreCheck, restoreCheckApp, RestoreCheckError, SCRATCH_SUFFIX, STALE_DUMP_HOURS, type RestoreCheckAppOptions, type RestoreCheckOptions, type RestoreCheckResult, type RestoreCheckRow, type RestoreVerdict, } from "./restore-check.js";
21
21
  export { provisionDatabase, ProvisionDatabaseError, REQUIRED_EXTENSIONS, type ProvisionDatabaseOptions, type ProvisionDatabaseResult, } from "./provision-database.js";
package/dist/index.js CHANGED
@@ -15,7 +15,7 @@ export { generate, GENERATOR_BIN, GENERATOR_CONFIG, NoGenerators, } from "./gen.
15
15
  export { dev, devBuildSha, DEV_COMPOSE_FILE } from "./dev.js";
16
16
  export { run, CommandFailed } from "./spawn.js";
17
17
  export { createLocalRunner, createSshRunner, shellQuote, sshExecArgv, sshTunnelArgv, RunnerError, DEFAULT_TUNNEL_READY_TIMEOUT_MS, } from "./runner.js";
18
- export { openDatabase, openDatabaseUrl, redactPasswords, DatabaseTransportError, DEFAULT_POSTGRES_PORT, } from "./database.js";
18
+ export { findPostgresContainer, openDatabase, openDatabaseUrl, redactPasswords, DatabaseTransportError, DEFAULT_POSTGRES_PORT, } from "./database.js";
19
19
  export { createLocalDirectoryBackupSource, createS3BackupSource, BackupSourceError, COOLIFY_BACKUP_DIR, } from "./backup-source.js";
20
- export { formatRestoreCheck, pgRestoreArgv, restoreCheck, restoreCheckApp, RestoreCheckError, SCRATCH_SUFFIX, STALE_DUMP_HOURS, } from "./restore-check.js";
20
+ export { formatRestoreCheck, pgRestoreArgv, pgRestoreInContainerArgv, restoreCheck, restoreCheckApp, RestoreCheckError, SCRATCH_SUFFIX, STALE_DUMP_HOURS, } from "./restore-check.js";
21
21
  export { provisionDatabase, ProvisionDatabaseError, REQUIRED_EXTENSIONS, } from "./provision-database.js";
@@ -77,9 +77,10 @@ export declare function invalidateStaleSecretSteps(state: AppStateStore): Promis
77
77
  export declare function runSteps<Context extends CloudContext>(steps: readonly Step<Context>[], context: Context): Promise<RunStepsResult>;
78
78
  /**
79
79
  * The keys a cloud `hf new` runs without: three have a default derived from another key, the two
80
- * provider keys are what the checklist warns about when they are unset, and the three Langfuse
81
- * keys are three ways of configuring one step — an org key, a project key pair, or neither, which
82
- * the step degrades to a warning and a checklist line.
80
+ * provider keys are what the checklist warns about when they are unset, the three Langfuse keys
81
+ * are three ways of configuring one step — an org key, a project key pair, or neither, which the
82
+ * step degrades to a warning and a checklist line — and the S3 storage is one the backup step
83
+ * discovers when the box has exactly one, and warns about when it cannot.
83
84
  */
84
85
  export declare const OPTIONAL_CLOUD_CONFIG: readonly ConfigKey[];
85
86
  /**
package/dist/new-cloud.js CHANGED
@@ -108,11 +108,13 @@ export async function runSteps(steps, context) {
108
108
  }
109
109
  /**
110
110
  * The keys a cloud `hf new` runs without: three have a default derived from another key, the two
111
- * provider keys are what the checklist warns about when they are unset, and the three Langfuse
112
- * keys are three ways of configuring one step — an org key, a project key pair, or neither, which
113
- * the step degrades to a warning and a checklist line.
111
+ * provider keys are what the checklist warns about when they are unset, the three Langfuse keys
112
+ * are three ways of configuring one step — an org key, a project key pair, or neither, which the
113
+ * step degrades to a warning and a checklist line — and the S3 storage is one the backup step
114
+ * discovers when the box has exactly one, and warns about when it cannot.
114
115
  */
115
116
  export const OPTIONAL_CLOUD_CONFIG = [
117
+ "HF_COOLIFY_S3_STORAGE_UUID",
116
118
  "HF_DB_HOST_INTERNAL",
117
119
  "HF_DB_CONTAINER",
118
120
  "HF_PG_ADMIN_USER",
@@ -108,10 +108,33 @@ export interface CoolifyBackupRequest {
108
108
  database_backup_retention_amount_locally?: number;
109
109
  database_backup_retention_days_locally?: number;
110
110
  }
111
+ /**
112
+ * What `PATCH /databases/{uuid}/backups/{scheduled_backup_uuid}` accepts of the fields E3 sets.
113
+ *
114
+ * Every field is optional upstream and only the ones a rerun reconciles are here: the schedule it
115
+ * finds was registered by an earlier `hf new`, and rewriting fields nobody set would undo whatever
116
+ * the operator changed in Coolify's own UI.
117
+ */
118
+ export interface CoolifyBackupUpdate {
119
+ frequency?: string;
120
+ enabled?: boolean;
121
+ save_s3?: boolean;
122
+ s3_storage_uuid?: string;
123
+ databases_to_backup?: string;
124
+ dump_all?: boolean;
125
+ }
111
126
  export interface CoolifyBackup {
112
127
  uuid: string;
113
128
  message?: string;
114
129
  }
130
+ /** An S3 storage as `GET /s3-storages` lists it; `is_usable` is Coolify's own validation verdict. */
131
+ export interface CoolifyS3Storage {
132
+ uuid: string;
133
+ name: string;
134
+ bucket?: string;
135
+ region?: string;
136
+ is_usable?: boolean;
137
+ }
115
138
  export interface CoolifyClientOptions {
116
139
  /** `HF_COOLIFY_URL` — the instance's origin, without `/api/v1`. */
117
140
  url: string;
@@ -153,6 +176,15 @@ export declare class CoolifyClient {
153
176
  }): Promise<CoolifyDeploymentRequest>;
154
177
  getDeployment(deploymentUuid: string): Promise<CoolifyDeployment>;
155
178
  createDatabaseBackup(databaseUuid: string, body: CoolifyBackupRequest): Promise<CoolifyBackup>;
179
+ updateDatabaseBackup(databaseUuid: string, backupUuid: string, body: CoolifyBackupUpdate): Promise<unknown>;
180
+ /**
181
+ * Every S3 storage the token's team has configured.
182
+ *
183
+ * A backup registered with `save_s3` and no `s3_storage_uuid` is accepted, then runs with
184
+ * `S3 storage configuration is missing` in its log and keeps the dump on the box alone — so the
185
+ * uuid has to come from somewhere, and this is the only place the API offers one.
186
+ */
187
+ listS3Storages(): Promise<CoolifyS3Storage[]>;
156
188
  /**
157
189
  * The database's scheduled backups.
158
190
  *
@@ -75,6 +75,23 @@ export class CoolifyClient {
75
75
  body,
76
76
  });
77
77
  }
78
+ async updateDatabaseBackup(databaseUuid, backupUuid, body) {
79
+ return await this.request({
80
+ method: "PATCH",
81
+ path: `/databases/${segment(databaseUuid)}/backups/${segment(backupUuid)}`,
82
+ body,
83
+ });
84
+ }
85
+ /**
86
+ * Every S3 storage the token's team has configured.
87
+ *
88
+ * A backup registered with `save_s3` and no `s3_storage_uuid` is accepted, then runs with
89
+ * `S3 storage configuration is missing` in its log and keeps the dump on the box alone — so the
90
+ * uuid has to come from somewhere, and this is the only place the API offers one.
91
+ */
92
+ async listS3Storages() {
93
+ return await this.request({ method: "GET", path: "/s3-storages" });
94
+ }
78
95
  /**
79
96
  * The database's scheduled backups.
80
97
  *
@@ -36,20 +36,26 @@ export interface RestoreCheckOptions {
36
36
  app: string;
37
37
  state: AppStateStore;
38
38
  source: BackupSource;
39
- /** Where `pg_restore` runs: the box, or this machine in a test. */
39
+ /** Where the dump file is and where `docker` is run: the box, or this machine in a test. */
40
40
  runner: Runner;
41
41
  /** How the scratch database is created and both sides are counted. */
42
42
  database: Database | string;
43
43
  /**
44
- * The cluster's admin URL **as the runner sees it**, for `pg_restore`.
44
+ * The Postgres container `pg_restore` runs inside.
45
45
  *
46
- * Not the same address as `database`: the laptop reaches the cluster through an `ssh -L`
47
- * forward onto a local port, and a `pg_restore` running on the far side of that forward has to
48
- * dial the box's own loopback. Defaults to `database`'s URL, which is what a test wants when
49
- * both sides are the same machine.
46
+ * The box host has no Postgres client tools Postgres runs only in Coolify's container — so a
47
+ * `pg_restore` on the host exits 127. The dump is the host's and is not mounted into the
48
+ * container, so it goes down `docker exec -i`'s stdin.
49
+ */
50
+ container: string;
51
+ /** The superuser inside the container; `HF_PG_ADMIN_USER`, default `postgres`. */
52
+ adminUser?: string;
53
+ /**
54
+ * @deprecated Unused: `pg_restore` no longer dials an address, it runs beside the server inside
55
+ * `container`. Removed in 0.2.0.
50
56
  */
51
57
  restoreAdminUrl?: string;
52
- /** The `pg_restore` binary on the runner. */
58
+ /** The `pg_restore` binary inside the container. */
53
59
  pgRestorePath?: string;
54
60
  now?: Date;
55
61
  }
@@ -64,13 +70,32 @@ export interface RestoreCheckOptions {
64
70
  * mismatch — has to leave the warning standing until a check actually passes.
65
71
  */
66
72
  export declare function restoreCheck(options: RestoreCheckOptions): Promise<RestoreCheckResult>;
67
- /** The argv `restoreCheck` runs — the array the test asserts against. */
73
+ /**
74
+ * The argv `restoreCheck` runs — the array the test asserts against.
75
+ *
76
+ * @deprecated The box host has no `pg_restore`; the restore runs inside the Postgres container.
77
+ * Use `pgRestoreInContainerArgv`. Removed in 0.2.0.
78
+ */
68
79
  export declare function pgRestoreArgv(options: {
69
80
  url: string;
70
81
  role: string;
71
82
  file: string;
72
83
  pgRestorePath?: string;
73
84
  }): string[];
85
+ /**
86
+ * The argv `restoreCheck` runs — the array the test asserts against.
87
+ *
88
+ * No `--dbname` URL: inside the container the server is on the local socket, so the admin user
89
+ * and the database name are all it takes and no password crosses an argv. The dump arrives on
90
+ * stdin, which is why there is no file argument either.
91
+ */
92
+ export declare function pgRestoreInContainerArgv(options: {
93
+ container: string;
94
+ adminUser: string;
95
+ database: string;
96
+ role: string;
97
+ pgRestorePath?: string;
98
+ }): string[];
74
99
  /** The table `hf restore-check` prints, and the two lines around it. */
75
100
  export declare function formatRestoreCheck(result: RestoreCheckResult): string[];
76
101
  export interface RestoreCheckAppOptions {
@@ -1,7 +1,7 @@
1
1
  import { quoteIdent } from "@hyperfixation/db";
2
2
  import { createLocalDirectoryBackupSource, createS3BackupSource, } from "./backup-source.js";
3
- import { loadOperatorConfig, pgAdminUser, postgresContainers, requireOperatorConfig, } from "./config.js";
4
- import { openDatabase, openDatabaseUrl, redactPasswords, DEFAULT_POSTGRES_PORT, } from "./database.js";
3
+ import { loadOperatorConfig, pgAdminUser, postgresContainers, requireOperatorConfig, DEFAULT_PG_ADMIN_USER, } from "./config.js";
4
+ import { findPostgresContainer, openDatabase, openDatabaseUrl, redactPasswords, } from "./database.js";
5
5
  import { deriveNames } from "./names.js";
6
6
  import { REQUIRED_EXTENSIONS } from "./provision-database.js";
7
7
  import { createSshRunner } from "./runner.js";
@@ -12,7 +12,6 @@ export const SCRATCH_SUFFIX = "_restore_check";
12
12
  const MAX_IDENTIFIER_BYTES = 63;
13
13
  /** Older than this and the dump gets a warning line; it never changes the exit code. */
14
14
  export const STALE_DUMP_HOURS = 36;
15
- const CLUSTER_ADMIN_DATABASE = "postgres";
16
15
  export class RestoreCheckError extends Error {
17
16
  constructor(message) {
18
17
  super(message);
@@ -45,11 +44,10 @@ export async function restoreCheck(options) {
45
44
  const db = typeof options.database === "string" ? openDatabaseUrl(options.database) : options.database;
46
45
  const clusterUrl = db.adminUrl();
47
46
  if (clusterUrl === undefined) {
48
- throw new RestoreCheckError(`a restore cannot be run over the ${db.kind} transport: pg_restore needs an address. ` +
49
- "Name the Postgres container HF_DB_CONTAINER, or HF_COOLIFY_POSTGRES_UUID so the " +
50
- "tunnel can discover one.");
47
+ throw new RestoreCheckError(`a restore cannot be checked over the ${db.kind} transport: counting the restored side ` +
48
+ "needs an address a client library can dial. Name the Postgres container — " +
49
+ "HF_DB_CONTAINER, or HF_COOLIFY_POSTGRES_UUID — so the tunnel can discover one.");
51
50
  }
52
- const restoreTarget = urlOnto(options.restoreAdminUrl ?? clusterUrl, scratchDatabase);
53
51
  const now = options.now ?? new Date();
54
52
  const dumpAgeHours = (now.getTime() - dump.takenAt.getTime()) / 3_600_000;
55
53
  try {
@@ -73,7 +71,7 @@ export async function restoreCheck(options) {
73
71
  await db.query(`GRANT CONNECT, CREATE ON DATABASE ${quoteIdent(scratchDatabase)} ` +
74
72
  `TO ${quoteIdent(names.migratorRole)}`);
75
73
  await scratch.query(`GRANT CREATE, USAGE ON SCHEMA public TO ${quoteIdent(names.migratorRole)}`);
76
- await runRestore(options, restoreTarget, names.migratorRole, dump.path);
74
+ await runRestore(options, scratchDatabase, names.migratorRole, dump.path);
77
75
  rows = compare(await countTables(db, names.databaseName), await countTables(scratch));
78
76
  }
79
77
  finally {
@@ -101,14 +99,16 @@ export async function restoreCheck(options) {
101
99
  await db.close();
102
100
  }
103
101
  }
104
- /** The argv `restoreCheck` runs — the array the test asserts against. */
102
+ /**
103
+ * The argv `restoreCheck` runs — the array the test asserts against.
104
+ *
105
+ * @deprecated The box host has no `pg_restore`; the restore runs inside the Postgres container.
106
+ * Use `pgRestoreInContainerArgv`. Removed in 0.2.0.
107
+ */
105
108
  export function pgRestoreArgv(options) {
106
109
  return [
107
110
  options.pgRestorePath ?? "pg_restore",
108
111
  "--no-owner",
109
- // Every object comes out owned by the migrator, as in the live database. `--no-comments`
110
- // because a dump's `COMMENT ON EXTENSION` belongs to the admin that created the extension,
111
- // and a comment nobody may set is not a restore failure.
112
112
  "--no-comments",
113
113
  `--role=${options.role}`,
114
114
  "--dbname",
@@ -116,15 +116,56 @@ export function pgRestoreArgv(options) {
116
116
  options.file,
117
117
  ];
118
118
  }
119
- async function runRestore(options, url, role, file) {
120
- const argv = pgRestoreArgv({ url, role, file, pgRestorePath: options.pgRestorePath });
121
- const result = await options.runner.exec(argv);
122
- if (result.code !== 0) {
123
- // `url` carries the cluster's admin password, and so does anything pg_restore echoed of it.
124
- throw new RestoreCheckError(`pg_restore exited ${String(result.code)} restoring ${file}: ` +
125
- redactPasswords(result.stderr.trim()));
126
- }
119
+ /**
120
+ * The argv `restoreCheck` runs the array the test asserts against.
121
+ *
122
+ * No `--dbname` URL: inside the container the server is on the local socket, so the admin user
123
+ * and the database name are all it takes and no password crosses an argv. The dump arrives on
124
+ * stdin, which is why there is no file argument either.
125
+ */
126
+ export function pgRestoreInContainerArgv(options) {
127
+ return [
128
+ "docker",
129
+ "exec",
130
+ "-i",
131
+ options.container,
132
+ options.pgRestorePath ?? "pg_restore",
133
+ "--no-owner",
134
+ // Every object comes out owned by the migrator, as in the live database. `--no-comments`
135
+ // because a dump's `COMMENT ON EXTENSION` belongs to the admin that created the extension,
136
+ // and a comment nobody may set is not a restore failure.
137
+ "--no-comments",
138
+ `--role=${options.role}`,
139
+ "-U",
140
+ options.adminUser,
141
+ "-d",
142
+ options.database,
143
+ ];
144
+ }
145
+ async function runRestore(options, database, role, file) {
146
+ const { container } = options;
147
+ const argv = pgRestoreInContainerArgv({
148
+ container,
149
+ adminUser: options.adminUser ?? DEFAULT_PG_ADMIN_USER,
150
+ database,
151
+ role,
152
+ pgRestorePath: options.pgRestorePath,
153
+ });
154
+ const result = await options.runner.exec(argv, { inputFile: file });
155
+ if (result.code === 0)
156
+ return;
157
+ // Anything `pg_restore` echoed of a connection string carries the cluster's admin password.
158
+ const stderr = redactPasswords(result.stderr.trim());
159
+ throw new RestoreCheckError(`pg_restore exited ${String(result.code)} restoring ${file} in ${container}: ${stderr}` +
160
+ (result.code === NOT_FOUND_EXIT
161
+ ? ` — exit ${String(NOT_FOUND_EXIT)} means "command not found". The box host has no ` +
162
+ "Postgres client tools; only the container does. Check that docker is on the box's " +
163
+ `PATH and that ${container} is the Postgres container (HF_DB_CONTAINER, or ` +
164
+ "HF_COOLIFY_POSTGRES_UUID)."
165
+ : ""));
127
166
  }
167
+ /** A shell's, and `docker exec`'s, "command not found". */
168
+ const NOT_FOUND_EXIT = 127;
128
169
  /**
129
170
  * Row counts for every table the app's data lives in: `hf_*`, plus every table carrying
130
171
  * `normalized_name`, which is how `@hyperfixation/db` spells a record table the app declared.
@@ -219,7 +260,8 @@ export async function restoreCheckApp(options) {
219
260
  const { HF_SSH_HOST } = requireOperatorConfig(config, ["HF_SSH_HOST"], { env });
220
261
  const runner = createSshRunner({ host: HF_SSH_HOST });
221
262
  const admin = { user: pgAdminUser(config), password: env.PGPASSWORD };
222
- const db = await openDatabase(runner, { admin, containers: postgresContainers(config) });
263
+ const containers = postgresContainers(config);
264
+ const db = await openDatabase(runner, { admin, containers });
223
265
  try {
224
266
  return await restoreCheck({
225
267
  app: options.app,
@@ -229,7 +271,8 @@ export async function restoreCheckApp(options) {
229
271
  : createLocalDirectoryBackupSource({ runner, directory: options.backupDir }),
230
272
  runner,
231
273
  database: db,
232
- restoreAdminUrl: boxAdminUrl(admin, db.boxAddress),
274
+ container: await restoreContainer(runner, db, containers),
275
+ adminUser: admin.user,
233
276
  });
234
277
  }
235
278
  finally {
@@ -237,20 +280,21 @@ export async function restoreCheckApp(options) {
237
280
  }
238
281
  }
239
282
  /**
240
- * The cluster as the box itself sees it, where `pg_restore` runs.
283
+ * The container `pg_restore` runs inside.
241
284
  *
242
- * `address` is whatever the tunnel settled on: with 5432 unpublished the box's loopback is no more
243
- * a listener for `pg_restore` than for the forward, and the container's address on the docker
244
- * network is what both have to dial.
285
+ * `openDatabase` already knows it whenever the box's loopback had no listener, which is the box
286
+ * Coolify builds; a box that does publish 5432 answers on the loopback and never looks, so the
287
+ * discovery runs here instead of leaving the check with nowhere to restore.
245
288
  */
246
- function boxAdminUrl(admin, address) {
247
- const url = new URL(`postgresql://${address?.host ?? "127.0.0.1"}`);
248
- url.port = String(address?.port ?? DEFAULT_POSTGRES_PORT);
249
- url.username = encodeURIComponent(admin.user);
250
- if (admin.password !== undefined)
251
- url.password = encodeURIComponent(admin.password);
252
- url.pathname = `/${CLUSTER_ADMIN_DATABASE}`;
253
- return url.toString();
289
+ async function restoreContainer(runner, db, containers) {
290
+ if (db.container !== undefined)
291
+ return db.container;
292
+ if (containers.length === 0) {
293
+ throw new RestoreCheckError("pg_restore has to run inside the Postgres container — the box host has no Postgres " +
294
+ "client tools — and no container was named. Set HF_DB_CONTAINER, or " +
295
+ "HF_COOLIFY_POSTGRES_UUID, in the operator config.");
296
+ }
297
+ return (await findPostgresContainer(runner, containers)).container;
254
298
  }
255
299
  function urlOnto(connectionString, database) {
256
300
  const url = new URL(connectionString);
package/dist/runner.d.ts CHANGED
@@ -1,6 +1,14 @@
1
1
  export interface ExecOptions {
2
2
  /** Written to the command's stdin and then closed. SQL goes here, never into argv. */
3
3
  input?: string;
4
+ /**
5
+ * A file **on the far side**, streamed into the command's stdin instead of `input`.
6
+ *
7
+ * How a dump on the box reaches a `pg_restore` that runs inside the Postgres container: the
8
+ * file is the host's and the process is the container's, so nothing but stdin spans the two.
9
+ * Takes precedence over `input`.
10
+ */
11
+ inputFile?: string;
4
12
  }
5
13
  export interface ExecResult {
6
14
  /** `null` when the command was killed by a signal or by `timeoutMs`. */
@@ -36,7 +44,7 @@ export declare class RunnerError extends Error {
36
44
  });
37
45
  }
38
46
  /** The argv `exec` runs, `ssh` excluded — the array the test asserts against. */
39
- export declare function sshExecArgv(host: string, command: readonly string[]): string[];
47
+ export declare function sshExecArgv(host: string, command: readonly string[], options?: ExecOptions): string[];
40
48
  export declare function sshTunnelArgv(host: string, localPort: number, remotePort: number, remoteHost?: string): string[];
41
49
  /** Where a forward lands when the caller names no host: the far side's own loopback. */
42
50
  export declare const TUNNEL_LOOPBACK = "127.0.0.1";
package/dist/runner.js CHANGED
@@ -1,4 +1,5 @@
1
1
  import { spawn } from "node:child_process";
2
+ import { createReadStream } from "node:fs";
2
3
  import { createServer, connect as connectTcp } from "node:net";
3
4
  export class RunnerError extends Error {
4
5
  constructor(message, options) {
@@ -31,11 +32,16 @@ const SSH_OPTIONS = [
31
32
  "ControlPath=none",
32
33
  ];
33
34
  /** The argv `exec` runs, `ssh` excluded — the array the test asserts against. */
34
- export function sshExecArgv(host, command) {
35
+ export function sshExecArgv(host, command, options = {}) {
35
36
  assertHost(host);
36
37
  // `ssh` hands the remote end one string and the login shell splits it, so the array has to be
37
- // re-quoted for that shell; nothing else in this file ever builds a shell word.
38
- return ["-T", ...SSH_OPTIONS, host, shellQuote(command)];
38
+ // re-quoted for that shell; nothing else in this file ever builds a shell word. `inputFile`
39
+ // becomes that shell's own `<` redirection, which is the only way a far-side file reaches the
40
+ // stdin of a far-side command without being pulled across the link first.
41
+ const remote = options.inputFile === undefined
42
+ ? shellQuote(command)
43
+ : `${shellQuote(command)} < ${shellQuote([options.inputFile])}`;
44
+ return ["-T", ...SSH_OPTIONS, host, remote];
39
45
  }
40
46
  export function sshTunnelArgv(host, localPort, remotePort, remoteHost = TUNNEL_LOOPBACK) {
41
47
  assertHost(host);
@@ -61,7 +67,13 @@ export function createSshRunner(options) {
61
67
  const ssh = options.sshPath ?? "ssh";
62
68
  assertHost(options.host);
63
69
  return {
64
- exec: async (command, execOptions) => await spawnCollecting(ssh, sshExecArgv(options.host, command), execOptions),
70
+ exec: async (command, execOptions) =>
71
+ // The redirection is the remote shell's, so `inputFile` is spent building the remote word
72
+ // and must not also be opened on this machine.
73
+ await spawnCollecting(ssh, sshExecArgv(options.host, command, execOptions), {
74
+ ...execOptions,
75
+ inputFile: undefined,
76
+ }),
65
77
  tunnel: async (remotePort, remoteHost = TUNNEL_LOOPBACK) => {
66
78
  const localPort = await freeLocalPort();
67
79
  const child = spawn(ssh, sshTunnelArgv(options.host, localPort, remotePort, remoteHost), {
@@ -134,19 +146,28 @@ async function spawnCollecting(bin, args, options = {}) {
134
146
  child.stderr?.on("data", (chunk) => {
135
147
  stderr += chunk.toString("utf8");
136
148
  });
149
+ const file = options.inputFile === undefined ? undefined : createReadStream(options.inputFile);
137
150
  const closed = new Promise((resolve, reject) => {
138
151
  // A child that exits before reading its stdin (`true`, a refused ssh) makes the write fail with
139
152
  // EPIPE; the exit code already says what happened. Any other stdin error is a real failure.
140
153
  child.stdin?.on("error", (error) => {
154
+ file?.destroy();
141
155
  if (error.code === "EPIPE")
142
156
  return;
143
157
  child.kill();
144
158
  reject(error);
145
159
  });
160
+ file?.once("error", (cause) => {
161
+ child.kill();
162
+ reject(new RunnerError(`could not read ${String(options.inputFile)} into stdin`, { cause }));
163
+ });
146
164
  child.once("error", reject);
147
165
  child.once("close", (exitCode) => resolve(exitCode));
148
166
  });
149
- child.stdin?.end(options.input ?? "");
167
+ if (file === undefined)
168
+ child.stdin?.end(options.input ?? "");
169
+ else if (child.stdin !== null)
170
+ file.pipe(child.stdin);
150
171
  const code = await closed;
151
172
  return { code, stdout, stderr };
152
173
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@hyperfixation/cli",
3
- "version": "0.1.4",
3
+ "version": "0.1.5",
4
4
  "license": "MIT",
5
5
  "description": "The hf binary and its Turborepo generator templates",
6
6
  "repository": {
@@ -29,15 +29,15 @@
29
29
  "!dist/test-support/**"
30
30
  ],
31
31
  "dependencies": {
32
- "@hyperfixation/auth": "0.1.4",
33
- "@hyperfixation/core": "0.1.4",
34
- "@hyperfixation/db": "0.1.4",
32
+ "@hyperfixation/auth": "0.1.5",
33
+ "@hyperfixation/core": "0.1.5",
34
+ "@hyperfixation/db": "0.1.5",
35
35
  "giget": "3.3.1",
36
36
  "pg": "^8.23.0"
37
37
  },
38
38
  "devDependencies": {
39
- "@hyperfixation/eslint-config": "0.1.4",
40
- "@hyperfixation/testing": "0.1.4",
39
+ "@hyperfixation/eslint-config": "0.1.5",
40
+ "@hyperfixation/testing": "0.1.5",
41
41
  "@microsoft/api-extractor": "^7.59.1",
42
42
  "@types/pg": "^8.23.1",
43
43
  "eslint": "^10.10.0",