@yunuseon/kayra 0.0.0-stage → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +80 -2
- package/ansible/.ansible-lint +13 -0
- package/ansible/ansible.cfg +12 -0
- package/ansible/playbooks/adopt.yml +35 -0
- package/ansible/playbooks/app.yml +21 -0
- package/ansible/playbooks/apps-prune.yml +30 -0
- package/ansible/playbooks/edge.yml +41 -0
- package/ansible/playbooks/nas.yml +47 -0
- package/ansible/playbooks/platform.yml +62 -0
- package/ansible/requirements.yml +7 -0
- package/ansible/roles/app/defaults/main.yml +9 -0
- package/ansible/roles/app/tasks/main.yml +107 -0
- package/ansible/roles/app/templates/app.env.j2 +9 -0
- package/ansible/roles/app/templates/casc-app.yaml.j2 +68 -0
- package/ansible/roles/backup-check/files/backup-check.service +9 -0
- package/ansible/roles/backup-check/files/backup-check.timer +10 -0
- package/ansible/roles/backup-check/tasks/main.yml +59 -0
- package/ansible/roles/backup-check/templates/backup-check.sh.j2 +117 -0
- package/ansible/roles/common/handlers/main.yml +16 -0
- package/ansible/roles/common/tasks/main.yml +100 -0
- package/ansible/roles/compose-stacks/tasks/main.yml +74 -0
- package/ansible/roles/docker/defaults/main.yml +3 -0
- package/ansible/roles/docker/files/buildkit-rootless +23 -0
- package/ansible/roles/docker/handlers/main.yml +12 -0
- package/ansible/roles/docker/tasks/main.yml +82 -0
- package/ansible/roles/edge-proxy/files/docker-compose.yml +13 -0
- package/ansible/roles/edge-proxy/handlers/main.yml +6 -0
- package/ansible/roles/edge-proxy/tasks/main.yml +34 -0
- package/ansible/roles/edge-proxy/templates/Caddyfile.j2 +25 -0
- package/ansible/roles/forgejo-config/tasks/main.yml +292 -0
- package/ansible/roles/platform/defaults/main.yml +76 -0
- package/ansible/roles/platform/tasks/main.yml +298 -0
- package/ansible/roles/platform/templates/authelia-users.yml.j2 +17 -0
- package/ansible/roles/platform/templates/platform.env.j2 +41 -0
- package/ansible/roles/restic-backup/files/docker-compose.postgres.yml +27 -0
- package/ansible/roles/restic-backup/files/docker-compose.yml +12 -0
- package/ansible/roles/restic-backup/handlers/main.yml +7 -0
- package/ansible/roles/restic-backup/tasks/main.yml +121 -0
- package/ansible/roles/restic-backup/templates/backup.env.j2 +9 -0
- package/ansible/roles/restic-backup/templates/backup.sh.j2 +28 -0
- package/ansible/roles/restic-backup/templates/docker-compose.override.yml.j2 +26 -0
- package/ansible/roles/restic-backup/templates/follow-restore-test.sh.j2 +28 -0
- package/ansible/roles/restic-backup/templates/restore-test.sh.j2 +51 -0
- package/ansible/roles/restic-backup/templates/root.crontab.j2 +6 -0
- package/ansible/roles/restic-backup/templates/ssh_config.j2 +12 -0
- package/ansible/roles/wireguard/handlers/main.yml +9 -0
- package/ansible/roles/wireguard/tasks/main.yml +33 -0
- package/ansible/roles/wireguard/templates/wg0.conf.j2 +12 -0
- package/bin/kayra.js +29 -0
- package/cli/dist/brand.js +41 -0
- package/cli/dist/cli.js +21 -0
- package/cli/dist/commands/doctor.js +79 -0
- package/cli/dist/commands/server.js +215 -0
- package/cli/dist/commands/status.js +43 -0
- package/cli/dist/fleet.js +134 -0
- package/cli/dist/interpret.js +98 -0
- package/cli/dist/load.js +49 -0
- package/cli/dist/main.js +94 -0
- package/cli/dist/plan.js +31 -0
- package/cli/dist/result.js +5 -0
- package/cloud-init.yml.tpl +14 -0
- package/compose/authelia/configuration.yml +138 -0
- package/compose/buildkit/buildkitd.toml +14 -0
- package/compose/docker-compose.acme-dns.yml +12 -0
- package/compose/docker-compose.ci.yml +156 -0
- package/compose/docker-compose.yml +83 -0
- package/compose/jenkins/Dockerfile +40 -0
- package/compose/jenkins/casc/00-platform.yaml +158 -0
- package/compose/jenkins/plugins.txt +97 -0
- package/compose/jenkins-agent/Dockerfile +24 -0
- package/compose/traefik/dynamic/authelia.yml +18 -0
- package/compose/traefik/dynamic/middlewares.yml +14 -0
- package/compose/traefik/dynamic/ratelimit.yml +21 -0
- package/compose/traefik/dynamic/tls.yml +9 -0
- package/compose/traefik/traefik.yml +46 -0
- package/hosts.conf +6 -0
- package/package.json +73 -4
- package/scripts/local/app-manifests.sh +26 -0
- package/scripts/local/authelia-dev-init.sh +58 -0
- package/scripts/local/certs.sh +55 -0
- package/scripts/local/home-image.sh +55 -0
- package/scripts/local/home-reach.sh +41 -0
- package/scripts/local/require-main.sh +38 -0
- package/scripts/local/setup-hosts.sh +30 -0
- package/scripts/server/app-purge +26 -0
- package/scripts/server/app-remove +30 -0
- package/scripts/server/jenkins-api +28 -0
- package/scripts/server/jenkins-app-apply +36 -0
- package/scripts/server/jenkins-reload +12 -0
- package/terraform/modules/hcloud-firewall/main.tf +54 -0
- package/terraform/modules/hcloud-firewall/outputs.tf +9 -0
- package/terraform/modules/hcloud-firewall/variables.tf +22 -0
- package/terraform/modules/hcloud-server/main.tf +62 -0
- package/terraform/modules/hcloud-server/outputs.tf +19 -0
- package/terraform/modules/hcloud-server/templates/inventory.ini.tpl +5 -0
- package/terraform/modules/hcloud-server/variables.tf +50 -0
- package/terraform/modules/multipass-instance/main.tf +67 -0
- package/terraform/modules/multipass-instance/outputs.tf +19 -0
- package/terraform/modules/multipass-instance/templates/inventory.ini.tpl +5 -0
- package/terraform/modules/multipass-instance/variables.tf +39 -0
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
import { within, } from "../brand.js";
|
|
2
|
+
import { run } from "../plan.js";
|
|
3
|
+
import { all, err, map, ok } from "../result.js";
|
|
4
|
+
/** The local scripts that read the fleet's files find them here, not beside themselves. */
|
|
5
|
+
const fleetEnv = ({ fleet }) => ({ FLEET_ROOT: fleet });
|
|
6
|
+
const script = ({ kayra }, name) => within(kayra, "scripts", "local", name);
|
|
7
|
+
const ansiblePath = ({ kayra }, ...segments) => within(kayra, "ansible", ...segments);
|
|
8
|
+
const inventory = (server) => within(server.dir, "inventory.ini");
|
|
9
|
+
const terraform = (server) => `-chdir=${within(server.dir, "terraform")}`;
|
|
10
|
+
/** How Ansible gets the vault password: from a file, or by asking. */
|
|
11
|
+
const vaultEnv = (server) => {
|
|
12
|
+
switch (server.vault.kind) {
|
|
13
|
+
case "file":
|
|
14
|
+
return { ANSIBLE_VAULT_PASSWORD_FILE: server.vault.file };
|
|
15
|
+
case "ask":
|
|
16
|
+
return { ANSIBLE_ASK_VAULT_PASS: "True" };
|
|
17
|
+
case "none":
|
|
18
|
+
return {};
|
|
19
|
+
}
|
|
20
|
+
};
|
|
21
|
+
const ansibleEnv = (engine, env) => ({ ...env, ANSIBLE_CONFIG: ansiblePath(engine, "ansible.cfg") });
|
|
22
|
+
/** Real servers change only from main as merged; a multipass one from anything. */
|
|
23
|
+
const fromMain = (engine, server) => ({
|
|
24
|
+
kind: "check",
|
|
25
|
+
label: `${server.name} may be changed from this checkout`,
|
|
26
|
+
probe: { kind: "exits-zero", argv: ["bash", script(engine, "require-main.sh"), server.name], env: fleetEnv(engine) },
|
|
27
|
+
onFailure: "stop",
|
|
28
|
+
});
|
|
29
|
+
/** Before Ansible's UNREACHABLE: why a home machine cannot be reached. */
|
|
30
|
+
const reachable = (engine, server) => server.target === "home"
|
|
31
|
+
? [
|
|
32
|
+
{
|
|
33
|
+
kind: "check",
|
|
34
|
+
label: `${server.name} answers over SSH`,
|
|
35
|
+
probe: {
|
|
36
|
+
kind: "exits-zero",
|
|
37
|
+
argv: ["bash", script(engine, "home-reach.sh"), server.name, server.kind],
|
|
38
|
+
env: fleetEnv(engine),
|
|
39
|
+
},
|
|
40
|
+
onFailure: "stop",
|
|
41
|
+
},
|
|
42
|
+
]
|
|
43
|
+
: [];
|
|
44
|
+
/** The pinned Ansible collections; a no-op once installed. */
|
|
45
|
+
const collections = (engine, env) => ({
|
|
46
|
+
kind: "run",
|
|
47
|
+
argv: ["ansible-galaxy", "collection", "install", "-r", ansiblePath(engine, "requirements.yml")],
|
|
48
|
+
env,
|
|
49
|
+
stdout: "discard",
|
|
50
|
+
});
|
|
51
|
+
const appSubdomains = (engine, server) => ({
|
|
52
|
+
kind: "query",
|
|
53
|
+
argv: ["bash", script(engine, "app-manifests.sh"), "--subdomains", server.name],
|
|
54
|
+
words: "many",
|
|
55
|
+
env: fleetEnv(engine),
|
|
56
|
+
});
|
|
57
|
+
const certs = (engine, domain, subdomains) => run(["bash", script(engine, "certs.sh"), domain, within(engine.fleet, ".cache", "certs"), ...subdomains]);
|
|
58
|
+
export const provision = (engine, server) => ok([
|
|
59
|
+
fromMain(engine, server),
|
|
60
|
+
...(server.vm ? [run(["bash", script(engine, "home-image.sh"), server.name], fleetEnv(engine))] : []),
|
|
61
|
+
...(server.terraform
|
|
62
|
+
? [run(["tofu", terraform(server), "init", "-input=false"]), run(["tofu", terraform(server), "apply", "-auto-approve"])]
|
|
63
|
+
: []),
|
|
64
|
+
]);
|
|
65
|
+
/** The host layer: the playbook for the server's kind. */
|
|
66
|
+
export const setup = (engine, server) => ok([
|
|
67
|
+
fromMain(engine, server),
|
|
68
|
+
// mkcert where Let's Encrypt cannot reach: test VMs, and home servers without DNS-01.
|
|
69
|
+
...(server.kind === "platform" && server.target !== "hetzner" && !server.acmeDns
|
|
70
|
+
? [certs(engine, server.domain, [appSubdomains(engine, server), ...server.stackHosts])]
|
|
71
|
+
: []),
|
|
72
|
+
...reachable(engine, server),
|
|
73
|
+
collections(engine, vaultEnv(server)),
|
|
74
|
+
run([
|
|
75
|
+
"ansible-playbook",
|
|
76
|
+
"-i",
|
|
77
|
+
inventory(server),
|
|
78
|
+
ansiblePath(engine, "playbooks", `${server.kind}.yml`),
|
|
79
|
+
"-e",
|
|
80
|
+
`server=${server.name}`,
|
|
81
|
+
"-e",
|
|
82
|
+
`server_dir=${server.dir}`,
|
|
83
|
+
"-e",
|
|
84
|
+
`domain=${server.domain}`,
|
|
85
|
+
"-e",
|
|
86
|
+
`target=${server.target}`,
|
|
87
|
+
"-e",
|
|
88
|
+
`kayra_version=${engine.version}`,
|
|
89
|
+
], ansibleEnv(engine, vaultEnv(server))),
|
|
90
|
+
]);
|
|
91
|
+
/** Register every app in server.yml, then remove the rest (their data kept). */
|
|
92
|
+
export const apps = (engine, server) => {
|
|
93
|
+
const env = vaultEnv(server);
|
|
94
|
+
const playbook = (name, ...extra) => run(["ansible-playbook", "-i", inventory(server), ansiblePath(engine, "playbooks", name), ...extra], ansibleEnv(engine, env));
|
|
95
|
+
return ok([
|
|
96
|
+
fromMain(engine, server),
|
|
97
|
+
...(server.kind === "platform"
|
|
98
|
+
? [
|
|
99
|
+
...server.apps.flatMap((app) => [
|
|
100
|
+
collections(engine, env),
|
|
101
|
+
playbook("app.yml", "-e", `server=${server.name}`, "-e", `server_dir=${server.dir}`, "-e", `domain=${server.domain}`, "-e", `{"app": ${app.entry}}`),
|
|
102
|
+
]),
|
|
103
|
+
collections(engine, env),
|
|
104
|
+
playbook("apps-prune.yml", "-e", `{"keep": ${JSON.stringify(server.apps.map((app) => app.name))}}`),
|
|
105
|
+
]
|
|
106
|
+
: []),
|
|
107
|
+
]);
|
|
108
|
+
};
|
|
109
|
+
export const hosts = (engine, server) => ok(server.target === "multipass"
|
|
110
|
+
? [
|
|
111
|
+
run([
|
|
112
|
+
"bash",
|
|
113
|
+
script(engine, "setup-hosts.sh"),
|
|
114
|
+
server.domain,
|
|
115
|
+
{ kind: "query", argv: ["tofu", terraform(server), "output", "-raw", "server_ip"], words: "one" },
|
|
116
|
+
appSubdomains(engine, server),
|
|
117
|
+
]),
|
|
118
|
+
]
|
|
119
|
+
: [{ kind: "note", text: `hosts: ${server.target} uses real DNS, nothing to do.` }]);
|
|
120
|
+
const sequence = (plans) => map(all(plans), (each) => each.flat());
|
|
121
|
+
export const deploy = (engine, server) => sequence([setup(engine, server), apps(engine, server)]);
|
|
122
|
+
export const up = (engine, server) => sequence([provision(engine, server), deploy(engine, server), hosts(engine, server)]);
|
|
123
|
+
export const down = (server) => server.terraform
|
|
124
|
+
? ok([
|
|
125
|
+
...(server.target === "hetzner"
|
|
126
|
+
? [
|
|
127
|
+
{
|
|
128
|
+
kind: "confirm",
|
|
129
|
+
prompt: `This destroys the Hetzner server '${server.name}' (${server.domain}). Type the name to confirm: `,
|
|
130
|
+
expect: server.name,
|
|
131
|
+
},
|
|
132
|
+
]
|
|
133
|
+
: []),
|
|
134
|
+
run(["tofu", terraform(server), "destroy", "-auto-approve"]),
|
|
135
|
+
])
|
|
136
|
+
: err(`${server.name} has no terraform/: nothing here creates it, so nothing destroys it`);
|
|
137
|
+
/** Once, for a server we did not create: the deploy user, added as root. */
|
|
138
|
+
export const adopt = (engine, server) => ok([
|
|
139
|
+
fromMain(engine, server),
|
|
140
|
+
collections(engine, {}),
|
|
141
|
+
run(["ansible-playbook", "-i", inventory(server), ansiblePath(engine, "playbooks", "adopt.yml")], ansibleEnv(engine, {})),
|
|
142
|
+
]);
|
|
143
|
+
/** A removed app's volumes: its data, irreversibly. */
|
|
144
|
+
export const appPurge = (engine, server, app) => ok([
|
|
145
|
+
fromMain(engine, server),
|
|
146
|
+
run([
|
|
147
|
+
"ansible",
|
|
148
|
+
"servers",
|
|
149
|
+
"-i",
|
|
150
|
+
inventory(server),
|
|
151
|
+
"--become",
|
|
152
|
+
"-m",
|
|
153
|
+
"ansible.builtin.command",
|
|
154
|
+
"-a",
|
|
155
|
+
`/opt/platform/bin/app-purge ${app}`,
|
|
156
|
+
], ansibleEnv(engine, {})),
|
|
157
|
+
]);
|
|
158
|
+
/**
|
|
159
|
+
* The restic-backup role's restore test: the latest backup, restored to a scratch dir and checked.
|
|
160
|
+
* It runs on the server, detached; kayra follows its log over SSH. Stopping kayra stops only the
|
|
161
|
+
* following, and running it again while a test runs follows that one instead of starting another.
|
|
162
|
+
*/
|
|
163
|
+
export const backupTest = (engine, server) => {
|
|
164
|
+
if (server.backupDir === undefined)
|
|
165
|
+
return err(`${server.name} has no restic_backup.dir in its vars.yml`);
|
|
166
|
+
if (server.inventory === undefined)
|
|
167
|
+
return err(`no inventory for '${server.name}' — provision it first.`);
|
|
168
|
+
return ok([
|
|
169
|
+
...reachable(engine, server),
|
|
170
|
+
run([
|
|
171
|
+
"ansible",
|
|
172
|
+
"servers",
|
|
173
|
+
"-i",
|
|
174
|
+
inventory(server),
|
|
175
|
+
"--become",
|
|
176
|
+
"-e",
|
|
177
|
+
`@${within(server.dir, "secrets.yml")}`,
|
|
178
|
+
"-m",
|
|
179
|
+
"ansible.builtin.command",
|
|
180
|
+
"-a",
|
|
181
|
+
`docker compose -f ${server.backupDir}/docker-compose.yml exec -d -T backup /etc/backup/restore-test.sh`,
|
|
182
|
+
], ansibleEnv(engine, vaultEnv(server))),
|
|
183
|
+
{ kind: "note", text: "Following the restore test. Ctrl-C stops following, not the test." },
|
|
184
|
+
run([
|
|
185
|
+
"ssh",
|
|
186
|
+
"-p",
|
|
187
|
+
server.inventory.port,
|
|
188
|
+
"-i",
|
|
189
|
+
server.inventory.key,
|
|
190
|
+
"-o",
|
|
191
|
+
"StrictHostKeyChecking=accept-new",
|
|
192
|
+
`${server.inventory.user}@${server.inventory.host}`,
|
|
193
|
+
"sh",
|
|
194
|
+
`${server.backupDir}/follow-restore-test.sh`,
|
|
195
|
+
]),
|
|
196
|
+
]);
|
|
197
|
+
};
|
|
198
|
+
export const ssh = (server) => server.inventory === undefined
|
|
199
|
+
? err(`no inventory for '${server.name}' — provision it first.`)
|
|
200
|
+
: ok([
|
|
201
|
+
run([
|
|
202
|
+
"ssh",
|
|
203
|
+
"-p",
|
|
204
|
+
server.inventory.port,
|
|
205
|
+
"-i",
|
|
206
|
+
server.inventory.key,
|
|
207
|
+
"-o",
|
|
208
|
+
"StrictHostKeyChecking=no",
|
|
209
|
+
"-o",
|
|
210
|
+
"UserKnownHostsFile=/dev/null",
|
|
211
|
+
`${server.inventory.user}@${server.inventory.host}`,
|
|
212
|
+
]),
|
|
213
|
+
]);
|
|
214
|
+
/** mkcert certs for a domain and its subdomains, for local dev and multipass. */
|
|
215
|
+
export const certsFor = (engine, domain, subdomains) => ok([certs(engine, domain, subdomains)]);
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { within } from "../brand.js";
|
|
2
|
+
import { capture } from "../interpret.js";
|
|
3
|
+
const describe = (machine) => {
|
|
4
|
+
switch (machine.kind) {
|
|
5
|
+
case "none":
|
|
6
|
+
return "not provisioned";
|
|
7
|
+
case "provisioned":
|
|
8
|
+
return `provisioned (${machine.ip ?? "?"})`;
|
|
9
|
+
case "unreadable":
|
|
10
|
+
return `state unreadable: ${machine.reason}`;
|
|
11
|
+
}
|
|
12
|
+
};
|
|
13
|
+
export const statusLine = (server, machine) => [
|
|
14
|
+
server.name.padEnd(10),
|
|
15
|
+
`kind=${server.kind.padEnd(9)} target=${server.target.padEnd(10)}`,
|
|
16
|
+
`domain=${server.domain.padEnd(24)}`,
|
|
17
|
+
`apps=${server.apps.map((app) => app.name).join(",").padEnd(10)}`,
|
|
18
|
+
describe(machine),
|
|
19
|
+
].join(" ");
|
|
20
|
+
/** Why tofu could not read the state, in a few words: the first line of its error. */
|
|
21
|
+
export const unreadable = (stderr) => {
|
|
22
|
+
if (stderr.includes("tofu init"))
|
|
23
|
+
return "tofu init needed";
|
|
24
|
+
const line = stderr
|
|
25
|
+
.split("\n")
|
|
26
|
+
.map((each) => each.replace(/^[│╷╵\s]+/, "").replace(/^Error:\s*/, "").trim())
|
|
27
|
+
.find((each) => each !== "");
|
|
28
|
+
return line ?? "tofu failed";
|
|
29
|
+
};
|
|
30
|
+
const machine = async (server) => {
|
|
31
|
+
if (!server.terraform)
|
|
32
|
+
return { kind: "none" };
|
|
33
|
+
const chdir = `-chdir=${within(server.dir, "terraform")}`;
|
|
34
|
+
const state = await capture(["tofu", chdir, "state", "list", "-no-color"], "capture");
|
|
35
|
+
if (state.code !== 0)
|
|
36
|
+
return { kind: "unreadable", reason: unreadable(state.stderr) };
|
|
37
|
+
if (state.stdout.trim() === "")
|
|
38
|
+
return { kind: "none" };
|
|
39
|
+
const ip = await capture(["tofu", chdir, "output", "-raw", "server_ip"], "capture");
|
|
40
|
+
return { kind: "provisioned", ip: ip.code === 0 ? ip.stdout.replace(/\n+$/, "") : undefined };
|
|
41
|
+
};
|
|
42
|
+
/** Every server and whether a machine exists for it. Reads state, changes nothing. */
|
|
43
|
+
export const status = async (fleet) => Promise.all(fleet.servers.map(async (server) => statusLine(server, await machine(server))));
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import { parse } from "yaml";
|
|
2
|
+
import { absolutePath, appName, domain, vaultPasswordFile, } from "./brand.js";
|
|
3
|
+
import { all, err, map, ok } from "./result.js";
|
|
4
|
+
export const serverKinds = ["platform", "edge", "nas"];
|
|
5
|
+
export const serverTargets = ["multipass", "hetzner", "home"];
|
|
6
|
+
const mapping = (value) => typeof value === "object" && value !== null && !Array.isArray(value) ? Object.fromEntries(Object.entries(value)) : undefined;
|
|
7
|
+
const fields = (text) => {
|
|
8
|
+
const value = parse(text);
|
|
9
|
+
if (value === null)
|
|
10
|
+
return ok({});
|
|
11
|
+
const result = mapping(value);
|
|
12
|
+
return result === undefined ? err("is not a YAML mapping") : ok(result);
|
|
13
|
+
};
|
|
14
|
+
const vaulted = (text) => text.startsWith("$ANSIBLE_VAULT");
|
|
15
|
+
const oneOf = (allowed, value, field) => {
|
|
16
|
+
const match = allowed.find((candidate) => candidate === value);
|
|
17
|
+
return match === undefined
|
|
18
|
+
? err(`${field} must be ${allowed.join(", ")}, not '${String(value)}'`)
|
|
19
|
+
: ok(match);
|
|
20
|
+
};
|
|
21
|
+
const vault = (descriptor) => {
|
|
22
|
+
const file = descriptor["vault_password_file"];
|
|
23
|
+
if (descriptor["vault"] !== true)
|
|
24
|
+
return ok({ kind: "none" });
|
|
25
|
+
if (file === undefined)
|
|
26
|
+
return ok({ kind: "ask" });
|
|
27
|
+
return typeof file === "string"
|
|
28
|
+
? map(absolutePath(file), (path) => ({ kind: "file", file: vaultPasswordFile(path) }))
|
|
29
|
+
: err("vault_password_file must be a path");
|
|
30
|
+
};
|
|
31
|
+
/** Apps live in their own repo: Jenkins clones it over HTTPS, and the app root stays inside it. */
|
|
32
|
+
const app = (value) => {
|
|
33
|
+
const entry = mapping(value);
|
|
34
|
+
if (entry === undefined)
|
|
35
|
+
return err("an app must be a mapping");
|
|
36
|
+
const name = typeof entry["name"] === "string" ? appName(entry["name"]) : err("an app needs a name");
|
|
37
|
+
if (!name.ok)
|
|
38
|
+
return name;
|
|
39
|
+
const repo = entry["repo"];
|
|
40
|
+
if (typeof repo !== "string" || !repo.startsWith("https://"))
|
|
41
|
+
return err(`app ${name.value}: repo must be an https:// URL`);
|
|
42
|
+
const path = entry["path"] ?? ".";
|
|
43
|
+
if (typeof path !== "string" || `/${path}/`.includes("/../"))
|
|
44
|
+
return err(`app ${name.value}: path must stay inside its repo`);
|
|
45
|
+
return ok({ name: name.value, repo, entry: JSON.stringify(value) });
|
|
46
|
+
};
|
|
47
|
+
const inventoryField = (inventory, key) => new RegExp(`${key}=([^ \\n]*)`).exec(inventory)?.[1] ?? "";
|
|
48
|
+
const inventoryOf = (text) => ({
|
|
49
|
+
host: inventoryField(text, "ansible_host"),
|
|
50
|
+
port: inventoryField(text, "ansible_port") || "22",
|
|
51
|
+
user: inventoryField(text, "ansible_user"),
|
|
52
|
+
key: inventoryField(text, "ansible_ssh_private_key_file"),
|
|
53
|
+
});
|
|
54
|
+
const stackHosts = (stacks, serverDomain) => stacks
|
|
55
|
+
.flatMap((compose) => [...compose.matchAll(/Host\(`([^`]*)`\)/g)].map((match) => match[1] ?? ""))
|
|
56
|
+
.map((host) => (host.endsWith(`.${serverDomain}`) ? host.slice(0, -serverDomain.length - 1) : host));
|
|
57
|
+
/** An app's config is plaintext, to be read and diffed; its secrets are vaulted wherever secrets.yml is. */
|
|
58
|
+
const appValuesProblem = (appValues, secretsMustBeVaulted) => appValues
|
|
59
|
+
.map(({ file, text }) => file.endsWith(".secrets.yml")
|
|
60
|
+
? secretsMustBeVaulted && !vaulted(text)
|
|
61
|
+
? `apps/${file} must be vault-encrypted`
|
|
62
|
+
: undefined
|
|
63
|
+
: vaulted(text)
|
|
64
|
+
? `apps/${file} is vault-encrypted; it holds config only — secrets go in apps/${file.replace(/\.yml$/, ".secrets.yml")}`
|
|
65
|
+
: undefined)
|
|
66
|
+
.find((problem) => problem !== undefined);
|
|
67
|
+
/** What every command may rely on, checked for every server whenever the fleet loads. */
|
|
68
|
+
export const parseServer = (files) => {
|
|
69
|
+
const descriptor = fields(files.descriptor);
|
|
70
|
+
if (!descriptor.ok)
|
|
71
|
+
return descriptor;
|
|
72
|
+
const kind = oneOf(serverKinds, descriptor.value["kind"] ?? "platform", "kind");
|
|
73
|
+
if (!kind.ok)
|
|
74
|
+
return kind;
|
|
75
|
+
const target = oneOf(serverTargets, descriptor.value["target"], "target");
|
|
76
|
+
if (!target.ok)
|
|
77
|
+
return target;
|
|
78
|
+
const declared = descriptor.value["domain"];
|
|
79
|
+
const serverDomain = typeof declared === "string" ? domain(declared) : err("domain is required");
|
|
80
|
+
if (!serverDomain.ok)
|
|
81
|
+
return serverDomain;
|
|
82
|
+
const serverVault = vault(descriptor.value);
|
|
83
|
+
if (!serverVault.ok)
|
|
84
|
+
return serverVault;
|
|
85
|
+
const declaredApps = descriptor.value["apps"] ?? [];
|
|
86
|
+
if (!Array.isArray(declaredApps))
|
|
87
|
+
return err("apps must be a list");
|
|
88
|
+
if (kind.value !== "platform" && declaredApps.length > 0)
|
|
89
|
+
return err("apps run on platform servers only");
|
|
90
|
+
const apps = all(declaredApps.map(app));
|
|
91
|
+
if (!apps.ok)
|
|
92
|
+
return apps;
|
|
93
|
+
if (target.value === "home" && files.inventory === undefined)
|
|
94
|
+
return err("target home needs a hand-written inventory.ini next to it");
|
|
95
|
+
if (target.value !== "home" && !files.terraform)
|
|
96
|
+
return err("no terraform/ next to it");
|
|
97
|
+
if (files.vars === undefined)
|
|
98
|
+
return err("no vars.yml next to it");
|
|
99
|
+
if (files.secrets === undefined)
|
|
100
|
+
return err("no secrets.yml next to it");
|
|
101
|
+
if (vaulted(files.vars))
|
|
102
|
+
return err("vars.yml is vault-encrypted; it holds settings only — secrets go in secrets.yml");
|
|
103
|
+
if (target.value !== "multipass" && serverVault.value.kind === "none")
|
|
104
|
+
return err(`target ${target.value} requires vault: true`);
|
|
105
|
+
if (target.value !== "multipass" && !vaulted(files.secrets))
|
|
106
|
+
return err(`target ${target.value} requires a vault-encrypted secrets.yml`);
|
|
107
|
+
const appValues = appValuesProblem(files.appValues, target.value !== "multipass");
|
|
108
|
+
if (appValues !== undefined)
|
|
109
|
+
return err(appValues);
|
|
110
|
+
const vars = fields(files.vars);
|
|
111
|
+
if (!vars.ok)
|
|
112
|
+
return err(`vars.yml ${vars.error}`);
|
|
113
|
+
const backupDir = mapping(vars.value["restic_backup"])?.["dir"];
|
|
114
|
+
return ok({
|
|
115
|
+
name: files.name,
|
|
116
|
+
dir: files.dir,
|
|
117
|
+
kind: kind.value,
|
|
118
|
+
target: target.value,
|
|
119
|
+
domain: serverDomain.value,
|
|
120
|
+
vault: serverVault.value,
|
|
121
|
+
secretsVaulted: vaulted(files.secrets),
|
|
122
|
+
apps: apps.value,
|
|
123
|
+
terraform: files.terraform,
|
|
124
|
+
vm: target.value === "home" && vars.value["vm"] !== undefined,
|
|
125
|
+
acmeDns: typeof vars.value["acme_dns"] === "string" && vars.value["acme_dns"] !== "",
|
|
126
|
+
inventory: files.inventory === undefined ? undefined : inventoryOf(files.inventory),
|
|
127
|
+
backupDir: typeof backupDir === "string" ? backupDir : undefined,
|
|
128
|
+
stackHosts: stackHosts(files.stacks, serverDomain.value),
|
|
129
|
+
});
|
|
130
|
+
};
|
|
131
|
+
export const serverNamed = (fleet, name) => {
|
|
132
|
+
const server = fleet.servers.find((candidate) => candidate.name === name);
|
|
133
|
+
return server === undefined ? err(`no server '${name}'`) : ok(server);
|
|
134
|
+
};
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { text } from "node:stream/consumers";
|
|
3
|
+
const spawned = async (argv, streams, env = {}) => {
|
|
4
|
+
const [program, ...args] = argv;
|
|
5
|
+
const child = spawn(program, args, {
|
|
6
|
+
env: { ...process.env, ...env },
|
|
7
|
+
stdio: [
|
|
8
|
+
"inherit",
|
|
9
|
+
streams.stdout === "show" ? "inherit" : streams.stdout === "capture" ? "pipe" : "ignore",
|
|
10
|
+
streams.stderr === "show" ? "inherit" : "pipe",
|
|
11
|
+
],
|
|
12
|
+
});
|
|
13
|
+
const stdout = child.stdout === null ? Promise.resolve("") : text(child.stdout);
|
|
14
|
+
const stderr = child.stderr === null ? Promise.resolve("") : text(child.stderr);
|
|
15
|
+
const exit = await new Promise((resolve) => {
|
|
16
|
+
child.on("error", resolve);
|
|
17
|
+
child.on("close", (code) => {
|
|
18
|
+
resolve(code ?? 1);
|
|
19
|
+
});
|
|
20
|
+
});
|
|
21
|
+
return exit instanceof Error
|
|
22
|
+
? { code: 127, stdout: "", stderr: exit.message }
|
|
23
|
+
: { code: exit, stdout: await stdout, stderr: await stderr };
|
|
24
|
+
};
|
|
25
|
+
/** A command's stdout, for kayra to read; its stderr reaches the operator unless captured too. */
|
|
26
|
+
export const capture = (argv, stderr = "show", env = {}) => spawned(argv, { stdout: "capture", stderr }, env);
|
|
27
|
+
const answer = (probe) => spawned(probe.kind === "on-path" ? ["sh", "-c", 'command -v "$1"', "sh", probe.program] : probe.argv, { stdout: "discard", stderr: "capture" }, probe.kind === "exits-zero" ? probe.env : {});
|
|
28
|
+
/** Each query's answer in place of it, as `$(…)` would: one word, or split into many. Undefined if one failed. */
|
|
29
|
+
const resolve = async (argv, io) => {
|
|
30
|
+
const words = await Promise.all(argv.map(async (value) => {
|
|
31
|
+
if (typeof value === "string")
|
|
32
|
+
return [value];
|
|
33
|
+
const exit = await capture(value.argv, "show", value.env);
|
|
34
|
+
if (exit.code !== 0) {
|
|
35
|
+
io.err(`kayra: ${value.argv.join(" ")} failed (exit ${String(exit.code)})`);
|
|
36
|
+
return undefined;
|
|
37
|
+
}
|
|
38
|
+
const output = exit.stdout.replace(/\n+$/, "");
|
|
39
|
+
return value.words === "one" ? [output] : output.split(/\s+/).filter((word) => word !== "");
|
|
40
|
+
}));
|
|
41
|
+
const [program, ...rest] = words.flatMap((word) => word ?? []);
|
|
42
|
+
return words.includes(undefined) || program === undefined ? undefined : [program, ...rest];
|
|
43
|
+
};
|
|
44
|
+
const lines = (text) => text
|
|
45
|
+
.trimEnd()
|
|
46
|
+
.split("\n")
|
|
47
|
+
.filter((line) => line !== "");
|
|
48
|
+
const goOn = { failed: false, stopped: false };
|
|
49
|
+
const check = async (step, io) => {
|
|
50
|
+
const exit = await answer(step.probe);
|
|
51
|
+
const hint = step.hint === undefined ? [] : [` → ${step.hint}`];
|
|
52
|
+
if (step.onFailure === "stop") {
|
|
53
|
+
if (exit.code === 0)
|
|
54
|
+
return goOn;
|
|
55
|
+
[...lines(exit.stderr), ...hint].forEach(io.err);
|
|
56
|
+
return { failed: true, stopped: true };
|
|
57
|
+
}
|
|
58
|
+
if (exit.code === 0) {
|
|
59
|
+
io.out(`ok ${step.label}`);
|
|
60
|
+
return goOn;
|
|
61
|
+
}
|
|
62
|
+
[`FAIL ${step.label}`, ...lines(exit.stderr).map((line) => ` ${line}`), ...hint].forEach(io.out);
|
|
63
|
+
return { failed: true, stopped: false };
|
|
64
|
+
};
|
|
65
|
+
const perform = async (step, io) => {
|
|
66
|
+
switch (step.kind) {
|
|
67
|
+
case "run": {
|
|
68
|
+
const argv = await resolve(step.argv, io);
|
|
69
|
+
if (argv === undefined)
|
|
70
|
+
return { failed: true, stopped: true };
|
|
71
|
+
const exit = await spawned(argv, { stdout: step.stdout, stderr: "show" }, step.env);
|
|
72
|
+
return exit.code === 0 ? goOn : { failed: true, stopped: true };
|
|
73
|
+
}
|
|
74
|
+
case "check":
|
|
75
|
+
return check(step, io);
|
|
76
|
+
case "confirm": {
|
|
77
|
+
const typed = await io.ask(step.prompt);
|
|
78
|
+
if (typed === step.expect)
|
|
79
|
+
return goOn;
|
|
80
|
+
io.out("Aborted.");
|
|
81
|
+
return { failed: false, stopped: true };
|
|
82
|
+
}
|
|
83
|
+
case "note":
|
|
84
|
+
io.out(step.text);
|
|
85
|
+
return goOn;
|
|
86
|
+
}
|
|
87
|
+
};
|
|
88
|
+
/** Runs a plan step by step, until it ends or a step stops it. True if nothing failed. */
|
|
89
|
+
export const interpret = async (plan, io) => {
|
|
90
|
+
const outcome = await plan.reduce(async (previous, step) => {
|
|
91
|
+
const so = await previous;
|
|
92
|
+
if (so.stopped)
|
|
93
|
+
return so;
|
|
94
|
+
const next = await perform(step, io);
|
|
95
|
+
return { failures: so.failures + (next.failed ? 1 : 0), stopped: next.stopped };
|
|
96
|
+
}, Promise.resolve({ failures: 0, stopped: false }));
|
|
97
|
+
return outcome.failures === 0;
|
|
98
|
+
};
|
package/cli/dist/load.js
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { fleetRoot, serverDir, serverName, within } from "./brand.js";
|
|
4
|
+
import { parseServer } from "./fleet.js";
|
|
5
|
+
import { all, err, map, ok } from "./result.js";
|
|
6
|
+
/** The fleet is the nearest directory, from `from` upwards, with a servers/defaults.yml. */
|
|
7
|
+
export const findFleetRoot = (from) => {
|
|
8
|
+
if (existsSync(join(from, "servers", "defaults.yml")))
|
|
9
|
+
return ok(fleetRoot(from));
|
|
10
|
+
const parent = within(from, "..");
|
|
11
|
+
return parent === from
|
|
12
|
+
? err("no fleet here: no servers/defaults.yml in this directory or above it")
|
|
13
|
+
: findFleetRoot(parent);
|
|
14
|
+
};
|
|
15
|
+
const text = (path) => (existsSync(path) ? readFileSync(path, "utf8") : undefined);
|
|
16
|
+
const stacks = (dir) => existsSync(join(dir, "stacks"))
|
|
17
|
+
? readdirSync(join(dir, "stacks"))
|
|
18
|
+
.toSorted()
|
|
19
|
+
.flatMap((stack) => text(join(dir, "stacks", stack, "compose.yaml")) ?? [])
|
|
20
|
+
: [];
|
|
21
|
+
const appValues = (dir) => existsSync(join(dir, "apps"))
|
|
22
|
+
? readdirSync(join(dir, "apps"))
|
|
23
|
+
.filter((file) => file.endsWith(".yml"))
|
|
24
|
+
.toSorted()
|
|
25
|
+
.map((file) => ({ file, text: readFileSync(join(dir, "apps", file), "utf8") }))
|
|
26
|
+
: [];
|
|
27
|
+
const readServer = (root, entry) => {
|
|
28
|
+
const name = serverName(entry);
|
|
29
|
+
if (!name.ok)
|
|
30
|
+
return err(`servers/${entry}: ${name.error}`);
|
|
31
|
+
const dir = serverDir(root, name.value);
|
|
32
|
+
const server = parseServer({
|
|
33
|
+
name: name.value,
|
|
34
|
+
dir,
|
|
35
|
+
descriptor: readFileSync(join(dir, "server.yml"), "utf8"),
|
|
36
|
+
vars: text(join(dir, "vars.yml")),
|
|
37
|
+
secrets: text(join(dir, "secrets.yml")),
|
|
38
|
+
inventory: text(join(dir, "inventory.ini")),
|
|
39
|
+
terraform: existsSync(join(dir, "terraform")),
|
|
40
|
+
stacks: stacks(dir),
|
|
41
|
+
appValues: appValues(dir),
|
|
42
|
+
});
|
|
43
|
+
return server.ok ? server : err(`servers/${entry}/server.yml: ${server.error}`);
|
|
44
|
+
};
|
|
45
|
+
export const loadFleet = (root) => map(all(readdirSync(join(root, "servers"), { withFileTypes: true })
|
|
46
|
+
.filter((entry) => entry.isDirectory() && existsSync(join(root, "servers", entry.name, "server.yml")))
|
|
47
|
+
.map((entry) => entry.name)
|
|
48
|
+
.toSorted()
|
|
49
|
+
.map((entry) => readServer(root, entry))), (servers) => ({ root, servers }));
|
package/cli/dist/main.js
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { run } from "@optique/run";
|
|
3
|
+
import { readFileSync } from "node:fs";
|
|
4
|
+
import { createInterface } from "node:readline/promises";
|
|
5
|
+
import { absolutePath, kayraRoot, kayraVersion, within } from "./brand.js";
|
|
6
|
+
import { parser } from "./cli.js";
|
|
7
|
+
import { doctor } from "./commands/doctor.js";
|
|
8
|
+
import * as server from "./commands/server.js";
|
|
9
|
+
import { status } from "./commands/status.js";
|
|
10
|
+
import { serverNamed } from "./fleet.js";
|
|
11
|
+
import { interpret } from "./interpret.js";
|
|
12
|
+
import { findFleetRoot, loadFleet } from "./load.js";
|
|
13
|
+
import { render } from "./plan.js";
|
|
14
|
+
import { flatMap, ok } from "./result.js";
|
|
15
|
+
const here = absolutePath(process.cwd());
|
|
16
|
+
const engine = absolutePath(import.meta.dirname);
|
|
17
|
+
if (!here.ok || !engine.ok)
|
|
18
|
+
throw new Error("node reported a relative path");
|
|
19
|
+
const kayra = kayraRoot(within(engine.value, "..", ".."));
|
|
20
|
+
const manifest = JSON.parse(readFileSync(within(kayra, "package.json"), "utf8"));
|
|
21
|
+
const version = kayraVersion(typeof manifest === "object" && manifest !== null && "version" in manifest ? manifest.version : undefined);
|
|
22
|
+
if (!version.ok)
|
|
23
|
+
throw new Error(`kayra's package.json: ${version.error}`);
|
|
24
|
+
const onServer = (on, target, command) => {
|
|
25
|
+
switch (command.name) {
|
|
26
|
+
case "up":
|
|
27
|
+
return server.up(on, target);
|
|
28
|
+
case "provision":
|
|
29
|
+
return server.provision(on, target);
|
|
30
|
+
case "deploy":
|
|
31
|
+
return server.deploy(on, target);
|
|
32
|
+
case "setup":
|
|
33
|
+
return server.setup(on, target);
|
|
34
|
+
case "apps":
|
|
35
|
+
return server.apps(on, target);
|
|
36
|
+
case "hosts":
|
|
37
|
+
return server.hosts(on, target);
|
|
38
|
+
case "ssh":
|
|
39
|
+
return server.ssh(target);
|
|
40
|
+
case "down":
|
|
41
|
+
return server.down(target);
|
|
42
|
+
case "adopt":
|
|
43
|
+
return server.adopt(on, target);
|
|
44
|
+
case "backup test":
|
|
45
|
+
return server.backupTest(on, target);
|
|
46
|
+
}
|
|
47
|
+
};
|
|
48
|
+
const plan = (fleet, command) => {
|
|
49
|
+
const on = { kayra, version: version.value, fleet: fleet.root };
|
|
50
|
+
if (command.name === "doctor")
|
|
51
|
+
return ok(doctor(fleet, kayra));
|
|
52
|
+
if (command.name === "certs")
|
|
53
|
+
return server.certsFor(on, command.domain, command.subdomains);
|
|
54
|
+
if (command.name === "app purge")
|
|
55
|
+
return flatMap(serverNamed(fleet, command.server), (target) => server.appPurge(on, target, command.app));
|
|
56
|
+
return flatMap(serverNamed(fleet, command.server), (target) => onServer(on, target, command));
|
|
57
|
+
};
|
|
58
|
+
const io = {
|
|
59
|
+
out: (line) => {
|
|
60
|
+
console.log(line);
|
|
61
|
+
},
|
|
62
|
+
err: (line) => {
|
|
63
|
+
console.error(line);
|
|
64
|
+
},
|
|
65
|
+
ask: async (prompt) => {
|
|
66
|
+
const terminal = createInterface({ input: process.stdin, output: process.stderr });
|
|
67
|
+
const answer = await terminal.question(prompt);
|
|
68
|
+
terminal.close();
|
|
69
|
+
return answer;
|
|
70
|
+
},
|
|
71
|
+
};
|
|
72
|
+
const fail = (message) => {
|
|
73
|
+
console.error(`kayra: ${message}`);
|
|
74
|
+
process.exit(1);
|
|
75
|
+
};
|
|
76
|
+
const fleet = flatMap(findFleetRoot(here.value), loadFleet);
|
|
77
|
+
if (!fleet.ok)
|
|
78
|
+
fail(fleet.error);
|
|
79
|
+
else {
|
|
80
|
+
const args = run(parser(fleet.value), { programName: "kayra", help: "both", completion: "both" });
|
|
81
|
+
if (args.command.name === "status") {
|
|
82
|
+
(await status(fleet.value)).forEach(io.out);
|
|
83
|
+
}
|
|
84
|
+
else {
|
|
85
|
+
const steps = plan(fleet.value, args.command);
|
|
86
|
+
if (!steps.ok)
|
|
87
|
+
fail(steps.error);
|
|
88
|
+
else if (args.dryRun)
|
|
89
|
+
render(steps.value).forEach(io.out);
|
|
90
|
+
// eslint-disable-next-line functional/immutable-data -- the exit status; process.exit() could cut piped output short
|
|
91
|
+
else
|
|
92
|
+
process.exitCode = (await interpret(steps.value, io)) ? 0 : 1;
|
|
93
|
+
}
|
|
94
|
+
}
|
package/cli/dist/plan.js
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
export const run = (argv, env = {}) => ({
|
|
2
|
+
kind: "run",
|
|
3
|
+
argv,
|
|
4
|
+
env,
|
|
5
|
+
stdout: "show",
|
|
6
|
+
});
|
|
7
|
+
const quote = (word) => (/^[\w@%+=:,./~-]+$/.test(word) ? word : `'${word.replaceAll("'", `'\\''`)}'`);
|
|
8
|
+
export const shell = (argv) => argv.map(quote).join(" ");
|
|
9
|
+
const assignments = (env = {}) => Object.entries(env)
|
|
10
|
+
.map(([key, value]) => `${key}=${quote(value)} `)
|
|
11
|
+
.join("");
|
|
12
|
+
const arg = (value) => {
|
|
13
|
+
if (typeof value === "string")
|
|
14
|
+
return quote(value);
|
|
15
|
+
const command = `$(${assignments(value.env)}${shell(value.argv)})`;
|
|
16
|
+
return value.words === "one" ? `"${command}"` : command;
|
|
17
|
+
};
|
|
18
|
+
const describeProbe = (probe) => probe.kind === "on-path" ? `command -v ${quote(probe.program)}` : assignments(probe.env) + shell(probe.argv);
|
|
19
|
+
const describe = (step) => {
|
|
20
|
+
switch (step.kind) {
|
|
21
|
+
case "run":
|
|
22
|
+
return `run ${assignments(step.env)}${step.argv.map(arg).join(" ")}${step.stdout === "discard" ? " >/dev/null" : ""}`;
|
|
23
|
+
case "check":
|
|
24
|
+
return `${step.onFailure === "stop" ? "guard " : "check "} ${step.label}: ${describeProbe(step.probe)}`;
|
|
25
|
+
case "confirm":
|
|
26
|
+
return `confirm ${step.prompt}(${step.expect})`;
|
|
27
|
+
case "note":
|
|
28
|
+
return `note ${step.text}`;
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
export const render = (plan) => plan.map(describe);
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export const ok = (value) => ({ ok: true, value });
|
|
2
|
+
export const err = (error) => ({ ok: false, error });
|
|
3
|
+
export const map = (result, f) => result.ok ? ok(f(result.value)) : result;
|
|
4
|
+
export const flatMap = (result, f) => (result.ok ? f(result.value) : result);
|
|
5
|
+
export const all = (results) => results.reduce((acc, result) => flatMap(acc, (values) => map(result, (value) => [...values, value])), ok([]));
|