@yunuseon/kayra 0.20.0 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/README.md +2 -2
  2. package/agent/skills/kayra-rollout/SKILL.md +1 -1
  3. package/ansible/.ansible-lint +2 -2
  4. package/ansible/playbooks/app.yml +1 -1
  5. package/ansible/playbooks/edge.yml +1 -1
  6. package/ansible/playbooks/nas.yml +1 -1
  7. package/ansible/playbooks/platform.yml +2 -2
  8. package/ansible/playbooks/restore-test.yml +24 -0
  9. package/ansible/roles/app/tasks/main.yml +20 -26
  10. package/ansible/roles/forgejo-config/tasks/main.yml +1 -1
  11. package/ansible/roles/platform/defaults/main.yml +1 -1
  12. package/ansible/roles/platform/tasks/main.yml +1 -1
  13. package/cli/dist/app-hook.js +6 -13
  14. package/cli/dist/app-launch.js +1 -1
  15. package/cli/dist/app-values.js +1 -1
  16. package/cli/dist/clients/mcp/run.js +1 -1
  17. package/cli/dist/clients/mcp/server.js +1 -1
  18. package/cli/dist/core/adapters/sops.js +19 -0
  19. package/cli/dist/core/app-values/store.js +46 -57
  20. package/cli/dist/core/fleet/brand.js +2 -2
  21. package/cli/dist/core/fleet/docs.js +1 -1
  22. package/cli/dist/core/fleet/parse.js +19 -29
  23. package/cli/dist/core/fleet/secrets.js +59 -0
  24. package/cli/dist/core/fleet/settings.js +5 -6
  25. package/cli/dist/core/interpreter/loop.js +8 -3
  26. package/cli/dist/core/interpreter/needs.js +54 -26
  27. package/cli/dist/core/interpreter/process.js +2 -2
  28. package/cli/dist/core/operations/catalog.js +1 -1
  29. package/cli/dist/core/operations/dispatch.js +1 -1
  30. package/cli/dist/core/plan/describe.js +2 -2
  31. package/cli/dist/core/recipes/app-hook.js +2 -2
  32. package/cli/dist/core/recipes/app-values.js +1 -2
  33. package/cli/dist/core/recipes/apps.js +5 -6
  34. package/cli/dist/core/recipes/backup.js +2 -15
  35. package/cli/dist/core/recipes/doctor.js +8 -11
  36. package/cli/dist/core/recipes/host.js +11 -11
  37. package/cli/dist/core/recipes/launch.js +3 -3
  38. package/cli/dist/core/recipes/steps.js +7 -20
  39. package/cli/dist/core/validation/facts.js +37 -27
  40. package/cli/dist/core/validation/findings.js +23 -4
  41. package/compose/authelia/configuration.yml +1 -1
  42. package/package.json +1 -1
  43. package/schema/app-values.json +1 -1
  44. package/schema/app.json +1 -1
  45. package/schema/server.json +2 -7
  46. package/scripts/local/authelia-dev-init.sh +2 -2
  47. package/scripts/server/app-restore +39 -0
  48. package/terraform/modules/platform-dns/tests/fleet/servers/nested/server.yml +0 -1
  49. package/terraform/modules/platform-dns/tests/fleet/servers/shop/server.yml +0 -1
  50. package/terraform/modules/platform-dns/tests/fleet/servers/uncloned/server.yml +0 -1
@@ -1,48 +1,58 @@
1
- import { existsSync } from "node:fs";
2
- import { homedir } from "node:os";
3
- import { join } from "node:path";
1
+ import { join, relative } from "node:path";
4
2
  import { textAt } from "../adapters/fs.js";
5
- import { isVaulted } from "../fleet/parse.js";
6
- import { fields } from "../../lib/yaml.js";
7
- import { capture } from "../interpreter/process.js";
3
+ import { appSecretsIn, hostSecretsIn, isEncrypted, secretsShape } from "../fleet/secrets.js";
4
+ import { fields, mapping } from "../../lib/yaml.js";
8
5
  import { typed } from "../../lib/typed.js";
9
6
  /** YAML as validate reads it: what it cannot read holds no values. */
10
7
  const valuesOf = (text) => {
11
8
  const parsed = text === undefined ? undefined : fields(text);
12
9
  return parsed?.ok === true ? parsed.value : typed({});
13
10
  };
14
- /** A secrets file's values: plaintext as is, a vault only with the server's password file; a missing file has none. */
15
- const secretsOf = async (server, file, text) => {
16
- if (text === undefined)
17
- return typed({ kind: "readable", values: {} });
18
- if (!isVaulted(text))
19
- return typed({ kind: "readable", values: valuesOf(text) });
20
- if (server.vault.kind !== "file")
21
- return typed({ kind: "locked", reason: "vaulted, and server.yml names no vault_password_file", hint: "vault_password_file in server.yml lets kayra open it" });
22
- const password = server.vault.file.replace(homedir(), "~");
23
- if (!existsSync(server.vault.file))
24
- return typed({ kind: "locked", reason: `vaulted, and there is no ${password} here`, hint: "put the vault's password there" });
25
- const opened = await capture(["ansible-vault", "view", "--vault-password-file", server.vault.file, join(server.dir, file)], "capture");
26
- return typed(opened.code === 0 ? { kind: "readable", values: valuesOf(opened.stdout) } : { kind: "locked", reason: `${password} does not open it`, hint: "is it this vault's password?" });
11
+ /** secrets.yml without its key: SOPS leaves the keys, and each value's type. */
12
+ const secretsOf = (text) => {
13
+ const shape = secretsShape(text);
14
+ return shape.ok ? typed({ kind: "readable", values: shape.value }) : typed({ kind: "locked", reason: shape.error });
27
15
  };
16
+ const ageRecipients = (value) => Object.freeze((typeof value === "string" ? value.split(",") : []).map((each) => each.trim()).filter((each) => each !== "").toSorted());
28
17
  /**
29
- * What validate needs of a server: its files as the fleet loaded them, its
30
- * vaults opened, and each app's app.yml from the fleet's clone (.cache/apps).
18
+ * Whom secrets.yml is encrypted to, and whom .sops.yaml's first rule matching
19
+ * its path says it should be: they drift when a key is added to .sops.yaml
20
+ * and `sops updatekeys` is not run.
31
21
  */
32
- export const gather = async (root, server) => {
22
+ const recipientsOf = (root, server, text) => {
23
+ if (!isEncrypted(text))
24
+ return undefined;
25
+ const path = relative(root, server.secrets.file);
26
+ const rules = mapping(valuesOf(textAt(join(root, ".sops.yaml"))))?.["creation_rules"];
27
+ const rule = (Array.isArray(rules) ? rules : [])
28
+ .map((each) => mapping(each) ?? {})
29
+ .find((each) => typeof each["path_regex"] !== "string" || new RegExp(each["path_regex"]).test(path));
30
+ const metadata = mapping(valuesOf(text)["sops"])?.["age"];
31
+ const actual = Object.freeze((Array.isArray(metadata) ? metadata : []).map((each) => mapping(each)?.["recipient"]).filter((each) => typeof each === "string").toSorted());
32
+ return typed({ wanted: rule === undefined ? undefined : ageRecipients(rule["age"]), actual });
33
+ };
34
+ /**
35
+ * What validate needs of a server: its files as the fleet loaded them,
36
+ * secrets.yml's keys (no key needed), and each app's app.yml from the
37
+ * fleet's clone (.cache/apps).
38
+ */
39
+ export const gather = (root, server) => {
33
40
  const appText = (file) => server.files.appValues.find((each) => each.file === file)?.text;
34
- const appSecrets = await Promise.all(server.apps.map(async (app) => [app.name, await secretsOf(server, join("apps", `${app.name}.secrets.yml`), appText(`${app.name}.secrets.yml`))]));
35
- return typed({
41
+ const secrets = secretsOf(server.files.secrets);
42
+ const all = secrets.kind === "readable" ? secrets.values : {};
43
+ return Promise.resolve(typed({
36
44
  server,
37
45
  vars: server.vars,
38
46
  varsText: server.files.vars,
39
- secrets: await secretsOf(server, "secrets.yml", server.files.secrets),
47
+ secrets: secrets.kind === "readable" ? { kind: "readable", values: hostSecretsIn(all) } : secrets,
48
+ recipients: recipientsOf(root, server, server.files.secrets),
40
49
  stackDirs: server.files.stacks,
41
50
  appConfig: Object.fromEntries(server.apps.map((app) => [app.name, valuesOf(appText(`${app.name}.yml`))])),
42
- appSecrets: Object.fromEntries(appSecrets),
51
+ appSecrets: Object.fromEntries(server.apps.map((app) => [app.name, secrets.kind === "readable" ? typed({ kind: "readable", values: appSecretsIn(all, app.name) }) : secrets])),
52
+ terraform: mapping(all["terraform"]) ?? {},
43
53
  manifests: Object.fromEntries(server.apps.flatMap((app) => {
44
54
  const manifest = textAt(join(root, ".cache", "apps", app.name, app.path, "app.yml"));
45
55
  return manifest === undefined ? Object.freeze([]) : Object.freeze([[app.name, valuesOf(manifest)]]);
46
56
  })),
47
- });
57
+ }));
48
58
  };
@@ -34,6 +34,25 @@ const secretsFindings = (facts) => {
34
34
  });
35
35
  return holds([...typeProblems, ...unused], `secrets.yml: ${count(Object.keys(values).length, "secret")}, each a setting or used by vars.yml`);
36
36
  };
37
+ /** secrets.yml is encrypted to exactly the age recipients .sops.yaml names for it. */
38
+ const recipientFindings = (facts) => {
39
+ const recipients = facts.recipients;
40
+ if (recipients === undefined)
41
+ return typed([]);
42
+ if (recipients.wanted === undefined)
43
+ return typed([problem("secrets.yml: no creation rule in .sops.yaml matches it", "add a rule whose path_regex matches it, with its age recipients")]);
44
+ const missing = recipients.wanted.filter((each) => !recipients.actual.includes(each));
45
+ const extra = recipients.actual.filter((each) => !recipients.wanted?.includes(each));
46
+ const fix = `sops updatekeys servers/${facts.server.name}/secrets.yml`;
47
+ return holds([
48
+ ...missing.map((each) => problem(`secrets.yml is not encrypted to ${each}, which .sops.yaml names`, fix)),
49
+ ...extra.map((each) => problem(`secrets.yml is encrypted to ${each}, which .sops.yaml no longer names`, `${fix}, then rotate what that key could read`)),
50
+ ], `secrets.yml: encrypted to the ${count(recipients.wanted.length, "recipient")} .sops.yaml names`);
51
+ };
52
+ /** secrets.yml's terraform: OpenTofu's variables are flat values. */
53
+ const terraformFindings = (facts) => typed(Object.entries(facts.terraform)
54
+ .filter(([, value]) => !["string", "number", "boolean"].includes(typeof value))
55
+ .map(([key]) => problem(`secrets.yml: terraform.${key} must be a string, number or boolean`)));
37
56
  /** compose_stacks.stacks and stacks/ name the same stacks. */
38
57
  const stackFindings = (facts) => {
39
58
  const listed = strings(mapping(facts.vars["compose_stacks"])["stacks"]);
@@ -80,24 +99,24 @@ const appFindings = (facts) => Object.freeze(facts.server.apps.flatMap((app) =>
80
99
  const secretKeys = parsed.data.secrets ?? [];
81
100
  const config = facts.appConfig[app.name] ?? {};
82
101
  const valuesFile = `apps/${app.name}.yml`;
83
- const secretsFile = `apps/${app.name}.secrets.yml`;
102
+ const secretsFile = "secrets.yml";
84
103
  const missingConfig = configKeys
85
104
  .filter((key) => !(key in config))
86
105
  .map((key) => problem(`${valuesFile}: no value for ${key}, which app.yml asks for`, `add ${key}: … to servers/${facts.server.name}/${valuesFile}`));
87
106
  const secrets = facts.appSecrets[app.name] ?? { kind: "readable", values: {} };
88
- const edit = facts.server.target === "multipass" ? "add it to" : "ansible-vault edit";
107
+ const edit = `kayra app secret generate ${facts.server.name} ${app.name}`;
89
108
  const missingSecrets = secrets.kind === "locked"
90
109
  ? secretKeys.length === 0
91
110
  ? []
92
111
  : [unchecked(`${secretsFile} not checked: ${secrets.reason}`, secrets.hint)]
93
112
  : secretKeys
94
113
  .filter((key) => !(key in secrets.values))
95
- .map((key) => problem(`${secretsFile}: no value for ${key}, which app.yml asks for`, `${edit} servers/${facts.server.name}/${secretsFile}`));
114
+ .map((key) => problem(`${secretsFile}: no app_secrets.${app.name}.${key}, which app.yml asks for`, `${edit} ${key} (or app secret import)`));
96
115
  return holds([...missingConfig, ...missingSecrets], `app ${app.name}: ${count(configKeys.length, "config value")} and ${count(secretKeys.length, "secret")}, as its app.yml asks`);
97
116
  }));
98
117
  const rank = { problem: 0, unchecked: 1, passed: 2 };
99
118
  /** Everything validate finds for one server: problems first, then what it could not check, then what holds. */
100
- export const findings = (facts) => Object.freeze([...secretsFindings(facts), ...stackFindings(facts), ...routeFindings(facts), ...appFindings(facts)].toSorted((a, b) => rank[a.result] - rank[b.result]));
119
+ export const findings = (facts) => Object.freeze([...secretsFindings(facts), ...recipientFindings(facts), ...terraformFindings(facts), ...stackFindings(facts), ...routeFindings(facts), ...appFindings(facts)].toSorted((a, b) => rank[a.result] - rank[b.result]));
101
120
  /** How a set of findings went as a whole: its worst. */
102
121
  export const overall = (found) => verdicts.find((verdict) => found.some((each) => each.result === verdict)) ?? "passed";
103
122
  /** How many problems, and how many checks could not look, across reports. */
@@ -56,7 +56,7 @@ identity_validation:
56
56
  authentication_backend:
57
57
  refresh_interval: 'always'
58
58
  # Both disabled so Authelia never writes back to users_database.yml. That
59
- # file is rendered from vault on every platform run and would otherwise be
59
+ # file is rendered from secrets.yml on every platform run and would otherwise be
60
60
  # silently reverted.
61
61
  password_reset:
62
62
  disable: true
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@yunuseon/kayra",
3
- "version": "0.20.0",
3
+ "version": "0.21.0",
4
4
  "description": "A self-hosted app platform and the CLI that runs it: Traefik, Authelia, a registry, rootless BuildKit and a small deployer per server, set up with OpenTofu and Ansible.",
5
5
  "license": "MIT",
6
6
  "author": "Yunus Öztürk",
@@ -11,5 +11,5 @@
11
11
  "boolean"
12
12
  ]
13
13
  },
14
- "title": "apps/<app>.yml and apps/<app>.secrets.yml"
14
+ "title": "apps/<app>.yml"
15
15
  }
package/schema/app.json CHANGED
@@ -85,7 +85,7 @@
85
85
  }
86
86
  },
87
87
  "secrets": {
88
- "description": "Secret keys; their values come from the fleet's apps/<app>.secrets.yml.",
88
+ "description": "Secret keys; their values come from the fleet's secrets.yml, under app_secrets.<app>.",
89
89
  "type": "array",
90
90
  "items": {
91
91
  "type": "string"
@@ -26,13 +26,8 @@
26
26
  "type": "string",
27
27
  "pattern": "^([a-z0-9]([a-z0-9-]*[a-z0-9])?\\.)+[a-z][a-z0-9-]*$"
28
28
  },
29
- "vault": {
30
- "default": false,
31
- "description": "secrets.yml is vault-encrypted (required outside multipass).",
32
- "type": "boolean"
33
- },
34
- "vault_password_file": {
35
- "description": "The vault password file; without it, Ansible asks.",
29
+ "age_key_file": {
30
+ "description": "The age identity that opens secrets.yml (SOPS); without it, SOPS's own default (SOPS_AGE_KEY_FILE, ~/.config/sops/age/keys.txt).",
36
31
  "type": "string"
37
32
  },
38
33
  "apps": {
@@ -2,7 +2,7 @@
2
2
  # Generate local Authelia secrets and a dev user database.
3
3
  #
4
4
  # Mirrors scripts/local/certs.sh: local-only material, generated on demand and
5
- # gitignored, never committed. Production gets both from Ansible + vault.
5
+ # gitignored, never committed. Production gets both from Ansible + secrets.yml.
6
6
  #
7
7
  # Usage: bash scripts/local/authelia-dev-init.sh <authelia-dir> <dev-email> <dev-password>
8
8
  set -euo pipefail
@@ -43,7 +43,7 @@ HASH="$(docker run --rm "${IMAGE}" \
43
43
 
44
44
  cat > "${USERS_FILE}" <<YML
45
45
  # Local development only — generated by scripts/local/authelia-dev-init.sh.
46
- # Production renders this from vault; see the platform Ansible role.
46
+ # Production renders this from secrets.yml; see the platform Ansible role.
47
47
  users:
48
48
  dev:
49
49
  disabled: false
@@ -0,0 +1,39 @@
1
+ #!/usr/bin/env bash
2
+ # =============================================================================
3
+ # app-restore <app> — an app that was deployed, yet runs no container now,
4
+ # deployed again at the release that ran.
5
+ #
6
+ # A Docker update or a reboot stops every container; one deployed before the
7
+ # platform gave each service a restart policy stays down. The deployer's ctl
8
+ # follows the run to its end, so this exits as the run did. An app never
9
+ # deployed (no release record) and one that runs are left alone.
10
+ #
11
+ # Installed by the platform role at <platform_dir>/bin; run by the app role
12
+ # after it registers the app. Run as root. Prints what it did on its last line:
13
+ # "never deployed", "runs", or "restored <sha>".
14
+ # =============================================================================
15
+ set -euo pipefail
16
+
17
+ app="${1:?Usage: app-restore <app>}"
18
+ platform_dir="${PLATFORM_DIR:-$(cd "$(dirname "$0")/.." && pwd)}"
19
+ releases="${platform_dir}/releases/${app}"
20
+
21
+ # releases/<app>/<UTC time>-<sha>.env, newest last; the deployer writes them.
22
+ live=""
23
+ if [ -d "$releases" ]; then
24
+ live="$(find "$releases" -maxdepth 1 -name '*.env' -exec basename {} \; | sort | tail -1 \
25
+ | sed -n 's/^[0-9]\{8\}T[0-9]\{6\}Z-\([0-9a-f]\{40\}\)\.env$/\1/p')"
26
+ fi
27
+ if [ -z "$live" ]; then
28
+ echo "never deployed"
29
+ exit 0
30
+ fi
31
+
32
+ if [ -n "$(docker ps -q --filter "label=com.docker.compose.project=${app}" --filter status=running --filter status=restarting)" ]; then
33
+ echo "runs"
34
+ exit 0
35
+ fi
36
+
37
+ cd "$platform_dir"
38
+ docker compose exec -T deployer kayra-deployer ctl deploy "$app" "$live"
39
+ echo "restored ${live}"
@@ -1,6 +1,5 @@
1
1
  kind: platform
2
2
  target: hetzner
3
3
  domain: shop.example.com
4
- vault: true
5
4
  apps:
6
5
  - {name: shop, repo: https://github.com/acme/shop.git}
@@ -1,7 +1,6 @@
1
1
  kind: platform
2
2
  target: hetzner
3
3
  domain: example.com
4
- vault: true
5
4
  apps:
6
5
  - {name: shop, repo: https://github.com/acme/shop.git}
7
6
  - {name: blog, repo: https://github.com/acme/blog.git, path: site}
@@ -1,6 +1,5 @@
1
1
  kind: platform
2
2
  target: hetzner
3
3
  domain: example.com
4
- vault: true
5
4
  apps:
6
5
  - {name: missing, repo: https://github.com/acme/missing.git}