@celilo/cli 0.3.30 → 0.4.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. package/drizzle/0005_module_operations.sql +12 -0
  2. package/drizzle/0006_base_module_aspects.sql +15 -0
  3. package/drizzle/0007_module_systems.sql +17 -0
  4. package/drizzle/meta/_journal.json +21 -0
  5. package/package.json +6 -5
  6. package/schemas/system_config.json +14 -28
  7. package/src/ansible/inventory.test.ts +46 -62
  8. package/src/ansible/inventory.ts +48 -25
  9. package/src/capabilities/registration.ts +25 -7
  10. package/src/capabilities/validation.test.ts +30 -0
  11. package/src/capabilities/validation.ts +8 -0
  12. package/src/cli/backup-rename.test.ts +95 -0
  13. package/src/cli/cli.test.ts +17 -23
  14. package/src/cli/command-registry.ts +199 -0
  15. package/src/cli/commands/backup-list.ts +1 -1
  16. package/src/cli/commands/events.ts +96 -0
  17. package/src/cli/commands/machine-add.ts +103 -59
  18. package/src/cli/commands/module-import.ts +153 -4
  19. package/src/cli/commands/module-remove.ts +86 -17
  20. package/src/cli/commands/module-status.ts +6 -2
  21. package/src/cli/commands/publish/alpha.test.ts +185 -0
  22. package/src/cli/commands/publish/alpha.ts +226 -0
  23. package/src/cli/commands/publish/changesets.test.ts +89 -0
  24. package/src/cli/commands/publish/changesets.ts +144 -0
  25. package/src/cli/commands/publish/consumer-pins.test.ts +155 -0
  26. package/src/cli/commands/publish/consumer-pins.ts +149 -0
  27. package/src/cli/commands/publish/execute.ts +131 -0
  28. package/src/cli/commands/publish/global-install.test.ts +154 -0
  29. package/src/cli/commands/publish/global-install.ts +171 -0
  30. package/src/cli/commands/publish/helpers.ts +227 -0
  31. package/src/cli/commands/publish/index.ts +365 -0
  32. package/src/cli/commands/publish/module-registry.test.ts +40 -0
  33. package/src/cli/commands/publish/module-registry.ts +64 -0
  34. package/src/cli/commands/publish/plan.ts +107 -0
  35. package/src/cli/commands/publish/preflight.ts +238 -0
  36. package/src/cli/commands/publish/types.ts +264 -0
  37. package/src/cli/commands/publish/workspace.test.ts +323 -0
  38. package/src/cli/commands/publish/workspace.ts +596 -0
  39. package/src/cli/commands/restore.ts +126 -0
  40. package/src/cli/commands/storage-add-local.ts +1 -1
  41. package/src/cli/commands/storage-add-s3.ts +1 -1
  42. package/src/cli/commands/subscribers-add.ts +68 -0
  43. package/src/cli/commands/subscribers-list.ts +48 -0
  44. package/src/cli/commands/subscribers-remove.ts +38 -0
  45. package/src/cli/commands/subscribers-serve.ts +77 -0
  46. package/src/cli/commands/subscribers-status.ts +33 -0
  47. package/src/cli/commands/subscribers-test.ts +71 -0
  48. package/src/cli/commands/system-apply-config-equivalence.test.ts +108 -0
  49. package/src/cli/commands/system-apply-config.test.ts +70 -0
  50. package/src/cli/commands/system-apply-config.ts +130 -0
  51. package/src/cli/commands/system-audit.ts +2 -1
  52. package/src/cli/commands/system-init-deprecation.test.ts +90 -0
  53. package/src/cli/commands/system-init.ts +36 -70
  54. package/src/cli/commands/system-update.ts +3 -2
  55. package/src/cli/completion.ts +22 -1
  56. package/src/cli/index.ts +214 -6
  57. package/src/cli/interactive-config.test.ts +19 -0
  58. package/src/cli/restore-command.test.ts +131 -0
  59. package/src/db/client.ts +42 -0
  60. package/src/db/schema.test.ts +13 -16
  61. package/src/db/schema.ts +161 -9
  62. package/src/hooks/capability-loader-firewall.test.ts +6 -15
  63. package/src/hooks/capability-loader.test.ts +2 -3
  64. package/src/hooks/capability-loader.ts +36 -2
  65. package/src/hooks/define-hook.test.ts +4 -0
  66. package/src/hooks/executor.test.ts +18 -0
  67. package/src/hooks/executor.ts +21 -2
  68. package/src/hooks/load-hook-config.test.ts +26 -24
  69. package/src/hooks/load-hook-config.ts +11 -2
  70. package/src/hooks/run-named-hook.ts +16 -0
  71. package/src/hooks/types.ts +9 -1
  72. package/src/manifest/contracts/v1.ts +70 -0
  73. package/src/manifest/schema.ts +262 -16
  74. package/src/manifest/validate-privileged.test.ts +84 -0
  75. package/src/manifest/validate.test.ts +156 -0
  76. package/src/manifest/validate.ts +69 -0
  77. package/src/module/import.ts +12 -0
  78. package/src/services/aspect-approvals.test.ts +231 -0
  79. package/src/services/aspect-approvals.ts +120 -0
  80. package/src/services/aspect-runner.test.ts +493 -0
  81. package/src/services/aspect-runner.ts +438 -0
  82. package/src/services/aspect-template-resolver.test.ts +101 -0
  83. package/src/services/aspect-template-resolver.ts +122 -0
  84. package/src/services/backup-create.ts +104 -25
  85. package/src/services/backup-envelope-roundtrip.test.ts +199 -0
  86. package/src/services/backup-in-flight-refusal.test.ts +163 -0
  87. package/src/services/backup-manifest.test.ts +115 -0
  88. package/src/services/backup-manifest.ts +163 -0
  89. package/src/services/backup-restore.ts +154 -19
  90. package/src/services/build-bus/delivery-events.ts +92 -0
  91. package/src/services/build-bus/event-factory.ts +54 -0
  92. package/src/services/build-bus/fan-out.test.ts +279 -0
  93. package/src/services/build-bus/fan-out.ts +161 -0
  94. package/src/services/build-bus/hook-dispatch-mgmt.test.ts +157 -0
  95. package/src/services/build-bus/hook-dispatch.test.ts +207 -0
  96. package/src/services/build-bus/hook-dispatch.ts +198 -0
  97. package/src/services/build-bus/hook-dispatcher.ts +115 -0
  98. package/src/services/build-bus/index.ts +41 -0
  99. package/src/services/build-bus/receiver-server.test.ts +179 -0
  100. package/src/services/build-bus/receiver-server.ts +159 -0
  101. package/src/services/build-bus/status.test.ts +212 -0
  102. package/src/services/build-bus/status.ts +213 -0
  103. package/src/services/build-bus/subscriber-store.ts +113 -0
  104. package/src/services/celilo-events.test.ts +70 -0
  105. package/src/services/celilo-events.ts +92 -0
  106. package/src/services/celilo-mgmt-hooks.test.ts +296 -0
  107. package/src/services/config-interview.ts +13 -95
  108. package/src/services/cross-module-data-manager.ts +2 -31
  109. package/src/services/cross-module-read.test.ts +250 -0
  110. package/src/services/cross-module-read.ts +232 -0
  111. package/src/services/deploy-validation.ts +7 -0
  112. package/src/services/deployed-systems.test.ts +235 -0
  113. package/src/services/deployed-systems.ts +308 -0
  114. package/src/services/dns-provider-backfill.ts +75 -0
  115. package/src/services/health-runner.ts +19 -3
  116. package/src/services/infrastructure-variable-resolver.test.ts +6 -32
  117. package/src/services/infrastructure-variable-resolver.ts +3 -13
  118. package/src/services/machine-detector.ts +104 -48
  119. package/src/services/machine-pool.ts +145 -2
  120. package/src/services/module-config.ts +78 -120
  121. package/src/services/module-deploy.ts +113 -40
  122. package/src/services/module-operations.test.ts +154 -0
  123. package/src/services/module-operations.ts +154 -0
  124. package/src/services/module-subscriptions.test.ts +58 -0
  125. package/src/services/module-subscriptions.ts +24 -1
  126. package/src/services/module-types-generator.test.ts +3 -3
  127. package/src/services/module-types-generator.ts +7 -2
  128. package/src/services/proxmox-reconcile.test.ts +333 -0
  129. package/src/services/proxmox-reconcile.ts +156 -0
  130. package/src/services/proxmox-state-recovery.ts +3 -24
  131. package/src/services/restore-from-file.test.ts +177 -0
  132. package/src/services/restore-from-file.ts +355 -0
  133. package/src/services/restore-preflight.test.ts +127 -0
  134. package/src/services/restore-preflight.ts +118 -0
  135. package/src/services/storage-providers/s3.ts +10 -2
  136. package/src/services/system-identity.ts +30 -0
  137. package/src/services/system-init.test.ts +64 -21
  138. package/src/services/system-init.ts +28 -26
  139. package/src/templates/generator.test.ts +7 -16
  140. package/src/templates/generator.ts +28 -115
  141. package/src/test-utils/integration.ts +5 -2
  142. package/src/types/infrastructure.ts +8 -0
  143. package/src/variables/computed/computed-integration.test.ts +191 -0
  144. package/src/variables/computed/computed.test.ts +177 -0
  145. package/src/variables/computed/evaluate.ts +271 -0
  146. package/src/variables/computed/marker.ts +53 -0
  147. package/src/variables/computed/parse.ts +262 -0
  148. package/src/variables/computed/provider-lookup.ts +130 -0
  149. package/src/variables/context.test.ts +89 -28
  150. package/src/variables/context.ts +196 -191
  151. package/src/variables/parser.ts +3 -3
  152. package/src/variables/resolver.test.ts +61 -0
  153. package/src/variables/resolver.ts +81 -0
  154. package/src/variables/types.ts +23 -1
  155. package/src/services/dns-auto-register.ts +0 -211
@@ -0,0 +1,115 @@
1
+ import { describe, expect, it } from 'bun:test';
2
+ import {
3
+ IncompatibleManifestError,
4
+ MANIFEST_SCHEMA_VERSION,
5
+ ManifestParseError,
6
+ assertCompatibleSchema,
7
+ buildManifest,
8
+ parseManifest,
9
+ } from './backup-manifest';
10
+
11
+ describe('backup-manifest', () => {
12
+ describe('buildManifest', () => {
13
+ it('produces a system manifest with the current schemaVersion', () => {
14
+ const m = buildManifest({ kind: 'system' });
15
+ expect(m.kind).toBe('system');
16
+ expect(m.schemaVersion).toBe(MANIFEST_SCHEMA_VERSION);
17
+ expect(m.moduleId).toBeUndefined();
18
+ expect(m.createdAt).toMatch(/^\d{4}-\d{2}-\d{2}T/);
19
+ });
20
+
21
+ it('produces a module manifest with moduleId/version', () => {
22
+ const m = buildManifest({
23
+ kind: 'module',
24
+ moduleId: 'homebridge',
25
+ moduleVersion: '1.2.3',
26
+ dataSchemaVersion: '2.0',
27
+ });
28
+ expect(m.kind).toBe('module');
29
+ expect(m.moduleId).toBe('homebridge');
30
+ expect(m.moduleVersion).toBe('1.2.3');
31
+ expect(m.dataSchemaVersion).toBe('2.0');
32
+ });
33
+
34
+ it('honors overrides for testability', () => {
35
+ const m = buildManifest({
36
+ kind: 'system',
37
+ now: new Date('2020-01-01T00:00:00Z'),
38
+ hostnameOverride: 'test-box',
39
+ celiloVersion: '9.9.9',
40
+ });
41
+ expect(m.createdAt).toBe('2020-01-01T00:00:00.000Z');
42
+ expect(m.hostname).toBe('test-box');
43
+ expect(m.celiloVersion).toBe('9.9.9');
44
+ });
45
+ });
46
+
47
+ describe('parseManifest', () => {
48
+ it('round-trips a built manifest', () => {
49
+ const built = buildManifest({ kind: 'system' });
50
+ const parsed = parseManifest(JSON.stringify(built));
51
+ expect(parsed).toEqual(built);
52
+ });
53
+
54
+ it('throws ManifestParseError on malformed JSON', () => {
55
+ expect(() => parseManifest('{not json')).toThrow(ManifestParseError);
56
+ });
57
+
58
+ it('throws ManifestParseError on missing required fields', () => {
59
+ expect(() => parseManifest(JSON.stringify({ kind: 'system' }))).toThrow(ManifestParseError);
60
+ });
61
+
62
+ it('throws ManifestParseError on bad schemaVersion format', () => {
63
+ expect(() =>
64
+ parseManifest(
65
+ JSON.stringify({
66
+ schemaVersion: 'not-a-version',
67
+ createdAt: '2020-01-01T00:00:00Z',
68
+ celiloVersion: '0.0.0',
69
+ hostname: 'h',
70
+ kind: 'system',
71
+ }),
72
+ ),
73
+ ).toThrow(ManifestParseError);
74
+ });
75
+
76
+ it('error message names the cause for operator triage', () => {
77
+ try {
78
+ parseManifest('not json');
79
+ } catch (err) {
80
+ expect((err as Error).message).toContain('not valid JSON');
81
+ return;
82
+ }
83
+ throw new Error('expected throw');
84
+ });
85
+ });
86
+
87
+ describe('assertCompatibleSchema', () => {
88
+ it('accepts the current schemaVersion', () => {
89
+ const m = buildManifest({ kind: 'system' });
90
+ expect(() => assertCompatibleSchema(m)).not.toThrow();
91
+ });
92
+
93
+ it('accepts the same major, different minor', () => {
94
+ const m = buildManifest({ kind: 'system' });
95
+ // Fudge the version to a higher minor of the same major.
96
+ m.schemaVersion = `${MANIFEST_SCHEMA_VERSION.split('.')[0]}.99`;
97
+ expect(() => assertCompatibleSchema(m)).not.toThrow();
98
+ });
99
+
100
+ it('rejects a different major with an actionable error', () => {
101
+ const m = buildManifest({ kind: 'system' });
102
+ const otherMajor = String(Number(MANIFEST_SCHEMA_VERSION.split('.')[0]) + 1);
103
+ m.schemaVersion = `${otherMajor}.0`;
104
+ try {
105
+ assertCompatibleSchema(m);
106
+ } catch (err) {
107
+ expect(err).toBeInstanceOf(IncompatibleManifestError);
108
+ expect((err as Error).message).toContain('envelope schema');
109
+ expect((err as Error).message).toContain('this celilo supports');
110
+ return;
111
+ }
112
+ throw new Error('expected throw');
113
+ });
114
+ });
115
+ });
@@ -0,0 +1,163 @@
1
+ /**
2
+ * Backup-artifact manifest — a single `manifest.json` file embedded at the
3
+ * root of every encrypted backup envelope (alongside the actual data). The
4
+ * manifest makes artifacts self-describing: an operator can hand a backup
5
+ * file to a different celilo install and the restore code can validate
6
+ * schemaVersion compatibility + dispatch to the right restore path
7
+ * without needing the source celilo's DB.
8
+ *
9
+ * The DB's `backups` table stays as the queryable index ("show me my
10
+ * backups") but is no longer authoritative — the artifact itself is.
11
+ *
12
+ * Schema versioning:
13
+ * - Bump the major when the file layout changes (e.g. renaming `data/`).
14
+ * - Bump the minor for additive fields.
15
+ * - Restore refuses an artifact whose major doesn't match.
16
+ */
17
+
18
+ import { hostname } from 'node:os';
19
+ import { z } from 'zod';
20
+
21
+ /** Current envelope-schema version. Bump the major on breaking changes. */
22
+ export const MANIFEST_SCHEMA_VERSION = '1.0' as const;
23
+
24
+ export const BackupManifestSchema = z.object({
25
+ schemaVersion: z.string().regex(/^\d+\.\d+$/),
26
+ createdAt: z.string(), // ISO 8601
27
+ celiloVersion: z.string(),
28
+ hostname: z.string(),
29
+ kind: z.enum(['system', 'module']),
30
+ /** Only present when kind='module'. */
31
+ moduleId: z.string().optional(),
32
+ /** Only present when kind='module'. */
33
+ moduleVersion: z.string().optional(),
34
+ /**
35
+ * Schema version of the on_backup hook's own data shape. Reported back
36
+ * by the module's hook output (`outputs.schema_version`) and threaded
37
+ * through to on_restore so the hook can migrate its own data. Distinct
38
+ * from `schemaVersion` (which versions the envelope, not the data).
39
+ */
40
+ dataSchemaVersion: z.string().optional(),
41
+ });
42
+
43
+ export type BackupManifest = z.infer<typeof BackupManifestSchema>;
44
+
45
+ interface BuildOptions {
46
+ kind: 'system' | 'module';
47
+ moduleId?: string;
48
+ moduleVersion?: string;
49
+ dataSchemaVersion?: string;
50
+ /**
51
+ * Override the timestamp / hostname / celiloVersion for tests. Production
52
+ * callers leave these at their defaults.
53
+ */
54
+ now?: Date;
55
+ hostnameOverride?: string;
56
+ celiloVersion?: string;
57
+ }
58
+
59
+ /**
60
+ * Construct a manifest object. Pure function — no I/O, no DB reads. The
61
+ * caller decides where to serialize it (`writeManifest()` below).
62
+ */
63
+ export function buildManifest(options: BuildOptions): BackupManifest {
64
+ const manifest: BackupManifest = {
65
+ schemaVersion: MANIFEST_SCHEMA_VERSION,
66
+ createdAt: (options.now ?? new Date()).toISOString(),
67
+ celiloVersion: options.celiloVersion ?? getCeliloVersion(),
68
+ hostname: options.hostnameOverride ?? hostname(),
69
+ kind: options.kind,
70
+ };
71
+ if (options.moduleId) manifest.moduleId = options.moduleId;
72
+ if (options.moduleVersion) manifest.moduleVersion = options.moduleVersion;
73
+ if (options.dataSchemaVersion) manifest.dataSchemaVersion = options.dataSchemaVersion;
74
+ return manifest;
75
+ }
76
+
77
+ let cachedCeliloVersion: string | null = null;
78
+ function getCeliloVersion(): string {
79
+ if (cachedCeliloVersion !== null) return cachedCeliloVersion;
80
+ try {
81
+ // Read our own package.json. Walking up from __dirname keeps this
82
+ // robust to the test directory layout (tests run from repo root).
83
+ const { readFileSync } = require('node:fs');
84
+ const { dirname, join } = require('node:path');
85
+ let dir: string = __dirname;
86
+ for (let i = 0; i < 8; i++) {
87
+ try {
88
+ const pkg = JSON.parse(readFileSync(join(dir, 'package.json'), 'utf-8'));
89
+ if (pkg.name === '@celilo/cli' || pkg.name === 'celilo') {
90
+ cachedCeliloVersion = String(pkg.version);
91
+ return cachedCeliloVersion;
92
+ }
93
+ } catch {
94
+ /* package.json doesn't exist in this dir or isn't ours */
95
+ }
96
+ const parent = dirname(dir);
97
+ if (parent === dir) break;
98
+ dir = parent;
99
+ }
100
+ } catch {
101
+ /* fall through to unknown */
102
+ }
103
+ cachedCeliloVersion = 'unknown';
104
+ return cachedCeliloVersion;
105
+ }
106
+
107
+ /**
108
+ * Parse + validate a manifest from a JSON string. Throws ManifestParseError
109
+ * with an actionable message on failure.
110
+ */
111
+ export function parseManifest(json: string): BackupManifest {
112
+ let raw: unknown;
113
+ try {
114
+ raw = JSON.parse(json);
115
+ } catch (err) {
116
+ const detail = err instanceof Error ? err.message : String(err);
117
+ throw new ManifestParseError(
118
+ `Backup artifact manifest.json is not valid JSON: ${detail}. The artifact may be corrupted or produced by an incompatible celilo version.`,
119
+ );
120
+ }
121
+ const parsed = BackupManifestSchema.safeParse(raw);
122
+ if (!parsed.success) {
123
+ throw new ManifestParseError(
124
+ `Backup artifact manifest.json is malformed: ${parsed.error.message}. The artifact may be corrupted or produced by an incompatible celilo version.`,
125
+ );
126
+ }
127
+ return parsed.data;
128
+ }
129
+
130
+ /**
131
+ * Verify that a backup artifact's schemaVersion is compatible with this
132
+ * celilo build. Throws with an operator-readable message on mismatch.
133
+ *
134
+ * Today the only check is "major version matches." A v1.x restorer will
135
+ * load v1.0, v1.1, ... but refuse v2.x.
136
+ */
137
+ export function assertCompatibleSchema(manifest: BackupManifest): void {
138
+ const expectedMajor = MANIFEST_SCHEMA_VERSION.split('.')[0];
139
+ const actualMajor = manifest.schemaVersion.split('.')[0];
140
+ if (expectedMajor !== actualMajor) {
141
+ throw new IncompatibleManifestError(
142
+ `Cannot restore backup: artifact was created with envelope schema ${manifest.schemaVersion}, this celilo supports ${MANIFEST_SCHEMA_VERSION}. Use a celilo build whose major version matches the artifact (or re-create the backup with this celilo).`,
143
+ manifest,
144
+ );
145
+ }
146
+ }
147
+
148
+ export class ManifestParseError extends Error {
149
+ constructor(message: string) {
150
+ super(message);
151
+ this.name = 'ManifestParseError';
152
+ }
153
+ }
154
+
155
+ export class IncompatibleManifestError extends Error {
156
+ constructor(
157
+ message: string,
158
+ public readonly manifest: BackupManifest,
159
+ ) {
160
+ super(message);
161
+ this.name = 'IncompatibleManifestError';
162
+ }
163
+ }
@@ -5,7 +5,7 @@
5
5
  */
6
6
 
7
7
  import { execSync } from 'node:child_process';
8
- import { copyFileSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
8
+ import { copyFileSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
9
9
  import { tmpdir } from 'node:os';
10
10
  import { join } from 'node:path';
11
11
  import { eq } from 'drizzle-orm';
@@ -19,7 +19,16 @@ import type { ModuleManifest } from '../manifest/schema';
19
19
  import { decryptSecret } from '../secrets/encryption';
20
20
  import { getOrCreateMasterKey } from '../secrets/master-key';
21
21
  import { shellEscape } from '../utils/shell';
22
+ import { assertCompatibleSchema, parseManifest } from './backup-manifest';
22
23
  import { createStorageProvider } from './backup-storage';
24
+ import { applyCrossModuleWriteRoot, moduleHasCrossModuleRead } from './cross-module-read';
25
+ import { getModuleSystems } from './deployed-systems';
26
+ import {
27
+ completeOperation,
28
+ failOperation,
29
+ refuseIfInFlight,
30
+ startOperation,
31
+ } from './module-operations';
23
32
 
24
33
  export interface RestoreResult {
25
34
  success: boolean;
@@ -31,6 +40,17 @@ export interface RestoreResult {
31
40
  * Restore a system state backup (replaces the Celilo database)
32
41
  */
33
42
  export async function restoreSystemStateBackup(backup: Backup): Promise<RestoreResult> {
43
+ // Refuse if any module operation is in flight (deploy/uninstall/backup/
44
+ // restore). Caller surfaces the InFlightError to the operator.
45
+ refuseIfInFlight();
46
+
47
+ // Start an operation row so concurrent attempts in another process see
48
+ // this restore as in-flight. The row's value is short-lived: once we
49
+ // replace the DB file below, the new DB won't have this row. That's
50
+ // acceptable — a system restore is a single-process operation by design,
51
+ // and a parallel CLI invocation would be a foot-gun in any case.
52
+ startOperation('__system__', 'restore');
53
+
34
54
  const provider = await createStorageProvider(backup.storageId);
35
55
  const tempDir = join(tmpdir(), `celilo-restore-${backup.id}`);
36
56
 
@@ -41,17 +61,50 @@ export async function restoreSystemStateBackup(backup: Backup): Promise<RestoreR
41
61
  const encryptedPath = join(tempDir, 'system.enc');
42
62
  await provider.download(backup.storagePath, encryptedPath);
43
63
 
44
- // Decrypt
64
+ // Decrypt → tar
45
65
  const encryptedData = JSON.parse(readFileSync(encryptedPath, 'utf-8'));
46
66
  const masterKey = await getOrCreateMasterKey();
47
67
  const base64Data = decryptSecret(encryptedData, masterKey);
48
- const dbData = Buffer.from(base64Data, 'base64');
68
+ const tarData = Buffer.from(base64Data, 'base64');
69
+
70
+ // Extract the envelope tar (manifest.json + celilo.db [+celilo.db-wal])
71
+ const envelopeDir = join(tempDir, 'envelope');
72
+ mkdirSync(envelopeDir, { recursive: true });
73
+ const tarPath = join(tempDir, 'envelope.tar');
74
+ writeFileSync(tarPath, tarData);
75
+ execSync(`tar -xf ${shellEscape(tarPath)} -C ${shellEscape(envelopeDir)}`);
49
76
 
50
- // Write restored database to temp file first
51
- const restoredDbPath = join(tempDir, 'celilo.db');
52
- writeFileSync(restoredDbPath, dbData);
77
+ // Read + validate manifest BEFORE touching the live DB. An
78
+ // incompatible artifact must not get past this point.
79
+ const manifestPath = join(envelopeDir, 'manifest.json');
80
+ if (!existsSync(manifestPath)) {
81
+ return {
82
+ success: false,
83
+ error:
84
+ 'System restore failed: artifact has no manifest.json. It may be from an older celilo (pre-envelope format) or corrupted.',
85
+ };
86
+ }
87
+ const manifest = parseManifest(readFileSync(manifestPath, 'utf-8'));
88
+ assertCompatibleSchema(manifest);
89
+ if (manifest.kind !== 'system') {
90
+ return {
91
+ success: false,
92
+ error: `System restore failed: artifact kind is '${manifest.kind}', expected 'system'.`,
93
+ };
94
+ }
95
+
96
+ const restoredDbPath = join(envelopeDir, 'celilo.db');
97
+ if (!existsSync(restoredDbPath)) {
98
+ return {
99
+ success: false,
100
+ error: 'System restore failed: artifact has no celilo.db.',
101
+ };
102
+ }
53
103
 
54
- // Close current database connection
104
+ // Close current database connection. After this point, the operation
105
+ // row from startOperation() above is unreachable (replaced by the
106
+ // restored DB's content) and we cannot meaningfully complete/fail it
107
+ // — but the restore IS the completion.
55
108
  closeDb();
56
109
 
57
110
  // Replace the database file
@@ -145,57 +198,137 @@ export async function restoreModuleBackup(
145
198
  };
146
199
  }
147
200
 
201
+ // Refuse if any module operation is in flight. Checked after we've
202
+ // validated the backup + module, so the operator gets the more useful
203
+ // error first.
204
+ refuseIfInFlight();
205
+
206
+ const opId = startOperation(backup.moduleId, 'restore');
148
207
  const provider = await createStorageProvider(backup.storageId);
149
208
  const tempDir = join(tmpdir(), `celilo-restore-${backup.id}`);
150
- const restoreDir = join(tempDir, 'artifacts');
209
+ const envelopeDir = join(tempDir, 'envelope');
151
210
 
152
211
  try {
153
- mkdirSync(restoreDir, { recursive: true });
212
+ mkdirSync(envelopeDir, { recursive: true });
154
213
 
155
214
  // Download encrypted archive
156
215
  const encryptedPath = join(tempDir, 'backup.tar.enc');
157
216
  await provider.download(backup.storagePath, encryptedPath);
158
217
 
159
- // Decrypt
218
+ // Decrypt → tar
160
219
  const encryptedData = JSON.parse(readFileSync(encryptedPath, 'utf-8'));
161
220
  const masterKey = await getOrCreateMasterKey();
162
221
  const base64Data = decryptSecret(encryptedData, masterKey);
163
222
  const tarData = Buffer.from(base64Data, 'base64');
164
223
 
165
- // Write tar and extract
166
- const tarPath = join(tempDir, 'backup.tar');
224
+ // Write tar and extract into envelopeDir. The envelope contains:
225
+ // manifest.json - validated below
226
+ // data/ - on_backup hook artifacts (passed to on_restore)
227
+ const tarPath = join(tempDir, 'envelope.tar');
167
228
  writeFileSync(tarPath, tarData);
168
- execSync(`tar -xf ${shellEscape(tarPath)} -C ${shellEscape(restoreDir)}`);
229
+ execSync(`tar -xf ${shellEscape(tarPath)} -C ${shellEscape(envelopeDir)}`);
230
+
231
+ // Read + validate envelope manifest BEFORE invoking the hook.
232
+ const manifestPath = join(envelopeDir, 'manifest.json');
233
+ if (!existsSync(manifestPath)) {
234
+ const err =
235
+ 'Module restore failed: artifact has no manifest.json. It may be from an older celilo (pre-envelope format) or corrupted.';
236
+ failOperation(opId, err);
237
+ return { success: false, error: err };
238
+ }
239
+ const envelopeManifest = parseManifest(readFileSync(manifestPath, 'utf-8'));
240
+ assertCompatibleSchema(envelopeManifest);
241
+ if (envelopeManifest.kind !== 'module') {
242
+ const err = `Module restore failed: artifact kind is '${envelopeManifest.kind}', expected 'module'.`;
243
+ failOperation(opId, err);
244
+ return { success: false, error: err };
245
+ }
246
+ if (envelopeManifest.moduleId && envelopeManifest.moduleId !== backup.moduleId) {
247
+ const err = `Module restore failed: artifact was created for module '${envelopeManifest.moduleId}', not '${backup.moduleId}'.`;
248
+ failOperation(opId, err);
249
+ return { success: false, error: err };
250
+ }
251
+
252
+ // The on_restore hook reads its data from envelope/data/ — that's
253
+ // where on_backup wrote it.
254
+ const restoreDataDir = join(envelopeDir, 'data');
255
+ if (!existsSync(restoreDataDir)) {
256
+ const err = 'Module restore failed: artifact has no data/ directory.';
257
+ failOperation(opId, err);
258
+ return { success: false, error: err };
259
+ }
169
260
 
170
261
  // Build context for hook execution
171
262
  const { configMap, secretMap } = await buildModuleContext(mod.id);
172
263
  const logger = createConsoleLogger(mod.id, 'on_restore');
173
264
 
174
- // Execute on_restore hook
265
+ // Cross-module-write privilege: if the manifest declares
266
+ // cross_module_read, hand the hook a writable staging dir. After
267
+ // a successful hook return, applyCrossModuleWriteRoot atomically
268
+ // moves each module's subtree into live storage. If the hook
269
+ // crashes/fails, the staging dir is discarded with no side effects.
270
+ const hookInputs: Record<string, unknown> = {
271
+ restore_dir: restoreDataDir,
272
+ schema_version: envelopeManifest.dataSchemaVersion ?? backup.schemaVersion ?? '',
273
+ };
274
+ let crossModuleWriteRoot: string | undefined;
275
+ if (moduleHasCrossModuleRead(manifest)) {
276
+ crossModuleWriteRoot = join(tempDir, 'cross-module-write');
277
+ mkdirSync(crossModuleWriteRoot, { recursive: true });
278
+ hookInputs.cross_module_write_root = crossModuleWriteRoot;
279
+ }
280
+
281
+ // Execute on_restore hook. Pass the data dir + the data-schema-version
282
+ // from the envelope manifest so the hook can migrate its own data if
283
+ // needed. Fall back to the backup record's stored value (kept for
284
+ // backups created before the envelope landed in this celilo build).
175
285
  const hookResult = await invokeHook(
176
286
  mod.sourcePath,
177
287
  'on_restore',
178
288
  manifest.celilo_contract,
179
289
  hookDef,
180
- {
181
- restore_dir: restoreDir,
182
- schema_version: backup.schemaVersion ?? '',
183
- },
290
+ hookInputs,
184
291
  configMap,
185
292
  secretMap,
186
293
  logger,
187
294
  {
188
295
  debug: false,
296
+ systems: getModuleSystems(backup.moduleId, db),
189
297
  },
190
298
  );
191
299
 
192
300
  if (!hookResult.success) {
301
+ const errMsg = hookResult.error ?? 'on_restore hook failed';
302
+ failOperation(opId, errMsg);
193
303
  return {
194
304
  success: false,
195
- error: hookResult.error ?? 'on_restore hook failed',
305
+ error: errMsg,
196
306
  };
197
307
  }
198
308
 
309
+ // Hook succeeded — apply the cross-module staging dir back onto
310
+ // live storage. Atomic per module (rename live → live.old + rename
311
+ // staged → live, with rollback on mid-loop failure). If the apply
312
+ // itself fails, fail the operation but leave the artifact intact —
313
+ // the operator can investigate and re-run.
314
+ if (crossModuleWriteRoot) {
315
+ try {
316
+ const applyResult = applyCrossModuleWriteRoot(crossModuleWriteRoot);
317
+ if (applyResult.applied.length > 0) {
318
+ logger.info(`Cross-module restore applied for: ${applyResult.applied.join(', ')}`);
319
+ }
320
+ if (applyResult.skipped.length > 0) {
321
+ logger.warn(
322
+ `Cross-module restore skipped (no staged data) for: ${applyResult.skipped.join(', ')}`,
323
+ );
324
+ }
325
+ } catch (applyErr) {
326
+ const errMsg = `cross_module_write_root apply failed: ${applyErr instanceof Error ? applyErr.message : String(applyErr)}`;
327
+ failOperation(opId, errMsg);
328
+ return { success: false, error: errMsg };
329
+ }
330
+ }
331
+
199
332
  // Run health check if requested and the module has one
200
333
  let healthCheckPassed: boolean | undefined;
201
334
  if (options.runHealthCheck && manifest.hooks?.health_check) {
@@ -210,11 +343,13 @@ export async function restoreModuleBackup(
210
343
  }
211
344
  }
212
345
 
346
+ completeOperation(opId);
213
347
  return {
214
348
  success: true,
215
349
  healthCheckPassed,
216
350
  };
217
351
  } catch (error) {
352
+ failOperation(opId, error);
218
353
  return {
219
354
  success: false,
220
355
  error: `Restore failed: ${error instanceof Error ? error.message : String(error)}`,
@@ -0,0 +1,92 @@
1
+ /**
2
+ * Local-bus event emitters for build-bus delivery outcomes.
3
+ *
4
+ * Phase 6 of [[v2/BUILD_BUS.md]] wants `celilo subscribers status`
5
+ * to show last-delivery state + recent failures per subscriber. The
6
+ * data source for that view is the local SQLite event bus — every
7
+ * publisher-side fan-out call emits one `webhook.delivered` or
8
+ * `webhook.failed` event here, and the status command queries them
9
+ * back.
10
+ *
11
+ * Two event types instead of one (`webhook.delivery` with ok bool):
12
+ * - `webhook.failed` filters cleanly for "show me what's broken"
13
+ * - `webhook.delivered` filters cleanly for "show me success rate"
14
+ * Either query is one indexed lookup against the events.type index
15
+ * on the bus.
16
+ */
17
+
18
+ import { defineEvents, openBus } from '@celilo/event-bus';
19
+ import type { DeliveryResult, PublishEvent } from '@celilo/event-bus/build-bus';
20
+ import { getEventBusPath } from '../../config/paths';
21
+
22
+ const NO_SCHEMAS = defineEvents({});
23
+
24
+ export const WEBHOOK_DELIVERED_EVENT = 'webhook.delivered';
25
+ export const WEBHOOK_FAILED_EVENT = 'webhook.failed';
26
+
27
+ /**
28
+ * Payload shape for `webhook.delivered` / `webhook.failed` events.
29
+ * Stored verbatim on the local bus; the status command queries with
30
+ * `recentEvents({ type, limit })` and aggregates.
31
+ */
32
+ export interface WebhookDeliveryPayload {
33
+ /** Subscriber display label (or URL if no name was set). */
34
+ subscriberLabel: string;
35
+ /** Subscriber webhook URL. Used as the aggregation key. */
36
+ subscriberUrl: string;
37
+ /** UUID of the event we attempted to deliver. */
38
+ eventId: string;
39
+ /** Package this event was about — convenience for status output. */
40
+ packageName: string;
41
+ packageVersion: string;
42
+ /** Dist-tag (e.g. 'latest', 'alpha'). */
43
+ tag: string;
44
+ /** Attempts spent (1 + retries). */
45
+ attempts: number;
46
+ /** Total ms spent including retries. */
47
+ durationMs: number;
48
+ /** HTTP status, if a response came back. Absent on network errors. */
49
+ status?: number;
50
+ /** Error message; only present on failed deliveries. */
51
+ error?: string;
52
+ }
53
+
54
+ function buildPayload(event: PublishEvent, result: DeliveryResult): WebhookDeliveryPayload {
55
+ return {
56
+ subscriberLabel: result.subscriber.name ?? result.subscriber.url,
57
+ subscriberUrl: result.subscriber.url,
58
+ eventId: event.eventId,
59
+ packageName: event.package.name,
60
+ packageVersion: event.package.version,
61
+ tag: event.tag,
62
+ attempts: result.attempts,
63
+ durationMs: result.durationMs,
64
+ status: result.status,
65
+ error: result.error,
66
+ };
67
+ }
68
+
69
+ /**
70
+ * Best-effort emit. Failures here (DB locked, disk full) get logged
71
+ * to stderr but never propagate — the actual webhook ALREADY
72
+ * succeeded (or failed) on the wire; recording the outcome is
73
+ * housekeeping.
74
+ */
75
+ function emitBest(type: string, payload: WebhookDeliveryPayload): void {
76
+ let bus: ReturnType<typeof openBus> | undefined;
77
+ try {
78
+ bus = openBus({ dbPath: getEventBusPath(), events: NO_SCHEMAS });
79
+ bus.emitRaw(type, payload);
80
+ } catch (err) {
81
+ const msg = err instanceof Error ? err.message : String(err);
82
+ console.warn(`[build-bus] failed to record ${type}: ${msg}`);
83
+ } finally {
84
+ bus?.close();
85
+ }
86
+ }
87
+
88
+ /** Record one delivery outcome on the local bus. */
89
+ export function recordDeliveryOutcome(event: PublishEvent, result: DeliveryResult): void {
90
+ const type = result.ok ? WEBHOOK_DELIVERED_EVENT : WEBHOOK_FAILED_EVENT;
91
+ emitBest(type, buildPayload(event, result));
92
+ }
@@ -0,0 +1,54 @@
1
+ /**
2
+ * Build PublishEvent payloads from the publish-flow output.
3
+ *
4
+ * `executePlan` returns a list of `{name, version}` for each
5
+ * published package. Each entry becomes a PublishEvent. The event's
6
+ * `registry` and `tag` flow from the publish mode (alpha/normal/
7
+ * promote) so subscribers can filter accurately — an alpha
8
+ * subscriber doesn't fire on real publishes.
9
+ */
10
+
11
+ import { randomUUID } from 'node:crypto';
12
+ import type { PublishEvent } from '@celilo/event-bus/build-bus';
13
+
14
+ export interface PublishedItem {
15
+ name: string;
16
+ version: string;
17
+ }
18
+
19
+ export interface EventFactoryInput {
20
+ published: PublishedItem[];
21
+ /** Per the publish mode: `latest` for normal, `alpha` for alpha. Promote ships under `latest`. */
22
+ tag: 'latest' | 'alpha';
23
+ /** Current git HEAD at publish time. Optional. */
24
+ gitHead?: string;
25
+ /** Defaults to "npm". The cross-machine event flow doesn't care. */
26
+ registry?: string;
27
+ /**
28
+ * Time + UUID injection for deterministic tests. Production
29
+ * omits, uses Date.now + crypto.randomUUID.
30
+ */
31
+ now?: () => Date;
32
+ newId?: () => string;
33
+ }
34
+
35
+ /**
36
+ * Build one PublishEvent per published package. Pure given the
37
+ * injectables — same input + same `now`/`newId` produces the same
38
+ * events.
39
+ */
40
+ export function eventsForPublished(input: EventFactoryInput): PublishEvent[] {
41
+ const tag = input.tag;
42
+ const registry = input.registry ?? 'npm';
43
+ const now = input.now ?? (() => new Date());
44
+ const newId = input.newId ?? (() => randomUUID());
45
+
46
+ return input.published.map(({ name, version }) => ({
47
+ eventId: newId(),
48
+ timestamp: now().toISOString(),
49
+ registry,
50
+ tag,
51
+ package: { name, version },
52
+ gitHead: input.gitHead,
53
+ }));
54
+ }