@celilo/cli 0.5.0-alpha.4 → 0.5.0-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/drizzle/0010_dns_internal_records.sql +12 -0
- package/drizzle/0011_backups_name.sql +1 -0
- package/drizzle/meta/_journal.json +14 -0
- package/package.json +2 -2
- package/src/ansible/inventory.test.ts +10 -10
- package/src/ansible/validation.test.ts +25 -15
- package/src/cli/command-registry.ts +13 -2
- package/src/cli/commands/events.test.ts +4 -4
- package/src/cli/commands/events.ts +2 -2
- package/src/cli/commands/service-add-proxmox.ts +9 -0
- package/src/cli/commands/system-doctor.ts +135 -40
- package/src/cli/commands/system-migrate.test.ts +40 -0
- package/src/cli/commands/system-migrate.ts +65 -0
- package/src/cli/completion.ts +1 -0
- package/src/cli/index.ts +7 -2
- package/src/config/paths.test.ts +61 -48
- package/src/db/client.ts +15 -146
- package/src/db/migrate.ts +14 -6
- package/src/db/schema-introspection.ts +88 -0
- package/src/db/schema.ts +38 -0
- package/src/hooks/capability-loader-firewall.test.ts +3 -3
- package/src/hooks/capability-loader.ts +24 -15
- package/src/infrastructure/property-extractor.test.ts +15 -0
- package/src/infrastructure/property-extractor.ts +12 -0
- package/src/manifest/schema.ts +7 -0
- package/src/manifest/validate.test.ts +53 -0
- package/src/services/bus-interview.test.ts +2 -2
- package/src/services/bus-secret-flow.test.ts +2 -2
- package/src/services/celilo-mgmt-hooks.test.ts +3 -2
- package/src/services/deploy-preflight.ts +25 -0
- package/src/services/deploy-validation.test.ts +2 -2
- package/src/services/dns-internal-records.test.ts +126 -0
- package/src/services/dns-internal-records.ts +119 -0
- package/src/services/dns-provider-backfill.test.ts +2 -2
- package/src/services/dns-registrations.test.ts +10 -10
- package/src/services/fleet-checks.test.ts +495 -0
- package/src/services/fleet-checks.ts +663 -0
- package/src/services/module-build.test.ts +43 -38
- package/src/templates/generator.test.ts +62 -12
- package/src/templates/generator.ts +69 -50
- package/src/test-utils/fixtures.test.ts +1 -1
- package/src/test-utils/integration-guard.ts +33 -0
- package/src/types/infrastructure.ts +6 -0
- package/src/variables/computed/computed-integration.test.ts +3 -3
- package/src/variables/computed/computed.test.ts +5 -5
- package/src/variables/declarative-derivation.test.ts +6 -6
package/src/config/paths.test.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
2
|
+
import { skipIntegration } from '../test-utils/integration-guard';
|
|
2
3
|
import { getDataDir, getDbPath, getMasterKeyPath, getModuleStoragePath } from './paths';
|
|
3
4
|
|
|
4
5
|
describe('paths configuration', () => {
|
|
@@ -34,18 +35,21 @@ describe('paths configuration', () => {
|
|
|
34
35
|
expect(path).toMatch(/celilo-data\/modules$/);
|
|
35
36
|
});
|
|
36
37
|
|
|
37
|
-
it(
|
|
38
|
-
|
|
39
|
-
|
|
38
|
+
it.skipIf(skipIntegration({ platform: 'darwin' }))(
|
|
39
|
+
'returns platform-specific path by default',
|
|
40
|
+
() => {
|
|
41
|
+
process.env.CELILO_DATA_DIR = undefined;
|
|
42
|
+
process.env.ENVIRONMENT = undefined;
|
|
40
43
|
|
|
41
|
-
|
|
44
|
+
const path = getModuleStoragePath();
|
|
42
45
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
46
|
+
if (process.platform === 'darwin') {
|
|
47
|
+
expect(path).toMatch(/Library\/Application Support\/celilo\/modules$/);
|
|
48
|
+
} else {
|
|
49
|
+
expect(path).toBe('/var/lib/celilo/modules');
|
|
50
|
+
}
|
|
51
|
+
},
|
|
52
|
+
);
|
|
49
53
|
|
|
50
54
|
it('prioritizes CELILO_DATA_DIR over ENVIRONMENT=dev', () => {
|
|
51
55
|
process.env.CELILO_DATA_DIR = '/custom/data';
|
|
@@ -73,18 +77,21 @@ describe('paths configuration', () => {
|
|
|
73
77
|
expect(path).toMatch(/celilo-data$/);
|
|
74
78
|
});
|
|
75
79
|
|
|
76
|
-
it(
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
80
|
+
it.skipIf(skipIntegration({ platform: 'darwin' }))(
|
|
81
|
+
'returns platform-specific path by default',
|
|
82
|
+
() => {
|
|
83
|
+
process.env.CELILO_DATA_DIR = undefined;
|
|
84
|
+
process.env.ENVIRONMENT = undefined;
|
|
85
|
+
|
|
86
|
+
const path = getDataDir();
|
|
87
|
+
|
|
88
|
+
if (process.platform === 'darwin') {
|
|
89
|
+
expect(path).toMatch(/Library\/Application Support\/celilo$/);
|
|
90
|
+
} else {
|
|
91
|
+
expect(path).toBe('/var/lib/celilo');
|
|
92
|
+
}
|
|
93
|
+
},
|
|
94
|
+
);
|
|
88
95
|
});
|
|
89
96
|
|
|
90
97
|
describe('getMasterKeyPath', () => {
|
|
@@ -103,19 +110,22 @@ describe('paths configuration', () => {
|
|
|
103
110
|
expect(path).toBe('/custom/data/master.key');
|
|
104
111
|
});
|
|
105
112
|
|
|
106
|
-
it(
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
113
|
+
it.skipIf(skipIntegration({ platform: 'darwin' }))(
|
|
114
|
+
'uses platform-specific data dir by default',
|
|
115
|
+
() => {
|
|
116
|
+
process.env.CELILO_MASTER_KEY_PATH = undefined;
|
|
117
|
+
process.env.CELILO_DATA_DIR = undefined;
|
|
118
|
+
process.env.ENVIRONMENT = undefined;
|
|
119
|
+
|
|
120
|
+
const path = getMasterKeyPath();
|
|
121
|
+
|
|
122
|
+
if (process.platform === 'darwin') {
|
|
123
|
+
expect(path).toMatch(/Library\/Application Support\/celilo\/master\.key$/);
|
|
124
|
+
} else {
|
|
125
|
+
expect(path).toBe('/var/lib/celilo/master.key');
|
|
126
|
+
}
|
|
127
|
+
},
|
|
128
|
+
);
|
|
119
129
|
});
|
|
120
130
|
|
|
121
131
|
describe('getDbPath', () => {
|
|
@@ -128,19 +138,22 @@ describe('paths configuration', () => {
|
|
|
128
138
|
expect(path).toBe('/custom/path/celilo.db');
|
|
129
139
|
});
|
|
130
140
|
|
|
131
|
-
it(
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
141
|
+
it.skipIf(skipIntegration({ platform: 'darwin' }))(
|
|
142
|
+
'returns data dir + celilo.db on macOS in production',
|
|
143
|
+
() => {
|
|
144
|
+
process.env.CELILO_DB_PATH = undefined;
|
|
145
|
+
process.env.CELILO_DATA_DIR = undefined;
|
|
146
|
+
process.env.ENVIRONMENT = undefined;
|
|
147
|
+
|
|
148
|
+
const path = getDbPath();
|
|
149
|
+
|
|
150
|
+
if (process.platform === 'darwin') {
|
|
151
|
+
expect(path).toMatch(/Library\/Application Support\/celilo\/celilo.db$/);
|
|
152
|
+
} else {
|
|
153
|
+
expect(path).toBe('/var/lib/celilo/celilo.db');
|
|
154
|
+
}
|
|
155
|
+
},
|
|
156
|
+
);
|
|
144
157
|
|
|
145
158
|
it('returns celilo-data/celilo.db in development mode', () => {
|
|
146
159
|
process.env.CELILO_DB_PATH = undefined;
|
package/src/db/client.ts
CHANGED
|
@@ -41,145 +41,6 @@ export function findMigrationsFolder(): string {
|
|
|
41
41
|
throw new Error(`Could not find drizzle migrations folder. Tried: ${candidates.join(', ')}`);
|
|
42
42
|
}
|
|
43
43
|
|
|
44
|
-
/**
|
|
45
|
-
* Check if database needs initialization (tables don't exist)
|
|
46
|
-
*/
|
|
47
|
-
function needsMigration(sqlite: Database): boolean {
|
|
48
|
-
try {
|
|
49
|
-
// Check if the modules table exists at all (new database)
|
|
50
|
-
const result = sqlite
|
|
51
|
-
.query("SELECT name FROM sqlite_master WHERE type='table' AND name='modules'")
|
|
52
|
-
.get();
|
|
53
|
-
if (!result) return true; // New database — run full migrations
|
|
54
|
-
|
|
55
|
-
// Existing database — apply incremental schema updates
|
|
56
|
-
// Each statement is wrapped in try/catch (no-op if already applied)
|
|
57
|
-
const alterStatements = [
|
|
58
|
-
'ALTER TABLE capabilities ADD zones text',
|
|
59
|
-
'ALTER TABLE machines ADD earmarked_module text',
|
|
60
|
-
// web_routes' subdomain/custom_domain columns were folded into a
|
|
61
|
-
// single `hostname` field by migration 0004. Don't re-add them
|
|
62
|
-
// here — the migration drops and recreates the table.
|
|
63
|
-
// Backup system tables (Phase 1)
|
|
64
|
-
`CREATE TABLE IF NOT EXISTS backup_storages (
|
|
65
|
-
id text PRIMARY KEY NOT NULL,
|
|
66
|
-
storage_id text NOT NULL UNIQUE,
|
|
67
|
-
name text NOT NULL,
|
|
68
|
-
provider_name text NOT NULL,
|
|
69
|
-
credentials_encrypted text NOT NULL,
|
|
70
|
-
provider_config text NOT NULL,
|
|
71
|
-
verified integer DEFAULT 0 NOT NULL,
|
|
72
|
-
verified_at integer,
|
|
73
|
-
verification_error text,
|
|
74
|
-
is_default integer DEFAULT 0 NOT NULL,
|
|
75
|
-
created_at integer DEFAULT (unixepoch()) NOT NULL,
|
|
76
|
-
updated_at integer DEFAULT (unixepoch()) NOT NULL
|
|
77
|
-
)`,
|
|
78
|
-
`CREATE TABLE IF NOT EXISTS backups (
|
|
79
|
-
id text PRIMARY KEY NOT NULL,
|
|
80
|
-
module_id text REFERENCES modules(id) ON DELETE SET NULL,
|
|
81
|
-
storage_id text NOT NULL REFERENCES backup_storages(id),
|
|
82
|
-
storage_path text NOT NULL,
|
|
83
|
-
backup_type text NOT NULL,
|
|
84
|
-
module_version text,
|
|
85
|
-
schema_version text,
|
|
86
|
-
size_bytes integer,
|
|
87
|
-
metadata text DEFAULT '{}' NOT NULL,
|
|
88
|
-
status text DEFAULT 'in_progress' NOT NULL,
|
|
89
|
-
error_message text,
|
|
90
|
-
started_at integer DEFAULT (unixepoch()) NOT NULL,
|
|
91
|
-
completed_at integer
|
|
92
|
-
)`,
|
|
93
|
-
// Backup naming support
|
|
94
|
-
'ALTER TABLE backups ADD name text',
|
|
95
|
-
// Module systems (v2/MODULE_SYSTEMS_ADDRESSING.md) — a module's 0..N
|
|
96
|
-
// deployed hosts. Fresh DBs get this via migration 0007; existing installs
|
|
97
|
-
// get it here. Replaces the scalar target_ip/vmid rows in module_configs.
|
|
98
|
-
`CREATE TABLE IF NOT EXISTS module_systems (
|
|
99
|
-
module_id text NOT NULL REFERENCES modules(id) ON DELETE cascade,
|
|
100
|
-
name text NOT NULL,
|
|
101
|
-
hostname text NOT NULL,
|
|
102
|
-
ipv4_address text NOT NULL,
|
|
103
|
-
zone text NOT NULL,
|
|
104
|
-
infra_type text NOT NULL,
|
|
105
|
-
machine_id text REFERENCES machines(id),
|
|
106
|
-
service_id text REFERENCES container_services(id),
|
|
107
|
-
vmid integer,
|
|
108
|
-
created_at integer DEFAULT (unixepoch()) NOT NULL,
|
|
109
|
-
updated_at integer DEFAULT (unixepoch()) NOT NULL,
|
|
110
|
-
PRIMARY KEY (module_id, name)
|
|
111
|
-
)`,
|
|
112
|
-
// Aspect consent decision (ISS-0027). Fresh DBs get this via migration
|
|
113
|
-
// 0008; existing installs get it here. Defaults to true so pre-existing
|
|
114
|
-
// rows (all approvals) keep running; false = a durable refusal.
|
|
115
|
-
'ALTER TABLE aspect_approvals ADD consented integer DEFAULT true NOT NULL',
|
|
116
|
-
];
|
|
117
|
-
|
|
118
|
-
for (const stmt of alterStatements) {
|
|
119
|
-
try {
|
|
120
|
-
sqlite.exec(stmt);
|
|
121
|
-
} catch {
|
|
122
|
-
// Column already exists — fine
|
|
123
|
-
}
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
// web_routes hostname migration (CADDY_HOSTNAME_LIST design).
|
|
127
|
-
// Drizzle's auto-migrate path (`migrate()`) only runs for fresh
|
|
128
|
-
// databases — for existing celilo installs we apply the schema
|
|
129
|
-
// change here. Phase 0 + no production users = destructive
|
|
130
|
-
// rebuild; modules repopulate routes on their next deploy.
|
|
131
|
-
try {
|
|
132
|
-
const cols = sqlite.query("SELECT name FROM pragma_table_info('web_routes')").all() as Array<{
|
|
133
|
-
name: string;
|
|
134
|
-
}>;
|
|
135
|
-
const hasHostname = cols.some((c) => c.name === 'hostname');
|
|
136
|
-
const hasOldColumns = cols.some((c) => c.name === 'subdomain' || c.name === 'custom_domain');
|
|
137
|
-
if (!hasHostname && cols.length > 0) {
|
|
138
|
-
// Old shape detected (or missing hostname) — drop and recreate.
|
|
139
|
-
// Wrap in a transaction so the table is never half-migrated.
|
|
140
|
-
sqlite.exec('BEGIN');
|
|
141
|
-
try {
|
|
142
|
-
sqlite.exec('DROP TABLE web_routes');
|
|
143
|
-
sqlite.exec(`CREATE TABLE web_routes (
|
|
144
|
-
id integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
|
145
|
-
slug text NOT NULL,
|
|
146
|
-
module_id text NOT NULL,
|
|
147
|
-
type text NOT NULL,
|
|
148
|
-
path text NOT NULL,
|
|
149
|
-
hostname text NOT NULL,
|
|
150
|
-
target_host text,
|
|
151
|
-
target_port integer,
|
|
152
|
-
websocket integer DEFAULT false NOT NULL,
|
|
153
|
-
content_hash text,
|
|
154
|
-
created_at integer DEFAULT (unixepoch()) NOT NULL,
|
|
155
|
-
updated_at integer DEFAULT (unixepoch()) NOT NULL,
|
|
156
|
-
FOREIGN KEY (module_id) REFERENCES modules(id) ON UPDATE no action ON DELETE cascade
|
|
157
|
-
)`);
|
|
158
|
-
sqlite.exec(
|
|
159
|
-
'CREATE UNIQUE INDEX web_routes_hostname_path_idx ON web_routes (hostname, path)',
|
|
160
|
-
);
|
|
161
|
-
sqlite.exec('COMMIT');
|
|
162
|
-
if (hasOldColumns) {
|
|
163
|
-
console.log(
|
|
164
|
-
'web_routes migrated to hostname-based schema. Modules will repopulate their routes on next deploy.',
|
|
165
|
-
);
|
|
166
|
-
}
|
|
167
|
-
} catch (err) {
|
|
168
|
-
sqlite.exec('ROLLBACK');
|
|
169
|
-
throw err;
|
|
170
|
-
}
|
|
171
|
-
}
|
|
172
|
-
} catch (err) {
|
|
173
|
-
console.error('Failed to migrate web_routes schema:', err);
|
|
174
|
-
throw err;
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
return false; // Schema is up to date
|
|
178
|
-
} catch {
|
|
179
|
-
return true;
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
|
|
183
44
|
/**
|
|
184
45
|
* Create database client and run migrations if needed
|
|
185
46
|
*/
|
|
@@ -209,12 +70,21 @@ export function createDbClient(config?: Partial<DatabaseConfig>) {
|
|
|
209
70
|
|
|
210
71
|
const db = drizzle(sqlite, { schema });
|
|
211
72
|
|
|
212
|
-
//
|
|
213
|
-
|
|
73
|
+
// Apply migrations on open (ISS-0100). Drizzle's migrator is the single
|
|
74
|
+
// migration mechanism for ALL DBs — fresh and existing — and the only place
|
|
75
|
+
// schema changes live (the old imperative hand-list is gone). migrate() is
|
|
76
|
+
// idempotent: it applies every migration newer than the latest recorded in
|
|
77
|
+
// `__drizzle_migrations` and no-ops once current.
|
|
78
|
+
//
|
|
79
|
+
// One-time caveat (ISS-0100): an existing DB from the hand-list era has a
|
|
80
|
+
// frozen `__drizzle_migrations` watermark; it must be remediated by hand
|
|
81
|
+
// (stamp the watermark to the latest migration + create any missing table)
|
|
82
|
+
// BEFORE this code opens it, or migrate() re-runs already-applied migrations
|
|
83
|
+
// and throws. `celilo system doctor` (checkSchemaDrift) detects the drift.
|
|
84
|
+
if (!readonly) {
|
|
214
85
|
try {
|
|
215
86
|
const migrationsFolder = findMigrationsFolder();
|
|
216
87
|
migrate(db, { migrationsFolder });
|
|
217
|
-
console.log('Database initialized with migrations');
|
|
218
88
|
} catch (error) {
|
|
219
89
|
console.error('Failed to run migrations:', error);
|
|
220
90
|
throw error;
|
|
@@ -222,10 +92,9 @@ export function createDbClient(config?: Partial<DatabaseConfig>) {
|
|
|
222
92
|
}
|
|
223
93
|
|
|
224
94
|
// One-time upgrade backfill for the target_ip → module_systems refactor
|
|
225
|
-
// (v2/MODULE_SYSTEMS_ADDRESSING.md).
|
|
226
|
-
//
|
|
227
|
-
//
|
|
228
|
-
// created before the refactor has its host data only in module_configs /
|
|
95
|
+
// (v2/MODULE_SYSTEMS_ADDRESSING.md). migrate() above has ensured the
|
|
96
|
+
// module_systems table exists (migration 0007). A deployment created before
|
|
97
|
+
// the refactor has its host data only in module_configs /
|
|
229
98
|
// ip_allocations / module_infrastructure and an EMPTY module_systems, so its
|
|
230
99
|
// modules resolve to no system and the migrated hooks throw "No deployed
|
|
231
100
|
// system found". This lifts that state across. Idempotent (skips modules
|
package/src/db/migrate.ts
CHANGED
|
@@ -1,17 +1,25 @@
|
|
|
1
1
|
import { migrate } from 'drizzle-orm/bun-sqlite/migrator';
|
|
2
|
-
import { closeDb, createDbClient, findMigrationsFolder } from './client';
|
|
2
|
+
import { type DbClient, closeDb, createDbClient, findMigrationsFolder } from './client';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
|
-
*
|
|
5
|
+
* Apply pending drizzle migrations to an open DB. Idempotent — drizzle applies
|
|
6
|
+
* only migrations newer than the latest recorded in `__drizzle_migrations`.
|
|
7
|
+
* The single migration mechanism (ISS-0100); createDbClient also calls this
|
|
8
|
+
* shape on open (auto-migrate).
|
|
9
|
+
*/
|
|
10
|
+
export function runMigrationsOn(db: DbClient): void {
|
|
11
|
+
migrate(db, { migrationsFolder: findMigrationsFolder() });
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Run database migrations (standalone entrypoint — `bun run src/db/migrate.ts`).
|
|
16
|
+
* createDbClient already migrates on open; this re-asserts for explicit use.
|
|
6
17
|
*/
|
|
7
18
|
export async function runMigrations(dbPath?: string) {
|
|
8
19
|
console.log('Running database migrations...');
|
|
9
|
-
|
|
10
20
|
const db = createDbClient(dbPath ? { path: dbPath } : undefined);
|
|
11
|
-
|
|
12
21
|
try {
|
|
13
|
-
|
|
14
|
-
await migrate(db, { migrationsFolder });
|
|
22
|
+
runMigrationsOn(db);
|
|
15
23
|
console.log('Migrations completed successfully');
|
|
16
24
|
} catch (error) {
|
|
17
25
|
console.error('Migration failed:', error);
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Schema introspection — compare the DB's actual tables/columns against the
|
|
3
|
+
* tables the running code's drizzle schema declares. The single source for
|
|
4
|
+
* "what does the code expect, and is it present?" shared by:
|
|
5
|
+
* - the migration baseline (db/migrate.ts) — to decide which migrations are
|
|
6
|
+
* already applied on an existing DB before running migrate() (ISS-0100), and
|
|
7
|
+
* - the doctor's schema-drift check (services/fleet-checks.ts) — to report
|
|
8
|
+
* missing tables/columns to the operator (ISS-0113).
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import type { Database } from 'bun:sqlite';
|
|
12
|
+
import { is } from 'drizzle-orm';
|
|
13
|
+
import { SQLiteTable, getTableConfig } from 'drizzle-orm/sqlite-core';
|
|
14
|
+
import * as dbSchema from './schema';
|
|
15
|
+
|
|
16
|
+
export interface SchemaDrift {
|
|
17
|
+
/** Tables the code's schema declares that the DB lacks. */
|
|
18
|
+
missingTables: string[];
|
|
19
|
+
/** `table.column` the code declares that the DB's table lacks. */
|
|
20
|
+
missingColumns: string[];
|
|
21
|
+
/** Total number of tables the code's schema declares. */
|
|
22
|
+
tableCount: number;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** Every table name + column names the drizzle schema declares. */
|
|
26
|
+
export function getSchemaTables(): Array<{ name: string; columns: string[] }> {
|
|
27
|
+
const out: Array<{ name: string; columns: string[] }> = [];
|
|
28
|
+
for (const value of Object.values(dbSchema)) {
|
|
29
|
+
if (!is(value, SQLiteTable)) continue;
|
|
30
|
+
const cfg = getTableConfig(value);
|
|
31
|
+
out.push({ name: cfg.name, columns: cfg.columns.map((c) => c.name) });
|
|
32
|
+
}
|
|
33
|
+
return out;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Names of all tables that physically exist in the DB. */
|
|
37
|
+
export function getExistingTables(sqlite: Database): Set<string> {
|
|
38
|
+
return new Set(
|
|
39
|
+
sqlite
|
|
40
|
+
.query<{ name: string }, []>("SELECT name FROM sqlite_master WHERE type='table'")
|
|
41
|
+
.all()
|
|
42
|
+
.map((r) => r.name),
|
|
43
|
+
);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Names of all indexes that physically exist in the DB. */
|
|
47
|
+
export function getExistingIndexes(sqlite: Database): Set<string> {
|
|
48
|
+
return new Set(
|
|
49
|
+
sqlite
|
|
50
|
+
.query<{ name: string }, []>("SELECT name FROM sqlite_master WHERE type='index'")
|
|
51
|
+
.all()
|
|
52
|
+
.map((r) => r.name),
|
|
53
|
+
);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Column names physically present on a table (empty if the table is absent). */
|
|
57
|
+
export function getExistingColumns(sqlite: Database, table: string): Set<string> {
|
|
58
|
+
// `table` is a schema/migration identifier (never user input) — safe to inline.
|
|
59
|
+
return new Set(
|
|
60
|
+
sqlite
|
|
61
|
+
.query<{ name: string }, []>(`SELECT name FROM pragma_table_info('${table}')`)
|
|
62
|
+
.all()
|
|
63
|
+
.map((r) => r.name),
|
|
64
|
+
);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Compare the running code's drizzle schema to the DB and report what's missing.
|
|
69
|
+
* Track-agnostic: it reads actual table/column presence, so it's honest whether
|
|
70
|
+
* the DB was migrated, baselined, or hand-patched.
|
|
71
|
+
*/
|
|
72
|
+
export function findSchemaDrift(sqlite: Database): SchemaDrift {
|
|
73
|
+
const present = getExistingTables(sqlite);
|
|
74
|
+
const missingTables: string[] = [];
|
|
75
|
+
const missingColumns: string[] = [];
|
|
76
|
+
const tables = getSchemaTables();
|
|
77
|
+
for (const t of tables) {
|
|
78
|
+
if (!present.has(t.name)) {
|
|
79
|
+
missingTables.push(t.name);
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
const cols = getExistingColumns(sqlite, t.name);
|
|
83
|
+
for (const c of t.columns) {
|
|
84
|
+
if (!cols.has(c)) missingColumns.push(`${t.name}.${c}`);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return { missingTables, missingColumns, tableCount: tables.length };
|
|
88
|
+
}
|
package/src/db/schema.ts
CHANGED
|
@@ -468,6 +468,44 @@ export const dnsRegistrations = sqliteTable(
|
|
|
468
468
|
}),
|
|
469
469
|
);
|
|
470
470
|
|
|
471
|
+
/**
|
|
472
|
+
* Internal split-horizon DNS A-record ledger. Mirrors `dns_registrations`
|
|
473
|
+
* but for the dns_internal capability (technitium/knot): the capability
|
|
474
|
+
* loader records every `dns_internal.registerRecord({type:'A'})` here so
|
|
475
|
+
* celilo has an offline, queryable record of what hostname → IP it asked
|
|
476
|
+
* the internal resolver to serve. Without this, the only source of truth
|
|
477
|
+
* is the resolver's own DB, requiring a live probe (ISS-0094 / ISS-0111).
|
|
478
|
+
*
|
|
479
|
+
* `celilo system doctor` reads this to assert service hostnames resolve to
|
|
480
|
+
* the firewall natIp (LAN-reachable) and not a zone-side container IP that
|
|
481
|
+
* a LAN device can't route to. Rows die with either module via FK cascade.
|
|
482
|
+
*/
|
|
483
|
+
export const dnsInternalRecords = sqliteTable(
|
|
484
|
+
'dns_internal_records',
|
|
485
|
+
{
|
|
486
|
+
id: integer('id').primaryKey({ autoIncrement: true }),
|
|
487
|
+
providerModuleId: text('provider_module_id')
|
|
488
|
+
.notNull()
|
|
489
|
+
.references(() => modules.id, { onDelete: 'cascade' }),
|
|
490
|
+
consumerModuleId: text('consumer_module_id')
|
|
491
|
+
.notNull()
|
|
492
|
+
.references(() => modules.id, { onDelete: 'cascade' }),
|
|
493
|
+
/** The registered hostname (e.g. "git-ssh.git.celilo.computer"). */
|
|
494
|
+
host: text('host').notNull(),
|
|
495
|
+
/** The A-record value celilo asked the resolver to serve. */
|
|
496
|
+
ip: text('ip').notNull(),
|
|
497
|
+
registeredAt: integer('registered_at', { mode: 'timestamp' })
|
|
498
|
+
.notNull()
|
|
499
|
+
.default(sql`(unixepoch())`),
|
|
500
|
+
},
|
|
501
|
+
(table) => ({
|
|
502
|
+
providerHostUnique: uniqueIndex('dns_internal_records_provider_host_idx').on(
|
|
503
|
+
table.providerModuleId,
|
|
504
|
+
table.host,
|
|
505
|
+
),
|
|
506
|
+
}),
|
|
507
|
+
);
|
|
508
|
+
|
|
471
509
|
/**
|
|
472
510
|
* Backup storage providers - destinations for backup archives
|
|
473
511
|
* Supports local filesystem and S3-compatible storage (AWS S3, MinIO, Backblaze B2, Wasabi)
|
|
@@ -37,7 +37,7 @@ export default defineCapabilityFunction({
|
|
|
37
37
|
capability: 'firewall',
|
|
38
38
|
handler: ({ config, secrets }) => ({
|
|
39
39
|
exposeService: async (opts) => ({
|
|
40
|
-
externalIp: '
|
|
40
|
+
externalIp: '203.0.113.10',
|
|
41
41
|
natIp: opts.internalIp,
|
|
42
42
|
}),
|
|
43
43
|
unexposeService: async () => {},
|
|
@@ -135,7 +135,7 @@ describe('Firewall Chain Building', () => {
|
|
|
135
135
|
ports: [80],
|
|
136
136
|
description: 'test',
|
|
137
137
|
});
|
|
138
|
-
expect(exposed.externalIp).toBe('
|
|
138
|
+
expect(exposed.externalIp).toBe('203.0.113.10');
|
|
139
139
|
});
|
|
140
140
|
|
|
141
141
|
test('builds chain with two providers (iptables → greenwave)', async () => {
|
|
@@ -206,7 +206,7 @@ describe('Firewall Chain Building', () => {
|
|
|
206
206
|
});
|
|
207
207
|
|
|
208
208
|
// External IP came from greenwave (leaf)
|
|
209
|
-
expect(exposed.externalIp).toBe('
|
|
209
|
+
expect(exposed.externalIp).toBe('203.0.113.10');
|
|
210
210
|
// NAT IP is iptables' NAT address
|
|
211
211
|
expect(exposed.natIp).toBe('192.168.0.253');
|
|
212
212
|
// iptables created local rules
|
|
@@ -31,6 +31,7 @@ import { decryptSecret } from '../secrets/encryption';
|
|
|
31
31
|
import { getOrCreateMasterKey } from '../secrets/master-key';
|
|
32
32
|
import { emitWebRoutesChangedAndWait } from '../services/celilo-events';
|
|
33
33
|
import { getModuleSystems } from '../services/deployed-systems';
|
|
34
|
+
import { withDnsInternalLedger } from '../services/dns-internal-records';
|
|
34
35
|
import { withDnsRegistrationLedger } from '../services/dns-registrations';
|
|
35
36
|
import { loadHookConfigMap } from './load-hook-config';
|
|
36
37
|
|
|
@@ -200,19 +201,27 @@ export async function loadCapabilityFunctions(
|
|
|
200
201
|
const providerConfig = await loadModuleConfig(capability.moduleId, db);
|
|
201
202
|
const providerSecrets = await loadModuleSecrets(capability.moduleId, masterKey, db);
|
|
202
203
|
|
|
203
|
-
// dns_registrar interfaces get
|
|
204
|
-
// every successful registerHost is recorded
|
|
205
|
-
//
|
|
206
|
-
//
|
|
207
|
-
//
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
204
|
+
// dns_registrar and dns_internal interfaces get a registration-ledger
|
|
205
|
+
// wrapper: every successful registerHost / registerRecord is recorded so
|
|
206
|
+
// celilo has an offline record of what it asked DNS to serve. The
|
|
207
|
+
// external ledger feeds the provider's refresh_registrations hook
|
|
208
|
+
// (DISPATCHER_DAEMON_AND_TIMER_EVENTS.md B2); the internal ledger feeds
|
|
209
|
+
// the doctor's natIp drift check (CELILO_DOCTOR_FLEET_DRIFT.md Phase 4).
|
|
210
|
+
// The loader is the one layer that knows both provider and consumer.
|
|
211
|
+
const ledgerCtx = {
|
|
212
|
+
db,
|
|
213
|
+
providerModuleId: capability.moduleId,
|
|
214
|
+
consumerModuleId: consumingModuleId,
|
|
215
|
+
};
|
|
216
|
+
const withLedger = (iface: unknown): unknown => {
|
|
217
|
+
if (capName === 'dns_registrar') {
|
|
218
|
+
return withDnsRegistrationLedger(iface as DnsRegistrarCapability, ledgerCtx);
|
|
219
|
+
}
|
|
220
|
+
if (capName === 'dns_internal') {
|
|
221
|
+
return withDnsInternalLedger(iface as DnsInternalCapability, ledgerCtx);
|
|
222
|
+
}
|
|
223
|
+
return iface;
|
|
224
|
+
};
|
|
216
225
|
|
|
217
226
|
try {
|
|
218
227
|
// Dynamically import the capability module. Try the default export
|
|
@@ -237,7 +246,7 @@ export async function loadCapabilityFunctions(
|
|
|
237
246
|
systems: getModuleSystems(capability.moduleId, db),
|
|
238
247
|
logger,
|
|
239
248
|
});
|
|
240
|
-
result[capName] =
|
|
249
|
+
result[capName] = withLedger(capabilityInterface);
|
|
241
250
|
debugLog(`${capName}: loaded via defineCapabilityFunction`);
|
|
242
251
|
continue;
|
|
243
252
|
}
|
|
@@ -253,7 +262,7 @@ export async function loadCapabilityFunctions(
|
|
|
253
262
|
);
|
|
254
263
|
|
|
255
264
|
if (capabilityInterface) {
|
|
256
|
-
result[capName] =
|
|
265
|
+
result[capName] = withLedger(
|
|
257
266
|
wrapWithLogging(capabilityInterface as object, logger, capName),
|
|
258
267
|
);
|
|
259
268
|
debugLog(`${capName}: loaded via legacy factory`);
|
|
@@ -78,6 +78,21 @@ describe('extractProxmoxProperties', () => {
|
|
|
78
78
|
});
|
|
79
79
|
});
|
|
80
80
|
|
|
81
|
+
test('omits vm_template when the service config has none (LXC services)', () => {
|
|
82
|
+
const properties = extractProxmoxProperties(100, '10.0.10.5', 'caddy', mockProxmoxConfig);
|
|
83
|
+
// Omitted, not empty — so a required-infrastructure `vm_template` var fails
|
|
84
|
+
// loudly rather than resolving to "".
|
|
85
|
+
expect('vm_template' in properties).toBe(false);
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
test('includes vm_template when the service config provides one (VM services)', () => {
|
|
89
|
+
const properties = extractProxmoxProperties(100, '10.0.10.5', 'builder', {
|
|
90
|
+
...mockProxmoxConfig,
|
|
91
|
+
vm_template: 'ubuntu-2204-cloudinit',
|
|
92
|
+
});
|
|
93
|
+
expect(properties.vm_template).toBe('ubuntu-2204-cloudinit');
|
|
94
|
+
});
|
|
95
|
+
|
|
81
96
|
test('converts vmid number to string', () => {
|
|
82
97
|
const properties = extractProxmoxProperties(12345, '10.0.20.15', 'caddy', mockProxmoxConfig);
|
|
83
98
|
|
|
@@ -47,6 +47,13 @@ export interface ProxmoxProviderConfig {
|
|
|
47
47
|
default_target_node: string;
|
|
48
48
|
lxc_template: string;
|
|
49
49
|
storage: string;
|
|
50
|
+
/**
|
|
51
|
+
* Cloud-init VM template to clone for `requires.system.type: vm` modules — the
|
|
52
|
+
* VM analogue of `lxc_template`. Optional: only Proxmox services that host VM
|
|
53
|
+
* modules configure it. A VM module declares `vm_template` as a *required*
|
|
54
|
+
* infrastructure var, so a service missing it fails loudly at resolution.
|
|
55
|
+
*/
|
|
56
|
+
vm_template?: string;
|
|
50
57
|
}
|
|
51
58
|
|
|
52
59
|
/**
|
|
@@ -72,6 +79,11 @@ export function extractProxmoxProperties(
|
|
|
72
79
|
target_node: providerConfig.default_target_node,
|
|
73
80
|
lxc_template: providerConfig.lxc_template,
|
|
74
81
|
storage: providerConfig.storage,
|
|
82
|
+
// VM clone source — present only when the service configures it. VM modules
|
|
83
|
+
// declare `vm_template` as a required infrastructure var (resolution errors
|
|
84
|
+
// if absent); LXC modules never reference it. Omitted (not empty) when unset
|
|
85
|
+
// so the resolver's required/optional handling stays correct.
|
|
86
|
+
...(providerConfig.vm_template ? { vm_template: providerConfig.vm_template } : {}),
|
|
75
87
|
};
|
|
76
88
|
}
|
|
77
89
|
|
package/src/manifest/schema.ts
CHANGED
|
@@ -311,6 +311,13 @@ export const SystemResourceSchema = z.object({
|
|
|
311
311
|
memory: z.number().int().positive().optional().describe('Recommended memory in MB'),
|
|
312
312
|
disk: z.number().int().positive().optional().describe('Recommended disk size in GB'),
|
|
313
313
|
storage: z.string().optional().describe('Proxmox storage backend (defaults to system config)'),
|
|
314
|
+
type: z
|
|
315
|
+
.enum(['lxc', 'vm'])
|
|
316
|
+
.default('lxc')
|
|
317
|
+
.describe(
|
|
318
|
+
'Proxmox provisioning type: lxc (default) or vm (qemu, for Docker / kernel-module workloads). ' +
|
|
319
|
+
'Modules declare this explicitly; celilo never infers it. Moot for machine-pool / external infra.',
|
|
320
|
+
),
|
|
314
321
|
zone: z
|
|
315
322
|
.enum(['internal', 'dmz', 'app', 'secure', 'external'])
|
|
316
323
|
.describe('Required security zone for this module'),
|