@celilo/cli 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +7 -6
- package/CELILO_SUBSYSTEMS.md +4 -2
- package/drizzle/0020_dns_registrations_drop_ip.sql +25 -0
- package/drizzle/0021_dns_registration_consumers.sql +63 -0
- package/drizzle/0022_dns_registrations_companion.sql +15 -0
- package/drizzle/0023_public_dns_evidence.sql +19 -0
- package/drizzle/meta/_journal.json +29 -1
- package/package.json +4 -4
- package/schemas/system_config.json +22 -11
- package/src/cli/commands/dns.ts +8 -4
- package/src/cli/commands/events.test.ts +66 -0
- package/src/cli/commands/events.ts +76 -1
- package/src/cli/commands/system-audit.ts +15 -0
- package/src/cli/commands/system-migrate.test.ts +25 -4
- package/src/cli/commands/system-update.ts +5 -0
- package/src/cli/completion.ts +1 -0
- package/src/cli/index.ts +4 -0
- package/src/cli/tui/audit-state.ts +2 -0
- package/src/db/dns-registrations-migration.test.ts +205 -0
- package/src/db/schema.ts +77 -8
- package/src/hooks/define-hook.test.ts +3 -3
- package/src/hooks/executor.test.ts +58 -0
- package/src/hooks/executor.ts +67 -7
- package/src/hooks/run-named-hook.ts +7 -1
- package/src/hooks/test-fixtures/silent-hook.ts +20 -0
- package/src/module/packaging/build.ts +14 -0
- package/src/services/alerting/builtin-monitors.ts +3 -0
- package/src/services/alerting/builtin-source.ts +23 -0
- package/src/services/audit/index.test.ts +2 -0
- package/src/services/audit/index.ts +3 -0
- package/src/services/audit/public-dns-source.ts +55 -0
- package/src/services/audit/public-dns.test.ts +209 -0
- package/src/services/audit/public-dns.ts +286 -0
- package/src/services/audit/types.ts +1 -0
- package/src/services/dns-registrations.test.ts +78 -16
- package/src/services/dns-registrations.ts +119 -19
- package/src/services/fleet-checks.test.ts +93 -1
- package/src/services/fleet-checks.ts +51 -10
- package/src/services/module-subscriptions.test.ts +9 -0
- package/src/services/public-dns-probe.test.ts +81 -0
- package/src/services/public-dns-probe.ts +156 -0
- package/src/services/update/orchestrator.test.ts +2 -0
|
@@ -1,11 +1,28 @@
|
|
|
1
1
|
import { Database } from 'bun:sqlite';
|
|
2
2
|
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
3
|
-
import { mkdtempSync, rmSync } from 'node:fs';
|
|
3
|
+
import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
|
4
4
|
import { tmpdir } from 'node:os';
|
|
5
5
|
import { join } from 'node:path';
|
|
6
|
-
import { closeDb } from '../../db/client';
|
|
6
|
+
import { closeDb, findMigrationsFolder } from '../../db/client';
|
|
7
7
|
import { handleSystemMigrate } from './system-migrate';
|
|
8
8
|
|
|
9
|
+
/**
|
|
10
|
+
* The newest migration, read from the journal rather than written down here.
|
|
11
|
+
*
|
|
12
|
+
* These assertions used to name `0019_backup_pid` literally, so every
|
|
13
|
+
* subsequent migration broke a test that has nothing to do with it. What is
|
|
14
|
+
* under test is that `--status` REPORTS the head and any gap below it, not
|
|
15
|
+
* which migration happens to be head today.
|
|
16
|
+
*/
|
|
17
|
+
function latestMigrationTag(): string {
|
|
18
|
+
const journal = JSON.parse(
|
|
19
|
+
readFileSync(join(findMigrationsFolder(), 'meta', '_journal.json'), 'utf8'),
|
|
20
|
+
) as { entries: { tag: string }[] };
|
|
21
|
+
const tag = journal.entries.at(-1)?.tag;
|
|
22
|
+
if (!tag) throw new Error('Migration journal is empty');
|
|
23
|
+
return tag;
|
|
24
|
+
}
|
|
25
|
+
|
|
9
26
|
describe('handleSystemMigrate', () => {
|
|
10
27
|
let dir: string;
|
|
11
28
|
|
|
@@ -51,7 +68,7 @@ describe('handleSystemMigrate', () => {
|
|
|
51
68
|
expect(result.success).toBe(true);
|
|
52
69
|
if (result.success) {
|
|
53
70
|
expect(result.message).toMatch(/Applied migrations: \d+/);
|
|
54
|
-
expect(result.message).toContain(
|
|
71
|
+
expect(result.message).toContain(latestMigrationTag());
|
|
55
72
|
expect(result.message).toContain('Pending: none');
|
|
56
73
|
expect(result.message).toContain('columns');
|
|
57
74
|
}
|
|
@@ -61,6 +78,7 @@ describe('handleSystemMigrate', () => {
|
|
|
61
78
|
await handleSystemMigrate();
|
|
62
79
|
closeDb();
|
|
63
80
|
// Rewind one migration, the way an upgrade that never ran would look.
|
|
81
|
+
const head = latestMigrationTag();
|
|
64
82
|
const raw = new Database(process.env.CELILO_DB_PATH as string);
|
|
65
83
|
raw.run(
|
|
66
84
|
'DELETE FROM `__drizzle_migrations` WHERE created_at = (SELECT MAX(created_at) FROM `__drizzle_migrations`)',
|
|
@@ -75,7 +93,10 @@ describe('handleSystemMigrate', () => {
|
|
|
75
93
|
|
|
76
94
|
expect(result.success).toBe(false);
|
|
77
95
|
if (!result.success) {
|
|
78
|
-
|
|
96
|
+
// The rewound migration is named as pending…
|
|
97
|
+
expect(result.error).toContain(head);
|
|
98
|
+
// …and the dropped COLUMN is reported independently, which is the
|
|
99
|
+
// thing a table count cannot see (celilo#604).
|
|
79
100
|
expect(result.error).toContain('backups.pid');
|
|
80
101
|
}
|
|
81
102
|
|
|
@@ -28,6 +28,7 @@ import { RegistryClient } from '../../registry/client';
|
|
|
28
28
|
import { runAudit } from '../../services/audit';
|
|
29
29
|
import { loadAbandonedOperations } from '../../services/audit/abandoned-operations';
|
|
30
30
|
import { fetchLatestCliVersion } from '../../services/audit/cli-version';
|
|
31
|
+
import { unusedPublicDnsProbe } from '../../services/audit/public-dns';
|
|
31
32
|
import { makeJournalReader, readAppliedMigrations } from '../../services/audit/schema';
|
|
32
33
|
import { createModuleBackup, createSystemStateBackup } from '../../services/backup-create';
|
|
33
34
|
import { runAllHealthChecks, runModuleHealthCheck } from '../../services/health-runner';
|
|
@@ -587,6 +588,10 @@ export async function handleSystemUpdate(
|
|
|
587
588
|
secretsDecryptable: { results: [] },
|
|
588
589
|
servicesReachable: { results: [] },
|
|
589
590
|
machinesReachable: { results: [] },
|
|
591
|
+
// Public reachability needs a network round trip per name; the update
|
|
592
|
+
// flow's partial audit does no probing. `system audit` and the scheduled
|
|
593
|
+
// monitor own this check.
|
|
594
|
+
publicDns: { records: [], probe: unusedPublicDnsProbe },
|
|
590
595
|
transportReads: { statuses: [], now: new Date(), staleAfterMs: 30 * 60_000 },
|
|
591
596
|
trustedSources: { firewalls: [] },
|
|
592
597
|
};
|
package/src/cli/completion.ts
CHANGED
package/src/cli/index.ts
CHANGED
|
@@ -32,6 +32,7 @@ import {
|
|
|
32
32
|
handleEventsEmit,
|
|
33
33
|
handleEventsFail,
|
|
34
34
|
handleEventsInstallDaemon,
|
|
35
|
+
handleEventsListFailed,
|
|
35
36
|
handleEventsListPending,
|
|
36
37
|
handleEventsListSubscribers,
|
|
37
38
|
handleEventsListUnanswered,
|
|
@@ -300,6 +301,7 @@ Subcommands:
|
|
|
300
301
|
list-subscribers List persistent bus subscribers
|
|
301
302
|
resync-subscriptions Rebuild subscribers from deployed modules' manifests (after a restore/migration)
|
|
302
303
|
list-pending [--subscriber] List pending deliveries
|
|
304
|
+
list-failed [--subscriber] List failed/abandoned deliveries with a true total
|
|
303
305
|
drain [--concurrency N] Process pending deliveries once and return
|
|
304
306
|
run [--poll-ms N] Run the long-running dispatcher (foreground)
|
|
305
307
|
emit <type> [<payload>] Emit an event (operator/test path)
|
|
@@ -1425,6 +1427,8 @@ export async function runCli(argv: string[]): Promise<CommandResult> {
|
|
|
1425
1427
|
return handleEventsResyncSubscriptions();
|
|
1426
1428
|
case 'list-pending':
|
|
1427
1429
|
return handleEventsListPending(parsed.args, parsed.flags);
|
|
1430
|
+
case 'list-failed':
|
|
1431
|
+
return handleEventsListFailed(parsed.args, parsed.flags);
|
|
1428
1432
|
case 'list-unanswered':
|
|
1429
1433
|
return handleEventsListUnanswered(parsed.args, parsed.flags);
|
|
1430
1434
|
case 'drain':
|
|
@@ -84,6 +84,7 @@ export const ALL_CATEGORIES: readonly DriftCategory[] = [
|
|
|
84
84
|
'secrets_decryptable',
|
|
85
85
|
'services_reachable',
|
|
86
86
|
'machines_reachable',
|
|
87
|
+
'public_dns',
|
|
87
88
|
'disk_space',
|
|
88
89
|
'transport_reads',
|
|
89
90
|
'trusted_sources',
|
|
@@ -105,6 +106,7 @@ export const CATEGORY_LABELS: Record<DriftCategory, string> = {
|
|
|
105
106
|
secrets_decryptable: 'Secrets',
|
|
106
107
|
services_reachable: 'Service reachability',
|
|
107
108
|
machines_reachable: 'Machine reachability',
|
|
109
|
+
public_dns: 'Public DNS reachability',
|
|
108
110
|
disk_space: 'Disk space',
|
|
109
111
|
transport_reads: 'Transport readability',
|
|
110
112
|
trusted_sources: 'Trusted networks',
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Migration gate for celilo#626 — the one class of defect the rest of the
|
|
3
|
+
* suite is structurally incapable of seeing.
|
|
4
|
+
*
|
|
5
|
+
* A `dns_registrations` row written before #464/#466 carried a literal `ip`,
|
|
6
|
+
* and `refresh_registrations` replayed it every 15 minutes as the address to
|
|
7
|
+
* publish. Once the ISP re-leased, that republished a dead address forever,
|
|
8
|
+
* reporting success each tick. Five public names went dark for nine days,
|
|
9
|
+
* including the apt repo and the module registry, while every in-fleet check
|
|
10
|
+
* stayed green (the split-horizon resolver answers with a reachable address).
|
|
11
|
+
*
|
|
12
|
+
* A thorough e2e gate for exactly this defect already existed and passed
|
|
13
|
+
* throughout — `modules/celilo-website/e2e/website-deploy-new-hostname.test.ts`
|
|
14
|
+
* moves the WAN address and fires the real refresh hook. It could only ever
|
|
15
|
+
* exercise rows the code under test wrote, because e2e always starts from an
|
|
16
|
+
* empty database. A suite that starts clean can prove the FORWARD invariant
|
|
17
|
+
* and says nothing about the installed base — which is where the outage was.
|
|
18
|
+
*
|
|
19
|
+
* So this test builds a database at the OLD schema, puts an armed legacy row
|
|
20
|
+
* in it, and runs the real migrator over it. Dropping the column IS the
|
|
21
|
+
* sweep: every armed row on every fleet disarms with the schema change,
|
|
22
|
+
* rather than waiting for someone to redeploy the module that owns it.
|
|
23
|
+
*
|
|
24
|
+
* See design.md D6.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import { Database } from 'bun:sqlite';
|
|
28
|
+
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
|
29
|
+
import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
30
|
+
import { tmpdir } from 'node:os';
|
|
31
|
+
import { join } from 'node:path';
|
|
32
|
+
import { drizzle } from 'drizzle-orm/bun-sqlite';
|
|
33
|
+
import { migrate } from 'drizzle-orm/bun-sqlite/migrator';
|
|
34
|
+
import { findMigrationsFolder } from './client';
|
|
35
|
+
|
|
36
|
+
/** The last migration before this change — the schema the outage ran on. */
|
|
37
|
+
const LAST_LEGACY_TAG = '0019_backup_pid';
|
|
38
|
+
|
|
39
|
+
interface JournalEntry {
|
|
40
|
+
idx: number;
|
|
41
|
+
version: string;
|
|
42
|
+
when: number;
|
|
43
|
+
tag: string;
|
|
44
|
+
breakpoints: boolean;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* A migrations folder truncated at `LAST_LEGACY_TAG`, so a database can be
|
|
49
|
+
* built at the pre-change schema using the real migration files rather than a
|
|
50
|
+
* hand-written approximation that could drift from them.
|
|
51
|
+
*/
|
|
52
|
+
function legacyMigrationsFolder(into: string): string {
|
|
53
|
+
const source = findMigrationsFolder();
|
|
54
|
+
const journal = JSON.parse(readFileSync(join(source, 'meta', '_journal.json'), 'utf8')) as {
|
|
55
|
+
version: string;
|
|
56
|
+
dialect: string;
|
|
57
|
+
entries: JournalEntry[];
|
|
58
|
+
};
|
|
59
|
+
const cutoff = journal.entries.findIndex((e) => e.tag === LAST_LEGACY_TAG);
|
|
60
|
+
if (cutoff === -1) throw new Error(`Journal has no entry for ${LAST_LEGACY_TAG}`);
|
|
61
|
+
const kept = journal.entries.slice(0, cutoff + 1);
|
|
62
|
+
|
|
63
|
+
mkdirSync(join(into, 'meta'), { recursive: true });
|
|
64
|
+
for (const entry of kept) {
|
|
65
|
+
cpSync(join(source, `${entry.tag}.sql`), join(into, `${entry.tag}.sql`));
|
|
66
|
+
}
|
|
67
|
+
writeFileSync(
|
|
68
|
+
join(into, 'meta', '_journal.json'),
|
|
69
|
+
JSON.stringify({ ...journal, entries: kept }, null, 2),
|
|
70
|
+
);
|
|
71
|
+
return into;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
describe('dns_registrations migration over a pre-#464 database', () => {
|
|
75
|
+
let tempDir: string;
|
|
76
|
+
let dbPath: string;
|
|
77
|
+
|
|
78
|
+
beforeEach(() => {
|
|
79
|
+
tempDir = mkdtempSync(join(tmpdir(), 'celilo-dnsmig-'));
|
|
80
|
+
dbPath = join(tempDir, 'legacy.db');
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
afterEach(() => {
|
|
84
|
+
rmSync(tempDir, { recursive: true, force: true });
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
function openLegacyDatabase(): Database {
|
|
88
|
+
const sqlite = new Database(dbPath, { create: true });
|
|
89
|
+
sqlite.run('PRAGMA foreign_keys = ON');
|
|
90
|
+
migrate(drizzle(sqlite), {
|
|
91
|
+
migrationsFolder: legacyMigrationsFolder(join(tempDir, 'legacy-migrations')),
|
|
92
|
+
});
|
|
93
|
+
return sqlite;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function seedModule(sqlite: Database, id: string): void {
|
|
97
|
+
sqlite.run(
|
|
98
|
+
'INSERT INTO modules (id, name, source_path, version, manifest_data) VALUES (?, ?, ?, ?, ?)',
|
|
99
|
+
[id, id, `/srv/${id}`, '1.0.0', JSON.stringify({ id })],
|
|
100
|
+
);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function runCurrentMigrations(sqlite: Database): void {
|
|
104
|
+
migrate(drizzle(sqlite), { migrationsFolder: findMigrationsFolder() });
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
test('the armed address is gone and the registration survives', () => {
|
|
108
|
+
const sqlite = openLegacyDatabase();
|
|
109
|
+
seedModule(sqlite, 'namecheap');
|
|
110
|
+
seedModule(sqlite, 'caddy');
|
|
111
|
+
|
|
112
|
+
// The shape an older celilo wrote: a literal address the refresh replayed.
|
|
113
|
+
// This is what no deploy can produce today, which is why the seed is here
|
|
114
|
+
// and not in e2e (CLAUDE.md's pre-seeding carve-out).
|
|
115
|
+
sqlite.run(
|
|
116
|
+
'INSERT INTO dns_registrations (provider_module_id, consumer_module_id, fqdn, ip) VALUES (?, ?, ?, ?)',
|
|
117
|
+
['namecheap', 'caddy', 'apt.celilo.computer', '71.36.112.98'],
|
|
118
|
+
);
|
|
119
|
+
|
|
120
|
+
runCurrentMigrations(sqlite);
|
|
121
|
+
|
|
122
|
+
const columns = (
|
|
123
|
+
sqlite.query('PRAGMA table_info(dns_registrations)').all() as { name: string }[]
|
|
124
|
+
).map((c) => c.name);
|
|
125
|
+
expect(columns).not.toContain('ip');
|
|
126
|
+
expect(columns).toContain('companion');
|
|
127
|
+
|
|
128
|
+
const rows = sqlite
|
|
129
|
+
.query('SELECT fqdn, provider_module_id, companion FROM dns_registrations')
|
|
130
|
+
.all() as {
|
|
131
|
+
fqdn: string;
|
|
132
|
+
provider_module_id: string;
|
|
133
|
+
companion: number;
|
|
134
|
+
}[];
|
|
135
|
+
expect(rows).toEqual([
|
|
136
|
+
{ fqdn: 'apt.celilo.computer', provider_module_id: 'namecheap', companion: 0 },
|
|
137
|
+
]);
|
|
138
|
+
|
|
139
|
+
sqlite.close();
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
test('the single consumer becomes the seed of the consumer set', () => {
|
|
143
|
+
const sqlite = openLegacyDatabase();
|
|
144
|
+
seedModule(sqlite, 'namecheap');
|
|
145
|
+
seedModule(sqlite, 'caddy');
|
|
146
|
+
seedModule(sqlite, 'authentik');
|
|
147
|
+
sqlite.run(
|
|
148
|
+
'INSERT INTO dns_registrations (provider_module_id, consumer_module_id, fqdn, ip) VALUES (?, ?, ?, ?)',
|
|
149
|
+
['namecheap', 'caddy', 'auth.lunacycle.net', '71.36.112.98'],
|
|
150
|
+
);
|
|
151
|
+
sqlite.run(
|
|
152
|
+
'INSERT INTO dns_registrations (provider_module_id, consumer_module_id, fqdn, ip) VALUES (?, ?, ?, ?)',
|
|
153
|
+
['namecheap', 'authentik', 'other.lunacycle.net', null],
|
|
154
|
+
);
|
|
155
|
+
|
|
156
|
+
runCurrentMigrations(sqlite);
|
|
157
|
+
|
|
158
|
+
const consumers = sqlite
|
|
159
|
+
.query(
|
|
160
|
+
`SELECT r.fqdn AS fqdn, c.module_id AS module_id
|
|
161
|
+
FROM dns_registration_consumers c
|
|
162
|
+
JOIN dns_registrations r ON r.id = c.registration_id
|
|
163
|
+
ORDER BY r.fqdn`,
|
|
164
|
+
)
|
|
165
|
+
.all() as { fqdn: string; module_id: string }[];
|
|
166
|
+
expect(consumers).toEqual([
|
|
167
|
+
{ fqdn: 'auth.lunacycle.net', module_id: 'caddy' },
|
|
168
|
+
{ fqdn: 'other.lunacycle.net', module_id: 'authentik' },
|
|
169
|
+
]);
|
|
170
|
+
|
|
171
|
+
sqlite.close();
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
test('a migrated row still dies with its provider, and now outlives a co-consumer', () => {
|
|
175
|
+
const sqlite = openLegacyDatabase();
|
|
176
|
+
seedModule(sqlite, 'namecheap');
|
|
177
|
+
seedModule(sqlite, 'caddy');
|
|
178
|
+
seedModule(sqlite, 'authentik');
|
|
179
|
+
sqlite.run(
|
|
180
|
+
'INSERT INTO dns_registrations (provider_module_id, consumer_module_id, fqdn, ip) VALUES (?, ?, ?, ?)',
|
|
181
|
+
['namecheap', 'caddy', 'auth.lunacycle.net', '71.36.112.98'],
|
|
182
|
+
);
|
|
183
|
+
|
|
184
|
+
runCurrentMigrations(sqlite);
|
|
185
|
+
|
|
186
|
+
// authentik also depends on the name, as it does on the live fleet — the
|
|
187
|
+
// recovery re-attributed it to caddy, and under the old single-valued
|
|
188
|
+
// column removing caddy would have cascade-deleted a name still served.
|
|
189
|
+
const registrationId = (
|
|
190
|
+
sqlite.query('SELECT id FROM dns_registrations').get() as { id: number }
|
|
191
|
+
).id;
|
|
192
|
+
sqlite.run(
|
|
193
|
+
'INSERT INTO dns_registration_consumers (registration_id, module_id) VALUES (?, ?)',
|
|
194
|
+
[registrationId, 'authentik'],
|
|
195
|
+
);
|
|
196
|
+
|
|
197
|
+
sqlite.run('DELETE FROM modules WHERE id = ?', ['caddy']);
|
|
198
|
+
expect(sqlite.query('SELECT COUNT(*) AS n FROM dns_registrations').get()).toEqual({ n: 1 });
|
|
199
|
+
|
|
200
|
+
sqlite.run('DELETE FROM modules WHERE id = ?', ['namecheap']);
|
|
201
|
+
expect(sqlite.query('SELECT COUNT(*) AS n FROM dns_registrations').get()).toEqual({ n: 0 });
|
|
202
|
+
|
|
203
|
+
sqlite.close();
|
|
204
|
+
});
|
|
205
|
+
});
|
package/src/db/schema.ts
CHANGED
|
@@ -592,9 +592,19 @@ export const trustedSources = sqliteTable(
|
|
|
592
592
|
* has successfully registered via dns_registrar.registerHost. Written
|
|
593
593
|
* framework-side by the capability loader (the only layer that knows
|
|
594
594
|
* both consumer and provider), read back to drive the provider's
|
|
595
|
-
* periodic refresh_registrations hook
|
|
596
|
-
*
|
|
597
|
-
*
|
|
595
|
+
* periodic refresh_registrations hook, the `public_dns` reachability
|
|
596
|
+
* check, and the `celilo dns registrations` view.
|
|
597
|
+
*
|
|
598
|
+
* It records WHICH MODULE ASKED FOR WHICH NAME and nothing else. In
|
|
599
|
+
* particular it does not store the published address: that is observable
|
|
600
|
+
* on demand from public DNS, and a stored copy is one careless read away
|
|
601
|
+
* from becoming an instruction again — which is exactly what took five
|
|
602
|
+
* public names dark for nine days (celilo#626).
|
|
603
|
+
*
|
|
604
|
+
* Rows die with their provider by FK cascade. The consumer side is a
|
|
605
|
+
* SET (`dns_registration_consumers`), not a column: the row must survive
|
|
606
|
+
* until its LAST consumer is removed. The remote DNS record itself stays
|
|
607
|
+
* (Namecheap DDNS has no delete API).
|
|
598
608
|
* See designs/DISPATCHER_DAEMON_AND_TIMER_EVENTS.md (B2).
|
|
599
609
|
*/
|
|
600
610
|
export const dnsRegistrations = sqliteTable(
|
|
@@ -604,12 +614,15 @@ export const dnsRegistrations = sqliteTable(
|
|
|
604
614
|
providerModuleId: text('provider_module_id')
|
|
605
615
|
.notNull()
|
|
606
616
|
.references(() => modules.id, { onDelete: 'cascade' }),
|
|
607
|
-
consumerModuleId: text('consumer_module_id')
|
|
608
|
-
.notNull()
|
|
609
|
-
.references(() => modules.id, { onDelete: 'cascade' }),
|
|
610
617
|
fqdn: text('fqdn').notNull(),
|
|
611
|
-
/**
|
|
612
|
-
|
|
618
|
+
/**
|
|
619
|
+
* True when celilo claimed this name on a module's behalf as the
|
|
620
|
+
* companion of a declared name (`www.<domain>` ↔ `<domain>`) rather
|
|
621
|
+
* than because a module asked for it. Companions are best effort at
|
|
622
|
+
* claim time, so the `public_dns` check carries the manual-registrar
|
|
623
|
+
* remediation for them.
|
|
624
|
+
*/
|
|
625
|
+
companion: integer('companion', { mode: 'boolean' }).notNull().default(false),
|
|
613
626
|
registeredAt: integer('registered_at', { mode: 'timestamp' })
|
|
614
627
|
.notNull()
|
|
615
628
|
.default(sql`(unixepoch())`),
|
|
@@ -623,6 +636,62 @@ export const dnsRegistrations = sqliteTable(
|
|
|
623
636
|
}),
|
|
624
637
|
);
|
|
625
638
|
|
|
639
|
+
/**
|
|
640
|
+
* Which modules currently depend on a DNS registration.
|
|
641
|
+
*
|
|
642
|
+
* Was a single `dns_registrations.consumer_module_id`, overwritten by
|
|
643
|
+
* every re-assert. That is not a label — the FK cascade on it decided
|
|
644
|
+
* when a LIVE record was forgotten, so recovering the fleet with
|
|
645
|
+
* `run-hook caddy on_install` re-attributed most names to caddy and
|
|
646
|
+
* removing caddy would then have cascade-deleted registrations that
|
|
647
|
+
* authentik and the site modules still served. Keep-first has the mirror
|
|
648
|
+
* failure. A set is the only model where the row dies exactly when the
|
|
649
|
+
* last module that wants the name goes away (design.md D5).
|
|
650
|
+
*
|
|
651
|
+
* The introducing module is the earliest row by `id`.
|
|
652
|
+
*/
|
|
653
|
+
export const dnsRegistrationConsumers = sqliteTable(
|
|
654
|
+
'dns_registration_consumers',
|
|
655
|
+
{
|
|
656
|
+
id: integer('id').primaryKey({ autoIncrement: true }),
|
|
657
|
+
registrationId: integer('registration_id')
|
|
658
|
+
.notNull()
|
|
659
|
+
.references(() => dnsRegistrations.id, { onDelete: 'cascade' }),
|
|
660
|
+
moduleId: text('module_id')
|
|
661
|
+
.notNull()
|
|
662
|
+
.references(() => modules.id, { onDelete: 'cascade' }),
|
|
663
|
+
firstSeenAt: integer('first_seen_at', { mode: 'timestamp' })
|
|
664
|
+
.notNull()
|
|
665
|
+
.default(sql`(unixepoch())`),
|
|
666
|
+
},
|
|
667
|
+
(table) => ({
|
|
668
|
+
registrationModuleUnique: uniqueIndex('dns_registration_consumers_unique_idx').on(
|
|
669
|
+
table.registrationId,
|
|
670
|
+
table.moduleId,
|
|
671
|
+
),
|
|
672
|
+
}),
|
|
673
|
+
);
|
|
674
|
+
|
|
675
|
+
/**
|
|
676
|
+
* Consecutive runs the `public_dns` check could obtain NO evidence about a
|
|
677
|
+
* subject (a served FQDN, or `system` for the echo service itself).
|
|
678
|
+
*
|
|
679
|
+
* A probe that could not run is not a pass. The one genuine external probe the
|
|
680
|
+
* fleet had was itself unreachable during celilo#626 and recorded that as
|
|
681
|
+
* *undetermined* with no check item at all, so a real outage produced complete
|
|
682
|
+
* silence. One blip must still stay quiet — a prober outage once paged three
|
|
683
|
+
* modules at once — so absences are counted rather than reported, and become
|
|
684
|
+
* their own finding only once they persist. A subject that answers has its row
|
|
685
|
+
* dropped, which is what makes the count consecutive.
|
|
686
|
+
*/
|
|
687
|
+
export const publicDnsEvidence = sqliteTable('public_dns_evidence', {
|
|
688
|
+
subject: text('subject').primaryKey(),
|
|
689
|
+
undeterminedRuns: integer('undetermined_runs').notNull().default(0),
|
|
690
|
+
lastCheckedAt: integer('last_checked_at', { mode: 'timestamp' })
|
|
691
|
+
.notNull()
|
|
692
|
+
.default(sql`(unixepoch())`),
|
|
693
|
+
});
|
|
694
|
+
|
|
626
695
|
/**
|
|
627
696
|
* Internal split-horizon DNS A-record ledger. Mirrors `dns_registrations`
|
|
628
697
|
* but for the dns_internal capability (technitium/knot): the capability
|
|
@@ -364,7 +364,7 @@ describe('defineCapabilityFunction', () => {
|
|
|
364
364
|
async registerHost(request: RegisterHostRequest): Promise<HookResult> {
|
|
365
365
|
return {
|
|
366
366
|
success: true,
|
|
367
|
-
outputs: {
|
|
367
|
+
outputs: { registered_fqdn: request.fqdn },
|
|
368
368
|
duration: 1,
|
|
369
369
|
};
|
|
370
370
|
},
|
|
@@ -378,9 +378,9 @@ describe('defineCapabilityFunction', () => {
|
|
|
378
378
|
logger: makeLogger(),
|
|
379
379
|
});
|
|
380
380
|
|
|
381
|
-
const result = await methods.registerHost({ fqdn: 'www.example.com'
|
|
381
|
+
const result = await methods.registerHost({ fqdn: 'www.example.com' });
|
|
382
382
|
expect(result.success).toBe(true);
|
|
383
|
-
expect(result.outputs.
|
|
383
|
+
expect(result.outputs.registered_fqdn).toBe('www.example.com');
|
|
384
384
|
});
|
|
385
385
|
|
|
386
386
|
test('works for the firewall capability', async () => {
|
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
executeHookScript,
|
|
7
7
|
invokeHook,
|
|
8
8
|
resolveHookScript,
|
|
9
|
+
resolveHookTimeouts,
|
|
9
10
|
validateHookInputs,
|
|
10
11
|
validateHookOutputs,
|
|
11
12
|
} from './executor';
|
|
@@ -212,6 +213,63 @@ describe('Hook Executor', () => {
|
|
|
212
213
|
}),
|
|
213
214
|
).rejects.toThrow('Hook execution failed: simulated error');
|
|
214
215
|
});
|
|
216
|
+
|
|
217
|
+
// celilo#622: the idle bound must be the caller's to set. The idle timer
|
|
218
|
+
// polls every 5s, so this costs ~6s of wall clock — it is the only
|
|
219
|
+
// assertion here that touches a real clock, which is why the *decision*
|
|
220
|
+
// lives in the pure resolveHookTimeouts below.
|
|
221
|
+
test('honors a caller-supplied idle timeout instead of the 30s default', async () => {
|
|
222
|
+
const { logger } = createCapturingLogger();
|
|
223
|
+
const scriptPath = join(FIXTURES_DIR, 'silent-hook.ts');
|
|
224
|
+
const context = {
|
|
225
|
+
config: { silent_ms: 8000 },
|
|
226
|
+
secrets: {},
|
|
227
|
+
systems: [],
|
|
228
|
+
logger,
|
|
229
|
+
debug: false,
|
|
230
|
+
screenshotDir: '/tmp',
|
|
231
|
+
capabilities: {},
|
|
232
|
+
};
|
|
233
|
+
|
|
234
|
+
// 8s of silence against a 1ms idle bound: killed at the first poll.
|
|
235
|
+
// Under the old hardcoded 30s idle this resolved instead.
|
|
236
|
+
await expect(executeHookScript(scriptPath, context, 60_000, 1)).rejects.toThrow(
|
|
237
|
+
'idle timeout exceeded',
|
|
238
|
+
);
|
|
239
|
+
}, 20_000);
|
|
240
|
+
});
|
|
241
|
+
|
|
242
|
+
describe('resolveHookTimeouts', () => {
|
|
243
|
+
test('no declaration: 60s total, 30s idle heuristic', () => {
|
|
244
|
+
expect(resolveHookTimeouts(undefined, false)).toEqual({
|
|
245
|
+
timeoutMs: 60_000,
|
|
246
|
+
idleTimeoutMs: 30_000,
|
|
247
|
+
});
|
|
248
|
+
});
|
|
249
|
+
|
|
250
|
+
// The bug in one assertion: namecheap declares 120000 and must get 120s,
|
|
251
|
+
// not 60s — and must NOT still be killed at 30s of silence, which would
|
|
252
|
+
// honor the declaration in name only and leave it exactly as broken.
|
|
253
|
+
test('a declared timeout sets the total AND replaces the idle heuristic', () => {
|
|
254
|
+
expect(resolveHookTimeouts(120_000, false)).toEqual({
|
|
255
|
+
timeoutMs: 120_000,
|
|
256
|
+
idleTimeoutMs: 120_000,
|
|
257
|
+
});
|
|
258
|
+
});
|
|
259
|
+
|
|
260
|
+
test('a declared timeout shorter than the idle default still wins', () => {
|
|
261
|
+
expect(resolveHookTimeouts(5_000, false)).toEqual({
|
|
262
|
+
timeoutMs: 5_000,
|
|
263
|
+
idleTimeoutMs: 5_000,
|
|
264
|
+
});
|
|
265
|
+
});
|
|
266
|
+
|
|
267
|
+
test('debug mode overrides any declaration', () => {
|
|
268
|
+
expect(resolveHookTimeouts(120_000, true)).toEqual({
|
|
269
|
+
timeoutMs: 600_000,
|
|
270
|
+
idleTimeoutMs: 600_000,
|
|
271
|
+
});
|
|
272
|
+
});
|
|
215
273
|
});
|
|
216
274
|
|
|
217
275
|
describe('invokeHook', () => {
|
package/src/hooks/executor.ts
CHANGED
|
@@ -37,7 +37,15 @@ import type { HookContext, HookDefinition, HookLogger, HookResult } from './type
|
|
|
37
37
|
/** Default total timeout: 60 seconds */
|
|
38
38
|
const DEFAULT_TIMEOUT_MS = 60_000;
|
|
39
39
|
|
|
40
|
-
/**
|
|
40
|
+
/**
|
|
41
|
+
* Default idle timeout: 30 seconds since last log message.
|
|
42
|
+
*
|
|
43
|
+
* This is a *heuristic* for hooks that declared no bound of their own — it
|
|
44
|
+
* catches a wedged hook faster than the total timeout would. It is NOT a
|
|
45
|
+
* contract, and it must not silently override one: when a hook or a
|
|
46
|
+
* subscription declares an explicit timeout, that declaration replaces this
|
|
47
|
+
* heuristic entirely (see `invokeHook`).
|
|
48
|
+
*/
|
|
41
49
|
const IDLE_TIMEOUT_MS = 30_000;
|
|
42
50
|
|
|
43
51
|
/**
|
|
@@ -123,12 +131,15 @@ export function resolveHookScript(modulePath: string, scriptPath: string): strin
|
|
|
123
131
|
* @param scriptPath - Absolute path to the hook script
|
|
124
132
|
* @param context - Hook context with config, secrets, logger, and inputs
|
|
125
133
|
* @param timeoutMs - Total timeout in milliseconds
|
|
134
|
+
* @param idleTimeoutMs - Kill after this much silence (no `ctx.logger` call).
|
|
135
|
+
* Pass `timeoutMs` to disable the idle heuristic — see `IDLE_TIMEOUT_MS`.
|
|
126
136
|
* @returns Hook result with outputs
|
|
127
137
|
*/
|
|
128
138
|
export async function executeHookScript(
|
|
129
139
|
scriptPath: string,
|
|
130
140
|
context: HookContext,
|
|
131
141
|
timeoutMs: number = DEFAULT_TIMEOUT_MS,
|
|
142
|
+
idleTimeoutMs: number = IDLE_TIMEOUT_MS,
|
|
132
143
|
): Promise<Record<string, unknown>> {
|
|
133
144
|
if (!existsSync(scriptPath)) {
|
|
134
145
|
throw new Error(`Hook script not found: ${scriptPath}`);
|
|
@@ -169,7 +180,7 @@ export async function executeHookScript(
|
|
|
169
180
|
createTimeoutPromise(timeoutMs, 'total'),
|
|
170
181
|
];
|
|
171
182
|
if (!context.debug) {
|
|
172
|
-
promises.push(createIdleTimeoutPromise(() => Date.now() - lastActivity,
|
|
183
|
+
promises.push(createIdleTimeoutPromise(() => Date.now() - lastActivity, idleTimeoutMs));
|
|
173
184
|
}
|
|
174
185
|
const result = await Promise.race(promises);
|
|
175
186
|
|
|
@@ -278,6 +289,52 @@ export interface InvokeHookOptions {
|
|
|
278
289
|
* skip the pre-flight check, matching the pre-Phase-3 behavior.
|
|
279
290
|
*/
|
|
280
291
|
requiredCapabilities?: string[];
|
|
292
|
+
/**
|
|
293
|
+
* Total timeout for THIS invocation, in milliseconds. Set by the caller
|
|
294
|
+
* that knows the per-invocation bound — for a bus delivery that's the
|
|
295
|
+
* subscription's `timeout_ms` (celilo#622). Takes precedence over the
|
|
296
|
+
* manifest hook's own `timeout`, being the more specific declaration.
|
|
297
|
+
*/
|
|
298
|
+
timeoutMs?: number;
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
/**
|
|
302
|
+
* Decide the total and idle bounds for one hook invocation.
|
|
303
|
+
*
|
|
304
|
+
* Planning function (Rule 10.4) — pure, so the crux below is testable
|
|
305
|
+
* without waiting on a real clock.
|
|
306
|
+
*
|
|
307
|
+
* **Idle-vs-total (celilo#622): an explicit declaration REPLACES the idle
|
|
308
|
+
* heuristic rather than being capped by it** — the idle bound becomes the
|
|
309
|
+
* total, which is the same as switching it off. The idle timer measures
|
|
310
|
+
* silence since the last `ctx.logger` call, which is a proxy for "this hook
|
|
311
|
+
* told us nothing about how long it needs". Once a hook or subscription
|
|
312
|
+
* states a bound, that proxy is not merely redundant but wrong: honoring a
|
|
313
|
+
* declared 120s total while still killing at 30s of silence honors the
|
|
314
|
+
* declaration in name only. That is exactly how namecheap's ddns
|
|
315
|
+
* `refresh_registrations` died — it declares `timeout_ms: 120000`, logs once
|
|
316
|
+
* per name, and a single slow upstream response is 30s of silence.
|
|
317
|
+
*
|
|
318
|
+
* The cost is deliberate: a hook that declares a bound and then truly wedges
|
|
319
|
+
* is now bounded only by that declared total, not killed early. That is the
|
|
320
|
+
* declaring subscriber's call to make, and the dispatcher kills the handler
|
|
321
|
+
* subprocess at the same `timeout_ms` regardless.
|
|
322
|
+
*
|
|
323
|
+
* @param declaredTimeoutMs - Explicitly declared total, or undefined
|
|
324
|
+
* @param debug - Debug mode uses a long total and no idle timer at all
|
|
325
|
+
*/
|
|
326
|
+
export function resolveHookTimeouts(
|
|
327
|
+
declaredTimeoutMs: number | undefined,
|
|
328
|
+
debug: boolean,
|
|
329
|
+
): { timeoutMs: number; idleTimeoutMs: number } {
|
|
330
|
+
const timeoutMs = debug
|
|
331
|
+
? 600_000 // 10 minutes for interactive debugging
|
|
332
|
+
: (declaredTimeoutMs ?? DEFAULT_TIMEOUT_MS);
|
|
333
|
+
|
|
334
|
+
return {
|
|
335
|
+
timeoutMs,
|
|
336
|
+
idleTimeoutMs: declaredTimeoutMs === undefined ? IDLE_TIMEOUT_MS : timeoutMs,
|
|
337
|
+
};
|
|
281
338
|
}
|
|
282
339
|
|
|
283
340
|
/**
|
|
@@ -449,15 +506,18 @@ export async function invokeHook(
|
|
|
449
506
|
}
|
|
450
507
|
}
|
|
451
508
|
|
|
452
|
-
//
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
509
|
+
// An explicitly declared bound — the subscription's `timeout_ms` for a bus
|
|
510
|
+
// delivery, else the manifest hook's own `timeout`. The subscription wins:
|
|
511
|
+
// it is the more specific declaration for this invocation.
|
|
512
|
+
const { timeoutMs, idleTimeoutMs } = resolveHookTimeouts(
|
|
513
|
+
options.timeoutMs ?? definition.timeout,
|
|
514
|
+
debug,
|
|
515
|
+
);
|
|
456
516
|
|
|
457
517
|
// Execute
|
|
458
518
|
try {
|
|
459
519
|
logger.info(`Executing hook: ${hookName}`);
|
|
460
|
-
const outputs = await executeHookScript(scriptPath, context, timeoutMs);
|
|
520
|
+
const outputs = await executeHookScript(scriptPath, context, timeoutMs, idleTimeoutMs);
|
|
461
521
|
|
|
462
522
|
// Validate outputs against the contract signature
|
|
463
523
|
const outputError = validateHookOutputs(signature, outputs);
|