@celilo/cli 0.13.2 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +3 -0
- package/CELILO_SUBSYSTEMS.md +71 -2
- package/docs/ALERTING.md +298 -0
- package/docs/INDEX.md +103 -0
- package/drizzle/0016_trusted_sources.sql +10 -0
- package/drizzle/0017_alerting.sql +127 -0
- package/drizzle/meta/_journal.json +15 -1
- package/package.json +3 -2
- package/schemas/system_config.json +9 -0
- package/src/ansible/inventory.ts +2 -2
- package/src/cli/commands/alerts-act.ts +107 -0
- package/src/cli/commands/alerts-list.ts +62 -0
- package/src/cli/commands/alerts-poll.ts +129 -0
- package/src/cli/commands/alerts-sweep.ts +156 -0
- package/src/cli/commands/module-list.ts +50 -3
- package/src/cli/commands/monitor.ts +178 -0
- package/src/cli/commands/notify-config.ts +453 -0
- package/src/cli/commands/system-audit.ts +2 -0
- package/src/cli/commands/system-update.ts +1 -0
- package/src/cli/completion.ts +26 -0
- package/src/cli/generate-zsh-completion.ts +2 -0
- package/src/cli/index.ts +58 -0
- package/src/cli/tui/audit-state.ts +2 -0
- package/src/db/schema.ts +358 -0
- package/src/hooks/capability-loader.ts +158 -46
- package/src/hooks/capability-map-coverage.test.ts +101 -0
- package/src/manifest/schema.ts +60 -1
- package/src/services/alerting/ack.test.ts +212 -0
- package/src/services/alerting/ack.ts +119 -0
- package/src/services/alerting/builtin-monitors.test.ts +132 -0
- package/src/services/alerting/builtin-monitors.ts +84 -0
- package/src/services/alerting/builtin-source.ts +82 -0
- package/src/services/alerting/coverage-source.ts +38 -0
- package/src/services/alerting/deferral.test.ts +161 -0
- package/src/services/alerting/delivery-loop.test.ts +396 -0
- package/src/services/alerting/deploy-hooks.test.ts +125 -0
- package/src/services/alerting/deploy-hooks.ts +111 -0
- package/src/services/alerting/escalation.test.ts +207 -0
- package/src/services/alerting/escalation.ts +151 -0
- package/src/services/alerting/format.test.ts +193 -0
- package/src/services/alerting/format.ts +150 -0
- package/src/services/alerting/health-coverage.ts +81 -0
- package/src/services/alerting/inbound-poller.test.ts +298 -0
- package/src/services/alerting/inbound-poller.ts +236 -0
- package/src/services/alerting/inbound.test.ts +201 -0
- package/src/services/alerting/inbound.ts +112 -0
- package/src/services/alerting/interview-responder.test.ts +169 -0
- package/src/services/alerting/interview-responder.ts +158 -0
- package/src/services/alerting/keys.test.ts +155 -0
- package/src/services/alerting/keys.ts +190 -0
- package/src/services/alerting/monitors.ts +185 -0
- package/src/services/alerting/notification-responder.test.ts +290 -0
- package/src/services/alerting/notification-responder.ts +260 -0
- package/src/services/alerting/notifier.ts +219 -0
- package/src/services/alerting/people.ts +178 -0
- package/src/services/alerting/quiet-hours.test.ts +140 -0
- package/src/services/alerting/quiet-hours.ts +99 -0
- package/src/services/alerting/reconcile.test.ts +190 -0
- package/src/services/alerting/reconcile.ts +166 -0
- package/src/services/alerting/run-monitor.test.ts +185 -0
- package/src/services/alerting/run-monitor.ts +177 -0
- package/src/services/alerting/store.test.ts +222 -0
- package/src/services/alerting/store.ts +289 -0
- package/src/services/alerting/suppression.test.ts +228 -0
- package/src/services/alerting/suppression.ts +142 -0
- package/src/services/alerting/sweep-runner.test.ts +229 -0
- package/src/services/alerting/sweep-runner.ts +204 -0
- package/src/services/alerting/sweep.test.ts +61 -0
- package/src/services/alerting/sweep.ts +41 -0
- package/src/services/alerting/tokens.test.ts +152 -0
- package/src/services/alerting/tokens.ts +119 -0
- package/src/services/alerting/transport-loader.ts +48 -0
- package/src/services/aspect-runner.ts +2 -2
- package/src/services/audit/index.test.ts +1 -0
- package/src/services/audit/index.ts +3 -0
- package/src/services/audit/trusted-sources.test.ts +137 -0
- package/src/services/audit/trusted-sources.ts +124 -0
- package/src/services/audit/types.ts +2 -1
- package/src/services/firewall-reach.ts +83 -0
- package/src/services/health-runner.test.ts +50 -0
- package/src/services/health-runner.ts +116 -82
- package/src/services/module-deploy.ts +32 -3
- package/src/services/ssh-key-manager.test.ts +14 -0
- package/src/services/ssh-key-manager.ts +12 -0
- package/src/services/system-config-validator.test.ts +31 -1
- package/src/services/trusted-sources.test.ts +221 -0
- package/src/services/trusted-sources.ts +159 -0
- package/src/services/update/orchestrator.test.ts +1 -0
- package/src/templates/generator.ts +6 -29
|
@@ -50,6 +50,7 @@ import type { ServiceCredentialsResult } from '../../services/audit/services-cre
|
|
|
50
50
|
import type { ServiceReachableResult } from '../../services/audit/services-reachable';
|
|
51
51
|
import type { TerraformPlanRunner } from '../../services/audit/terraform-plan';
|
|
52
52
|
import { getServiceCredentials, listContainerServices } from '../../services/container-service';
|
|
53
|
+
import { collectFirewallReach } from '../../services/firewall-reach';
|
|
53
54
|
import { runAllHealthChecks } from '../../services/health-runner';
|
|
54
55
|
import { listMachines } from '../../services/machine-pool';
|
|
55
56
|
import { parseStoredConfigValue } from '../../services/module-config';
|
|
@@ -436,6 +437,7 @@ async function buildAuditDeps(onProgress?: (msg: string) => void) {
|
|
|
436
437
|
secretsDecryptable: { results: secretResults },
|
|
437
438
|
servicesReachable: { results: serviceReachableResults },
|
|
438
439
|
machinesReachable: { results: machineReachableResults },
|
|
440
|
+
trustedSources: { firewalls: collectFirewallReach(db) },
|
|
439
441
|
};
|
|
440
442
|
}
|
|
441
443
|
|
package/src/cli/completion.ts
CHANGED
|
@@ -37,16 +37,21 @@ export async function getCompletions(words: string[], current: number): Promise<
|
|
|
37
37
|
'commands',
|
|
38
38
|
'dns',
|
|
39
39
|
'completion',
|
|
40
|
+
'alerts',
|
|
41
|
+
'escalation-policy',
|
|
40
42
|
'events',
|
|
41
43
|
'help',
|
|
42
44
|
'hook',
|
|
43
45
|
'ipam',
|
|
44
46
|
'machine',
|
|
45
47
|
'module',
|
|
48
|
+
'monitor',
|
|
46
49
|
'package',
|
|
50
|
+
'person',
|
|
47
51
|
'proxmox',
|
|
48
52
|
'publish',
|
|
49
53
|
'registry',
|
|
54
|
+
'route',
|
|
50
55
|
'restore',
|
|
51
56
|
'service',
|
|
52
57
|
'status',
|
|
@@ -449,6 +454,27 @@ export async function getCompletions(words: string[], current: number): Promise<
|
|
|
449
454
|
return filterSuggestions(subcommands, args[1] || '');
|
|
450
455
|
}
|
|
451
456
|
|
|
457
|
+
// Person / route / escalation-policy subcommands
|
|
458
|
+
if (command === 'person' && currentIndex === 1) {
|
|
459
|
+
return filterSuggestions(['list', 'add', 'remove'], args[1] || '');
|
|
460
|
+
}
|
|
461
|
+
if (command === 'route' && currentIndex === 1) {
|
|
462
|
+
return filterSuggestions(['list', 'add', 'remove'], args[1] || '');
|
|
463
|
+
}
|
|
464
|
+
if (command === 'escalation-policy' && currentIndex === 1) {
|
|
465
|
+
return filterSuggestions(['list', 'add', 'step', 'assign', 'remove'], args[1] || '');
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
// Monitor subcommands
|
|
469
|
+
if (command === 'monitor' && currentIndex === 1) {
|
|
470
|
+
return filterSuggestions(['list', 'add', 'run', 'enable', 'disable'], args[1] || '');
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
// Alerts subcommands
|
|
474
|
+
if (command === 'alerts' && currentIndex === 1) {
|
|
475
|
+
return filterSuggestions(['list', 'ack', 'silence', 'resolve', 'sweep', 'poll'], args[1] || '');
|
|
476
|
+
}
|
|
477
|
+
|
|
452
478
|
// Storage subcommands
|
|
453
479
|
if (command === 'storage' && currentIndex === 1) {
|
|
454
480
|
const subcommands = ['add', 'list', 'remove', 'verify', 'set-default'];
|
|
@@ -357,6 +357,8 @@ _celilo_system_config_keys() {
|
|
|
357
357
|
'network.internal.gateway:Internal gateway IP'
|
|
358
358
|
'network.secure-mgmt.subnet:Control-plane subnet (celilo-mgr own network)'
|
|
359
359
|
'network.secure-mgmt.gateway:Control-plane gateway IP'
|
|
360
|
+
'network.vpn.subnet:VPN client subnet (WireGuard remote access)'
|
|
361
|
+
'firewall.trusted_subnets:Extra subnets that reach every managed zone (comma-separated CIDRs)'
|
|
360
362
|
'dns.primary:Primary DNS server'
|
|
361
363
|
'dns.fallback:Fallback DNS servers'
|
|
362
364
|
'routing.internal_gateway:Internal gateway IP'
|
package/src/cli/index.ts
CHANGED
|
@@ -178,6 +178,11 @@ Usage:
|
|
|
178
178
|
Commands:
|
|
179
179
|
status Show system and module status
|
|
180
180
|
audit Top-level alias for 'system audit'
|
|
181
|
+
alerts View alerts raised by monitors
|
|
182
|
+
monitor Manage what celilo watches (health checks on a schedule)
|
|
183
|
+
person Manage people celilo can reach
|
|
184
|
+
route Manage how each person is reached (transport + address)
|
|
185
|
+
escalation-policy Manage who gets paged, and in what order
|
|
181
186
|
events SQLite event-bus operations (status, tail, run dispatcher, etc.)
|
|
182
187
|
capability View registered module capabilities
|
|
183
188
|
dns View DNS bookkeeping (registrations ledger)
|
|
@@ -1708,6 +1713,59 @@ export async function runCli(argv: string[]): Promise<CommandResult> {
|
|
|
1708
1713
|
}
|
|
1709
1714
|
}
|
|
1710
1715
|
|
|
1716
|
+
if (parsed.command === 'person') {
|
|
1717
|
+
const { handlePerson } = await import('./commands/notify-config');
|
|
1718
|
+
return handlePerson(parsed.subcommand, parsed.args, parsed.flags);
|
|
1719
|
+
}
|
|
1720
|
+
|
|
1721
|
+
if (parsed.command === 'route') {
|
|
1722
|
+
const { handleRoute } = await import('./commands/notify-config');
|
|
1723
|
+
return handleRoute(parsed.subcommand, parsed.args, parsed.flags);
|
|
1724
|
+
}
|
|
1725
|
+
|
|
1726
|
+
if (parsed.command === 'escalation-policy') {
|
|
1727
|
+
const { handleEscalationPolicy } = await import('./commands/notify-config');
|
|
1728
|
+
return handleEscalationPolicy(parsed.subcommand, parsed.args, parsed.flags);
|
|
1729
|
+
}
|
|
1730
|
+
|
|
1731
|
+
if (parsed.command === 'monitor') {
|
|
1732
|
+
const { handleMonitor } = await import('./commands/monitor');
|
|
1733
|
+
return handleMonitor(parsed.subcommand, parsed.args, parsed.flags);
|
|
1734
|
+
}
|
|
1735
|
+
|
|
1736
|
+
if (parsed.command === 'alerts') {
|
|
1737
|
+
if (parsed.subcommand === 'ack') {
|
|
1738
|
+
const { handleAlertsAck } = await import('./commands/alerts-act');
|
|
1739
|
+
return handleAlertsAck(parsed.args, parsed.flags);
|
|
1740
|
+
}
|
|
1741
|
+
if (parsed.subcommand === 'silence') {
|
|
1742
|
+
const { handleAlertsSilence } = await import('./commands/alerts-act');
|
|
1743
|
+
return handleAlertsSilence(parsed.args, parsed.flags);
|
|
1744
|
+
}
|
|
1745
|
+
if (parsed.subcommand === 'resolve') {
|
|
1746
|
+
const { handleAlertsResolve } = await import('./commands/alerts-act');
|
|
1747
|
+
return handleAlertsResolve(parsed.args);
|
|
1748
|
+
}
|
|
1749
|
+
if (parsed.subcommand === 'poll') {
|
|
1750
|
+
const { handleAlertsPoll } = await import('./commands/alerts-poll');
|
|
1751
|
+
return handleAlertsPoll(parsed.flags);
|
|
1752
|
+
}
|
|
1753
|
+
if (parsed.subcommand === 'sweep') {
|
|
1754
|
+
const { handleAlertsSweep } = await import('./commands/alerts-sweep');
|
|
1755
|
+
return handleAlertsSweep();
|
|
1756
|
+
}
|
|
1757
|
+
if (!parsed.subcommand || parsed.subcommand === 'list') {
|
|
1758
|
+
const alertsFlagError = checkFlags('alerts', 'list', parsed.flags, parsed.args);
|
|
1759
|
+
if (alertsFlagError) return alertsFlagError;
|
|
1760
|
+
const { handleAlertsList } = await import('./commands/alerts-list');
|
|
1761
|
+
return handleAlertsList(parsed.args, parsed.flags);
|
|
1762
|
+
}
|
|
1763
|
+
return {
|
|
1764
|
+
success: false,
|
|
1765
|
+
error: `Unknown alerts subcommand: ${parsed.subcommand}\n\nUse: list, ack, silence, resolve, sweep, poll`,
|
|
1766
|
+
};
|
|
1767
|
+
}
|
|
1768
|
+
|
|
1711
1769
|
if (parsed.command === 'storage') {
|
|
1712
1770
|
if (parsed.flags.help || parsed.flags.h) {
|
|
1713
1771
|
return displayStorageHelp();
|
|
@@ -83,6 +83,7 @@ export const ALL_CATEGORIES: readonly DriftCategory[] = [
|
|
|
83
83
|
'secrets_decryptable',
|
|
84
84
|
'services_reachable',
|
|
85
85
|
'machines_reachable',
|
|
86
|
+
'trusted_sources',
|
|
86
87
|
];
|
|
87
88
|
|
|
88
89
|
export const CATEGORY_LABELS: Record<DriftCategory, string> = {
|
|
@@ -100,6 +101,7 @@ export const CATEGORY_LABELS: Record<DriftCategory, string> = {
|
|
|
100
101
|
secrets_decryptable: 'Secrets',
|
|
101
102
|
services_reachable: 'Service reachability',
|
|
102
103
|
machines_reachable: 'Machine reachability',
|
|
104
|
+
trusted_sources: 'Trusted networks',
|
|
103
105
|
};
|
|
104
106
|
|
|
105
107
|
/** Total lines we keep across the command log before evicting oldest. */
|
package/src/db/schema.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { sql } from 'drizzle-orm';
|
|
2
2
|
import {
|
|
3
|
+
index,
|
|
3
4
|
integer,
|
|
4
5
|
primaryKey,
|
|
5
6
|
sqliteTable,
|
|
@@ -503,6 +504,39 @@ export const portForwards = sqliteTable(
|
|
|
503
504
|
}),
|
|
504
505
|
);
|
|
505
506
|
|
|
507
|
+
/**
|
|
508
|
+
* Trusted-source registry — the desired-state store for "this subnet may reach
|
|
509
|
+
* every managed zone", the sibling of `port_forwards` and for the same reason: a
|
|
510
|
+
* rule that lives only in the applied ruleset belongs to no module, so the next
|
|
511
|
+
* converge correctly removes it. The admin VPN sat in exactly that position and
|
|
512
|
+
* needed a shell script re-adding its rules every second.
|
|
513
|
+
*
|
|
514
|
+
* Distinct from a port forward: that publishes one backend on specific ports;
|
|
515
|
+
* this is a whole origin subnet permitted to initiate into the segmented tiers.
|
|
516
|
+
*/
|
|
517
|
+
export const trustedSources = sqliteTable(
|
|
518
|
+
'trusted_sources',
|
|
519
|
+
{
|
|
520
|
+
id: integer('id').primaryKey({ autoIncrement: true }),
|
|
521
|
+
/** The firewall host that renders this trust (config.firewallIp). */
|
|
522
|
+
firewallIp: text('firewall_ip').notNull(),
|
|
523
|
+
/** Subnet CIDR permitted to reach every managed zone. */
|
|
524
|
+
subnet: text('subnet').notNull(),
|
|
525
|
+
description: text('description').notNull().default(''),
|
|
526
|
+
/** Module that registered it — reach into every tier must never be anonymous. */
|
|
527
|
+
registeredBy: text('registered_by').notNull(),
|
|
528
|
+
createdAt: integer('created_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
|
|
529
|
+
},
|
|
530
|
+
(table) => ({
|
|
531
|
+
// One row per (firewall, subnet) — the store delete-then-inserts on this
|
|
532
|
+
// tuple so re-registering the same subnet is an idempotent upsert.
|
|
533
|
+
trustedSourceUnique: uniqueIndex('trusted_sources_unique_idx').on(
|
|
534
|
+
table.firewallIp,
|
|
535
|
+
table.subnet,
|
|
536
|
+
),
|
|
537
|
+
}),
|
|
538
|
+
);
|
|
539
|
+
|
|
506
540
|
/**
|
|
507
541
|
* DNS registration ledger — one row per (provider, fqdn) the framework
|
|
508
542
|
* has successfully registered via dns_registrar.registerHost. Written
|
|
@@ -742,6 +776,312 @@ export const apiPrincipals = sqliteTable('api_principals', {
|
|
|
742
776
|
updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
|
|
743
777
|
});
|
|
744
778
|
|
|
779
|
+
// ---------------------------------------------------------------------------
|
|
780
|
+
// Alerting (openspec/changes/add-alerting)
|
|
781
|
+
// ---------------------------------------------------------------------------
|
|
782
|
+
|
|
783
|
+
/**
|
|
784
|
+
* Alert severity. Only `critical` pages.
|
|
785
|
+
*
|
|
786
|
+
* A check item's `warn` status always yields `warning`; a `fail` yields the
|
|
787
|
+
* monitor's configured severity. A monitor deliberately configured as
|
|
788
|
+
* `warning` therefore never pages even when its checks fail — the operator's
|
|
789
|
+
* "record this but don't wake me" knob. See design D6.
|
|
790
|
+
*/
|
|
791
|
+
export type AlertSeverity = 'warning' | 'critical';
|
|
792
|
+
|
|
793
|
+
/** What a monitor runs. `module_hook` = a module's health_check hook. */
|
|
794
|
+
export type MonitorKind = 'module_hook' | 'builtin_check';
|
|
795
|
+
|
|
796
|
+
/**
|
|
797
|
+
* Whether a monitor run executed at all. The load-bearing distinction: a run
|
|
798
|
+
* that could not execute produces an empty failing set for reasons that say
|
|
799
|
+
* nothing about the underlying checks, so it must never resolve anything.
|
|
800
|
+
* See design D5.
|
|
801
|
+
*/
|
|
802
|
+
export type MonitorRunOutcome = 'success' | 'error';
|
|
803
|
+
|
|
804
|
+
export type AlertState = 'pending' | 'firing' | 'acked' | 'suppressed' | 'resolved';
|
|
805
|
+
|
|
806
|
+
/**
|
|
807
|
+
* People celilo can reach. Deliberately independent of any transport or
|
|
808
|
+
* module — a person exists before any notification module is deployed.
|
|
809
|
+
*/
|
|
810
|
+
export const people = sqliteTable('people', {
|
|
811
|
+
id: text('id').primaryKey(), // UUID
|
|
812
|
+
/** User-facing kebab-case name (e.g. "peter"). Never a UUID in CLI output. */
|
|
813
|
+
name: text('name').notNull().unique(),
|
|
814
|
+
/** IANA timezone (e.g. "America/Los_Angeles") — quiet hours are local to it. */
|
|
815
|
+
timezone: text('timezone').notNull(),
|
|
816
|
+
/** Quiet-hours window as local "HH:MM"; both null = always reachable. */
|
|
817
|
+
quietHoursStart: text('quiet_hours_start'),
|
|
818
|
+
quietHoursEnd: text('quiet_hours_end'),
|
|
819
|
+
createdAt: integer('created_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
|
|
820
|
+
updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
|
|
821
|
+
});
|
|
822
|
+
|
|
823
|
+
/**
|
|
824
|
+
* A person's address on a transport, plus the policy for using it.
|
|
825
|
+
*
|
|
826
|
+
* Addresses live HERE, never in the transport module's config — otherwise
|
|
827
|
+
* adding a recipient would require redeploying the module. signal-cli holds
|
|
828
|
+
* one credential (its own registration); recipients are addresses, not
|
|
829
|
+
* credentials. See design D8.
|
|
830
|
+
*/
|
|
831
|
+
export const routes = sqliteTable(
|
|
832
|
+
'routes',
|
|
833
|
+
{
|
|
834
|
+
id: text('id').primaryKey(), // UUID
|
|
835
|
+
personId: text('person_id')
|
|
836
|
+
.notNull()
|
|
837
|
+
.references(() => people.id, { onDelete: 'cascade' }),
|
|
838
|
+
/** Module providing the `notification` capability. */
|
|
839
|
+
transportModuleId: text('transport_module_id')
|
|
840
|
+
.notNull()
|
|
841
|
+
.references(() => modules.id, { onDelete: 'cascade' }),
|
|
842
|
+
/** Transport-specific address (phone number, email, topic). */
|
|
843
|
+
address: text('address').notNull(),
|
|
844
|
+
/** Alerts below this severity are not delivered here. */
|
|
845
|
+
severityFloor: text('severity_floor').$type<AlertSeverity>().notNull().default('warning'),
|
|
846
|
+
/**
|
|
847
|
+
* Whether replies can arrive over this route — set from whether the
|
|
848
|
+
* transport implements `receive`. A route that cannot ack never stops
|
|
849
|
+
* escalation (spec: *Unidirectional transport cannot acknowledge*).
|
|
850
|
+
*/
|
|
851
|
+
canAck: integer('can_ack', { mode: 'boolean' }).notNull().default(false),
|
|
852
|
+
enabled: integer('enabled', { mode: 'boolean' }).notNull().default(true),
|
|
853
|
+
verifiedAt: integer('verified_at', { mode: 'timestamp' }),
|
|
854
|
+
createdAt: integer('created_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
|
|
855
|
+
updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
|
|
856
|
+
},
|
|
857
|
+
(table) => ({
|
|
858
|
+
uniqueAddress: unique().on(table.personId, table.transportModuleId, table.address),
|
|
859
|
+
}),
|
|
860
|
+
);
|
|
861
|
+
|
|
862
|
+
/**
|
|
863
|
+
* Named, reusable escalation policy. Steps reference ROUTES rather than
|
|
864
|
+
* people — "page Peter" is ambiguous about which transport to use.
|
|
865
|
+
*/
|
|
866
|
+
export const escalationPolicies = sqliteTable('escalation_policies', {
|
|
867
|
+
id: text('id').primaryKey(), // UUID
|
|
868
|
+
/** User-facing kebab-case name (e.g. "default", "critical"). */
|
|
869
|
+
name: text('name').notNull().unique(),
|
|
870
|
+
/**
|
|
871
|
+
* Reserved escape hatch (design D13): quiet hours otherwise defer ALL
|
|
872
|
+
* severities. Unused in MVP — the column exists so enabling it later is not
|
|
873
|
+
* a migration.
|
|
874
|
+
*/
|
|
875
|
+
bypassQuietHours: integer('bypass_quiet_hours', { mode: 'boolean' }).notNull().default(false),
|
|
876
|
+
createdAt: integer('created_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
|
|
877
|
+
updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
|
|
878
|
+
});
|
|
879
|
+
|
|
880
|
+
/** One ordered step of an escalation policy. Delay is from escalation start. */
|
|
881
|
+
export const escalationSteps = sqliteTable(
|
|
882
|
+
'escalation_steps',
|
|
883
|
+
{
|
|
884
|
+
policyId: text('policy_id')
|
|
885
|
+
.notNull()
|
|
886
|
+
.references(() => escalationPolicies.id, { onDelete: 'cascade' }),
|
|
887
|
+
/** 0-based order within the policy. */
|
|
888
|
+
stepIndex: integer('step_index').notNull(),
|
|
889
|
+
routeId: text('route_id')
|
|
890
|
+
.notNull()
|
|
891
|
+
.references(() => routes.id, { onDelete: 'cascade' }),
|
|
892
|
+
/** Minutes after escalation begins. Step 0 is normally 0. */
|
|
893
|
+
delayMinutes: integer('delay_minutes').notNull(),
|
|
894
|
+
},
|
|
895
|
+
(table) => ({
|
|
896
|
+
pk: primaryKey({ columns: [table.policyId, table.stepIndex] }),
|
|
897
|
+
}),
|
|
898
|
+
);
|
|
899
|
+
|
|
900
|
+
/**
|
|
901
|
+
* What gets checked, how often, and how failures are routed.
|
|
902
|
+
*
|
|
903
|
+
* The module manifest's `hooks.health_check.interval` is only a SUGGESTION;
|
|
904
|
+
* this row is the effective schedule and an operator edit survives module
|
|
905
|
+
* upgrades. See design D3.
|
|
906
|
+
*/
|
|
907
|
+
export const monitors = sqliteTable(
|
|
908
|
+
'monitors',
|
|
909
|
+
{
|
|
910
|
+
id: text('id').primaryKey(), // UUID
|
|
911
|
+
kind: text('kind').$type<MonitorKind>().notNull(),
|
|
912
|
+
/** Module id for `module_hook`; audit check name for `builtin_check`. */
|
|
913
|
+
target: text('target').notNull(),
|
|
914
|
+
intervalMinutes: integer('interval_minutes').notNull(),
|
|
915
|
+
/** Severity applied to `fail` items. Only `critical` pages. */
|
|
916
|
+
severity: text('severity').$type<AlertSeverity>().notNull().default('critical'),
|
|
917
|
+
/**
|
|
918
|
+
* False for monitors watching the alerting system itself — a cascading
|
|
919
|
+
* failure must not silence the component reporting it (design S4).
|
|
920
|
+
*/
|
|
921
|
+
suppressible: integer('suppressible', { mode: 'boolean' }).notNull().default(true),
|
|
922
|
+
enabled: integer('enabled', { mode: 'boolean' }).notNull().default(true),
|
|
923
|
+
escalationPolicyId: text('escalation_policy_id').references(() => escalationPolicies.id, {
|
|
924
|
+
onDelete: 'set null',
|
|
925
|
+
}),
|
|
926
|
+
lastRunAt: integer('last_run_at', { mode: 'timestamp' }),
|
|
927
|
+
createdAt: integer('created_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
|
|
928
|
+
updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
|
|
929
|
+
},
|
|
930
|
+
(table) => ({
|
|
931
|
+
uniqueTarget: unique().on(table.kind, table.target),
|
|
932
|
+
}),
|
|
933
|
+
);
|
|
934
|
+
|
|
935
|
+
/**
|
|
936
|
+
* Record of each monitor execution. Exists so reconciliation can branch on
|
|
937
|
+
* whether the last run actually ran (design D5) — without a persisted
|
|
938
|
+
* outcome there is nothing to distinguish "found nothing wrong" from
|
|
939
|
+
* "couldn't look".
|
|
940
|
+
*/
|
|
941
|
+
export const monitorRuns = sqliteTable('monitor_runs', {
|
|
942
|
+
id: integer('id').primaryKey({ autoIncrement: true }),
|
|
943
|
+
monitorId: text('monitor_id')
|
|
944
|
+
.notNull()
|
|
945
|
+
.references(() => monitors.id, { onDelete: 'cascade' }),
|
|
946
|
+
ranAt: integer('ran_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
|
|
947
|
+
outcome: text('outcome').$type<MonitorRunOutcome>().notNull(),
|
|
948
|
+
/** Populated when outcome is `error` (ssh failure, timeout, hook threw). */
|
|
949
|
+
errorMessage: text('error_message'),
|
|
950
|
+
});
|
|
951
|
+
|
|
952
|
+
/**
|
|
953
|
+
* A deliberate, time-boxed suppression source. Today only deploys create
|
|
954
|
+
* these — a deploy is the same suppression mechanism as a machine-down alert,
|
|
955
|
+
* with a window as the source instead of an ancestor alert (design D7).
|
|
956
|
+
*/
|
|
957
|
+
export const suppressionWindows = sqliteTable('suppression_windows', {
|
|
958
|
+
id: text('id').primaryKey(), // UUID
|
|
959
|
+
source: text('source').$type<'deploy'>().notNull(),
|
|
960
|
+
scopeModuleId: text('scope_module_id')
|
|
961
|
+
.notNull()
|
|
962
|
+
.references(() => modules.id, { onDelete: 'cascade' }),
|
|
963
|
+
startedAt: integer('started_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
|
|
964
|
+
/** Null while open. Set on deploy completion, including on failure. */
|
|
965
|
+
endsAt: integer('ends_at', { mode: 'timestamp' }),
|
|
966
|
+
});
|
|
967
|
+
|
|
968
|
+
/**
|
|
969
|
+
* The alert lifecycle. One row per alert key per firing episode: a resolve
|
|
970
|
+
* followed by a re-fire mints a new row, so history is preserved.
|
|
971
|
+
*
|
|
972
|
+
* `acked`, `suppressed`, and `silenced` are deliberately THREE separate
|
|
973
|
+
* concerns (design S5) — collapsing any two is how alerting systems become
|
|
974
|
+
* untrustworthy.
|
|
975
|
+
*/
|
|
976
|
+
export const alerts = sqliteTable(
|
|
977
|
+
'alerts',
|
|
978
|
+
{
|
|
979
|
+
id: text('id').primaryKey(), // UUID
|
|
980
|
+
/** Stable key, e.g. `module:caddy/check:cert-validity`. See design D4. */
|
|
981
|
+
key: text('key').notNull(),
|
|
982
|
+
/**
|
|
983
|
+
* Mirrors `key` while the alert is live; set to NULL on resolve. Backs the
|
|
984
|
+
* "one live alert per key" unique index below — SQLite treats NULLs in a
|
|
985
|
+
* unique index as distinct, so any number of resolved rows may share a key
|
|
986
|
+
* while at most one live row may hold it. Written only by the
|
|
987
|
+
* reconciliation code that sets `state`; the two must move together.
|
|
988
|
+
*/
|
|
989
|
+
activeKey: text('active_key'),
|
|
990
|
+
monitorId: text('monitor_id')
|
|
991
|
+
.notNull()
|
|
992
|
+
.references(() => monitors.id, { onDelete: 'cascade' }),
|
|
993
|
+
state: text('state').$type<AlertState>().notNull().default('pending'),
|
|
994
|
+
severity: text('severity').$type<AlertSeverity>().notNull(),
|
|
995
|
+
firstFiredAt: integer('first_fired_at', { mode: 'timestamp' })
|
|
996
|
+
.notNull()
|
|
997
|
+
.default(sql`(unixepoch())`),
|
|
998
|
+
lastSeenAt: integer('last_seen_at', { mode: 'timestamp' })
|
|
999
|
+
.notNull()
|
|
1000
|
+
.default(sql`(unixepoch())`),
|
|
1001
|
+
/**
|
|
1002
|
+
* Notification is withheld until this instant, so an ancestor firing
|
|
1003
|
+
* moments later can establish suppression first, and so a failure that
|
|
1004
|
+
* clears immediately never pages (design S1).
|
|
1005
|
+
*/
|
|
1006
|
+
graceUntil: integer('grace_until', { mode: 'timestamp' }).notNull(),
|
|
1007
|
+
/** Set when an ancestor ALERT is suppressing this one. */
|
|
1008
|
+
suppressedByAlertId: text('suppressed_by_alert_id'),
|
|
1009
|
+
/** Set when a suppression WINDOW (e.g. a deploy) is suppressing this one. */
|
|
1010
|
+
suppressedByWindowId: text('suppressed_by_window_id').references(() => suppressionWindows.id, {
|
|
1011
|
+
onDelete: 'set null',
|
|
1012
|
+
}),
|
|
1013
|
+
/** Escalation clock origin after suppression lifts — NOT firstFiredAt (S3). */
|
|
1014
|
+
unsuppressedAt: integer('unsuppressed_at', { mode: 'timestamp' }),
|
|
1015
|
+
/**
|
|
1016
|
+
* True between un-suppression and the next SUCCESSFUL run. While set, the
|
|
1017
|
+
* alert does not notify: if it recovered along with its ancestor it
|
|
1018
|
+
* resolves quietly instead of paging (design S2).
|
|
1019
|
+
*/
|
|
1020
|
+
awaitingConfirmation: integer('awaiting_confirmation', { mode: 'boolean' })
|
|
1021
|
+
.notNull()
|
|
1022
|
+
.default(false),
|
|
1023
|
+
ackedBy: text('acked_by').references(() => people.id, { onDelete: 'set null' }),
|
|
1024
|
+
ackedAt: integer('acked_at', { mode: 'timestamp' }),
|
|
1025
|
+
/** Deliberate operator silence — distinct from suppression (S5). */
|
|
1026
|
+
silencedUntil: integer('silenced_until', { mode: 'timestamp' }),
|
|
1027
|
+
escalationStep: integer('escalation_step').notNull().default(0),
|
|
1028
|
+
nextEscalationAt: integer('next_escalation_at', { mode: 'timestamp' }),
|
|
1029
|
+
/**
|
|
1030
|
+
* Quiet hours defer DELIVERY, never the escalation clock (D13). When a step
|
|
1031
|
+
* falls due inside someone's window the step is still taken — the step
|
|
1032
|
+
* index advances and the next one is scheduled — and only the message
|
|
1033
|
+
* waits, held here until the window ends.
|
|
1034
|
+
*/
|
|
1035
|
+
deferredUntil: integer('deferred_until', { mode: 'timestamp' }),
|
|
1036
|
+
/** Route the deferred message is owed to. */
|
|
1037
|
+
deferredRouteId: text('deferred_route_id').references(() => routes.id, {
|
|
1038
|
+
onDelete: 'set null',
|
|
1039
|
+
}),
|
|
1040
|
+
escalationPolicyId: text('escalation_policy_id').references(() => escalationPolicies.id, {
|
|
1041
|
+
onDelete: 'set null',
|
|
1042
|
+
}),
|
|
1043
|
+
message: text('message').notNull(),
|
|
1044
|
+
details: text('details'),
|
|
1045
|
+
resolvedAt: integer('resolved_at', { mode: 'timestamp' }),
|
|
1046
|
+
},
|
|
1047
|
+
(table) => ({
|
|
1048
|
+
/**
|
|
1049
|
+
* One live alert per key. Resolved rows are exempt so history accumulates:
|
|
1050
|
+
* `activeKey` mirrors `key` while live and is NULL once resolved, and
|
|
1051
|
+
* SQLite treats NULLs in a unique index as distinct.
|
|
1052
|
+
*/
|
|
1053
|
+
liveKey: uniqueIndex('alerts_live_key_idx').on(table.activeKey),
|
|
1054
|
+
keyLookup: index('alerts_key_idx').on(table.key),
|
|
1055
|
+
}),
|
|
1056
|
+
);
|
|
1057
|
+
|
|
1058
|
+
/**
|
|
1059
|
+
* One outbound message, and the token that authorises a reply to it.
|
|
1060
|
+
*
|
|
1061
|
+
* Per DELIVERY, not per alert: the token identifies WHO replied, which is
|
|
1062
|
+
* what "an ack from the secondary is broadcast to everyone paged" needs, and
|
|
1063
|
+
* doubles as the audit trail (design D10).
|
|
1064
|
+
*/
|
|
1065
|
+
export const notificationDeliveries = sqliteTable('notification_deliveries', {
|
|
1066
|
+
id: text('id').primaryKey(), // UUID
|
|
1067
|
+
/** Short operator-typable token, unique across live deliveries. */
|
|
1068
|
+
token: text('token').notNull().unique(),
|
|
1069
|
+
kind: text('kind').$type<'alert' | 'interview'>().notNull(),
|
|
1070
|
+
/**
|
|
1071
|
+
* `alerts.id` when kind is `alert`; a BUS event id when kind is
|
|
1072
|
+
* `interview`. Deliberately not a foreign key — the event bus is a separate
|
|
1073
|
+
* SQLite database, so no FK can span it.
|
|
1074
|
+
*/
|
|
1075
|
+
targetId: text('target_id').notNull(),
|
|
1076
|
+
routeId: text('route_id')
|
|
1077
|
+
.notNull()
|
|
1078
|
+
.references(() => routes.id, { onDelete: 'cascade' }),
|
|
1079
|
+
sentAt: integer('sent_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
|
|
1080
|
+
expiresAt: integer('expires_at', { mode: 'timestamp' }).notNull(),
|
|
1081
|
+
/** Set when a reply consumed this token. */
|
|
1082
|
+
consumedAt: integer('consumed_at', { mode: 'timestamp' }),
|
|
1083
|
+
});
|
|
1084
|
+
|
|
745
1085
|
/**
|
|
746
1086
|
* Type exports for use in application code
|
|
747
1087
|
*/
|
|
@@ -785,3 +1125,21 @@ export type AspectApproval = typeof aspectApprovals.$inferSelect;
|
|
|
785
1125
|
export type NewAspectApproval = typeof aspectApprovals.$inferInsert;
|
|
786
1126
|
export type ApiPrincipal = typeof apiPrincipals.$inferSelect;
|
|
787
1127
|
export type NewApiPrincipal = typeof apiPrincipals.$inferInsert;
|
|
1128
|
+
export type Person = typeof people.$inferSelect;
|
|
1129
|
+
export type NewPerson = typeof people.$inferInsert;
|
|
1130
|
+
export type Route = typeof routes.$inferSelect;
|
|
1131
|
+
export type NewRoute = typeof routes.$inferInsert;
|
|
1132
|
+
export type EscalationPolicy = typeof escalationPolicies.$inferSelect;
|
|
1133
|
+
export type NewEscalationPolicy = typeof escalationPolicies.$inferInsert;
|
|
1134
|
+
export type EscalationStep = typeof escalationSteps.$inferSelect;
|
|
1135
|
+
export type NewEscalationStep = typeof escalationSteps.$inferInsert;
|
|
1136
|
+
export type Monitor = typeof monitors.$inferSelect;
|
|
1137
|
+
export type NewMonitor = typeof monitors.$inferInsert;
|
|
1138
|
+
export type MonitorRun = typeof monitorRuns.$inferSelect;
|
|
1139
|
+
export type NewMonitorRun = typeof monitorRuns.$inferInsert;
|
|
1140
|
+
export type SuppressionWindow = typeof suppressionWindows.$inferSelect;
|
|
1141
|
+
export type NewSuppressionWindow = typeof suppressionWindows.$inferInsert;
|
|
1142
|
+
export type Alert = typeof alerts.$inferSelect;
|
|
1143
|
+
export type NewAlert = typeof alerts.$inferInsert;
|
|
1144
|
+
export type NotificationDelivery = typeof notificationDeliveries.$inferSelect;
|
|
1145
|
+
export type NewNotificationDelivery = typeof notificationDeliveries.$inferInsert;
|