@celilo/cli 2.2.1 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +2 -1
- package/CELILO_SUBSYSTEMS.md +1 -1
- package/README.md +1 -1
- package/drizzle/0032_module_jail_policies.sql +28 -0
- package/drizzle/0033_build_bus_hook_runs.sql +37 -0
- package/drizzle/meta/_journal.json +15 -1
- package/package.json +3 -3
- package/src/__integration__/container-services-cli.integration.test.ts +1 -1
- package/src/api/serve.ts +13 -1
- package/src/api/sessions.test.ts +3 -3
- package/src/api-clients/proxmox.ts +10 -7
- package/src/cli/backup-rename.test.ts +3 -3
- package/src/cli/cli.test.ts +1 -1
- package/src/cli/commands/events.test.ts +2 -2
- package/src/cli/commands/firewall-interface-list.test.ts +2 -2
- package/src/cli/commands/machine-list.test.ts +57 -0
- package/src/cli/commands/machine-list.ts +35 -4
- package/src/cli/commands/module-health.test.ts +35 -0
- package/src/cli/commands/module-health.ts +12 -3
- package/src/cli/commands/module-import-registry.test.ts +1 -1
- package/src/cli/commands/module-jail.test.ts +242 -0
- package/src/cli/commands/module-jail.ts +227 -0
- package/src/cli/commands/module-list-jail.test.ts +136 -0
- package/src/cli/commands/module-list.ts +30 -3
- package/src/cli/commands/module-publish.test.ts +28 -6
- package/src/cli/commands/module-publish.ts +14 -12
- package/src/cli/commands/module-update.test.ts +22 -7
- package/src/cli/commands/module-update.ts +73 -18
- package/src/cli/commands/module-upgrade-gate.test.ts +154 -0
- package/src/cli/commands/module-upgrade.test.ts +115 -1
- package/src/cli/commands/module-upgrade.ts +66 -4
- package/src/cli/commands/module-verify.test.ts +1 -1
- package/src/cli/commands/publish/execute.ts +4 -1
- package/src/cli/commands/publish/helpers.ts +4 -3
- package/src/cli/commands/publish/index.ts +23 -1
- package/src/cli/commands/publish/module-registry.test.ts +24 -1
- package/src/cli/commands/publish/module-registry.ts +23 -4
- package/src/cli/commands/publish/plan.test.ts +64 -0
- package/src/cli/commands/publish/plan.ts +52 -19
- package/src/cli/commands/publish/types.ts +20 -0
- package/src/cli/commands/registry-owner.test.ts +1 -1
- package/src/cli/commands/registry-token.test.ts +1 -1
- package/src/cli/commands/subscribers-install-daemon.test.ts +44 -0
- package/src/cli/commands/subscribers-install-daemon.ts +107 -0
- package/src/cli/commands/subscribers-serve.test.ts +22 -0
- package/src/cli/commands/subscribers-serve.ts +22 -4
- package/src/cli/commands/system-audit.ts +34 -11
- package/src/cli/commands/system-doctor-remediation-gate.test.ts +295 -0
- package/src/cli/commands/system-doctor.test.ts +76 -10
- package/src/cli/commands/system-doctor.ts +34 -3
- package/src/cli/commands/system-init-deprecation.test.ts +1 -1
- package/src/cli/commands/system-update.ts +16 -6
- package/src/cli/completion.ts +18 -3
- package/src/cli/flag-surface-gate.test.ts +279 -0
- package/src/cli/fuel-gauge.ts +12 -4
- package/src/cli/index.ts +80 -2
- package/src/cli/json-output.test.ts +81 -0
- package/src/cli/parser.test.ts +37 -1
- package/src/cli/restore-command.test.ts +3 -3
- package/src/cli/restore-migration-failure.test.ts +2 -2
- package/src/cli/tui/audit-state.ts +2 -0
- package/src/cli/types.ts +9 -0
- package/src/config/paths.test.ts +19 -19
- package/src/db/client.test.ts +46 -1
- package/src/db/client.ts +26 -0
- package/src/db/schema.ts +63 -0
- package/src/hooks/broker.test.ts +9 -3
- package/src/hooks/capability-loader-firewall.test.ts +9 -1
- package/src/hooks/capability-loader.ts +8 -0
- package/src/hooks/executor.test.ts +91 -14
- package/src/hooks/executor.ts +31 -1
- package/src/hooks/hook-jail-toolchain-reach.test.ts +8 -3
- package/src/hooks/hook-jail-unreachability.test.ts +27 -20
- package/src/hooks/hook-trespass.test.ts +22 -6
- package/src/hooks/jail.test.ts +105 -17
- package/src/hooks/jail.ts +47 -10
- package/src/hooks/run-named-hook.ts +19 -16
- package/src/hooks/test-fixtures/artifact-writing-hook.ts +0 -1
- package/src/hooks/test-fixtures/capability-calling-hook.ts +20 -13
- package/src/hooks/test-fixtures/jail-probe-hook.ts +9 -1
- package/src/hooks/test-fixtures/jail-toolchain-hook.ts +7 -2
- package/src/hooks/test-fixtures/runaway-hook.ts +0 -1
- package/src/hooks/test-fixtures/sigterm-ignoring-hook.ts +0 -1
- package/src/hooks/test-fixtures/silent-hook.ts +0 -1
- package/src/hooks/test-fixtures/store-writing-hook.ts +20 -11
- package/src/hooks/test-fixtures/success-hook.ts +4 -4
- package/src/manifest/contracts/v1.ts +32 -14
- package/src/manifest/json-schema-roundtrip.test.ts +1 -1
- package/src/manifest/schema.ts +66 -16
- package/src/manifest/validate.test.ts +47 -0
- package/src/policy/capability-shape-baseline.ts +63 -21
- package/src/policy/capability-shape-drift.test.ts +53 -1
- package/src/policy/capability-shape.test.ts +105 -0
- package/src/policy/capability-shape.ts +283 -2
- package/src/policy/module-script-scan.ts +52 -68
- package/src/policy/no-hand-built-ssh.test.ts +5 -1
- package/src/policy/no-swallowed-refusal.test.ts +14 -14
- package/src/registry/client.test.ts +67 -2
- package/src/registry/client.ts +7 -7
- package/src/secrets/storage.test.ts +70 -4
- package/src/secrets/storage.ts +71 -1
- package/src/services/alerting/keys.test.ts +4 -0
- package/src/services/alerting/keys.ts +12 -2
- package/src/services/audit/health.test.ts +155 -2
- package/src/services/audit/health.ts +76 -2
- package/src/services/audit/index.test.ts +23 -1
- package/src/services/audit/index.ts +8 -2
- package/src/services/audit/interface-classification.test.ts +16 -5
- package/src/services/audit/interface-classification.ts +25 -2
- package/src/services/audit/jail-exemptions.test.ts +42 -0
- package/src/services/audit/jail-exemptions.ts +44 -0
- package/src/services/audit/module-integrity.test.ts +23 -1
- package/src/services/audit/module-integrity.ts +7 -2
- package/src/services/audit/module-versions.ts +5 -1
- package/src/services/audit/public-dns.test.ts +20 -0
- package/src/services/audit/public-dns.ts +7 -2
- package/src/services/audit/recurrence-gate.test.ts +225 -0
- package/src/services/audit/trusted-sources.test.ts +17 -0
- package/src/services/audit/trusted-sources.ts +21 -0
- package/src/services/audit/types.ts +2 -1
- package/src/services/backup-create.ts +9 -0
- package/src/services/backup-envelope-roundtrip.test.ts +1 -1
- package/src/services/backup-in-flight-refusal.test.ts +1 -1
- package/src/services/build-bus/hook-dispatch-executor.test.ts +269 -0
- package/src/services/build-bus/hook-dispatch-mgmt.test.ts +105 -105
- package/src/services/build-bus/hook-dispatch-path.test.ts +95 -0
- package/src/services/build-bus/hook-dispatch.test.ts +86 -116
- package/src/services/build-bus/hook-dispatch.ts +99 -121
- package/src/services/build-bus/hook-dispatcher.ts +143 -17
- package/src/services/build-bus/receiver-daemon.test.ts +189 -0
- package/src/services/build-bus/receiver-daemon.ts +355 -0
- package/src/services/build-bus/self-update.ts +156 -0
- package/src/services/bus-ensure-flow.test.ts +1 -1
- package/src/services/bus-interview-park.test.ts +2 -2
- package/src/services/bus-interview.test.ts +2 -2
- package/src/services/bus-secret-flow.test.ts +1 -1
- package/src/services/capability-compat.test.ts +90 -0
- package/src/services/capability-compat.ts +128 -0
- package/src/services/celilo-events.test.ts +1 -1
- package/src/services/celilo-mgmt-hooks.test.ts +23 -5
- package/src/services/container-service.test.ts +1 -1
- package/src/services/cross-module-read.test.ts +2 -2
- package/src/services/deploy-preflight.ts +7 -0
- package/src/services/deploy-terraform.ts +38 -1
- package/src/services/deployed-systems.ts +1 -1
- package/src/services/events-daemon.test.ts +57 -0
- package/src/services/events-daemon.ts +76 -0
- package/src/services/firewall-reach.ts +21 -8
- package/src/services/fleet-checks.test.ts +159 -4
- package/src/services/fleet-checks.ts +206 -3
- package/src/services/health-runner.test.ts +87 -2
- package/src/services/health-runner.ts +83 -16
- package/src/services/infrastructure-selector.test.ts +1 -1
- package/src/services/jail-exemptions.test.ts +125 -0
- package/src/services/jail-exemptions.ts +81 -0
- package/src/services/machine-pool.test.ts +1 -1
- package/src/services/module-deploy-prune.test.ts +89 -0
- package/src/services/module-deploy.ts +84 -129
- package/src/services/module-subscriptions.test.ts +2 -2
- package/src/services/module-types-drift.test.ts +1 -1
- package/src/services/module-validator/capability-versions.test.ts +13 -2
- package/src/services/network-discovery.test.ts +64 -1
- package/src/services/network-discovery.ts +30 -5
- package/src/services/responder-probe.test.ts +1 -1
- package/src/services/restore-from-file.test.ts +4 -4
- package/src/services/restore-preflight.test.ts +1 -1
- package/src/services/ssh-key-manager.test.ts +2 -2
- package/src/services/system-state-stage.test.ts +2 -2
- package/src/services/terraform-safety.test.ts +83 -0
- package/src/services/terraform-safety.ts +53 -0
- package/src/services/update/orchestrator.test.ts +3 -1
- package/src/test-utils/bus-responder.ts +1 -1
- package/tsconfig.json +2 -13
|
@@ -7,13 +7,20 @@ import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
|
|
7
7
|
import { mkdtempSync, rmSync } from 'node:fs';
|
|
8
8
|
import { tmpdir } from 'node:os';
|
|
9
9
|
import { join } from 'node:path';
|
|
10
|
-
import { eq } from 'drizzle-orm';
|
|
10
|
+
import { and, eq } from 'drizzle-orm';
|
|
11
11
|
import { drizzle } from 'drizzle-orm/bun-sqlite';
|
|
12
12
|
import { migrate } from 'drizzle-orm/bun-sqlite/migrator';
|
|
13
13
|
import type { DbClient } from '../db/client';
|
|
14
14
|
import * as schema from '../db/schema';
|
|
15
|
-
import { systemSecrets } from '../db/schema';
|
|
16
|
-
import {
|
|
15
|
+
import { secrets, systemSecrets } from '../db/schema';
|
|
16
|
+
import { decryptSecret } from './encryption';
|
|
17
|
+
import { getOrCreateMasterKey } from './master-key';
|
|
18
|
+
import {
|
|
19
|
+
getSystemSecret,
|
|
20
|
+
recordProviderSecret,
|
|
21
|
+
storeSystemSecret,
|
|
22
|
+
systemSecretExists,
|
|
23
|
+
} from './storage';
|
|
17
24
|
|
|
18
25
|
describe('secret storage', () => {
|
|
19
26
|
let testDir: string;
|
|
@@ -36,7 +43,7 @@ describe('secret storage', () => {
|
|
|
36
43
|
afterEach(() => {
|
|
37
44
|
// Cleanup
|
|
38
45
|
rmSync(testDir, { recursive: true, force: true });
|
|
39
|
-
process.env.CELILO_MASTER_KEY_PATH
|
|
46
|
+
delete process.env.CELILO_MASTER_KEY_PATH;
|
|
40
47
|
});
|
|
41
48
|
|
|
42
49
|
test('stores and retrieves system secret', async () => {
|
|
@@ -112,4 +119,63 @@ describe('secret storage', () => {
|
|
|
112
119
|
const decrypted = await getSystemSecret('test.long', testDb);
|
|
113
120
|
expect(decrypted).toBe(longSecret);
|
|
114
121
|
});
|
|
122
|
+
|
|
123
|
+
describe('recordProviderSecret (hook-owned-state D3)', () => {
|
|
124
|
+
const providerRow = (moduleId: string, name: string) =>
|
|
125
|
+
testDb
|
|
126
|
+
.select()
|
|
127
|
+
.from(secrets)
|
|
128
|
+
.where(and(eq(secrets.moduleId, moduleId), eq(secrets.name, name)))
|
|
129
|
+
.get();
|
|
130
|
+
|
|
131
|
+
const decryptRow = async (moduleId: string, name: string) => {
|
|
132
|
+
const row = providerRow(moduleId, name);
|
|
133
|
+
expect(row).toBeDefined();
|
|
134
|
+
if (!row) throw new Error('provider secret row missing');
|
|
135
|
+
const mk = await getOrCreateMasterKey();
|
|
136
|
+
return decryptSecret(
|
|
137
|
+
{ encryptedValue: row.encryptedValue, iv: row.iv, authTag: row.authTag },
|
|
138
|
+
mk,
|
|
139
|
+
);
|
|
140
|
+
};
|
|
141
|
+
|
|
142
|
+
test('records an encrypted secret onto the provider module', async () => {
|
|
143
|
+
await recordProviderSecret(testDb, 'some-provider', 'api_key', 'minted-value');
|
|
144
|
+
|
|
145
|
+
expect(await decryptRow('some-provider', 'api_key')).toBe('minted-value');
|
|
146
|
+
// Encrypted on disk, never the plaintext.
|
|
147
|
+
const row = providerRow('some-provider', 'api_key');
|
|
148
|
+
expect(row && row.encryptedValue !== 'minted-value').toBe(true);
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
test('upserts on (moduleId, name) instead of duplicating', async () => {
|
|
152
|
+
await recordProviderSecret(testDb, 'some-provider', 'api_key', 'first');
|
|
153
|
+
await recordProviderSecret(testDb, 'some-provider', 'api_key', 'second');
|
|
154
|
+
|
|
155
|
+
const rows = testDb.select().from(secrets).all();
|
|
156
|
+
expect(rows).toHaveLength(1);
|
|
157
|
+
expect(await decryptRow('some-provider', 'api_key')).toBe('second');
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
test('keeps two providers with the same secret name distinct', async () => {
|
|
161
|
+
await recordProviderSecret(testDb, 'provider-a', 'api_key', 'value-a');
|
|
162
|
+
await recordProviderSecret(testDb, 'provider-b', 'api_key', 'value-b');
|
|
163
|
+
|
|
164
|
+
const rows = testDb.select().from(secrets).all();
|
|
165
|
+
expect(rows).toHaveLength(2);
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
test('refuses an empty module id, name, or value', async () => {
|
|
169
|
+
await expect(recordProviderSecret(testDb, '', 'api_key', 'v')).rejects.toThrow(
|
|
170
|
+
'providerModuleId is required',
|
|
171
|
+
);
|
|
172
|
+
await expect(recordProviderSecret(testDb, 'p', ' ', 'v')).rejects.toThrow(
|
|
173
|
+
'name is required',
|
|
174
|
+
);
|
|
175
|
+
await expect(recordProviderSecret(testDb, 'p', 'api_key', '')).rejects.toThrow(
|
|
176
|
+
'value is required',
|
|
177
|
+
);
|
|
178
|
+
expect(testDb.select().from(secrets).all()).toHaveLength(0);
|
|
179
|
+
});
|
|
180
|
+
});
|
|
115
181
|
});
|
package/src/secrets/storage.ts
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* Reusable functions for storing encrypted secrets
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
-
import { eq } from 'drizzle-orm';
|
|
6
|
+
import { and, eq } from 'drizzle-orm';
|
|
7
7
|
import type { DbClient } from '../db/client';
|
|
8
8
|
import { getDb } from '../db/client';
|
|
9
9
|
import { systemSecrets } from '../db/schema';
|
|
@@ -104,3 +104,73 @@ export function systemSecretExists(key: string, db: DbClient = getDb()): boolean
|
|
|
104
104
|
|
|
105
105
|
return secret !== undefined;
|
|
106
106
|
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Record a secret on a PROVIDER module on behalf of the framework.
|
|
110
|
+
*
|
|
111
|
+
* This is the named, single-purpose path that replaced the generic
|
|
112
|
+
* return-value capture sweep (openspec/changes/hook-owned-state, D3/D5):
|
|
113
|
+
* `container_created` used to persist whatever the provider hook returned
|
|
114
|
+
* onto the provider module, keyed by the provider's own id — a write a
|
|
115
|
+
* per-module hook accessor cannot express and must never be able to. The
|
|
116
|
+
* framework still needs to mint provider secrets on a deploy's authority,
|
|
117
|
+
* so that write keeps a broker-side home: an explicit call in framework
|
|
118
|
+
* code, not reachable from any hook, not driven by a hook return value.
|
|
119
|
+
*
|
|
120
|
+
* Policy function (Rule 10.1) - validates name before storing
|
|
121
|
+
* Execution function - performs encryption and database write
|
|
122
|
+
*
|
|
123
|
+
* @param db - Database client
|
|
124
|
+
* @param providerModuleId - The module that OWNS the secret (the provider)
|
|
125
|
+
* @param name - Secret name
|
|
126
|
+
* @param value - Plaintext secret value
|
|
127
|
+
* @throws Error if name or value is empty
|
|
128
|
+
*/
|
|
129
|
+
export async function recordProviderSecret(
|
|
130
|
+
db: DbClient,
|
|
131
|
+
providerModuleId: string,
|
|
132
|
+
name: string,
|
|
133
|
+
value: string,
|
|
134
|
+
): Promise<void> {
|
|
135
|
+
if (!providerModuleId.trim()) {
|
|
136
|
+
throw new Error('recordProviderSecret: providerModuleId is required');
|
|
137
|
+
}
|
|
138
|
+
if (!name.trim()) {
|
|
139
|
+
throw new Error('recordProviderSecret: secret name is required');
|
|
140
|
+
}
|
|
141
|
+
if (!value.trim()) {
|
|
142
|
+
throw new Error('recordProviderSecret: secret value is required');
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const { secrets } = await import('../db/schema');
|
|
146
|
+
const masterKey = await getOrCreateMasterKey();
|
|
147
|
+
const encrypted = encryptSecret(value, masterKey);
|
|
148
|
+
|
|
149
|
+
const existing = db
|
|
150
|
+
.select()
|
|
151
|
+
.from(secrets)
|
|
152
|
+
.where(and(eq(secrets.moduleId, providerModuleId), eq(secrets.name, name)))
|
|
153
|
+
.get();
|
|
154
|
+
|
|
155
|
+
if (existing) {
|
|
156
|
+
db.update(secrets)
|
|
157
|
+
.set({
|
|
158
|
+
encryptedValue: encrypted.encryptedValue,
|
|
159
|
+
iv: encrypted.iv,
|
|
160
|
+
authTag: encrypted.authTag,
|
|
161
|
+
updatedAt: new Date(),
|
|
162
|
+
})
|
|
163
|
+
.where(eq(secrets.id, existing.id))
|
|
164
|
+
.run();
|
|
165
|
+
} else {
|
|
166
|
+
db.insert(secrets)
|
|
167
|
+
.values({
|
|
168
|
+
moduleId: providerModuleId,
|
|
169
|
+
name,
|
|
170
|
+
encryptedValue: encrypted.encryptedValue,
|
|
171
|
+
iv: encrypted.iv,
|
|
172
|
+
authTag: encrypted.authTag,
|
|
173
|
+
})
|
|
174
|
+
.run();
|
|
175
|
+
}
|
|
176
|
+
}
|
|
@@ -116,6 +116,10 @@ describe('severityForItemStatus', () => {
|
|
|
116
116
|
expect(severityForItemStatus('fail', 'critical')).toBe('critical');
|
|
117
117
|
expect(severityForItemStatus('fail', 'warning')).toBe('warning');
|
|
118
118
|
});
|
|
119
|
+
|
|
120
|
+
test('skip produces no alert — a check that could not run is not a failure', () => {
|
|
121
|
+
expect(severityForItemStatus('skip', 'critical')).toBeNull();
|
|
122
|
+
});
|
|
119
123
|
});
|
|
120
124
|
|
|
121
125
|
describe('failingKeysFromHealthItems', () => {
|
|
@@ -134,14 +134,24 @@ export function parentModuleKey(key: string): string | null {
|
|
|
134
134
|
* and never pages. That gives module authors somewhere to put "cert expires in
|
|
135
135
|
* 20 days" without anyone having to decide whether it is a 3am problem.
|
|
136
136
|
*
|
|
137
|
+
* `skip` produces no alert either: a check that could not perform its probe
|
|
138
|
+
* is not a failure. Note the honest cost — a previously-failing key whose
|
|
139
|
+
* item now reports skip leaves this run's failing set, so the reconciler
|
|
140
|
+
* reads "recovered" off a run that measured nothing. That is the same hazard
|
|
141
|
+
* class design D5 handles at run level (a run that could not execute is the
|
|
142
|
+
* caller's branch); there is no per-item sticky-key mechanism here, so a
|
|
143
|
+
* module whose credential disappeared needs the audit's `unmeasured` finding
|
|
144
|
+
* to be seen (celilo#1326).
|
|
145
|
+
*
|
|
137
146
|
* See design D6.
|
|
138
147
|
*/
|
|
139
148
|
export function severityForItemStatus(
|
|
140
|
-
itemStatus: 'pass' | 'warn' | 'fail',
|
|
149
|
+
itemStatus: 'pass' | 'warn' | 'fail' | 'skip',
|
|
141
150
|
monitorSeverity: AlertSeverity,
|
|
142
151
|
): AlertSeverity | null {
|
|
143
152
|
if (itemStatus === 'pass') return null;
|
|
144
153
|
if (itemStatus === 'warn') return 'warning';
|
|
154
|
+
if (itemStatus === 'skip') return null;
|
|
145
155
|
return monitorSeverity;
|
|
146
156
|
}
|
|
147
157
|
|
|
@@ -155,7 +165,7 @@ export interface FailingKey {
|
|
|
155
165
|
|
|
156
166
|
export interface HealthCheckItemLike {
|
|
157
167
|
name: string;
|
|
158
|
-
status: 'pass' | 'warn' | 'fail';
|
|
168
|
+
status: 'pass' | 'warn' | 'fail' | 'skip';
|
|
159
169
|
message: string;
|
|
160
170
|
details?: string;
|
|
161
171
|
}
|
|
@@ -1,14 +1,167 @@
|
|
|
1
1
|
import { describe, expect, test } from 'bun:test';
|
|
2
2
|
import type { HealthCheckResult } from '../health-runner';
|
|
3
3
|
import { auditHealth } from './health';
|
|
4
|
+
import { computeVerdict } from './types';
|
|
4
5
|
|
|
5
6
|
describe('auditHealth', () => {
|
|
6
|
-
test('
|
|
7
|
+
test('a healthy module produces no finding; a module with no hook is unmeasured, never green', async () => {
|
|
7
8
|
const results: HealthCheckResult[] = [
|
|
8
9
|
{ moduleId: 'caddy', status: 'healthy', checks: [] },
|
|
9
10
|
{ moduleId: 'iptables', status: 'no-checks', checks: [] },
|
|
10
11
|
];
|
|
11
|
-
|
|
12
|
+
const result = await auditHealth({ results });
|
|
13
|
+
expect(result).toHaveLength(1);
|
|
14
|
+
expect(result[0]).toMatchObject({
|
|
15
|
+
category: 'health',
|
|
16
|
+
severity: 'unmeasured',
|
|
17
|
+
code: 'health_unmeasured',
|
|
18
|
+
subject: 'iptables',
|
|
19
|
+
actionable: false,
|
|
20
|
+
});
|
|
21
|
+
expect(result[0].message).toContain('no health_check hook');
|
|
22
|
+
expect(computeVerdict(result)).toBe('UNKNOWN');
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
test('a module whose checks all report skip is unmeasured and names the skipped checks', async () => {
|
|
26
|
+
// The smoketest_bot_token incident, as an audit finding: the hook ran,
|
|
27
|
+
// every meaningful check refused to guess, and the fleet must not read
|
|
28
|
+
// READY off that.
|
|
29
|
+
const results: HealthCheckResult[] = [
|
|
30
|
+
{
|
|
31
|
+
moduleId: 'smoketest',
|
|
32
|
+
status: 'no-checks',
|
|
33
|
+
checks: [
|
|
34
|
+
{
|
|
35
|
+
name: 'bot_token_valid',
|
|
36
|
+
status: 'skip',
|
|
37
|
+
message: 'smoketest_bot_token not yet provisioned',
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
name: 'webhook_reachable',
|
|
41
|
+
status: 'skip',
|
|
42
|
+
message: 'smoketest_bot_token not yet provisioned',
|
|
43
|
+
},
|
|
44
|
+
],
|
|
45
|
+
},
|
|
46
|
+
];
|
|
47
|
+
const result = await auditHealth({ results });
|
|
48
|
+
expect(result).toHaveLength(1);
|
|
49
|
+
expect(result[0]).toMatchObject({
|
|
50
|
+
severity: 'unmeasured',
|
|
51
|
+
code: 'health_unmeasured',
|
|
52
|
+
subject: 'smoketest',
|
|
53
|
+
actionable: true,
|
|
54
|
+
remediation: 'celilo module health smoketest --debug',
|
|
55
|
+
});
|
|
56
|
+
expect(result[0].message).toContain('all 2 health checks skipped');
|
|
57
|
+
expect(result[0].details).toContain('bot_token_valid');
|
|
58
|
+
expect(result[0].details).toContain('not yet provisioned');
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
test('skipped checks on an otherwise healthy module render unmeasured', async () => {
|
|
62
|
+
const results: HealthCheckResult[] = [
|
|
63
|
+
{
|
|
64
|
+
moduleId: 'caddy',
|
|
65
|
+
status: 'healthy',
|
|
66
|
+
checks: [
|
|
67
|
+
{ name: 'config_valid', status: 'pass', message: 'config parses' },
|
|
68
|
+
{
|
|
69
|
+
name: 'origin_reachable',
|
|
70
|
+
status: 'skip',
|
|
71
|
+
message: 'peer unreachable from control plane',
|
|
72
|
+
},
|
|
73
|
+
],
|
|
74
|
+
},
|
|
75
|
+
];
|
|
76
|
+
const result = await auditHealth({ results });
|
|
77
|
+
expect(result).toHaveLength(1);
|
|
78
|
+
expect(result[0]).toMatchObject({
|
|
79
|
+
severity: 'unmeasured',
|
|
80
|
+
code: 'health_checks_skipped',
|
|
81
|
+
subject: 'caddy',
|
|
82
|
+
});
|
|
83
|
+
expect(result[0].message).toContain('1 of 2');
|
|
84
|
+
expect(result[0].details).toContain('origin_reachable');
|
|
85
|
+
expect(computeVerdict(result)).toBe('UNKNOWN');
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
test('a degraded module with a skipped check yields both a drift and an unmeasured finding', async () => {
|
|
89
|
+
const results: HealthCheckResult[] = [
|
|
90
|
+
{
|
|
91
|
+
moduleId: 'iptables',
|
|
92
|
+
status: 'degraded',
|
|
93
|
+
checks: [
|
|
94
|
+
{
|
|
95
|
+
name: 'persistence_drift',
|
|
96
|
+
status: 'warn',
|
|
97
|
+
message: '1 live rule(s) absent from rules.v4',
|
|
98
|
+
},
|
|
99
|
+
{ name: 'ssh_access', status: 'skip', message: 'fw01 unreachable' },
|
|
100
|
+
],
|
|
101
|
+
},
|
|
102
|
+
];
|
|
103
|
+
const result = await auditHealth({ results });
|
|
104
|
+
expect(result).toHaveLength(2);
|
|
105
|
+
expect(result.map((f) => f.severity).sort()).toEqual(['drift', 'unmeasured']);
|
|
106
|
+
expect(computeVerdict(result)).toBe('UNKNOWN');
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
test('a waived module still emits unmeasured, annotated with who and why', async () => {
|
|
110
|
+
// The waiver annotates the finding; it never replaces it. Severity
|
|
111
|
+
// stays `unmeasured` so the fleet verdict stays UNKNOWN (D3). Zero
|
|
112
|
+
// unmeasured findings after a waiver would mean the waiver re-greened.
|
|
113
|
+
const results: HealthCheckResult[] = [
|
|
114
|
+
{
|
|
115
|
+
moduleId: 'namecheap',
|
|
116
|
+
status: 'no-checks',
|
|
117
|
+
checks: [],
|
|
118
|
+
waiver: {
|
|
119
|
+
reason: 'API-only: a real check is hard and may not be worth forcing',
|
|
120
|
+
by: 'peba',
|
|
121
|
+
at: '2026-09-09',
|
|
122
|
+
},
|
|
123
|
+
},
|
|
124
|
+
];
|
|
125
|
+
const result = await auditHealth({ results });
|
|
126
|
+
expect(result).toHaveLength(1);
|
|
127
|
+
expect(result[0]).toMatchObject({
|
|
128
|
+
severity: 'unmeasured',
|
|
129
|
+
code: 'health_unmeasured',
|
|
130
|
+
subject: 'namecheap',
|
|
131
|
+
});
|
|
132
|
+
expect(result[0].message).toContain(
|
|
133
|
+
'WAIVED by peba (2026-09-09): API-only: a real check is hard and may not be worth forcing',
|
|
134
|
+
);
|
|
135
|
+
expect(computeVerdict(result)).toBe('UNKNOWN');
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
test('a waiver does not green an all-skipped module either', async () => {
|
|
139
|
+
const results: HealthCheckResult[] = [
|
|
140
|
+
{
|
|
141
|
+
moduleId: 'homebridge',
|
|
142
|
+
status: 'no-checks',
|
|
143
|
+
checks: [{ name: 'bridge_reachable', status: 'skip', message: 'no operator time' }],
|
|
144
|
+
waiver: {
|
|
145
|
+
reason: 'deferred: should eventually have one, no operator time now',
|
|
146
|
+
by: 'peba',
|
|
147
|
+
at: '2026-09-09',
|
|
148
|
+
},
|
|
149
|
+
},
|
|
150
|
+
];
|
|
151
|
+
const result = await auditHealth({ results });
|
|
152
|
+
expect(result).toHaveLength(1);
|
|
153
|
+
expect(result[0]).toMatchObject({ severity: 'unmeasured', code: 'health_unmeasured' });
|
|
154
|
+
expect(result[0].message).toContain('all 1 health checks skipped');
|
|
155
|
+
expect(result[0].message).toContain('WAIVED by peba');
|
|
156
|
+
expect(computeVerdict(result)).toBe('UNKNOWN');
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
test('a module without a waiver is not annotated', async () => {
|
|
160
|
+
const results: HealthCheckResult[] = [
|
|
161
|
+
{ moduleId: 'iptables', status: 'no-checks', checks: [] },
|
|
162
|
+
];
|
|
163
|
+
const result = await auditHealth({ results });
|
|
164
|
+
expect(result[0].message).not.toContain('WAIVED');
|
|
12
165
|
});
|
|
13
166
|
|
|
14
167
|
test('drift finding for unhealthy module', async () => {
|
|
@@ -5,24 +5,98 @@
|
|
|
5
5
|
* informational (DRIFT, not BLOCKED) — the user might want to update
|
|
6
6
|
* even if some module is currently unhealthy (an update could fix it).
|
|
7
7
|
*
|
|
8
|
+
* Unmeasured is reported, never folded into green: a module with no
|
|
9
|
+
* health_check hook, and a module whose checks all reported skip, have
|
|
10
|
+
* contributed no measurement at all, and both render the fleet UNKNOWN
|
|
11
|
+
* rather than READY (celilo#1326). The same rule reaches one level down —
|
|
12
|
+
* a module that measured something but skipped a check gets an
|
|
13
|
+
* `unmeasured` finding beside its verdict, so a VERIFIED module whose
|
|
14
|
+
* meaningful probe was skipped is still visible.
|
|
15
|
+
*
|
|
8
16
|
* The health-runner result is passed in directly so this file stays
|
|
9
17
|
* pure for testing. The aggregator wires `runAllHealthChecks` to it.
|
|
10
18
|
*/
|
|
11
19
|
|
|
20
|
+
import type { HealthWaiver } from '../../manifest/schema';
|
|
12
21
|
import type { HealthCheckResult } from '../health-runner';
|
|
13
22
|
import type { DriftFinding } from './types';
|
|
14
23
|
|
|
24
|
+
/** Render a waiver as the annotation carried on the `unmeasured` finding. */
|
|
25
|
+
export function waiverAnnotation(waiver: HealthWaiver): string {
|
|
26
|
+
return `WAIVED by ${waiver.by} (${waiver.at}): ${waiver.reason}`;
|
|
27
|
+
}
|
|
28
|
+
|
|
15
29
|
export interface HealthAuditDeps {
|
|
16
30
|
results: HealthCheckResult[];
|
|
17
31
|
}
|
|
18
32
|
|
|
33
|
+
function skippedChecksFinding(r: HealthCheckResult, code: string, message: string): DriftFinding {
|
|
34
|
+
const details = r.checks
|
|
35
|
+
.filter((c) => c.status === 'skip')
|
|
36
|
+
.map((c) => ` • ${c.name}: ${c.message}`)
|
|
37
|
+
.join('\n');
|
|
38
|
+
|
|
39
|
+
return {
|
|
40
|
+
category: 'health',
|
|
41
|
+
severity: 'unmeasured',
|
|
42
|
+
code,
|
|
43
|
+
message,
|
|
44
|
+
details: details || undefined,
|
|
45
|
+
remediation: `celilo module health ${r.moduleId} --debug`,
|
|
46
|
+
actionable: true,
|
|
47
|
+
subject: r.moduleId,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
|
|
19
51
|
export async function auditHealth(deps: HealthAuditDeps): Promise<DriftFinding[]> {
|
|
20
52
|
const findings: DriftFinding[] = [];
|
|
21
53
|
|
|
22
54
|
for (const r of deps.results) {
|
|
23
|
-
|
|
55
|
+
const skipped = r.checks.filter((c) => c.status === 'skip');
|
|
56
|
+
const measured = r.checks.filter((c) => c.status !== 'skip');
|
|
57
|
+
|
|
58
|
+
// A waiver annotates an `unmeasured` finding; it never replaces it.
|
|
59
|
+
// The severity stays `unmeasured` so the fleet verdict stays UNKNOWN
|
|
60
|
+
// (D3) — zero unmeasured findings after a waiver would mean the
|
|
61
|
+
// waiver re-greened the board, which is the failure it guards.
|
|
62
|
+
const waiverNote = r.waiver ? ` — ${waiverAnnotation(r.waiver)}` : '';
|
|
63
|
+
|
|
64
|
+
if (r.status === 'no-checks') {
|
|
65
|
+
// Either the module declares no health_check hook, or the hook ran
|
|
66
|
+
// and every check reported skip. Neither measured anything.
|
|
67
|
+
findings.push(
|
|
68
|
+
skipped.length > 0
|
|
69
|
+
? skippedChecksFinding(
|
|
70
|
+
r,
|
|
71
|
+
'health_unmeasured',
|
|
72
|
+
`${r.moduleId}: all ${skipped.length} health checks skipped — nothing measured${waiverNote}`,
|
|
73
|
+
)
|
|
74
|
+
: {
|
|
75
|
+
category: 'health',
|
|
76
|
+
severity: 'unmeasured',
|
|
77
|
+
code: 'health_unmeasured',
|
|
78
|
+
message: `${r.moduleId}: no health_check hook — never measured${waiverNote}`,
|
|
79
|
+
remediation: `Add a health_check hook to the ${r.moduleId} module manifest`,
|
|
80
|
+
actionable: false,
|
|
81
|
+
subject: r.moduleId,
|
|
82
|
+
},
|
|
83
|
+
);
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
if (skipped.length > 0) {
|
|
88
|
+
findings.push(
|
|
89
|
+
skippedChecksFinding(
|
|
90
|
+
r,
|
|
91
|
+
'health_checks_skipped',
|
|
92
|
+
`${r.moduleId}: ${skipped.length} of ${r.checks.length} health checks skipped${waiverNote}`,
|
|
93
|
+
),
|
|
94
|
+
);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
if (r.status === 'healthy') continue;
|
|
24
98
|
|
|
25
|
-
const failingChecks =
|
|
99
|
+
const failingChecks = measured
|
|
26
100
|
.filter((c) => c.status === 'fail' || c.status === 'warn')
|
|
27
101
|
.map((c) => ` • ${c.name}: ${c.message}`)
|
|
28
102
|
.join('\n');
|
|
@@ -32,6 +32,7 @@ const emptyDeps = {
|
|
|
32
32
|
moduleConfigs: { modules: [] },
|
|
33
33
|
moduleIntegrity: { results: [] },
|
|
34
34
|
detectWithoutConverge: { modules: [] },
|
|
35
|
+
jailExemptions: { exemptions: [] },
|
|
35
36
|
health: { results: [] },
|
|
36
37
|
backups: { modules: [] },
|
|
37
38
|
abandonedOperations: { records: [] },
|
|
@@ -41,8 +42,9 @@ const emptyDeps = {
|
|
|
41
42
|
secretsDecryptable: { results: [] },
|
|
42
43
|
servicesReachable: { results: [] },
|
|
43
44
|
machinesReachable: { results: [] },
|
|
45
|
+
diskSpace: { results: [] },
|
|
44
46
|
transportReads: { statuses: [], now: new Date(), staleAfterMs: 30 * 60_000 },
|
|
45
|
-
trustedSources: { firewalls: [] },
|
|
47
|
+
trustedSources: { firewalls: [], unreachableFirewalls: [] },
|
|
46
48
|
publicDns: { records: [], probe: unusedPublicDnsProbe },
|
|
47
49
|
};
|
|
48
50
|
|
|
@@ -88,6 +90,26 @@ describe('runAudit', () => {
|
|
|
88
90
|
expect(report.findings.some((f) => f.category === 'schema')).toBe(true);
|
|
89
91
|
});
|
|
90
92
|
|
|
93
|
+
test('disk_space is wired into the report (celilo#1336)', async () => {
|
|
94
|
+
const events: string[] = [];
|
|
95
|
+
const report = await runAudit(
|
|
96
|
+
{
|
|
97
|
+
...emptyDeps,
|
|
98
|
+
diskSpace: {
|
|
99
|
+
results: [{ hostname: 'celilo-mgr', ipAddress: '10.0.0.5', usedPercent: 96 }],
|
|
100
|
+
},
|
|
101
|
+
},
|
|
102
|
+
(event) => events.push(`${event.category}:${event.phase}`),
|
|
103
|
+
);
|
|
104
|
+
|
|
105
|
+
// 96% is at the blocked threshold, so the wired category alone flips
|
|
106
|
+
// the fleet verdict — which is the point of wiring it.
|
|
107
|
+
expect(report.verdict).toBe('BLOCKED');
|
|
108
|
+
expect(report.findings.map((f) => f.category)).toContain('disk_space');
|
|
109
|
+
expect(events).toContain('disk_space:start');
|
|
110
|
+
expect(events).toContain('disk_space:end');
|
|
111
|
+
});
|
|
112
|
+
|
|
91
113
|
test('aggregates findings across all categories', async () => {
|
|
92
114
|
const report = await runAudit({
|
|
93
115
|
...emptyDeps,
|
|
@@ -22,7 +22,9 @@ import {
|
|
|
22
22
|
type DetectWithoutConvergeAuditDeps,
|
|
23
23
|
auditDetectWithoutConverge,
|
|
24
24
|
} from './detect-without-converge';
|
|
25
|
+
import { type DiskSpaceAuditDeps, auditDiskSpace } from './disk-space';
|
|
25
26
|
import { type HealthAuditDeps, auditHealth } from './health';
|
|
27
|
+
import { type JailExemptionsAuditDeps, auditJailExemptions } from './jail-exemptions';
|
|
26
28
|
import { type MachinesReachableAuditDeps, auditMachinesReachable } from './machines-reachable';
|
|
27
29
|
import { type ModuleConfigsAuditDeps, auditModuleConfigs } from './module-configs';
|
|
28
30
|
import { type ModuleIntegrityAuditDeps, auditModuleIntegrity } from './module-integrity';
|
|
@@ -60,6 +62,7 @@ export interface AuditDeps {
|
|
|
60
62
|
moduleConfigs: ModuleConfigsAuditDeps;
|
|
61
63
|
moduleIntegrity: ModuleIntegrityAuditDeps;
|
|
62
64
|
detectWithoutConverge: DetectWithoutConvergeAuditDeps;
|
|
65
|
+
jailExemptions: JailExemptionsAuditDeps;
|
|
63
66
|
health: HealthAuditDeps;
|
|
64
67
|
backups: BackupsAuditDeps;
|
|
65
68
|
abandonedOperations: AbandonedOperationsAuditDeps;
|
|
@@ -70,6 +73,7 @@ export interface AuditDeps {
|
|
|
70
73
|
servicesReachable: ServicesReachableAuditDeps;
|
|
71
74
|
machinesReachable: MachinesReachableAuditDeps;
|
|
72
75
|
publicDns: PublicDnsAuditDeps;
|
|
76
|
+
diskSpace: DiskSpaceAuditDeps;
|
|
73
77
|
transportReads: TransportReadsAuditDeps;
|
|
74
78
|
trustedSources: TrustedSourcesAuditDeps;
|
|
75
79
|
/** Defaults to `Date.now()`-based ISO string. */
|
|
@@ -95,8 +99,8 @@ export async function runAudit(
|
|
|
95
99
|
): Promise<SystemAuditReport> {
|
|
96
100
|
// Wrap each category's promise so the caller sees per-category
|
|
97
101
|
// start/end events. `start` fires synchronously when this function
|
|
98
|
-
// schedules the category; in practice all
|
|
99
|
-
// since Promise.all spawns them in parallel.
|
|
102
|
+
// schedules the category; in practice all of them fire roughly at
|
|
103
|
+
// once since Promise.all spawns them in parallel.
|
|
100
104
|
const wrap = (category: DriftCategory, p: Promise<DriftFinding[]>): Promise<DriftFinding[]> => {
|
|
101
105
|
onProgress?.({ category, phase: 'start' });
|
|
102
106
|
return p.then((findings) => {
|
|
@@ -119,6 +123,7 @@ export async function runAudit(
|
|
|
119
123
|
'detect_without_converge',
|
|
120
124
|
Promise.resolve(auditDetectWithoutConverge(deps.detectWithoutConverge)),
|
|
121
125
|
),
|
|
126
|
+
wrap('jail_exemptions', Promise.resolve(auditJailExemptions(deps.jailExemptions))),
|
|
122
127
|
wrap('health', auditHealth(deps.health)),
|
|
123
128
|
wrap('backups', auditBackups(deps.backups)),
|
|
124
129
|
wrap(
|
|
@@ -132,6 +137,7 @@ export async function runAudit(
|
|
|
132
137
|
wrap('services_reachable', auditServicesReachable(deps.servicesReachable)),
|
|
133
138
|
wrap('machines_reachable', auditMachinesReachable(deps.machinesReachable)),
|
|
134
139
|
wrap('public_dns', auditPublicDns(deps.publicDns)),
|
|
140
|
+
wrap('disk_space', Promise.resolve(auditDiskSpace(deps.diskSpace))),
|
|
135
141
|
wrap('transport_reads', auditTransportReads(deps.transportReads)),
|
|
136
142
|
wrap('trusted_sources', auditTrustedSources(deps.trustedSources)),
|
|
137
143
|
]);
|
|
@@ -207,14 +207,25 @@ describe('describeClassification — the picture, not a finding', () => {
|
|
|
207
207
|
|
|
208
208
|
describe('auditInterfaceClassification — every firewall, flattened', () => {
|
|
209
209
|
test('reports across several firewalls, each subjected on its own hostname', () => {
|
|
210
|
-
const findings = auditInterfaceClassification(
|
|
211
|
-
|
|
212
|
-
liveFleetShape({ hostname: 'fw-branch' }),
|
|
213
|
-
|
|
210
|
+
const findings = auditInterfaceClassification({
|
|
211
|
+
unreachableFirewalls: [],
|
|
212
|
+
views: [liveFleetShape(), liveFleetShape({ hostname: 'fw-branch' })],
|
|
213
|
+
});
|
|
214
214
|
expect(findings.map((f) => f.subject).sort()).toEqual(['fw-branch', 'fw-main']);
|
|
215
215
|
});
|
|
216
216
|
|
|
217
217
|
test('no firewalls means no findings', () => {
|
|
218
|
-
expect(auditInterfaceClassification([])).toEqual([]);
|
|
218
|
+
expect(auditInterfaceClassification({ views: [], unreachableFirewalls: [] })).toEqual([]);
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
test('a firewall whose interfaces could not be read is unmeasured, not silent (D7)', () => {
|
|
222
|
+
const findings = auditInterfaceClassification({
|
|
223
|
+
views: [liveFleetShape()],
|
|
224
|
+
unreachableFirewalls: ['fw-branch'],
|
|
225
|
+
});
|
|
226
|
+
const unmeasured = findings.filter((f) => f.severity === 'unmeasured');
|
|
227
|
+
expect(unmeasured).toHaveLength(1);
|
|
228
|
+
expect(unmeasured[0].subject).toBe('fw-branch');
|
|
229
|
+
expect(unmeasured[0].code).toBe('interface_classification_unmeasured');
|
|
219
230
|
});
|
|
220
231
|
});
|
|
@@ -142,8 +142,31 @@ export function auditFirewallInterfaces(view: FirewallInterfaceView): DriftFindi
|
|
|
142
142
|
}
|
|
143
143
|
|
|
144
144
|
/** Every firewall's findings, flattened. */
|
|
145
|
-
export
|
|
146
|
-
|
|
145
|
+
export interface InterfaceClassificationAuditDeps {
|
|
146
|
+
views: FirewallInterfaceView[];
|
|
147
|
+
/**
|
|
148
|
+
* Firewalls celilo knows about but whose interface state could not be read
|
|
149
|
+
* over SSH. Each is an `unmeasured` finding: an interface survey that
|
|
150
|
+
* cannot see a firewall has measured nothing about it (D7).
|
|
151
|
+
*/
|
|
152
|
+
unreachableFirewalls: string[];
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
export function auditInterfaceClassification(
|
|
156
|
+
deps: InterfaceClassificationAuditDeps,
|
|
157
|
+
): DriftFinding[] {
|
|
158
|
+
const unreachable = deps.unreachableFirewalls.map((firewallIp): DriftFinding => {
|
|
159
|
+
return {
|
|
160
|
+
category: 'interface_classification',
|
|
161
|
+
severity: 'unmeasured',
|
|
162
|
+
code: 'interface_classification_unmeasured',
|
|
163
|
+
subject: firewallIp,
|
|
164
|
+
message: `${firewallIp}: interfaces could not be read, so classification is unknown`,
|
|
165
|
+
remediation: `Check SSH reachability of ${firewallIp}, then re-audit. This finding records that the interfaces were not read, not that they are classified.`,
|
|
166
|
+
actionable: true,
|
|
167
|
+
};
|
|
168
|
+
});
|
|
169
|
+
return [...unreachable, ...deps.views.flatMap(auditFirewallInterfaces)];
|
|
147
170
|
}
|
|
148
171
|
|
|
149
172
|
/**
|