@celilo/cli 2.1.0 → 2.2.1

package/src/hooks/broker.test.ts

@@ -18,7 +18,9 @@ import { join } from 'node:path';
 import { MissingProviderInputError } from '@celilo/capabilities';
 import { capabilityShape } from './broker';
 import { executeHookScript } from './executor';
+import type { HookStoreBackend, HookStores } from './hook-store';
 import { createCapturingLogger } from './logger';
+import { configStore, secretStore } from './test-fixtures/store-backed';
 import type { HookContext } from './types';
 
 const FIXTURES = join(__dirname, 'test-fixtures');
@@ -49,8 +51,8 @@ async function runCapabilityHook(): Promise<Record<string, unknown>> {
   const dir = mkdtempSync(join(tmpdir(), 'celilo-broker-'));
   try {
     const context: HookContext = {
-      config: {},
-      secrets: {},
+      config: configStore(),
+      secrets: secretStore(),
       systems: [],
       logger: createCapturingLogger().logger,
       debug: false,
@@ -67,6 +69,84 @@ async function runCapabilityHook(): Promise<Record<string, unknown>> {
   }
 }
 
+/**
+ * In-memory stand-ins for the broker-side stores. The storage mechanics have
+ * their own suite (`hook-store.test.ts`, real DB); what this layer must prove
+ * is the ROUND TRIP — a write in the hook's process landing in the broker's
+ * store and the read coming back.
+ */
+function memoryStores(): HookStores {
+  const makeBackend = (
+    kind: 'secret' | 'hook-owned config',
+    declared: string[],
+  ): HookStoreBackend => {
+    const rows = new Map<string, string>();
+    const assertDeclared = (name: string) => {
+      if (!declared.includes(name)) {
+        throw new Error(
+          `Module 'test-module' has no declared ${kind} '${name}'. Declared ${kind} names: ${declared.join(', ')}.`,
+        );
+      }
+    };
+    return {
+      get: (name) => {
+        assertDeclared(name);
+        return Promise.resolve(rows.get(name));
+      },
+      set: (name, value) => {
+        assertDeclared(name);
+        rows.set(name, value);
+        return Promise.resolve();
+      },
+      delete: (name) => {
+        assertDeclared(name);
+        rows.delete(name);
+        return Promise.resolve();
+      },
+      applyTransaction: (ops) => {
+        for (const entry of ops) {
+          assertDeclared(entry.name);
+          if (entry.op === 'set') rows.set(entry.name, entry.value ?? '');
+          else rows.delete(entry.name);
+        }
+        return Promise.resolve();
+      },
+    };
+  };
+  return {
+    secrets: makeBackend('secret', ['bot_token', 'api_key']),
+    config: makeBackend('hook-owned config', ['public_ip']),
+    declaredSecretNames: ['bot_token', 'api_key'],
+    declaredConfigNames: ['public_ip'],
+  };
+}
+
+async function runStoreHook(
+  contextData: Record<string, unknown>,
+): Promise<Record<string, unknown>> {
+  const dir = mkdtempSync(join(tmpdir(), 'celilo-broker-store-'));
+  try {
+    const context = {
+      config: { mapOnlyValue: 'from-the-context-frame' },
+      secrets: {},
+      systems: [],
+      logger: createCapturingLogger().logger,
+      debug: false,
+      screenshotDir: dir,
+      stateDir: dir,
+      capabilities: {},
+      ...contextData,
+    } as unknown as HookContext;
+    return await executeHookScript(join(FIXTURES, 'store-writing-hook.ts'), context, {
+      timeoutMs: 30_000,
+      idleTimeoutMs: 30_000,
+      hookStores: () => Promise.resolve(memoryStores()),
+    });
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
+
 describe('capabilityShape', () => {
   test('splits functions from data', () => {
     const shape = capabilityShape(demoCapabilities());
@@ -149,3 +229,27 @@ describe('capability calls across the boundary', () => {
     expect(orphans).toEqual([]);
   }, 30_000);
 });
+
+describe('hook-owned-state store calls across the boundary', () => {
+  test('every store operation survives the round trip', async () => {
+    const outputs = await runStoreHook({});
+
+    expect(outputs.secretRoundTrip).toBe('token-value');
+    expect(outputs.configRoundTrip).toBe('203.0.113.7');
+    // The old map surface still answers beside the new methods.
+    expect(outputs.mapRead).toBe('from-the-context-frame');
+    expect(outputs.deletedIsGone).toBeUndefined();
+
+    // D4: plain sets land immediately; a transaction discards on throw.
+    expect(outputs.transactionA).toBe('committed-a');
+    expect(outputs.transactionB).toBe('committed-b');
+    expect(outputs.discarded).toBe('hook failed midway');
+    expect(outputs.afterDiscard).toBe('committed-a');
+
+    // D2: the undeclared name is an ERROR naming the module and the set. The
+    // message text itself is pinned by hook-store.test.ts; here it only has
+    // to have survived the wire intact.
+    expect(outputs.undeclaredSecret).toContain('not_declared');
+    expect(outputs.undeclaredSecret).toContain('bot_token');
+  }, 30_000);
+});

package/src/hooks/broker.ts

@@ -23,7 +23,10 @@ import { type Server, type Socket, createServer } from 'node:net';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import {
+  type BufferedStoreOp,
   type CapabilityShape,
+  type ChildFrame,
+  HOOK_PROTOCOL_VERSION,
   type HookError,
   createLineReader,
   encodeFrame,
@@ -31,8 +34,22 @@ import {
   serializeError,
   versionMismatch,
 } from './hook-protocol';
+import type { HookStores } from './hook-store';
 import type { HookLogger } from './types';
 
+/** The answering half's view of one store frame. */
+type StoreCallFrame = Extract<ChildFrame, { type: 'store' }>;
+
+/**
+ * A lazy provider of the module's two hook state stores.
+ *
+ * Lazy because building the stores reads the module row and its manifest, and
+ * because the secrets store touches the master key only when an operation
+ * actually needs it. A hook that never calls `context.secrets` should pay
+ * none of that.
+ */
+export type HookStoresProvider = () => Promise<HookStores>;
+
 /**
  * How long to wait after the child exits for its last frame to arrive. Short,
  * because by this point the writer is already gone and the bytes are either in
@@ -56,6 +73,13 @@ export interface BrokerOptions {
   logger: HookLogger;
   /** Called on every frame, so the idle timer measures the whole channel. */
   onActivity: () => void;
+  /**
+   * The module's hook-owned-state stores (hook-owned-state task 3.5). Absent
+   * means this run has none wired, and a `store` frame from the hook is
+   * REFUSED with an error naming the gap — a write that vanishes silently is
+   * the exact defect the accessor exists to remove.
+   */
+  stores?: HookStoresProvider;
 }
 
 export interface Broker {
@@ -163,6 +187,8 @@ export async function startBroker(options: BrokerOptions): Promise<Broker> {
   let connection: Socket | undefined;
   // Already resolved: no connection means nothing left to deliver.
   let connectionClosed: Promise<void> = Promise.resolve();
+  // The module's stores, resolved from the provider on first use.
+  let storesCache: HookStores | undefined;
 
   const server: Server = createServer((socket) => {
     if (connection) {
@@ -200,7 +226,7 @@ export async function startBroker(options: BrokerOptions): Promise<Broker> {
           }
           send({
             type: 'context',
-            protocolVersion: 1,
+            protocolVersion: HOOK_PROTOCOL_VERSION,
             scriptPath: options.scriptPath,
             context: options.context,
           });
@@ -216,6 +242,10 @@ export async function startBroker(options: BrokerOptions): Promise<Broker> {
           void dispatch(frame.id, frame.capability, frame.method, frame.args, send);
           return;
 
+        case 'store':
+          void dispatchStore(frame, send);
+          return;
+
         case 'result':
           outcome = { ok: true, outputs: frame.outputs };
           return;
@@ -285,6 +315,66 @@ export async function startBroker(options: BrokerOptions): Promise<Broker> {
     server.listen(socketPath, resolve);
   });
 
+  /**
+   * Answer one hook-owned-state store call.
+   *
+   * The stores resolve on first use and stay cached for the run. Everything
+   * that can fail — an undeclared name, a missing module row, a store that was
+   * never wired — answers as a `throw` frame rather than crashing the broker,
+   * so the hook's own `try`/`catch` sees it the way an in-process throw would
+   * behave.
+   */
+  async function dispatchStore(
+    frame: StoreCallFrame,
+    send: (frame: Parameters<typeof encodeFrame>[0]) => void,
+  ): Promise<void> {
+    if (stopped) {
+      send({
+        type: 'throw',
+        id: frame.id,
+        error: {
+          name: 'Error',
+          message: `Hook run has ended; refusing context.${frame.store}.${frame.method}.`,
+        },
+      });
+      return;
+    }
+
+    try {
+      if (!options.stores) {
+        throw new Error(
+          `No hook state stores were wired for this run, so context.${frame.store} is unavailable. This is a celilo bug: every hook invocation must receive both stores (hook-owned-state task 3.5).`,
+        );
+      }
+      storesCache ??= await options.stores();
+      const backend = storesCache[frame.store];
+
+      let value: unknown;
+      switch (frame.method) {
+        case 'get':
+          value = await backend.get(frame.args[0] as string);
+          break;
+        case 'set':
+          await backend.set(frame.args[0] as string, frame.args[1] as string);
+          value = null;
+          break;
+        case 'delete':
+          await backend.delete(frame.args[0] as string);
+          value = null;
+          break;
+        case 'transaction':
+          await backend.applyTransaction(frame.args[0] as BufferedStoreOp[]);
+          value = null;
+          break;
+      }
+      options.onActivity();
+      send({ type: 'return', id: frame.id, value });
+    } catch (error) {
+      options.onActivity();
+      send({ type: 'throw', id: frame.id, error: serializeError(error) });
+    }
+  }
+
   return {
     socketPath,
     drained: () =>

package/src/hooks/capability-loader-firewall.test.ts

@@ -302,6 +302,39 @@ describe('Firewall Chain Building', () => {
       expect(config.isolateTransitNetwork).toBe(true);
     });
 
+    test('single provider: the recorded interface_baseline is forwarded as a parsed list', async () => {
+      // The recorded baseline is what turns an undeclared interface from a
+      // refusal into an isolation. The single-provider iptables firewall (the
+      // production shape: the ISP router is not a celilo module) is built by
+      // buildCapabilityInterface, and THIS is the construction site that
+      // dropped the field — the baseline was written to module config and
+      // never read back, so the module converged in onboarding mode forever
+      // (ce-qzxm).
+      installReporter('iptables', '{"has_external":true}', '["dmz","app","secure"]');
+      upsertModuleConfig(db, 'iptables', 'firewall_ip', '192.168.0.254');
+      upsertModuleConfig(db, 'iptables', 'nat_ip', '192.168.0.253');
+      upsertModuleConfig(db, 'iptables', 'interface_baseline', 'eth0, eth1, eth2, eth3');
+
+      const config = await loadedConfig();
+      // A parsed array of NAMES, as the module's D5 split consumes it — not
+      // the raw comma string, and not with the write-side padding intact.
+      expect(config.interfaceBaseline).toEqual(['eth0', 'eth1', 'eth2', 'eth3']);
+    });
+
+    test('single provider: an empty interface_baseline arrives undefined, meaning onboarding', async () => {
+      // Empty and absent are the same thing, and both mean "never converged
+      // cleanly": treating an empty string as a baseline of nothing would make
+      // every interface new. The loader normalizes this so the module sees
+      // one shape for one meaning.
+      installReporter('iptables', '{"has_external":true}', '["dmz"]');
+      upsertModuleConfig(db, 'iptables', 'firewall_ip', '192.168.0.254');
+      upsertModuleConfig(db, 'iptables', 'nat_ip', '192.168.0.253');
+      upsertModuleConfig(db, 'iptables', 'interface_baseline', '');
+
+      const config = await loadedConfig();
+      expect(config.interfaceBaseline).toBeUndefined();
+    });
+
     test('chained provider: the downstream layer gets them too', async () => {
       // greenwave owns the WAN, so iptables is built through buildFirewallChain
       // rather than the single-provider path. That is a SECOND construction
@@ -324,10 +357,14 @@ describe('Firewall Chain Building', () => {
       upsertModuleConfig(db, 'iptables', 'nat_ip', '192.168.0.253');
       upsertModuleConfig(db, 'iptables', 'default_route_zone', 'internal');
       upsertModuleConfig(db, 'iptables', 'isolate_transit_network', true);
+      upsertModuleConfig(db, 'iptables', 'interface_baseline', 'eth0,eth1');
 
       const config = await loadedConfig();
       expect(config.defaultRouteZone).toBe('internal');
       expect(config.isolateTransitNetwork).toBe(true);
+      // The downstream construction site already forwarded this; pinned so the
+      // two sites cannot drift apart again.
+      expect(config.interfaceBaseline).toEqual(['eth0', 'eth1']);
     });
 
     test('unset settings arrive undefined, not as a wrong default', async () => {

package/src/hooks/capability-loader.test.ts

@@ -7,7 +7,9 @@ import { mkdirSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import type { HookLogger, RouteReadView } from '@celilo/capabilities';
+import { and, eq } from 'drizzle-orm';
 import type { DbClient } from '../db/client';
+import { moduleConfigs } from '../db/schema';
 import { upsertModuleConfig } from '../services/module-config';
 import { cleanupTestDatabase, setupTestDatabase } from '../test-utils/database';
 import { loadCapabilityFunctions } from './capability-loader';
@@ -40,6 +42,36 @@ export default function createDhcpServer(context) {
 }
 `;
 
+/**
+ * A factory that WRITES through the hook-owned-state accessor, so the loader's
+ * store view is proven end to end: the method's write lands as a hook-owned
+ * row, and an undeclared name throws instead of landing. Branded
+ * (defineCapabilityFunction, the Phase 8 pattern) — the branded path is the
+ * one the loader hands the store views to. The temp dir the fixture is
+ * dropped into has no node_modules link, so the import uses the workspace
+ * package by absolute path, the same trick capability-loader-firewall uses.
+ */
+const TEST_STORE_WRITING_DHCP_MODULE = `
+import { defineCapabilityFunction } from '${join(__dirname, '../../../..', 'packages/capabilities/src/index.ts')}';
+
+export default defineCapabilityFunction({
+  capability: 'dhcp_server',
+  handler: (context) => ({
+    async setDnsServers() {},
+    async getDnsServers() { return [context.config.marker]; },
+    async setDomainName() {},
+    async getDomainName() { return context.config.marker; },
+    async claimIp() {
+      await context.config.set('public_ip', '203.0.113.7');
+      return 'claimed';
+    },
+    async claimUndeclared() {
+      await context.config.set('no_such_key', 'x');
+    },
+  }),
+});
+`;
+
 describe('Capability Loader', () => {
   let db: DbClient;
   let tempDir: string;
@@ -134,6 +166,52 @@ describe('Capability Loader', () => {
     expect(await dhcp.getDomainName()).toBe('internal-dhcp');
   });
 
+  test('a provider factory writes hook-owned config through the injected store view', async () => {
+    // The store view must do more than carry the plain record a factory reads.
+    // A method writing `context.config.set` lands a hook-owned row (validated
+    // against the PROVIDER's manifest, source column 'hook'), and an
+    // undeclared name throws instead of writing a phantom row.
+    const modulePath = join(tempDir, 'storewriting-vpn');
+    const scriptsDir = join(modulePath, 'scripts');
+    mkdirSync(scriptsDir, { recursive: true });
+    writeFileSync(join(scriptsDir, 'dhcp-server-functions.ts'), TEST_STORE_WRITING_DHCP_MODULE);
+    const manifest = JSON.stringify({
+      variables: { owns: [{ name: 'public_ip', type: 'string', source: 'hook' }] },
+    });
+    db.$client.run(
+      `INSERT INTO modules (id, name, version, source_path, manifest_data) VALUES ('storewriting-vpn', 'StoreWritingVpn', '1.0.0', '${modulePath}', '${manifest.replace(/'/g, "''")}')`,
+    );
+    db.$client.run(
+      `INSERT INTO capabilities (module_id, capability_name, version, data, registered_at) VALUES ('storewriting-vpn', 'dhcp_server', '1.0.0', '{}', unixepoch())`,
+    );
+    upsertModuleConfig(db, 'storewriting-vpn', 'marker', 'storewriting-vpn');
+
+    const result = await loadCapabilityFunctions('store-consumer', db, noopLogger);
+    const dhcp = result.dhcp_server as {
+      claimIp(): Promise<string>;
+      claimUndeclared(): Promise<void>;
+    };
+
+    expect(await dhcp.claimIp()).toBe('claimed');
+    const row = db
+      .select()
+      .from(moduleConfigs)
+      .where(
+        and(eq(moduleConfigs.moduleId, 'storewriting-vpn'), eq(moduleConfigs.key, 'public_ip')),
+      )
+      .get();
+    expect(row?.valueJson).toBe('"203.0.113.7"');
+    expect(row?.source).toBe('hook');
+
+    // The store round-trips text, so the map read the NEXT load takes sees the
+    // same string the write sent.
+    const reloaded = await loadCapabilityFunctions('store-consumer', db, noopLogger);
+    const dhcpAgain = reloaded.dhcp_server as { getDnsServers(): Promise<string[]> };
+    expect(await dhcpAgain.getDnsServers()).toEqual(['storewriting-vpn']);
+
+    await expect(dhcp.claimUndeclared()).rejects.toThrow(/no declared hook-owned config/);
+  });
+
   test('falls back to a zone-agnostic provider when no explicit zone provider exists', async () => {
     const modulePath = join(tempDir, 'upstream-dhcp');
     const scriptsDir = join(modulePath, 'scripts');

package/src/hooks/capability-loader.ts

@@ -39,7 +39,7 @@ import {
   systemConfig,
   webRoutes,
 } from '../db/schema';
-import { resolveModuleWebRoot } from '../module/web-root';
+import { resolveModuleStateWebRoot, resolveModuleWebRoot } from '../module/web-root';
 import { decryptSecret } from '../secrets/encryption';
 import { getOrCreateMasterKey } from '../secrets/master-key';
 import { buildControlPlaneApi } from '../services/api-principal-enrolment';
@@ -60,6 +60,9 @@ import {
 import { resolveComputedFields } from '../variables/computed/evaluate';
 import { containsComputedMarker } from '../variables/computed/marker';
 import { buildProviderLookup } from '../variables/computed/provider-lookup';
+import type { BufferedStoreOp } from './hook-protocol';
+import { type HookStores, createHookStores } from './hook-store';
+import { type StoreTransport, buildStoreView } from './hook-store-proxy';
 import { loadHookConfigMap } from './load-hook-config';
 
 /**
@@ -260,6 +263,13 @@ export async function loadCapabilityFunctions(
   // This allows modules to optionally consume capabilities at runtime
   // (e.g., caddy uses dns_registrar if available, falls back if not).
   const verbose = process.env.CELILO_DEBUG === '1';
+
+  // The broker-side stores, resolved from the PROVIDER on the first store
+  // call this consumer load makes, and shared across every capability it
+  // builds. Nothing pays for the manifest read until a factory actually
+  // writes or reads through the accessor. See the storeTransport construction
+  // at the factory-invocation site below.
+  let storesPromise: Promise<HookStores> | undefined;
   const debugLog = verbose
     ? (msg: string) => process.stderr.write(`[capability-loader] ${msg}\n`)
     : (_msg: string) => {};
@@ -392,10 +402,35 @@ export async function loadCapabilityFunctions(
       // Phase 8 path: branded defineCapabilityFunction factory. Call with
       // the canonical { config, secrets, logger } context. The factory
       // applies wrapWithLogging internally so we don't double-wrap.
+      //
+      // Both maps arrive as the hook-owned-state view (buildStoreView): the
+      // plain record the factory reads, PLUS get/set/delete/transaction that
+      // reach the broker-side stores (createHookStores), validated against
+      // the PROVIDER module's own manifest. The transport is in-process — a
+      // capability factory runs here in celilo's process, not in the jail —
+      // but the view is the same object a jailed hook sees, so a provider's
+      // self-config write goes through the same validated door regardless of
+      // which process it runs in (hook-owned-state tasks 0.7 and 5.5 shape).
       if (isCompiledCapabilityFactory(exported)) {
+        const storeTransport: StoreTransport = (store, method, args) => {
+          storesPromise ??= createHookStores(db, capability.moduleId);
+          return storesPromise.then((backend: HookStores): Promise<unknown> => {
+            const one = backend[store];
+            switch (method) {
+              case 'transaction':
+                return one.applyTransaction(args[0] as readonly BufferedStoreOp[]);
+              case 'set':
+                return one.set(args[0] as string, args[1] as string);
+              case 'delete':
+                return one.delete(args[0] as string);
+              default:
+                return one.get(args[0] as string);
+            }
+          });
+        };
         const capabilityInterface = exported({
-          config: providerConfig,
-          secrets: providerSecrets,
+          config: buildStoreView('config', providerConfig, storeTransport),
+          secrets: buildStoreView('secrets', providerSecrets, storeTransport),
           systems: getModuleSystems(capability.moduleId, db),
           logger,
           // WHO IS CALLING, so a provider can scope per-consumer state to them.
@@ -408,6 +443,10 @@ export async function loadCapabilityFunctions(
           // without this they could not find the caller's web root now that
           // `sourceDir` has left the request (D10 amendment).
           consumerWebRoot: resolveModuleWebRoot(consumingModuleId, db),
+          // WHERE THE CALLER'S GENERATED SITE FILES ARE (celilo#1265), by the
+          // same seam. A jailed hook cannot write into its own web root, so a
+          // module-provided web provider needs the state overlay to honor it.
+          consumerStateWebRoot: resolveModuleStateWebRoot(consumingModuleId, db),
         });
         // Stamp here too, not only on the legacy path: a consumer that cannot
         // get what it needs must be able to name WHICH provider could not give
@@ -594,6 +633,7 @@ export async function loadCapabilityFunctions(
         // Resolved here, not in the capability: on a converge there is no
         // consumer process to ask (D10 amendment).
         webRoot: resolveModuleWebRoot(consumingModuleId, db),
+        stateWebRoot: resolveModuleStateWebRoot(consumingModuleId, db),
         logger,
         config: providerConfig,
         secrets: providerSecrets,
@@ -789,6 +829,15 @@ function buildCapabilityInterface(
           trustedSubnets: zones?.trustedSubnets ?? [],
           controlPlaneSubnet: zones?.controlPlaneSubnet,
           frontedSubnets: zones?.frontedSubnets ?? [],
+          // The recorded baseline (D12), from this firewall's own module config.
+          // Absent means the box has never converged cleanly, so an interface
+          // celilo cannot attribute refuses rather than being disabled. The
+          // downstream-chain site below forwards this too; the single-provider
+          // firewall (an iptables box with the WAN held by a non-celilo ISP
+          // router) is built HERE, and this is the site that dropped it — the
+          // baseline was recorded but never read back, so every converge stayed
+          // in onboarding mode and refused instead of isolating (ce-qzxm).
+          interfaceBaseline: parseInterfaceBaseline(config.interface_baseline),
           // Operator settings, forwarded verbatim. `parseStoredConfigValue`
           // preserves each manifest-declared type, so the boolean arrives as a
           // boolean and needs no coercion here.

package/src/hooks/define-hook.test.ts

@@ -30,6 +30,7 @@ import {
   isCompiledCapabilityFactory,
   isCompiledHook,
 } from '@celilo/capabilities';
+import { configStore, secretStore } from './test-fixtures/store-backed';
 
 function makeLogger(): HookLogger {
   return {
@@ -42,8 +43,8 @@ function makeLogger(): HookLogger {
 
 function makeContext(overrides: Partial<HookContext> = {}): HookContext {
   return {
-    config: {},
-    secrets: {},
+    config: configStore(),
+    secrets: secretStore(),
     systems: [],
     logger: makeLogger(),
     consumerModuleId: 'test-consumer',
@@ -152,7 +153,7 @@ describe('defineHook', () => {
       },
     });
 
-    await hook(makeContext({ config: { foo: 'bar' } }));
+    await hook(makeContext({ config: configStore({ foo: 'bar' }) }));
 
     expect(seenConfig).toEqual({ foo: 'bar' });
   });