@nylorun/runtime 0.13.0-beta → 0.15.0-beta

package/dist/host/main.js

@@ -2,33 +2,48 @@
  * Runtime Host process entry (`@nylorun/runtime/server`).
  *
  * Reads `NYLORUN_HOME` (falls back to `~/.nylorun`), and hands an environment
- * snapshot and argv to `parseStackConfig` (container listen mode, `--role`,
+ * snapshot and argv to `parseStackConfig` (container listen mode, `--service`,
  * stack endpoints). Absolute Host root is resolved once and passed down.
  *
  * Composition: `createInfra` builds the Postgres pool, Durable Session
  * Execution and Durable Streams from the endpoints; `createHostExecution`
- * shares one execution across the Tenants; the Tenant store is Postgres
- * (`NYLORUN_DATABASE_URL`, required); `/ready` reports the infrastructure
- * checks. See the startup order in `main()`. Tests compose a Host without this
+ * builds the process's execution; the Host serves the one Tenant its Postgres
+ * database holds (`NYLORUN_DATABASE_URL`, required), and creates it there on
+ * first start (`NYLORUN_TENANT_ID`, `NYLORUN_TENANT_NAME`,
+ * `NYLORUN_DERIVED_PRINCIPALS`; tenancy.md §4); with S2, a process running core
+ * runs the stream relay, which feeds the Tenant's streams from the record over
+ * logical replication (one process at a time holds the slot); `/ready` reports
+ * the Tenant and the infrastructure checks. A process running the gates service (the local
+ * stack's `gateway` container) starts only the gate (`runGates`): it needs
+ * neither host.json nor host-credentials.json. See the startup order in `main()`. Tests compose a Host without this
  * entry, with `createHost` and an injected Tenant module.
  */
 import { readFileSync, mkdirSync, existsSync } from "node:fs";
 import { createRequire } from "node:module";
 import { homedir } from "node:os";
 import { resolve } from "node:path";
-import { hostPaths, trashSqliteTenants } from "../tenant/paths.js";
+import { hostPaths } from "../tenant/paths.js";
 import { createTenantModule } from "../tenant/module.js";
-import { createPostgresTenantStore } from "../tenant/store-pg.js";
+import { hostPrincipals } from "../tenant/principals.js";
+import { createPostgresTenantOpener } from "../tenant/store-pg.js";
 import { openTenantRuntime } from "../tenant/runtime.js";
-import { createTenantStreams, deleteTenantStreams } from "../tenant/streams.js";
+import { createPgoutputSource } from "../adapters/replication/pgoutput.js";
+import { assertLogicalReplication } from "../store/postgres/connect.js";
+import { createPostgresRecordReader } from "../store/postgres/record.js";
+import { createStreamRelay } from "../streams/relay/core.js";
 import { configForFactory } from "./config-for.js";
 import { createHost } from "./create-host.js";
 import { createHostExecution } from "./execution.js";
 import { baselineEnvironment } from "./environment.js";
 import { HostListenError, EXIT_PORT_IN_USE, EXIT_NON_LOOPBACK, } from "./http.js";
 import { createHostLogger } from "./logger.js";
-import { describeEndpoints, parseStackConfig } from "./stack-config.js";
+import { describeEndpoints, describeServices, parseStackConfig, } from "./stack-config.js";
 import { createExecution, createInfra } from "../infra/index.js";
+import { createDatabase } from "../infra/database.js";
+import { startGates } from "./gates.js";
+import { httpToolGate } from "../gates/tool-client.js";
+import { httpKeys } from "../keys/client.js";
+import { httpModelGate } from "../gates/http-client.js";
 const nodeRequire = createRequire(import.meta.url);
 function coreVersion() {
     try {
@@ -46,20 +61,74 @@ function resolveHostRoot() {
     const root = fromEnv && fromEnv.length > 0 ? fromEnv : resolve(homedir(), ".nylorun");
     return resolve(root);
 }
+/** The slot's lag in WAL bytes, when the source can tell. */
+async function lagOf(source) {
+    const lagBytes = await source.lag?.().catch(() => undefined);
+    return lagBytes === undefined ? {} : { lagBytes };
+}
+/**
+ * The gates service: the Model Gate's listener over the Postgres pool and the Host's tenant
+ * directory. Writes nothing to the Host root (the local stack mounts it read-only).
+ */
+async function runGates(stack) {
+    const gates = stack.gates;
+    if (!stack.endpoints.databaseUrl)
+        throw new Error("NYLORUN_DATABASE_URL is required for --service gates: the gate reads Tenant vaults from Postgres (`nylorun start` sets it)");
+    const logger = createHostLogger();
+    logger.info("host_stack_config", {
+        services: [...stack.services],
+        ...(stack.packing ? { packing: stack.packing } : {}),
+        endpoints: describeEndpoints(stack.endpoints),
+    });
+    const database = createDatabase(stack);
+    let server;
+    try {
+        server = await startGates({
+            gates,
+            database,
+            hostRoot: resolveHostRoot(),
+            logger,
+            ...(stack.delivery ? { delivery: stack.delivery } : {}),
+            ...(stack.services.has("keys") ? { keys: true } : {}),
+        });
+    }
+    catch (error) {
+        await database.end({ timeout: 5 });
+        if (error instanceof HostListenError) {
+            logger.error("listen_failed", { message: error.message, exitCode: error.exitCode });
+            process.exitCode = error.exitCode;
+            return;
+        }
+        throw error;
+    }
+    logger.info("gates_ready", { url: server.url });
+    let stopping = false;
+    for (const signal of ["SIGINT", "SIGTERM"])
+        process.on(signal, () => {
+            if (stopping)
+                return;
+            stopping = true;
+            void server
+                .close()
+                .then(() => database.end({ timeout: 5 }))
+                .finally(() => process.exit(0));
+        });
+}
 export async function main() {
     const stack = parseStackConfig(process.env, process.argv.slice(2));
+    if (stack.services.has("gates") || stack.services.has("keys"))
+        return runGates(stack);
     const hostRoot = resolveHostRoot();
     const paths = hostPaths(hostRoot);
     mkdirSync(paths.home, { recursive: true });
     mkdirSync(paths.tmp, { recursive: true });
-    mkdirSync(paths.tenants, { recursive: true });
     if (!existsSync(paths.config)) {
         throw new Error(`Missing host.json at ${paths.config}; run \`nylorun start\` first`);
     }
     if (!existsSync(paths.credentials)) {
         throw new Error(`Missing host-credentials.json at ${paths.credentials}; run \`nylorun start\` first`);
     }
-    // Tenants are Postgres schemas. Only a Host outside a container (a local
+    // The Tenant is a Postgres database. Only a Host outside a container (a local
     // development Host) may run without S2, on in-process streams.
     if (!stack.endpoints.databaseUrl) {
         throw new Error("NYLORUN_DATABASE_URL is required: the Postgres URL of the Session Store (`nylorun start` sets it)");
@@ -70,13 +139,20 @@ export async function main() {
     const config = loadJson(paths.config);
     const credentials = loadJson(paths.credentials);
     const logger = createHostLogger();
+    if (stack.deprecatedRole)
+        logger.warn("deprecated_flag", {
+            flag: "--role",
+            use: `--service ${describeServices(stack.services)}`,
+        });
     logger.info("host_stack_config", {
-        role: stack.role,
+        services: [...stack.services],
+        ...(stack.packing ? { packing: stack.packing } : {}),
+        ...(stack.services.has("loop")
+            ? { modelGate: stack.modelGate ? stack.modelGate.url : "in-process" }
+            : {}),
         mode: stack.listen ? "container" : "local",
         endpoints: describeEndpoints(stack.endpoints),
     });
-    // Tenants from the SQLite Runtime are not migrated: move them out of the way.
-    trashSqliteTenants(hostRoot, logger);
     const infra = createInfra(stack, { logger });
     const database = infra.database;
     if (!database)
@@ -89,46 +165,86 @@ export async function main() {
         hostConfig: config,
         logger,
         baseline,
+        ...(stack.delivery ? { delivery: stack.delivery } : {}),
     });
-    // One Durable Session Execution for every Tenant this process opens: Restate
-    // when its endpoints are set, else the in-process memory execution. An
-    // invocation for a Tenant that is not open here opens it on demand.
+    // The process's Durable Session Execution: Restate when its endpoints are set, else the
+    // in-process memory execution. An invocation that arrives while the Tenant opens waits for it.
     const hostExecution = createHostExecution({
         execution: infra.execution ?? createExecution(stack),
-        role: stack.role,
+        services: stack.services,
         resolve: (tenantId) => module.worker(tenantId),
         logger,
     });
     // Durable Streams: S2 when configured (always in container mode). Without
-    // them (a local Host only) each Tenant keeps in-process streams, whose history
+    // them (a local Host only) the Tenant keeps in-process streams, whose history
     // does not survive a restart.
     const streams = infra.streams;
-    const store = createPostgresTenantStore({
-        hostRoot,
-        sql: database,
-        configFor,
-        logger,
-        openRuntime: (tenantConfig, opened) => openTenantRuntime(tenantConfig, {
-            execution: hostExecution.tenantExecution,
-            ...(streams ? { streams } : {}),
-            ...opened,
-        }),
-    });
+    // With S2, the stream relay feeds the Tenant's streams from the record. Every process
+    // running core runs one, once the Tenant is open (the relay reads its id); the replication
+    // slot lets exactly one be active. Without S2, the Tenant relays its own commits.
+    let relay;
+    let relayLag;
+    if (streams && stack.services.has("core"))
+        await assertLogicalReplication(database);
+    const startRelay = (tenantId) => {
+        if (!streams || !stack.services.has("core") || relay)
+            return;
+        const source = createPgoutputSource({
+            connectionString: stack.endpoints.databaseUrl,
+            tenantId,
+            log: (message, fields) => logger.info(message, fields),
+        });
+        relay = createStreamRelay({
+            source,
+            record: createPostgresRecordReader(database, { tenantId }),
+            streams,
+            log: (message, fields) => logger.info(message, fields),
+        });
+        const status = relay.status;
+        relayLag = async () => ({ ...status(), ...(await lagOf(source)) });
+        relay.start();
+    };
+    // With the gates service the Tenant's vault-backed model calls, remote MCP calls and Action
+    // deliveries cross it, and this process never reads a model or MCP credential.
+    const modelGate = stack.modelGate
+        ? httpModelGate({ url: stack.modelGate.url, token: stack.modelGate.token })
+        : undefined;
+    const toolGate = stack.modelGate
+        ? httpToolGate({ url: stack.modelGate.url, token: stack.modelGate.token })
+        : undefined;
+    // With the keys service, vault writes and token signing cross it, and this process never
+    // reads the vault key (F4.2).
+    const keys = stack.keys ? httpKeys({ url: stack.keys.url, token: stack.keys.token }) : undefined;
+    const tenantSettings = stack.tenant ?? { name: "default", derivedPrincipals: ["project"] };
     const module = createTenantModule({
-        store,
+        open: createPostgresTenantOpener({
+            hostRoot,
+            sql: database,
+            create: {
+                ...(tenantSettings.id ? { tenantId: tenantSettings.id } : {}),
+                name: tenantSettings.name,
+                principals: hostPrincipals({
+                    adminKey: credentials.adminKey,
+                    derived: tenantSettings.derivedPrincipals,
+                }),
+            },
+            configFor,
+            logger,
+            openRuntime: (tenantConfig, opened) => openTenantRuntime(tenantConfig, {
+                execution: hostExecution.tenantExecution,
+                ...(modelGate ? { modelGate } : {}),
+                ...(toolGate ? { toolGate } : {}),
+                ...(keys ? { keys } : {}),
+                ...(streams ? { streams, hostRelay: true } : {}),
+                ...opened,
+            }),
+        }),
         logger,
-        // A new Tenant's basin is created with it (opening it repairs a failure);
-        // a deleted Tenant's sweep stops re-arming and its basin goes.
-        ...(streams
-            ? { onCreated: (tenantId) => createTenantStreams(streams, tenantId) }
-            : {}),
-        onDeleted: async (tenantId) => {
-            const failed = (await Promise.allSettled([
-                hostExecution.disarm(tenantId),
-                ...(streams ? [deleteTenantStreams(streams, tenantId)] : []),
-            ])).flatMap((result) => (result.status === "rejected" ? [result.reason] : []));
-            if (failed.length > 0)
-                throw new AggregateError(failed, "Deleted Tenant cleanup failed");
+        onOpen: (handle) => startRelay(handle.envelope.id),
+        relayStatus: async () => {
+            if (!relayLag)
+                throw new Error("no relay");
+            return relayLag();
         },
     });
     const options = {
@@ -140,29 +256,47 @@ export async function main() {
         coreVersion: coreVersion(),
         ...(stack.listen ? { listen: stack.listen } : {}),
         ...(stack.publicUrl ? { publicUrl: stack.publicUrl } : {}),
+        // Container mode (the local stack) allows browsers unless told not to; a Host started
+        // from host.json only when it says so. Without publishable keys nothing is reachable.
+        browserAccess: stack.browserAccess ?? (stack.listen ? true : config.browserAccess === true),
+        // The Admin API on its own listener: from the container environment, or from host.json
+        // (loopback, on the same host) when the Host runs outside a container.
+        ...(stack.operator
+            ? { operator: stack.operator }
+            : !stack.listen && typeof config.adminPort === "number"
+                ? { operator: { host: config.host, port: config.adminPort } }
+                : {}),
         ...(infra.readiness ? { readiness: infra.readiness } : {}),
         // SIGTERM and POST /v1/admin/host/shutdown both close the Host this way:
-        // stop the Worker, close the Tenants, then end the infrastructure clients.
+        // stop the Worker and the relay, close the Tenant, then end the infrastructure clients.
         shutdown: {
-            beforeTenants: () => hostExecution.stop(),
+            beforeTenants: async () => {
+                await hostExecution.stop();
+                await relay?.stop();
+            },
             afterTenants: () => infra.close(),
         },
     };
     const host = createHost(options);
-    // Startup order. The Worker starts first: opening a Tenant arms its sweep
+    // Startup order. The Worker starts first: opening the Tenant arms its sweep
     // through Restate's ingress, which answers 404 until a Worker has registered
-    // the services, so no Tenant may open before `start` (the listener opens
-    // Tenants on demand). An api-role process serves no Worker endpoint, so it
-    // can open Tenants only once some Worker process has registered. Container
-    // mode runs one `--role all` process, which registers here. Then the
-    // listener starts (and marks discovery done), and every listed Tenant's
-    // sweep is re-armed, which recovers wakes lost with Restate's state (§14.8).
+    // the services, so the Tenant may not open before `start`. A core-only
+    // process serves no Worker endpoint, so it can open the Tenant only once some
+    // Worker process has registered (until then the open is retried). Container
+    // mode runs one `--service core,loop` process, which registers here. Then the
+    // listener starts and opens the Tenant (migrating its database, creating it on
+    // first start); opening arms its sweep, which recovers wakes lost with
+    // Restate's state (§14.8), and starts the stream relay. A Tenant that cannot
+    // be opened leaves the Host listening but not ready, with the cause in
+    // `/v1/admin/status`.
     try {
         await hostExecution.start();
         await host.listen();
     }
     catch (error) {
         await hostExecution.stop().catch(() => undefined);
+        await module.close().catch(() => undefined);
+        await relay?.stop().catch(() => undefined);
         await infra.close();
         if (error instanceof HostListenError) {
             logger.error("listen_failed", {
@@ -174,16 +308,13 @@ export async function main() {
         }
         throw error;
     }
-    const tenants = await module.list();
-    await hostExecution
-        .armAll(tenants.filter((t) => t.state === "open").map((t) => t.id))
-        .catch((error) => logger.warn("tenant_sweeps_not_armed", {
-        message: error instanceof Error ? error.message : String(error),
-    }));
+    const tenant = module.tenant();
     logger.info("host_ready", {
         url: host.url,
         hostId: config.hostId,
-        tenants: tenants.length,
+        tenantId: tenant.id,
+        tenant: tenant.state,
+        ...(tenant.cause ? { cause: tenant.cause.code } : {}),
     });
     void host.closed.then(() => process.exit(0));
     for (const signal of ["SIGINT", "SIGTERM"]) {

package/dist/host/stack-config.d.ts

@@ -1,21 +1,20 @@
-/**
- * Stack configuration for the Runtime process, parsed from an environment
- * snapshot and argv. Pure: `host/main.ts` passes the process environment and
- * arguments; nothing here reads ambient state.
- *
- * Two listen modes:
- * - **local** (no `NYLORUN_LISTEN_*` / `NYLORUN_ALLOWED_HOSTS`): the Host binds
- *   what `host.json` names, loopback only unless `allowNonLoopback`.
- * - **container**: the Host binds `NYLORUN_LISTEN_HOST:NYLORUN_LISTEN_PORT`
- *   (default `0.0.0.0:4000`) and accepts only the `Host` headers in
- *   `NYLORUN_ALLOWED_HOSTS`, plus the loopback forms of the listen port for
- *   probes from inside the container.
- *
- * The Postgres, Restate and S2 endpoints are parsed and validated here;
- * `infra/*` builds the clients from them.
- */
+/** A Runtime service this release has. */
+export type RuntimeService = "core" | "loop" | "gates" | "keys";
+export type RuntimeServices = ReadonlySet<RuntimeService>;
+export declare const RUNTIME_SERVICES: readonly RuntimeService[];
+/** Where the gates service listens by default. */
+export declare const DEFAULT_GATES_LISTEN_PORT = 4100;
+/** What a process runs without `--service`: core and loop, as `--role all` did. */
+export declare const DEFAULT_SERVICES: RuntimeServices;
+/** The deprecated `--role` values, and the services each stands for. */
 export type RuntimeRole = "api" | "worker" | "all";
-export declare const RUNTIME_ROLES: readonly RuntimeRole[];
+/** `services` as the `--service` value that selects them, e.g. `core,loop`. */
+export declare function describeServices(services: RuntimeServices): string;
+export interface ServiceSelection {
+    services: RuntimeServices;
+    /** Set when the process was started with the deprecated `--role`. */
+    deprecatedRole?: RuntimeRole;
+}
 export declare const DEFAULT_CONTAINER_LISTEN_HOST = "0.0.0.0";
 export declare const DEFAULT_CONTAINER_LISTEN_PORT = 4000;
 export interface ContainerListen {
@@ -43,8 +42,43 @@ export interface StackEndpoints {
      */
     restateIdentityKeys?: string[];
 }
+/** The gates service's listener and the token callers must present (`NYLORUN_GATES_*`). */
+export interface GatesConfig {
+    listen: ContainerListen;
+    /** `NYLORUN_GATES_TOKEN`: the bearer the loop presents; at least 32 bytes as hex. */
+    token: string;
+}
+/** Where the loop reaches the gates service (`NYLORUN_GATES_URL`, `NYLORUN_GATES_TOKEN`). */
+export interface ModelGateEndpoint {
+    url: string;
+    token: string;
+}
 export interface StackConfig {
-    role: RuntimeRole;
+    /** The Runtime services this process runs. */
+    services: RuntimeServices;
+    /** Present when the process runs gates. */
+    gates?: GatesConfig;
+    /**
+     * Present when the process runs core or loop and `NYLORUN_GATES_URL` is set: its model calls,
+     * remote MCP calls and Action deliveries cross the gates service. Required for loop in
+     * container mode; outside a container (a development Host, tests) the loop may make them in
+     * its own process.
+     */
+    modelGate?: ModelGateEndpoint;
+    /**
+     * Present when the process runs core or loop and `NYLORUN_KEYS_URL`, or else
+     * `NYLORUN_GATES_URL`, is set (F4.2): its vault writes and token signing cross the gateway's
+     * keys service, with `NYLORUN_GATES_TOKEN`, and it never reads the vault key. A loop in a
+     * container always has it, since it requires `NYLORUN_GATES_URL`.
+     */
+    keys?: ModelGateEndpoint;
+    /**
+     * How the stack packs services into containers (`NYLORUN_PACKING`), for the startup log:
+     * `combined` (the local stack: runtime and gateway) or `split` (one container per service).
+     */
+    packing?: "combined" | "split";
+    /** Set when the process was started with the deprecated `--role` (logged at startup). */
+    deprecatedRole?: RuntimeRole;
     /** Present in container mode; absent means bind what host.json names. */
     listen?: ContainerListen;
     endpoints: StackEndpoints;
@@ -54,12 +88,50 @@ export interface StackConfig {
      * means nothing outside the container.
      */
     publicUrl?: string;
+    /**
+     * `NYLORUN_BROWSER_ACCESS` (`on` or `off`): whether browser requests may reach Tenant
+     * routes. Absent means the Host's default (on in container mode).
+     */
+    browserAccess?: boolean;
+    /**
+     * The operator listener in container mode (`NYLORUN_ADMIN_LISTEN_PORT`, `…_HOST`,
+     * `…_ALLOWED_HOSTS`). Absent: one listener serves the Admin API and the Tenant API.
+     */
+    operator?: ContainerListen;
+    /**
+     * How the Runtime may call Action endpoints: `NYLORUN_ENDPOINT_LOOPBACK=docker-host` (the local
+     * stack: `localhost` means the machine that runs Docker), `NYLORUN_ENDPOINT_PRIVATE`
+     * (`allow` or `refuse`) and `NYLORUN_ENDPOINT_HTTP` (`allow` or `refuse`).
+     */
+    delivery?: {
+        loopback?: "docker-host";
+        privateAddresses?: "allow" | "refuse";
+        allowHttp?: boolean;
+    };
+    /** Present when the process runs core or loop: who its Tenant is on first start. */
+    tenant?: TenantSettings;
+}
+/**
+ * The Tenant a Host creates when its database holds none (tenancy.md §4). Later starts open the
+ * Tenant the database holds; only principals missing from it are added.
+ */
+export interface TenantSettings {
+    /** `NYLORUN_TENANT_ID`: the new Tenant's id. Default: a new id. */
+    id?: string;
+    /** `NYLORUN_TENANT_NAME`. Default `default`. */
+    name: string;
+    /**
+     * `NYLORUN_DERIVED_PRINCIPALS`: comma-separated application principals whose keys the admin
+     * key derives (`deriveTenantKey`). Default `project`.
+     */
+    derivedPrincipals: readonly string[];
 }
 export declare class StackConfigError extends Error {
     constructor(message: string);
 }
 type EnvSnapshot = Readonly<Record<string, string | undefined>>;
-export declare function parseRole(argv: readonly string[]): RuntimeRole;
+/** Parses `--service a,b` (or the deprecated `--role`); throws `StackConfigError`. */
+export declare function parseServices(argv: readonly string[]): ServiceSelection;
 /**
  * Normalize one `Host` header allowlist entry to lowercase `name:port`.
  * IPv6 literals must be bracketed (`[::1]:4000`).