@clawscarf/cli 0.1.0-alpha.1

package/scripts/deployment/networks.js

@@ -0,0 +1,354 @@
+import { lstat, readFile } from "node:fs/promises";
+import { isIP } from "node:net";
+import { join } from "node:path";
+import { z } from "zod";
+import ipaddr from "ipaddr.js";
+import { isDeepStrictEqual } from "node:util";
+import { LocalSetupError, run } from "./process.js";
+import { ensurePrivateFile, resourceNames } from "./state.js";
+const networkId = z.string().regex(/^[a-f0-9]{64}$/);
+const intentSchema = z.strictObject({
+    ownerId: z.uuid(),
+    purpose: z.enum(["companion", "runtime", "browser", "machine"]),
+    name: z.string().min(1),
+});
+const browserAddressSchema = z.strictObject({
+    subnet: z.string(),
+    address: z.ipv4(),
+    gateway: z.string().optional(),
+});
+const receiptSchema = intentSchema
+    .extend({
+    id: networkId,
+    isolated: browserAddressSchema.optional(),
+})
+    .refine((receipt) => ["browser", "machine"].includes(receipt.purpose) ===
+    (receipt.isolated !== undefined));
+const emptyOptions = z.record(z.string(), z.never()).nullable();
+const bridgeOptions = z
+    .strictObject({
+    "com.docker.network.enable_ipv4": z.literal("true").optional(),
+    "com.docker.network.enable_ipv6": z.literal("false").optional(),
+})
+    .nullable();
+const inspectedSchema = z.object({
+    Id: networkId,
+    Name: z.string(),
+    Driver: z.literal("bridge"),
+    Scope: z.literal("local"),
+    Internal: z.literal(false),
+    Ingress: z.literal(false),
+    Attachable: z.boolean(),
+    EnableIPv6: z.literal(false),
+    Options: bridgeOptions,
+    Labels: z.record(z.string(), z.string()),
+    IPAM: z.object({
+        Driver: z.literal("default"),
+        Options: emptyOptions,
+        Config: z
+            .array(z.object({ Gateway: z.string().optional() }))
+            .refine((configs) => configs.some((config) => config.Gateway && isIP(config.Gateway) === 4)),
+    }),
+});
+const browserInspectedSchema = inspectedSchema.extend({
+    Internal: z.literal(true),
+    Attachable: z.literal(false),
+    Options: z.strictObject({
+        "com.docker.network.enable_ipv4": z.literal("true").optional(),
+        "com.docker.network.enable_ipv6": z.literal("false").optional(),
+        "com.docker.network.bridge.gateway_mode_ipv4": z.literal("isolated"),
+    }),
+    IPAM: z.object({
+        Driver: z.literal("default"),
+        Options: emptyOptions,
+        Config: z.tuple([
+            z.strictObject({
+                Subnet: z.string(),
+                Gateway: z.string().optional(),
+                IPRange: z.literal("").optional(),
+                AuxiliaryAddresses: z.record(z.string(), z.never()).optional(),
+            }),
+        ]),
+    }),
+});
+function reservedAddress(config) {
+    try {
+        if (!ipaddr.IPv4.isValidCIDRFourPartDecimal(config.Subnet))
+            changed();
+        const [base, prefix] = ipaddr.IPv4.parseCIDR(config.Subnet);
+        const first = ipaddr.IPv4.networkAddressFromCIDR(config.Subnet);
+        const last = ipaddr.IPv4.broadcastAddressFromCIDR(config.Subnet);
+        if (base.toString() !== first.toString() ||
+            prefix > 29 ||
+            first.range() !== "private" ||
+            last.range() !== "private")
+            changed();
+        // The final usable host avoids Docker's usual low-address dynamic allocation.
+        const bytes = last.toByteArray();
+        const end = bytes[3];
+        if (end === undefined || end < 1)
+            changed();
+        bytes[3] = end - 1;
+        const address = ipaddr.fromByteArray(bytes).toString();
+        if (config.Gateway) {
+            if (!ipaddr.IPv4.isValidFourPartDecimal(config.Gateway))
+                changed();
+            const gateway = ipaddr.IPv4.parse(config.Gateway);
+            if (!gateway.match([first, prefix]) ||
+                [first.toString(), last.toString(), address].includes(gateway.toString()))
+                changed();
+        }
+        return {
+            subnet: config.Subnet,
+            address,
+            ...(config.Gateway !== undefined ? { gateway: config.Gateway } : {}),
+        };
+    }
+    catch {
+        changed();
+    }
+}
+function changed() {
+    throw new LocalSetupError("network_identity_changed", "A required network or its private ownership record differs from this installation. Inspect it before continuing; setup will not adopt or replace it.");
+}
+function uncertain(purpose) {
+    throw new LocalSetupError("network_outcome_unknown", `Creation of this installation's ${purpose} network was attempted but its owned network is absent. Inspect Docker network capacity and this installation before continuing; setup will not automatically allocate it again.`);
+}
+async function record(path) {
+    try {
+        const metadata = await lstat(path);
+        if (!metadata.isFile() ||
+            (metadata.mode & 0o077) !== 0 ||
+            metadata.uid !== process.getuid?.())
+            changed();
+        return JSON.parse(await readFile(path, "utf8"));
+    }
+    catch (error) {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT")
+            return undefined;
+        if (error instanceof LocalSetupError)
+            throw error;
+        changed();
+    }
+}
+function equalIntent(actual, expected) {
+    return (actual.ownerId === expected.ownerId &&
+        actual.purpose === expected.purpose &&
+        actual.name === expected.name);
+}
+async function records(directory, intent) {
+    const base = join(directory, "private", `network-${intent.purpose}`);
+    const intentPath = `${base}-create.json`;
+    const receiptPath = `${base}-receipt.json`;
+    const pending = await record(intentPath);
+    const rawReceipt = await record(receiptPath);
+    if (pending !== undefined) {
+        const parsed = intentSchema.safeParse(pending);
+        if (!parsed.success || !equalIntent(parsed.data, intent))
+            changed();
+    }
+    let receipt;
+    if (rawReceipt !== undefined) {
+        const parsed = receiptSchema.safeParse(rawReceipt);
+        if (!parsed.success ||
+            !equalIntent(parsed.data, intent) ||
+            pending === undefined)
+            changed();
+        receipt = parsed.data;
+    }
+    return { pending: pending !== undefined, receipt, intentPath, receiptPath };
+}
+async function observe(intent, command) {
+    try {
+        const output = await command("docker", [
+            "network",
+            "ls",
+            "--no-trunc",
+            "--format",
+            '{"id":{{json .ID}},"name":{{json .Name}}}',
+        ]);
+        const rows = output.trim()
+            ? output
+                .trim()
+                .split("\n")
+                .map((line) => z
+                .object({ id: networkId, name: z.string() })
+                .parse(JSON.parse(line)))
+            : [];
+        const matching = rows.filter((row) => row.name === intent.name);
+        if (matching.length > 1)
+            changed();
+        const found = matching[0];
+        if (!found)
+            return undefined;
+        const format = '{"Id":{{json .Id}},"Name":{{json .Name}},"Driver":{{json .Driver}},"Scope":{{json .Scope}},"Internal":{{json .Internal}},"Ingress":{{json .Ingress}},"Attachable":{{json .Attachable}},"EnableIPv6":{{json .EnableIPv6}},"Options":{{json .Options}},"Labels":{{json .Labels}},"IPAM":{{json .IPAM}}}';
+        const raw = JSON.parse(await command("docker", [
+            "network",
+            "inspect",
+            "--format",
+            format,
+            found.id,
+        ]));
+        const browser = ["browser", "machine"].includes(intent.purpose)
+            ? browserInspectedSchema.safeParse(raw)
+            : undefined;
+        const result = browser ?? inspectedSchema.safeParse(raw);
+        if (!result.success)
+            changed();
+        const network = result.data;
+        if (network.Id !== found.id ||
+            network.Name !== intent.name ||
+            network.Labels["clawscarf.installation"] !== intent.ownerId ||
+            network.Labels["clawscarf.network-purpose"] !== intent.purpose ||
+            network.Attachable !== (intent.purpose === "runtime"))
+            changed();
+        return {
+            id: network.Id,
+            ...(browser?.success
+                ? { isolated: reservedAddress(browser.data.IPAM.Config[0]) }
+                : {}),
+        };
+    }
+    catch (error) {
+        if (error instanceof LocalSetupError &&
+            error.code === "network_identity_changed")
+            throw error;
+        throw new LocalSetupError("network_lookup_incomplete", "Docker network ownership could not be completely observed. No absence or allocation decision was made; inspect Docker before retrying.");
+    }
+}
+function intents(state) {
+    const names = resourceNames(state);
+    return [
+        {
+            ownerId: state.ownerId,
+            purpose: "companion",
+            name: `${names.project}_default`,
+        },
+        { ownerId: state.ownerId, purpose: "runtime", name: names.sandbox },
+        ...(state.input.browser
+            ? [
+                {
+                    ownerId: state.ownerId,
+                    purpose: "browser",
+                    name: `${names.project}_browser`,
+                },
+                {
+                    ownerId: state.ownerId,
+                    purpose: "machine",
+                    name: `${names.project}_machine`,
+                },
+            ]
+            : []),
+    ];
+}
+/** Caller holds the installation lock. Reserve actual networks before volumes or services. */
+export async function ensureLocalNetworks(directory, state, command = run) {
+    for (const intent of intents(state)) {
+        const saved = await records(directory, intent);
+        let observed = await observe(intent, command);
+        let id = observed?.id;
+        if (!id && saved.pending)
+            uncertain(intent.purpose);
+        if (saved.receipt &&
+            (id !== saved.receipt.id ||
+                !isDeepStrictEqual(observed?.isolated, saved.receipt.isolated)))
+            changed();
+        if (id && !saved.pending)
+            changed();
+        if (!id) {
+            await ensurePrivateFile(saved.intentPath, JSON.stringify(intent) + "\n");
+            let returnedId;
+            try {
+                returnedId = (await command("docker", [
+                    "network",
+                    "create",
+                    "--driver",
+                    "bridge",
+                    ...(intent.purpose === "runtime" ? ["--attachable"] : []),
+                    ...(["browser", "machine"].includes(intent.purpose)
+                        ? [
+                            "--internal",
+                            "--opt",
+                            "com.docker.network.bridge.gateway_mode_ipv4=isolated",
+                        ]
+                        : []),
+                    "--label",
+                    `clawscarf.installation=${intent.ownerId}`,
+                    "--label",
+                    `clawscarf.network-purpose=${intent.purpose}`,
+                    intent.name,
+                ])).trim();
+            }
+            catch {
+                // A lost response may follow allocation. Reconcile exact observed ownership only.
+            }
+            observed = await observe(intent, command);
+            id = observed?.id;
+            if (!id)
+                uncertain(intent.purpose);
+            if (returnedId !== undefined && returnedId !== id)
+                changed();
+        }
+        if (!saved.receipt)
+            await ensurePrivateFile(saved.receiptPath, JSON.stringify({
+                ...intent,
+                id,
+                ...(observed?.isolated ? { isolated: observed.isolated } : {}),
+            }) + "\n");
+    }
+}
+/** Read-only startup check: never allocate or repair a missing reservation. */
+export async function verifyLocalNetworks(directory, state, command = run) {
+    for (const intent of intents(state)) {
+        const saved = await records(directory, intent);
+        if (!saved.receipt)
+            throw new LocalSetupError("network_unprepared", "Required network reservations are not confirmed. Inspect and resume preparation before starting this installation.");
+        const observed = await observe(intent, command);
+        if (!observed)
+            uncertain(intent.purpose);
+        if (observed.id !== saved.receipt.id ||
+            !isDeepStrictEqual(observed.isolated, saved.receipt.isolated))
+            changed();
+    }
+}
+/** Re-observe the isolated reservation before generating its static address and source ACL. */
+export async function observedBrowserAddress(directory, state, command = run) {
+    const intent = intents(state).find((value) => value.purpose === "browser");
+    if (!intent)
+        return undefined;
+    const saved = await records(directory, intent);
+    if (!saved.receipt)
+        throw new LocalSetupError("network_unprepared", "The browser network reservation is not confirmed. Resume preparation before configuring browser access.");
+    const observed = await observe(intent, command);
+    if (!observed)
+        uncertain(intent.purpose);
+    if (observed.id !== saved.receipt.id ||
+        !observed.isolated ||
+        !isDeepStrictEqual(observed.isolated, saved.receipt.isolated))
+        changed();
+    return observed.isolated.address;
+}
+/** Separate private bridge: only the browser node, its ingress and DNS attach. */
+export async function observedBrowserMachineAddresses(directory, state, command = run) {
+    const intent = intents(state).find((value) => value.purpose === "machine");
+    if (!intent)
+        return undefined;
+    const saved = await records(directory, intent);
+    if (!saved.receipt)
+        throw new LocalSetupError("network_unprepared", "The private browser-node network is not prepared.");
+    const observed = await observe(intent, command);
+    if (!observed ||
+        observed.id !== saved.receipt.id ||
+        !observed.isolated ||
+        !isDeepStrictEqual(observed.isolated, saved.receipt.isolated))
+        changed();
+    const bytes = ipaddr.IPv4.parse(observed.isolated.address).toByteArray();
+    const last = bytes[3];
+    if (last === undefined || last < 3)
+        changed();
+    const ingress = [...bytes.slice(0, 3), last - 1].join(".");
+    const dns = [...bytes.slice(0, 3), last - 2].join(".");
+    if ([ingress, dns].includes(observed.isolated.gateway ?? ""))
+        changed();
+    return { node: observed.isolated.address, ingress, dns };
+}

package/scripts/deployment/policy.js

@@ -0,0 +1,56 @@
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { parseDocument } from "yaml";
+import { z } from "zod";
+import { LocalSetupError } from "./process.js";
+import { ensurePrivateFile } from "./state.js";
+export function initialRuntimePolicy(source, models, connections) {
+    const document = parseDocument(source);
+    if (document.errors.length || document.warnings.length)
+        throw new LocalSetupError("invalid_runtime_policy", "The shipped runtime policy is invalid.");
+    const raw = document.toJS();
+    const parsed = z
+        .looseObject({ network_policies: z.record(z.string(), z.never()) })
+        .safeParse(raw);
+    if (!parsed.success)
+        throw new LocalSetupError("invalid_runtime_policy", "Initial model setup requires the shipped deny-by-default runtime policy.");
+    return {
+        ...parsed.data,
+        network_policies: {
+            ...(connections
+                ? {
+                    connections_broker: {
+                        name: "Connections broker",
+                        endpoints: [
+                            {
+                                host: connections.network.host,
+                                port: connections.network.port,
+                                tls: "skip",
+                            },
+                        ],
+                        binaries: [{ path: connections.network.binary }],
+                    },
+                }
+                : {}),
+            ...(models
+                ? {
+                    model_gateway: {
+                        name: "Model gateway",
+                        endpoints: [
+                            {
+                                host: models.network.host,
+                                port: models.network.port,
+                                tls: "skip",
+                            },
+                        ],
+                        binaries: [{ path: models.network.binary }],
+                    },
+                }
+                : {}),
+        },
+    };
+}
+export async function prepareRuntimePolicy(directory, models, connections) {
+    const policy = initialRuntimePolicy(await readFile(new URL("../../deploy/openshell/policy.yaml", import.meta.url), "utf8"), models, connections);
+    await ensurePrivateFile(join(directory, "private/runtime-policy.json"), JSON.stringify(policy));
+}

package/scripts/deployment/preflight.js

@@ -0,0 +1,124 @@
+import { constants } from "node:fs";
+import { access, stat } from "node:fs/promises";
+import { createServer } from "node:net";
+import { z } from "zod";
+import { resourceNames } from "./state.js";
+import { LocalSetupError, run } from "./process.js";
+/** A bind probe detects current conflicts; it does not reserve a port for startup. */
+export async function portAvailable(port, host) {
+    const server = createServer((socket) => {
+        socket.destroy();
+    });
+    try {
+        await new Promise((resolve, reject) => {
+            server.once("error", reject);
+            server.listen({ host, port, exclusive: true }, resolve);
+        });
+        return true;
+    }
+    catch (error) {
+        if (error instanceof Error &&
+            "code" in error &&
+            error.code === "EADDRINUSE")
+            return false;
+        throw new LocalSetupError("port_check_failed", `Cannot check local port ${String(port)}. Check local network permissions before retrying.`);
+    }
+    finally {
+        if (server.listening)
+            await new Promise((resolve, reject) => server.close((error) => {
+                if (error)
+                    reject(error);
+                else
+                    resolve();
+            }));
+    }
+}
+async function ownedListener(state, name, port, host, command) {
+    const services = {
+        database: "postgres",
+        native: "application",
+        nativeWidgets: "widgets",
+        application: "companion",
+        widgets: "companion",
+        management: "companion",
+        browser: "browser-relay",
+    };
+    const service = services[name] ?? name;
+    const project = resourceNames(state).project;
+    const ids = (await command("docker", [
+        "container",
+        "ls",
+        "--quiet",
+        "--no-trunc",
+        "--filter",
+        `label=clawscarf.installation=${state.ownerId}`,
+        "--filter",
+        `label=com.docker.compose.project=${project}`,
+        "--filter",
+        `label=com.docker.compose.service=${service}`,
+    ]))
+        .trim()
+        .split("\n")
+        .filter(Boolean);
+    if (ids.length !== 1 || !/^[a-f0-9]{64}$/.test(ids[0] ?? ""))
+        return false;
+    const id = ids[0];
+    if (!id)
+        return false;
+    const format = '{"running":{{json .State.Running}},"owner":{{json (index .Config.Labels "clawscarf.installation")}},"project":{{json (index .Config.Labels "com.docker.compose.project")}},"service":{{json (index .Config.Labels "com.docker.compose.service")}},"ports":{{json .NetworkSettings.Ports}}}';
+    const result = z
+        .object({
+        running: z.literal(true),
+        owner: z.literal(state.ownerId),
+        project: z.literal(project),
+        service: z.literal(service),
+        ports: z.record(z.string(), z
+            .array(z.object({ HostIp: z.string(), HostPort: z.string() }))
+            .nullable()),
+    })
+        .safeParse(JSON.parse(await command("docker", [
+        "container",
+        "inspect",
+        "--format",
+        format,
+        id,
+    ])));
+    return (result.success &&
+        Object.values(result.data.ports).some((bindings) => bindings?.some((binding) => binding.HostIp === host && binding.HostPort === String(port))));
+}
+/** Repeated starts may reuse listeners belonging to this installation only. */
+export async function verifyLocalPorts(state, command = run) {
+    const ports = {
+        ...state.input.ports,
+        ...(state.input.modelGateway
+            ? { models: state.input.modelGateway.port }
+            : {}),
+        ...(state.input.browser ? { browser: state.input.browser.port } : {}),
+    };
+    for (const [name, port] of Object.entries(ports)) {
+        const host = state.input.team.certificateFile &&
+            ["application", "widgets"].includes(name)
+            ? "0.0.0.0"
+            : "127.0.0.1";
+        if (await portAvailable(port, host))
+            continue;
+        if (await ownedListener(state, name, port, host, command))
+            continue;
+        throw new LocalSetupError("port_in_use", `The ${name} port ${host}:${String(port)} is already in use. Stop its current owner or select different ports for a new installation before retrying.`);
+    }
+}
+export async function verifyLocalExecutables(state) {
+    for (const [name, path] of [
+        ["OpenShell CLI", state.input.openshellCli],
+        ["OpenShell gateway", state.input.openshellGateway],
+    ]) {
+        try {
+            await access(path, constants.X_OK);
+            if (!(await stat(path)).isFile())
+                throw Error("Not an executable file.");
+        }
+        catch {
+            throw new LocalSetupError("executable_unavailable", `The configured ${name} executable is unavailable. Install the pinned executable and check its permissions before retrying.`);
+        }
+    }
+}