@clawscarf/cli 0.1.0-alpha.1

package/scripts/installation/prerequisites.js

@@ -0,0 +1,170 @@
+import { dirname, resolve } from "node:path";
+import { createInterface } from "node:readline";
+import { spawn } from "node:child_process";
+import { LocalSetupError, run } from "../deployment/process.js";
+import { portAvailable, verifyLocalPorts } from "../deployment/preflight.js";
+import { readState } from "../deployment/state.js";
+import { openshellGatewayImage, verifyRuntimeImage, } from "../deployment/images.js";
+import { installationSchema, } from "./configuration.js";
+import { readRuntime, assertReleaseCapabilities } from "./setup.js";
+import { readJson } from "./files.js";
+import { acquireRuntimeTools } from "./runtime.js";
+import { InstallationError } from "./errors.js";
+/** No configuration, credentials or cloud account is needed for these checks. */
+export async function checkHost(command = run) {
+    if (process.platform !== "darwin" || process.arch !== "arm64")
+        throw new InstallationError("unsupported_platform", "This release requires macOS on Apple Silicon with Docker Desktop.");
+    let os;
+    try {
+        os = await command("docker", ["info", "--format", "{{.OSType}}"]);
+    }
+    catch (error) {
+        const missing = error instanceof LocalSetupError &&
+            error.commandFailure?.reason === "spawn" &&
+            error.commandFailure.systemCode === "ENOENT";
+        throw new InstallationError("unavailable", missing
+            ? "Docker is not installed or is not on PATH. Install Docker Desktop, start it, then run configure again."
+            : "Cannot reach Docker. Start Docker Desktop and wait until it is running, then try again.");
+    }
+    if (os.trim() !== "linux")
+        throw new InstallationError("unsupported_platform", "Docker must run Linux containers.");
+    try {
+        await command("docker", ["compose", "version"]);
+    }
+    catch {
+        throw new InstallationError("unavailable", "Docker Compose is unavailable. Install or update Docker Desktop, then try again.");
+    }
+}
+/** Stream only Docker's layer progress, never registry error payloads or credentials. */
+export function pullImage(image, report, signal) {
+    return new Promise((resolve, reject) => {
+        const child = spawn("docker", ["pull", image], {
+            stdio: ["ignore", "pipe", "pipe"],
+            ...(signal ? { signal } : {}),
+            timeout: 30 * 60_000,
+        });
+        let last = 0;
+        let spawnFailed = false;
+        const readers = [child.stdout, child.stderr].map((stream) => createInterface({ input: stream }));
+        for (const reader of readers)
+            reader.on("line", (line) => {
+                if (/^[a-f0-9]{12}: (?:Pulling fs layer|Waiting|Downloading|Extracting|Verifying Checksum|Download complete|Pull complete)(?:\s+\[[=> ]*\]\s+[\d.]+[kMGT]?B\/[\d.]+[kMGT]?B)?$/.test(line) &&
+                    Date.now() - last >= 500) {
+                    report(`${image.split("@")[0] ?? image} — ${line}`);
+                    last = Date.now();
+                }
+            });
+        child.once("error", () => {
+            spawnFailed = true;
+        });
+        child.once("close", (code) => {
+            readers.forEach((reader) => {
+                reader.close();
+            });
+            if (signal?.aborted)
+                reject(signal.reason instanceof Error
+                    ? signal.reason
+                    : new Error("Download cancelled."));
+            else if (spawnFailed)
+                reject(new InstallationError("unavailable", "Image download could not start. Check Docker and try again."));
+            else if (code === 0)
+                resolve();
+            else
+                reject(new InstallationError("unavailable", `Could not download ${image}. Check network access and registry authentication, then run configure again.`));
+        });
+    });
+}
+export async function checkNewInstallationPorts(exposure) {
+    const ports = exposure.mode === "local"
+        ? [exposure.applicationPort, exposure.widgetPort]
+        : [
+            Number(new URL(exposure.applicationOrigin).port || 443),
+            Number(new URL(exposure.widgetOrigin).port || 443),
+        ];
+    const host = exposure.mode === "local" ? "127.0.0.1" : "0.0.0.0";
+    for (const port of ports)
+        if (!(await portAvailable(port, host)))
+            throw new InstallationError("unavailable", `Port ${host}:${String(port)} is already in use. Stop its current owner or choose different listener ports in configure.`);
+}
+/** Runs before cloud registration; tools and registry images may be fetched. */
+export async function checkInstallationPrerequisites(configFile, options = {}, command = run, pull = pullImage) {
+    const report = options.report ?? (() => { });
+    const config = installationSchema.parse(await readJson(configFile));
+    const base = dirname(resolve(configFile));
+    const { release, releaseFile } = await readRuntime(resolve(base, config.releaseFile));
+    assertReleaseCapabilities({ release }, config);
+    await acquireRuntimeTools(releaseFile, options);
+    report("Runtime tools verified");
+    let state;
+    try {
+        state = await readState(resolve(base, config.stateDirectory));
+    }
+    catch (error) {
+        if (!(error instanceof Error && "code" in error && error.code === "ENOENT"))
+            throw error;
+    }
+    if (state)
+        await verifyLocalPorts(state, command);
+    else {
+        await checkNewInstallationPorts(config.exposure);
+    }
+    if (config.packOperator)
+        await command(resolve(base, config.packOperator.pythonExecutable), ["-c", "from openshell.sandbox import SandboxClient"], { timeout: 10000 });
+    const images = [
+        ...new Set([
+            release.images.gateway,
+            release.images.companion,
+            release.images.openshellClient,
+            release.images.postgres,
+            openshellGatewayImage,
+            ...(config.models.mode === "litellm" && release.images.models
+                ? [release.images.models]
+                : []),
+            ...(config.browser.enabled && release.images.browser
+                ? [
+                    ...Object.values(release.images.browser),
+                    ...(release.images.relay ? [release.images.relay] : []),
+                ]
+                : []),
+        ]),
+    ];
+    const missing = [];
+    for (const image of images) {
+        options.signal?.throwIfAborted();
+        try {
+            await command("docker", ["image", "inspect", image]);
+        }
+        catch (error) {
+            if (!(error instanceof LocalSetupError &&
+                error.commandFailure?.reason === "exit" &&
+                error.commandFailure.exitCode === 1))
+                throw error;
+            missing.push(image);
+        }
+    }
+    if (missing.length) {
+        await checkHost(command);
+        const local = missing.filter((image) => image.startsWith("sha256:"));
+        if (local.length)
+            throw new InstallationError("unavailable", `Missing local development images: ${local.join(", ")}. Rebuild the development images and update the runtime definition; these image IDs cannot be downloaded.`);
+        if (!options.acquire)
+            throw new InstallationError("unavailable", `Missing images: ${missing.join(", ")}. Run configure to download them.`);
+        report(`Downloading ${String(missing.length)} required images (first setup can take several minutes):\n${missing.map((image) => `  ${image}`).join("\n")}`);
+        for (const image of missing) {
+            options.signal?.throwIfAborted();
+            report(`Downloading ${image}`);
+            await pull(image, report, options.signal);
+            await command("docker", ["image", "inspect", image]);
+        }
+    }
+    else
+        report(`All ${String(images.length)} required images are already available`);
+    await verifyRuntimeImage(release.images.gateway, command);
+    options.signal?.throwIfAborted();
+    return {
+        state: "prerequisites_available",
+        platform: `${process.platform}-${process.arch}`,
+        release: release.version,
+        images: images.length,
+    };
+}

package/scripts/installation/recipes/catalog.js

@@ -0,0 +1,55 @@
+import { readdir } from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { recipeSchema, recipesSchema } from "./definition.js";
+import { readJson } from "../files.js";
+import { modelCatalogSchema } from "../../models/catalog.js";
+import { openPack } from "../../packs/source.js";
+import { recipeModelRoutes } from "../models.js";
+import { InstallationError } from "../errors.js";
+const packageRoot = fileURLToPath(new URL("../../../", import.meta.url));
+export async function readRecipe(file) {
+    const recipe = recipeSchema.parse(await readJson(file));
+    return { ...recipe, runtime: resolve(dirname(file), recipe.runtime) };
+}
+/** Small, local package assets. Runtime images and executables are separate artifacts. */
+export async function installationCatalog(root = packageRoot) {
+    const entries = await readdir(join(root, "recipes"), { withFileTypes: true });
+    const recipes = recipesSchema.parse(await Promise.all(entries
+        .filter((entry) => entry.isDirectory())
+        .sort((a, b) => a.name.localeCompare(b.name))
+        .map((entry) => readRecipe(join(root, "recipes", entry.name, "recipe.json")))));
+    const modelCatalog = modelCatalogSchema.parse(await readJson(join(root, "deploy/models/catalog.json")));
+    const packEntries = await readdir(join(root, "packs"), {
+        withFileTypes: true,
+    });
+    const packs = await Promise.all(packEntries
+        .filter((entry) => entry.isDirectory())
+        .sort((a, b) => a.name.localeCompare(b.name))
+        .map(async (entry) => {
+        const directory = join(root, "packs", entry.name);
+        const pack = await openPack(directory);
+        return {
+            id: pack.manifest.id,
+            directory,
+            members: pack.manifest.members.map((member) => member.id),
+        };
+    }));
+    if (new Set(packs.map((pack) => pack.id)).size !== packs.length)
+        throw new InstallationError("invalid_configuration", "Pack IDs must be unique.");
+    for (const recipe of recipes)
+        validateRecipe(recipe, { packs, modelCatalog });
+    return { recipes, packs, modelCatalog };
+}
+export function validateRecipe(recipe, catalog) {
+    recipeModelRoutes(recipe.models, catalog.modelCatalog);
+    const selected = new Set();
+    for (const entry of recipe.packs) {
+        const pack = catalog.packs.find((pack) => pack.id === entry.id);
+        for (const member of entry.members) {
+            if (!pack?.members.includes(member) || selected.has(member))
+                throw new InstallationError("invalid_configuration", `Recipe ${recipe.id} selects a missing or duplicate pack agent: ${entry.id}/${member}.`);
+            selected.add(member);
+        }
+    }
+}

package/scripts/installation/recipes/definition.js

@@ -0,0 +1,32 @@
+import { z } from "zod";
+import { installationSchema } from "../configuration.js";
+/** Recipes are CLI-owned defaults, never executable installation hooks. */
+export const recipeSchema = z.strictObject({
+    id: z.string().regex(/^[a-z][a-z0-9-]{0,63}$/),
+    version: z.string().regex(/^\d+\.\d+\.\d+(?:-[A-Za-z0-9.-]+)?$/),
+    runtime: z.string().min(1),
+    name: z.string().min(1).max(100),
+    description: z.string().min(1).max(500),
+    maturity: z.enum(["example", "supported"]),
+    defaults: z.strictObject({
+        resources: installationSchema.shape.resources,
+        browser: installationSchema.shape.browser,
+        connections: z.strictObject({ enabled: z.boolean() }).optional(),
+    }),
+    models: z.strictObject({
+        model: z.string().min(1),
+        provider: z.string().min(1),
+        reasoning: z.enum(["low", "medium", "high"]).optional(),
+    }),
+    packs: z
+        .array(z.strictObject({
+        id: z.string().regex(/^[a-z][a-z0-9-]{0,63}$/),
+        members: installationSchema.shape.packs.element.shape.members,
+    }))
+        .max(32)
+        .default([]),
+});
+export const recipesSchema = z
+    .array(recipeSchema)
+    .max(32)
+    .refine((recipes) => new Set(recipes.map((recipe) => recipe.id)).size === recipes.length, "Recipe IDs must be unique.");

package/scripts/installation/reconfigure.js

@@ -0,0 +1,255 @@
+import { applyConnectionSettings } from "../deployment/connection-settings.js";
+import { readFile } from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+import { isDeepStrictEqual } from "node:util";
+import { z } from "zod";
+import { compose } from "../deployment/compose.js";
+import { setRuntimeCredentialModels } from "../models/credentials.js";
+import { configurationChanges, resolveConfigurationInputs, } from "./configure.js";
+import { ModelConfigurationError } from "../../runtime/model-contract.js";
+import { loadGatewayConfiguration, prepareModelGateway, } from "../deployment/model-gateway.js";
+import { readInitialConnectionsEndpoint, loadInitialConnections, prepareInitialConnections, } from "../deployment/connections.js";
+import { configureStoppedRuntimeModels } from "../models/runtime.js";
+import { loadInitialModels } from "../deployment/models.js";
+import { readState, resourceNames, withInstallationLock, writePrivate, } from "../deployment/state.js";
+import { requireNoUpgrade } from "../deployment/upgrade-state.js";
+import { run } from "../deployment/process.js";
+import { selectionSchema } from "./packs.js";
+import { installationSchema } from "./configuration.js";
+import { resolveInstallation } from "./resolve.js";
+import { readJson, fingerprint } from "./files.js";
+import { InstallationError } from "./errors.js";
+function unsupported(message) {
+    throw new InstallationError("change_unsupported", message);
+}
+/** Deliberately bounded: initial preparation is never used to replace retained state. */
+export async function planSettingsChange(configFile, reapply) {
+    const config = installationSchema.parse(await readJson(configFile));
+    const directory = resolve(dirname(resolve(configFile)), config.stateDirectory);
+    const state = await readState(directory);
+    const old = state.input;
+    const desired = await resolveInstallation(configFile, [
+        old.ports.controller,
+        old.ports.management,
+        old.ports.native,
+        old.ports.nativeWidgets,
+        old.ports.database,
+        old.browser?.port ?? 65534,
+        old.modelGateway?.port ?? 65533,
+    ]);
+    const next = desired.input;
+    const previousPacks = selectionSchema.parse(await readJson(join(directory, "packs.json")));
+    if (!desired.packSelection.python && previousPacks.python)
+        desired.packSelection.python = previousPacks.python;
+    const { models: _oldModels, modelGateway: oldGateway, connections: oldConnections, ...oldFixed } = old;
+    const { models: _nextModels, modelGateway: nextGateway, connections: nextConnections, ...nextFixed } = next;
+    if (!isDeepStrictEqual(oldFixed, nextFixed) ||
+        !isDeepStrictEqual(await readJson(join(directory, "release.json")), desired.release))
+        unsupported("Settings changes cannot replace the release, identity, addresses, access, resources or execution protection.");
+    if (Boolean(oldGateway) !== Boolean(nextGateway) ||
+        (oldGateway &&
+            nextGateway &&
+            (oldGateway.image !== nextGateway.image ||
+                oldGateway.port !== nextGateway.port)))
+        unsupported("Switching between bundled and external model gateways requires a deployment change.");
+    if (!nextGateway) {
+        const before = await loadInitialModels(old.models);
+        const after = await loadInitialModels(next.models);
+        if (!before || !after || !isDeepStrictEqual(before.network, after.network))
+            unsupported("Changing the external model gateway address requires updating its protected network policy.");
+    }
+    const previousEndpoint = await readInitialConnectionsEndpoint(directory);
+    const connections = await loadInitialConnections(next.connections);
+    await prepareInitialConnections(directory, connections, {
+        state: { ...state, input: next },
+        apply: false,
+    });
+    const modelConfiguration = nextGateway
+        ? (await loadGatewayConfiguration(nextGateway.configurationFile, nextGateway.upstreamEnvironmentFile)).configuration
+        : (await loadInitialModels(next.models))?.configuration;
+    const pending = z
+        .strictObject({
+        ownerId: z.literal(state.ownerId),
+        settingsPending: z.string().optional(),
+        settingsCandidate: z.string().optional(),
+        settingsReapply: z.enum(["models", "connections"]).optional(),
+    })
+        .parse(await readJson(join(directory, "prepared.json")));
+    if (pending.settingsPending &&
+        pending.settingsPending !== desired.fingerprint)
+        unsupported("Finish the pending settings change with its original candidate before selecting another configuration.");
+    if (pending.settingsReapply && reapply && pending.settingsReapply !== reapply)
+        unsupported("Resume the pending change before selecting another capability to reapply.");
+    reapply ??= pending.settingsReapply;
+    const scopes = await configurationChanges(installationSchema.parse(await readJson(join(directory, "settings.json"))), resolveConfigurationInputs(config, dirname(resolve(configFile))));
+    if (reapply)
+        scopes[reapply] = true;
+    return {
+        directory,
+        state,
+        desired,
+        previousEndpoint,
+        scopes,
+        reapply,
+        changes: {
+            models: modelConfiguration
+                ? {
+                    enabled: modelConfiguration.models
+                        .filter((model) => model.enabled)
+                        .map((model) => model.id),
+                    default: modelConfiguration.defaultModel,
+                    reasoning: modelConfiguration.thinkingDefault ?? "unchanged",
+                }
+                : undefined,
+            connections: {
+                from: oldConnections?.mode ?? "disabled",
+                to: nextConnections?.mode ?? "disabled",
+            },
+            packs: {
+                selected: desired.packSelection.packs.flatMap((pack) => pack.members),
+                removed: previousPacks.packs
+                    .flatMap((pack) => pack.members)
+                    .filter((member) => !desired.packSelection.packs.some((pack) => pack.members.includes(member))),
+            },
+        },
+        fingerprint: fingerprint(JSON.stringify({
+            scopes,
+            reapply,
+            desired: desired.fingerprint,
+            state,
+            previousPacks,
+            accepted: await readFile(join(directory, "inputs.sha256"), "utf8"),
+            pending: pending.settingsPending ?? null,
+        })),
+        resuming: !!pending.settingsPending,
+    };
+}
+/** Explicit retained capability changes; native users and individual model overrides remain native-owned. */
+export async function reconfigureInstallation(configFile, expectedFingerprint, reapply) {
+    const planned = await planSettingsChange(configFile, reapply);
+    return withInstallationLock(planned.directory, async () => {
+        const checked = await planSettingsChange(configFile, reapply);
+        if (checked.fingerprint !== expectedFingerprint)
+            throw new InstallationError("stale_plan", "Settings changed after preview. Review them again.");
+        const { directory, state, desired, scopes } = checked;
+        if (!checked.resuming && !Object.values(scopes).some(Boolean))
+            return { state: "unchanged", restartRequired: false, directory };
+        await requireNoUpgrade(directory);
+        const names = resourceNames(state);
+        for (const filter of [
+            `label=com.docker.compose.project=${names.project}`,
+            `volume=${names.volume}`,
+        ])
+            if ((await run("docker", [
+                "container",
+                "ls",
+                "--quiet",
+                "--filter",
+                filter,
+            ])).trim())
+                unsupported("Stop this installation before applying settings.");
+        const volume = z
+            .object({
+            Name: z.literal(names.volume),
+            Labels: z.record(z.string(), z.string()),
+        })
+            .parse(JSON.parse(await run("docker", [
+            "volume",
+            "inspect",
+            "--format",
+            '{"Name":{{json .Name}},"Labels":{{json .Labels}}}',
+            names.volume,
+        ])));
+        if (volume.Labels["clawscarf.installation"] !== state.ownerId)
+            unsupported("The native home volume does not belong to this installation.");
+        const prepared = join(directory, "prepared.json");
+        let models = await loadInitialModels(state.input.models);
+        const gateway = desired.input.modelGateway;
+        const loaded = gateway
+            ? await loadGatewayConfiguration(gateway.configurationFile, gateway.upstreamEnvironmentFile)
+            : undefined;
+        if (!gateway)
+            models = await loadInitialModels(desired.input.models);
+        if (!models || (loaded && !loaded.configuration.defaultModel))
+            throw new InstallationError("invalid_configuration", "Model settings and an enabled default are required.");
+        if (loaded && loaded.configuration.defaultModel)
+            models.configuration = {
+                ...loaded.configuration,
+                defaultModel: loaded.configuration.defaultModel,
+                baseUrl: models.configuration.baseUrl,
+            };
+        // Check the packaged helper and native validation before changing retained service files.
+        if (scopes.models)
+            await configureStoppedRuntimeModels({
+                image: state.input.runtimeImage,
+                volume: names.volume,
+                configuration: models.configuration,
+                credential: models.credential,
+                apply: false,
+            });
+        // Startup already requires this exact prepared record. An interrupted mutation must not announce readiness.
+        await writePrivate(prepared, JSON.stringify({
+            ownerId: state.ownerId,
+            settingsPending: desired.fingerprint,
+            settingsCandidate: resolve(configFile),
+            ...(checked.reapply ? { settingsReapply: checked.reapply } : {}),
+        }));
+        let stage = "model gateway configuration";
+        try {
+            if (scopes.models) {
+                await prepareModelGateway(directory, { ...state, input: desired.input }, { replaceConfiguration: true });
+                if (gateway && loaded)
+                    try {
+                        stage = "model credential permissions";
+                        await compose(directory, [
+                            "up",
+                            "-d",
+                            "--wait",
+                            "--wait-timeout",
+                            "120",
+                            "models",
+                        ]);
+                        await setRuntimeCredentialModels({
+                            origin: `https://127.0.0.1:${String(gateway.port)}`,
+                            masterKeyFile: join(directory, "private/models/master-key"),
+                            keyFile: join(directory, "private/models/runtime-key"),
+                            caFile: join(directory, "private/management-ca.pem"),
+                            models: loaded.configuration.models
+                                .filter((model) => model.enabled)
+                                .map((model) => model.id),
+                        });
+                    }
+                    finally {
+                        await compose(directory, ["stop", "models", "models-database"]);
+                    }
+                stage = "native model configuration";
+                await configureStoppedRuntimeModels({
+                    image: state.input.runtimeImage,
+                    volume: names.volume,
+                    configuration: models.configuration,
+                    credential: models.credential,
+                    apply: true,
+                });
+            }
+            stage = "Connections configuration";
+            if (scopes.connections &&
+                (state.input.connections || desired.input.connections))
+                await applyConnectionSettings(directory, { ...state, input: desired.input }, checked.previousEndpoint, desired.connectorCredentialFile);
+            stage = "accepted settings";
+            if (scopes.models)
+                await writePrivate(join(directory, "private/model-bootstrap.json"), JSON.stringify(models));
+            await writePrivate(join(directory, "packs.json"), JSON.stringify(desired.packSelection));
+            await writePrivate(join(directory, "installation.json"), JSON.stringify({ ...state, input: desired.input }, null, 2));
+            await writePrivate(join(directory, "inputs.sha256"), desired.fingerprint);
+            await writePrivate(join(directory, "settings.json"), JSON.stringify({
+                ...resolveConfigurationInputs(desired.config, dirname(resolve(configFile))),
+                stateDirectory: directory,
+            }, null, 2));
+            await writePrivate(prepared, JSON.stringify({ ownerId: state.ownerId }));
+            return { state: "configured", restartRequired: true, directory };
+        }
+        catch (error) {
+            throw new InstallationError("unavailable", `Settings were not confirmed during ${stage}; the installation remains stopped. Run configure --directory again to review and explicitly resume the same change. The gateway key is observed first and matching native settings are not repeated.${error instanceof ModelConfigurationError ? ` Native result: ${error.code}.` : ""}`);
+        }
+    });
+}

package/scripts/installation/requirements.js

@@ -0,0 +1,23 @@
+import { openPack } from "../packs/source.js";
+/** Draft feedback only; preview/apply still revalidate real inputs and native state. */
+export async function packRequirements(config) {
+    const issues = [];
+    const members = new Set();
+    for (const selected of config.packs) {
+        const pack = await openPack(selected.directory);
+        for (const id of selected.members) {
+            const member = pack.manifest.members.find((entry) => entry.id === id);
+            if (!member || members.has(id)) {
+                issues.push(`Pack agent ${id} must exist and be selected only once.`);
+                continue;
+            }
+            members.add(id);
+            if (member.requirements.model !== "none" && !config.models)
+                issues.push(`${id} requires Models to be configured.`);
+            if (member.requirements.connections.length &&
+                (config.connections.mode === "disabled" || !selected.bindingsFile))
+                issues.push(`${id} requires Connections and account bindings.`);
+        }
+    }
+    return issues;
+}