cloving 0.3.19 → 1.0.1

package/dist/src/service/service.js

@@ -0,0 +1,212 @@
+import { lstat, rm } from "node:fs/promises";
+import { join } from "node:path";
+import { authorityTiming, openAuthority, } from "../broker/actor/authority.js";
+import { socketAddress } from "../broker/socket-address.js";
+import { changeSignal } from "./changes.js";
+import { decision, questions } from "./decision-handler.js";
+import { events } from "./event-handler.js";
+import { expiryScan } from "./expiry.js";
+import { commandGateway, workerCommandBound } from "./gateway.js";
+import { workflowRegistry, } from "./registry.js";
+import { failed } from "./replies.js";
+import { runCommand, status } from "./run-handlers.js";
+import { submissionHandler } from "./submission-handler.js";
+import { connectionLimits, listen, reachable } from "./transport.js";
+import { workerPort } from "./worker-port.js";
+export const serviceTiming = {
+    authority: authorityTiming,
+    expiryScanMilliseconds: 1_000,
+    livenessMilliseconds: 600_000,
+    followMilliseconds: 250,
+    resolverMilliseconds: 30_000,
+    idleMilliseconds: connectionLimits.idleMilliseconds,
+};
+export const serviceLimits = {
+    textBytes: 65_536,
+    documentBytes: 262_144,
+    pendingWorkerCommands: workerCommandBound,
+};
+function refused(reason, detail) {
+    return { kind: "refused", reason, detail };
+}
+async function privateDirectory(directory) {
+    const entry = await lstat(directory).catch(() => undefined);
+    if (entry === undefined || !entry.isDirectory())
+        return refused("insecure-runtime-directory", "not a directory");
+    const owned = entry.uid === process.getuid?.();
+    const mode = entry.mode & 0o777;
+    if (owned && (mode & 0o077) === 0)
+        return undefined;
+    return refused("insecure-runtime-directory", owned ? `mode ${mode.toString(8)}` : "owned by a different user");
+}
+async function endpoint(directory) {
+    const insecure = await privateDirectory(directory);
+    if (insecure !== undefined)
+        return insecure;
+    const address = socketAddress(join(directory, "control.sock"));
+    if (address.kind === "refused")
+        return address;
+    if (await reachable(address.path))
+        return refused("service-already-running", directory);
+    return address.path;
+}
+function wallClock() {
+    return new Date().toISOString();
+}
+function dispatcher(context) {
+    const run = submissionHandler(context);
+    const operations = new Map([
+        ["run", (request) => run(request.parameters)],
+        ["status", (request) => status(context, request.parameters)],
+        ["questions", (request) => questions(context, request.parameters)],
+        ["decision", (request) => decision(context, request.parameters)],
+        ["run-command", (request) => runCommand(context, request.parameters)],
+        [
+            "events",
+            (request, connection) => events(context, { parameters: request.parameters, connection }),
+        ],
+    ]);
+    return (request, connection) => {
+        const operation = operations.get(request.operation);
+        if (operation === undefined)
+            return Promise.resolve(failed("unknown-operation", {
+                detail: `The service has no operation ${request.operation}.`,
+            }));
+        return operation(request, connection);
+    };
+}
+async function started(authority, setup) {
+    const { options, path, timing, registry, gateway, host } = setup;
+    const clock = options.clock ?? wallClock;
+    const { textBytes, documentBytes } = options.limits ?? serviceLimits;
+    await rm(path, { force: true });
+    const changes = changeSignal(authority.changes.subscribe);
+    const listener = await listen({
+        path,
+        limits: { idleMilliseconds: timing.idleMilliseconds },
+        handle: dispatcher({
+            authority,
+            registry,
+            gateway,
+            clock,
+            changed: changes.next,
+            livenessMilliseconds: timing.livenessMilliseconds,
+            followMilliseconds: timing.followMilliseconds,
+            resolverMilliseconds: timing.resolverMilliseconds,
+            textBytes,
+            documentBytes,
+            worldReports: () => host?.reports?.() ?? [],
+        }),
+    });
+    const stopping = new AbortController();
+    const delivering = Promise.allSettled([authority.deliver(stopping.signal)]);
+    const expiry = expiryScan({
+        authority,
+        gateway,
+        clock,
+        intervalMilliseconds: timing.expiryScanMilliseconds,
+    });
+    return {
+        socket: path,
+        epoch: authority.epoch,
+        worker: setup.worker,
+        close: async () => {
+            changes.close();
+            await expiry.stop();
+            await listener.close();
+            stopping.abort();
+            await delivering;
+            await released(authority, host);
+        },
+    };
+}
+async function released(authority, host) {
+    try {
+        await host?.close();
+    }
+    finally {
+        await authority.close();
+    }
+}
+export async function startService(options) {
+    const path = await endpoint(options.runtimeDirectory);
+    if (typeof path !== "string")
+        return path;
+    const timing = options.timing ?? serviceTiming;
+    const registry = workflowRegistry(options.workflows);
+    const opening = await openAuthority(options.stateDirectory, {
+        delivery: options.delivery,
+        timing: timing.authority,
+        available: registry.available,
+        ...(options.effects === undefined ? {} : { effects: options.effects }),
+        ...(options.clock === undefined ? {} : { clock: options.clock }),
+    });
+    if (opening.kind === "refused")
+        return opening;
+    const { authority } = opening;
+    const gateway = serviceGateway(authority, { options, registry });
+    const worker = workerPort({ gateway, authority });
+    const host = await hosted(options, hostPorts(authority, worker));
+    if (host.kind === "refused") {
+        await authority.close();
+        return host;
+    }
+    try {
+        return {
+            kind: "started",
+            service: await started(authority, {
+                options,
+                path,
+                timing,
+                registry,
+                gateway,
+                worker,
+                host: host.host,
+            }),
+        };
+    }
+    catch (error) {
+        const unclosed = await released(authority, host.host).then(() => "", (failure) => `; the close failed: ${message(failure)}`);
+        return refused("transport-failure", `${message(error)}${unclosed}`);
+    }
+}
+function serviceGateway(authority, parts) {
+    const limits = parts.options.limits ?? serviceLimits;
+    return commandGateway({
+        authority,
+        registry: parts.registry,
+        clock: parts.options.clock ?? wallClock,
+        textBytes: limits.textBytes,
+        pendingWorkerCommands: limits.pendingWorkerCommands,
+    });
+}
+function hostPorts(authority, worker) {
+    return {
+        state: authority.state,
+        submit: worker.submit,
+        input: async (runId) => {
+            const digest = (await authority.state()).details[runId]?.input;
+            return typeof digest === "string"
+                ? await authority.queries.document(digest)
+                : undefined;
+        },
+        document: authority.queries.document,
+        requestEffect: worker.requestEffect,
+        reportEffect: worker.reportEffect,
+        openGate: worker.openGate,
+        registerWait: worker.registerWait,
+        operation: authority.effects.operation,
+        workflowId: async (runId) => (await authority.queries.workflowIds())[runId],
+    };
+}
+function message(error) {
+    return String(Reflect.get(Object(error), "message"));
+}
+async function hosted(options, ports) {
+    try {
+        return { kind: "started", host: await options.host?.(ports) };
+    }
+    catch (error) {
+        return refused("host-unavailable", message(error));
+    }
+}

package/dist/src/service/submission-handler.d.ts

@@ -0,0 +1,4 @@
+import { type Fields, type Reply } from "../protocol.ts";
+import { type ServiceContext } from "./replies.ts";
+export declare function runIdentity(commandId: string): string;
+export declare function submissionHandler(context: ServiceContext): (parameters: Fields) => Promise<Reply>;

package/dist/src/service/submission-handler.js

@@ -0,0 +1,206 @@
+import { createHash } from "node:crypto";
+import { setTimeout as delay } from "node:timers/promises";
+import { isIdentifier as isWorkflowIdentifier } from "../authority/commands.js";
+import { documentDigest } from "../broker/decision-restore.js";
+import { record } from "../protocol.js";
+import { checkResolution, checkSubmission, normalizedRequest, } from "../submission.js";
+import { committed, failed, invalidRequest, isIdentifier, publicReceipt, rejection, result, } from "./replies.js";
+export function runIdentity(commandId) {
+    return `run-${createHash("sha256").update(commandId).digest("hex").slice(0, 32)}`;
+}
+function resolverIdentity(text) {
+    const separator = text.lastIndexOf(":");
+    return { id: text.slice(0, separator), version: text.slice(separator + 1) };
+}
+function admitted(stored, outcome) {
+    const { command } = stored;
+    return result("run-admitted", {
+        runId: stored.runId,
+        workflowId: outcome.workflowId,
+        commandId: stored.commandId,
+        replayed: outcome.replayed,
+        receipt: publicReceipt(stored),
+        binding: {
+            workflowPin: command.workflow,
+            resolver: resolverIdentity(String(command.resolver)),
+            requestDigest: command.submission,
+            executionInputDigest: command.input,
+        },
+    });
+}
+function conflict(commandId) {
+    return failed("conflicting-command", {
+        commandId,
+        detail: "The command ID is known with a different request.",
+    });
+}
+function replayed(existing, request) {
+    const submission = record(request.submission);
+    const same = existing.command.kind === "admit-run" &&
+        submission !== undefined &&
+        documentDigest(normalizedRequest(request.workflowId, submission)) ===
+            existing.command.submission;
+    return same
+        ? admitted(existing, { workflowId: request.workflowId, replayed: true })
+        : conflict(existing.commandId);
+}
+function effectBudget(workflow) {
+    if (workflow.effectLimit === undefined)
+        return {};
+    return {
+        effectLimit: workflow.effectLimit,
+        capabilities: JSON.stringify([...workflow.manifest.capabilities].sort()),
+    };
+}
+function usable(workflow, workflowId) {
+    if (workflow === undefined)
+        return {
+            kind: "unusable",
+            reply: failed("unknown-workflow", {
+                detail: `No admitted workflow has the ID ${workflowId}.`,
+            }),
+        };
+    if (workflow.availability !== "available")
+        return {
+            kind: "unusable",
+            reply: failed("workflow-unavailable", {
+                detail: `The workflow ${workflowId} is not available. A required capability did not pass admission.`,
+            }),
+        };
+    return { kind: "usable", workflow };
+}
+async function resolved(context, admission) {
+    const { workflow, workflowId, commandId, submission } = admission;
+    const limit = new AbortController();
+    const timer = new AbortController();
+    const request = { workflowId, submission, commandId, signal: limit.signal };
+    const call = Promise.resolve(request)
+        .then((given) => workflow.resolver.resolve(given))
+        .then((value) => checkResolution(value) ?? "failure")
+        .catch(() => "failure");
+    const expired = delay(context.resolverMilliseconds, "timeout", {
+        signal: timer.signal,
+    });
+    try {
+        const outcome = await Promise.race([call, expired]);
+        if (outcome === "timeout")
+            limit.abort();
+        return outcome;
+    }
+    finally {
+        timer.abort();
+    }
+}
+function unresolved(context, outcome) {
+    const { kind, commandId } = outcome;
+    if (kind === "timeout")
+        return failed("resolver-timeout", {
+            commandId,
+            detail: `The resolver gave no result in ${String(context.resolverMilliseconds)} ms. Send the command again with the same command ID.`,
+        });
+    return failed("resolver-failure", {
+        commandId,
+        detail: "The resolver of the workflow failed or gave a result that is not valid. Nothing is admitted.",
+    });
+}
+async function admit(context, admission) {
+    const { workflow, workflowId, commandId, normalized } = admission;
+    const resolution = await resolved(context, admission);
+    if (typeof resolution === "string")
+        return unresolved(context, { kind: resolution, commandId });
+    if (resolution.kind === "rejected")
+        return failed(resolution.code, { commandId, detail: resolution.detail });
+    const { input } = resolution;
+    if (Buffer.byteLength(input) > context.documentBytes)
+        return unresolved(context, { kind: "failure", commandId });
+    const receipt = await context.gateway.execute({
+        principal: "operator",
+        command: {
+            kind: "admit-run",
+            commandId,
+            runId: runIdentity(commandId),
+            workflow: workflow.pin,
+            submission: documentDigest(normalized),
+            input: documentDigest(input),
+            resolver: `${workflow.resolver.id}:${workflow.resolver.version}`,
+            ...effectBudget(workflow),
+        },
+        documents: [normalized, input],
+    });
+    if (!committed(receipt))
+        return rejection(receipt, { commandId });
+    const stored = await context.authority.queries.receipt(commandId);
+    if (stored === undefined)
+        return conflict(commandId);
+    return admitted(stored, {
+        workflowId,
+        replayed: receipt.kind === "duplicate",
+    });
+}
+async function submitted(context, parameters) {
+    const { workflowId, commandId } = parameters;
+    if (!isWorkflowIdentifier(workflowId) || !isIdentifier(commandId))
+        return invalidRequest("The workflow ID or the command ID is not valid.");
+    const existing = await context.authority.queries.receipt(commandId);
+    if (existing !== undefined)
+        return replayed(existing, {
+            workflowId,
+            submission: parameters["submission"],
+        });
+    const found = usable(context.registry.byId(workflowId), workflowId);
+    if (found.kind === "unusable")
+        return found.reply;
+    const contract = found.workflow.submission;
+    const check = checkSubmission({
+        ...contract,
+        maximumBytes: Math.min(contract.maximumBytes, context.documentBytes),
+    }, {
+        workflowId,
+        submission: parameters["submission"],
+        source: parameters["source"],
+    });
+    if (check.kind === "rejected")
+        return failed(check.code, { commandId, detail: check.detail });
+    if (found.workflow.admission === "when-idle" &&
+        (await activeWorkflow(context, workflowId)))
+        return failed("workflow-active", {
+            commandId,
+            detail: `The workflow ${workflowId} already has an active run.`,
+        });
+    return admit(context, {
+        workflow: found.workflow,
+        workflowId,
+        commandId,
+        submission: check.submission,
+        normalized: check.normalized,
+    });
+}
+async function activeWorkflow(context, workflowId) {
+    const workflows = await context.authority.queries.workflowIds();
+    const state = await context.authority.state();
+    return Object.values(state.runs).some((run) => workflows[run.runId] === workflowId &&
+        !["completed", "failed", "cancelled"].includes(run.status));
+}
+export function submissionHandler(context) {
+    const pending = new Map();
+    return (parameters) => {
+        const keys = [`command:${String(parameters["commandId"])}`];
+        const workflowId = String(parameters["workflowId"]);
+        if (context.registry.byId(workflowId)?.admission === "when-idle")
+            keys.push(`workflow:${workflowId}`);
+        const next = () => submitted(context, parameters);
+        const previous = keys
+            .map((key) => pending.get(key))
+            .filter((reply) => reply !== undefined);
+        const reply = Promise.allSettled(previous).then(next);
+        for (const key of keys)
+            pending.set(key, reply);
+        const settled = () => {
+            for (const key of keys)
+                if (pending.get(key) === reply)
+                    pending.delete(key);
+        };
+        reply.then(settled, settled);
+        return reply;
+    };
+}

package/dist/src/service/transport.d.ts

@@ -0,0 +1,16 @@
+import { type Reply, type ServiceRequest } from "../protocol.ts";
+import type { Connection } from "./event-handler.ts";
+export type Handle = (request: ServiceRequest, connection: Connection) => Promise<Reply | undefined>;
+export interface Listener {
+    readonly close: () => Promise<void>;
+}
+export interface ConnectionLimits {
+    readonly idleMilliseconds: number;
+}
+export declare const connectionLimits: ConnectionLimits;
+export declare function listen(options: {
+    readonly path: string;
+    readonly handle: Handle;
+    readonly limits?: ConnectionLimits;
+}): Promise<Listener>;
+export declare function reachable(path: string): Promise<boolean>;

package/dist/src/service/transport.js

@@ -0,0 +1,141 @@
+import { chmod, rm } from "node:fs/promises";
+import net from "node:net";
+import { eventLine, maximumFrameBytes, parseRequest, replyLine, } from "../protocol.js";
+function tooLarge() {
+    return {
+        kind: "error",
+        error: {
+            code: "request-too-large",
+            detail: `A request has at most ${String(maximumFrameBytes)} bytes.`,
+        },
+    };
+}
+function unavailable(error) {
+    return {
+        kind: "error",
+        error: {
+            code: "authority-unavailable",
+            detail: String(Reflect.get(Object(error), "message")),
+        },
+    };
+}
+function flushed(socket, limit) {
+    return new Promise((settle) => {
+        const settled = (outcome) => () => {
+            clearTimeout(timer);
+            socket.off("drain", onDrain);
+            socket.off("close", onStall);
+            if (outcome === "stalled")
+                socket.destroy();
+            settle(outcome === "drained");
+        };
+        const onDrain = settled("drained");
+        const onStall = settled("stalled");
+        const timer = setTimeout(onStall, limit);
+        socket.once("drain", onDrain);
+        socket.once("close", onStall);
+    });
+}
+function served(socket, serving) {
+    const { handle, limits } = serving;
+    let buffered = "";
+    let handled = false;
+    const connection = {
+        send: (events) => {
+            if (socket.destroyed)
+                return Promise.resolve(false);
+            const written = events.map((event) => socket.write(eventLine(event)));
+            return written.includes(false)
+                ? flushed(socket, limits.idleMilliseconds)
+                : Promise.resolve(true);
+        },
+        closed: () => socket.destroyed || socket.readableEnded,
+    };
+    socket.setTimeout(limits.idleMilliseconds, () => {
+        if (!handled)
+            socket.destroy();
+    });
+    const finish = (reply) => {
+        if (socket.destroyed)
+            return;
+        if (reply === undefined)
+            socket.destroy();
+        else
+            socket.end(replyLine(reply));
+        setTimeout(() => socket.destroy(), limits.idleMilliseconds).unref();
+    };
+    socket.setEncoding("utf8");
+    socket.on("error", () => undefined);
+    socket.on("data", (data) => {
+        if (handled)
+            return;
+        buffered += data;
+        const end = buffered.indexOf("\n");
+        if (end < 0 && Buffer.byteLength(buffered) <= maximumFrameBytes)
+            return;
+        handled = true;
+        if (end < 0 || end > maximumFrameBytes) {
+            finish(tooLarge());
+            return;
+        }
+        const parsed = parseRequest(buffered.slice(0, end));
+        if (parsed.kind === "error") {
+            finish(parsed);
+            return;
+        }
+        Promise.resolve(parsed.request)
+            .then((request) => handle(request, connection))
+            .then(finish, (error) => {
+            finish(unavailable(error));
+        });
+    });
+    socket.on("end", () => {
+        if (!handled)
+            socket.end();
+    });
+}
+export const connectionLimits = {
+    idleMilliseconds: 30_000,
+};
+export async function listen(options) {
+    const { path, handle } = options;
+    const limits = options.limits ?? connectionLimits;
+    const sockets = new Set();
+    const server = net.createServer({ allowHalfOpen: true }, (socket) => {
+        sockets.add(socket);
+        socket.on("close", () => sockets.delete(socket));
+        served(socket, { handle, limits });
+    });
+    await new Promise((ready, refuse) => {
+        server.once("error", refuse);
+        server.listen(path, () => {
+            server.off("error", refuse);
+            ready();
+        });
+    });
+    await chmod(path, 0o600);
+    return {
+        close: async () => {
+            for (const socket of sockets)
+                socket.destroy();
+            await new Promise((closed) => {
+                server.close(() => {
+                    closed();
+                });
+            });
+            await rm(path, { force: true });
+        },
+    };
+}
+export function reachable(path) {
+    return new Promise((settle) => {
+        const socket = net.connect(path);
+        socket.on("connect", () => {
+            socket.destroy();
+            settle(true);
+        });
+        socket.on("error", () => {
+            settle(false);
+        });
+    });
+}

package/dist/src/service/worker-port.d.ts

@@ -0,0 +1,85 @@
+import type { Authority, OperationView, StoreReceipt, StoredDisposition } from "../broker/actor/authority.ts";
+import type { Command, Gateway } from "./gateway.ts";
+interface Identified {
+    readonly runId: string;
+    readonly commandId: string;
+}
+interface Revised extends Identified {
+    readonly expectedRevision: number;
+}
+export interface GateOpening extends Revised {
+    readonly gateId: string;
+    readonly kind: "question" | "approval";
+    readonly prompt: string;
+    readonly subject: string;
+    readonly expiresAt?: string;
+}
+export interface WaitRegistration extends Identified {
+    readonly gateId: string;
+}
+export interface ObservationReport extends Identified {
+    readonly condition: string;
+    readonly reason: string;
+    readonly evidence?: string;
+    readonly retryAt?: string;
+    readonly operationId?: string;
+    readonly attempt?: number;
+    readonly limit?: number;
+}
+export interface EffectScope {
+    readonly phase: string;
+    readonly operation: string;
+    readonly round: number;
+    readonly key?: string;
+    readonly stage?: {
+        readonly id: string;
+        readonly visit: string;
+        readonly branch: string;
+    };
+}
+export interface EffectRequest extends Identified {
+    readonly scope: EffectScope;
+    readonly capability: string;
+    readonly method: string;
+    readonly target: string;
+    readonly payload: string;
+    readonly units: number;
+    readonly resource?: string;
+    readonly transport?: string;
+    readonly source?: string;
+    readonly sourceGeneration?: number;
+    readonly epoch?: number;
+}
+export interface EffectReport extends Identified {
+    readonly operationId: string;
+    readonly inputDigest: string;
+    readonly leaseGeneration: number;
+    readonly worker: string;
+    readonly outcome: "completed" | "failed" | "stopped";
+    readonly result?: string;
+    readonly reason?: string;
+    readonly evidence?: string;
+    readonly epoch?: number;
+}
+export interface EffectReply {
+    readonly receipt: StoreReceipt;
+    readonly disposition: StoredDisposition | null;
+    readonly operation: OperationView | null;
+}
+export interface WorkerPort {
+    readonly submit: (command: Command, documents?: readonly string[]) => Promise<StoreReceipt>;
+    readonly started: (run: Revised) => Promise<StoreReceipt>;
+    readonly openGate: (opening: GateOpening) => Promise<StoreReceipt>;
+    readonly registerWait: (wait: WaitRegistration) => Promise<StoreReceipt>;
+    readonly observe: (report: ObservationReport) => Promise<StoreReceipt>;
+    readonly completed: (run: Revised) => Promise<StoreReceipt>;
+    readonly failed: (run: Revised) => Promise<StoreReceipt>;
+    readonly requestEffect: (request: EffectRequest) => Promise<EffectReply>;
+    readonly reportEffect: (report: EffectReport) => Promise<EffectReply>;
+}
+interface Parts {
+    readonly gateway: Gateway;
+    readonly authority: Authority;
+}
+export declare function workerPort(parts: Parts): WorkerPort;
+export {};