hammerkit 1.27.0-next.9 → 2.0.0-next.2

package/dist/server/run-request.js

@@ -1,12 +1,17 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.runRequestSchema = exports.runOptionsSchema = exports.MAX_ARGS_BYTES = exports.MAX_ARGS = void 0;
+exports.runRequestSchema = exports.runOptionsSchema = exports.MAX_ARGS_BYTES = exports.MAX_ARGS = exports.MAX_ENDPOINTS = exports.SERVICE_RUN_NAME_PATTERN = void 0;
 const zod_1 = require("zod");
 const units_1 = require("../utils/units");
 const bundle_1 = require("../remote/bundle");
 const repo_1 = require("./repo");
 const runs_1 = require("./runs");
 const name = zod_1.z.string().min(1).max(200);
+// The name of a service run, one DNS label: the hosts of its endpoints carry it, and a run of
+// the same caller with the same name replaces it
+exports.SERVICE_RUN_NAME_PATTERN = /^[a-z0-9]([a-z0-9-]{0,38}[a-z0-9])?$/;
+// endpoints one service run may publish
+exports.MAX_ENDPOINTS = 5;
 // what one run may pass after `--`
 exports.MAX_ARGS = 100;
 exports.MAX_ARGS_BYTES = 16 * 1024;
@@ -38,6 +43,8 @@ exports.runOptionsSchema = zod_1.z
     cacheReadOnly: zod_1.z.boolean().optional(),
     // as --timeout
     timeout: duration.optional(),
+    // as --max-timeout: caps every task, also one with a longer timeout of its own
+    maxTimeout: duration.optional(),
     // as --no-skip-deps when false
     skipDeps: zod_1.z.boolean().optional(),
     // as the text after `--`: appended to the last command of the one task the run
@@ -87,18 +94,66 @@ exports.runRequestSchema = zod_1.z
         zod_1.z.object({ type: zod_1.z.literal('upload'), snapshot: zod_1.z.string().regex(bundle_1.HASH_PATTERN) }).strict(),
         // a sandbox of the caller's, see POST /v1/sandboxes
         zod_1.z
-            .object({ type: zod_1.z.literal('sandbox'), id: zod_1.z.string().regex(runs_1.SANDBOX_ID_PATTERN, 'expected a sandbox id') })
+            .object({
+            type: zod_1.z.literal('sandbox'),
+            id: zod_1.z.string().regex(runs_1.SANDBOX_ID_PATTERN, 'expected a sandbox id'),
+            // a check: run on a copy of the files a push of the sandbox would commit, so what
+            // passes counts for that commit (remote verify)
+            check: zod_1.z.literal(true).optional(),
+        })
             .strict(),
     ]),
     options: exports.runOptionsSchema.default({}),
+    // a service run: these services of the build file, started as `hammerkit up` starts
+    // them and kept up until `ttl` ends or the run is cancelled, instead of tasks
+    services: zod_1.z.array(name).max(20).default([]),
+    // how long the services stay once they are ready, whether they sleep in between or not;
+    // `runs.up.ttl` when left out. At its end they are removed.
+    ttl: duration.optional(),
+    // how long the services stay up without a request to one of the endpoints, then they
+    // sleep until the next request wakes them; `runs.up.sleepTtl` when left out
+    sleepTtl: duration.optional(),
+    // a second service run of the caller with the same name replaces the first; the branch
+    // of the source when left out
+    name: zod_1.z
+        .string()
+        .regex(exports.SERVICE_RUN_NAME_PATTERN, 'expected lower case letters, digits and -, at most 40, as a DNS label')
+        .optional(),
+    // ports of the run's services published at hosts of their own, behind the server's sign-in
+    endpoints: zod_1.z
+        .array(zod_1.z.object({ service: name, port: zod_1.z.number().int().min(1).max(65535) }).strict())
+        .max(exports.MAX_ENDPOINTS)
+        .default([]),
 })
     .strict()
-    .refine((request) => request.tasks.length > 0 || (request.options.filter ?? []).length > 0, {
+    .refine((request) => request.services.length > 0 || request.tasks.length > 0 || (request.options.filter ?? []).length > 0, {
     message: 'name at least one task, or filter by labels',
     path: ['tasks'],
 })
+    .refine((request) => request.services.length === 0 || request.tasks.length === 0, {
+    message: 'a run names tasks or services, not both',
+    path: ['services'],
+})
+    .refine((request) => request.services.length === 0 ||
+    ((request.options.filter ?? []).length === 0 && (request.options.exclude ?? []).length === 0), { message: 'a service run names its services, labels do not select them', path: ['options', 'filter'] })
+    .refine((request) => request.services.length === 0 || (request.options.args ?? []).length === 0, {
+    message: 'a service run takes no arguments',
+    path: ['options', 'args'],
+})
+    .refine((request) => request.services.length > 0 ||
+    (request.ttl === undefined &&
+        request.sleepTtl === undefined &&
+        request.name === undefined &&
+        request.endpoints.length === 0), { message: 'ttl, sleepTtl, name and endpoints are for a run of services', path: ['services'] })
+    .refine((request) => new Set(request.endpoints.map((endpoint) => `${endpoint.service}:${endpoint.port}`)).size ===
+    request.endpoints.length, { message: 'an endpoint is named twice', path: ['endpoints'] })
     .refine((request) => (request.options.args ?? []).length === 0 || request.tasks.length === 1, {
     message: 'arguments go to one task, name exactly one in tasks',
     path: ['options', 'args'],
+})
+    .refine((request) => request.source.type !== 'sandbox' || !request.source.check || (request.options.args ?? []).length === 0, { message: 'a check runs the task as the build file says, it takes no arguments', path: ['options', 'args'] })
+    .refine((request) => request.source.type !== 'sandbox' || !request.source.check || request.services.length === 0, {
+    message: 'a check runs tasks, a run of services keeps them up and passes nothing',
+    path: ['services'],
 });
 //# sourceMappingURL=run-request.js.map

package/dist/server/run-worker.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.workerMain = exports.taskCauses = exports.runTasks = exports.getRunNetwork = exports.workerInputSchema = void 0;
+exports.workerMain = exports.readInput = exports.taskCauses = exports.describeFailures = exports.runTasks = exports.getEndpointTargets = exports.publishEndpoints = exports.getRunNetwork = exports.workerInputSchema = void 0;
 const readline_1 = require("readline");
 const zod_1 = require("zod");
 const cli_1 = require("../cli");
@@ -21,6 +21,12 @@ const network_schema_1 = require("../schema/network-schema");
 const work_network_1 = require("../planner/work-network");
 const run_plan_1 = require("./run-plan");
 const runs_1 = require("./runs");
+const resources_1 = require("../kubernetes/resources");
+const remove_stale_items_1 = require("../kubernetes/remove-stale-items");
+const service_rollouts_1 = require("../kubernetes/service-rollouts");
+const cluster_state_1 = require("../kubernetes/cluster-state");
+const kubernetes_instance_1 = require("../kubernetes/kubernetes-instance");
+const check_keys_1 = require("../optimizer/check-keys");
 // one log line of a task is cut here, the registry caps how many events a run keeps
 const MAX_LOG_TEXT = 8 * 1024;
 /**
@@ -49,6 +55,9 @@ exports.workerInputSchema = zod_1.z
     kubernetes: zod_1.z
         .custom((value) => typeof value === 'object' && value !== null && value.type === 'kubernetes')
         .optional(),
+    // on the server's docker host, the runtime a container that adds capabilities runs in
+    // (`backend.capabilities.runtime`)
+    capabilityRuntime: zod_1.z.string().min(1).optional(),
     // the most any task or service reaches besides its needs, as `--network`: `runs.network`
     network: network_schema_1.networkSchema.optional(),
     // for a run on a sandbox, `sandboxes.network`: the run gets the lower of the two
@@ -57,8 +66,19 @@ exports.workerInputSchema = zod_1.z
     // volumes (see scopeServiceVolumes)
     volumeScope: zod_1.z
         .string()
-        .regex(/^[a-z]+-[0-9a-f]{24}$/)
+        .regex(/^(?:[a-z]+|up-[0-9a-f]{8})-[0-9a-f]{24}$/)
         .optional(),
+    // a service run: the services it keeps up instead of tasks, and the ports the server
+    // reaches as endpoints
+    up: zod_1.z
+        .object({
+        services: zod_1.z.array(zod_1.z.string()).min(1),
+        endpoints: zod_1.z.array(zod_1.z.object({ service: zod_1.z.string(), port: zod_1.z.number().int() }).strict()),
+    })
+        .strict()
+        .optional(),
+    // the run's results count as checks: report each task's check key before it runs
+    check: zod_1.z.boolean().optional(),
 })
     .strict();
 // The ceiling of the run, combined here and not in the server: the lower of two allow lists
@@ -68,6 +88,60 @@ function getRunNetwork(input) {
     return input.sandboxNetwork ? (0, work_network_1.limitNetwork)((0, work_network_1.parseWorkNetwork)(input.sandboxNetwork), runs) : runs;
 }
 exports.getRunNetwork = getRunNetwork;
+/**
+ * The ports of the services of a service run as the server publishes them: on a host the
+ * ports of endpoints go to a free port of the loopback interface, where only the server
+ * reaches them, and no other port goes to the host, whatever the build file says (`up`
+ * would publish them on every interface, where two runs collide). On a cluster nothing is
+ * published, the server reaches the Service of each.
+ */
+function publishEndpoints(workTree, endpoints, cluster) {
+    for (const service of Object.values(workTree.services)) {
+        if (!('ports' in service.data)) {
+            continue;
+        }
+        service.data.ports = service.data.ports.map((port) => !cluster &&
+            endpoints.some((endpoint) => endpoint.service === service.name && endpoint.port === port.containerPort)
+            ? { containerPort: port.containerPort, hostPort: 0, hostIp: '127.0.0.1' }
+            : { containerPort: port.containerPort, hostPort: null });
+    }
+}
+exports.publishEndpoints = publishEndpoints;
+// Where the server reaches each endpoint, once every service the run keeps up is ready;
+// null until then.
+function getEndpointTargets(workTree, up, kubernetes) {
+    const services = Object.values(workTree.services).filter((service) => up.services.includes(service.name));
+    if (services.length < up.services.length || services.some((service) => service.state.current.type !== 'running')) {
+        return null;
+    }
+    const targets = [];
+    for (const endpoint of up.endpoints) {
+        const service = services.find((candidate) => candidate.name === endpoint.service);
+        const state = service?.state.current;
+        if (!service || state?.type !== 'running') {
+            return null;
+        }
+        if (kubernetes) {
+            targets.push({
+                service: endpoint.service,
+                port: endpoint.port,
+                address: { host: `${(0, resources_1.getResourceName)(service, kubernetes)}.${kubernetes.namespace}.svc`, port: endpoint.port },
+            });
+            continue;
+        }
+        const published = state.published?.find((candidate) => candidate.containerPort === endpoint.port);
+        if (!published) {
+            throw new Error(`${endpoint.service} is up, but its port ${endpoint.port} was not published`);
+        }
+        targets.push({
+            service: endpoint.service,
+            port: endpoint.port,
+            address: { host: published.host, port: published.port },
+        });
+    }
+    return targets;
+}
+exports.getEndpointTargets = getEndpointTargets;
 /**
  * Executes the tasks of a run with the engine of `hammerkit run`. The caller says which
  * process this is in: it reports what happens through `emit` and stops when `abortCtrl`
@@ -75,7 +149,9 @@ exports.getRunNetwork = getRunNetwork;
  */
 async function runTasks(input, cwd, processEnvs, abortCtrl, 
 // whether the run was stopped from outside; the engine aborts too when a task fails
-cancelled, emit) {
+cancelled, emit, 
+// a service run: its services are ready, at these addresses
+up) {
     const secrets = (0, redact_1.createSecretRegistry)();
     const status = (0, work_item_status_1.statusConsole)((0, empty_writable_1.emptyWritable)(), secrets);
     // the process starts in a directory of the server, the tasks work on the run's files
@@ -84,7 +160,12 @@ cancelled, emit) {
     // is what is streamed: it is redacted with the secrets of this run
     const environment = {
         processEnvs,
-        hosted: { catalog: input.catalog, accounts: input.accounts, kubernetes: input.kubernetes },
+        hosted: {
+            catalog: input.catalog,
+            accounts: input.accounts,
+            kubernetes: input.kubernetes,
+            capabilityRuntime: input.capabilityRuntime,
+        },
         abortCtrl,
         cwd: work,
         file: (0, get_file_context_1.getFileContext)(work),
@@ -117,15 +198,27 @@ cancelled, emit) {
         const options = input.options;
         const execOptions = {
             logMode: 'live',
-            cacheDefault: options.cache ?? 'checksum',
+            // a check takes a cached result only for the same files, never for the same dates
+            cacheDefault: input.check && options.cache === 'modify-date' ? 'checksum' : options.cache ?? 'checksum',
             cacheReadOnly: options.cacheReadOnly ?? false,
             timeout: options.timeout ? (0, units_1.parseDuration)(options.timeout) : null,
+            maxTimeout: options.maxTimeout ? (0, units_1.parseDuration)(options.maxTimeout) : null,
             skipDeps: options.skipDeps !== false,
             network: getRunNetwork(input),
             // why a task was not taken from the cache goes into its result
             explain: true,
         };
-        for (const { name, scope } of (0, run_plan_1.getRunScopes)(input.tasks, options)) {
+        const scopes = (0, run_plan_1.getRunScopes)(input.tasks, options, input.up?.services);
+        if (input.check) {
+            // the files as they are before any task wrote to them, a second task's too: a first
+            // one that formats or generates into src would key it by files no commit holds
+            for (const { scope } of scopes) {
+                for (const [task, { key }] of Object.entries(await (0, check_keys_1.getCheckKeys)(input.entry, environment, scope))) {
+                    emit({ type: 'key', task, key });
+                }
+            }
+        }
+        for (const { name, scope } of scopes) {
             const { ctx, scope: parsed } = await (0, schema_parser_1.createParseContext)(input.entry, environment, {
                 directory: input.directory,
             });
@@ -134,23 +227,75 @@ cancelled, emit) {
             if (input.volumeScope && !input.kubernetes) {
                 (0, hosted_volumes_1.scopeServiceVolumes)(workTree, input.volumeScope);
             }
+            if (input.up) {
+                publishEndpoints(workTree, input.up.endpoints, !!input.kubernetes);
+            }
             const cli = (0, cli_1.getCli)(workTree, environment);
-            if (cli.tasks().length === 0) {
+            if (cli.tasks().length === 0 && (!input.up || cli.services().length === 0)) {
                 throw new Error(`no tasks found for ${name}`);
             }
-            const execution = cli.exec({ ...execOptions, workers: cli.defaultWorkers() });
+            const execution = input.up
+                ? cli.up({ ...execOptions, workers: cli.defaultWorkers(), daemon: false, wait: 'ready' })
+                : cli.exec({ ...execOptions, workers: cli.defaultWorkers() });
             const seen = new Map();
-            const report = () => reportTasks(emit, execution.state.current, seen, (text) => secrets.redact(text));
+            const failedServices = new Set();
+            let ready = false;
+            const report = () => {
+                reportTasks(emit, execution.state.current, seen, (text) => secrets.redact(text));
+                reportServiceFailures(emit, execution.state.current, failedServices, (text) => secrets.redact(text));
+                if (input.up && up && !ready) {
+                    const targets = getEndpointTargets(execution.state.current, input.up, input.kubernetes);
+                    if (targets) {
+                        ready = true;
+                        up(targets);
+                    }
+                }
+            };
             const watch = execution.state.on('run-events', report);
             try {
                 const result = await execution.start();
                 report();
+                // on a cluster the services stay in their Deployments once they are ready, until
+                // the namespace of the run goes
+                if (input.up && input.kubernetes && ready && result.success && !abortCtrl.signal.aborted) {
+                    if (input.kubernetes.server?.serviceRun) {
+                        // a run that moved here from another commit: what it made for items it has no more
+                        const ids = [...Object.values(workTree.tasks), ...Object.values(workTree.services)].map((item) => item.id());
+                        await (0, remove_stale_items_1.removeStaleItems)((0, kubernetes_instance_1.createKubernetesInstances)(input.kubernetes), input.kubernetes, ids).catch((e) => emit({
+                            type: 'log',
+                            task: 'hammerkit',
+                            stream: 'stderr',
+                            text: `what the run made at its previous commit could not be removed: ${(0, log_1.getErrorMessage)(e)}`,
+                        }));
+                    }
+                    await new Promise((resolve) => abortCtrl.signal.addEventListener('abort', () => resolve(), { once: true }));
+                    return;
+                }
+                if (input.up && ready) {
+                    if (!abortCtrl.signal.aborted) {
+                        throw new Error('the services stopped on their own');
+                    }
+                    return;
+                }
                 if (!result.success) {
-                    throw new Error(cancelled() ? 'the run was cancelled' : `${name} failed`);
+                    if (cancelled()) {
+                        throw new Error('the run was cancelled');
+                    }
+                    if (input.kubernetes) {
+                        // what the cluster said, before the server removes what the run made
+                        await reportClusterState(input.kubernetes, execution.state.current, !!input.up, emit, (text) => secrets.redact(text));
+                    }
+                    // a service run names what failed and why: its services have no results of their own
+                    const failures = input.up ? describeFailures(execution.state.current, (text) => secrets.redact(text)) : [];
+                    throw new Error(failures.length > 0 ? failures.join('; ').slice(0, runs_1.MAX_TASK_ERROR) : `${name} failed`);
                 }
             }
             finally {
                 watch.close();
+                if (input.up && input.kubernetes?.server?.serviceRun && !ready && service_rollouts_1.serviceRollouts.size > 0) {
+                    // a move (or a start) whose services did not all get up: the run stays where it was
+                    await service_rollouts_1.serviceRollouts.undo((0, kubernetes_instance_1.createKubernetesInstances)(input.kubernetes), (text) => emit({ type: 'log', task: 'hammerkit', stream: 'stderr', text }));
+                }
             }
         }
     }
@@ -159,6 +304,82 @@ cancelled, emit) {
     }
 }
 exports.runTasks = runTasks;
+// What the cluster said of a run that failed, into its log: the server removes its pods
+// once it ended.
+async function reportClusterState(kubernetes, workTree, ownNamespace, emit, redact) {
+    let lines;
+    try {
+        lines = await (0, cluster_state_1.readClusterState)((0, kubernetes_instance_1.createKubernetesInstances)(kubernetes), kubernetes, workTree, ownNamespace);
+    }
+    catch (e) {
+        lines = [`it could not be read: ${(0, log_1.getErrorMessage)(e)}`];
+    }
+    if (lines.length === 0) {
+        return;
+    }
+    for (const text of [`what the cluster said in ${kubernetes.namespace}:`, ...lines.map((line) => `  ${line}`)]) {
+        emit({ type: 'log', task: 'hammerkit', stream: 'stderr', text: redact(text).slice(0, MAX_LOG_TEXT) });
+    }
+}
+// A service has no result of its own in the run: why it failed goes into the log, under
+// its name, once.
+function reportServiceFailures(emit, workTree, said, redact) {
+    for (const service of (0, plan_work_tasks_1.iterateWorkServices)(workTree)) {
+        const state = service.state.current;
+        if (state.type === 'error' && !said.has(service.name)) {
+            said.add(service.name);
+            if (lastError(service) === state.errorMessage) {
+                // the runtime said it already
+                continue;
+            }
+            emit({
+                type: 'log',
+                task: service.name,
+                stream: 'stderr',
+                text: redact(state.errorMessage).slice(0, MAX_LOG_TEXT),
+            });
+        }
+    }
+}
+// The last error the runtime of an item wrote to its status.
+function lastError(item) {
+    let last;
+    for (const message of item.status.read()) {
+        if (message.level === 'error') {
+            last = message.message;
+        }
+    }
+    return last;
+}
+/**
+ * What failed in a run, one line each, for its error: the services and tasks that failed,
+ * with their reason. Those that only stopped because another failed are left out.
+ */
+function describeFailures(workTree, redact) {
+    const failures = [];
+    for (const service of (0, plan_work_tasks_1.iterateWorkServices)(workTree)) {
+        const state = service.state.current;
+        if (state.type === 'error') {
+            failures.push(`${service.name}: ${redact(state.errorMessage)}`);
+        }
+        else if (state.type === 'end' && state.reason === 'crash') {
+            // on Docker a service that does not get ready ends, and its runtime says why
+            const why = lastError(service);
+            failures.push(`${service.name}: ${why ? redact(why) : 'it stopped on its own'}`);
+        }
+    }
+    for (const task of (0, plan_work_tasks_1.iterateWorkTasks)(workTree)) {
+        const state = task.state.current;
+        if (state.type === 'error') {
+            failures.push(`${task.name}: ${redact(state.errorMessage)}`);
+        }
+        else if (state.type === 'crash') {
+            failures.push(`${task.name}: exit code ${state.exitCode}`);
+        }
+    }
+    return failures;
+}
+exports.describeFailures = describeFailures;
 // Why a task ran, as a task event carries it: the first causes, masked and cut short.
 function taskCauses(causes, redact) {
     return causes?.length
@@ -217,9 +438,13 @@ redact) {
 // The run is the first line of the standard input. The server keeps it open and writes
 // `cancel` on it to stop the run: that works the same on every platform, where a signal
 // on Windows ends the process without letting it stop its tasks.
+// `onEnd` is told when the input ends before the run does: the server is gone.
 function readInput(stdin, cancel) {
     const lines = (0, readline_1.createInterface)({ input: stdin, crlfDelay: Infinity });
     let first;
+    let closing = false;
+    let ended = false;
+    let onEnd;
     const input = new Promise((resolve) => {
         lines.on('line', (line) => {
             if (first === undefined) {
@@ -230,10 +455,29 @@ function readInput(stdin, cancel) {
                 cancel();
             }
         });
-        lines.on('close', () => resolve(first ?? ''));
+        lines.on('close', () => {
+            resolve(first ?? '');
+            if (!closing) {
+                ended = true;
+                onEnd?.();
+            }
+        });
     });
-    return { input, close: () => lines.close() };
+    return {
+        input,
+        close: () => {
+            closing = true;
+            lines.close();
+        },
+        onEnd: (listener) => {
+            onEnd = listener;
+            if (ended) {
+                listener();
+            }
+        },
+    };
 }
+exports.readInput = readInput;
 /**
  * The process of one run: reads its input, executes the tasks in the current directory
  * and reports as lines of JSON. It exits 0 when it could say how the run ended, and the
@@ -260,8 +504,13 @@ async function workerMain(stdin, stdout, processEnvs, cwd, abortCtrl) {
         say({ kind: 'result', ok: false, error: `the run could not be read: ${e instanceof Error ? e.message : e}` });
         return;
     }
+    if (input.up) {
+        // services stay until they are stopped: without the server that would stop them, at
+        // their ttl or on a cancel, nothing would
+        reader.onEnd(stop);
+    }
     try {
-        await runTasks(input, cwd, processEnvs, abortCtrl, () => cancelled, (event) => say({ kind: 'event', event }));
+        await runTasks(input, cwd, processEnvs, abortCtrl, () => cancelled, (event) => say({ kind: 'event', event }), (targets) => say({ kind: 'up', targets }));
         say({ kind: 'result', ok: true });
     }
     catch (e) {