@kici-dev/shared 0.4.0 → 0.6.0

package/dist/cold-store/chunk-id.js

@@ -14,14 +14,15 @@ import { sha256 } from "@kici-dev/core";
 * at a trillion chunks, birthday probability is ~5e-8.
 */
 function computeChunkId(args) {
-	return sha256([
+	const input = [
 		args.db,
 		args.table,
 		args.tenantId,
 		args.partitionDate,
 		String(args.minRowId),
 		String(args.maxRowId)
-	].join("|")).slice(0, 16);
+	].join("|");
+	return sha256(input).slice(0, 16);
 }
 //#endregion
 export { computeChunkId };

package/dist/cold-store/config.js

@@ -6,9 +6,9 @@ import "../rolldown-runtime-ClRpJifh.js";
 */
 const DEFAULT_TABLE_CONFIG = {
 	warmTtlDays: 30,
-	minWarmTenantBytes: 5 * 1024 * 1024,
-	minChunkBytes: 1 * 1024 * 1024,
-	maxChunkBytes: 50 * 1024 * 1024,
+	minWarmTenantBytes: 5242880,
+	minChunkBytes: 1048576,
+	maxChunkBytes: 52428800,
 	maxRowsPerCycle: 5e4,
 	enabled: true
 };

package/dist/container-runtime.d.ts

@@ -0,0 +1,148 @@
+/**
+ * Container-runtime primitives shared by every site that starts a container for
+ * KiCI: the orchestrator's container scaler backend, its bare-metal backend in
+ * container mode, and the AGENT's own job-container sandbox.
+ *
+ * They live in `@kici-dev/shared` rather than in the orchestrator because the
+ * agent needs the same runtime injection: when an ordinary agent nests a job
+ * container from a customer's image, that image is no more likely to ship Node
+ * than the one a scaler spawns. Two copies of "materialize the KiCI runtime"
+ * would be two chances to disagree on the parts that are easy to get wrong —
+ * the volume key, the root-owned copy, and the self-verification.
+ *
+ * They are separate composable functions rather than one `spawnAgentContainer`
+ * that owns the whole sequence, because the create/start step is genuinely
+ * caller-specific — network isolation, label sets, bind lists and log capture
+ * differ per caller — while the steps below are identical everywhere.
+ *
+ * `dockerode` is a TYPE-ONLY import here: every function takes an already-built
+ * client, so `@kici-dev/shared` does not depend on it at runtime and a consumer
+ * that never imports this module never pulls it in.
+ */
+import { z } from 'zod';
+import type Docker from 'dockerode';
+/**
+ * When to pull an image.
+ *
+ * Defined here rather than in the orchestrator's scaler types because
+ * `pullImageIfMissing` lives here and both the orchestrator and the agent call
+ * it. The orchestrator's `scaler/types.ts` re-exports it, so the scaler config
+ * schema and every operator-facing value stay exactly as they were.
+ */
+export declare const ImagePullPolicy: z.ZodEnum<{
+    Always: "Always";
+    IfNotPresent: "IfNotPresent";
+    Never: "Never";
+}>;
+export type ImagePullPolicy = z.infer<typeof ImagePullPolicy>;
+/** Registry credentials in the shape a container runtime expects. */
+export interface RegistryAuthconfig {
+    username: string;
+    password: string;
+    serveraddress: string;
+}
+export interface PullImageOptions {
+    docker: Docker;
+    image: string;
+    /** Absent means an anonymous pull. */
+    authconfig?: RegistryAuthconfig | undefined;
+    signal?: AbortSignal | undefined;
+    /** Progress sink; callers surface this as a scaler event. */
+    onProgress?: ((message: string) => void) | undefined;
+    /**
+     * When to pull. Defaults to `IfNotPresent`, which is right for KiCI's own
+     * agent images (pinned and immutable, so re-pulling every spawn only storms
+     * the registry). A label set on a moving tag sets `Always`.
+     */
+    pullPolicy?: ImagePullPolicy | undefined;
+}
+/**
+ * Pull `image` according to the pull policy. Returns whether a pull ran.
+ *
+ * Authenticated when an authconfig is supplied — a private registry otherwise
+ * fails the pull with a 401 that reads like a missing image.
+ */
+export declare function pullImageIfMissing(opts: PullImageOptions): Promise<boolean>;
+/** Read-only mount point of the KiCI-provisioned runtime inside a container. */
+export declare const RUNTIME_MOUNT = "/opt/kici";
+/** Read-only mount point of the injected Node tree inside a job container. */
+export declare const RUNTIME_NODE_MOUNT = "/opt/kici/node";
+/**
+ * Bind spec that injects the KiCI runtime into a container.
+ *
+ * Read-only is load-bearing: a job that could rewrite the runtime would control
+ * the interpreter every later step runs under.
+ */
+export declare function runtimeInjectBind(hostRuntimeDir: string): string;
+/**
+ * Command that starts the agent from the INJECTED runtime.
+ *
+ * Only meaningful when the runtime is mounted. A spawned job image has its own
+ * default CMD — python's shell, a node REPL, whatever the customer's image
+ * declares — so without overriding it the container starts that instead and no
+ * agent ever registers. The job then waits for an agent that will never arrive.
+ *
+ * Absolute on both halves: the image is not required to ship Node, and the
+ * agent lives inside the runtime tree rather than at the image's own /app.
+ */
+export declare function injectedAgentCommand(): string[];
+/**
+ * Which part of an agent image's `/opt/kici` tree a caller needs.
+ *
+ * - `all` — node PLUS the agent application. What a spawn that runs the AGENT
+ *   ITSELF out of the volume needs (the per-job-image topologies), because the
+ *   entrypoint resolves inside the mounted tree.
+ * - `node` — the Node tree alone, mounted at `/opt/kici/node`. What an agent
+ *   nesting a job container needs: the runner bundle is bind-mounted from that
+ *   agent's OWN build, so the runner and the agent driving it can never be two
+ *   different versions.
+ *
+ * They are separate volumes rather than one tree mounted twice because the
+ * sandbox binds its runner and loader hook at fixed paths under `/opt/kici`.
+ * Mounting the whole tree read-only at `/opt/kici` would put those two binds
+ * INSIDE a read-only mount, whose mountpoints cannot be created.
+ */
+export declare const RuntimeSubtree: z.ZodEnum<{
+    all: "all";
+    node: "node";
+}>;
+export type RuntimeSubtree = z.infer<typeof RuntimeSubtree>;
+/**
+ * Name of the shared volume holding the KiCI runtime, keyed by image IDENTITY.
+ *
+ * Keyed by the image's content id, never its name:tag. A tag moves — `:stg`,
+ * `:latest`, and every E2E tag are rebuilt in place — so a name-keyed volume is
+ * reused after the image it was copied from has been replaced, which is exactly
+ * the "silently reusing the previous version's binaries" failure this is
+ * supposed to prevent. It bit: a volume populated from a pre-/opt/kici image was
+ * reused after the rebuild, and every spawned container failed to start because
+ * the runtime it mounted was empty.
+ *
+ * The subtree is part of the name for the same reason: an `all` volume and a
+ * `node` volume have different roots, so sharing one name would mount a tree
+ * whose `bin/node` is one level off.
+ */
+export declare function runtimeVolumeName(imageId: string, subtree?: RuntimeSubtree): string;
+export interface EnsureRuntimeVolumeOptions {
+    docker: Docker;
+    /** The published kici-agent image, which carries /opt/kici. */
+    agentImage: string;
+    /** Which part of the tree to materialize. Defaults to the whole runtime. */
+    subtree?: RuntimeSubtree | undefined;
+    signal?: AbortSignal | undefined;
+    onProgress?: ((message: string) => void) | undefined;
+}
+/**
+ * Materialize the KiCI runtime into a named volume that job containers mount.
+ *
+ * The runtime lives inside the `kici-agent` image, but a bind mount needs a
+ * HOST path — and the orchestrator (or the agent) may itself be containerized,
+ * so it cannot assume /opt/kici exists on the host filesystem. Copying the tree
+ * out of the image into a named volume once, then mounting that volume, works
+ * the same whether the caller runs on bare metal or in a container.
+ *
+ * Idempotent: the volume is created once per (agent image, subtree) and reused.
+ * Returns the volume name to mount.
+ */
+export declare function ensureRuntimeVolume(opts: EnsureRuntimeVolumeOptions): Promise<string>;
+//# sourceMappingURL=container-runtime.d.ts.map

package/dist/container-runtime.js

@@ -0,0 +1,298 @@
+import "./rolldown-runtime-ClRpJifh.js";
+import { z } from "zod";
+//#region src/container-runtime.ts
+/**
+* Container-runtime primitives shared by every site that starts a container for
+* KiCI: the orchestrator's container scaler backend, its bare-metal backend in
+* container mode, and the AGENT's own job-container sandbox.
+*
+* They live in `@kici-dev/shared` rather than in the orchestrator because the
+* agent needs the same runtime injection: when an ordinary agent nests a job
+* container from a customer's image, that image is no more likely to ship Node
+* than the one a scaler spawns. Two copies of "materialize the KiCI runtime"
+* would be two chances to disagree on the parts that are easy to get wrong —
+* the volume key, the root-owned copy, and the self-verification.
+*
+* They are separate composable functions rather than one `spawnAgentContainer`
+* that owns the whole sequence, because the create/start step is genuinely
+* caller-specific — network isolation, label sets, bind lists and log capture
+* differ per caller — while the steps below are identical everywhere.
+*
+* `dockerode` is a TYPE-ONLY import here: every function takes an already-built
+* client, so `@kici-dev/shared` does not depend on it at runtime and a consumer
+* that never imports this module never pulls it in.
+*/
+/**
+* When to pull an image.
+*
+* Defined here rather than in the orchestrator's scaler types because
+* `pullImageIfMissing` lives here and both the orchestrator and the agent call
+* it. The orchestrator's `scaler/types.ts` re-exports it, so the scaler config
+* schema and every operator-facing value stay exactly as they were.
+*/
+const ImagePullPolicy = z.enum([
+	"Always",
+	"IfNotPresent",
+	"Never"
+]);
+/**
+* Pull `image` according to the pull policy. Returns whether a pull ran.
+*
+* Authenticated when an authconfig is supplied — a private registry otherwise
+* fails the pull with a 401 that reads like a missing image.
+*/
+async function pullImageIfMissing(opts) {
+	const { docker, image, authconfig, signal } = opts;
+	const policy = opts.pullPolicy ?? ImagePullPolicy.enum.IfNotPresent;
+	if (policy === ImagePullPolicy.enum.Never) return false;
+	if (policy === ImagePullPolicy.enum.IfNotPresent) try {
+		await docker.getImage(image).inspect({ ...signal ? { abortSignal: signal } : {} });
+		return false;
+	} catch {}
+	if (signal?.aborted) throw signal.reason ?? /* @__PURE__ */ new Error("container spawn aborted");
+	opts.onProgress?.(`pulling image ${image}`);
+	const stream = await docker.pull(image, {
+		...signal ? { abortSignal: signal } : {},
+		...authconfig ? { authconfig } : {}
+	});
+	await new Promise((resolve, reject) => {
+		docker.modem.followProgress(stream, (err) => err ? reject(err) : resolve());
+	});
+	return true;
+}
+/** Read-only mount point of the KiCI-provisioned runtime inside a container. */
+const RUNTIME_MOUNT = "/opt/kici";
+/** Read-only mount point of the injected Node tree inside a job container. */
+const RUNTIME_NODE_MOUNT = `${RUNTIME_MOUNT}/node`;
+/**
+* Bind spec that injects the KiCI runtime into a container.
+*
+* Read-only is load-bearing: a job that could rewrite the runtime would control
+* the interpreter every later step runs under.
+*/
+function runtimeInjectBind(hostRuntimeDir) {
+	return `${hostRuntimeDir}:${RUNTIME_MOUNT}:ro`;
+}
+/**
+* Command that starts the agent from the INJECTED runtime.
+*
+* Only meaningful when the runtime is mounted. A spawned job image has its own
+* default CMD — python's shell, a node REPL, whatever the customer's image
+* declares — so without overriding it the container starts that instead and no
+* agent ever registers. The job then waits for an agent that will never arrive.
+*
+* Absolute on both halves: the image is not required to ship Node, and the
+* agent lives inside the runtime tree rather than at the image's own /app.
+*/
+function injectedAgentCommand() {
+	return [`${RUNTIME_MOUNT}/node/bin/node`, `${RUNTIME_MOUNT}/app/packages/agent/dist/server.js`];
+}
+/**
+* Which part of an agent image's `/opt/kici` tree a caller needs.
+*
+* - `all` — node PLUS the agent application. What a spawn that runs the AGENT
+*   ITSELF out of the volume needs (the per-job-image topologies), because the
+*   entrypoint resolves inside the mounted tree.
+* - `node` — the Node tree alone, mounted at `/opt/kici/node`. What an agent
+*   nesting a job container needs: the runner bundle is bind-mounted from that
+*   agent's OWN build, so the runner and the agent driving it can never be two
+*   different versions.
+*
+* They are separate volumes rather than one tree mounted twice because the
+* sandbox binds its runner and loader hook at fixed paths under `/opt/kici`.
+* Mounting the whole tree read-only at `/opt/kici` would put those two binds
+* INSIDE a read-only mount, whose mountpoints cannot be created.
+*/
+const RuntimeSubtree = z.enum(["all", "node"]);
+/**
+* Name of the shared volume holding the KiCI runtime, keyed by image IDENTITY.
+*
+* Keyed by the image's content id, never its name:tag. A tag moves — `:stg`,
+* `:latest`, and every E2E tag are rebuilt in place — so a name-keyed volume is
+* reused after the image it was copied from has been replaced, which is exactly
+* the "silently reusing the previous version's binaries" failure this is
+* supposed to prevent. It bit: a volume populated from a pre-/opt/kici image was
+* reused after the rebuild, and every spawned container failed to start because
+* the runtime it mounted was empty.
+*
+* The subtree is part of the name for the same reason: an `all` volume and a
+* `node` volume have different roots, so sharing one name would mount a tree
+* whose `bin/node` is one level off.
+*/
+function runtimeVolumeName(imageId, subtree = RuntimeSubtree.enum.all) {
+	const slug = imageId.replace(/^sha256:/, "").replace(/[^a-zA-Z0-9_.-]/g, "-").slice(0, 32);
+	return `kici-runtime-${subtree === RuntimeSubtree.enum.node ? "node-" : ""}${slug}`;
+}
+/** Where the populator and the marker probe mount the volume being filled. */
+const RUNTIME_OUT = "/kici-runtime-out";
+/**
+* Marker file proving the volume holds a COMPLETE runtime.
+*
+* Volume existence is not population. A volume is created before the copy runs,
+* so a concurrent caller's `inspect()` succeeds against a volume that is still
+* being filled, and it mounts a half-copied tree — which surfaces far from the
+* cause, typically as a zod ESM SyntaxError from a partially written module.
+* The marker is written LAST, after the populator's own `test -x` checks, so it
+* can only appear on a tree that already verified.
+*/
+const RUNTIME_COMPLETE_MARKER = `${RUNTIME_OUT}/.kici-runtime-complete`;
+/**
+* The self-verifying copy each subtree runs in the populator container.
+*
+* The verification is not decoration. `cp -a` of an absent or empty source
+* exits 0, leaving an EMPTY volume — and the spawn then creates a container
+* whose runtime mount has no node, which dies at start with no error recorded
+* anywhere near the cause.
+*
+* The completion marker is the LAST command in both branches: written any
+* earlier it would land on a tree that has not been verified, which is the
+* exact failure it exists to prevent.
+*/
+function populatorCommand(subtree) {
+	const out = RUNTIME_OUT;
+	if (subtree === RuntimeSubtree.enum.node) return `set -e; cp -a ${RUNTIME_MOUNT}/node/. ${out}/; test -x ${out}/bin/node; touch ${RUNTIME_COMPLETE_MARKER}`;
+	return `set -e; cp -a ${RUNTIME_MOUNT}/. ${out}/; test -x ${out}/node/bin/node; test -f ${out}/app/packages/agent/dist/server.js; touch ${RUNTIME_COMPLETE_MARKER}`;
+}
+/** What the populator container printed, for an error that has to explain itself. */
+async function populatorOutput(container) {
+	try {
+		const buf = await container.logs({
+			stdout: true,
+			stderr: true,
+			tail: 20
+		});
+		return Buffer.isBuffer(buf) ? buf.toString("utf-8").trim() : String(buf).trim();
+	} catch {
+		return "";
+	}
+}
+/**
+* Materialize the KiCI runtime into a named volume that job containers mount.
+*
+* The runtime lives inside the `kici-agent` image, but a bind mount needs a
+* HOST path — and the orchestrator (or the agent) may itself be containerized,
+* so it cannot assume /opt/kici exists on the host filesystem. Copying the tree
+* out of the image into a named volume once, then mounting that volume, works
+* the same whether the caller runs on bare metal or in a container.
+*
+* Idempotent: the volume is created once per (agent image, subtree) and reused.
+* Returns the volume name to mount.
+*/
+async function ensureRuntimeVolume(opts) {
+	const { docker, agentImage, signal } = opts;
+	const subtree = opts.subtree ?? RuntimeSubtree.enum.all;
+	await pullImageIfMissing({
+		docker,
+		image: agentImage,
+		...signal ? { signal } : {}
+	});
+	const name = runtimeVolumeName((await docker.getImage(agentImage).inspect()).Id ?? agentImage, subtree);
+	const inFlight = inFlightRuntimeVolumes.get(name);
+	if (inFlight) return await inFlight;
+	const populating = materializeRuntimeVolume({
+		...opts,
+		subtree,
+		name
+	}).finally(() => {
+		inFlightRuntimeVolumes.delete(name);
+	});
+	inFlightRuntimeVolumes.set(name, populating);
+	return await populating;
+}
+/** Per-volume populate in flight right now, so concurrent callers share one. */
+const inFlightRuntimeVolumes = /* @__PURE__ */ new Map();
+/**
+* Does this volume carry the completion marker?
+*
+* A short-lived container is the only way to read a named volume's contents:
+* the daemon owns the mount, and the caller may itself be containerized, so
+* there is no host path to stat. Mounted the same way the populator mounts it
+* (root, same bind) so the probe sees exactly what the populator wrote.
+*
+* Any failure to answer reads as NOT populated. Repopulating a good volume
+* costs one redundant copy; mounting an unverified one is the bug.
+*/
+async function runtimeVolumeIsPopulated(args) {
+	const { docker, name, agentImage } = args;
+	let probe;
+	try {
+		probe = await docker.createContainer({
+			Image: agentImage,
+			User: "0:0",
+			Cmd: [
+				"sh",
+				"-c",
+				`test -f ${RUNTIME_COMPLETE_MARKER}`
+			],
+			Labels: { "kici-managed": "true" },
+			HostConfig: {
+				Binds: [`${name}:${RUNTIME_OUT}`],
+				AutoRemove: false
+			}
+		});
+		await probe.start();
+		return (await probe.wait()).StatusCode === 0;
+	} catch {
+		return false;
+	} finally {
+		await probe?.remove({ force: true }).catch(() => void 0);
+	}
+}
+/** Populate the volume, reusing it only when the marker proves it is complete. */
+async function materializeRuntimeVolume(opts) {
+	const { docker, agentImage, subtree, name } = opts;
+	const exists = await docker.getVolume(name).inspect().then(() => true).catch(() => false);
+	if (exists && await runtimeVolumeIsPopulated({
+		docker,
+		name,
+		agentImage
+	})) return name;
+	opts.onProgress?.(`materializing the KiCI runtime from ${agentImage}`);
+	await docker.createVolume({
+		Name: name,
+		Labels: { "kici-managed": "true" }
+	});
+	const populator = await docker.createContainer({
+		Image: agentImage,
+		User: "0:0",
+		Cmd: [
+			"sh",
+			"-c",
+			populatorCommand(subtree)
+		],
+		Labels: { "kici-managed": "true" },
+		HostConfig: {
+			Binds: [`${name}:${RUNTIME_OUT}`],
+			AutoRemove: false
+		}
+	});
+	let populatorDropped = false;
+	const dropPopulator = async () => {
+		if (populatorDropped) return;
+		populatorDropped = true;
+		await populator.remove({ force: true }).catch(() => void 0);
+	};
+	try {
+		await populator.start();
+		const result = await populator.wait();
+		if (result.StatusCode !== 0) {
+			const output = await populatorOutput(populator);
+			throw new Error(`runtime copy exited ${result.StatusCode ?? "unknown"}` + (output ? `: ${output}` : " with no output"));
+		}
+	} catch (err) {
+		await dropPopulator();
+		if (!exists && !await runtimeVolumeIsPopulated({
+			docker,
+			name,
+			agentImage
+		})) await docker.getVolume(name).remove({ force: true }).catch(() => void 0);
+		throw new Error(`Failed to materialize the KiCI runtime from ${agentImage}: ${err instanceof Error ? err.message : String(err)}. The image must carry /opt/kici/node and /opt/kici/app.`);
+	} finally {
+		await dropPopulator();
+	}
+	return name;
+}
+//#endregion
+export { ImagePullPolicy, RUNTIME_MOUNT, RUNTIME_NODE_MOUNT, RuntimeSubtree, ensureRuntimeVolume, injectedAgentCommand, pullImageIfMissing, runtimeInjectBind, runtimeVolumeName };
+
+//# sourceMappingURL=container-runtime.js.map

package/dist/container-runtime.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=container-runtime.test.d.ts.map