@camstack/addon-training 0.0.0-stage → 0.1.1

package/dist/orchestrator/training.addon.js

@@ -0,0 +1,1712 @@
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const require_dist = require("../dist-WGwGDX6_.js");
+let node_path = require("node:path");
+node_path = require_dist.__toESM(node_path);
+let node_crypto = require("node:crypto");
+let node_fs = require("node:fs");
+let node_fs_promises = require("node:fs/promises");
+//#region src/orchestrator/api-ports.ts
+var DETECTOR_STEP = "object-detection";
+var REGISTRY_CAP = "custom-model-registry";
+/** The provider cap, routed to ONE provider by the `addonId` every call carries. */
+function providerPort(api) {
+	const p = api.trainingProvider;
+	return {
+		describe: () => p.describe.query({}),
+		validateCredentials: (i) => p.validateCredentials.mutate(i),
+		estimateJob: (i) => p.estimateJob.query(i),
+		uploadDataset: (i) => p.uploadDataset.mutate(i),
+		submitJob: (i) => p.submitJob.mutate(i),
+		getJob: (i) => p.getJob.query(i),
+		listJobEvents: (i) => p.listJobEvents.query(i),
+		cancelJob: (i) => p.cancelJob.mutate(i),
+		readResult: (i) => p.readResult.mutate(i),
+		offerArtifact: (i) => p.offerArtifact.mutate(i),
+		cleanupJob: (i) => p.cleanupJob.mutate(i),
+		listJobs: (i) => p.listJobs.query(i)
+	};
+}
+function datasetPort(api) {
+	return { offerArchive: async (input) => {
+		const offered = await api.pipelineAnalytics.offerRetrainArchive.mutate({
+			filter: { ...input.filter },
+			layout: input.layout
+		});
+		return {
+			ticket: offered.ticket,
+			frames: offered.frames
+		};
+	} };
+}
+/** Model Studio's registry: the hub's provider of `custom-model-registry`. */
+function registryPort(api) {
+	return { importBundle: async (input) => {
+		const hub = (await api.addons.listCapabilityProviders.query({ capName: REGISTRY_CAP })).find((p) => !p.addonId.includes("@"));
+		if (hub === void 0) throw new Error("no custom-model registry on the hub — is Model Studio installed?");
+		return api.customModelRegistry.importModelBundle.mutate({
+			...input,
+			addonId: hub.addonId
+		});
+	} };
+}
+function pinPort(api) {
+	const orchestrator = api.pipelineOrchestrator;
+	return {
+		listDetectorDevices: async () => {
+			const agents = await orchestrator.listAgentSettings.query();
+			const out = [];
+			for (const agent of agents) {
+				if (agent.settings.detect === false) continue;
+				const node = await orchestrator.getNodeInferenceDevices.query({ nodeId: agent.nodeId });
+				for (const device of node.devices) {
+					if (!device.enabled || device.exclusion !== null) continue;
+					out.push({
+						nodeId: agent.nodeId,
+						deviceKey: device.key,
+						pinnedModelId: device.steps?.["object-detection"]?.modelId ?? device.defaultModelId
+					});
+				}
+			}
+			return out;
+		},
+		getOverride: async (deviceId, nodeId, deviceKey) => {
+			const patch = (await orchestrator.getCameraStepOverrides.query({ deviceId }))?.[nodeId]?.[deviceKey]?.[DETECTOR_STEP];
+			if (patch === void 0) return null;
+			return {
+				...patch.modelId !== void 0 ? { modelId: patch.modelId } : {},
+				...patch.settings !== void 0 ? { settings: { ...patch.settings } } : {}
+			};
+		},
+		setOverride: async (deviceId, nodeId, deviceKey, patch) => {
+			await orchestrator.setCameraStepOverride.mutate({
+				deviceId,
+				agentNodeId: nodeId,
+				deviceKey,
+				addonId: DETECTOR_STEP,
+				patch: patch === null ? null : {
+					...patch.modelId !== void 0 ? { modelId: patch.modelId } : {},
+					...patch.settings !== void 0 ? { settings: { ...patch.settings } } : {}
+				}
+			});
+		}
+	};
+}
+//#endregion
+//#region src/orchestrator/artifact-staging.ts
+/**
+* The hub-side staging of a job's artefacts: `<dir>/<jobId>/bundle.tar` and
+* `result.json`. A bundle arrives over `ctx.peerBytes`, bounded by the size
+* result.json declared and hashed after it lands; it leaves the same way to
+* Model Studio. The whole job directory goes when the result expires (14 days
+* after ready / registration) or is discarded.
+*/
+var BUNDLE = "bundle.tar";
+var RESULT = "result.json";
+async function sha256File(file) {
+	const hash = (0, node_crypto.createHash)("sha256");
+	for await (const chunk of (0, node_fs.createReadStream)(file)) hash.update(chunk);
+	return hash.digest("hex");
+}
+/** Job ids are uuids; anything else never becomes a path. */
+function jobDir(root, jobId) {
+	if (!/^[0-9a-f-]{36}$/.test(jobId)) throw new Error(`not a job id: ${jobId}`);
+	return node_path.default.join(root, jobId);
+}
+function createDiskArtifacts(root, peer) {
+	return {
+		receiveBundle: async (jobId, ticket, maxBytes) => {
+			const dir = jobDir(root, jobId);
+			await (0, node_fs_promises.mkdir)(dir, { recursive: true });
+			const target = node_path.default.join(dir, BUNDLE);
+			const opened = await peer.open(ticket);
+			if (opened.kind === "refused") throw new Error(`bundle offer refused: ${opened.code} — ${opened.detail}`);
+			const written = await opened.body.writeToFile(target, { maxBytes });
+			if (written.kind === "refused") throw new Error(`bundle transfer refused: ${written.code} — ${written.detail}`);
+			return {
+				bytes: written.bytes,
+				sha256: await sha256File(target)
+			};
+		},
+		writeResult: async (jobId, text) => {
+			const dir = jobDir(root, jobId);
+			await (0, node_fs_promises.mkdir)(dir, { recursive: true });
+			await (0, node_fs_promises.writeFile)(node_path.default.join(dir, RESULT), text, "utf8");
+		},
+		readResult: async (jobId) => {
+			try {
+				return await (0, node_fs_promises.readFile)(node_path.default.join(jobDir(root, jobId), RESULT), "utf8");
+			} catch {
+				return null;
+			}
+		},
+		offerBundle: async (jobId) => {
+			const offered = await peer.offerFile({
+				path: node_path.default.join(jobDir(root, jobId), BUNDLE),
+				contentType: "application/x-tar",
+				label: "training-bundle"
+			});
+			if (offered.kind === "refused") throw new Error(`bundle offer refused: ${offered.code} — ${offered.detail}`);
+			return offered.ticket;
+		},
+		remove: async (jobId) => {
+			await (0, node_fs_promises.rm)(jobDir(root, jobId), {
+				recursive: true,
+				force: true
+			});
+		}
+	};
+}
+//#endregion
+//#region src/orchestrator/base-models.ts
+var HF_MODELS = "https://huggingface.co/camstack/camstack-models/resolve/main";
+var TIER_LABEL = {
+	t: "Tiny",
+	s: "Small",
+	m: "Medium",
+	c: "Compact"
+};
+function base(tier, res, floors, supplied = {}) {
+	return {
+		modelId: `yolov9${tier}-${String(res)}`,
+		label: `YOLOv9 ${TIER_LABEL[tier]} @${String(res)}`,
+		weights: `yolov9${tier}.pt`,
+		imgsz: res,
+		baselineOnnxUrl: `${HF_MODELS}/objectDetection/yolov9/onnx/camstack-yolov9${tier}-${String(res)}.onnx`,
+		baselineFloors: floors,
+		recommended: tier === "m" && res === 640,
+		tier,
+		suppliedLabelFloors: supplied.fp32 ?? {},
+		suppliedLabelFloorsInt8: supplied.int8 ?? {}
+	};
+}
+var BASE_MODELS = [
+	base("m", 640, {
+		person: .6,
+		vehicle: .4,
+		animal: .55
+	}, { int8: { "animal-classifier": .7 } }),
+	base("m", 320, {
+		person: .7,
+		vehicle: .5,
+		animal: .55
+	}, { int8: { "animal-classifier": .75 } }),
+	base("s", 640, {
+		person: .7,
+		vehicle: .4,
+		animal: .6
+	}),
+	base("s", 320, {
+		person: .7,
+		vehicle: .55,
+		animal: .4
+	}),
+	base("c", 640, {
+		person: .55,
+		vehicle: .45,
+		animal: .3
+	}, { int8: { "animal-classifier": .75 } }),
+	base("c", 320, {
+		person: .7,
+		vehicle: .45,
+		animal: .45
+	}),
+	base("t", 640, {
+		person: .65,
+		vehicle: .5,
+		animal: .5
+	}),
+	base("t", 320, {
+		person: .55,
+		vehicle: .45,
+		animal: .3
+	})
+];
+function findBaseModel(modelId) {
+	return BASE_MODELS.find((m) => m.modelId === modelId) ?? null;
+}
+/** The public shape (`listBaseModels`) — the internal fields stay here. */
+function publicBaseModel(spec) {
+	return {
+		modelId: spec.modelId,
+		label: spec.label,
+		weights: spec.weights,
+		imgsz: spec.imgsz,
+		baselineOnnxUrl: spec.baselineOnnxUrl,
+		baselineFloors: { ...spec.baselineFloors },
+		recommended: spec.recommended
+	};
+}
+function clip(text) {
+	const line = text.replace(/\s+$/u, "");
+	return line.length > 500 ? `${line.slice(0, 499)}…` : line;
+}
+/** Provider log chunks may hold several lines, or half of one; split on newlines. */
+function splitLogLines(lines) {
+	return lines.flatMap((l) => l.text.split(/\r?\n/u)).filter((t) => t.trim().length > 0);
+}
+function applyLogLines(job, lines) {
+	let stage = job.stage;
+	let epoch = job.progress.epoch;
+	let epochs = job.progress.epochs;
+	let etaSec = job.progress.etaSec;
+	const series = [...job.progress.series];
+	const tail = [...job.logTail];
+	for (const text of splitLogLines(lines)) {
+		tail.push(clip(text));
+		const event = require_dist.parseTrainingEventLine(text);
+		if (event === null) continue;
+		if (event.type === "stage") stage = event.stage;
+		if (event.type === "epoch") {
+			epoch = event.epoch;
+			epochs = event.epochs;
+			etaSec = event.etaSec ?? null;
+			const point = {
+				epoch: event.epoch,
+				map50: event.map50 ?? null,
+				map50_95: event.map50_95 ?? null
+			};
+			if (series.at(-1)?.epoch === point.epoch) series[series.length - 1] = point;
+			else series.push(point);
+		}
+	}
+	return {
+		stage,
+		progress: {
+			epoch,
+			epochs,
+			etaSec,
+			series: series.slice(-300)
+		},
+		logTail: tail.slice(-200)
+	};
+}
+//#endregion
+//#region src/orchestrator/job-mutator.ts
+var JobMutator = class {
+	jobs;
+	now;
+	chains = /* @__PURE__ */ new Map();
+	constructor(jobs, now) {
+		this.jobs = jobs;
+		this.now = now;
+	}
+	/** Apply `fn` to the CURRENT row; `null` from `fn` = write nothing. Returns the row after. */
+	async update(id, fn) {
+		return this.locked(id, async () => {
+			const job = await this.jobs.get(id);
+			if (job === null) return null;
+			const next = fn(job);
+			if (next === null) return job;
+			const stamped = {
+				...next,
+				updatedAt: this.now()
+			};
+			await this.jobs.put(stamped);
+			return stamped;
+		});
+	}
+	/** Run `body` under the job's lock (for a step that must not interleave). */
+	locked(id, body) {
+		const run = (this.chains.get(id) ?? Promise.resolve()).then(body, body);
+		const settled = run.then(() => void 0, () => void 0);
+		this.chains.set(id, settled);
+		settled.then(() => {
+			if (this.chains.get(id) === settled) this.chains.delete(id);
+		});
+		return run;
+	}
+};
+var DETECTOR_CLASSES = [
+	"person",
+	"vehicle",
+	"animal"
+];
+var PRESETS = {
+	quick: {
+		epochs: 15,
+		freeze: 10,
+		replay: 800
+	},
+	standard: {
+		epochs: 40,
+		freeze: 10,
+		replay: 1500
+	},
+	thorough: {
+		epochs: 80,
+		freeze: 0,
+		replay: 2500
+	}
+};
+/** Training images per second on an L4 at 640 (unmeasured first guess). */
+var L4_IMAGES_PER_SEC_640 = {
+	t: 180,
+	s: 120,
+	m: 60,
+	c: 45
+};
+/** Throughput relative to an L4 (unmeasured first guess, from public benchmarks). */
+var GPU_SPEED = {
+	T4: .6,
+	L4: 1,
+	A10G: 1.3,
+	L40S: 2.2,
+	"A100-40GB": 2.5,
+	"A100-80GB": 2.6,
+	H100: 3.5
+};
+/** Pull, replay download, evaluation and export — a fixed slice of every run. */
+var OVERHEAD_SEC = 900;
+/** A detector frame in the archive (a full-resolution JPEG), on average. */
+var BYTES_PER_FRAME = 350 * 1024;
+function cameraHoldout(stats) {
+	const withFrames = stats.cameras.filter((c) => c.completeFrames > 0).toSorted((a, b) => b.completeFrames - a.completeFrames || a.deviceId - b.deviceId);
+	if (withFrames.length < 6) return [];
+	return withFrames.slice(1, 3).map((c) => c.deviceId);
+}
+function supportOf(stats, cameras) {
+	const out = Object.fromEntries(DETECTOR_CLASSES.map((c) => [c, 0]));
+	for (const cell of stats.cells) {
+		if (!cameras.has(cell.deviceId)) continue;
+		out[cell.macroClass] = (out[cell.macroClass] ?? 0) + cell.subjects;
+	}
+	return out;
+}
+function planSplit(stats, requestedHoldout) {
+	const holdout = requestedHoldout ?? cameraHoldout(stats);
+	const warnings = [];
+	if (holdout.length === 0) {
+		warnings.push(`Fewer than ${String(6)} cameras: the last ${String(3)} days of every camera are held out instead of whole cameras.`);
+		return {
+			split: {
+				holdoutCameras: [],
+				holdoutLastDays: 3,
+				holdoutSupport: null,
+				trainSubjects: stats.subjects
+			},
+			warnings
+		};
+	}
+	const support = supportOf(stats, new Set(holdout));
+	const heldSubjects = Object.values(support).reduce((a, b) => a + b, 0);
+	for (const cls of DETECTOR_CLASSES) {
+		const n = support[cls] ?? 0;
+		if (n < 20) warnings.push(`${cls}: ${String(n)} held-out positives < ${String(20)} — its floor will inherit the baseline's.`);
+	}
+	return {
+		split: {
+			holdoutCameras: [...holdout],
+			holdoutLastDays: 0,
+			holdoutSupport: support,
+			trainSubjects: Math.max(0, stats.subjects - heldSubjects)
+		},
+		warnings
+	};
+}
+function estimateGpuSeconds(input) {
+	const preset = PRESETS[input.preset];
+	const images = input.frames + preset.replay;
+	const resFactor = (input.base.imgsz / 640) ** 2;
+	const throughput = L4_IMAGES_PER_SEC_640[input.base.tier] / resFactor * (GPU_SPEED[input.gpuType] ?? 1);
+	const nominal = images * preset.epochs / throughput + OVERHEAD_SEC;
+	return {
+		gpuSecondsLow: Math.round(nominal * .7),
+		gpuSecondsHigh: Math.round(nominal * 1.6)
+	};
+}
+function datasetBytesEstimate(frames) {
+	return frames * BYTES_PER_FRAME;
+}
+function uploadMinutes(bytes, mbps = 20) {
+	return Math.round(bytes * 8 / (mbps * 1e6) / 60 * 10) / 10;
+}
+/**
+* The default cost cap: the provider's own setting (operator decision
+* 2026-10-08 — it depends on the account), else 1.5× the high estimate.
+*/
+function defaultCostCap(usdHigh, providerCap) {
+	if (providerCap > 0) return providerCap;
+	return Math.max(1, Math.ceil(usdHigh * 1.5 * 100) / 100);
+}
+function knownGpu(gpus, gpuType) {
+	return gpus.some((g) => g.id === gpuType);
+}
+//#endregion
+//#region src/orchestrator/job-spec.ts
+/**
+* `job.json` for the trainer, built from a job record — every field filled
+* here, none left to a default (defaults do not run on the addon cap path,
+* and the trainer's spec has none either). Validated against the shared
+* contract before it leaves the hub.
+*/
+/** The trainer's evaluation and gate parameters — one place, one value. */
+var JOB_EVAL_DEFAULTS = {
+	valFraction: .15,
+	seed: 1,
+	batch: 16,
+	lr0: .002,
+	patience: 10,
+	calibrationFrames: 300,
+	minSupport: 20,
+	floorClamp: .15,
+	int8MaxMap50Drop: .02
+};
+/** The catalog id of a job's result: `custom-<base>-<first 8 of the job key>`. */
+function resultModelId(job) {
+	return `custom-${job.request.baseModelId}-${job.id.replaceAll("-", "").slice(0, 8)}`;
+}
+function buildJobSpec(input) {
+	const { job, base } = input;
+	const preset = PRESETS[job.request.preset];
+	const spec = {
+		schema: require_dist.TRAINING_JOB_SCHEMA,
+		jobKey: job.id,
+		target: {
+			kind: "detector",
+			step: "object-detection"
+		},
+		base: {
+			modelId: base.modelId,
+			weights: base.weights,
+			imgsz: base.imgsz
+		},
+		baseline: {
+			modelId: base.modelId,
+			onnxUrl: base.baselineOnnxUrl,
+			imgsz: base.imgsz,
+			floors: { ...base.baselineFloors }
+		},
+		dataset: {
+			versionId: input.versionId,
+			annotationHash: input.annotationHash,
+			sha256: input.datasetSha256
+		},
+		split: {
+			holdoutCameras: [...job.split.holdoutCameras],
+			holdoutLastDays: job.split.holdoutLastDays,
+			valFraction: JOB_EVAL_DEFAULTS.valFraction,
+			seed: JOB_EVAL_DEFAULTS.seed
+		},
+		hyper: {
+			epochs: preset.epochs,
+			batch: JOB_EVAL_DEFAULTS.batch,
+			lr0: JOB_EVAL_DEFAULTS.lr0,
+			freeze: job.request.freeze ?? preset.freeze,
+			patience: JOB_EVAL_DEFAULTS.patience,
+			seed: JOB_EVAL_DEFAULTS.seed
+		},
+		replay: {
+			source: "coco2017",
+			count: preset.replay
+		},
+		exports: [
+			"onnx",
+			"openvino-fp16",
+			"openvino-int8",
+			"coreml-fp16"
+		],
+		calibration: { maxFrames: JOB_EVAL_DEFAULTS.calibrationFrames },
+		floors: {
+			minSupport: JOB_EVAL_DEFAULTS.minSupport,
+			clamp: JOB_EVAL_DEFAULTS.floorClamp
+		},
+		int8Gate: { maxMap50Drop: JOB_EVAL_DEFAULTS.int8MaxMap50Drop },
+		catalog: {
+			id: resultModelId(job),
+			name: `${base.label} — ${job.name}`
+		}
+	};
+	return require_dist.TrainingJobSpecSchema.parse(spec);
+}
+function refused(reason, detail, dataNotice = "") {
+	return {
+		plan: {
+			refusal: {
+				reason,
+				detail
+			},
+			warnings: [],
+			split: null,
+			estimate: null,
+			defaults: null,
+			dataNotice
+		},
+		accepted: null
+	};
+}
+function dataNoticeFor(provider, region, frames) {
+	const where = region === "" ? "in the region the provider picks" : `in region "${region}"`;
+	const grace = provider.defaults.failedDatasetGraceHours;
+	return `${String(frames)} frames and their boxes are uploaded to YOUR ${provider.label} account, ${where}. ${provider.dataResidencyNote} They are deleted there as soon as the result is fetched (${String(grace)} h after a failure, so it can be inspected), and they are never put in a log, an environment variable or the job file. The result stays on this hub for ${String(14)} days; nothing is registered until you click Register.`;
+}
+async function planJob(request, deps) {
+	const version = await deps.versions.get(request.datasetVersionId);
+	if (version === null) return refused("unknown-dataset", `dataset version "${request.datasetVersionId}" does not exist`);
+	const provider = (await deps.provider.describe({})).find((p) => p.addonId === request.addonId);
+	if (provider === void 0) return refused("unknown-provider", `no training provider "${request.addonId}"`);
+	const notice = dataNoticeFor(provider, request.region, version.frameIds.length);
+	if (!provider.configured) return refused("provider-not-configured", `${provider.label} has no credentials yet — set them in its settings`, notice);
+	const base = findBaseModel(request.baseModelId);
+	if (base === null) return refused("unknown-base-model", `"${request.baseModelId}" is not a trainable base`, notice);
+	if (!knownGpu(provider.gpuTypes, request.gpuType)) return refused("unknown-gpu", `${provider.label} offers no "${request.gpuType}"`, notice);
+	const datasetBytes = datasetBytesEstimate(version.frameIds.length);
+	if (datasetBytes > provider.maxDatasetBytes) return refused("dataset-too-large", `about ${String(Math.round(datasetBytes / 1e6))} MB is over ${provider.label}'s limit`, notice);
+	const { split, warnings } = planSplit(version.stats, request.holdoutCameras);
+	if (split.trainSubjects <= 0) return refused("no-training-subjects", "no boxed subject is left to train on once the holdout is set aside", notice);
+	const gpu = estimateGpuSeconds({
+		frames: version.frameIds.length,
+		base,
+		preset: request.preset,
+		gpuType: request.gpuType
+	});
+	const usd = await deps.provider.estimateJob({
+		addonId: request.addonId,
+		gpuType: request.gpuType,
+		region: request.region,
+		datasetBytes,
+		gpuSecondsLow: gpu.gpuSecondsLow,
+		gpuSecondsHigh: gpu.gpuSecondsHigh
+	});
+	const maxCostUsd = defaultCostCap(usd.usdHigh, provider.defaults.costCapUsd);
+	const allWarnings = [...warnings];
+	if (usd.usdHigh > maxCostUsd) allWarnings.push(`The high estimate ($${usd.usdHigh.toFixed(2)}) is above the cost cap ($${maxCostUsd.toFixed(2)}): the job may be stopped before it finishes.`);
+	if (version.stats.completeness < .5) allWarnings.push("Fewer than half of the frames are marked complete: evaluation uses complete frames only, so the comparison rests on few frames.");
+	return {
+		plan: {
+			refusal: null,
+			warnings: allWarnings,
+			split,
+			estimate: {
+				datasetBytes,
+				uploadMinutesAt20Mbps: uploadMinutes(datasetBytes),
+				gpuSecondsLow: gpu.gpuSecondsLow,
+				gpuSecondsHigh: gpu.gpuSecondsHigh,
+				usdLow: usd.usdLow,
+				usdHigh: usd.usdHigh,
+				notes: ["GPU time is a first estimate (not yet measured on real runs); the cost cap bounds a wrong guess.", ...usd.notes]
+			},
+			defaults: {
+				maxCostUsd,
+				maxRuntimeSec: provider.defaults.maxRuntimeSec
+			},
+			dataNotice: notice
+		},
+		accepted: {
+			version,
+			provider,
+			base,
+			split
+		}
+	};
+}
+//#endregion
+//#region src/orchestrator/job-engine.ts
+/**
+* The job state machine (D805): start → upload → submit → poll → collect →
+* ready, with every failure path named and every terminal path cleaning the
+* cloud up.
+*
+* Registry-driven, not timer-driven: the reconcile tick compares every
+* non-terminal job with what the provider says (on boot, then every 30 s), so
+* a hub restart mid-run loses nothing — the job record and the provider's
+* jobKey tag are the whole truth. Background work (the upload, the collect)
+* is tracked in memory; a job found in one of those states with no task
+* behind it was interrupted by a restart and is handled as such.
+*/
+var RECONCILE_INTERVAL_MS = 3e4;
+var ORPHAN_SWEEP_INTERVAL_MS = 1440 * 6e4;
+var ARTIFACT_RETENTION_MS = 14 * (1440 * 6e4);
+/** The provider enforces maxRuntimeSec; past it by this much, the hub cancels too. */
+var RUNTIME_GRACE_MS = 10 * 6e4;
+var EVENT_PAGE = 500;
+var EVENT_PAGES_PER_POLL = 5;
+/** States the reconcile tick polls the provider for. */
+var POLLED = new Set([
+	"submitted",
+	"starting",
+	"running",
+	"cancelling"
+]);
+/** States from which nothing more happens in the cloud. */
+var SETTLED = new Set([
+	"ready",
+	"registered",
+	"canary",
+	"failed",
+	"cancelled",
+	"lost",
+	"expired",
+	"discarded"
+]);
+/** A job in one of these holds (or is about to hold) a GPU. */
+var ACTIVE = new Set([
+	"uploading",
+	"submitted",
+	"starting",
+	"running",
+	"collecting",
+	"cancelling"
+]);
+function errText(err) {
+	return err instanceof Error ? err.message : String(err);
+}
+var TrainingEngine = class {
+	ports;
+	trainer;
+	mutator;
+	tasks = /* @__PURE__ */ new Map();
+	constructor(ports, trainer) {
+		this.ports = ports;
+		this.trainer = trainer;
+		this.mutator = new JobMutator(ports.jobs, ports.now);
+	}
+	/** Await every background task (tests, shutdown). */
+	async settle() {
+		while (this.tasks.size > 0) await Promise.allSettled([...this.tasks.values()]);
+	}
+	async startJob(input) {
+		const existing = await this.ports.jobs.findByIdempotencyKey(input.idempotencyKey);
+		if (existing !== null) return {
+			kind: "started",
+			job: existing
+		};
+		const active = (await this.ports.jobs.list()).find((j) => ACTIVE.has(j.state));
+		if (active !== void 0) return {
+			kind: "refused",
+			reason: "job-already-running",
+			detail: `"${active.name}" is still ${active.state}`
+		};
+		const planned = await planJob(input, {
+			versions: this.ports.versions,
+			provider: this.ports.provider
+		});
+		if (planned.plan.refusal !== null || planned.accepted === null) {
+			const refusal = planned.plan.refusal ?? {
+				reason: "unknown-dataset",
+				detail: "no plan"
+			};
+			return {
+				kind: "refused",
+				reason: refusal.reason,
+				detail: refusal.detail
+			};
+		}
+		const { version, split } = planned.accepted;
+		const now = this.ports.now();
+		const { name, maxCostUsd, maxRuntimeSec, idempotencyKey, ...request } = input;
+		const job = {
+			id: this.ports.newId(),
+			idempotencyKey,
+			name,
+			createdAt: now,
+			updatedAt: now,
+			state: "uploading",
+			stage: null,
+			failure: null,
+			request,
+			maxCostUsd,
+			maxRuntimeSec,
+			trainer: this.trainer,
+			datasetLayout: version.layout,
+			dataset: null,
+			split,
+			provider: {
+				state: null,
+				providerJobId: null,
+				startedAt: null,
+				endedAt: null,
+				billedSeconds: 0,
+				costUsd: 0,
+				costIsEstimate: true,
+				lastPolledAt: null
+			},
+			progress: {
+				epoch: null,
+				epochs: null,
+				etaSec: null,
+				series: []
+			},
+			logTail: [],
+			logCursor: null,
+			artifacts: null,
+			cloudDatasetDeletedAt: null,
+			cloudCleanupDueAt: null,
+			registeredModelIds: [],
+			canary: null
+		};
+		await this.ports.jobs.put(job);
+		this.ports.logger.info("training job started", { meta: {
+			jobId: job.id,
+			provider: request.addonId,
+			gpu: request.gpuType,
+			frames: version.frameIds.length
+		} });
+		this.runTask(job.id, () => this.uploadAndSubmit(job.id));
+		return {
+			kind: "started",
+			job
+		};
+	}
+	async cancelJob(id) {
+		const job = await this.ports.jobs.get(id);
+		if (job === null) throw new Error(`training job "${id}" does not exist`);
+		if (!ACTIVE.has(job.state) || job.state === "cancelling") return job;
+		const moved = await this.mutator.update(id, (j) => ({
+			...j,
+			state: "cancelling"
+		}));
+		if (job.state !== "uploading") try {
+			await this.ports.provider.cancelJob({
+				addonId: job.request.addonId,
+				jobKey: id
+			});
+		} catch (err) {
+			this.ports.logger.warn("training cancel call failed — the next reconcile retries", { meta: {
+				jobId: id,
+				error: errText(err)
+			} });
+		}
+		return moved ?? job;
+	}
+	/** One reconcile pass over every job. Never throws; each job fails alone. */
+	async tick() {
+		const jobs = await this.ports.jobs.list();
+		const now = this.ports.now();
+		for (const job of jobs) try {
+			await this.reconcile(job, now);
+		} catch (err) {
+			this.ports.logger.warn("training reconcile failed for one job", { meta: {
+				jobId: job.id,
+				state: job.state,
+				error: errText(err)
+			} });
+		}
+	}
+	/** Cloud jobs we hold no live record of are cancelled and cleaned (they burn credit). */
+	async sweepOrphans() {
+		const known = new Map((await this.ports.jobs.list()).map((j) => [j.id, j]));
+		const swept = [];
+		for (const provider of await this.ports.provider.describe({})) {
+			if (!provider.configured) continue;
+			for (const status of await this.ports.provider.listJobs({ addonId: provider.addonId })) {
+				const record = known.get(status.jobKey);
+				const live = record !== void 0 && !SETTLED.has(record.state);
+				const stillRunning = status.state === "pending" || status.state === "starting" || status.state === "running";
+				if (live || record !== void 0 && !stillRunning) continue;
+				this.ports.logger.warn("training orphan swept — a cloud job with no live record", { meta: {
+					jobKey: status.jobKey,
+					provider: provider.addonId,
+					state: status.state
+				} });
+				await this.ports.provider.cancelJob({
+					addonId: provider.addonId,
+					jobKey: status.jobKey
+				});
+				await this.ports.provider.cleanupJob({
+					addonId: provider.addonId,
+					jobKey: status.jobKey
+				});
+				swept.push(status.jobKey);
+			}
+		}
+		return swept;
+	}
+	runTask(id, body) {
+		if (this.tasks.has(id)) return;
+		const task = body().catch((err) => {
+			this.ports.logger.warn("training background step failed", { meta: {
+				jobId: id,
+				error: errText(err)
+			} });
+		}).finally(() => this.tasks.delete(id));
+		this.tasks.set(id, task);
+	}
+	async uploadAndSubmit(id) {
+		const job = await this.ports.jobs.get(id);
+		if (job === null) return;
+		const version = await this.ports.versions.get(job.request.datasetVersionId);
+		const base = findBaseModel(job.request.baseModelId);
+		if (version === null || base === null) {
+			await this.fail(id, "upload-failed", null, "the dataset version or base model disappeared");
+			return;
+		}
+		let uploaded;
+		try {
+			const offer = await this.ports.datasets.offerArchive({
+				filter: {
+					frameIds: [...version.frameIds],
+					...version.completeOnly ? { completeOnly: true } : {}
+				},
+				layout: version.layout
+			});
+			uploaded = await this.ports.provider.uploadDataset({
+				addonId: job.request.addonId,
+				jobKey: id,
+				ticket: offer.ticket
+			});
+		} catch (err) {
+			await this.fail(id, "upload-failed", "prepare", errText(err));
+			await this.cleanupCloud(id);
+			return;
+		}
+		const afterUpload = await this.mutator.update(id, (j) => ({
+			...j,
+			dataset: {
+				bytes: uploaded.bytes,
+				sha256: uploaded.sha256,
+				uploadedAt: this.ports.now()
+			}
+		}));
+		if (afterUpload?.state === "cancelling") {
+			await this.finishCancelled(id);
+			return;
+		}
+		try {
+			const spec = buildJobSpec({
+				job: afterUpload ?? job,
+				base,
+				versionId: version.id,
+				annotationHash: version.annotationHash,
+				datasetSha256: uploaded.sha256
+			});
+			const status = await this.ports.provider.submitJob({
+				addonId: job.request.addonId,
+				jobKey: id,
+				trainer: job.trainer,
+				jobSpecJson: JSON.stringify(spec),
+				gpuType: job.request.gpuType,
+				region: job.request.region,
+				maxRuntimeSec: job.maxRuntimeSec
+			});
+			await this.mutator.update(id, (j) => this.withStatus({
+				...j,
+				state: "submitted"
+			}, status));
+		} catch (err) {
+			await this.fail(id, "submit-failed", "prepare", errText(err));
+			await this.cleanupCloud(id);
+		}
+	}
+	async reconcile(job, now) {
+		if (job.state === "uploading" && !this.tasks.has(job.id)) {
+			await this.fail(job.id, "hub-restart-during-upload", "prepare", "the hub restarted while the dataset was uploading");
+			await this.cleanupCloud(job.id);
+			return;
+		}
+		if (POLLED.has(job.state)) {
+			await this.poll(job, now);
+			return;
+		}
+		if (job.state === "collecting") {
+			this.runTask(job.id, () => this.collect(job.id));
+			return;
+		}
+		await this.housekeep(job, now);
+	}
+	async housekeep(job, now) {
+		if (job.cloudDatasetDeletedAt === null && SETTLED.has(job.state)) {
+			if (job.state !== "failed" || (job.cloudCleanupDueAt ?? 0) <= now) await this.cleanupCloud(job.id);
+		}
+		if (job.artifacts !== null && job.artifacts.expiresAt <= now) {
+			await this.ports.artifacts.remove(job.id);
+			await this.mutator.update(job.id, (j) => ({
+				...j,
+				artifacts: null,
+				state: j.state === "ready" ? "expired" : j.state
+			}));
+			this.ports.logger.info("training artefacts expired from the hub", { meta: { jobId: job.id } });
+		}
+	}
+	async poll(job, now) {
+		const ref = {
+			addonId: job.request.addonId,
+			jobKey: job.id
+		};
+		const status = await this.ports.provider.getJob(ref);
+		let cursor = job.logCursor ?? void 0;
+		let progressed = {
+			stage: job.stage,
+			progress: job.progress,
+			logTail: job.logTail
+		};
+		for (let page = 0; page < EVENT_PAGES_PER_POLL; page += 1) {
+			const batch = await this.ports.provider.listJobEvents({
+				...ref,
+				max: EVENT_PAGE,
+				...cursor !== void 0 ? { cursor } : {}
+			});
+			progressed = applyLogLines(progressed, batch.lines);
+			cursor = batch.cursor;
+			if (batch.lines.length < EVENT_PAGE) break;
+		}
+		const updated = await this.mutator.update(job.id, (j) => this.withStatus({
+			...j,
+			...progressed,
+			logCursor: cursor ?? null
+		}, status));
+		if (updated === null) return;
+		await this.afterPoll(updated, status, now);
+	}
+	async afterPoll(job, status, now) {
+		const ref = {
+			addonId: job.request.addonId,
+			jobKey: job.id
+		};
+		const running = status.state === "pending" || status.state === "starting" || status.state === "running";
+		if (job.state === "cancelling") {
+			if (!running) await this.finishCancelled(job.id);
+			return;
+		}
+		if (running && job.provider.costUsd >= job.maxCostUsd) {
+			await this.ports.provider.cancelJob(ref);
+			await this.fail(job.id, "cost-cap", job.stage, `spent $${job.provider.costUsd.toFixed(2)} of a $${job.maxCostUsd.toFixed(2)} cap`);
+			return;
+		}
+		const started = job.provider.startedAt;
+		if (running && started !== null && now - started > job.maxRuntimeSec * 1e3 + RUNTIME_GRACE_MS) {
+			await this.ports.provider.cancelJob(ref);
+			await this.fail(job.id, "runtime-cap", job.stage, `ran past ${String(job.maxRuntimeSec)} s`);
+			return;
+		}
+		if (status.state === "succeeded") {
+			await this.mutator.update(job.id, (j) => ({
+				...j,
+				state: "collecting"
+			}));
+			this.runTask(job.id, () => this.collect(job.id));
+			return;
+		}
+		if (status.state === "failed") {
+			const detail = await this.failureDetail(job);
+			await this.fail(job.id, "trainer-failed", job.stage, detail ?? status.failureReason ?? `exit ${String(status.exitCode ?? "?")}`);
+			return;
+		}
+		if (status.state === "cancelled") {
+			await this.fail(job.id, "provider-error", job.stage, "the provider stopped the job");
+			return;
+		}
+		if (status.state === "lost") await this.mutator.update(job.id, (j) => ({
+			...j,
+			state: "lost"
+		}));
+	}
+	async failureDetail(job) {
+		try {
+			const raw = await this.ports.provider.readResult({
+				addonId: job.request.addonId,
+				jobKey: job.id
+			});
+			if (!raw.found || raw.text === void 0) return null;
+			const parsed = require_dist.TrainingResultSchema.safeParse(JSON.parse(raw.text));
+			const error = parsed.success ? parsed.data.error : null;
+			return error === null ? null : `${error.stage}: ${error.code} — ${error.message}`;
+		} catch {
+			return null;
+		}
+	}
+	async collect(id) {
+		const job = await this.ports.jobs.get(id);
+		if (job === null || job.state !== "collecting") return;
+		const ref = {
+			addonId: job.request.addonId,
+			jobKey: id
+		};
+		try {
+			const raw = await this.ports.provider.readResult(ref);
+			if (!raw.found || raw.text === void 0) {
+				await this.fail(id, "result-invalid", "package", "the job ended without a result.json");
+				return;
+			}
+			const parsed = require_dist.TrainingResultSchema.safeParse(JSON.parse(raw.text));
+			if (!parsed.success) {
+				await this.fail(id, "result-invalid", "package", parsed.error.issues[0]?.message ?? "unreadable");
+				return;
+			}
+			const result = parsed.data;
+			if (result.status !== "succeeded" || result.bundle == null || result.catalogDraft == null || result.files === void 0) {
+				const why = result.error === null ? "the result names no bundle" : `${result.error.stage}: ${result.error.message}`;
+				await this.fail(id, result.status === "failed" ? "trainer-failed" : "result-invalid", "package", why);
+				return;
+			}
+			const { ticket } = await this.ports.provider.offerArtifact({
+				...ref,
+				path: "bundle.tar"
+			});
+			const got = await this.ports.artifacts.receiveBundle(id, ticket, result.bundle.bytes);
+			if (got.sha256 !== result.bundle.sha256 || got.bytes !== result.bundle.bytes) {
+				await this.ports.artifacts.remove(id);
+				await this.fail(id, "checksum-mismatch", "package", "the fetched bundle does not match result.json");
+				return;
+			}
+			await this.ports.artifacts.writeResult(id, raw.text);
+			const now = this.ports.now();
+			await this.mutator.update(id, (j) => ({
+				...j,
+				state: "ready",
+				artifacts: {
+					bundleBytes: got.bytes,
+					bundleSha256: got.sha256,
+					fetchedAt: now,
+					expiresAt: now + ARTIFACT_RETENTION_MS
+				}
+			}));
+			this.ports.logger.info("training result fetched and verified", { meta: {
+				jobId: id,
+				bytes: got.bytes
+			} });
+			await this.cleanupCloud(id);
+		} catch (err) {
+			await this.fail(id, "fetch-failed", "package", errText(err));
+		}
+	}
+	async finishCancelled(id) {
+		await this.mutator.update(id, (j) => ({
+			...j,
+			state: "cancelled"
+		}));
+		await this.cleanupCloud(id);
+	}
+	/** Delete the cloud copy (dataset first). A failure is retried by the next tick. */
+	async cleanupCloud(id) {
+		const job = await this.ports.jobs.get(id);
+		if (job === null || job.cloudDatasetDeletedAt !== null) return;
+		try {
+			const { deleted } = await this.ports.provider.cleanupJob({
+				addonId: job.request.addonId,
+				jobKey: id
+			});
+			await this.mutator.update(id, (j) => ({
+				...j,
+				cloudDatasetDeletedAt: this.ports.now(),
+				cloudCleanupDueAt: null
+			}));
+			this.ports.logger.info("training cloud copy deleted", { meta: {
+				jobId: id,
+				deleted: [...deleted]
+			} });
+		} catch (err) {
+			this.ports.logger.warn("training cloud cleanup failed — retried on the next reconcile", { meta: {
+				jobId: id,
+				error: errText(err)
+			} });
+		}
+	}
+	async fail(id, reason, stage, detail) {
+		let grace = 0;
+		const job = await this.ports.jobs.get(id);
+		if (job !== null && reason === "trainer-failed") grace = ((await this.ports.provider.describe({})).find((p) => p.addonId === job.request.addonId)?.defaults.failedDatasetGraceHours ?? 0) * 60 * 6e4;
+		await this.mutator.update(id, (j) => ({
+			...j,
+			state: "failed",
+			failure: {
+				reason,
+				stage,
+				detail
+			},
+			cloudCleanupDueAt: this.ports.now() + grace
+		}));
+		this.ports.logger.warn("training job failed", { meta: {
+			jobId: id,
+			reason,
+			stage,
+			detail
+		} });
+	}
+	withStatus(job, status) {
+		const mapped = job.state === "cancelling" ? "cancelling" : status.state === "pending" ? "submitted" : status.state === "starting" ? "starting" : status.state === "running" ? "running" : job.state;
+		return {
+			...job,
+			state: mapped,
+			provider: {
+				state: status.state,
+				providerJobId: status.providerJobId ?? job.provider.providerJobId,
+				startedAt: status.startedAt ?? job.provider.startedAt,
+				endedAt: status.endedAt ?? job.provider.endedAt,
+				billedSeconds: status.billedSeconds,
+				costUsd: status.costUsd,
+				costIsEstimate: status.costIsEstimate,
+				lastPolledAt: this.ports.now()
+			}
+		};
+	}
+};
+//#endregion
+//#region src/orchestrator/return-flow.ts
+/**
+* The return flow (D805, D806): a verified result → Model Studio → a canary
+* on chosen cameras → rollback. Every step is an operator click; nothing here
+* runs on its own (operator decision 2026-10-08: never auto-register).
+*
+* Register hands the staged bundle to `custom-model-registry.importModelBundle`
+* over `ctx.peerBytes`, with the catalog rows completed here (labels, licence,
+* the base's D741 floors) and every format URL rewritten to
+* `camstack-local://`. A canary pins the camera's root detector, on every
+* accelerator it can land on, to the variant that matches what that
+* accelerator ran (INT8 where the base pin was INT8), and remembers EXACTLY
+* what it replaced so a rollback restores it.
+*/
+var DETECTOR_STEP_ID = "object-detection";
+var TRAINER_SOURCE_URL = "https://huggingface.co/camstack/camstack-trainer";
+function localFormat(id, fmt) {
+	if (fmt === void 0) return void 0;
+	return {
+		url: `camstack-local://${id}/${fmt.url}`,
+		sizeMB: fmt.sizeMB,
+		revision: fmt.revision,
+		...fmt.isDirectory !== void 0 ? { isDirectory: fmt.isDirectory } : {},
+		...fmt.files !== void 0 ? { files: [...fmt.files] } : {},
+		...fmt.runtimes !== void 0 ? { runtimes: [...fmt.runtimes] } : {}
+	};
+}
+/** A trainer draft completed into a registry row. */
+function catalogEntryFromDraft(input) {
+	const { draft, job, result, base } = input;
+	const supplied = input.int8 ? base.suppliedLabelFloorsInt8 : base.suppliedLabelFloors;
+	const formats = {};
+	const onnx = localFormat(draft.id, draft.formats.onnx);
+	const openvino = localFormat(draft.id, draft.formats.openvino);
+	const coreml = localFormat(draft.id, draft.formats.coreml);
+	if (onnx !== void 0) formats.onnx = onnx;
+	if (openvino !== void 0) formats.openvino = openvino;
+	if (coreml !== void 0) formats.coreml = coreml;
+	return {
+		id: draft.id,
+		name: draft.name,
+		description: `${draft.description} Fine-tuned from ${base.modelId} by CamStack Training (job ${job.id}, dataset ${job.request.datasetVersionId}).`,
+		inputSize: { ...draft.inputSize },
+		labels: [],
+		preprocessMode: "letterbox",
+		defaultClassFloors: { ...draft.defaultClassFloors },
+		...Object.keys(supplied).length > 0 ? { suppliedLabelFloors: { ...supplied } } : {},
+		license: {
+			weights: "AGPL-3.0-only",
+			code: "AGPL-3.0-only",
+			upstream: {
+				name: "Ultralytics YOLOv9",
+				url: "https://github.com/ultralytics/ultralytics"
+			},
+			attribution: `© Ultralytics, AGPL-3.0 (YOLOv9 weights); ${result.licence}. Corresponding source: ${TRAINER_SOURCE_URL} at revision ${result.trainer.revision}.`,
+			modifications: `fine-tuned on the operator's own frames (dataset ${job.request.datasetVersionId}) and exported by CamStack Training`
+		},
+		formats
+	};
+}
+/** Which registered variant each (camera, accelerator) gets: INT8 where the base pin was INT8. */
+function planCanary(input) {
+	return input.deviceIds.flatMap((deviceId) => input.devices.map((d) => ({
+		deviceId,
+		nodeId: d.nodeId,
+		deviceKey: d.deviceKey,
+		modelId: input.int8Id !== null && d.pinnedModelId?.endsWith("-int8") === true ? input.int8Id : input.baseId
+	})));
+}
+var ReturnFlow = class {
+	ports;
+	mutator;
+	constructor(ports, mutator) {
+		this.ports = ports;
+		this.mutator = mutator;
+	}
+	async readResult(job) {
+		if (job.artifacts === null) return null;
+		const text = await this.ports.artifacts.readResult(job.id);
+		if (text === null) return null;
+		const parsed = require_dist.TrainingResultSchema.safeParse(JSON.parse(text));
+		return parsed.success ? parsed.data : null;
+	}
+	async register(id, canaryDeviceIds) {
+		const job = await this.requireJob(id);
+		if (job.state !== "ready") throw new Error(`only a ready result can be registered (this one is ${job.state})`);
+		const result = await this.readResult(job);
+		const base = findBaseModel(job.request.baseModelId);
+		if (result === null || result.catalogDraft == null || result.files === void 0 || job.artifacts === null || base === null) throw new Error("the staged result is missing or unreadable — fetch it again or discard the job");
+		const entries = [catalogEntryFromDraft({
+			draft: result.catalogDraft.base,
+			job,
+			result,
+			base,
+			int8: false
+		})];
+		if (result.catalogDraft.int8 !== null) entries.push(catalogEntryFromDraft({
+			draft: result.catalogDraft.int8,
+			job,
+			result,
+			base,
+			int8: true
+		}));
+		const ticket = await this.ports.artifacts.offerBundle(id);
+		const answer = await this.ports.registry.importBundle({
+			stepId: DETECTOR_STEP_ID,
+			entries,
+			bundle: {
+				ticket,
+				bytes: job.artifacts.bundleBytes,
+				sha256: job.artifacts.bundleSha256
+			},
+			files: result.files.map((f) => ({
+				path: f.path,
+				bytes: f.bytes,
+				sha256: f.sha256
+			})),
+			distribute: true
+		});
+		if (answer.kind === "refused") throw new Error(`Model Studio refused the bundle: ${answer.reason} — ${answer.detail}`);
+		const now = this.ports.now();
+		const registered = await this.mutator.update(id, (j) => ({
+			...j,
+			state: "registered",
+			registeredModelIds: [...answer.modelIds],
+			artifacts: j.artifacts === null ? null : {
+				...j.artifacts,
+				expiresAt: now + ARTIFACT_RETENTION_MS
+			}
+		}));
+		this.ports.logger.info("training result registered", { meta: {
+			jobId: id,
+			modelIds: [...answer.modelIds],
+			distributed: answer.distributed.length
+		} });
+		if (canaryDeviceIds.length > 0) return this.canary(id, canaryDeviceIds);
+		return registered ?? job;
+	}
+	async canary(id, deviceIds) {
+		const job = await this.requireJob(id);
+		if (job.state !== "registered" && job.state !== "canary") throw new Error("register the result before a canary");
+		const [baseId, int8Id] = job.registeredModelIds;
+		if (baseId === void 0) throw new Error("no registered model to canary");
+		const already = new Set(job.canary?.deviceIds ?? []);
+		const fresh = deviceIds.filter((d) => !already.has(d));
+		const writes = planCanary({
+			deviceIds: fresh,
+			devices: await this.ports.pins.listDetectorDevices(),
+			baseId,
+			int8Id: int8Id ?? null
+		});
+		const previous = [];
+		for (const w of writes) {
+			const before = await this.ports.pins.getOverride(w.deviceId, w.nodeId, w.deviceKey);
+			previous.push({
+				deviceId: w.deviceId,
+				nodeId: w.nodeId,
+				deviceKey: w.deviceKey,
+				patch: before === null ? null : { ...before }
+			});
+			await this.ports.pins.setOverride(w.deviceId, w.nodeId, w.deviceKey, { modelId: w.modelId });
+		}
+		const updated = await this.mutator.update(id, (j) => ({
+			...j,
+			state: "canary",
+			canary: {
+				deviceIds: [...already, ...fresh],
+				since: j.canary?.since ?? this.ports.now(),
+				previous: [...j.canary?.previous ?? [], ...previous]
+			}
+		}));
+		this.ports.logger.info("training canary pinned", { meta: {
+			jobId: id,
+			cameras: fresh,
+			writes: writes.length
+		} });
+		return updated ?? job;
+	}
+	async rollback(id) {
+		const job = await this.requireJob(id);
+		if (job.canary === null) return job;
+		for (const p of job.canary.previous) await this.ports.pins.setOverride(p.deviceId, p.nodeId, p.deviceKey, p.patch);
+		const updated = await this.mutator.update(id, (j) => ({
+			...j,
+			state: "registered",
+			canary: null
+		}));
+		this.ports.logger.info("training canary rolled back", { meta: {
+			jobId: id,
+			restored: job.canary.previous.length
+		} });
+		return updated ?? job;
+	}
+	async discard(id) {
+		const job = await this.requireJob(id);
+		if (job.state !== "ready") throw new Error(`only an unregistered result can be discarded (this one is ${job.state})`);
+		await this.ports.artifacts.remove(id);
+		return await this.mutator.update(id, (j) => ({
+			...j,
+			state: "discarded",
+			artifacts: null
+		})) ?? job;
+	}
+	async requireJob(id) {
+		const job = await this.ports.jobs.get(id);
+		if (job === null) throw new Error(`training job "${id}" does not exist`);
+		return job;
+	}
+};
+//#endregion
+//#region src/orchestrator/stores.ts
+/**
+* The orchestrator's two durable tables, both in the settings store.
+*
+* Rows are validated on the way OUT (a row a newer build wrote, or a hand
+* edit, never reaches the state machine as a half-shaped job) and dropped
+* with a log line when they do not parse — a job that cannot be read cannot
+* be reconciled, and saying so beats acting on a guess.
+*/
+/**
+* @durable class=ledger owner=training
+*   write="one row per training job, rewritten on every state change by the job engine
+*     (upload, submit, each reconcile poll, collect, register, canary, rollback)."
+*   retention="none — a job row is the record of what ran, what it cost and what was
+*     registered; it leaves only when the operator deletes it. Its staged artefacts expire
+*     separately (14 days after ready/registration) and the row says so."
+*/
+var TRAINING_JOBS_COLLECTION = "training:jobs";
+/**
+* @durable class=ledger owner=training
+*   write="one row per dataset version, inserted once (cut by pipeline-analytics, or moved
+*     from its local store on first sight of this capability) and never updated."
+*   retention="none — rows leave only when the operator deletes the version."
+*/
+var TRAINING_DATASET_VERSIONS_COLLECTION = "training:dataset-versions";
+var JOB_COLUMNS = [
+	{
+		name: "id",
+		type: "TEXT",
+		primaryKey: true,
+		notNull: true
+	},
+	{
+		name: "state",
+		type: "TEXT",
+		notNull: true
+	},
+	{
+		name: "createdAt",
+		type: "INTEGER",
+		notNull: true
+	},
+	{
+		name: "idempotencyKey",
+		type: "TEXT",
+		notNull: true
+	},
+	{
+		name: "job",
+		type: "JSON",
+		notNull: true
+	}
+];
+var JOB_INDEXES = [{
+	name: "idx_training_jobs_idem",
+	columns: ["idempotencyKey"],
+	unique: true
+}, {
+	name: "idx_training_jobs_created",
+	columns: ["createdAt"]
+}];
+var VERSION_COLUMNS = [
+	{
+		name: "id",
+		type: "TEXT",
+		primaryKey: true,
+		notNull: true
+	},
+	{
+		name: "nameKey",
+		type: "TEXT",
+		notNull: true
+	},
+	{
+		name: "createdAt",
+		type: "INTEGER",
+		notNull: true
+	},
+	{
+		name: "row",
+		type: "JSON",
+		notNull: true
+	}
+];
+var VERSION_INDEXES = [{
+	name: "idx_training_versions_name_key",
+	columns: ["nameKey"],
+	unique: true
+}, {
+	name: "idx_training_versions_created",
+	columns: ["createdAt"]
+}];
+function parseJson(value) {
+	if (typeof value !== "string") return value;
+	try {
+		return JSON.parse(value);
+	} catch {
+		return;
+	}
+}
+function recordOf(value) {
+	return value !== null && typeof value === "object" && !Array.isArray(value) ? { ...value } : {};
+}
+async function declareTrainingCollections(store) {
+	await store.declareCollection.mutate({
+		collection: TRAINING_JOBS_COLLECTION,
+		columns: [...JOB_COLUMNS],
+		indexes: [...JOB_INDEXES]
+	});
+	await store.declareCollection.mutate({
+		collection: TRAINING_DATASET_VERSIONS_COLLECTION,
+		columns: [...VERSION_COLUMNS],
+		indexes: [...VERSION_INDEXES]
+	});
+}
+var SettingsJobStore = class {
+	store;
+	logger;
+	constructor(store, logger) {
+		this.store = store;
+		this.logger = logger;
+	}
+	put = async (job) => {
+		await this.store.set.mutate({
+			collection: TRAINING_JOBS_COLLECTION,
+			key: job.id,
+			value: {
+				state: job.state,
+				createdAt: job.createdAt,
+				idempotencyKey: job.idempotencyKey,
+				job
+			}
+		});
+	};
+	get = async (id) => {
+		const row = await this.store.get.query({
+			collection: TRAINING_JOBS_COLLECTION,
+			key: id
+		});
+		return row === null || row === void 0 ? null : this.parse(id, recordOf(row));
+	};
+	list = async () => {
+		return (await this.store.query.query({
+			collection: TRAINING_JOBS_COLLECTION,
+			filter: { orderBy: {
+				field: "createdAt",
+				direction: "desc"
+			} }
+		})).flatMap((r) => {
+			const job = this.parse(r.id, recordOf(r.data));
+			return job === null ? [] : [job];
+		});
+	};
+	findByIdempotencyKey = async (key) => {
+		const row = (await this.store.query.query({
+			collection: TRAINING_JOBS_COLLECTION,
+			filter: {
+				where: { idempotencyKey: key },
+				limit: 1
+			}
+		}))[0];
+		return row === void 0 ? null : this.parse(row.id, recordOf(row.data));
+	};
+	parse(id, data) {
+		const parsed = require_dist.TrainingJobSchema.safeParse(parseJson(data["job"]));
+		if (parsed.success) return parsed.data;
+		this.logger.warn("training job row does not parse — skipped", { meta: {
+			jobId: id,
+			issue: parsed.error.issues[0]?.message ?? "unknown"
+		} });
+		return null;
+	}
+};
+var nameKeyOf = (name) => name.trim().toLowerCase();
+var SettingsVersionStore = class {
+	store;
+	logger;
+	constructor(store, logger) {
+		this.store = store;
+		this.logger = logger;
+	}
+	/** Insert once; an existing id is left as it is (a version is immutable). */
+	put = async (row) => {
+		if (await this.get(row.id) !== null) return false;
+		await this.store.set.mutate({
+			collection: TRAINING_DATASET_VERSIONS_COLLECTION,
+			key: row.id,
+			value: {
+				nameKey: nameKeyOf(row.name),
+				createdAt: row.createdAt,
+				row
+			}
+		});
+		return true;
+	};
+	get = async (id) => {
+		const row = await this.store.get.query({
+			collection: TRAINING_DATASET_VERSIONS_COLLECTION,
+			key: id
+		});
+		return row === null || row === void 0 ? null : this.parse(id, recordOf(row));
+	};
+	list = async () => {
+		return (await this.store.query.query({
+			collection: TRAINING_DATASET_VERSIONS_COLLECTION,
+			filter: { orderBy: {
+				field: "createdAt",
+				direction: "desc"
+			} }
+		})).flatMap((r) => {
+			const parsed = this.parse(r.id, recordOf(r.data));
+			return parsed === null ? [] : [parsed];
+		});
+	};
+	findIdByName = async (name) => {
+		return (await this.store.query.query({
+			collection: "training:dataset-versions",
+			filter: {
+				where: { nameKey: nameKeyOf(name) },
+				limit: 1
+			}
+		}))[0]?.id ?? null;
+	};
+	remove = async (id) => {
+		if (await this.get(id) === null) return false;
+		await this.store.delete.mutate({
+			collection: TRAINING_DATASET_VERSIONS_COLLECTION,
+			key: id
+		});
+		return true;
+	};
+	parse(id, data) {
+		const parsed = require_dist.TrainingDatasetVersionRowSchema.safeParse(parseJson(data["row"]));
+		if (parsed.success) return parsed.data;
+		this.logger.warn("training dataset version row does not parse — skipped", { meta: {
+			versionId: id,
+			issue: parsed.error.issues[0]?.message ?? "unknown"
+		} });
+		return null;
+	}
+};
+//#endregion
+//#region src/orchestrator/trainer-source.ts
+var TRAINER_SOURCE = {
+	repo: "camstack/camstack-trainer",
+	revision: "ebeed67a296241fcb5b0d97cf7c85dd12a743db8"
+};
+//#endregion
+//#region src/orchestrator/training.addon.ts
+/**
+* `training` — the cloud-training orchestrator (D805). Hub-only.
+*
+* Wires the engine to `ctx.api` and the hub's disk, provides the `training`
+* capability and the Training page, and runs the reconcile loop: once at boot
+* (a restart loses nothing — the job rows and the provider's jobKey tags are
+* the whole truth), then every 30 s, plus a daily orphan sweep.
+*/
+/** The first orphan sweep waits for the providers to register after a boot. */
+var FIRST_SWEEP_DELAY_MS = 5 * 6e4;
+var TrainingAddon = class extends require_dist.BaseAddon {
+	pages = [{
+		id: "training",
+		label: "Training",
+		icon: "graduation-cap",
+		path: "/addon/training",
+		remoteName: "addon_training_page",
+		bundle: "remoteEntry.js",
+		section: "detection"
+	}];
+	engine = null;
+	timers = [];
+	ticking = false;
+	constructor() {
+		super({});
+	}
+	async onInitialize() {
+		const api = this.ctx.api;
+		const peer = this.ctx.peerBytes;
+		if (peer === void 0) throw new Error("training needs the peer-bytes facility");
+		await declareTrainingCollections(api.settingsStore);
+		const logger = this.ctx.logger;
+		const versions = new SettingsVersionStore(api.settingsStore, logger);
+		const ports = {
+			provider: providerPort(api),
+			datasets: datasetPort(api),
+			versions,
+			jobs: new SettingsJobStore(api.settingsStore, logger),
+			artifacts: createDiskArtifacts(node_path.default.join(this.ctx.dataDir, "training", "jobs"), peer),
+			registry: registryPort(api),
+			pins: pinPort(api),
+			now: () => Date.now(),
+			newId: () => crypto.randomUUID(),
+			logger
+		};
+		const engine = new TrainingEngine(ports, TRAINER_SOURCE);
+		const flow = new ReturnFlow(ports, engine.mutator);
+		this.engine = engine;
+		const provider = {
+			putDatasetVersion: async (row) => ({ stored: await versions.put(row) }),
+			getDatasetVersion: async ({ id }) => versions.get(id),
+			listDatasetVersions: async () => (await versions.list()).map(({ frameIds, ...rest }) => ({
+				...rest,
+				frameCount: frameIds.length
+			})),
+			findDatasetVersionIdByName: async ({ name }) => ({ id: await versions.findIdByName(name) }),
+			removeDatasetVersion: async ({ id }) => ({ removed: await versions.remove(id) }),
+			listBaseModels: async () => BASE_MODELS.map(publicBaseModel),
+			planJob: async (request) => (await planJob(request, {
+				versions,
+				provider: ports.provider
+			})).plan,
+			startJob: async (input) => engine.startJob(input),
+			listJobs: async () => ports.jobs.list(),
+			getJob: async ({ id }) => ports.jobs.get(id),
+			cancelJob: async ({ id }) => engine.cancelJob(id),
+			getJobResult: async ({ id }) => {
+				const job = await ports.jobs.get(id);
+				return job === null ? null : flow.readResult(job);
+			},
+			registerResult: async ({ id, canaryDeviceIds }) => flow.register(id, canaryDeviceIds),
+			canaryResult: async ({ id, deviceIds }) => flow.canary(id, deviceIds),
+			rollbackCanary: async ({ id }) => flow.rollback(id),
+			discardResult: async ({ id }) => flow.discard(id)
+		};
+		const pagesProvider = {
+			id: this.ctx.id,
+			listPages: () => this.pages
+		};
+		this.schedule(0, RECONCILE_INTERVAL_MS, () => this.reconcile());
+		this.schedule(FIRST_SWEEP_DELAY_MS, ORPHAN_SWEEP_INTERVAL_MS, async () => {
+			const swept = await engine.sweepOrphans();
+			if (swept.length > 0) logger.warn("training orphan sweep stopped cloud jobs", { meta: { swept: [...swept] } });
+		});
+		return { providers: [{
+			capability: require_dist.trainingCapability,
+			provider
+		}, {
+			capability: require_dist.addonPagesSourceCapability,
+			provider: pagesProvider
+		}] };
+	}
+	async onShutdown() {
+		for (const t of this.timers) clearTimeout(t);
+		this.timers = [];
+		this.engine = null;
+	}
+	async reconcile() {
+		if (this.ticking || this.engine === null) return;
+		this.ticking = true;
+		try {
+			await this.engine.tick();
+		} finally {
+			this.ticking = false;
+		}
+	}
+	/** A self-rescheduling timer that never overlaps itself and never throws out. */
+	schedule(firstDelayMs, everyMs, body) {
+		const run = () => {
+			body().catch((err) => {
+				this.ctx.logger.warn("training scheduled pass failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
+			}).finally(() => {
+				if (this.engine !== null) this.timers.push(setTimeout(run, everyMs));
+			});
+		};
+		this.timers.push(setTimeout(run, firstDelayMs));
+	}
+};
+//#endregion
+exports.TrainingAddon = TrainingAddon;