@ucdjs/pipelines-executor 0.0.1-beta.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025-PRESENT Lucas Nørgård
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,27 @@
+# @ucdjs/pipelines-executor
+
+[![npm version][npm-version-src]][npm-version-href]
+[![npm downloads][npm-downloads-src]][npm-downloads-href]
+[![codecov][codecov-src]][codecov-href]
+
+> [!IMPORTANT]
+> This is an internal package. It may change without warning and is not subject to semantic versioning. Use at your own risk.
+
+A collection of core pipeline functionalities for the UCD project.
+
+## Installation
+
+```bash
+npm install @ucdjs/pipelines-executor
+```
+
+## 📄 License
+
+Published under [MIT License](./LICENSE).
+
+[npm-version-src]: https://img.shields.io/npm/v/@ucdjs/pipelines-executor?style=flat&colorA=18181B&colorB=4169E1
+[npm-version-href]: https://npmjs.com/package/@ucdjs/pipelines-executor
+[npm-downloads-src]: https://img.shields.io/npm/dm/@ucdjs/pipelines-executor?style=flat&colorA=18181B&colorB=4169E1
+[npm-downloads-href]: https://npmjs.com/package/@ucdjs/pipelines-executor
+[codecov-src]: https://img.shields.io/codecov/c/gh/ucdjs/ucd?style=flat&colorA=18181B&colorB=4169E1
+[codecov-href]: https://codecov.io/gh/ucdjs/ucd

package/dist/index.d.mts

@@ -0,0 +1,87 @@
+import { AnyPipelineDefinition, PipelineError, PipelineEvent, PipelineGraph } from "@ucdjs/pipelines-core";
+import { PipelineArtifactDefinition } from "@ucdjs/pipelines-artifacts";
+
+//#region src/cache.d.ts
+interface CacheKey {
+  routeId: string;
+  version: string;
+  inputHash: string;
+  artifactHashes: Record<string, string>;
+}
+interface CacheEntry<TOutput = unknown> {
+  key: CacheKey;
+  output: TOutput[];
+  producedArtifacts: Record<string, unknown>;
+  createdAt: string;
+  meta?: Record<string, unknown>;
+}
+declare function serializeCacheKey(key: CacheKey): string;
+interface CacheStore {
+  get: (key: CacheKey) => Promise<CacheEntry | undefined>;
+  set: (entry: CacheEntry) => Promise<void>;
+  has: (key: CacheKey) => Promise<boolean>;
+  delete: (key: CacheKey) => Promise<boolean>;
+  clear: () => Promise<void>;
+  stats?: () => Promise<CacheStats>;
+}
+interface CacheStats {
+  entries: number;
+  sizeBytes?: number;
+  hits?: number;
+  misses?: number;
+}
+interface CacheOptions {
+  enabled?: boolean;
+  hashFn?: (content: string) => string;
+}
+declare function createMemoryCacheStore(): CacheStore;
+declare function defaultHashFn(content: string): string;
+declare function hashArtifact(value: unknown): string;
+//#endregion
+//#region src/types.d.ts
+interface PipelineSummary {
+  versions: string[];
+  totalFiles: number;
+  matchedFiles: number;
+  skippedFiles: number;
+  fallbackFiles: number;
+  totalOutputs: number;
+  durationMs: number;
+}
+type ExecutionStatus = "running" | "completed" | "failed";
+interface PipelineExecutionResult {
+  id: string;
+  data: unknown[];
+  graph: PipelineGraph;
+  errors: PipelineError[];
+  summary: PipelineSummary;
+  status: ExecutionStatus;
+}
+interface PipelineExecutorOptions {
+  artifacts?: PipelineArtifactDefinition[];
+  cacheStore?: CacheStore;
+  onEvent?: (event: PipelineEvent) => void | Promise<void>;
+}
+interface PipelineExecutorRunOptions {
+  cache?: boolean;
+  versions?: string[];
+}
+interface PipelineExecutor {
+  run: (pipelines: AnyPipelineDefinition[], options?: PipelineExecutorRunOptions) => Promise<PipelineExecutionResult[]>;
+}
+//#endregion
+//#region src/executor.d.ts
+declare function createPipelineExecutor(options: PipelineExecutorOptions): PipelineExecutor;
+//#endregion
+//#region src/log-context.d.ts
+interface PipelineExecutionContext {
+  executionId: string;
+  spanId?: string;
+  event?: PipelineEvent;
+}
+declare function getPipelineExecutionContext(): PipelineExecutionContext | undefined;
+declare function runWithPipelineExecutionContext<T>(context: PipelineExecutionContext, fn: () => T | Promise<T>): T | Promise<T>;
+declare function withPipelineSpan<T>(spanId: string, fn: () => T | Promise<T>): T | Promise<T>;
+declare function withPipelineEvent<T>(event: PipelineEvent, fn: () => T | Promise<T>): T | Promise<T>;
+//#endregion
+export { type CacheEntry, type CacheKey, type CacheOptions, type CacheStats, type CacheStore, type ExecutionStatus, type PipelineExecutionResult, type PipelineExecutor, type PipelineExecutorOptions, type PipelineExecutorRunOptions, type PipelineSummary, createMemoryCacheStore, createPipelineExecutor, defaultHashFn, getPipelineExecutionContext, hashArtifact, runWithPipelineExecutionContext, serializeCacheKey, withPipelineEvent, withPipelineSpan };

package/dist/index.mjs

@@ -0,0 +1,869 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import { applyTransforms, getExecutionLayers, resolveMultipleSourceFiles } from "@ucdjs/pipelines-core";
+import { PipelineGraphBuilder } from "@ucdjs/pipelines-graph";
+import { isGlobalArtifact } from "@ucdjs/pipelines-artifacts";
+
+//#region src/cache.ts
+function serializeCacheKey(key) {
+	const artifactHashStr = Object.entries(key.artifactHashes).sort(([a], [b]) => a.localeCompare(b)).map(([id, hash]) => `${id}:${hash}`).join(",");
+	return `${key.routeId}|${key.version}|${key.inputHash}|${artifactHashStr}`;
+}
+function createMemoryCacheStore() {
+	const cache = /* @__PURE__ */ new Map();
+	let hits = 0;
+	let misses = 0;
+	return {
+		async get(key) {
+			const serialized = serializeCacheKey(key);
+			const entry = cache.get(serialized);
+			if (entry) hits++;
+			else misses++;
+			return entry;
+		},
+		async set(entry) {
+			const serialized = serializeCacheKey(entry.key);
+			cache.set(serialized, entry);
+		},
+		async has(key) {
+			const serialized = serializeCacheKey(key);
+			return cache.has(serialized);
+		},
+		async delete(key) {
+			const serialized = serializeCacheKey(key);
+			return cache.delete(serialized);
+		},
+		async clear() {
+			cache.clear();
+			hits = 0;
+			misses = 0;
+		},
+		async stats() {
+			return {
+				entries: cache.size,
+				hits,
+				misses
+			};
+		}
+	};
+}
+function defaultHashFn(content) {
+	let hash = 5381;
+	for (let i = 0; i < content.length; i++) hash = (hash << 5) + hash ^ content.charCodeAt(i);
+	return (hash >>> 0).toString(16).padStart(8, "0");
+}
+function hashArtifact(value) {
+	if (value === null || value === void 0) return "null";
+	if (typeof value === "string") return defaultHashFn(value);
+	if (value instanceof Map) return defaultHashFn(Array.from(value.entries()).sort(([a], [b]) => String(a).localeCompare(String(b))).map(([k, v]) => `${String(k)}=${String(v)}`).join(";"));
+	if (value instanceof Set) return defaultHashFn(Array.from(value).map(String).sort().join(";"));
+	if (Array.isArray(value)) return defaultHashFn(JSON.stringify(value));
+	if (typeof value === "object") return defaultHashFn(JSON.stringify(value));
+	return defaultHashFn(String(value));
+}
+
+//#endregion
+//#region src/log-context.ts
+const storage = new AsyncLocalStorage();
+function getPipelineExecutionContext() {
+	return storage.getStore();
+}
+function runWithPipelineExecutionContext(context, fn) {
+	return storage.run(context, fn);
+}
+function withPipelineSpan(spanId, fn) {
+	const current = storage.getStore();
+	if (!current) return fn();
+	return storage.run({
+		...current,
+		spanId
+	}, fn);
+}
+function withPipelineEvent(event, fn) {
+	const current = storage.getStore();
+	if (!current) return fn();
+	return storage.run({
+		...current,
+		spanId: event.spanId,
+		event
+	}, fn);
+}
+
+//#endregion
+//#region src/executor/events.ts
+function createEventEmitter(options) {
+	const { onEvent } = options;
+	let eventCounter = 0;
+	const nextEventId = () => `evt_${Date.now()}_${++eventCounter}`;
+	let spanCounter = 0;
+	const nextSpanId = () => `span_${Date.now()}_${++spanCounter}`;
+	const emit = async (event) => {
+		const fullEvent = {
+			...event,
+			id: event.id ?? nextEventId(),
+			spanId: event.spanId ?? nextSpanId()
+		};
+		await withPipelineEvent(fullEvent, async () => {
+			await onEvent?.(fullEvent);
+		});
+	};
+	return {
+		emit,
+		nextEventId,
+		nextSpanId
+	};
+}
+async function emitWithSpan(spanId, fn) {
+	await withPipelineSpan(spanId, fn);
+}
+
+//#endregion
+//#region src/executor/cache-helpers.ts
+async function tryLoadCachedResult(options) {
+	const { cacheStore, routeId, version, fileContent, artifactsMap } = options;
+	const partialKey = {
+		routeId,
+		version,
+		inputHash: defaultHashFn(fileContent),
+		artifactHashes: {}
+	};
+	const cachedEntry = await cacheStore.get(partialKey);
+	if (!cachedEntry) return {
+		result: null,
+		hit: false
+	};
+	const currentArtifactHashes = {};
+	for (const id of Object.keys(cachedEntry.key.artifactHashes)) if (id in artifactsMap) currentArtifactHashes[id] = hashArtifact(artifactsMap[id]);
+	if (!Object.keys(cachedEntry.key.artifactHashes).every((id) => currentArtifactHashes[id] === cachedEntry.key.artifactHashes[id])) return {
+		result: null,
+		hit: false
+	};
+	return {
+		result: {
+			outputs: cachedEntry.output,
+			emittedArtifacts: cachedEntry.producedArtifacts,
+			consumedArtifactIds: Object.keys(cachedEntry.key.artifactHashes)
+		},
+		hit: true
+	};
+}
+async function buildCacheKey(routeId, version, fileContent, artifactsMap, consumedArtifactIds) {
+	const artifactHashes = {};
+	for (const id of consumedArtifactIds) if (id in artifactsMap) artifactHashes[id] = hashArtifact(artifactsMap[id]);
+	return {
+		routeId,
+		version,
+		inputHash: defaultHashFn(fileContent),
+		artifactHashes
+	};
+}
+async function storeCacheEntry(options) {
+	const { cacheStore, cacheKey, outputs, emittedArtifacts } = options;
+	const cacheEntry = {
+		key: cacheKey,
+		output: outputs,
+		producedArtifacts: emittedArtifacts,
+		createdAt: (/* @__PURE__ */ new Date()).toISOString()
+	};
+	await cacheStore.set(cacheEntry);
+}
+
+//#endregion
+//#region src/executor/processing-queue.ts
+function createProcessingQueue(concurrency) {
+	const queue = [];
+	let running = 0;
+	let resolveIdle = null;
+	const runNext = async () => {
+		if (running >= concurrency || queue.length === 0) {
+			if (running === 0 && queue.length === 0 && resolveIdle) resolveIdle();
+			return;
+		}
+		running++;
+		const task = queue.shift();
+		try {
+			await task();
+		} finally {
+			running--;
+			runNext();
+		}
+	};
+	return {
+		add: async (task) => {
+			queue.push(task);
+			runNext();
+		},
+		drain: () => {
+			if (running === 0 && queue.length === 0) return Promise.resolve();
+			return new Promise((resolve) => {
+				resolveIdle = resolve;
+			});
+		}
+	};
+}
+
+//#endregion
+//#region src/executor/source-adapter.ts
+function createSourceAdapter(pipeline) {
+	if (pipeline.inputs.length === 0) throw new Error("Pipeline requires at least one input source");
+	const backends = /* @__PURE__ */ new Map();
+	for (const input of pipeline.inputs) backends.set(input.id, input.backend);
+	return {
+		listFiles: async (version) => resolveMultipleSourceFiles(pipeline.inputs, version),
+		readFile: async (file) => {
+			const sourceFile = file;
+			if ("source" in sourceFile && sourceFile.source) {
+				const backend = backends.get(sourceFile.source.id);
+				if (backend) return backend.readFile(file);
+			}
+			const firstBackend = backends.values().next().value;
+			if (firstBackend) return firstBackend.readFile(file);
+			throw new Error(`No backend found for file: ${file.path}`);
+		}
+	};
+}
+function createParseContext(file, source) {
+	let cachedContent = null;
+	return {
+		file,
+		readContent: async () => {
+			if (cachedContent === null) cachedContent = await source.readFile(file);
+			return cachedContent;
+		},
+		async *readLines() {
+			const lines = (await source.readFile(file)).split(/\r?\n/);
+			for (const line of lines) yield line;
+		},
+		isComment: (line) => line.startsWith("#") || line.trim() === ""
+	};
+}
+
+//#endregion
+//#region src/executor/route-execution.ts
+function createRouteResolveContext(options) {
+	const { version, file, routeId, artifactsMap, emittedArtifacts, emitsDefinition, onArtifactEmit, onArtifactGet } = options;
+	return {
+		version,
+		file,
+		getArtifact: (key) => {
+			if (!(key in artifactsMap)) throw new Error(`Artifact "${key}" not found. Make sure a route that produces this artifact runs before route "${routeId}".`);
+			onArtifactGet?.(key);
+			return artifactsMap[key];
+		},
+		emitArtifact: (id, value) => {
+			if (emitsDefinition) {
+				const def = emitsDefinition[id];
+				if (def) {
+					const result = def.schema.safeParse(value);
+					if (!result.success) throw new Error(`Artifact "${id}" validation failed: ${result.error.message}`);
+				}
+			}
+			emittedArtifacts[id] = value;
+			onArtifactEmit?.(id, value);
+		},
+		normalizeEntries: (entries) => {
+			return entries.sort((a, b) => {
+				const aStart = a.range?.split("..")[0] ?? a.codePoint ?? "";
+				const bStart = b.range?.split("..")[0] ?? b.codePoint ?? "";
+				return aStart.localeCompare(bStart);
+			});
+		},
+		now: () => (/* @__PURE__ */ new Date()).toISOString()
+	};
+}
+async function processRoute(options) {
+	const { file, route, artifactsMap, source, version, emit, spanId } = options;
+	const parseStartTime = performance.now();
+	const parseSpanId = spanId();
+	await withPipelineSpan(parseSpanId, async () => {
+		await emit({
+			type: "parse:start",
+			file,
+			routeId: route.id,
+			spanId: parseSpanId,
+			timestamp: performance.now()
+		});
+	});
+	const parseCtx = createParseContext(file, source);
+	let rows = route.parser(parseCtx);
+	const collectedRows = [];
+	const filteredRows = filterRows(rows, file, route.filter, collectedRows);
+	if (route.transforms && route.transforms.length > 0) rows = applyTransforms({
+		version,
+		file
+	}, filteredRows, route.transforms);
+	else rows = filteredRows;
+	await withPipelineSpan(parseSpanId, async () => {
+		await emit({
+			type: "parse:end",
+			file,
+			routeId: route.id,
+			rowCount: collectedRows.length,
+			durationMs: performance.now() - parseStartTime,
+			spanId: parseSpanId,
+			timestamp: performance.now()
+		});
+	});
+	const resolveStartTime = performance.now();
+	const resolveSpanId = spanId();
+	await withPipelineSpan(resolveSpanId, async () => {
+		await emit({
+			type: "resolve:start",
+			file,
+			routeId: route.id,
+			spanId: resolveSpanId,
+			timestamp: performance.now()
+		});
+	});
+	const emittedArtifacts = {};
+	const consumedArtifactIds = [];
+	const resolveCtx = createRouteResolveContext({
+		version,
+		file,
+		routeId: route.id,
+		artifactsMap,
+		emittedArtifacts,
+		emitsDefinition: route.emits,
+		onArtifactEmit: async (id) => {
+			await withPipelineSpan(resolveSpanId, async () => {
+				await emit({
+					type: "artifact:produced",
+					artifactId: `${route.id}:${id}`,
+					routeId: route.id,
+					version,
+					spanId: resolveSpanId,
+					timestamp: performance.now()
+				});
+			});
+		},
+		onArtifactGet: async (id) => {
+			if (!consumedArtifactIds.includes(id)) {
+				consumedArtifactIds.push(id);
+				await withPipelineSpan(resolveSpanId, async () => {
+					await emit({
+						type: "artifact:consumed",
+						artifactId: id,
+						routeId: route.id,
+						version,
+						spanId: resolveSpanId,
+						timestamp: performance.now()
+					});
+				});
+			}
+		}
+	});
+	const outputs = await route.resolver(resolveCtx, rows);
+	const outputArray = Array.isArray(outputs) ? outputs : [outputs];
+	await withPipelineSpan(resolveSpanId, async () => {
+		await emit({
+			type: "resolve:end",
+			file,
+			routeId: route.id,
+			outputCount: outputArray.length,
+			durationMs: performance.now() - resolveStartTime,
+			spanId: resolveSpanId,
+			timestamp: performance.now()
+		});
+	});
+	return {
+		outputs: outputArray,
+		emittedArtifacts,
+		consumedArtifactIds
+	};
+}
+async function processFallback(options) {
+	const { file, fallback, artifactsMap, source, version, emit, spanId } = options;
+	const parseStartTime = performance.now();
+	const parseSpanId = spanId();
+	await withPipelineSpan(parseSpanId, async () => {
+		await emit({
+			type: "parse:start",
+			file,
+			routeId: "__fallback__",
+			spanId: parseSpanId,
+			timestamp: performance.now()
+		});
+	});
+	const parseCtx = createParseContext(file, source);
+	const rows = fallback.parser(parseCtx);
+	const collectedRows = [];
+	const filteredRows = filterRows(rows, file, fallback.filter, collectedRows);
+	await withPipelineSpan(parseSpanId, async () => {
+		await emit({
+			type: "parse:end",
+			file,
+			routeId: "__fallback__",
+			rowCount: collectedRows.length,
+			durationMs: performance.now() - parseStartTime,
+			spanId: parseSpanId,
+			timestamp: performance.now()
+		});
+	});
+	const resolveStartTime = performance.now();
+	const resolveSpanId = spanId();
+	await withPipelineSpan(resolveSpanId, async () => {
+		await emit({
+			type: "resolve:start",
+			file,
+			routeId: "__fallback__",
+			spanId: resolveSpanId,
+			timestamp: performance.now()
+		});
+	});
+	const emittedArtifacts = {};
+	const resolveCtx = {
+		version,
+		file,
+		getArtifact: (id) => {
+			if (!(id in artifactsMap)) throw new Error(`Artifact "${String(id)}" not found.`);
+			return artifactsMap[id];
+		},
+		emitArtifact: (id, value) => {
+			emittedArtifacts[id] = value;
+		},
+		normalizeEntries: (entries) => {
+			return entries.sort((a, b) => {
+				const aStart = a.range?.split("..")[0] ?? a.codePoint ?? "";
+				const bStart = b.range?.split("..")[0] ?? b.codePoint ?? "";
+				return aStart.localeCompare(bStart);
+			});
+		},
+		now: () => (/* @__PURE__ */ new Date()).toISOString()
+	};
+	const outputs = await fallback.resolver(resolveCtx, filteredRows);
+	const outputArray = Array.isArray(outputs) ? outputs : [outputs];
+	await withPipelineSpan(resolveSpanId, async () => {
+		await emit({
+			type: "resolve:end",
+			file,
+			routeId: "__fallback__",
+			outputCount: outputArray.length,
+			durationMs: performance.now() - resolveStartTime,
+			spanId: resolveSpanId,
+			timestamp: performance.now()
+		});
+	});
+	return outputArray;
+}
+async function* filterRows(rows, file, filter, collector) {
+	for await (const row of rows) {
+		collector.push(row);
+		if (!filter) {
+			yield row;
+			continue;
+		}
+		if (filter({
+			file,
+			row: { property: row.property }
+		})) yield row;
+	}
+}
+function recordEmittedArtifacts(options) {
+	const { routeId, emittedArtifacts, routeEmits, artifactsMap, globalArtifactsMap } = options;
+	for (const [artifactName, artifactValue] of Object.entries(emittedArtifacts)) {
+		const prefixedKey = `${routeId}:${artifactName}`;
+		const artifactDef = routeEmits?.[artifactName];
+		if (artifactDef && isGlobalArtifact(artifactDef)) globalArtifactsMap[prefixedKey] = artifactValue;
+		else artifactsMap[prefixedKey] = artifactValue;
+	}
+}
+
+//#endregion
+//#region src/executor/run-pipeline.ts
+async function runPipeline(options) {
+	const { pipeline, runOptions = {}, cacheStore, artifacts: globalArtifacts, events } = options;
+	const { cache: enableCache = true, versions: runVersions } = runOptions;
+	const useCache = enableCache && cacheStore != null;
+	const versionsToRun = runVersions ?? pipeline.versions;
+	const source = createSourceAdapter(pipeline);
+	const graph = new PipelineGraphBuilder();
+	const startTime = performance.now();
+	const outputs = [];
+	const errors = [];
+	let totalFiles = 0;
+	let matchedFiles = 0;
+	let skippedFiles = 0;
+	let fallbackFiles = 0;
+	const dag = pipeline.dag;
+	const pipelineSpanId = events.nextSpanId();
+	await emitWithSpan(pipelineSpanId, () => events.emit({
+		type: "pipeline:start",
+		versions: versionsToRun,
+		spanId: pipelineSpanId,
+		timestamp: performance.now()
+	}));
+	for (const version of versionsToRun) {
+		const versionStartTime = performance.now();
+		const versionSpanId = events.nextSpanId();
+		await emitWithSpan(versionSpanId, () => events.emit({
+			type: "version:start",
+			version,
+			spanId: versionSpanId,
+			timestamp: performance.now()
+		}));
+		const sourceNodeId = graph.addSourceNode(version);
+		const artifactsMap = {};
+		const globalArtifactsMap = {};
+		let versionFiles = null;
+		async function listVersionFiles() {
+			if (!versionFiles) versionFiles = await source.listFiles(version);
+			return versionFiles;
+		}
+		for (const artifactDef of globalArtifacts) {
+			const artifactStartTime = performance.now();
+			const artifactSpanId = events.nextSpanId();
+			await emitWithSpan(artifactSpanId, () => events.emit({
+				type: "artifact:start",
+				artifactId: artifactDef.id,
+				version,
+				spanId: artifactSpanId,
+				timestamp: performance.now()
+			}));
+			const artifactNodeId = graph.addArtifactNode(artifactDef.id, version);
+			graph.addEdge(sourceNodeId, artifactNodeId, "provides");
+			try {
+				let rows;
+				if (artifactDef.filter && artifactDef.parser) {
+					const files = await listVersionFiles();
+					for (const file of files) if (artifactDef.filter({ file })) {
+						rows = artifactDef.parser(createParseContext(file, source));
+						break;
+					}
+				}
+				const value = await artifactDef.build({ version }, rows);
+				artifactsMap[artifactDef.id] = value;
+			} catch (err) {
+				const pipelineError = {
+					scope: "artifact",
+					message: err instanceof Error ? err.message : String(err),
+					error: err,
+					artifactId: artifactDef.id,
+					version
+				};
+				errors.push(pipelineError);
+				await emitWithSpan(artifactSpanId, () => events.emit({
+					type: "error",
+					error: pipelineError,
+					spanId: artifactSpanId,
+					timestamp: performance.now()
+				}));
+			}
+			await emitWithSpan(artifactSpanId, () => events.emit({
+				type: "artifact:end",
+				artifactId: artifactDef.id,
+				version,
+				durationMs: performance.now() - artifactStartTime,
+				spanId: artifactSpanId,
+				timestamp: performance.now()
+			}));
+		}
+		const files = await listVersionFiles();
+		totalFiles += files.length;
+		const filesToProcess = pipeline.include ? files.filter((file) => pipeline.include({ file })) : files;
+		const executionLayers = getExecutionLayers(dag);
+		const processedFiles = /* @__PURE__ */ new Set();
+		for (const layer of executionLayers) {
+			const processingQueue = createProcessingQueue(pipeline.concurrency);
+			const layerRoutes = pipeline.routes.filter((route) => layer.includes(route.id));
+			for (const route of layerRoutes) {
+				const matchingFiles = filesToProcess.filter((file) => {
+					const filterCtx = {
+						file,
+						source: file.source
+					};
+					return route.filter(filterCtx);
+				});
+				for (const file of matchingFiles) {
+					processedFiles.add(file.path);
+					await processingQueue.add(async () => {
+						const fileNodeId = graph.addFileNode(file);
+						graph.addEdge(sourceNodeId, fileNodeId, "provides");
+						matchedFiles++;
+						const routeNodeId = graph.addRouteNode(route.id, version);
+						graph.addEdge(fileNodeId, routeNodeId, "matched");
+						const fileSpanId = events.nextSpanId();
+						await emitWithSpan(fileSpanId, () => events.emit({
+							type: "file:matched",
+							file,
+							routeId: route.id,
+							spanId: fileSpanId,
+							timestamp: performance.now()
+						}));
+						await withPipelineSpan(fileSpanId, async () => {
+							try {
+								const routeCacheEnabled = useCache && route.cache !== false;
+								let result = null;
+								let fileContent;
+								if (routeCacheEnabled && cacheStore) {
+									fileContent = await source.readFile(file);
+									const combinedArtifacts = {
+										...artifactsMap,
+										...globalArtifactsMap
+									};
+									const cachedResult = await tryLoadCachedResult({
+										cacheStore,
+										routeId: route.id,
+										version,
+										fileContent,
+										artifactsMap: combinedArtifacts
+									});
+									if (cachedResult.hit && cachedResult.result) {
+										result = cachedResult.result;
+										await events.emit({
+											type: "cache:hit",
+											routeId: route.id,
+											file,
+											version,
+											spanId: fileSpanId,
+											timestamp: performance.now()
+										});
+									} else await events.emit({
+										type: "cache:miss",
+										routeId: route.id,
+										file,
+										version,
+										spanId: fileSpanId,
+										timestamp: performance.now()
+									});
+								}
+								if (!result) {
+									result = await processRoute({
+										file,
+										route,
+										artifactsMap: {
+											...artifactsMap,
+											...globalArtifactsMap
+										},
+										source,
+										version,
+										emit: (event) => events.emit({
+											...event,
+											spanId: fileSpanId
+										}),
+										spanId: events.nextSpanId
+									});
+									if (routeCacheEnabled && cacheStore) {
+										fileContent ??= await source.readFile(file);
+										const cacheArtifacts = {
+											...artifactsMap,
+											...globalArtifactsMap
+										};
+										await storeCacheEntry({
+											cacheStore,
+											cacheKey: await buildCacheKey(route.id, version, fileContent, cacheArtifacts, result.consumedArtifactIds),
+											outputs: result.outputs,
+											emittedArtifacts: result.emittedArtifacts
+										});
+										await events.emit({
+											type: "cache:store",
+											routeId: route.id,
+											file,
+											version,
+											spanId: fileSpanId,
+											timestamp: performance.now()
+										});
+									}
+								}
+								recordEmittedArtifacts({
+									routeId: route.id,
+									emittedArtifacts: result.emittedArtifacts,
+									routeEmits: route.emits,
+									artifactsMap,
+									globalArtifactsMap
+								});
+								for (const output of result.outputs) {
+									const outputIndex = outputs.length;
+									outputs.push(output);
+									const outputNodeId = graph.addOutputNode(outputIndex, version, output.property);
+									graph.addEdge(routeNodeId, outputNodeId, "resolved");
+								}
+							} catch (err) {
+								const pipelineError = {
+									scope: "route",
+									message: err instanceof Error ? err.message : String(err),
+									error: err,
+									file,
+									routeId: route.id,
+									version
+								};
+								errors.push(pipelineError);
+								await events.emit({
+									type: "error",
+									error: pipelineError,
+									spanId: fileSpanId,
+									timestamp: performance.now()
+								});
+							}
+						});
+					});
+				}
+			}
+			await processingQueue.drain();
+		}
+		for (const file of filesToProcess) {
+			if (processedFiles.has(file.path)) continue;
+			if (pipeline.fallback) if (!pipeline.fallback.filter || pipeline.fallback.filter({ file })) {
+				fallbackFiles++;
+				const fileNodeId = graph.addFileNode(file);
+				graph.addEdge(sourceNodeId, fileNodeId, "provides");
+				const fallbackSpanId = events.nextSpanId();
+				await emitWithSpan(fallbackSpanId, () => events.emit({
+					type: "file:fallback",
+					file,
+					spanId: fallbackSpanId,
+					timestamp: performance.now()
+				}));
+				try {
+					const fallbackOutputs = await withPipelineSpan(fallbackSpanId, () => processFallback({
+						file,
+						fallback: pipeline.fallback,
+						artifactsMap: {
+							...artifactsMap,
+							...globalArtifactsMap
+						},
+						source,
+						version,
+						emit: (event) => events.emit({
+							...event,
+							spanId: fallbackSpanId
+						}),
+						spanId: events.nextSpanId
+					}));
+					for (const output of fallbackOutputs) {
+						const outputIndex = outputs.length;
+						outputs.push(output);
+						const outputNodeId = graph.addOutputNode(outputIndex, version, output.property);
+						graph.addEdge(fileNodeId, outputNodeId, "resolved");
+					}
+				} catch (err) {
+					const pipelineError = {
+						scope: "file",
+						message: err instanceof Error ? err.message : String(err),
+						error: err,
+						file,
+						version
+					};
+					errors.push(pipelineError);
+					await emitWithSpan(fallbackSpanId, () => events.emit({
+						type: "error",
+						error: pipelineError,
+						spanId: fallbackSpanId,
+						timestamp: performance.now()
+					}));
+				}
+			} else {
+				skippedFiles++;
+				await emitWithSpan(versionSpanId, () => events.emit({
+					type: "file:skipped",
+					file,
+					reason: "filtered",
+					spanId: versionSpanId,
+					timestamp: performance.now()
+				}));
+			}
+			else {
+				skippedFiles++;
+				if (pipeline.strict) {
+					const pipelineError = {
+						scope: "file",
+						message: `No matching route for file: ${file.path}`,
+						file,
+						version
+					};
+					errors.push(pipelineError);
+					await emitWithSpan(versionSpanId, () => events.emit({
+						type: "error",
+						error: pipelineError,
+						spanId: versionSpanId,
+						timestamp: performance.now()
+					}));
+				} else await emitWithSpan(versionSpanId, () => events.emit({
+					type: "file:skipped",
+					file,
+					reason: "no-match",
+					spanId: versionSpanId,
+					timestamp: performance.now()
+				}));
+			}
+		}
+		await emitWithSpan(versionSpanId, () => events.emit({
+			type: "version:end",
+			version,
+			durationMs: performance.now() - versionStartTime,
+			spanId: versionSpanId,
+			timestamp: performance.now()
+		}));
+	}
+	const durationMs = performance.now() - startTime;
+	await emitWithSpan(pipelineSpanId, () => events.emit({
+		type: "pipeline:end",
+		durationMs,
+		spanId: pipelineSpanId,
+		timestamp: performance.now()
+	}));
+	const summary = {
+		versions: versionsToRun,
+		totalFiles,
+		matchedFiles,
+		skippedFiles,
+		fallbackFiles,
+		totalOutputs: outputs.length,
+		durationMs
+	};
+	const status = errors.length === 0 ? "completed" : "failed";
+	return {
+		id: pipeline.id,
+		data: outputs,
+		graph: graph.build(),
+		errors,
+		summary,
+		status
+	};
+}
+
+//#endregion
+//#region src/executor.ts
+function createPipelineExecutor(options) {
+	const { artifacts: globalArtifacts = [], cacheStore, onEvent } = options;
+	const events = createEventEmitter({ onEvent });
+	const run = async (pipelinesToRun, runOptions = {}) => {
+		const results = [];
+		for (const pipeline of pipelinesToRun) try {
+			results.push(await runPipeline({
+				pipeline,
+				runOptions,
+				cacheStore,
+				artifacts: globalArtifacts,
+				events
+			}));
+		} catch (err) {
+			results.push({
+				id: pipeline.id,
+				data: [],
+				graph: {
+					nodes: [],
+					edges: []
+				},
+				errors: [{
+					scope: "pipeline",
+					message: err instanceof Error ? err.message : String(err),
+					error: err
+				}],
+				summary: {
+					versions: pipeline.versions,
+					totalFiles: 0,
+					matchedFiles: 0,
+					skippedFiles: 0,
+					fallbackFiles: 0,
+					totalOutputs: 0,
+					durationMs: 0
+				},
+				status: "failed"
+			});
+		}
+		return results;
+	};
+	return { run };
+}
+
+//#endregion
+export { createMemoryCacheStore, createPipelineExecutor, defaultHashFn, getPipelineExecutionContext, hashArtifact, runWithPipelineExecutionContext, serializeCacheKey, withPipelineEvent, withPipelineSpan };

package/package.json

@@ -0,0 +1,55 @@
+{
+  "name": "@ucdjs/pipelines-executor",
+  "version": "0.0.1-beta.1",
+  "type": "module",
+  "author": {
+    "name": "Lucas Nørgård",
+    "email": "lucasnrgaard@gmail.com",
+    "url": "https://luxass.dev"
+  },
+  "license": "MIT",
+  "homepage": "https://github.com/ucdjs/ucd",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/ucdjs/ucd.git",
+    "directory": "packages/pipelines/pipeline-executor"
+  },
+  "bugs": {
+    "url": "https://github.com/ucdjs/ucd/issues"
+  },
+  "exports": {
+    ".": "./dist/index.mjs",
+    "./package.json": "./package.json"
+  },
+  "types": "./dist/index.d.mts",
+  "files": [
+    "dist"
+  ],
+  "engines": {
+    "node": ">=22.18"
+  },
+  "dependencies": {
+    "@ucdjs/pipelines-core": "0.0.1-beta.1",
+    "@ucdjs/pipelines-artifacts": "0.0.1-beta.1",
+    "@ucdjs/pipelines-graph": "0.0.1-beta.1"
+  },
+  "devDependencies": {
+    "@luxass/eslint-config": "7.2.0",
+    "eslint": "10.0.0",
+    "publint": "0.3.17",
+    "tsdown": "0.20.3",
+    "typescript": "5.9.3",
+    "@ucdjs-tooling/tsdown-config": "1.0.0",
+    "@ucdjs-tooling/tsconfig": "1.0.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsdown --tsconfig=./tsconfig.build.json",
+    "dev": "tsdown --watch",
+    "clean": "git clean -xdf dist node_modules",
+    "lint": "eslint .",
+    "typecheck": "tsc --noEmit -p tsconfig.build.json"
+  }
+}