@ucdjs/pipelines-core 0.0.1-beta.1

package/dist/index.mjs

@@ -0,0 +1,338 @@
+import { n as resolveMultipleSourceFiles, r as resolveSourceFiles, t as definePipelineSource } from "./source-EtWCktj_.mjs";
+import { n as definePipelineTransform, t as applyTransforms } from "./transform-BDsUG3EV.mjs";
+import picomatch from "picomatch";
+
+//#region src/dependencies.ts
+function parseDependency(dep) {
+	if (dep.startsWith("route:")) {
+		const routeId = dep.slice(6);
+		if (!routeId) throw new Error(`Invalid route dependency format: ${dep}. Expected "route:<id>" with non-empty id`);
+		return {
+			type: "route",
+			routeId
+		};
+	}
+	if (dep.startsWith("artifact:")) {
+		const rest = dep.slice(9);
+		const sepIndex = rest.indexOf(":");
+		if (sepIndex <= 0 || sepIndex >= rest.length - 1) throw new Error(`Invalid dependency format: ${dep}. Expected "route:<id>" or "artifact:<routeId>:<artifactName>"`);
+		return {
+			type: "artifact",
+			routeId: rest.slice(0, sepIndex),
+			artifactName: rest.slice(sepIndex + 1)
+		};
+	}
+	throw new Error(`Invalid dependency format: ${dep}. Expected "route:<id>" or "artifact:<routeId>:<artifactName>"`);
+}
+function isRouteDependency(dep) {
+	return dep.startsWith("route:");
+}
+function isArtifactDependency(dep) {
+	return dep.startsWith("artifact:");
+}
+function createRouteDependency(routeId) {
+	return `route:${routeId}`;
+}
+function createArtifactDependency(routeId, artifactName) {
+	return `artifact:${routeId}:${artifactName}`;
+}
+
+//#endregion
+//#region src/dag.ts
+function buildDAG(routes) {
+	const errors = [];
+	const nodes = /* @__PURE__ */ new Map();
+	const artifactsByRoute = /* @__PURE__ */ new Map();
+	const seenIds = /* @__PURE__ */ new Map();
+	for (let i = 0; i < routes.length; i++) {
+		const route = routes[i];
+		if (!route) continue;
+		const id = route.id;
+		if (seenIds.has(id)) errors.push({
+			type: "duplicate-route",
+			message: `Duplicate route ID "${id}" found at index ${seenIds.get(id)} and ${i}`,
+			details: { routeId: id }
+		});
+		else seenIds.set(id, i);
+	}
+	if (errors.length > 0) return {
+		valid: false,
+		errors
+	};
+	const routeIds = new Set(routes.map((r) => r.id));
+	for (const route of routes) {
+		const emittedArtifacts = /* @__PURE__ */ new Set();
+		if (route.emits) for (const artifactName of Object.keys(route.emits)) emittedArtifacts.add(`${route.id}:${artifactName}`);
+		artifactsByRoute.set(route.id, emittedArtifacts);
+		nodes.set(route.id, {
+			id: route.id,
+			dependencies: /* @__PURE__ */ new Set(),
+			dependents: /* @__PURE__ */ new Set(),
+			emittedArtifacts
+		});
+	}
+	for (const route of routes) {
+		const node = nodes.get(route.id);
+		if (!route.depends) continue;
+		for (const dep of route.depends) {
+			const parsed = parseDependency(dep);
+			if (isRouteDependency(dep)) {
+				if (!routeIds.has(parsed.routeId)) {
+					errors.push({
+						type: "missing-route",
+						message: `Route "${route.id}" depends on non-existent route "${parsed.routeId}"`,
+						details: {
+							routeId: route.id,
+							dependencyId: parsed.routeId
+						}
+					});
+					continue;
+				}
+				node.dependencies.add(parsed.routeId);
+				nodes.get(parsed.routeId).dependents.add(route.id);
+			} else if (isArtifactDependency(dep)) {
+				if (parsed.type !== "artifact") continue;
+				if (!routeIds.has(parsed.routeId)) {
+					errors.push({
+						type: "missing-route",
+						message: `Route "${route.id}" depends on artifact from non-existent route "${parsed.routeId}"`,
+						details: {
+							routeId: route.id,
+							dependencyId: parsed.routeId
+						}
+					});
+					continue;
+				}
+				const routeArtifacts = artifactsByRoute.get(parsed.routeId);
+				const artifactKey = `${parsed.routeId}:${parsed.artifactName}`;
+				if (!routeArtifacts?.has(artifactKey)) {
+					errors.push({
+						type: "missing-artifact",
+						message: `Route "${route.id}" depends on non-existent artifact "${parsed.artifactName}" from route "${parsed.routeId}"`,
+						details: {
+							routeId: route.id,
+							dependencyId: artifactKey
+						}
+					});
+					continue;
+				}
+				node.dependencies.add(parsed.routeId);
+				nodes.get(parsed.routeId).dependents.add(route.id);
+			}
+		}
+	}
+	const cycleResult = detectCycle(nodes);
+	if (cycleResult) errors.push({
+		type: "cycle",
+		message: `Circular dependency detected: ${cycleResult.join(" -> ")}`,
+		details: { cycle: cycleResult }
+	});
+	if (errors.length > 0) return {
+		valid: false,
+		errors
+	};
+	return {
+		valid: true,
+		errors: [],
+		dag: {
+			nodes,
+			executionOrder: topologicalSort(nodes)
+		}
+	};
+}
+function detectCycle(nodes) {
+	const visited = /* @__PURE__ */ new Set();
+	const recursionStack = /* @__PURE__ */ new Set();
+	const path = [];
+	function dfs(nodeId) {
+		visited.add(nodeId);
+		recursionStack.add(nodeId);
+		path.push(nodeId);
+		const node = nodes.get(nodeId);
+		if (node) {
+			for (const depId of node.dependencies) if (!visited.has(depId)) {
+				const cycle = dfs(depId);
+				if (cycle) return cycle;
+			} else if (recursionStack.has(depId)) {
+				const cycleStart = path.indexOf(depId);
+				return [...path.slice(cycleStart), depId];
+			}
+		}
+		path.pop();
+		recursionStack.delete(nodeId);
+		return null;
+	}
+	for (const nodeId of nodes.keys()) if (!visited.has(nodeId)) {
+		const cycle = dfs(nodeId);
+		if (cycle) return cycle;
+	}
+	return null;
+}
+function topologicalSort(nodes) {
+	const result = [];
+	const visited = /* @__PURE__ */ new Set();
+	const temp = /* @__PURE__ */ new Set();
+	function visit(nodeId) {
+		if (temp.has(nodeId)) return;
+		if (visited.has(nodeId)) return;
+		temp.add(nodeId);
+		const node = nodes.get(nodeId);
+		if (node) for (const depId of node.dependencies) visit(depId);
+		temp.delete(nodeId);
+		visited.add(nodeId);
+		result.push(nodeId);
+	}
+	for (const nodeId of nodes.keys()) if (!visited.has(nodeId)) visit(nodeId);
+	return result;
+}
+function getExecutionLayers(dag) {
+	const layers = [];
+	const scheduled = /* @__PURE__ */ new Set();
+	const remaining = new Set(dag.nodes.keys());
+	while (remaining.size > 0) {
+		const layer = [];
+		for (const nodeId of remaining) if ([...dag.nodes.get(nodeId).dependencies].every((dep) => scheduled.has(dep))) layer.push(nodeId);
+		if (layer.length === 0) break;
+		for (const nodeId of layer) {
+			remaining.delete(nodeId);
+			scheduled.add(nodeId);
+		}
+		layers.push(layer);
+	}
+	return layers;
+}
+
+//#endregion
+//#region src/filters.ts
+function byName(name) {
+	return (ctx) => ctx.file.name === name;
+}
+function byDir(dir) {
+	return (ctx) => ctx.file.dir === dir;
+}
+function byExt(ext) {
+	if (ext === "") return (ctx) => ctx.file.ext === "";
+	const normalizedExt = ext.startsWith(".") ? ext : `.${ext}`;
+	return (ctx) => ctx.file.ext === normalizedExt;
+}
+function byGlob(pattern) {
+	const matcher = picomatch(pattern);
+	return (ctx) => matcher(ctx.file.path);
+}
+function byPath(pathPattern) {
+	if (typeof pathPattern === "string") return (ctx) => ctx.file.path === pathPattern;
+	return (ctx) => pathPattern.test(ctx.file.path);
+}
+function byProp(pattern) {
+	if (typeof pattern === "string") return (ctx) => ctx.row?.property === pattern;
+	return (ctx) => !!ctx.row?.property && pattern.test(ctx.row.property);
+}
+function bySource(sourceIds) {
+	const ids = Array.isArray(sourceIds) ? sourceIds : [sourceIds];
+	return (ctx) => ctx.source != null && ids.includes(ctx.source.id);
+}
+function and(...filters) {
+	return (ctx) => filters.every((f) => f(ctx));
+}
+function or(...filters) {
+	return (ctx) => filters.some((f) => f(ctx));
+}
+function not(filter) {
+	return (ctx) => !filter(ctx);
+}
+function always() {
+	return () => true;
+}
+function never() {
+	return () => false;
+}
+
+//#endregion
+//#region src/pipeline.ts
+/**
+* Define a pipeline configuration.
+*
+* This returns a pure data structure describing the pipeline.
+* To execute the pipeline, pass it to a pipeline executor.
+*
+* @example
+* ```ts
+* const pipeline = definePipeline({
+*   id: "my-pipeline",
+*   name: "My Pipeline",
+*   versions: ["16.0.0"],
+*   inputs: [mySource],
+*   routes: [myRoute],
+* });
+*
+* // Execute with an executor
+* const executor = createPipelineExecutor({ pipelines: [pipeline] });
+* const result = await executor.run();
+* ```
+*/
+function definePipeline(options) {
+	const dagResult = buildDAG(options.routes);
+	if (!dagResult.valid) {
+		const errorMessages = dagResult.errors.map((e) => e.message).join("\n  ");
+		throw new Error(`Pipeline "${options.id}" has invalid route dependencies:\n  ${errorMessages}`);
+	}
+	return {
+		_type: "pipeline-definition",
+		id: options.id,
+		name: options.name,
+		description: options.description,
+		versions: options.versions,
+		inputs: options.inputs,
+		routes: options.routes,
+		include: options.include,
+		strict: options.strict ?? false,
+		concurrency: options.concurrency ?? 4,
+		fallback: options.fallback,
+		onEvent: options.onEvent,
+		dag: dagResult.dag,
+		tags: options.tags ?? []
+	};
+}
+function isPipelineDefinition(value) {
+	return typeof value === "object" && value !== null && "_type" in value && typeof value._type === "string" && value._type === "pipeline-definition" && "id" in value && typeof value.id === "string" && "inputs" in value && Array.isArray(value.inputs) && "routes" in value && Array.isArray(value.routes);
+}
+function getPipelineRouteIds(pipeline) {
+	return pipeline.routes.map((route) => route.id);
+}
+function getPipelineSourceIds(pipeline) {
+	return pipeline.inputs.map((source) => source.id);
+}
+
+//#endregion
+//#region src/route.ts
+function definePipelineRoute(definition) {
+	return definition;
+}
+
+//#endregion
+//#region src/utils/arrays.ts
+/**
+* Split a line into exactly 2 fields.
+* Returns a tuple [first, second] or null if not enough fields.
+*
+* After null check, TypeScript knows both elements exist and are strings.
+*/
+function splitTwoFields(line, delimiter) {
+	const parts = line.split(delimiter);
+	if (parts.length < 2) return null;
+	return [parts[0], parts[1]];
+}
+/**
+* Split a line with minimum field count check.
+* Returns the array or null if not enough fields.
+*
+* Use when you need more than 2 fields (e.g., UnicodeData with 14 fields).
+* Access fields with nullish coalescing: `fields[0] ?? ""`
+*/
+function splitMinFields(line, delimiter, minFields) {
+	const parts = line.split(delimiter);
+	return parts.length >= minFields ? parts : null;
+}
+
+//#endregion
+export { always, and, applyTransforms, buildDAG, byDir, byExt, byGlob, byName, byPath, byProp, bySource, createArtifactDependency, createRouteDependency, definePipeline, definePipelineRoute, definePipelineSource, definePipelineTransform, getExecutionLayers, getPipelineRouteIds, getPipelineSourceIds, isArtifactDependency, isPipelineDefinition, isRouteDependency, never, not, or, parseDependency, resolveMultipleSourceFiles, resolveSourceFiles, splitMinFields, splitTwoFields };

package/dist/source-DYH0SvDh.d.mts

@@ -0,0 +1,37 @@
+import { n as FileContext, s as PipelineFilter } from "./types-t7nns1Hx.mjs";
+
+//#region src/source.d.ts
+interface StreamOptions {
+  chunkSize?: number;
+  start?: number;
+  end?: number;
+}
+interface FileMetadata {
+  size: number;
+  hash?: string;
+  lastModified?: string;
+}
+interface SourceBackend {
+  listFiles: (version: string) => Promise<FileContext[]>;
+  readFile: (file: FileContext) => Promise<string>;
+  readFileStream?: (file: FileContext, options?: StreamOptions) => AsyncIterable<Uint8Array>;
+  getMetadata?: (file: FileContext) => Promise<FileMetadata>;
+}
+interface PipelineSourceDefinition<TId extends string = string> {
+  id: TId;
+  backend: SourceBackend;
+  includes?: PipelineFilter;
+  excludes?: PipelineFilter;
+}
+interface SourceFileContext extends FileContext {
+  source: {
+    id: string;
+  };
+}
+declare function definePipelineSource<const TId extends string>(definition: PipelineSourceDefinition<TId>): PipelineSourceDefinition<TId>;
+declare function resolveSourceFiles(source: PipelineSourceDefinition, version: string): Promise<SourceFileContext[]>;
+declare function resolveMultipleSourceFiles(sources: PipelineSourceDefinition[] | readonly PipelineSourceDefinition[], version: string): Promise<SourceFileContext[]>;
+type InferSourceId<T> = T extends PipelineSourceDefinition<infer TId> ? TId : never;
+type InferSourceIds<T extends readonly PipelineSourceDefinition[]> = { [K in keyof T]: InferSourceId<T[K]> }[number];
+//#endregion
+export { SourceBackend as a, definePipelineSource as c, PipelineSourceDefinition as i, resolveMultipleSourceFiles as l, InferSourceId as n, SourceFileContext as o, InferSourceIds as r, StreamOptions as s, FileMetadata as t, resolveSourceFiles as u };

package/dist/source-EtWCktj_.mjs

@@ -0,0 +1,26 @@
+//#region src/source.ts
+function definePipelineSource(definition) {
+	return definition;
+}
+async function resolveSourceFiles(source, version) {
+	return (await source.backend.listFiles(version)).filter((file) => {
+		const ctx = { file };
+		if (source.includes && !source.includes(ctx)) return false;
+		if (source.excludes && source.excludes(ctx)) return false;
+		return true;
+	}).map((file) => ({
+		...file,
+		source: { id: source.id }
+	}));
+}
+async function resolveMultipleSourceFiles(sources, version) {
+	const filesByPath = /* @__PURE__ */ new Map();
+	for (const source of sources) {
+		const files = await resolveSourceFiles(source, version);
+		for (const file of files) filesByPath.set(file.path, file);
+	}
+	return Array.from(filesByPath.values());
+}
+
+//#endregion
+export { resolveMultipleSourceFiles as n, resolveSourceFiles as r, definePipelineSource as t };

package/dist/sources/index.d.mts

@@ -0,0 +1,44 @@
+import { n as FileContext } from "../types-t7nns1Hx.mjs";
+import { a as SourceBackend, i as PipelineSourceDefinition } from "../source-DYH0SvDh.mjs";
+
+//#region src/sources/http.d.ts
+interface HttpBackendOptions {
+  baseUrl: string;
+  headers?: Record<string, string>;
+  timeout?: number;
+}
+declare const UNICODE_ORG_BASE_URL = "https://www.unicode.org/Public/";
+declare function createHttpBackend(options: HttpBackendOptions): SourceBackend;
+interface HttpSourceOptions extends HttpBackendOptions {
+  id?: string;
+}
+declare function createHttpSource(): PipelineSourceDefinition<"http">;
+declare function createHttpSource<TId extends string>(options: HttpSourceOptions & {
+  id: TId;
+}): PipelineSourceDefinition<TId>;
+declare function createUnicodeOrgSource(): PipelineSourceDefinition<"unicode-org">;
+declare function createUnicodeOrgSource<TId extends string>(id: TId): PipelineSourceDefinition<TId>;
+declare const unicodeOrgSource: PipelineSourceDefinition<"unicode-org">;
+//#endregion
+//#region src/sources/memory.d.ts
+interface MemoryFile {
+  path: string;
+  content: string;
+  dir?: FileContext["dir"];
+}
+interface MemoryBackendOptions {
+  files: Record<string, MemoryFile[]>;
+}
+declare function createMemoryBackend(options: MemoryBackendOptions): SourceBackend;
+interface MemorySourceOptions {
+  id?: string;
+  files: Record<string, MemoryFile[]>;
+}
+declare function createMemorySource(options: MemorySourceOptions & {
+  id?: undefined;
+}): PipelineSourceDefinition<"memory">;
+declare function createMemorySource<TId extends string>(options: MemorySourceOptions & {
+  id: TId;
+}): PipelineSourceDefinition<TId>;
+//#endregion
+export { type HttpBackendOptions, type HttpSourceOptions, type MemoryBackendOptions, type MemoryFile, type MemorySourceOptions, UNICODE_ORG_BASE_URL, createHttpBackend, createHttpSource, createMemoryBackend, createMemorySource, createUnicodeOrgSource, unicodeOrgSource };

package/dist/sources/index.mjs

@@ -0,0 +1,113 @@
+import { t as definePipelineSource } from "../source-EtWCktj_.mjs";
+
+//#region src/sources/http.ts
+const UNICODE_ORG_BASE_URL = "https://www.unicode.org/Public/";
+function createHttpBackend(options) {
+	const { baseUrl, headers = {}, timeout = 3e4 } = options;
+	const normalizedBaseUrl = baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`;
+	return {
+		async listFiles(version) {
+			throw new Error(`HTTP backend does not support listing files. Use a file manifest or provide explicit file list for version ${version}.`);
+		},
+		async readFile(file) {
+			const url = `${normalizedBaseUrl}${file.version}/${file.path}`;
+			const controller = new AbortController();
+			const timeoutId = setTimeout(() => controller.abort(), timeout);
+			try {
+				const response = await fetch(url, {
+					headers,
+					signal: controller.signal
+				});
+				if (!response.ok) throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
+				return response.text();
+			} finally {
+				clearTimeout(timeoutId);
+			}
+		},
+		async getMetadata(file) {
+			const url = `${normalizedBaseUrl}${file.version}/${file.path}`;
+			const controller = new AbortController();
+			const timeoutId = setTimeout(() => controller.abort(), timeout);
+			try {
+				const response = await fetch(url, {
+					method: "HEAD",
+					headers,
+					signal: controller.signal
+				});
+				if (!response.ok) throw new Error(`Failed to get metadata for ${url}: ${response.status}`);
+				const contentLength = response.headers.get("content-length");
+				const lastModified = response.headers.get("last-modified");
+				return {
+					size: contentLength ? Number.parseInt(contentLength, 10) : 0,
+					lastModified: lastModified || void 0
+				};
+			} finally {
+				clearTimeout(timeoutId);
+			}
+		}
+	};
+}
+function createHttpSource(options) {
+	return definePipelineSource({
+		id: options?.id ?? "http",
+		backend: createHttpBackend(options ?? { baseUrl: "" })
+	});
+}
+function createUnicodeOrgSource(id) {
+	return createHttpSource({
+		id: id ?? "unicode-org",
+		baseUrl: UNICODE_ORG_BASE_URL
+	});
+}
+const unicodeOrgSource = createUnicodeOrgSource();
+
+//#endregion
+//#region src/sources/memory.ts
+function getFileContext(version, file) {
+	const path = file.path;
+	const parts = path.split("/");
+	const name = parts[parts.length - 1];
+	if (!name) throw new Error(`Invalid file path: ${file.path}`);
+	const extIndex = name.lastIndexOf(".");
+	const ext = extIndex >= 0 ? name.slice(extIndex) : "";
+	return {
+		version,
+		dir: file.dir || parts[0] || "ucd",
+		path,
+		name,
+		ext
+	};
+}
+function createMemoryBackend(options) {
+	const { files } = options;
+	return {
+		async listFiles(version) {
+			const versionFiles = files[version];
+			if (!versionFiles) return [];
+			return versionFiles.map((f) => getFileContext(version, f));
+		},
+		async readFile(file) {
+			const versionFiles = files[file.version];
+			if (!versionFiles) throw new Error(`Version ${file.version} not found in memory backend`);
+			const memFile = versionFiles.find((f) => f.path === file.path);
+			if (!memFile) throw new Error(`File ${file.path} not found in version ${file.version}`);
+			return memFile.content;
+		},
+		async getMetadata(file) {
+			const versionFiles = files[file.version];
+			if (!versionFiles) throw new Error(`Version ${file.version} not found in memory backend`);
+			const memFile = versionFiles.find((f) => f.path === file.path);
+			if (!memFile) throw new Error(`File ${file.path} not found in version ${file.version}`);
+			return { size: new TextEncoder().encode(memFile.content).length };
+		}
+	};
+}
+function createMemorySource(options) {
+	return definePipelineSource({
+		id: options?.id ?? "memory",
+		backend: createMemoryBackend({ files: options?.files ?? {} })
+	});
+}
+
+//#endregion
+export { UNICODE_ORG_BASE_URL, createHttpBackend, createHttpSource, createMemoryBackend, createMemorySource, createUnicodeOrgSource, unicodeOrgSource };

package/dist/transform-BDsUG3EV.mjs

@@ -0,0 +1,62 @@
+//#region src/transform.ts
+/**
+* Creates and returns a validated pipeline transform definition.
+*
+* This function is primarily a type-safe way to define transforms. It ensures that
+* the transform adheres to the `PipelineTransformDefinition` interface and preserves
+* type information for chaining.
+*
+* @typeParam TInput - The input type accepted by the transform.
+* @typeParam TOutput - The output type produced by the transform.
+* @param {PipelineTransformDefinition<TInput, TOutput>} definition - The transform definition.
+* @returns {PipelineTransformDefinition<TInput, TOutput>} The same definition, typed correctly.
+*
+* @example
+* ```ts
+* const uppercase = definePipelineTransform({
+*   id: 'uppercase',
+*   fn: async function* (_ctx, rows) {
+*     for await (const row of rows) {
+*       yield row.toUpperCase();
+*     }
+*   },
+* });
+* ```
+*/
+function definePipelineTransform(definition) {
+	return definition;
+}
+/**
+* Applies a sequence of transforms to an async iterable, composing them together.
+*
+* This function threads the output of one transform into the input of the next,
+* creating a pipeline. All iteration is lazy—values are pulled through the pipeline
+* only as they are consumed.
+*
+* @typeParam TInput - The input type of the first transform.
+* @param {TransformContext} ctx - The context to pass to each transform.
+* @param {AsyncIterable<TInput>} rows - The initial data source.
+* @param {readonly PipelineTransformDefinition<any, any>[]} transforms - The transforms to apply in order.
+* @returns {AsyncIterable<unknown>} An async iterable of the final output (typed as `unknown` since
+*                                    the result type depends on the transform sequence).
+*
+* @remarks
+* The output type can be narrowed using the `ChainTransforms` utility type if the
+* exact sequence of transforms is known at compile time.
+*
+* @example
+* ```ts
+* const output = applyTransforms(ctx, sourceRows, [
+*   definePipelineTransform({ id: 'filter', fn: filterFn }),
+*   definePipelineTransform({ id: 'map', fn: mapFn }),
+* ]);
+* ```
+*/
+async function* applyTransforms(ctx, rows, transforms) {
+	let current = rows;
+	for (const transform of transforms) current = transform.fn(ctx, current);
+	yield* current;
+}
+
+//#endregion
+export { definePipelineTransform as n, applyTransforms as t };