@ucdjs/pipelines-core 0.0.1-beta.6 → 0.0.1-beta.8

package/dist/tracing/index.d.mts

@@ -0,0 +1,178 @@
+import { n as FileContext } from "../types-B_R0NiDI.mjs";
+
+//#region src/tracing/events.d.ts
+type PipelineErrorScope = "pipeline" | "version" | "file" | "route";
+interface PipelineError {
+  scope: PipelineErrorScope;
+  message: string;
+  error?: unknown;
+  file?: FileContext;
+  routeId?: string;
+  version?: string;
+}
+//#endregion
+//#region src/tracing/types.d.ts
+type PipelineTraceKind = "pipeline" | "version" | "source.provided" | "source.listing" | "file.route" | "file.matched" | "file.fallback" | "file.skipped" | "file.queued" | "file.dequeued" | "parse" | "resolve" | "cache.hit" | "cache.miss" | "cache.store" | "output.produced" | "output" | "output.written" | "dependency.resolve" | "error" | (string & {});
+interface PipelineTraceBase<TKind extends PipelineTraceKind> {
+  id: string;
+  kind: TKind;
+  pipelineId: string;
+  traceId: string;
+  spanId?: string;
+  parentSpanId?: string;
+  timestamp: number;
+  schemaVersion?: number;
+}
+interface PipelineSpanRecord extends PipelineTraceBase<"pipeline"> {
+  versions: string[];
+  startTimestamp: number;
+  durationMs: number;
+}
+interface VersionSpanRecord extends PipelineTraceBase<"version"> {
+  version: string;
+  startTimestamp: number;
+  durationMs: number;
+}
+interface SourceProvidedTraceRecord extends PipelineTraceBase<"source.provided"> {
+  version: string;
+  file?: FileContext;
+}
+interface SourceListingSpanRecord extends PipelineTraceBase<"source.listing"> {
+  version: string;
+  fileCount: number;
+  startTimestamp: number;
+  durationMs: number;
+}
+interface FileRouteSpanRecord extends PipelineTraceBase<"file.route"> {
+  version: string;
+  file: FileContext;
+  routeId: string;
+  startTimestamp: number;
+  durationMs: number;
+}
+interface FileMatchedTraceRecord extends PipelineTraceBase<"file.matched"> {
+  version: string;
+  file: FileContext;
+  routeId: string;
+}
+interface FileFallbackTraceRecord extends PipelineTraceBase<"file.fallback"> {
+  version: string;
+  file: FileContext;
+}
+interface FileSkippedTraceRecord extends PipelineTraceBase<"file.skipped"> {
+  version: string;
+  file: FileContext;
+  reason: "no-match" | "filtered";
+}
+interface FileQueuedTraceRecord extends PipelineTraceBase<"file.queued"> {
+  version: string;
+  file: FileContext;
+  routeId: string;
+}
+interface FileDequeuedTraceRecord extends PipelineTraceBase<"file.dequeued"> {
+  version: string;
+  file: FileContext;
+  routeId: string;
+  waitDurationMs: number;
+}
+interface ParseSpanRecord extends PipelineTraceBase<"parse"> {
+  version: string;
+  file: FileContext;
+  routeId: string;
+  rowCount: number;
+  filteredRowCount: number;
+  startTimestamp: number;
+  durationMs: number;
+}
+interface ResolveSpanRecord extends PipelineTraceBase<"resolve"> {
+  version: string;
+  file: FileContext;
+  routeId: string;
+  outputCount: number;
+  startTimestamp: number;
+  durationMs: number;
+}
+interface CacheHitTraceRecord extends PipelineTraceBase<"cache.hit"> {
+  version: string;
+  routeId: string;
+  file: FileContext;
+}
+interface CacheMissTraceRecord extends PipelineTraceBase<"cache.miss"> {
+  version: string;
+  routeId: string;
+  file: FileContext;
+}
+interface CacheStoreTraceRecord extends PipelineTraceBase<"cache.store"> {
+  version: string;
+  routeId: string;
+  file: FileContext;
+}
+type CacheTraceRecord = CacheHitTraceRecord | CacheMissTraceRecord | CacheStoreTraceRecord;
+interface OutputProducedTraceRecord extends PipelineTraceBase<"output.produced"> {
+  version: string;
+  routeId: string;
+  file?: FileContext;
+  outputIndex: number;
+  property?: string;
+}
+interface OutputTraceRecord extends PipelineTraceBase<"output"> {
+  version: string;
+  routeId: string;
+  file: FileContext;
+  outputIndex: number;
+  outputId: string;
+  property?: string;
+  sink: string;
+  format: "json" | "text";
+  locator: string;
+}
+interface OutputWrittenTraceRecord extends PipelineTraceBase<"output.written"> {
+  version: string;
+  routeId: string;
+  file: FileContext;
+  outputIndex: number;
+  outputId: string;
+  property?: string;
+  sink: string;
+  locator: string;
+  status: "written" | "failed";
+  error?: string;
+}
+interface DependencyResolveTraceRecord extends PipelineTraceBase<"dependency.resolve"> {
+  version: string;
+  file: FileContext;
+  routeId: string;
+  dependsOnRouteId: string;
+  status: "resolved" | "missing";
+}
+interface ErrorTraceRecord extends PipelineTraceBase<"error"> {
+  error: PipelineError;
+  stack?: string;
+}
+type PipelineTraceRecord = PipelineSpanRecord | VersionSpanRecord | SourceProvidedTraceRecord | SourceListingSpanRecord | FileRouteSpanRecord | FileMatchedTraceRecord | FileFallbackTraceRecord | FileSkippedTraceRecord | FileQueuedTraceRecord | FileDequeuedTraceRecord | ParseSpanRecord | ResolveSpanRecord | CacheHitTraceRecord | CacheMissTraceRecord | CacheStoreTraceRecord | OutputProducedTraceRecord | OutputTraceRecord | OutputWrittenTraceRecord | DependencyResolveTraceRecord | ErrorTraceRecord;
+type PipelineTraceRecordByKind<TKind extends PipelineTraceKind> = Extract<PipelineTraceRecord, {
+  kind: TKind;
+}>;
+type PipelineTraceInput = { [K in PipelineTraceKind]: Omit<PipelineTraceRecordByKind<K>, "id" | "traceId" | "spanId" | "parentSpanId" | "timestamp"> }[PipelineTraceKind];
+type PipelineTraceEmitInput = { [K in PipelineTraceKind]: Omit<PipelineTraceRecordByKind<K>, "id" | "traceId" | "spanId" | "parentSpanId" | "timestamp" | "pipelineId"> }[PipelineTraceKind];
+interface PipelineOutputManifestEntry {
+  outputIndex: number;
+  outputId: string;
+  routeId: string;
+  pipelineId: string;
+  version: string;
+  property?: string;
+  sink: string;
+  format: "json" | "text";
+  locator: string;
+  status: "resolved" | "written" | "failed";
+  error?: string;
+}
+//#endregion
+//#region src/tracing/utils.d.ts
+declare function buildOutputManifestFromTraces(traces: readonly PipelineTraceRecord[]): PipelineOutputManifestEntry[];
+declare const PIPELINE_TRACE_PHASES: readonly ["Pipeline", "Version", "Parse", "Resolve", "File", "Cache", "Error", "Other"];
+type PipelineTracePhase = typeof PIPELINE_TRACE_PHASES[number];
+declare function getTracePhase(kind: PipelineTraceKind): PipelineTracePhase;
+//#endregion
+export { CacheHitTraceRecord, CacheMissTraceRecord, CacheStoreTraceRecord, CacheTraceRecord, DependencyResolveTraceRecord, ErrorTraceRecord, FileDequeuedTraceRecord, FileFallbackTraceRecord, FileMatchedTraceRecord, FileQueuedTraceRecord, FileRouteSpanRecord, FileSkippedTraceRecord, OutputProducedTraceRecord, OutputTraceRecord, OutputWrittenTraceRecord, PIPELINE_TRACE_PHASES, ParseSpanRecord, PipelineError, PipelineErrorScope, PipelineOutputManifestEntry, PipelineSpanRecord, PipelineTraceEmitInput, PipelineTraceInput, PipelineTraceKind, PipelineTracePhase, PipelineTraceRecord, PipelineTraceRecordByKind, ResolveSpanRecord, SourceListingSpanRecord, SourceProvidedTraceRecord, VersionSpanRecord, buildOutputManifestFromTraces, getTracePhase };

package/dist/tracing/index.mjs

@@ -0,0 +1,60 @@
+//#region src/tracing/utils.ts
+function buildOutputManifestFromTraces(traces) {
+	const manifest = /* @__PURE__ */ new Map();
+	for (const trace of traces) {
+		if (trace.kind === "output") {
+			const key = getOutputManifestKey(trace.pipelineId, trace.version, trace.routeId, trace.outputIndex, trace.outputId, trace.locator);
+			manifest.set(key, {
+				outputIndex: trace.outputIndex,
+				outputId: trace.outputId,
+				routeId: trace.routeId,
+				pipelineId: trace.pipelineId,
+				version: trace.version,
+				property: trace.property,
+				sink: trace.sink,
+				format: trace.format,
+				locator: trace.locator,
+				status: "resolved"
+			});
+			continue;
+		}
+		if (trace.kind === "output.written") {
+			const key = getOutputManifestKey(trace.pipelineId, trace.version, trace.routeId, trace.outputIndex, trace.outputId, trace.locator);
+			const entry = manifest.get(key);
+			if (entry) manifest.set(key, {
+				...entry,
+				status: trace.status,
+				error: trace.error
+			});
+		}
+	}
+	return [...manifest.values()].toSorted((left, right) => {
+		return left.pipelineId.localeCompare(right.pipelineId) || left.version.localeCompare(right.version) || left.outputIndex - right.outputIndex || left.routeId.localeCompare(right.routeId) || left.outputId.localeCompare(right.outputId) || left.locator.localeCompare(right.locator);
+	});
+}
+function getOutputManifestKey(pipelineId, version, routeId, outputIndex, outputId, locator) {
+	return `${pipelineId}:${version}:${routeId}:${outputIndex}:${outputId}:${locator}`;
+}
+const PIPELINE_TRACE_PHASES = [
+	"Pipeline",
+	"Version",
+	"Parse",
+	"Resolve",
+	"File",
+	"Cache",
+	"Error",
+	"Other"
+];
+function getTracePhase(kind) {
+	if (kind === "error") return "Error";
+	if (kind === "pipeline") return "Pipeline";
+	if (kind === "version") return "Version";
+	if (kind === "parse") return "Parse";
+	if (kind === "resolve") return "Resolve";
+	if (kind === "source.listing" || kind.startsWith("file.") || kind.startsWith("source.")) return "File";
+	if (kind.startsWith("cache.")) return "Cache";
+	if (kind.startsWith("output.")) return "Other";
+	return "Other";
+}
+//#endregion
+export { PIPELINE_TRACE_PHASES, buildOutputManifestFromTraces, getTracePhase };

package/dist/{transform-BSrhinRr.d.mts → transform-BkBLzZAP.d.mts}

@@ -1,9 +1,10 @@
-import { n as FileContext } from "./types-t7nns1Hx.mjs";
+import { l as PipelineLogger, n as FileContext } from "./types-B_R0NiDI.mjs";
 
 //#region src/transform.d.ts
 interface TransformContext {
   version: string;
   file: FileContext;
+  logger: PipelineLogger;
 }
 type PipelineTransformFunction<TInput, TOutput> = (ctx: TransformContext, rows: AsyncIterable<TInput>) => AsyncIterable<TOutput>;
 interface PipelineTransformDefinition<TInput = unknown, TOutput = unknown> {
@@ -61,7 +62,7 @@ type ChainTransforms<TInput, TTransforms extends readonly PipelineTransformDefin
  * Applies a sequence of transforms to an async iterable, composing them together.
  *
  * This function threads the output of one transform into the input of the next,
- * creating a pipeline. All iteration is lazy—values are pulled through the pipeline
+ * creating a pipeline. All iteration is lazy-values are pulled through the pipeline
  * only as they are consumed.
  *
  * @typeParam TInput - The input type of the first transform.

package/dist/{transform-BDsUG3EV.mjs → transform-C7y-ZJKs.mjs}

@@ -30,7 +30,7 @@ function definePipelineTransform(definition) {
 * Applies a sequence of transforms to an async iterable, composing them together.
 *
 * This function threads the output of one transform into the input of the next,
-* creating a pipeline. All iteration is lazy—values are pulled through the pipeline
+* creating a pipeline. All iteration is lazy-values are pulled through the pipeline
 * only as they are consumed.
 *
 * @typeParam TInput - The input type of the first transform.
@@ -57,6 +57,5 @@ async function* applyTransforms(ctx, rows, transforms) {
 	for (const transform of transforms) current = transform.fn(ctx, current);
 	yield* current;
 }
-
 //#endregion
-export { definePipelineTransform as n, applyTransforms as t };
+export { definePipelineTransform as n, applyTransforms as t };

package/dist/transforms/index.d.mts

@@ -1,5 +1,5 @@
-import { a as ParsedRow } from "../types-t7nns1Hx.mjs";
-import { a as PipelineTransformDefinition } from "../transform-BSrhinRr.mjs";
+import { a as ParsedRow } from "../types-B_R0NiDI.mjs";
+import { a as PipelineTransformDefinition } from "../transform-BkBLzZAP.mjs";
 
 //#region src/transforms/deduplicate.d.ts
 declare const deduplicateRows: PipelineTransformDefinition<ParsedRow, ParsedRow>;

package/dist/transforms/index.mjs

@@ -1,5 +1,4 @@
-import { n as definePipelineTransform } from "../transform-BDsUG3EV.mjs";
-
+import { n as definePipelineTransform } from "../transform-C7y-ZJKs.mjs";
 //#region src/transforms/deduplicate.ts
 function getRowKey(row) {
 	if (row.kind === "point" && row.codePoint) return `point:${row.codePoint}`;
@@ -47,7 +46,6 @@ function createDeduplicateTransform(options = {}) {
 		}
 	});
 }
-
 //#endregion
 //#region src/transforms/expand-ranges.ts
 function hexToNumber$1(hex) {
@@ -95,11 +93,11 @@ function createExpandRangesTransform(options = {}) {
 		}
 	});
 }
-
 //#endregion
 //#region src/transforms/normalize.ts
+const LEADING_ZEROES_RE = /^0+/;
 function normalizeHex(hex) {
-	return hex.toUpperCase().replace(/^0+/, "") || "0";
+	return hex.toUpperCase().replace(LEADING_ZEROES_RE, "") || "0";
 }
 function padHex(hex, length = 4) {
 	return hex.toUpperCase().padStart(length, "0");
@@ -132,7 +130,6 @@ function createNormalizeTransform(padLength = 4) {
 		}
 	});
 }
-
 //#endregion
 //#region src/transforms/sort.ts
 function hexToNumber(hex) {
@@ -169,6 +166,5 @@ function createSortTransform(options = {}) {
 		}
 	});
 }
-
 //#endregion
-export { createDeduplicateTransform, createExpandRangesTransform, createNormalizeTransform, createSortTransform, deduplicateRows, expandRanges, normalizeCodePoints, sortByCodePoint };
+export { createDeduplicateTransform, createExpandRangesTransform, createNormalizeTransform, createSortTransform, deduplicateRows, expandRanges, normalizeCodePoints, sortByCodePoint };

package/dist/{types-t7nns1Hx.d.mts → types-B_R0NiDI.d.mts}

@@ -1,4 +1,11 @@
 //#region src/types.d.ts
+type PipelineLogLevel = "debug" | "info" | "warn" | "error";
+interface PipelineLogger {
+  debug: (message: string, meta?: Record<string, unknown>) => void;
+  info: (message: string, meta?: Record<string, unknown>) => void;
+  warn: (message: string, meta?: Record<string, unknown>) => void;
+  error: (message: string, meta?: Record<string, unknown>) => void;
+}
 interface FileContext {
   /**
    * The Unicode version being processed (e.g., "16.0.0").
@@ -45,6 +52,10 @@ interface FilterContext {
      */
     id: string;
   };
+  /**
+   * Execution-aware logger for pipeline code.
+   */
+  logger: PipelineLogger;
 }
 type PipelineFilter = (ctx: FilterContext) => boolean;
 interface ParsedRow {
@@ -90,6 +101,10 @@ interface ParseContext {
    * The file being parsed.
    */
   file: FileContext;
+  /**
+   * Execution-aware logger for pipeline code.
+   */
+  logger: PipelineLogger;
   /**
    * Read the raw content of the file.
    */
@@ -158,45 +173,5 @@ interface PropertyJson {
    */
   meta?: Record<string, unknown>;
 }
-interface ResolveContext<TArtifacts extends Record<string, unknown> = Record<string, unknown>> {
-  /**
-   * The Unicode version being processed.
-   */
-  version: string;
-  /**
-   * The file being resolved.
-   */
-  file: FileContext;
-  /**
-   * Get an artifact by ID.
-   */
-  getArtifact: <K extends keyof TArtifacts>(id: K) => TArtifacts[K];
-  /**
-   * Emit an artifact for subsequent routes.
-   */
-  emitArtifact: <K extends string, V>(id: K, value: V) => void;
-  /**
-   * Normalize and sort entries by code point range.
-   */
-  normalizeEntries: (entries: ResolvedEntry[]) => ResolvedEntry[];
-  /**
-   * Get current timestamp in ISO 8601 format.
-   */
-  now: () => string;
-}
-type ResolverFn<TArtifacts extends Record<string, unknown> = Record<string, unknown>, TOutput = PropertyJson[]> = (ctx: ResolveContext<TArtifacts>, rows: AsyncIterable<ParsedRow>) => Promise<TOutput>;
-/**
- * Output configuration for a route.
- */
-interface RouteOutput {
-  /**
-   * Custom output directory.
-   */
-  dir?: string;
-  /**
-   * Custom file name generator.
-   */
-  fileName?: (pj: PropertyJson) => string;
-}
 //#endregion
-export { ParsedRow as a, PropertyJson as c, ResolverFn as d, RouteOutput as f, ParseContext as i, ResolveContext as l, FileContext as n, ParserFn as o, RowContext as p, FilterContext as r, PipelineFilter as s, DefaultRange as t, ResolvedEntry as u };
+export { ParsedRow as a, PipelineLogLevel as c, ResolvedEntry as d, RowContext as f, ParseContext as i, PipelineLogger as l, FileContext as n, ParserFn as o, FilterContext as r, PipelineFilter as s, DefaultRange as t, PropertyJson as u };

package/dist/types-BxE_sFho.mjs

@@ -0,0 +1,19 @@
+//#region src/outputs/types.ts
+function filesystemSink(options = {}) {
+	return {
+		type: "filesystem",
+		...options
+	};
+}
+function normalizeRouteOutputs(route) {
+	return (route.outputs?.length ? route.outputs : [{ id: "default" }]).map((output, index) => {
+		return {
+			id: output.id ?? `output-${index + 1}`,
+			sink: output.sink,
+			format: output.format ?? "json",
+			path: output.path
+		};
+	});
+}
+//#endregion
+export { normalizeRouteOutputs as n, filesystemSink as t };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ucdjs/pipelines-core",
-  "version": "0.0.1-beta.6",
+  "version": "0.0.1-beta.8",
   "type": "module",
   "author": {
     "name": "Lucas Nørgård",
@@ -17,9 +17,14 @@
   "bugs": {
     "url": "https://github.com/ucdjs/ucd/issues"
   },
+  "imports": {
+    "#builtin-transforms/*": "./src/transforms/*.ts"
+  },
   "exports": {
     ".": "./dist/index.mjs",
+    "./outputs": "./dist/outputs/index.mjs",
     "./sources": "./dist/sources/index.mjs",
+    "./tracing": "./dist/tracing/index.mjs",
     "./transforms": "./dist/transforms/index.mjs",
     "./package.json": "./package.json"
   },
@@ -31,20 +36,20 @@
     "node": ">=24.13"
   },
   "dependencies": {
-    "picomatch": "4.0.3",
+    "picomatch": "4.0.4",
     "zod": "4.3.6",
-    "@ucdjs-internal/shared": "0.1.1-beta.6"
+    "@ucdjs-internal/shared": "0.1.1-beta.8"
   },
   "devDependencies": {
-    "@luxass/eslint-config": "7.2.0",
+    "@luxass/eslint-config": "7.4.1",
     "@types/picomatch": "4.0.2",
-    "eslint": "10.0.0",
-    "publint": "0.3.17",
-    "tsdown": "0.20.3",
+    "eslint": "10.1.0",
+    "publint": "0.3.18",
+    "tsdown": "0.21.4",
     "tsx": "4.21.0",
-    "typescript": "5.9.3",
-    "@ucdjs-tooling/tsdown-config": "1.0.0",
-    "@ucdjs-tooling/tsconfig": "1.0.0"
+    "typescript": "6.0.2",
+    "@ucdjs-tooling/tsconfig": "1.0.0",
+    "@ucdjs-tooling/tsdown-config": "1.0.0"
   },
   "publishConfig": {
     "access": "public"

package/dist/source-EtWCktj_.mjs

@@ -1,26 +0,0 @@
-//#region src/source.ts
-function definePipelineSource(definition) {
-	return definition;
-}
-async function resolveSourceFiles(source, version) {
-	return (await source.backend.listFiles(version)).filter((file) => {
-		const ctx = { file };
-		if (source.includes && !source.includes(ctx)) return false;
-		if (source.excludes && source.excludes(ctx)) return false;
-		return true;
-	}).map((file) => ({
-		...file,
-		source: { id: source.id }
-	}));
-}
-async function resolveMultipleSourceFiles(sources, version) {
-	const filesByPath = /* @__PURE__ */ new Map();
-	for (const source of sources) {
-		const files = await resolveSourceFiles(source, version);
-		for (const file of files) filesByPath.set(file.path, file);
-	}
-	return Array.from(filesByPath.values());
-}
-
-//#endregion
-export { resolveMultipleSourceFiles as n, resolveSourceFiles as r, definePipelineSource as t };