@ucdjs/pipelines-core 0.0.1-beta.1

package/dist/transform-BSrhinRr.d.mts

@@ -0,0 +1,88 @@
+import { n as FileContext } from "./types-t7nns1Hx.mjs";
+
+//#region src/transform.d.ts
+interface TransformContext {
+  version: string;
+  file: FileContext;
+}
+type PipelineTransformFunction<TInput, TOutput> = (ctx: TransformContext, rows: AsyncIterable<TInput>) => AsyncIterable<TOutput>;
+interface PipelineTransformDefinition<TInput = unknown, TOutput = unknown> {
+  id: string;
+  fn: PipelineTransformFunction<TInput, TOutput>;
+}
+type AnyPipelineTransformDefinition = PipelineTransformDefinition<any, any>;
+/**
+ * Creates and returns a validated pipeline transform definition.
+ *
+ * This function is primarily a type-safe way to define transforms. It ensures that
+ * the transform adheres to the `PipelineTransformDefinition` interface and preserves
+ * type information for chaining.
+ *
+ * @typeParam TInput - The input type accepted by the transform.
+ * @typeParam TOutput - The output type produced by the transform.
+ * @param {PipelineTransformDefinition<TInput, TOutput>} definition - The transform definition.
+ * @returns {PipelineTransformDefinition<TInput, TOutput>} The same definition, typed correctly.
+ *
+ * @example
+ * ```ts
+ * const uppercase = definePipelineTransform({
+ *   id: 'uppercase',
+ *   fn: async function* (_ctx, rows) {
+ *     for await (const row of rows) {
+ *       yield row.toUpperCase();
+ *     }
+ *   },
+ * });
+ * ```
+ */
+declare function definePipelineTransform<TInput, TOutput>(definition: PipelineTransformDefinition<TInput, TOutput>): PipelineTransformDefinition<TInput, TOutput>;
+type InferTransformInput<T> = T extends PipelineTransformDefinition<infer TInput, unknown> ? TInput : never;
+type InferTransformOutput<T> = T extends PipelineTransformDefinition<unknown, infer TOutput> ? TOutput : never;
+/**
+ * Recursively composes a sequence of transform definitions into a single output type.
+ *
+ * Given an input type and a sequence of transforms, this type infers the final output
+ * type by threading the output of each transform into the input of the next.
+ *
+ * @typeParam TInput - The initial input type.
+ * @typeParam TTransforms - A readonly tuple of transform definitions, starting with one
+ *                          that accepts `TInput`.
+ *
+ * @example
+ * ```ts
+ * type T1 = PipelineTransformDefinition<string, number>;
+ * type T2 = PipelineTransformDefinition<number, boolean>;
+ * type Result = ChainTransforms<string, [T1, T2]>; // boolean
+ * ```
+ */
+type ChainTransformsHelper<TInput, TTransforms extends readonly PipelineTransformDefinition<any, any>[]> = TTransforms extends readonly [PipelineTransformDefinition<TInput, infer Output>, ...infer Rest] ? ChainTransformsHelper<Output, Extract<Rest, readonly PipelineTransformDefinition<any, any>[]>> : TInput;
+type ChainTransforms<TInput, TTransforms extends readonly PipelineTransformDefinition<any, any>[]> = ChainTransformsHelper<TInput, TTransforms>;
+/**
+ * Applies a sequence of transforms to an async iterable, composing them together.
+ *
+ * This function threads the output of one transform into the input of the next,
+ * creating a pipeline. All iteration is lazy—values are pulled through the pipeline
+ * only as they are consumed.
+ *
+ * @typeParam TInput - The input type of the first transform.
+ * @param {TransformContext} ctx - The context to pass to each transform.
+ * @param {AsyncIterable<TInput>} rows - The initial data source.
+ * @param {readonly PipelineTransformDefinition<any, any>[]} transforms - The transforms to apply in order.
+ * @returns {AsyncIterable<unknown>} An async iterable of the final output (typed as `unknown` since
+ *                                    the result type depends on the transform sequence).
+ *
+ * @remarks
+ * The output type can be narrowed using the `ChainTransforms` utility type if the
+ * exact sequence of transforms is known at compile time.
+ *
+ * @example
+ * ```ts
+ * const output = applyTransforms(ctx, sourceRows, [
+ *   definePipelineTransform({ id: 'filter', fn: filterFn }),
+ *   definePipelineTransform({ id: 'map', fn: mapFn }),
+ * ]);
+ * ```
+ */
+declare function applyTransforms<TInput>(ctx: TransformContext, rows: AsyncIterable<TInput>, transforms: readonly PipelineTransformDefinition<any, any>[]): AsyncIterable<unknown>;
+//#endregion
+export { PipelineTransformDefinition as a, definePipelineTransform as c, InferTransformOutput as i, ChainTransforms as n, TransformContext as o, InferTransformInput as r, applyTransforms as s, AnyPipelineTransformDefinition as t };

package/dist/transforms/index.d.mts

@@ -0,0 +1,33 @@
+import { a as ParsedRow } from "../types-t7nns1Hx.mjs";
+import { a as PipelineTransformDefinition } from "../transform-BSrhinRr.mjs";
+
+//#region src/transforms/deduplicate.d.ts
+declare const deduplicateRows: PipelineTransformDefinition<ParsedRow, ParsedRow>;
+type DeduplicateStrategy = "first" | "last" | "merge";
+interface DeduplicateOptions {
+  strategy?: DeduplicateStrategy;
+  keyFn?: (row: ParsedRow) => string;
+}
+declare function createDeduplicateTransform(options?: DeduplicateOptions): PipelineTransformDefinition<ParsedRow, ParsedRow>;
+//#endregion
+//#region src/transforms/expand-ranges.d.ts
+declare const expandRanges: PipelineTransformDefinition<ParsedRow, ParsedRow>;
+interface ExpandRangesOptions {
+  maxExpansion?: number;
+}
+declare function createExpandRangesTransform(options?: ExpandRangesOptions): PipelineTransformDefinition<ParsedRow, ParsedRow>;
+//#endregion
+//#region src/transforms/normalize.d.ts
+declare const normalizeCodePoints: PipelineTransformDefinition<ParsedRow, ParsedRow>;
+declare function createNormalizeTransform(padLength?: number): PipelineTransformDefinition<ParsedRow, ParsedRow>;
+//#endregion
+//#region src/transforms/sort.d.ts
+declare const sortByCodePoint: PipelineTransformDefinition<ParsedRow, ParsedRow>;
+type SortDirection = "asc" | "desc";
+interface SortOptions {
+  direction?: SortDirection;
+  keyFn?: (row: ParsedRow) => number;
+}
+declare function createSortTransform(options?: SortOptions): PipelineTransformDefinition<ParsedRow, ParsedRow>;
+//#endregion
+export { type DeduplicateOptions, type DeduplicateStrategy, type ExpandRangesOptions, type SortDirection, type SortOptions, createDeduplicateTransform, createExpandRangesTransform, createNormalizeTransform, createSortTransform, deduplicateRows, expandRanges, normalizeCodePoints, sortByCodePoint };

package/dist/transforms/index.mjs

@@ -0,0 +1,174 @@
+import { n as definePipelineTransform } from "../transform-BDsUG3EV.mjs";
+
+//#region src/transforms/deduplicate.ts
+function getRowKey(row) {
+	if (row.kind === "point" && row.codePoint) return `point:${row.codePoint}`;
+	if (row.kind === "range" && row.start && row.end) return `range:${row.start}..${row.end}`;
+	if (row.kind === "sequence" && row.sequence) return `seq:${row.sequence.join("-")}`;
+	return `unknown:${JSON.stringify(row)}`;
+}
+const deduplicateRows = definePipelineTransform({
+	id: "deduplicate-rows",
+	async *fn(_ctx, rows) {
+		const seen = /* @__PURE__ */ new Set();
+		for await (const row of rows) {
+			const key = getRowKey(row);
+			if (!seen.has(key)) {
+				seen.add(key);
+				yield row;
+			}
+		}
+	}
+});
+function createDeduplicateTransform(options = {}) {
+	const { strategy = "first", keyFn = getRowKey } = options;
+	if (strategy === "last") return definePipelineTransform({
+		id: "deduplicate-rows-last",
+		async *fn(_ctx, rows) {
+			const byKey = /* @__PURE__ */ new Map();
+			for await (const row of rows) {
+				const key = keyFn(row);
+				byKey.set(key, row);
+			}
+			yield* byKey.values();
+		}
+	});
+	return definePipelineTransform({
+		id: "deduplicate-rows-first",
+		async *fn(_ctx, rows) {
+			const seen = /* @__PURE__ */ new Set();
+			for await (const row of rows) {
+				const key = keyFn(row);
+				if (!seen.has(key)) {
+					seen.add(key);
+					yield row;
+				}
+			}
+		}
+	});
+}
+
+//#endregion
+//#region src/transforms/expand-ranges.ts
+function hexToNumber$1(hex) {
+	return Number.parseInt(hex, 16);
+}
+function numberToHex(num) {
+	return num.toString(16).toUpperCase().padStart(4, "0");
+}
+const expandRanges = definePipelineTransform({
+	id: "expand-ranges",
+	async *fn(_ctx, rows) {
+		for await (const row of rows) if (row.kind === "range" && row.start && row.end) {
+			const start = hexToNumber$1(row.start);
+			const end = hexToNumber$1(row.end);
+			for (let i = start; i <= end; i++) yield {
+				...row,
+				kind: "point",
+				codePoint: numberToHex(i),
+				start: void 0,
+				end: void 0
+			};
+		} else yield row;
+	}
+});
+function createExpandRangesTransform(options = {}) {
+	const { maxExpansion = 1e4 } = options;
+	return definePipelineTransform({
+		id: "expand-ranges-limited",
+		async *fn(_ctx, rows) {
+			for await (const row of rows) if (row.kind === "range" && row.start && row.end) {
+				const start = hexToNumber$1(row.start);
+				const end = hexToNumber$1(row.end);
+				if (end - start + 1 > maxExpansion) {
+					yield row;
+					continue;
+				}
+				for (let i = start; i <= end; i++) yield {
+					...row,
+					kind: "point",
+					codePoint: numberToHex(i),
+					start: void 0,
+					end: void 0
+				};
+			} else yield row;
+		}
+	});
+}
+
+//#endregion
+//#region src/transforms/normalize.ts
+function normalizeHex(hex) {
+	return hex.toUpperCase().replace(/^0+/, "") || "0";
+}
+function padHex(hex, length = 4) {
+	return hex.toUpperCase().padStart(length, "0");
+}
+const normalizeCodePoints = definePipelineTransform({
+	id: "normalize-code-points",
+	async *fn(_ctx, rows) {
+		for await (const row of rows) {
+			const normalized = { ...row };
+			if (normalized.codePoint) normalized.codePoint = padHex(normalizeHex(normalized.codePoint));
+			if (normalized.start) normalized.start = padHex(normalizeHex(normalized.start));
+			if (normalized.end) normalized.end = padHex(normalizeHex(normalized.end));
+			if (normalized.sequence) normalized.sequence = normalized.sequence.map((cp) => padHex(normalizeHex(cp)));
+			yield normalized;
+		}
+	}
+});
+function createNormalizeTransform(padLength = 4) {
+	return definePipelineTransform({
+		id: `normalize-code-points-${padLength}`,
+		async *fn(_ctx, rows) {
+			for await (const row of rows) {
+				const normalized = { ...row };
+				if (normalized.codePoint) normalized.codePoint = padHex(normalizeHex(normalized.codePoint), padLength);
+				if (normalized.start) normalized.start = padHex(normalizeHex(normalized.start), padLength);
+				if (normalized.end) normalized.end = padHex(normalizeHex(normalized.end), padLength);
+				if (normalized.sequence) normalized.sequence = normalized.sequence.map((cp) => padHex(normalizeHex(cp), padLength));
+				yield normalized;
+			}
+		}
+	});
+}
+
+//#endregion
+//#region src/transforms/sort.ts
+function hexToNumber(hex) {
+	return Number.parseInt(hex, 16);
+}
+function getRowSortKey(row) {
+	if (row.codePoint) return hexToNumber(row.codePoint);
+	if (row.start) return hexToNumber(row.start);
+	if (row.sequence && row.sequence.length > 0) {
+		const first = row.sequence[0];
+		if (first) return hexToNumber(first);
+	}
+	return 0;
+}
+const sortByCodePoint = definePipelineTransform({
+	id: "sort-by-code-point",
+	async *fn(_ctx, rows) {
+		const collected = [];
+		for await (const row of rows) collected.push(row);
+		collected.sort((a, b) => getRowSortKey(a) - getRowSortKey(b));
+		yield* collected;
+	}
+});
+function createSortTransform(options = {}) {
+	const { direction = "asc", keyFn = getRowSortKey } = options;
+	const multiplier = direction === "asc" ? 1 : -1;
+	return definePipelineTransform({
+		id: `sort-${direction}`,
+		async *fn(_ctx, rows) {
+			const collected = [];
+			for await (const row of rows) collected.push(row);
+			collected.sort((a, b) => multiplier * (keyFn(a) - keyFn(b)));
+			yield* collected;
+		}
+	});
+}
+
+//#endregion
+export { createDeduplicateTransform, createExpandRangesTransform, createNormalizeTransform, createSortTransform, deduplicateRows, expandRanges, normalizeCodePoints, sortByCodePoint };

package/dist/types-t7nns1Hx.d.mts

@@ -0,0 +1,202 @@
+//#region src/types.d.ts
+interface FileContext {
+  /**
+   * The Unicode version being processed (e.g., "16.0.0").
+   */
+  version: string;
+  /**
+   * The directory category of the file.
+   */
+  dir: "ucd" | "extracted" | "auxiliary" | "emoji" | "unihan" | (string & {});
+  /**
+   * The relative path from the version root (e.g., "ucd/LineBreak.txt").
+   */
+  path: string;
+  /**
+   * The file name (e.g., "LineBreak.txt").
+   */
+  name: string;
+  /**
+   * The file extension (e.g., ".txt").
+   */
+  ext: string;
+}
+interface RowContext {
+  /**
+   * The property name for multi-property files (e.g., "NFKC_Casefold").
+   */
+  property?: string;
+}
+interface FilterContext {
+  /**
+   * The file context.
+   */
+  file: FileContext;
+  /**
+   * The row context (only defined during row-level filtering).
+   */
+  row?: RowContext;
+  /**
+   * The source context (only defined when using multiple sources).
+   */
+  source?: {
+    /**
+     * The source ID.
+     */
+    id: string;
+  };
+}
+type PipelineFilter = (ctx: FilterContext) => boolean;
+interface ParsedRow {
+  /**
+   * The source file path relative to the version root.
+   */
+  sourceFile: string;
+  /**
+   * The kind of entry.
+   */
+  kind: "range" | "point" | "sequence" | "alias";
+  /**
+   * Start of range (hex string, e.g., "0041").
+   */
+  start?: string;
+  /**
+   * End of range (hex string, e.g., "005A").
+   */
+  end?: string;
+  /**
+   * Single code point (hex string).
+   */
+  codePoint?: string;
+  /**
+   * Sequence of code points (hex strings).
+   */
+  sequence?: string[];
+  /**
+   * Property name for multi-property files.
+   */
+  property?: string;
+  /**
+   * The value(s) associated with this entry.
+   */
+  value?: string | string[];
+  /**
+   * Additional metadata (comments, line numbers, etc.).
+   */
+  meta?: Record<string, unknown>;
+}
+interface ParseContext {
+  /**
+   * The file being parsed.
+   */
+  file: FileContext;
+  /**
+   * Read the raw content of the file.
+   */
+  readContent: () => Promise<string>;
+  /**
+   * Read the file line by line.
+   */
+  readLines: () => AsyncIterable<string>;
+  /**
+   * Check if a line is a comment.
+   */
+  isComment: (line: string) => boolean;
+}
+type ParserFn = (ctx: ParseContext) => AsyncIterable<ParsedRow>;
+interface ResolvedEntry {
+  /**
+   * Range in "XXXX..YYYY" format (hex, inclusive).
+   */
+  range?: `${string}..${string}`;
+  /**
+   * Single code point in hex.
+   */
+  codePoint?: string;
+  /**
+   * Sequence of code points.
+   */
+  sequence?: string[];
+  /**
+   * The value(s) for this entry.
+   */
+  value: string | string[];
+}
+interface DefaultRange {
+  /**
+   * The range this default applies to.
+   */
+  range: `${string}..${string}`;
+  /**
+   * The default value.
+   */
+  value: string | string[];
+}
+interface PropertyJson {
+  /**
+   * The Unicode version (e.g., "16.0.0").
+   */
+  version: string;
+  /**
+   * The property name (e.g., "Line_Break").
+   */
+  property: string;
+  /**
+   * The source file name (e.g., "LineBreak.txt").
+   */
+  file: string;
+  /**
+   * The resolved entries.
+   */
+  entries: ResolvedEntry[];
+  /**
+   * Default ranges from @missing (in encounter order).
+   */
+  defaults?: DefaultRange[];
+  /**
+   * Additional metadata.
+   */
+  meta?: Record<string, unknown>;
+}
+interface ResolveContext<TArtifacts extends Record<string, unknown> = Record<string, unknown>> {
+  /**
+   * The Unicode version being processed.
+   */
+  version: string;
+  /**
+   * The file being resolved.
+   */
+  file: FileContext;
+  /**
+   * Get an artifact by ID.
+   */
+  getArtifact: <K extends keyof TArtifacts>(id: K) => TArtifacts[K];
+  /**
+   * Emit an artifact for subsequent routes.
+   */
+  emitArtifact: <K extends string, V>(id: K, value: V) => void;
+  /**
+   * Normalize and sort entries by code point range.
+   */
+  normalizeEntries: (entries: ResolvedEntry[]) => ResolvedEntry[];
+  /**
+   * Get current timestamp in ISO 8601 format.
+   */
+  now: () => string;
+}
+type ResolverFn<TArtifacts extends Record<string, unknown> = Record<string, unknown>, TOutput = PropertyJson[]> = (ctx: ResolveContext<TArtifacts>, rows: AsyncIterable<ParsedRow>) => Promise<TOutput>;
+/**
+ * Output configuration for a route.
+ */
+interface RouteOutput {
+  /**
+   * Custom output directory.
+   */
+  dir?: string;
+  /**
+   * Custom file name generator.
+   */
+  fileName?: (pj: PropertyJson) => string;
+}
+//#endregion
+export { ParsedRow as a, PropertyJson as c, ResolverFn as d, RouteOutput as f, ParseContext as i, ResolveContext as l, FileContext as n, ParserFn as o, RowContext as p, FilterContext as r, PipelineFilter as s, DefaultRange as t, ResolvedEntry as u };

package/package.json

@@ -0,0 +1,59 @@
+{
+  "name": "@ucdjs/pipelines-core",
+  "version": "0.0.1-beta.1",
+  "type": "module",
+  "author": {
+    "name": "Lucas Nørgård",
+    "email": "lucasnrgaard@gmail.com",
+    "url": "https://luxass.dev"
+  },
+  "license": "MIT",
+  "homepage": "https://github.com/ucdjs/ucd",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/ucdjs/ucd.git",
+    "directory": "packages/pipelines/pipeline-core"
+  },
+  "bugs": {
+    "url": "https://github.com/ucdjs/ucd/issues"
+  },
+  "exports": {
+    ".": "./dist/index.mjs",
+    "./sources": "./dist/sources/index.mjs",
+    "./transforms": "./dist/transforms/index.mjs",
+    "./package.json": "./package.json"
+  },
+  "types": "./dist/index.d.mts",
+  "files": [
+    "dist"
+  ],
+  "engines": {
+    "node": ">=22.18"
+  },
+  "dependencies": {
+    "picomatch": "4.0.3",
+    "zod": "4.3.6",
+    "@ucdjs-internal/shared": "0.1.1-beta.1"
+  },
+  "devDependencies": {
+    "@luxass/eslint-config": "7.2.0",
+    "@types/picomatch": "4.0.2",
+    "eslint": "10.0.0",
+    "publint": "0.3.17",
+    "tsdown": "0.20.3",
+    "tsx": "4.21.0",
+    "typescript": "5.9.3",
+    "@ucdjs-tooling/tsdown-config": "1.0.0",
+    "@ucdjs-tooling/tsconfig": "1.0.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsdown --tsconfig=./tsconfig.build.json",
+    "dev": "tsdown --watch",
+    "clean": "git clean -xdf dist node_modules",
+    "lint": "eslint .",
+    "typecheck": "tsc --noEmit -p tsconfig.build.json"
+  }
+}