@ucdjs/codegen 0.0.0 → 0.0.1

package/dist/fields.d.mts

@@ -0,0 +1,2 @@
+import { a as generateFields, i as GenerateFieldsOptions, n as ProcessedFieldsFile, r as runFieldsCodegen, t as FieldsCodegenOptions } from "./run-Cgm8iCzh.mjs";
+export { FieldsCodegenOptions, GenerateFieldsOptions, ProcessedFieldsFile, generateFields, runFieldsCodegen };

package/dist/fields.mjs

@@ -0,0 +1,2 @@
+import { n as generateFields, t as runFieldsCodegen } from "./run-bH3vZ-ci.mjs";
+export { generateFields, runFieldsCodegen };

package/dist/index.d.mts

@@ -1,48 +1,2 @@
-import { LanguageModel } from "ai";
-
-//#region src/index.d.ts
-interface CodegenFile {
-  /**
-   * The filePath to the data file.
-   */
-  filePath: string;
-  /**
-   * The version of the data file.
-   */
-  version: string;
-}
-interface ProcessedFile {
-  fields: {
-    type: string;
-    name: string;
-    description: string;
-  }[];
-  code: string;
-  fileName: string;
-  version: string;
-}
-interface CodegenOptions {
-  /**
-   * Files to generate structures for.
-   */
-  files: CodegenFile[];
-  /**
-   * The OpenAI API key to use for generating the fields.
-   */
-  openaiKey?: string;
-  /**
-   * The OpenAI model to use for generating fields.
-   * NOTE:
-   * This is good for testing purposes, where you
-   * can provide a mock model to test the generation.
-   *
-   * If not provided, it will create a new OpenAI instance
-   * with the default model.
-   *
-   * SEE: https://ai-sdk.dev/docs/ai-sdk-core/testing
-   */
-  model?: LanguageModel;
-}
-declare function runCodegen(options: CodegenOptions): Promise<ProcessedFile[]>;
-//#endregion
-export { CodegenFile, CodegenOptions, ProcessedFile, runCodegen };
+import { a as generateFields, c as processFile, i as GenerateFieldsOptions, n as ProcessedFieldsFile, o as CodegenFile, r as runFieldsCodegen, s as ProcessDataFile, t as FieldsCodegenOptions } from "./run-Cgm8iCzh.mjs";
+export { type CodegenFile, type FieldsCodegenOptions, type GenerateFieldsOptions, type ProcessDataFile, type ProcessedFieldsFile, generateFields, processFile, runFieldsCodegen };

package/dist/index.mjs

@@ -1,243 +1,2 @@
-import { readFile } from "node:fs/promises";
-import path from "node:path";
-import { dedent, sanitizeIdentifier, toPascalCase, toSnakeCase } from "@luxass/utils";
-import { createConcurrencyLimiter } from "@ucdjs-internal/shared";
-import { RawDataFile } from "@unicode-utils/core";
-import { createOpenAI } from "@ai-sdk/openai";
-import { generateObject } from "ai";
-import { z } from "zod";
-//#region src/fields.ts
-const SYSTEM_PROMPT = dedent`
-<system_prompt>
-  <role>Expert TypeScript field extractor for Unicode data files</role>
-
-  <task>
-    <input>
-      Text description: {{INPUT}}
-    </input>
-    <output>JSON array of field objects with name, type, and description</output>
-  </task>
-
-  <critical_rule>
-    YOU MUST ALWAYS OUTPUT A VALID JSON ARRAY OF OBJECTS WITH THIS STRUCTURE:
-    [
-      {
-        "name": "actual_field_name",
-        "type": "valid_typescript_type",
-        "description": "Description of the field"
-      }
-    ]
-
-    NEVER use generic names like "field_0", "field_1", etc. - extract the ACTUAL field names.
-    NEVER use invalid TypeScript types like "union" - use proper union syntax with pipe symbol.
-  </critical_rule>
-
-  <field_detection>
-    For Unicode data files, fields are typically described in patterns like:
-    - Lines starting with "# Field 0: Name" - where "Name" is the actual field name
-    - Table headers or column definitions
-    - Property descriptions in documentation
-
-    DO NOT use "field_0", "field_1" as field names - extract the REAL field names.
-    DO NOT use "type_name" as a field name unless it's explicitly mentioned as a field.
-
-    Example: "# Field 0: Code_Point" should produce a field named "code_point" (NOT "field_0").
-  </field_detection>
-
-  <type_mapping>
-    ONLY USE THESE VALID TYPESCRIPT TYPES:
-    - string - For text, identifiers, character codes, etc.
-    - number - For numeric values, indices, counts
-    - boolean - For true/false flags
-    - string[] - For arrays of strings
-    - number[] - For arrays of numbers
-    - Array<string> - Alternative syntax for string arrays
-    - Array<number> - Alternative syntax for number arrays
-    - Record<string, string> - For string to string mappings
-    - Record<string, number> - For string to number mappings
-    - Record<string, unknown> - For objects with unknown structure
-    - unknown - When type cannot be determined
-    - any - Only as a last resort when type is truly variable
-
-    For union types with string literals, ALWAYS use double quotes and pipe symbol:
-    - "\"value1\" | \"value2\" | \"value3\""
-
-    CRITICAL: Special values handling:
-    1. If a value contains angle brackets, like <none>, FIRST remove the angle brackets: "none" (NOT "<none>")
-    2. THEN ALWAYS wrap the value in quotes if it's a string literal: "\"none\""
-    3. For union types of string literals, EACH value MUST be in quotes:
-       - Correct: "\"R\" | \"L\" | \"D\" | \"C\" | \"U\" | \"T\""
-       - Incorrect: "R | L | D | C | U | T" (missing quotes around each value)
-       - Incorrect: "none" (missing quotes if it's a string literal)
-
-    NEVER use these invalid types:
-    - "union" - This is not a valid TypeScript type
-    - "object" - Too generic, use Record<> instead
-    - "array" - Too generic, use proper array syntax instead
-    - "map" - Use Record<> instead
-    - "none" - Use "\"none\"" or never instead
-    - "list" - Use proper array syntax instead
-
-    ALWAYS wrap the final converted type in double quotes in the JSON output.
-    ALWAYS wrap each value in a union type of string literals in quotes.
-  </type_mapping>
-
-  <examples>
-    Input example:
-    \`\`\`
-    # ArabicShaping.txt
-    # Field 0: Code point
-    # Field 1: Name
-    # Field 2: Joining_Type (R = Right_Joining, L = Left_Joining, D = Dual_Joining, C = Join_Causing, U = Non_Joining, T = Transparent)
-    # Field 3: Joining_Group
-    \`\`\`
-
-    Correct output:
-    [
-      {
-        "name": "code_point",
-        "type": "string",
-        "description": "The code point of a character, in hexadecimal form"
-      },
-      {
-        "name": "name",
-        "type": "string",
-        "description": "A short schematic name for the character"
-      },
-      {
-        "name": "joining_type",
-        "type": "\"R\" | \"L\" | \"D\" | \"C\" | \"U\" | \"T\"",
-        "description": "Defines the joining type (R = Right_Joining, L = Left_Joining, D = Dual_Joining, C = Join_Causing, U = Non_Joining, T = Transparent)"
-      },
-      {
-        "name": "joining_group",
-        "type": "string",
-        "description": "Defines the joining group, based schematically on character names"
-      }
-    ]
-
-    Example with special value:
-    Input: "# Field 4: Value (<none> or specific value)"
-
-    Correct output for this field:
-    {
-      "name": "value",
-      "type": "\"none\" | string",
-      "description": "The value, which can be none or a specific value"
-    }
-  </examples>
-
-  <validation>
-    Before outputting, verify:
-    - Field names are the ACTUAL field names from the documentation, NOT generic "field_0" style names
-    - All field names are in snake_case
-    - All types are valid TypeScript types (string, number, boolean, arrays, or proper union types)
-    - NEVER use the word "union" as a type - use proper TypeScript syntax with the pipe symbol
-    - String literal values in union types are ALWAYS wrapped in quotes (e.g., "\"value1\" | \"value2\"")
-    - Special values like "none" (from <none>) are properly quoted as string literals: "\"none\""
-    - Each field has a clear, specific description
-    - Output is a valid JSON array of objects
-  </validation>
-
-  <error_handling>
-    If NO fields can be detected in the input:
-    - Return an empty JSON array: []
-    - DO NOT return error messages or explanations in the JSON output
-    - DO NOT attempt to create fields when none are clearly defined
-
-    Example when no fields are detected:
-    Input: "This is some text without any field definitions"
-
-    Correct output:
-    []
-
-    If SOME fields are unclear but others are detectable:
-    - Only include the fields that can be clearly identified
-    - Omit any fields that cannot be confidently extracted
-    - Follow all validation rules for the fields that are included
-  </error_handling>
-
-  <format>JSON array of field objects</format>
-</system_prompt>
-`;
-async function generateFields(options) {
-	const { datafile, apiKey, model } = options;
-	if (!apiKey && !model) return null;
-	const openai = model != null ? null : createOpenAI({ apiKey });
-	try {
-		return (await generateObject({
-			model: model ?? openai("gpt-4o-mini"),
-			schema: z.object({ fields: z.array(z.object({
-				name: z.string(),
-				type: z.string(),
-				description: z.string()
-			})) }),
-			prompt: SYSTEM_PROMPT.replace("{{INPUT}}", datafile.heading)
-		})).object.fields;
-	} catch (err) {
-		console.error("error generating fields:", err);
-		return null;
-	}
-}
-//#endregion
-//#region src/index.ts
-const TXT_EXTENSION_RE = /\.txt$/;
-function buildInterface(name, fields, opts) {
-	return `${opts?.export ? "export " : ""}interface ${name} {\n${Object.entries(fields).map(([key, type]) => `  ${key}: ${type};`).join("\n")}\n}`;
-}
-function buildStringArray(values) {
-	return `[${values.map((v) => `"${v}"`).join(", ")}]`;
-}
-async function runCodegen(options) {
-	const inputFiles = options.files;
-	const limit = createConcurrencyLimiter(10);
-	if (!options.openaiKey && !options.model) throw new Error("Either openaiKey or model must be provided");
-	const processPromises = inputFiles.map(({ filePath, version }) => limit(() => processFile({
-		filePath,
-		openaiKey: options.openaiKey,
-		version,
-		model: options.model
-	})));
-	return Promise.all(processPromises).then((results) => results.filter((result) => result !== null));
-}
-async function processFile(request) {
-	const { filePath, openaiKey, version, model } = request;
-	try {
-		console.log(`Processing file: ${filePath}`);
-		const content = await readFile(filePath, "utf-8");
-		const fileName = path.basename(filePath).replace(TXT_EXTENSION_RE, "");
-		const fields = await generateFields({
-			datafile: new RawDataFile(content),
-			apiKey: openaiKey,
-			model
-		});
-		if (fields == null) {
-			console.error(`Error generating fields for file: ${filePath}`);
-			return null;
-		}
-		let code = ``;
-		const properties = {};
-		for (const field of fields) {
-			if (properties[field.name] != null) {
-				console.error(`Duplicate field name ${field.name} in file ${filePath}. Skipping field.`);
-				continue;
-			}
-			properties[field.name] = field.type;
-		}
-		code += `// This file is generated by ucd codegen. Do not edit this file directly.\n`;
-		code += `// Unicode Version: ${version}\n\n`;
-		code += `${buildInterface(sanitizeIdentifier(toPascalCase(fileName)), properties, { export: true })}\n\n`;
-		code += `export const ${sanitizeIdentifier(toSnakeCase(fileName)).toUpperCase()}_FIELDS = ${buildStringArray(fields.map((f) => f.name))};\n`;
-		return {
-			fields,
-			fileName,
-			version,
-			code
-		};
-	} catch (err) {
-		console.error(`Error processing file ${filePath}:`, err);
-		return null;
-	}
-}
-//#endregion
-export { runCodegen };
+import { n as generateFields, r as processFile, t as runFieldsCodegen } from "./run-bH3vZ-ci.mjs";
+export { generateFields, processFile, runFieldsCodegen };

package/dist/run-Cgm8iCzh.d.mts

@@ -0,0 +1,98 @@
+import { RawDataFile } from "@unicode-utils/core";
+import { LanguageModel } from "ai";
+
+//#region src/process.d.ts
+type CodegenFile = {
+  /**
+   * The file path to the UCD data file on disk.
+   */
+  filePath: string;
+  /**
+   * The Unicode version this file belongs to.
+   */
+  version: string;
+} | {
+  /**
+   * Pre-loaded file content. When provided, no disk read is performed.
+   */
+  content: string;
+  /**
+   * The file name (without extension) used to derive the interface and constant names.
+   */
+  fileName: string;
+  /**
+   * The Unicode version this file belongs to.
+   */
+  version: string;
+};
+type ProcessDataFile<T> = (datafile: RawDataFile, fileName: string, version: string) => Promise<T | null>;
+/**
+ * Reads a UCD data file from disk, parses it into a {@link RawDataFile}, and
+ * passes the result to the provided `processor` callback.
+ *
+ * This is the shared file-loading primitive used by all codegen runners.
+ * Implement a custom `processor` to add new kinds of code generation without
+ * duplicating the I/O and error-handling logic.
+ *
+ * @example
+ * ```ts
+ * const result = await processFile("./data/ArabicShaping.txt", "16.0",
+ *   async (datafile, fileName, version) => { ... }
+ * );
+ * ```
+ */
+declare function processFile<T>(filePath: string, version: string, processor: ProcessDataFile<T>): Promise<T | null>;
+//#endregion
+//#region src/fields/generate.d.ts
+interface GenerateFieldsOptions {
+  /**
+   * The parsed UCD data file to extract fields from.
+   */
+  datafile: RawDataFile;
+  /**
+   * The language model to use for field extraction.
+   */
+  model: LanguageModel;
+}
+declare function generateFields(options: GenerateFieldsOptions): Promise<{
+  name: string;
+  type: string;
+  description: string;
+}[] | null>;
+//#endregion
+//#region src/fields/run.d.ts
+interface ProcessedFieldsFile {
+  fields: {
+    type: string;
+    name: string;
+    description: string;
+  }[];
+  code: string;
+  fileName: string;
+  version: string;
+}
+interface FieldsCodegenOptions {
+  /**
+   * Files to generate field definitions for.
+   */
+  files: CodegenFile[];
+  /**
+   * OpenAI API key. Required when no custom `model` is provided.
+   */
+  openaiKey?: string;
+  /**
+   * OpenAI model ID to use when `openaiKey` is provided.
+   * @default "gpt-4o-mini"
+   */
+  modelId?: string;
+  /**
+   * A custom language model. Useful for testing or non-OpenAI providers.
+   * When provided, `openaiKey` and `modelId` are ignored.
+   *
+   * @see https://ai-sdk.dev/docs/ai-sdk-core/testing
+   */
+  model?: LanguageModel;
+}
+declare function runFieldsCodegen(options: FieldsCodegenOptions): Promise<ProcessedFieldsFile[]>;
+//#endregion
+export { generateFields as a, processFile as c, GenerateFieldsOptions as i, ProcessedFieldsFile as n, CodegenFile as o, runFieldsCodegen as r, ProcessDataFile as s, FieldsCodegenOptions as t };

package/dist/run-bH3vZ-ci.mjs

@@ -0,0 +1,142 @@
+import { createOpenAI } from "@ai-sdk/openai";
+import { sanitizeIdentifier, toPascalCase, toSnakeCase } from "@luxass/utils";
+import { createConcurrencyLimiter } from "@ucdjs-internal/shared";
+import { RawDataFile } from "@unicode-utils/core";
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { Output, generateText } from "ai";
+import { z } from "zod";
+//#region src/knitwork.ts
+function buildInterface(name, fields, opts) {
+	return `${opts?.export ? "export " : ""}interface ${name} {\n${Object.entries(fields).map(([key, type]) => `  ${key}: ${type};`).join("\n")}\n}`;
+}
+function buildStringArray(values) {
+	if (values.length === 0) return "[]";
+	return `[${values.map((v) => `"${v}"`).join(", ")}]`;
+}
+//#endregion
+//#region src/process.ts
+const TXT_EXTENSION_RE = /\.txt$/;
+/**
+* Reads a UCD data file from disk, parses it into a {@link RawDataFile}, and
+* passes the result to the provided `processor` callback.
+*
+* This is the shared file-loading primitive used by all codegen runners.
+* Implement a custom `processor` to add new kinds of code generation without
+* duplicating the I/O and error-handling logic.
+*
+* @example
+* ```ts
+* const result = await processFile("./data/ArabicShaping.txt", "16.0",
+*   async (datafile, fileName, version) => { ... }
+* );
+* ```
+*/
+async function processFile(filePath, version, processor) {
+	try {
+		console.log(`Processing file: ${filePath}`);
+		const content = await readFile(filePath, "utf-8");
+		const fileName = path.basename(filePath).replace(TXT_EXTENSION_RE, "");
+		return await processor(new RawDataFile(content), fileName, version);
+	} catch (err) {
+		console.error(`Error processing file ${filePath}:`, err);
+		return null;
+	}
+}
+//#endregion
+//#region src/fields/generate.ts
+const SYSTEM_PROMPT = `
+Extract TypeScript field definitions from a Unicode data file header.
+
+## Field naming
+- Use snake_case for all field names.
+- Extract the REAL name from patterns like "# Field 0: Code_Point" → "code_point". Never use "field_0", "field_1", etc.
+
+## Types
+Valid TypeScript only: string, number, boolean, string[], number[], Array<string>, Array<number>, Record<string, string>, Record<string, number>, Record<string, unknown>, unknown.
+- String literal unions: each value quoted with pipe — "\"R\" | \"L\" | \"D\""
+- Angle-bracket values like <none> → remove brackets and quote: "\"none\""
+- Never use: union, object, array, map, list, none (unquoted)
+
+## Output
+Return { "fields": [] } when no fields are detectable. Only include fields you can confidently identify.
+
+## Example
+Input:
+# Field 0: Code_Point
+# Field 1: Name
+# Field 2: Joining_Type (R = Right_Joining, L = Left_Joining, D = Dual_Joining, C = Join_Causing, U = Non_Joining, T = Transparent)
+# Field 3: Joining_Group
+
+Output:
+{
+  "fields": [
+    { "name": "code_point", "type": "string", "description": "Unicode code point in hexadecimal" },
+    { "name": "name", "type": "string", "description": "Schematic name for the character" },
+    { "name": "joining_type", "type": "\\"R\\" | \\"L\\" | \\"D\\" | \\"C\\" | \\"U\\" | \\"T\\"", "description": "Joining type (R=Right, L=Left, D=Dual, C=Join_Causing, U=Non_Joining, T=Transparent)" },
+    { "name": "joining_group", "type": "string", "description": "Joining group based on character names" }
+  ]
+}
+`.trim();
+async function generateFields(options) {
+	const { datafile, model } = options;
+	try {
+		return (await generateText({
+			model,
+			system: SYSTEM_PROMPT,
+			prompt: datafile.heading,
+			output: Output.object({ schema: z.object({ fields: z.array(z.object({
+				name: z.string(),
+				type: z.string(),
+				description: z.string()
+			})) }) })
+		})).output.fields;
+	} catch (err) {
+		console.error("error generating fields:", err);
+		return null;
+	}
+}
+//#endregion
+//#region src/fields/run.ts
+async function generateFieldsCode(datafile, fileName, version, model) {
+	const fields = await generateFields({
+		datafile,
+		model
+	});
+	if (fields == null) {
+		console.error(`Error generating fields for file: ${fileName}`);
+		return null;
+	}
+	const properties = {};
+	for (const field of fields) {
+		if (properties[field.name] != null) {
+			console.error(`Duplicate field name ${field.name} in file ${fileName}. Skipping field.`);
+			continue;
+		}
+		properties[field.name] = field.type;
+	}
+	let code = `// This file is generated by ucd codegen. Do not edit this file directly.\n`;
+	code += `// Unicode Version: ${version}\n\n`;
+	code += `${buildInterface(sanitizeIdentifier(toPascalCase(fileName)), properties, { export: true })}\n\n`;
+	code += `export const ${sanitizeIdentifier(toSnakeCase(fileName)).toUpperCase()}_FIELDS = ${buildStringArray(Object.keys(properties))};\n`;
+	return {
+		fields,
+		fileName,
+		version,
+		code
+	};
+}
+async function runFieldsCodegen(options) {
+	if (!options.openaiKey && !options.model) throw new Error("Either openaiKey or model must be provided");
+	const model = options.model ?? createOpenAI({ apiKey: options.openaiKey })(options.modelId ?? "gpt-4o-mini");
+	const limit = createConcurrencyLimiter(10);
+	const processPromises = options.files.map((file) => {
+		if ("content" in file) return limit(async () => {
+			return generateFieldsCode(new RawDataFile(file.content), file.fileName, file.version, model);
+		});
+		return limit(() => processFile(file.filePath, file.version, (datafile, fileName, ver) => generateFieldsCode(datafile, fileName, ver, model)));
+	});
+	return Promise.all(processPromises).then((results) => results.filter((r) => r !== null));
+}
+//#endregion
+export { generateFields as n, processFile as r, runFieldsCodegen as t };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ucdjs/codegen",
-  "version": "0.0.0",
+  "version": "0.0.1",
   "type": "module",
   "author": {
     "name": "Lucas Nørgård",
@@ -19,6 +19,7 @@
   },
   "exports": {
     ".": "./dist/index.mjs",
+    "./fields": "./dist/fields.mjs",
     "./package.json": "./package.json"
   },
   "types": "./dist/index.d.mts",
@@ -34,7 +35,7 @@
     "@unicode-utils/core": "0.12.0-beta.27",
     "ai": "6.0.138",
     "zod": "4.3.6",
-    "@ucdjs-internal/shared": "0.1.1-beta.10"
+    "@ucdjs-internal/shared": "0.2.0"
   },
   "devDependencies": {
     "@luxass/eslint-config": "7.4.2",