@ucdjs/codegen 0.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026-PRESENT Lucas Nørgård
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,21 @@
+# @ucdjs/codegen
+
+[![npm version][npm-version-src]][npm-version-href]
+[![npm downloads][npm-downloads-src]][npm-downloads-href]
+
+This package provides utilities for generating code from UCD data and models.
+
+## Installation
+
+```bash
+npm install @ucdjs/codegen
+```
+
+## 📄 License
+
+Published under [MIT License](./LICENSE).
+
+[npm-version-src]: https://img.shields.io/npm/v/@ucdjs/codegen?style=flat&colorA=18181B&colorB=4169E1
+[npm-version-href]: https://npmjs.com/package/@ucdjs/codegen
+[npm-downloads-src]: https://img.shields.io/npm/dm/@ucdjs/codegen?style=flat&colorA=18181B&colorB=4169E1
+[npm-downloads-href]: https://npmjs.com/package/@ucdjs/codegen

package/dist/index.d.mts

@@ -0,0 +1,48 @@
+import { LanguageModel } from "ai";
+
+//#region src/index.d.ts
+interface CodegenFile {
+  /**
+   * The filePath to the data file.
+   */
+  filePath: string;
+  /**
+   * The version of the data file.
+   */
+  version: string;
+}
+interface ProcessedFile {
+  fields: {
+    type: string;
+    name: string;
+    description: string;
+  }[];
+  code: string;
+  fileName: string;
+  version: string;
+}
+interface CodegenOptions {
+  /**
+   * Files to generate structures for.
+   */
+  files: CodegenFile[];
+  /**
+   * The OpenAI API key to use for generating the fields.
+   */
+  openaiKey?: string;
+  /**
+   * The OpenAI model to use for generating fields.
+   * NOTE:
+   * This is good for testing purposes, where you
+   * can provide a mock model to test the generation.
+   *
+   * If not provided, it will create a new OpenAI instance
+   * with the default model.
+   *
+   * SEE: https://ai-sdk.dev/docs/ai-sdk-core/testing
+   */
+  model?: LanguageModel;
+}
+declare function runCodegen(options: CodegenOptions): Promise<ProcessedFile[]>;
+//#endregion
+export { CodegenFile, CodegenOptions, ProcessedFile, runCodegen };

package/dist/index.mjs

@@ -0,0 +1,243 @@
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { dedent, sanitizeIdentifier, toPascalCase, toSnakeCase } from "@luxass/utils";
+import { createConcurrencyLimiter } from "@ucdjs-internal/shared";
+import { RawDataFile } from "@unicode-utils/core";
+import { createOpenAI } from "@ai-sdk/openai";
+import { generateObject } from "ai";
+import { z } from "zod";
+//#region src/fields.ts
+const SYSTEM_PROMPT = dedent`
+<system_prompt>
+  <role>Expert TypeScript field extractor for Unicode data files</role>
+
+  <task>
+    <input>
+      Text description: {{INPUT}}
+    </input>
+    <output>JSON array of field objects with name, type, and description</output>
+  </task>
+
+  <critical_rule>
+    YOU MUST ALWAYS OUTPUT A VALID JSON ARRAY OF OBJECTS WITH THIS STRUCTURE:
+    [
+      {
+        "name": "actual_field_name",
+        "type": "valid_typescript_type",
+        "description": "Description of the field"
+      }
+    ]
+
+    NEVER use generic names like "field_0", "field_1", etc. - extract the ACTUAL field names.
+    NEVER use invalid TypeScript types like "union" - use proper union syntax with pipe symbol.
+  </critical_rule>
+
+  <field_detection>
+    For Unicode data files, fields are typically described in patterns like:
+    - Lines starting with "# Field 0: Name" - where "Name" is the actual field name
+    - Table headers or column definitions
+    - Property descriptions in documentation
+
+    DO NOT use "field_0", "field_1" as field names - extract the REAL field names.
+    DO NOT use "type_name" as a field name unless it's explicitly mentioned as a field.
+
+    Example: "# Field 0: Code_Point" should produce a field named "code_point" (NOT "field_0").
+  </field_detection>
+
+  <type_mapping>
+    ONLY USE THESE VALID TYPESCRIPT TYPES:
+    - string - For text, identifiers, character codes, etc.
+    - number - For numeric values, indices, counts
+    - boolean - For true/false flags
+    - string[] - For arrays of strings
+    - number[] - For arrays of numbers
+    - Array<string> - Alternative syntax for string arrays
+    - Array<number> - Alternative syntax for number arrays
+    - Record<string, string> - For string to string mappings
+    - Record<string, number> - For string to number mappings
+    - Record<string, unknown> - For objects with unknown structure
+    - unknown - When type cannot be determined
+    - any - Only as a last resort when type is truly variable
+
+    For union types with string literals, ALWAYS use double quotes and pipe symbol:
+    - "\"value1\" | \"value2\" | \"value3\""
+
+    CRITICAL: Special values handling:
+    1. If a value contains angle brackets, like <none>, FIRST remove the angle brackets: "none" (NOT "<none>")
+    2. THEN ALWAYS wrap the value in quotes if it's a string literal: "\"none\""
+    3. For union types of string literals, EACH value MUST be in quotes:
+       - Correct: "\"R\" | \"L\" | \"D\" | \"C\" | \"U\" | \"T\""
+       - Incorrect: "R | L | D | C | U | T" (missing quotes around each value)
+       - Incorrect: "none" (missing quotes if it's a string literal)
+
+    NEVER use these invalid types:
+    - "union" - This is not a valid TypeScript type
+    - "object" - Too generic, use Record<> instead
+    - "array" - Too generic, use proper array syntax instead
+    - "map" - Use Record<> instead
+    - "none" - Use "\"none\"" or never instead
+    - "list" - Use proper array syntax instead
+
+    ALWAYS wrap the final converted type in double quotes in the JSON output.
+    ALWAYS wrap each value in a union type of string literals in quotes.
+  </type_mapping>
+
+  <examples>
+    Input example:
+    \`\`\`
+    # ArabicShaping.txt
+    # Field 0: Code point
+    # Field 1: Name
+    # Field 2: Joining_Type (R = Right_Joining, L = Left_Joining, D = Dual_Joining, C = Join_Causing, U = Non_Joining, T = Transparent)
+    # Field 3: Joining_Group
+    \`\`\`
+
+    Correct output:
+    [
+      {
+        "name": "code_point",
+        "type": "string",
+        "description": "The code point of a character, in hexadecimal form"
+      },
+      {
+        "name": "name",
+        "type": "string",
+        "description": "A short schematic name for the character"
+      },
+      {
+        "name": "joining_type",
+        "type": "\"R\" | \"L\" | \"D\" | \"C\" | \"U\" | \"T\"",
+        "description": "Defines the joining type (R = Right_Joining, L = Left_Joining, D = Dual_Joining, C = Join_Causing, U = Non_Joining, T = Transparent)"
+      },
+      {
+        "name": "joining_group",
+        "type": "string",
+        "description": "Defines the joining group, based schematically on character names"
+      }
+    ]
+
+    Example with special value:
+    Input: "# Field 4: Value (<none> or specific value)"
+
+    Correct output for this field:
+    {
+      "name": "value",
+      "type": "\"none\" | string",
+      "description": "The value, which can be none or a specific value"
+    }
+  </examples>
+
+  <validation>
+    Before outputting, verify:
+    - Field names are the ACTUAL field names from the documentation, NOT generic "field_0" style names
+    - All field names are in snake_case
+    - All types are valid TypeScript types (string, number, boolean, arrays, or proper union types)
+    - NEVER use the word "union" as a type - use proper TypeScript syntax with the pipe symbol
+    - String literal values in union types are ALWAYS wrapped in quotes (e.g., "\"value1\" | \"value2\"")
+    - Special values like "none" (from <none>) are properly quoted as string literals: "\"none\""
+    - Each field has a clear, specific description
+    - Output is a valid JSON array of objects
+  </validation>
+
+  <error_handling>
+    If NO fields can be detected in the input:
+    - Return an empty JSON array: []
+    - DO NOT return error messages or explanations in the JSON output
+    - DO NOT attempt to create fields when none are clearly defined
+
+    Example when no fields are detected:
+    Input: "This is some text without any field definitions"
+
+    Correct output:
+    []
+
+    If SOME fields are unclear but others are detectable:
+    - Only include the fields that can be clearly identified
+    - Omit any fields that cannot be confidently extracted
+    - Follow all validation rules for the fields that are included
+  </error_handling>
+
+  <format>JSON array of field objects</format>
+</system_prompt>
+`;
+async function generateFields(options) {
+	const { datafile, apiKey, model } = options;
+	if (!apiKey && !model) return null;
+	const openai = model != null ? null : createOpenAI({ apiKey });
+	try {
+		return (await generateObject({
+			model: model ?? openai("gpt-4o-mini"),
+			schema: z.object({ fields: z.array(z.object({
+				name: z.string(),
+				type: z.string(),
+				description: z.string()
+			})) }),
+			prompt: SYSTEM_PROMPT.replace("{{INPUT}}", datafile.heading)
+		})).object.fields;
+	} catch (err) {
+		console.error("error generating fields:", err);
+		return null;
+	}
+}
+//#endregion
+//#region src/index.ts
+const TXT_EXTENSION_RE = /\.txt$/;
+function buildInterface(name, fields, opts) {
+	return `${opts?.export ? "export " : ""}interface ${name} {\n${Object.entries(fields).map(([key, type]) => `  ${key}: ${type};`).join("\n")}\n}`;
+}
+function buildStringArray(values) {
+	return `[${values.map((v) => `"${v}"`).join(", ")}]`;
+}
+async function runCodegen(options) {
+	const inputFiles = options.files;
+	const limit = createConcurrencyLimiter(10);
+	if (!options.openaiKey && !options.model) throw new Error("Either openaiKey or model must be provided");
+	const processPromises = inputFiles.map(({ filePath, version }) => limit(() => processFile({
+		filePath,
+		openaiKey: options.openaiKey,
+		version,
+		model: options.model
+	})));
+	return Promise.all(processPromises).then((results) => results.filter((result) => result !== null));
+}
+async function processFile(request) {
+	const { filePath, openaiKey, version, model } = request;
+	try {
+		console.log(`Processing file: ${filePath}`);
+		const content = await readFile(filePath, "utf-8");
+		const fileName = path.basename(filePath).replace(TXT_EXTENSION_RE, "");
+		const fields = await generateFields({
+			datafile: new RawDataFile(content),
+			apiKey: openaiKey,
+			model
+		});
+		if (fields == null) {
+			console.error(`Error generating fields for file: ${filePath}`);
+			return null;
+		}
+		let code = ``;
+		const properties = {};
+		for (const field of fields) {
+			if (properties[field.name] != null) {
+				console.error(`Duplicate field name ${field.name} in file ${filePath}. Skipping field.`);
+				continue;
+			}
+			properties[field.name] = field.type;
+		}
+		code += `// This file is generated by ucd codegen. Do not edit this file directly.\n`;
+		code += `// Unicode Version: ${version}\n\n`;
+		code += `${buildInterface(sanitizeIdentifier(toPascalCase(fileName)), properties, { export: true })}\n\n`;
+		code += `export const ${sanitizeIdentifier(toSnakeCase(fileName)).toUpperCase()}_FIELDS = ${buildStringArray(fields.map((f) => f.name))};\n`;
+		return {
+			fields,
+			fileName,
+			version,
+			code
+		};
+	} catch (err) {
+		console.error(`Error processing file ${filePath}:`, err);
+		return null;
+	}
+}
+//#endregion
+export { runCodegen };

package/package.json

@@ -0,0 +1,59 @@
+{
+  "name": "@ucdjs/codegen",
+  "version": "0.0.0",
+  "type": "module",
+  "author": {
+    "name": "Lucas Nørgård",
+    "email": "lucasnrgaard@gmail.com",
+    "url": "https://luxass.dev"
+  },
+  "license": "MIT",
+  "homepage": "https://github.com/ucdjs/ucd",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/ucdjs/ucd.git",
+    "directory": "packages/codegen"
+  },
+  "bugs": {
+    "url": "https://github.com/ucdjs/ucd/issues"
+  },
+  "exports": {
+    ".": "./dist/index.mjs",
+    "./package.json": "./package.json"
+  },
+  "types": "./dist/index.d.mts",
+  "files": [
+    "dist"
+  ],
+  "engines": {
+    "node": ">=24.13"
+  },
+  "dependencies": {
+    "@ai-sdk/openai": "3.0.48",
+    "@luxass/utils": "2.7.3",
+    "@unicode-utils/core": "0.12.0-beta.27",
+    "ai": "6.0.138",
+    "zod": "4.3.6",
+    "@ucdjs-internal/shared": "0.1.1-beta.10"
+  },
+  "devDependencies": {
+    "@luxass/eslint-config": "7.4.2",
+    "eslint": "10.1.0",
+    "publint": "0.3.18",
+    "tsdown": "0.21.7",
+    "typescript": "6.0.2",
+    "vitest-testdirs": "4.4.3",
+    "@ucdjs-tooling/tsconfig": "1.0.0",
+    "@ucdjs-tooling/tsdown-config": "1.0.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsdown --tsconfig=./tsconfig.build.json",
+    "dev": "tsdown --watch",
+    "clean": "git clean -xdf dist node_modules",
+    "lint": "eslint .",
+    "typecheck": "tsc --noEmit -p tsconfig.build.json"
+  }
+}