@ucdjs/schema-gen 0.2.0 → 0.2.2

package/README.md

@@ -2,6 +2,7 @@
 
 [![npm version][npm-version-src]][npm-version-href]
 [![npm downloads][npm-downloads-src]][npm-downloads-href]
+[![codecov][codecov-src]][codecov-href]
 
 Utilities for working with the Unicode Character Database (UCD).
 
@@ -19,3 +20,5 @@ Published under [MIT License](./LICENSE).
 [npm-version-href]: https://npmjs.com/package/@ucdjs/schema-gen
 [npm-downloads-src]: https://img.shields.io/npm/dm/@ucdjs/schema-gen?style=flat&colorA=18181B&colorB=4169E1
 [npm-downloads-href]: https://npmjs.com/package/@ucdjs/schema-gen
+[codecov-src]: https://img.shields.io/codecov/c/gh/ucdjs/ucd?style=flat&colorA=18181B&colorB=4169E1
+[codecov-href]: https://codecov.io/gh/ucdjs/ucd

package/dist/index.d.ts

@@ -1,17 +1,48 @@
-import { RawDataFile } from "@luxass/unicode-utils";
+import { LanguageModel } from "ai";
 
-//#region src/fields.d.ts
-interface GenerateFieldsOptions {
-    /**
-     * The data file to generate fields for.
-     */
-    datafile: RawDataFile;
-    /**
-     * The OpenAI API key to use for generating fields.
-     */
-    apiKey: string;
+//#region src/index.d.ts
+interface SchemaGenFile {
+  /**
+   * The filePath to the data file.
+   */
+  filePath: string;
+  /**
+   * The version of the data file.
+   */
+  version: string;
 }
-declare function generateFields(options: GenerateFieldsOptions): Promise<string | null>;
-
+interface ProcessedFile {
+  fields: {
+    type: string;
+    name: string;
+    description: string;
+  }[];
+  code: string;
+  fileName: string;
+  version: string;
+}
+interface SchemaGenOptions {
+  /**
+   * Files to generate structures for.
+   */
+  files: SchemaGenFile[];
+  /**
+   * The OpenAI API key to use for generating the schema.
+   */
+  openaiKey?: string;
+  /**
+   * The OpenAI model to use for generating fields.
+   * NOTE:
+   * This is good for testing purposes, where you
+   * can provide a mock model to test the generation.
+   *
+   * If not provided, it will create a new OpenAI instance
+   * with the default model.
+   *
+   * SEE: https://ai-sdk.dev/docs/ai-sdk-core/testing
+   */
+  model?: LanguageModel;
+}
+declare function runSchemagen(options: SchemaGenOptions): Promise<ProcessedFile[]>;
 //#endregion
-export { generateFields };
+export { ProcessedFile, SchemaGenFile, SchemaGenOptions, runSchemagen };

package/dist/index.js

@@ -1,71 +1,182 @@
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { RawDataFile } from "@luxass/unicode-utils-old";
+import { dedent, sanitizeIdentifier, toPascalCase, toSnakeCase } from "@luxass/utils";
+import { createConcurrencyLimiter } from "@ucdjs-internal/shared";
+import { genArrayFromRaw, genInterface } from "knitwork";
 import { createOpenAI } from "@ai-sdk/openai";
-import { dedent } from "@luxass/utils";
 import { generateObject } from "ai";
 import { z } from "zod";
 
 //#region src/fields.ts
-const SYSTEM_PROMOT = dedent`
-      <system_prompt>
-        <role>Expert TypeScript code generator specializing in interfaces and documentation</role>
-
-        <task>
-          <input>Text description: {{INPUT}}</input>
-          <output>TypeScript interface with comprehensive JSDoc comments</output>
-        </task>
-
-        <requirements>
-          <field_processing>
-            - Extract all relevant fields from text
-            - Convert field names to snake_case
-            - Preserve original order
-          </field_processing>
-
-          <documentation>
-            - JSDoc for each property only
-            - Document union types with double quotes (no enums)
-            - Explain constraints and formats with examples
-            - No JSDoc for the main interface
-          </documentation>
-
-          <structure>
-            - Use the file name from the first line of the input (ignoring version numbers) as the interface name
-            - Single interface named after the input file name without version numbers (e.g., ArabicShaping not ArabicShaping-16.0.0)
-            - No additional interfaces or arrays
-            - Create ordered keys array named [INTERFACE_NAME]_FIELDS using SCREAMING_SNAKE_CASE
-            - Convert the interface name to SCREAMING_SNAKE_CASE by inserting underscores between camelCase words
-            - Use double quotes for field names in the keys array
-            - No example data or additional declarations
-            - Export all variables and interfaces
-          </structure>
-
-          <formatting>
-            - Use 2 space indentation
-            - Use trailing commas in all multi-line structures
-            - Use trailing commas in arrays and interfaces
-          </formatting>
-        </requirements>
-
-        <format>Single TypeScript code block containing only the interface and fields array, both exported</format>
-
-        <examples>
-          - Interface: NamedSequences -> Fields array: NAMED_SEQUENCES_FIELDS
-          - Interface: NamedSequencesProv -> Fields array: NAMED_SEQUENCES_PROV_FIELDS
-          - Interface: ArabicShaping -> Fields array: ARABIC_SHAPING_FIELDS
-        </examples>
-      </system_prompt>
+const SYSTEM_PROMPT = dedent`
+<system_prompt>
+  <role>Expert TypeScript field extractor for Unicode data files</role>
+
+  <task>
+    <input>
+      Text description: {{INPUT}}
+    </input>
+    <output>JSON array of field objects with name, type, and description</output>
+  </task>
+
+  <critical_rule>
+    YOU MUST ALWAYS OUTPUT A VALID JSON ARRAY OF OBJECTS WITH THIS STRUCTURE:
+    [
+      {
+        "name": "actual_field_name",
+        "type": "valid_typescript_type",
+        "description": "Description of the field"
+      }
+    ]
+
+    NEVER use generic names like "field_0", "field_1", etc. - extract the ACTUAL field names.
+    NEVER use invalid TypeScript types like "union" - use proper union syntax with pipe symbol.
+  </critical_rule>
+
+  <field_detection>
+    For Unicode data files, fields are typically described in patterns like:
+    - Lines starting with "# Field 0: Name" - where "Name" is the actual field name
+    - Table headers or column definitions
+    - Property descriptions in documentation
+
+    DO NOT use "field_0", "field_1" as field names - extract the REAL field names.
+    DO NOT use "type_name" as a field name unless it's explicitly mentioned as a field.
+
+    Example: "# Field 0: Code_Point" should produce a field named "code_point" (NOT "field_0").
+  </field_detection>
+
+  <type_mapping>
+    ONLY USE THESE VALID TYPESCRIPT TYPES:
+    - string - For text, identifiers, character codes, etc.
+    - number - For numeric values, indices, counts
+    - boolean - For true/false flags
+    - string[] - For arrays of strings
+    - number[] - For arrays of numbers
+    - Array<string> - Alternative syntax for string arrays
+    - Array<number> - Alternative syntax for number arrays
+    - Record<string, string> - For string to string mappings
+    - Record<string, number> - For string to number mappings
+    - Record<string, unknown> - For objects with unknown structure
+    - unknown - When type cannot be determined
+    - any - Only as a last resort when type is truly variable
+
+    For union types with string literals, ALWAYS use double quotes and pipe symbol:
+    - "\"value1\" | \"value2\" | \"value3\""
+
+    CRITICAL: Special values handling:
+    1. If a value contains angle brackets, like <none>, FIRST remove the angle brackets: "none" (NOT "<none>")
+    2. THEN ALWAYS wrap the value in quotes if it's a string literal: "\"none\""
+    3. For union types of string literals, EACH value MUST be in quotes:
+       - Correct: "\"R\" | \"L\" | \"D\" | \"C\" | \"U\" | \"T\""
+       - Incorrect: "R | L | D | C | U | T" (missing quotes around each value)
+       - Incorrect: "none" (missing quotes if it's a string literal)
+
+    NEVER use these invalid types:
+    - "union" - This is not a valid TypeScript type
+    - "object" - Too generic, use Record<> instead
+    - "array" - Too generic, use proper array syntax instead
+    - "map" - Use Record<> instead
+    - "none" - Use "\"none\"" or never instead
+    - "list" - Use proper array syntax instead
+
+    ALWAYS wrap the final converted type in double quotes in the JSON output.
+    ALWAYS wrap each value in a union type of string literals in quotes.
+  </type_mapping>
+
+  <examples>
+    Input example:
+    \`\`\`
+    # ArabicShaping.txt
+    # Field 0: Code point
+    # Field 1: Name
+    # Field 2: Joining_Type (R = Right_Joining, L = Left_Joining, D = Dual_Joining, C = Join_Causing, U = Non_Joining, T = Transparent)
+    # Field 3: Joining_Group
+    \`\`\`
+
+    Correct output:
+    [
+      {
+        "name": "code_point",
+        "type": "string",
+        "description": "The code point of a character, in hexadecimal form"
+      },
+      {
+        "name": "name",
+        "type": "string",
+        "description": "A short schematic name for the character"
+      },
+      {
+        "name": "joining_type",
+        "type": "\"R\" | \"L\" | \"D\" | \"C\" | \"U\" | \"T\"",
+        "description": "Defines the joining type (R = Right_Joining, L = Left_Joining, D = Dual_Joining, C = Join_Causing, U = Non_Joining, T = Transparent)"
+      },
+      {
+        "name": "joining_group",
+        "type": "string",
+        "description": "Defines the joining group, based schematically on character names"
+      }
+    ]
+
+    Example with special value:
+    Input: "# Field 4: Value (<none> or specific value)"
+
+    Correct output for this field:
+    {
+      "name": "value",
+      "type": "\"none\" | string",
+      "description": "The value, which can be none or a specific value"
+    }
+  </examples>
+
+  <validation>
+    Before outputting, verify:
+    - Field names are the ACTUAL field names from the documentation, NOT generic "field_0" style names
+    - All field names are in snake_case
+    - All types are valid TypeScript types (string, number, boolean, arrays, or proper union types)
+    - NEVER use the word "union" as a type - use proper TypeScript syntax with the pipe symbol
+    - String literal values in union types are ALWAYS wrapped in quotes (e.g., "\"value1\" | \"value2\"")
+    - Special values like "none" (from <none>) are properly quoted as string literals: "\"none\""
+    - Each field has a clear, specific description
+    - Output is a valid JSON array of objects
+  </validation>
+
+  <error_handling>
+    If NO fields can be detected in the input:
+    - Return an empty JSON array: []
+    - DO NOT return error messages or explanations in the JSON output
+    - DO NOT attempt to create fields when none are clearly defined
+
+    Example when no fields are detected:
+    Input: "This is some text without any field definitions"
+
+    Correct output:
+    []
+
+    If SOME fields are unclear but others are detectable:
+    - Only include the fields that can be clearly identified
+    - Omit any fields that cannot be confidently extracted
+    - Follow all validation rules for the fields that are included
+  </error_handling>
+
+  <format>JSON array of field objects</format>
+</system_prompt>
 `;
 async function generateFields(options) {
-	const { datafile, apiKey } = options;
+	const { datafile, apiKey, model } = options;
 	if (datafile.heading == null) return null;
-	if (!apiKey) return null;
-	const openai = createOpenAI({ apiKey });
+	if (!apiKey && !model) return null;
+	const openai = model != null ? null : createOpenAI({ apiKey });
 	try {
-		const result = await generateObject({
-			model: openai("gpt-4o-mini"),
-			schema: z.object({ code: z.string().describe("A TypeScript code block containing the JSDoc commented interface definition and the corresponding keys array with original casing.") }),
-			prompt: SYSTEM_PROMOT.replace("{{INPUT}}", datafile.heading)
-		});
-		return result.object.code;
+		return (await generateObject({
+			model: model ?? openai("gpt-4o-mini"),
+			schema: z.object({ fields: z.array(z.object({
+				name: z.string(),
+				type: z.string(),
+				description: z.string()
+			})) }),
+			prompt: SYSTEM_PROMPT.replace("{{INPUT}}", datafile.heading)
+		})).object.fields;
 	} catch (err) {
 		console.error("error generating fields:", err);
 		return null;
@@ -73,4 +184,63 @@ async function generateFields(options) {
 }
 
 //#endregion
-export { generateFields };
+//#region src/index.ts
+async function runSchemagen(options) {
+	const inputFiles = options.files;
+	const limit = createConcurrencyLimiter(10);
+	if (!options.openaiKey && !options.model) throw new Error("Either openaiKey or model must be provided");
+	const processPromises = inputFiles.map(({ filePath, version }) => limit(() => processFile({
+		filePath,
+		openaiKey: options.openaiKey,
+		version,
+		model: options.model
+	})));
+	return Promise.all(processPromises).then((results) => results.filter((result) => result !== null));
+}
+async function processFile(request) {
+	const { filePath, openaiKey, version, model } = request;
+	try {
+		console.log(`Processing file: ${filePath}`);
+		const content = await readFile(filePath, "utf-8");
+		const fileName = path.basename(filePath).replace(/\.txt$/, "");
+		const datafile = new RawDataFile(content, fileName);
+		if (datafile.heading == null) {
+			console.error(`heading for file ${filePath} is null. Skipping file.`);
+			return null;
+		}
+		const fields = await generateFields({
+			datafile,
+			apiKey: openaiKey,
+			model
+		});
+		if (fields == null) {
+			console.error(`Error generating fields for file: ${filePath}`);
+			return null;
+		}
+		let code = ``;
+		const properties = {};
+		for (const field of fields) {
+			if (properties[field.name] != null) {
+				console.error(`Duplicate field name ${field.name} in file ${filePath}. Skipping field.`);
+				continue;
+			}
+			properties[field.name] = field.type;
+		}
+		code += `// This file is generated by ucd codegen. Do not edit this file directly.\n`;
+		code += `// Unicode Version: ${version}\n\n`;
+		code += `${genInterface(sanitizeIdentifier(toPascalCase(fileName)), properties, { export: true })}\n\n`;
+		code += `export const ${sanitizeIdentifier(toSnakeCase(fileName)).toUpperCase()}_FIELDS = ${genArrayFromRaw(fields.map((f) => `"${f.name}"`))};\n`;
+		return {
+			fields,
+			fileName,
+			version,
+			code
+		};
+	} catch (error) {
+		console.error(`Error processing file ${filePath}:`, error);
+		return null;
+	}
+}
+
+//#endregion
+export { runSchemagen };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ucdjs/schema-gen",
-  "version": "0.2.0",
+  "version": "0.2.2",
   "type": "module",
   "author": {
     "name": "Lucas Nørgård",
@@ -27,26 +27,33 @@
   "files": [
     "dist"
   ],
+  "engines": {
+    "node": ">=22.18"
+  },
   "dependencies": {
-    "@ai-sdk/openai": "^1.3.20",
-    "@luxass/unicode-utils": "^0.4.1",
-    "@luxass/utils": "^1.4.0",
-    "ai": "^4.3.10",
-    "zod": "^3.24.3"
+    "@ai-sdk/openai": "2.0.32",
+    "@luxass/unicode-utils-old": "npm:@luxass/unicode-utils@0.11.0",
+    "@luxass/utils": "2.7.2",
+    "ai": "5.0.48",
+    "knitwork": "1.2.0",
+    "zod": "4.1.11",
+    "@ucdjs-internal/shared": "0.1.0"
   },
   "devDependencies": {
-    "@luxass/eslint-config": "^4.18.1",
-    "eslint": "^9.25.1",
-    "publint": "^0.3.12",
-    "tsdown": "v0.9.3",
-    "typescript": "^5.8.3",
-    "vitest-testdirs": "^3.0.1"
+    "@luxass/eslint-config": "6.0.1",
+    "eslint": "9.38.0",
+    "publint": "0.3.15",
+    "tsdown": "0.15.9",
+    "typescript": "5.9.3",
+    "vitest-testdirs": "4.2.1",
+    "@ucdjs-tooling/tsconfig": "1.0.0",
+    "@ucdjs-tooling/tsdown-config": "1.0.0"
   },
   "publishConfig": {
     "access": "public"
   },
   "scripts": {
-    "build": "tsdown",
+    "build": "tsdown --tsconfig=./tsconfig.build.json",
     "dev": "tsdown --watch",
     "clean": "git clean -xdf dist node_modules",
     "lint": "eslint .",