aiex-cli 0.1.1-beta.5 → 0.1.1-beta.7

package/dist/default-prompts.json

@@ -0,0 +1,4 @@
+{
+  "systemTemplate": "You are a professional data extraction assistant. Your task is to extract structured data from text and return a JSON object based on the data structure definition provided below.\n\n{schema}\n\nExtraction requirements:\n1. Extract strictly according to the field names and types defined in the structure\n2. If the text lacks information for a field, set that field to null\n3. Do not add fields that do not exist in the structure definition\n4. Maintain data accuracy and completeness",
+  "userTemplate": "Please extract data from the following text:\n{text}"
+}

package/dist/{generate-drizzle-schema-DpHYeu2z.mjs → generate-drizzle-schema-B6ocPcWd.mjs}

@@ -3,15 +3,16 @@ import os from "node:os";
 import path from "node:path";
 import process from "node:process";
 import { fileURLToPath } from "node:url";
-import Database from "better-sqlite3";
 import { execa } from "execa";
 import { readFile, writeFile } from "jsonfile";
 import Conf from "conf";
 import { z } from "zod";
+import Database from "better-sqlite3";
+import { Kysely, SqliteDialect, sql } from "kysely";
 
 //#region package.json
 var name = "aiex-cli";
-var version = "0.1.1-beta.5";
+var version = "0.1.1-beta.7";
 var description = "JSON Schema → SQLite with AI-powered data extraction";
 var package_default = {
 	name,
@@ -147,6 +148,53 @@ function seedConfig(config = createConfig()) {
 	if (!config.has("version")) config.set("version", version);
 }
 
+//#endregion
+//#region assets/default-prompts.json
+var systemTemplate = "You are a professional data extraction assistant. Your task is to extract structured data from text and return a JSON object based on the data structure definition provided below.\n\n{schema}\n\nExtraction requirements:\n1. Extract strictly according to the field names and types defined in the structure\n2. If the text lacks information for a field, set that field to null\n3. Do not add fields that do not exist in the structure definition\n4. Maintain data accuracy and completeness";
+var userTemplate = "Please extract data from the following text:\n{text}";
+
+//#endregion
+//#region src/domain/ai/prompts.ts
+const PLACEHOLDER_SCHEMA = "{schema}";
+const PLACEHOLDER_TEXT = "{text}";
+const DEFAULT_EXTRACTION_SYSTEM_TEMPLATE = systemTemplate;
+const DEFAULT_EXTRACTION_USER_TEMPLATE = userTemplate;
+const DEFAULT_PROMPT_CONFIG = {
+	systemTemplate: DEFAULT_EXTRACTION_SYSTEM_TEMPLATE,
+	userTemplate: DEFAULT_EXTRACTION_USER_TEMPLATE
+};
+const EVIDENCE_INSTRUCTIONS = `Evidence requirements:
+- Also return a top-level "_evidence" object.
+- For each top-level scalar field you extracted from the text, include "_evidence.<field>.quote".
+- The quote must be an exact contiguous substring copied from the input text.
+- Do not invent offsets. Only provide quotes.
+- If no exact quote supports a field, omit that field from "_evidence".`;
+const CORRECTION_SYSTEM_PROMPT = `You are a precise data correction assistant. Your task is to correct validation errors in a previously generated JSON object to make it comply with the provided JSON Schema.
+        
+CRITICAL RULES:
+1. Only correct the fields that failed validation.
+2. Preserve all other correctly extracted fields and their values exactly.
+3. Return ONLY the corrected JSON object. No explanations, no markdown blocks other than JSON.`;
+function buildCorrectionUserPrompt(input) {
+	return `The JSON data you generated previously failed validation. Please correct it.
+
+[Original Text]
+${input.text || "Data is contained in the attached file."}
+
+[JSON Schema Definition]
+${JSON.stringify(input.schema, null, 2)}
+
+[Previously Generated Invalid JSON]
+${input.invalidJson}
+
+[Validation Error Details]
+${input.error}
+
+${input.includeEvidenceInstructions ? EVIDENCE_INSTRUCTIONS : ""}
+
+Please output the corrected JSON object now:`;
+}
+
 //#endregion
 //#region src/domain/ai/types.ts
 const PDF_CONVERTER_KINDS = [
@@ -156,8 +204,6 @@ const PDF_CONVERTER_KINDS = [
 	"mineru_api",
 	"external"
 ];
-const PLACEHOLDER_SCHEMA = "{schema}";
-const PLACEHOLDER_TEXT = "{text}";
 const DEFAULT_MODELS = [{
 	name: "qwen-plus",
 	capabilities: {
@@ -177,19 +223,6 @@ const DEFAULT_PROVIDER_CONFIG = {
 	models: [...DEFAULT_MODELS],
 	timeout: 300
 };
-const DEFAULT_PROMPT_CONFIG = {
-	systemTemplate: `You are a professional data extraction assistant. Your task is to extract structured data from text and return a JSON object based on the data structure definition provided below.
-
-{schema}
-
-Extraction requirements:
-1. Extract strictly according to the field names and types defined in the structure
-2. If the text lacks information for a field, set that field to null
-3. Do not add fields that do not exist in the structure definition
-4. Maintain data accuracy and completeness`,
-	userTemplate: `Please extract data from the following text:
-{text}`
-};
 const DEFAULT_EXTRACTION_CONFIG = { outputDir: ".aiex/extracted" };
 const DEFAULT_MINERU_CONFIG = {
 	command: "mineru",
@@ -420,15 +453,15 @@ function describeColumnType(columnType) {
 	switch (columnType.class) {
 		case "text": return {
 			drizzleType: columnType.mode === "json" ? `text({ mode: 'json' })` : "text()",
-			sqliteType: "text"
+			databaseType: "text"
 		};
 		case "integer": return {
 			drizzleType: columnType.mode ? `integer({ mode: '${columnType.mode}' })` : "integer()",
-			sqliteType: "integer"
+			databaseType: "integer"
 		};
 		case "real": return {
 			drizzleType: "real()",
-			sqliteType: "real"
+			databaseType: "real"
 		};
 	}
 }
@@ -448,7 +481,7 @@ function mapColumnToReport(schemaPath, table, property, column, relation) {
 		table,
 		column: column.name,
 		drizzleType: columnType.drizzleType,
-		sqliteType: columnType.sqliteType,
+		databaseType: columnType.databaseType,
 		nullable: column.isNullable,
 		primary: column.isPrimary,
 		unique: column.isUnique,
@@ -554,7 +587,7 @@ function parseNestedObject(propName, property, parentTableName, warnings, mappin
 		table: nestedTableName,
 		column: "id",
 		drizzleType: "integer().primaryKey({ autoIncrement: true })",
-		sqliteType: "integer",
+		databaseType: "integer",
 		nullable: false,
 		primary: true,
 		unique: false,
@@ -579,7 +612,7 @@ function parseNestedObject(propName, property, parentTableName, warnings, mappin
 		table: nestedTableName,
 		column: `${parentTableName}_id`,
 		drizzleType: "integer().references(...)",
-		sqliteType: "integer",
+		databaseType: "integer",
 		nullable: false,
 		primary: false,
 		unique: false,
@@ -884,6 +917,262 @@ async function addToGitignore(aiexDir, fileName) {
 	}
 }
 
+//#endregion
+//#region src/infrastructure/extraction/insert-extracted-data.ts
+function convertValue(value, column) {
+	if (value === null || value === void 0) return null;
+	const mode = column.columnType.class !== "real" ? column.columnType.mode : void 0;
+	if (mode === "json") return typeof value === "string" ? value : JSON.stringify(value);
+	if (mode === "boolean") return value ? 1 : 0;
+	if (mode === "timestamp" || mode === "timestamp_ms") {
+		if (typeof value === "string") {
+			const ms = Date.parse(value);
+			if (Number.isNaN(ms)) return value;
+			return mode === "timestamp_ms" ? ms : Math.floor(ms / 1e3);
+		}
+		return value;
+	}
+	return value;
+}
+function buildInsertSql(table, data) {
+	const columns = [];
+	const values = [];
+	for (const col of table.columns) {
+		if (col.isAutoIncrement) continue;
+		const value = data[col.name];
+		if (value === void 0) {
+			if (col.default !== void 0) {
+				columns.push(col.name);
+				values.push(convertValue(col.default, col));
+			}
+			continue;
+		}
+		columns.push(col.name);
+		values.push(convertValue(value, col));
+	}
+	const placeholders = values.map(() => "?").join(", ");
+	return {
+		sql: `INSERT INTO ${table.name} (${columns.join(", ")}) VALUES (${placeholders})`,
+		values
+	};
+}
+function insertTableRow({ db, table, data, parentRowId, foreignKeyColumn }) {
+	const rowData = { ...data };
+	if (parentRowId !== void 0 && foreignKeyColumn) rowData[foreignKeyColumn] = parentRowId;
+	const { sql: sql$1, values } = buildInsertSql(table, rowData);
+	const info = db.prepare(sql$1).run(...values);
+	return Number(info.lastInsertRowid);
+}
+function parseDataByColumns(data, schema, table) {
+	const result = {};
+	if ("properties" in schema) {
+		const s = schema;
+		for (const [propName, prop] of Object.entries(s.properties)) {
+			if (prop.nested?.enabled) continue;
+			if (prop.type === "array" && prop.items?.nested?.enabled) continue;
+			const colName = toSnakeCase(propName);
+			if (table.columns.some((c) => c.name === colName && c.isAutoIncrement)) continue;
+			if (propName in data) result[colName] = data[propName];
+		}
+	}
+	if (schema.table?.timestamps) {
+		if (!("created_at" in result)) result.created_at = Math.floor(Date.now() / 1e3);
+		if (!("updated_at" in result)) result.updated_at = Math.floor(Date.now() / 1e3);
+	}
+	return result;
+}
+function insertExtractedData(db, schema, data) {
+	const inserted = [];
+	try {
+		const parseResult = parseJsonSchema(schema);
+		const mainTable = parseResult.tables[0];
+		db.transaction(() => {
+			const mainRowId = insertTableRow({
+				db,
+				table: mainTable,
+				data: parseDataByColumns(data, schema, mainTable)
+			});
+			inserted.push({
+				table: mainTable.name,
+				rowId: mainRowId
+			});
+			for (const revRel of parseResult.reverseRelations) {
+				const rel = parseResult.relations.find((r) => r.fromTable === revRel.toTable && r.toTable === revRel.fromTable);
+				if (!rel) continue;
+				const propEntry = Object.entries(schema.properties).find(([key]) => toSnakeCase(key) === revRel.name && key in data);
+				if (!propEntry) continue;
+				const [propName] = propEntry;
+				const nestedValue = data[propName];
+				if (nestedValue === null || nestedValue === void 0) continue;
+				const nestedTable = parseResult.tables.find((t$1) => t$1.name === revRel.toTable);
+				if (!nestedTable) continue;
+				if (revRel.type === "has-one") {
+					const rowId = insertTableRow({
+						db,
+						table: nestedTable,
+						data: parseDataByColumns(nestedValue, schema.properties[propName], nestedTable),
+						parentRowId: mainRowId,
+						foreignKeyColumn: rel.fromColumn
+					});
+					inserted.push({
+						table: revRel.toTable,
+						rowId
+					});
+				} else if (revRel.type === "has-many") {
+					const items = nestedValue;
+					for (const item of items) {
+						const rowId = insertTableRow({
+							db,
+							table: nestedTable,
+							data: parseDataByColumns(item, schema.properties[propName].items, nestedTable),
+							parentRowId: mainRowId,
+							foreignKeyColumn: rel.fromColumn
+						});
+						inserted.push({
+							table: revRel.toTable,
+							rowId
+						});
+					}
+				}
+			}
+			return mainRowId;
+		})();
+		return {
+			success: true,
+			tablesInserted: inserted
+		};
+	} catch (e) {
+		return {
+			success: false,
+			tablesInserted: inserted,
+			error: e instanceof Error ? e.message : String(e)
+		};
+	}
+}
+
+//#endregion
+//#region src/infrastructure/database/sqlite-database.ts
+const INTERNAL_ROWID_COLUMN = "__aiex_rowid";
+function createReadonlyQueryDb(databasePath) {
+	return new Kysely({ dialect: new SqliteDialect({ database: new Database(databasePath, { readonly: true }) }) });
+}
+var SqliteProjectDatabase = class {
+	dialect = "sqlite";
+	constructor(databasePath) {
+		this.databasePath = databasePath;
+	}
+	async exists() {
+		try {
+			return (await fs.stat(this.databasePath)).isFile();
+		} catch {
+			return false;
+		}
+	}
+	async listTableNames() {
+		let db = null;
+		try {
+			db = createReadonlyQueryDb(this.databasePath);
+			return (await sql`
+        select name
+        from sqlite_master
+        where type = 'table' and name not like 'sqlite_%' and name not like '_%'
+        order by name
+      `.execute(db)).rows.map((row) => row.name);
+		} finally {
+			await db?.destroy();
+		}
+	}
+	async verifyTables(tableNames) {
+		const db = new Database(this.databasePath, { readonly: true });
+		try {
+			const missing = tableNames.filter((table) => {
+				return !db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`).get(table);
+			});
+			return {
+				ok: missing.length === 0,
+				missing
+			};
+		} catch (error) {
+			return {
+				ok: false,
+				missing: [],
+				error: error instanceof Error ? error.message : String(error)
+			};
+		} finally {
+			db.close();
+		}
+	}
+	insertExtracted(schema, data) {
+		const db = new Database(this.databasePath);
+		try {
+			return insertExtractedData(db, schema, data);
+		} finally {
+			db.close();
+		}
+	}
+	async readTableRows(query) {
+		const { tableName, page, pageSize, search, sortField, sortOrder, all } = query;
+		const db = createReadonlyQueryDb(this.databasePath);
+		try {
+			if ((await sql`
+        select name
+        from sqlite_master
+        where type = 'table' and name = ${tableName}
+      `.execute(db)).rows.length === 0) throw new Error(`Table not found: ${tableName}`);
+			const columns = (await sql`
+        pragma table_info(${sql.table(tableName)})
+      `.execute(db)).rows.map((col) => ({
+				name: col.name,
+				type: col.type,
+				notNull: !!col.notnull,
+				pk: !!col.pk
+			}));
+			const searchConditions = columns.map((col) => sql`${sql.ref(col.name)} like ${`%${search}%`}`);
+			const searchCondition = search ? sql`where ${sql.join(searchConditions, sql` or `)}` : sql``;
+			const sortColumn = columns.find((col) => col.name === sortField);
+			const orderBy = sortColumn ? sql`order by ${sql.ref(sortColumn.name)} ${sql.raw(sortOrder === "desc" ? "desc" : "asc")}` : sql``;
+			const total = (await sql`
+        select count(*) as count
+        from ${sql.table(tableName)}
+        ${searchCondition}
+      `.execute(db)).rows[0]?.count ?? 0;
+			const offset = (page - 1) * pageSize;
+			const totalPages = all ? 1 : Math.max(1, Math.ceil(total / pageSize));
+			const result = all ? await sql`
+            select rowid as ${sql.raw(INTERNAL_ROWID_COLUMN)}, *
+            from ${sql.table(tableName)}
+            ${searchCondition}
+            ${orderBy}
+          `.execute(db) : await sql`
+            select rowid as ${sql.raw(INTERNAL_ROWID_COLUMN)}, *
+            from ${sql.table(tableName)}
+            ${searchCondition}
+            ${orderBy}
+            limit ${pageSize}
+            offset ${offset}
+          `.execute(db);
+			const rowIds = result.rows.map((row) => {
+				const rowId = row[INTERNAL_ROWID_COLUMN];
+				return rowId === null || rowId === void 0 ? void 0 : String(rowId);
+			});
+			return {
+				columns,
+				rows: result.rows.map(({ [INTERNAL_ROWID_COLUMN]: _rowid, ...row }) => row),
+				rowIds,
+				total,
+				page,
+				pageSize,
+				totalPages
+			};
+		} finally {
+			await db.destroy();
+		}
+	}
+};
+function createProjectDatabase(config) {
+	return new SqliteProjectDatabase(config.databasePath);
+}
+
 //#endregion
 //#region src/locales/en.ts
 const en = {
@@ -1288,19 +1577,6 @@ const en = {
 		},
 		ocrFallbackOptions: { localAuto: "Vision model or local OCR" }
 	},
-	prompt: {
-		defaultSystem: `You are a professional data extraction assistant. Your task is to extract structured data from text and return a JSON object based on the data structure definition provided below.
-
-{schema}
-
-Extraction requirements:
-1. Extract data strictly according to the field names and types defined in the structure
-2. If a field's information is missing from the text, set that field to null
-3. Do not add fields that are not in the structure definition
-4. Maintain data accuracy and completeness`,
-		defaultUser: `Please extract data from the following text:
-{text}`
-	},
 	fields: {
 		config: "Fields Config",
 		fewShotExamples: "Few-Shot Examples",
@@ -1357,7 +1633,7 @@ async function initI18n(lng) {
 			fallbackLng: "en",
 			resources: {
 				"en": { translation: en },
-				"zh-CN": { translation: await import("./zh-CN-BAGJklRp.mjs").then((m) => m.zhCN) }
+				"zh-CN": { translation: await import("./zh-CN-CJiDMnGe.mjs").then((m) => m.zhCN) }
 			},
 			interpolation: { escapeValue: false },
 			returnNull: false
@@ -1479,6 +1755,7 @@ async function checkImageOcrAvailability(imagePath, runtime = defaultRuntime) {
 //#region src/infrastructure/schema/migration-config.ts
 function createMigrationConfig(cwd) {
 	return {
+		databaseDialect: "sqlite",
 		schemaPath: `${cwd}/.aiex/schema`,
 		drizzleSchemaPath: `${cwd}/.aiex/drizzle/schema.ts`,
 		migrationsPath: `${cwd}/.aiex/migrations`,
@@ -1617,26 +1894,18 @@ async function collectDoctorDiagnostics(options = {}) {
 			error: error instanceof Error ? error.message : String(error)
 		});
 	}
-	let dbExists = false;
-	if (dirExists) try {
-		dbExists = (await fs.stat(migConfig.databasePath)).isFile();
-	} catch {
-		dbExists = false;
-	}
+	const database = createProjectDatabase(migConfig);
+	const dbExists = dirExists ? await database.exists() : false;
 	let databaseTablesOk = null;
 	let missingDatabaseTables = [];
 	if (dbExists && expectedTables.size > 0) {
-		const db = new Database(migConfig.databasePath, { readonly: true });
-		try {
-			missingDatabaseTables = [...expectedTables].filter((table) => {
-				return !db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`).get(table);
-			});
-			databaseTablesOk = missingDatabaseTables.length === 0;
-		} catch (error) {
+		const tableCheck = await database.verifyTables([...expectedTables]);
+		if (tableCheck.error) {
 			databaseTablesOk = false;
-			errors.push(`Could not inspect database tables: ${error instanceof Error ? error.message : String(error)}`);
-		} finally {
-			db.close();
+			errors.push(`Could not inspect database tables: ${tableCheck.error}`);
+		} else {
+			missingDatabaseTables = tableCheck.missing;
+			databaseTablesOk = tableCheck.ok;
 		}
 	} else if (dbExists) databaseTablesOk = true;
 	let migrationCount = 0;
@@ -1788,4 +2057,4 @@ function generateDrizzleSchema(result) {
 }
 
 //#endregion
-export { package_default as A, DEFAULT_PROMPT_CONFIG as C, seedConfig as D, createConfig as E, description as O, DEFAULT_MINERU_CONFIG as S, PLACEHOLDER_TEXT as T, doctorDiagnosticsSeverityRows as _, recognizeImageText as a, DEFAULT_LITEPARSE_CONFIG as b, t as c, writeAIConfig as d, AIConfigSchema as f, buildDoctorDiagnostics as g, toSnakeCase as h, generateDrizzleConfig as i, version as j, name as k, getDefaultAIConfig as l, parseJsonSchema as m, collectDoctorDiagnostics as n, shouldUseImageOcrFallback as o, JsonSchemaDefinitionSchema as p, createMigrationConfig as r, initI18n as s, generateDrizzleSchema as t, readAIConfig as u, doctorDiagnosticsTableRows as v, PLACEHOLDER_SCHEMA as w, DEFAULT_MINERU_API_CONFIG as x, formatDoctorDiagnosticsJson as y };
+export { seedConfig as A, CORRECTION_SYSTEM_PROMPT as C, PLACEHOLDER_TEXT as D, PLACEHOLDER_SCHEMA as E, name as M, package_default as N, buildCorrectionUserPrompt as O, version as P, DEFAULT_MINERU_CONFIG as S, EVIDENCE_INSTRUCTIONS as T, doctorDiagnosticsSeverityRows as _, recognizeImageText as a, DEFAULT_LITEPARSE_CONFIG as b, t as c, readAIConfig as d, writeAIConfig as f, buildDoctorDiagnostics as g, parseJsonSchema as h, generateDrizzleConfig as i, description as j, createConfig as k, createProjectDatabase as l, JsonSchemaDefinitionSchema as m, collectDoctorDiagnostics as n, shouldUseImageOcrFallback as o, AIConfigSchema as p, createMigrationConfig as r, initI18n as s, generateDrizzleSchema as t, getDefaultAIConfig as u, doctorDiagnosticsTableRows as v, DEFAULT_PROMPT_CONFIG as w, DEFAULT_MINERU_API_CONFIG as x, formatDoctorDiagnosticsJson as y };

package/dist/index.d.mts

@@ -326,7 +326,7 @@ interface SchemaMappingEntry {
   table: string;
   column: string;
   drizzleType: string;
-  sqliteType: 'text' | 'integer' | 'real';
+  databaseType: 'text' | 'integer' | 'real';
   nullable: boolean;
   primary: boolean;
   unique: boolean;
@@ -362,6 +362,7 @@ interface ParseResult {
   mapping?: SchemaMappingEntry[];
 }
 interface MigrationConfig {
+  databaseDialect: 'sqlite';
   schemaPath: string;
   drizzleSchemaPath: string;
   migrationsPath: string;

package/dist/index.mjs

@@ -1,3 +1,3 @@
-import { _ as doctorDiagnosticsSeverityRows, g as buildDoctorDiagnostics, i as generateDrizzleConfig, m as parseJsonSchema, n as collectDoctorDiagnostics, p as JsonSchemaDefinitionSchema, r as createMigrationConfig, t as generateDrizzleSchema, v as doctorDiagnosticsTableRows, y as formatDoctorDiagnosticsJson } from "./generate-drizzle-schema-DpHYeu2z.mjs";
+import { _ as doctorDiagnosticsSeverityRows, g as buildDoctorDiagnostics, h as parseJsonSchema, i as generateDrizzleConfig, m as JsonSchemaDefinitionSchema, n as collectDoctorDiagnostics, r as createMigrationConfig, t as generateDrizzleSchema, v as doctorDiagnosticsTableRows, y as formatDoctorDiagnosticsJson } from "./generate-drizzle-schema-B6ocPcWd.mjs";
 
 export { JsonSchemaDefinitionSchema, buildDoctorDiagnostics, collectDoctorDiagnostics, createMigrationConfig, doctorDiagnosticsSeverityRows, doctorDiagnosticsTableRows, formatDoctorDiagnosticsJson, generateDrizzleConfig, generateDrizzleSchema, parseJsonSchema };

package/dist/infrastructure/schema/migrate-helper.mjs

@@ -3,8 +3,8 @@ import fs from "node:fs/promises";
 import path from "node:path";
 import process from "node:process";
 import { fileURLToPath } from "node:url";
-import Database from "better-sqlite3";
 import { readFile, writeFile } from "jsonfile";
+import Database from "better-sqlite3";
 import * as esbuild from "esbuild";
 import lockfile from "proper-lockfile";