aiex-cli 0.1.1-beta.1 → 0.1.1-beta.3

package/README.md

@@ -32,7 +32,7 @@ aiex watch -s invoice -d ./watch_folder # watch folder daemon for automatic extr
 
 ## ✨ Features
 
-- **JSON Schema → SQLite** — Define tables as JSON Schema files, generate Drizzle ORM schema, and migrate to SQLite
+- **AIEX JSON Schema → SQLite** — Define tables with a Drizzle-backed JSON Schema dialect, generate Drizzle ORM schema, and migrate to SQLite
 - **Web Configuration & Viewer** — Browser-based UI for designing schemas, configuring integrations, previewing prompts, and browsing extracted data
 - **AI Extraction** — Extract structured data from files (text, images, PDFs) using any OpenAI-compatible provider (OpenAI, Anthropic, Ollama, DeepSeek, local models, etc.)
 - **Interactive Mode** — Run `aiex extract` without arguments for a guided extraction workflow
@@ -61,7 +61,7 @@ Opens a browser UI where you can visually design and manage your schemas, config
 aiex schema
 ```
 
-Converts your JSON Schema files into a SQLite database with full migration support.
+Converts AIEX JSON Schema files into a SQLite database with full migration support. AIEX uses a Drizzle-backed schema dialect rather than the full JSON Schema specification; see [Docs/schema-dialect.md](Docs/schema-dialect.md) for the supported mapping surface.
 
 ### 3. Extract Data
 
@@ -107,6 +107,7 @@ Runs a background watcher daemon to monitor a folder for new incoming files (suc
 | --- | --- |
 | `aiex schema` | Parse JSON Schema files and migrate to SQLite |
 | `aiex schema --generate` | Generate Drizzle schema code only (skip migration) |
+| `aiex schema --force` | Allow a high-risk schema migration after reviewing the migration risk report |
 | `aiex web` | Launch visual schema/configuration UI and data viewer in browser |
 | `aiex extract` | Interactive mode — prompts for schema and file/directory input |
 | `aiex extract -s <name> -f <file>` | Extract structured data from a file and insert into SQLite database |

package/dist/cli.mjs

@@ -1,4 +1,4 @@
-import { A as package_default, C as DEFAULT_PROMPT_CONFIG, D as seedConfig, E as createConfig, O as description, S as DEFAULT_MINERU_CONFIG, T as PLACEHOLDER_TEXT, _ as doctorDiagnosticsSeverityRows, a as recognizeImageText, b as DEFAULT_LITEPARSE_CONFIG, c as t, d as writeAIConfig, f as AIConfigSchema, h as toSnakeCase, j as version, k as name, l as getDefaultAIConfig, m as parseJsonSchema, n as collectDoctorDiagnostics, o as shouldUseImageOcrFallback, p as JsonSchemaDefinitionSchema, r as createMigrationConfig, s as initI18n, t as generateDrizzleSchema, u as readAIConfig, v as doctorDiagnosticsTableRows, w as PLACEHOLDER_SCHEMA, x as DEFAULT_MINERU_API_CONFIG, y as formatDoctorDiagnosticsJson } from "./generate-drizzle-schema-CaSMqQWx.mjs";
+import { A as package_default, C as DEFAULT_PROMPT_CONFIG, D as seedConfig, E as createConfig, O as description, S as DEFAULT_MINERU_CONFIG, T as PLACEHOLDER_TEXT, _ as doctorDiagnosticsSeverityRows, a as recognizeImageText, b as DEFAULT_LITEPARSE_CONFIG, c as t, d as writeAIConfig, f as AIConfigSchema, h as toSnakeCase, j as version, k as name, l as getDefaultAIConfig, m as parseJsonSchema, n as collectDoctorDiagnostics, o as shouldUseImageOcrFallback, p as JsonSchemaDefinitionSchema, r as createMigrationConfig, s as initI18n, t as generateDrizzleSchema, u as readAIConfig, v as doctorDiagnosticsTableRows, w as PLACEHOLDER_SCHEMA, x as DEFAULT_MINERU_API_CONFIG, y as formatDoctorDiagnosticsJson } from "./generate-drizzle-schema-DbJcRtaQ.mjs";
 import { createRequire } from "node:module";
 import fs from "node:fs/promises";
 import os from "node:os";
@@ -1162,16 +1162,16 @@ function getFileHash(filePath) {
 //#endregion
 //#region src/domain/ai-extraction/evidence.ts
 const NUMERIC_RE = /^-?\d+(?:\.\d+)?$/;
-function isRecord$2(value) {
+function isRecord$3(value) {
 	return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 function stripEvidence(data) {
-	if (!isRecord$2(data)) return { data };
+	if (!isRecord$3(data)) return { data };
 	const { _evidence, ...businessData } = data;
-	if (!isRecord$2(_evidence)) return { data: businessData };
+	if (!isRecord$3(_evidence)) return { data: businessData };
 	return {
 		data: businessData,
-		rawEvidence: Object.fromEntries(Object.entries(_evidence).filter(([, value]) => isRecord$2(value)).map(([field, value]) => [field, value]))
+		rawEvidence: Object.fromEntries(Object.entries(_evidence).filter(([, value]) => isRecord$3(value)).map(([field, value]) => [field, value]))
 	};
 }
 function findExactUnique(text$1, quote) {
@@ -1203,7 +1203,7 @@ function quoteContainsValue(quote, value, property) {
 	return false;
 }
 function verifyFieldEvidence(input) {
-	if (!input.text || !isRecord$2(input.data) || !input.rawEvidence) return void 0;
+	if (!input.text || !isRecord$3(input.data) || !input.rawEvidence) return void 0;
 	const verified = {};
 	for (const [field, raw] of Object.entries(input.rawEvidence)) {
 		const property = input.schema.properties[field];
@@ -1437,7 +1437,7 @@ function propertyToExtractionSchema(property) {
 	}
 	return { type: nullableType(property.type) };
 }
-function isRecord$1(value) {
+function isRecord$2(value) {
 	return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 function schemaToExtractionOutputSchema(schema) {
@@ -1475,7 +1475,7 @@ function validatePropertyValue(path$1, property, value, issues) {
 			}
 			return;
 		case "object":
-			if (!isRecord$1(value)) {
+			if (!isRecord$2(value)) {
 				issues.push(`${path$1}: expected object or null`);
 				return;
 			}
@@ -1498,7 +1498,7 @@ function validateProperties(basePath, properties, data, issues) {
 	}
 }
 function validateExtractedData(schema, data) {
-	if (!isRecord$1(data)) return {
+	if (!isRecord$2(data)) return {
 		success: false,
 		error: "Extracted data must be a JSON object."
 	};
@@ -1519,11 +1519,11 @@ const EVIDENCE_INSTRUCTIONS = `Evidence requirements:
 - The quote must be an exact contiguous substring copied from the input text.
 - Do not invent offsets. Only provide quotes.
 - If no exact quote supports a field, omit that field from "_evidence".`;
-function isRecord(value) {
+function isRecord$1(value) {
 	return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 function withEvidenceSchema(schema) {
-	const properties = isRecord(schema.properties) ? schema.properties : {};
+	const properties = isRecord$1(schema.properties) ? schema.properties : {};
 	const required = Array.isArray(schema.required) ? schema.required : [];
 	return {
 		...schema,
@@ -2748,6 +2748,73 @@ function cancel$1(msg) {
 	process.exitCode = 0;
 }
 
+//#endregion
+//#region src/domain/schema/migration-risk.ts
+function keyOf(entry) {
+	return `${entry.table}.${entry.column}`;
+}
+function maxSeverity(items) {
+	if (items.some((item) => item.severity === "high")) return "high";
+	if (items.some((item) => item.severity === "medium")) return "medium";
+	if (items.some((item) => item.severity === "low")) return "low";
+	return "none";
+}
+function addRisk(items, severity, kind, table, column, message) {
+	items.push({
+		severity,
+		kind,
+		table,
+		column,
+		message
+	});
+}
+function enumValues(entry) {
+	return new Set((entry.constraints?.enumValues ?? []).map((value) => JSON.stringify(value)));
+}
+function isEnumNarrowed(previous, next) {
+	const prev = enumValues(previous);
+	const current = enumValues(next);
+	if (prev.size === 0 || current.size === 0 || current.size >= prev.size) return false;
+	return [...current].every((value) => prev.has(value));
+}
+function enumChanged(previous, next) {
+	const prev = enumValues(previous);
+	const current = enumValues(next);
+	if (prev.size === 0 && current.size === 0) return false;
+	if (prev.size !== current.size) return true;
+	return [...prev].some((value) => !current.has(value));
+}
+function analyzeMigrationRisk(previousEntries, nextEntries) {
+	const items = [];
+	const previousByKey = new Map(previousEntries.map((entry) => [keyOf(entry), entry]));
+	const nextByKey = new Map(nextEntries.map((entry) => [keyOf(entry), entry]));
+	const previousTables = new Set(previousEntries.map((entry) => entry.table));
+	const nextTables = new Set(nextEntries.map((entry) => entry.table));
+	for (const table of previousTables) if (!nextTables.has(table)) addRisk(items, "high", "table_removed", table, void 0, `Table "${table}" will be removed.`);
+	for (const table of nextTables) if (!previousTables.has(table)) addRisk(items, "low", "table_added", table, void 0, `Table "${table}" will be added.`);
+	for (const [key, previous] of previousByKey) {
+		const next = nextByKey.get(key);
+		if (!next) {
+			addRisk(items, "high", "column_removed", previous.table, previous.column, `Column "${key}" will be removed.`);
+			continue;
+		}
+		if (previous.sqliteType !== next.sqliteType || previous.drizzleType !== next.drizzleType) addRisk(items, "high", "column_type_changed", previous.table, previous.column, `Column "${key}" type changes from ${previous.drizzleType} to ${next.drizzleType}.`);
+		if (previous.nullable && !next.nullable) addRisk(items, "high", "nullable_tightened", previous.table, previous.column, `Column "${key}" changes from nullable to not null.`);
+		else if (!previous.nullable && next.nullable) addRisk(items, "medium", "nullable_relaxed", previous.table, previous.column, `Column "${key}" changes from not null to nullable.`);
+		if (!previous.unique && next.unique) addRisk(items, "high", "unique_added", previous.table, previous.column, `Column "${key}" adds a unique constraint.`);
+		if (previous.primary !== next.primary) addRisk(items, "high", "primary_changed", previous.table, previous.column, `Column "${key}" primary key status changes.`);
+		if (isEnumNarrowed(previous, next)) addRisk(items, "high", "enum_narrowed", previous.table, previous.column, `Column "${key}" enum values are narrowed.`);
+		else if (enumChanged(previous, next)) addRisk(items, "medium", "enum_changed", previous.table, previous.column, `Column "${key}" enum values change.`);
+	}
+	for (const [key, next] of nextByKey) if (!previousByKey.has(key)) addRisk(items, next.nullable ? "low" : "medium", "column_added", next.table, next.column, `Column "${key}" will be added.`);
+	const level = maxSeverity(items);
+	return {
+		level,
+		items,
+		hasHighRisk: level === "high"
+	};
+}
+
 //#endregion
 //#region src/infrastructure/runtime/package-paths.ts
 const __filename = fileURLToPath(import.meta.url);
@@ -2772,6 +2839,102 @@ function resolveHelperPath() {
 	}
 }
 
+//#endregion
+//#region src/domain/schema/dialect.ts
+const TOP_LEVEL_KEYS = new Set([
+	"$schema",
+	"title",
+	"description",
+	"type",
+	"table",
+	"properties",
+	"required",
+	"examples"
+]);
+const PROPERTY_KEYS = new Set([
+	"description",
+	"type",
+	"format",
+	"pattern",
+	"enum",
+	"primary",
+	"autoIncrement",
+	"unique",
+	"default",
+	"maxLength",
+	"minLength",
+	"minimum",
+	"maximum",
+	"examples",
+	"xPrompt",
+	"drizzle",
+	"nested",
+	"foreignKey",
+	"properties",
+	"items",
+	"required"
+]);
+const DRIZZLE_KEYS = new Set(["mode"]);
+const NESTED_KEYS = new Set(["enabled", "relation"]);
+const FOREIGN_KEY_KEYS = new Set(["table", "column"]);
+const TABLE_KEYS = new Set([
+	"name",
+	"timestamps",
+	"softDelete"
+]);
+const UNSUPPORTED_JSON_SCHEMA_KEYWORDS = new Set([
+	"oneOf",
+	"anyOf",
+	"allOf",
+	"not",
+	"if",
+	"then",
+	"else",
+	"const",
+	"contains",
+	"prefixItems",
+	"additionalItems",
+	"additionalProperties",
+	"patternProperties",
+	"propertyNames",
+	"dependentRequired",
+	"dependentSchemas",
+	"dependencies",
+	"unevaluatedItems",
+	"unevaluatedProperties",
+	"multipleOf",
+	"exclusiveMinimum",
+	"exclusiveMaximum"
+]);
+function isRecord(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function warnUnsupportedKey(warnings, path$1, key) {
+	const reason = UNSUPPORTED_JSON_SCHEMA_KEYWORDS.has(key) ? "is not part of the AIEX Drizzle-backed schema dialect and cannot be mapped reliably" : "is not recognized by the AIEX Drizzle-backed schema dialect";
+	warnings.push(`${path$1}.${key} ${reason}.`);
+}
+function inspectRecordKeys(warnings, value, path$1, allowedKeys) {
+	if (!isRecord(value)) return;
+	for (const key of Object.keys(value)) if (!allowedKeys.has(key)) warnUnsupportedKey(warnings, path$1, key);
+}
+function inspectProperty(warnings, property, path$1) {
+	if (!isRecord(property)) return;
+	inspectRecordKeys(warnings, property, path$1, PROPERTY_KEYS);
+	inspectRecordKeys(warnings, property.drizzle, `${path$1}.drizzle`, DRIZZLE_KEYS);
+	inspectRecordKeys(warnings, property.nested, `${path$1}.nested`, NESTED_KEYS);
+	inspectRecordKeys(warnings, property.foreignKey, `${path$1}.foreignKey`, FOREIGN_KEY_KEYS);
+	if (isRecord(property.properties)) for (const [name$1, child] of Object.entries(property.properties)) inspectProperty(warnings, child, `${path$1}.properties.${name$1}`);
+	if (property.items) inspectProperty(warnings, property.items, `${path$1}.items`);
+}
+function collectDialectWarnings(schema, filePath) {
+	if (!isRecord(schema)) return [];
+	const warnings = [];
+	inspectRecordKeys(warnings, schema, filePath, TOP_LEVEL_KEYS);
+	inspectRecordKeys(warnings, schema.table, `${filePath}.table`, TABLE_KEYS);
+	if (isRecord(schema.properties)) for (const [name$1, property] of Object.entries(schema.properties)) inspectProperty(warnings, property, `${filePath}.properties.${name$1}`);
+	return warnings;
+}
+
 //#endregion
 //#region src/application/schema/parse-all-schemas.ts
 function formatZodError(error, filePath) {
@@ -2782,6 +2945,7 @@ function parseAllSchemas(entries) {
 	const relations = [];
 	const reverseRelations = [];
 	const warnings = [];
+	const mapping = [];
 	for (const { filePath, content } of entries) {
 		let schema;
 		try {
@@ -2793,11 +2957,16 @@ function parseAllSchemas(entries) {
 			};
 		}
 		try {
+			warnings.push(...collectDialectWarnings(schema, filePath));
 			const result = parseJsonSchema(JsonSchemaDefinitionSchema.parse(schema));
 			tables.push(...result.tables);
 			relations.push(...result.relations);
 			reverseRelations.push(...result.reverseRelations);
 			warnings.push(...result.warnings);
+			mapping.push(...(result.mapping ?? []).map((entry) => ({
+				...entry,
+				schemaPath: `${filePath}${entry.schemaPath.slice(1)}`
+			})));
 		} catch (e) {
 			if (e instanceof ZodError) return {
 				success: false,
@@ -2813,6 +2982,7 @@ function parseAllSchemas(entries) {
 			relations,
 			reverseRelations,
 			warnings,
+			mapping,
 			drizzleCode: generateDrizzleSchema({
 				tables,
 				relations,
@@ -2826,6 +2996,24 @@ function parseAllSchemas(entries) {
 //#endregion
 //#region src/application/schema/schema-sync.ts
 const execFileAsync = promisify(execFile);
+const NO_RISK_REPORT = {
+	level: "none",
+	items: [],
+	hasHighRisk: false
+};
+function schemaMapPath(config) {
+	return path.join(path.dirname(config.drizzleSchemaPath), "schema-map.json");
+}
+async function readPreviousSchemaMap(config) {
+	try {
+		const content = await fs.readFile(schemaMapPath(config), "utf-8");
+		const parsed = JSON.parse(content);
+		if (Array.isArray(parsed.baselineEntries)) return parsed.baselineEntries;
+		return Array.isArray(parsed.entries) ? parsed.entries : [];
+	} catch {
+		return [];
+	}
+}
 async function listSchemaFiles(schemaDir) {
 	try {
 		return (await fs.readdir(schemaDir)).filter((f) => f.endsWith(".json")).map((f) => path.join(schemaDir, f)).sort();
@@ -2833,7 +3021,8 @@ async function listSchemaFiles(schemaDir) {
 		return [];
 	}
 }
-async function generateSchemaFromFiles(schemaFiles, config) {
+async function generateSchemaFromFiles(schemaFiles, config, options = {}) {
+	const previousMapping = await readPreviousSchemaMap(config);
 	const result = parseAllSchemas(await Promise.all(schemaFiles.map(async (filePath) => {
 		return {
 			filePath,
@@ -2846,17 +3035,30 @@ async function generateSchemaFromFiles(schemaFiles, config) {
 		warnings: [],
 		schemaCount: schemaFiles.length,
 		tables: 0,
-		relations: 0
+		relations: 0,
+		mappingEntries: 0,
+		riskReport: NO_RISK_REPORT
 	};
-	const { tables, relations, reverseRelations, warnings, drizzleCode } = result.data;
+	const { tables, relations, reverseRelations, warnings, mapping, drizzleCode } = result.data;
+	const riskReport = previousMapping.length > 0 ? analyzeMigrationRisk(previousMapping, mapping) : NO_RISK_REPORT;
 	await fs.mkdir(path.dirname(config.drizzleSchemaPath), { recursive: true });
 	await fs.writeFile(config.drizzleSchemaPath, drizzleCode);
+	await fs.writeFile(schemaMapPath(config), `${JSON.stringify({
+		generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
+		dialect: "aiex-drizzle-sqlite",
+		entries: mapping,
+		baselineEntries: riskReport.hasHighRisk && !options.force ? previousMapping : void 0,
+		warnings,
+		migrationRisk: riskReport
+	}, null, 2)}\n`);
 	return {
 		success: true,
 		warnings,
 		schemaCount: schemaFiles.length,
 		tables: tables.length,
-		relations: relations.length + reverseRelations.length
+		relations: relations.length + reverseRelations.length,
+		mappingEntries: mapping.length,
+		riskReport
 	};
 }
 function parseMigrationOutput(stdout, stderr) {
@@ -2908,24 +3110,34 @@ async function runSchemaSync(config, options = {}) {
 		warnings: [],
 		schemaCount: 0,
 		tables: 0,
-		relations: 0
+		relations: 0,
+		mappingEntries: 0,
+		riskReport: NO_RISK_REPORT
 	};
-	const generated = await generateSchemaFromFiles(schemaFiles, config);
+	const generated = await generateSchemaFromFiles(schemaFiles, config, { force: options.force });
 	if (!generated.success) return {
 		success: false,
 		error: generated.error,
 		warnings: generated.warnings,
 		schemaCount: generated.schemaCount,
 		tables: generated.tables,
-		relations: generated.relations
-	};
-	if (options.generateOnly) return {
-		success: true,
-		warnings: generated.warnings,
-		schemaCount: generated.schemaCount,
-		tables: generated.tables,
-		relations: generated.relations
+		relations: generated.relations,
+		mappingEntries: generated.mappingEntries,
+		riskReport: generated.riskReport
 	};
+	if (options.generateOnly || generated.riskReport.hasHighRisk && !options.force) {
+		const blockedByRisk = generated.riskReport.hasHighRisk && !options.force && !options.generateOnly;
+		return {
+			success: !blockedByRisk,
+			error: blockedByRisk ? t("errors.schema.highRiskMigrationBlocked") : void 0,
+			warnings: generated.warnings,
+			schemaCount: generated.schemaCount,
+			tables: generated.tables,
+			relations: generated.relations,
+			mappingEntries: generated.mappingEntries,
+			riskReport: generated.riskReport
+		};
+	}
 	const migration = await runSchemaMigration(config, options.migrationName);
 	return {
 		success: migration.success,
@@ -2934,6 +3146,8 @@ async function runSchemaSync(config, options = {}) {
 		schemaCount: generated.schemaCount,
 		tables: generated.tables,
 		relations: generated.relations,
+		mappingEntries: generated.mappingEntries,
+		riskReport: generated.riskReport,
 		migration
 	};
 }
@@ -2955,6 +3169,11 @@ const schemaCommand = defineCommand({
 		name: {
 			type: "string",
 			description: t("command.schema.args.name")
+		},
+		force: {
+			type: "boolean",
+			description: t("command.schema.args.force"),
+			default: false
 		}
 	},
 	async run({ args }) {
@@ -2969,7 +3188,7 @@ const schemaCommand = defineCommand({
 		}
 		const s1 = spinner();
 		s1.start(t("command.schema.generating"));
-		const generated = await generateSchemaFromFiles(schemaFiles, config);
+		const generated = await generateSchemaFromFiles(schemaFiles, config, { force: args.force });
 		for (const warning of generated.warnings) consola.warn(warning);
 		if (generated.success) consola.success(t("command.schema.generated", {
 			path: pc.cyan(".aiex/drizzle/schema.ts"),
@@ -2981,10 +3200,21 @@ const schemaCommand = defineCommand({
 			failCommand(t("common.failed"));
 			return;
 		}
+		if (generated.riskReport.items.length > 0) {
+			consola.info(t("command.schema.riskSummary", {
+				level: generated.riskReport.level,
+				count: generated.riskReport.items.length
+			}));
+			for (const item of generated.riskReport.items) consola.warn(`[${item.severity}] ${item.message}`);
+		}
 		if (args.generate) {
 			outro(t("command.schema.runWithoutGenerate"));
 			return;
 		}
+		if (generated.riskReport.hasHighRisk && !args.force) {
+			failCommand(t("command.schema.highRiskBlocked", { flag: pc.cyan("--force") }));
+			return;
+		}
 		const s2 = spinner();
 		s2.start(t("command.schema.runningMigrations"));
 		const migration = await runSchemaMigration(config, args.name);
@@ -16648,12 +16878,13 @@ function schemaRoutes(config) {
 	app.post("/migrate", async (c) => {
 		try {
 			await ensureDir();
-			const result = await runSchemaSync(config);
+			const result = await runSchemaSync(config, { force: c.req.query("force") === "true" });
 			if (!result.success) {
-				const status = result.schemaCount === 0 ? 400 : 500;
+				const status = result.schemaCount === 0 ? 400 : result.riskReport.hasHighRisk ? 409 : 500;
 				return c.json({
 					success: false,
-					error: result.error || t("server.migrationFailed")
+					error: result.error || t("server.migrationFailed"),
+					riskReport: result.riskReport
 				}, status);
 			}
 			return c.json({
@@ -16662,7 +16893,8 @@ function schemaRoutes(config) {
 				tag: result.migration?.tag,
 				tables: result.tables,
 				relations: result.relations,
-				warnings: result.warnings
+				warnings: result.warnings,
+				riskReport: result.riskReport
 			});
 		} catch (error) {
 			return c.json({