aiex-cli 0.0.0 → 0.0.1-beta.2

package/README.md

@@ -21,9 +21,10 @@ npm install -g aiex-cli
 ```
 
 ```bash
-cd my-project && aiex schema --init   # set up your schema directory
-aiex schema "*.json"                   # generate SQLite from JSON Schema
-aiex extract -s invoice -f invoice.pdf # extract data with AI
+aiex schema --init                      # set up .aiex/schema/ directory
+aiex schema                             # generate SQLite from JSON Schema files
+aiex extract -s invoice -f invoice.pdf  # extract data with AI
+aiex extract -s invoice -f invoice.pdf --db  # extract and insert into database
 ```
 
 <br>
@@ -47,6 +48,8 @@ aiex schema --init
 
 Creates a `.aiex/` directory with example schemas to get you started.
 
+Add your own JSON Schema files to `.aiex/schema/` (one file per table), then run `aiex schema` to migrate them into the database.
+
 ### 2. Visual Editor
 
 ```bash
@@ -58,7 +61,7 @@ Opens a browser UI where you can visually design and manage your schemas, config
 ### 3. Generate Database
 
 ```bash
-aiex schema "*.json"
+aiex schema
 ```
 
 Converts your JSON Schema files into a SQLite database with full migration support.
@@ -72,11 +75,12 @@ aiex extract -s <schema> -t <text>  # from text
 
 The AI reads your document and outputs structured JSON matching your schema.
 
-**Example:**
+**Examples:**
 ```bash
-aiex extract -s paper -f research.pdf
+aiex extract -s paper -f research.pdf   # save result to .aiex/extracted/
+aiex extract -s paper -f research.pdf --db  # also insert into SQLite database
 ```
-Creates `output.json` with fields like `title`, `firstAuthor`, `journal`, `year` — exactly as defined in your schema.
+Saves the extracted result to `.aiex/extracted/<schema-name>-<timestamp>.json` with fields like `title`, `firstAuthor`, `journal`, `year` — exactly as defined in your schema. Add `--db` to also insert the data directly into the SQLite database.
 
 <br>
 
@@ -85,10 +89,11 @@ Creates `output.json` with fields like `title`, `firstAuthor`, `journal`, `year`
 | Command | Description |
 | --- | --- |
 | `aiex schema --init` | Scaffold `.aiex/` directory with example schemas |
-| `aiex schema <files>` | Parse JSON Schema files and migrate to SQLite |
+| `aiex schema` | Parse JSON Schema files and migrate to SQLite |
 | `aiex schema --generate` | Generate Drizzle schema code only (skip migration) |
 | `aiex web` | Launch visual schema editor in browser |
-| `aiex extract -s <name>` | Extract structured data from documents via AI |
+| `aiex extract -s <name> -f <file>` | Extract structured data from documents via AI |
+| `aiex extract -s <name> -f <file> --db` | Extract and insert into SQLite database |
 | `aiex doctor` | System and configuration diagnostics |
 
 <br>

package/dist/cli.mjs

@@ -1,15 +1,16 @@
-import { C as formatDoctorDiagnosticsJson, S as doctorDiagnosticsTableRows, a as writeAIConfig, b as generateDrizzleSchema, c as PLACEHOLDER_TEXT, d as seedConfig, f as description$1, g as createMigrationConfig, h as version, i as readAIConfig, l as AIConfigSchema, m as package_default, n as getDefaultAIConfig, o as DEFAULT_PROMPT_CONFIG, p as name, r as maskApiKey, s as PLACEHOLDER_SCHEMA, t as collectDoctorDiagnostics, u as createConfig, v as JsonSchemaDefinitionSchema, y as parseJsonSchema } from "./doctor-Wic10PLt.mjs";
+import { C as doctorDiagnosticsTableRows, a as writeAIConfig, b as toSnakeCase, c as PLACEHOLDER_TEXT, d as seedConfig, f as description$1, g as createMigrationConfig, h as version, i as readAIConfig, l as AIConfigSchema, m as package_default, n as getDefaultAIConfig, o as DEFAULT_PROMPT_CONFIG, p as name, r as maskApiKey, s as PLACEHOLDER_SCHEMA, t as collectDoctorDiagnostics, u as createConfig, v as JsonSchemaDefinitionSchema, w as formatDoctorDiagnosticsJson, x as generateDrizzleSchema, y as parseJsonSchema } from "./doctor-Ddn9BjxC.mjs";
 import { createRequire } from "node:module";
 import path from "node:path";
 import process from "node:process";
 import { fileURLToPath } from "node:url";
 import { ZodError } from "zod";
-import fs, { glob } from "node:fs/promises";
+import fs from "node:fs/promises";
 import { defineCommand, runMain } from "citty";
 import updateNotifier from "update-notifier";
 import CliTable3 from "cli-table3";
 import { consola } from "consola";
 import { intro, outro, spinner } from "@clack/prompts";
+import Database from "better-sqlite3";
 import pc from "picocolors";
 import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
 import { Output, generateText, jsonSchema } from "ai";
@@ -9174,6 +9175,143 @@ async function extractStructuredData(input) {
 	}
 }
 
+//#endregion
+//#region src/core/ai-extraction/inserter.ts
+const DRIZZLE_MODE_RE = /mode:\s*'(\w+)'/;
+function extractDrizzleMode(column) {
+	return column.drizzleType.match(DRIZZLE_MODE_RE)?.[1];
+}
+function convertValue(value, column) {
+	if (value === null || value === void 0) return null;
+	const mode = extractDrizzleMode(column);
+	if (mode === "json") return typeof value === "string" ? value : JSON.stringify(value);
+	if (mode === "boolean") return value ? 1 : 0;
+	if (mode === "timestamp" || mode === "timestamp_ms") {
+		if (typeof value === "string") {
+			const ms = Date.parse(value);
+			if (Number.isNaN(ms)) return value;
+			return mode === "timestamp_ms" ? ms : Math.floor(ms / 1e3);
+		}
+		return value;
+	}
+	return value;
+}
+function buildInsertSql(table, data) {
+	const columns = [];
+	const values = [];
+	for (const col of table.columns) {
+		if (col.isAutoIncrement) continue;
+		const value = data[col.name];
+		if (value === void 0) {
+			if (col.defaultValue !== void 0) {
+				columns.push(col.name);
+				values.push(convertValue(JSON.parse(col.defaultValue), col));
+			}
+			continue;
+		}
+		columns.push(col.name);
+		values.push(convertValue(value, col));
+	}
+	const placeholders = values.map(() => "?").join(", ");
+	return {
+		sql: `INSERT INTO ${table.name} (${columns.join(", ")}) VALUES (${placeholders})`,
+		values
+	};
+}
+function insertTableRow({ db, table, data, parentRowId, foreignKeyColumn }) {
+	const rowData = { ...data };
+	if (parentRowId !== void 0 && foreignKeyColumn) rowData[foreignKeyColumn] = parentRowId;
+	const { sql, values } = buildInsertSql(table, rowData);
+	const info = db.prepare(sql).run(...values);
+	return Number(info.lastInsertRowid);
+}
+function parseDataByColumns(data, schema, table) {
+	const result = {};
+	if ("properties" in schema) {
+		const s = schema;
+		for (const [propName, prop] of Object.entries(s.properties)) {
+			if (prop.nested?.enabled) continue;
+			if (prop.type === "array" && prop.items?.nested?.enabled) continue;
+			const colName = toSnakeCase(propName);
+			if (table.columns.some((c) => c.name === colName && c.isAutoIncrement)) continue;
+			if (propName in data) result[colName] = data[propName];
+		}
+	}
+	if (schema.table?.timestamps) {
+		if (!("created_at" in result)) result.created_at = Math.floor(Date.now() / 1e3);
+		if (!("updated_at" in result)) result.updated_at = Math.floor(Date.now() / 1e3);
+	}
+	return result;
+}
+function insertExtractedData(db, schema, data) {
+	const inserted = [];
+	try {
+		const parseResult = parseJsonSchema(schema);
+		const mainTable = parseResult.tables[0];
+		db.transaction(() => {
+			const mainRowId = insertTableRow({
+				db,
+				table: mainTable,
+				data: parseDataByColumns(data, schema, mainTable)
+			});
+			inserted.push({
+				table: mainTable.name,
+				rowId: mainRowId
+			});
+			for (const revRel of parseResult.reverseRelations) {
+				const rel = parseResult.relations.find((r) => r.fromTable === revRel.toTable && r.toTable === revRel.fromTable);
+				if (!rel) continue;
+				const propEntry = Object.entries(schema.properties).find(([key]) => toSnakeCase(key) === revRel.name && key in data);
+				if (!propEntry) continue;
+				const [propName] = propEntry;
+				const nestedValue = data[propName];
+				if (nestedValue === null || nestedValue === void 0) continue;
+				const nestedTable = parseResult.tables.find((t) => t.name === revRel.toTable);
+				if (!nestedTable) continue;
+				if (revRel.type === "has-one") {
+					const rowId = insertTableRow({
+						db,
+						table: nestedTable,
+						data: parseDataByColumns(nestedValue, schema.properties[propName], nestedTable),
+						parentRowId: mainRowId,
+						foreignKeyColumn: rel.fromColumn
+					});
+					inserted.push({
+						table: revRel.toTable,
+						rowId
+					});
+				} else if (revRel.type === "has-many") {
+					const items = nestedValue;
+					for (const item of items) {
+						const rowId = insertTableRow({
+							db,
+							table: nestedTable,
+							data: parseDataByColumns(item, schema.properties[propName].items, nestedTable),
+							parentRowId: mainRowId,
+							foreignKeyColumn: rel.fromColumn
+						});
+						inserted.push({
+							table: revRel.toTable,
+							rowId
+						});
+					}
+				}
+			}
+			return mainRowId;
+		})();
+		return {
+			success: true,
+			tablesInserted: inserted
+		};
+	} catch (e) {
+		return {
+			success: false,
+			tablesInserted: inserted,
+			error: e instanceof Error ? e.message : String(e)
+		};
+	}
+}
+
 //#endregion
 //#region src/core/ai-extraction/snapshot.ts
 async function savePromptSnapshot(schema, aiexDir) {
@@ -9197,6 +9335,25 @@ const IMAGE_EXTENSIONS = new Set([
 	"bmp",
 	"svg"
 ]);
+async function ensureDatabaseReady(dbPath, schema) {
+	try {
+		await fs.access(dbPath);
+	} catch {
+		return `Database not found at ${pc.cyan(".aiex/database.db")}. Run ${pc.cyan("aiex schema")} first to create the database.`;
+	}
+	try {
+		const result = parseJsonSchema(schema);
+		const db = new Database(dbPath);
+		try {
+			for (const table of result.tables) if (!db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`).get(table.name)) return `Table "${table.name}" not found in database. Run ${pc.cyan("aiex schema")} first to create tables.`;
+		} finally {
+			db.close();
+		}
+	} catch (e) {
+		return `Cannot verify database: ${e instanceof Error ? e.message : String(e)}`;
+	}
+	return null;
+}
 const extractCommand = defineCommand({
 	meta: {
 		name: "extract",
@@ -9218,6 +9375,12 @@ const extractCommand = defineCommand({
 			type: "string",
 			alias: "f",
 			description: "File path (text/image/PDF) to extract from"
+		},
+		db: {
+			type: "boolean",
+			alias: "d",
+			description: "Insert extracted data into SQLite database",
+			default: false
 		}
 	},
 	async run({ args }) {
@@ -9302,6 +9465,37 @@ const extractCommand = defineCommand({
 		s.stop("Extraction complete");
 		if (result.outputPath) consola.success(`Result saved: ${pc.cyan(result.outputPath)}`);
 		if (result.tokensUsed) consola.info(pc.gray(`Token usage: prompt=${result.tokensUsed.prompt}, completion=${result.tokensUsed.completion}, total=${result.tokensUsed.total}`));
+		if (args.db && result.data) {
+			const s2 = spinner();
+			s2.start("Inserting into database...");
+			const dbError = await ensureDatabaseReady(config.databasePath, schema);
+			if (dbError) {
+				s2.stop("Database not ready");
+				consola.error(dbError);
+				outro("Failed!");
+				return;
+			}
+			try {
+				const db = new Database(config.databasePath);
+				try {
+					const insertResult = insertExtractedData(db, schema, result.data);
+					if (insertResult.success) s2.stop(`Inserted into ${insertResult.tablesInserted.length} table(s)`);
+					else {
+						s2.stop("Database insert failed");
+						consola.error(insertResult.error || "Unknown error");
+						outro("Failed!");
+						return;
+					}
+				} finally {
+					db.close();
+				}
+			} catch (e) {
+				s2.stop("Database insert failed");
+				consola.error(e instanceof Error ? e.message : String(e));
+				outro("Failed!");
+				return;
+			}
+		}
 		outro("Done!");
 	}
 });
@@ -9309,17 +9503,6 @@ const extractCommand = defineCommand({
 //#endregion
 //#region src/commands/schema.ts
 const execFileAsync$1 = promisify(execFile);
-async function resolveSchemaFiles(pattern) {
-	const cwd = process.cwd();
-	const absPattern = path.isAbsolute(pattern) ? pattern : path.resolve(cwd, pattern);
-	const files = [];
-	try {
-		for await (const entry of glob(absPattern)) files.push(entry);
-	} catch {
-		if (await fs.access(absPattern).then(() => true, () => false)) files.push(absPattern);
-	}
-	return files.filter((f) => f.endsWith(".json"));
-}
 async function generateFromFiles(schemaFiles, config) {
 	const result = parseAllSchemas(await Promise.all(schemaFiles.map(async (filePath) => {
 		return {
@@ -9375,11 +9558,6 @@ const schemaCommand = defineCommand({
 		description: "Sync JSON Schema to SQLite database"
 	},
 	args: {
-		files: {
-			type: "positional",
-			description: "JSON Schema file path(s), supports glob patterns",
-			required: false
-		},
 		init: {
 			type: "boolean",
 			alias: "i",
@@ -9597,18 +9775,18 @@ const schemaCommand = defineCommand({
 			await fs.writeFile(path.join(config.schemaPath, "post.json"), `${JSON.stringify(postSchema, null, 2)}\n`);
 			consola.success(`Initialized ${pc.cyan(".aiex/")} with example schemas`);
 			consola.info("Example includes: User (with preferences has-one), Post (with comments has-many)");
-			outro("Run: aiex schema .aiex/schema/*.json");
+			outro("Run: aiex schema");
 			return;
 		}
-		if (!args.files) {
-			consola.error("Missing required argument: FILES");
-			consola.info("Use --init to initialize with an example schema");
-			outro("Failed!");
-			return;
+		let schemaFiles;
+		try {
+			schemaFiles = await fs.readdir(config.schemaPath);
+			schemaFiles = schemaFiles.filter((f) => f.endsWith(".json")).map((f) => path.join(config.schemaPath, f));
+		} catch {
+			schemaFiles = [];
 		}
-		const schemaFiles = await resolveSchemaFiles(args.files);
 		if (schemaFiles.length === 0) {
-			consola.error(`No schema files found for: ${args.files}`);
+			consola.error(`No schema files found in ${pc.cyan(".aiex/schema/")}`);
 			consola.info("Use --init to initialize with an example schema");
 			outro("Failed!");
 			return;

package/dist/{doctor-Wic10PLt.mjs → doctor-Ddn9BjxC.mjs}

@@ -411,7 +411,7 @@ function generateDrizzleConfig() {
 //#endregion
 //#region package.json
 var name = "aiex-cli";
-var version = "0.0.0";
+var version = "0.0.1-beta.2";
 var description = "JSON Schema → SQLite with AI-powered data extraction";
 var package_default = {
 	name,
@@ -708,4 +708,4 @@ async function collectDoctorDiagnostics(options = {}) {
 }
 
 //#endregion
-export { formatDoctorDiagnosticsJson as C, doctorDiagnosticsTableRows as S, generateDrizzleConfig as _, writeAIConfig as a, generateDrizzleSchema as b, PLACEHOLDER_TEXT as c, seedConfig as d, description as f, createMigrationConfig as g, version as h, readAIConfig as i, AIConfigSchema as l, package_default as m, getDefaultAIConfig as n, DEFAULT_PROMPT_CONFIG as o, name as p, maskApiKey as r, PLACEHOLDER_SCHEMA as s, collectDoctorDiagnostics as t, createConfig as u, JsonSchemaDefinitionSchema as v, buildDoctorDiagnostics as x, parseJsonSchema as y };
+export { doctorDiagnosticsTableRows as C, buildDoctorDiagnostics as S, generateDrizzleConfig as _, writeAIConfig as a, toSnakeCase as b, PLACEHOLDER_TEXT as c, seedConfig as d, description as f, createMigrationConfig as g, version as h, readAIConfig as i, AIConfigSchema as l, package_default as m, getDefaultAIConfig as n, DEFAULT_PROMPT_CONFIG as o, name as p, maskApiKey as r, PLACEHOLDER_SCHEMA as s, collectDoctorDiagnostics as t, createConfig as u, JsonSchemaDefinitionSchema as v, formatDoctorDiagnosticsJson as w, generateDrizzleSchema as x, parseJsonSchema as y };

package/dist/index.mjs

@@ -1,3 +1,3 @@
-import { C as formatDoctorDiagnosticsJson, S as doctorDiagnosticsTableRows, _ as generateDrizzleConfig, b as generateDrizzleSchema, g as createMigrationConfig, t as collectDoctorDiagnostics, v as JsonSchemaDefinitionSchema, x as buildDoctorDiagnostics, y as parseJsonSchema } from "./doctor-Wic10PLt.mjs";
+import { C as doctorDiagnosticsTableRows, S as buildDoctorDiagnostics, _ as generateDrizzleConfig, g as createMigrationConfig, t as collectDoctorDiagnostics, v as JsonSchemaDefinitionSchema, w as formatDoctorDiagnosticsJson, x as generateDrizzleSchema, y as parseJsonSchema } from "./doctor-Ddn9BjxC.mjs";
 
 export { JsonSchemaDefinitionSchema, buildDoctorDiagnostics, collectDoctorDiagnostics, createMigrationConfig, doctorDiagnosticsTableRows, formatDoctorDiagnosticsJson, generateDrizzleConfig, generateDrizzleSchema, parseJsonSchema };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "aiex-cli",
   "type": "module",
-  "version": "0.0.0",
+  "version": "0.0.1-beta.2",
   "description": "JSON Schema → SQLite with AI-powered data extraction",
   "author": "OSpoon <zxin088@gmail.com>",
   "license": "MIT",