aiex-cli 0.0.1-beta.28 → 0.0.1-beta.29

package/dist/cli.mjs

@@ -1,6 +1,7 @@
-import { C as formatDoctorDiagnosticsJson, S as doctorDiagnosticsTableRows, _ as description, a as parseJsonSchema, b as version, c as getDefaultAIConfig, d as DEFAULT_PROMPT_CONFIG, f as PLACEHOLDER_SCHEMA, g as seedConfig, h as createConfig, i as JsonSchemaDefinitionSchema, l as readAIConfig, m as AIConfigSchema, n as createMigrationConfig, o as toSnakeCase, p as PLACEHOLDER_TEXT, s as generateDrizzleSchema, t as collectDoctorDiagnostics, u as writeAIConfig, v as name, y as package_default } from "./doctor-collector-D2q6iD_e.mjs";
+import { C as doctorDiagnosticsTableRows, _ as seedConfig, a as parseJsonSchema, b as package_default, c as getDefaultAIConfig, d as DEFAULT_MINERU_CONFIG, f as DEFAULT_PROMPT_CONFIG, g as createConfig, h as AIConfigSchema, i as JsonSchemaDefinitionSchema, l as readAIConfig, m as PLACEHOLDER_TEXT, n as createMigrationConfig, o as toSnakeCase, p as PLACEHOLDER_SCHEMA, s as generateDrizzleSchema, t as collectDoctorDiagnostics, u as writeAIConfig, v as description, w as formatDoctorDiagnosticsJson, x as version, y as name } from "./doctor-collector-j2dG7dG1.mjs";
 import { createRequire } from "node:module";
 import fs from "node:fs/promises";
+import os from "node:os";
 import path from "node:path";
 import process from "node:process";
 import { ZodError } from "zod";
@@ -15,9 +16,11 @@ import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
 import { LangfuseSpanProcessor } from "@langfuse/otel";
 import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
 import { APICallError, Output, generateText, jsonSchema } from "ai";
+import pRetry from "p-retry";
 import fs$1 from "node:fs";
 import Database from "better-sqlite3";
 import picomatch from "picomatch";
+import { execa } from "execa";
 import { Buffer } from "node:buffer";
 import { extractText, getMeta } from "unpdf";
 import { execFile } from "node:child_process";
@@ -12780,23 +12783,25 @@ function lookupModelCapabilities(modelName) {
 //#endregion
 //#region src/utils/retry.ts
 async function withRetry(fn, onRetry, maxRetries = 5) {
-	let lastError;
-	for (let attempt = 0; attempt <= maxRetries; attempt++) try {
-		return await fn();
-	} catch (error) {
-		const err = error instanceof Error ? error : new Error(String(error));
-		lastError = err;
-		if (!(err instanceof APICallError && err.isRetryable && attempt < maxRetries)) throw err;
-		const delayMs = 1e3 * 2 ** attempt + Math.round(Math.random() * 500);
-		onRetry?.({
-			attempt: attempt + 1,
-			maxRetries,
-			delayMs,
-			statusCode: err.statusCode
-		});
-		await new Promise((resolve) => setTimeout(resolve, delayMs));
-	}
-	throw lastError ?? /* @__PURE__ */ new Error("Retry failed after all attempts");
+	return pRetry(async () => fn(), {
+		retries: maxRetries,
+		factor: 2,
+		minTimeout: 1e3,
+		randomize: true,
+		onFailedAttempt({ error, attemptNumber, retriesLeft }) {
+			if (!(error instanceof APICallError) || !error.isRetryable || retriesLeft <= 0) return;
+			const baseDelayMs = 1e3 * 2 ** (attemptNumber - 1);
+			onRetry?.({
+				attempt: attemptNumber,
+				maxRetries,
+				delayMs: baseDelayMs,
+				statusCode: error.statusCode
+			});
+		},
+		shouldRetry({ error }) {
+			return error instanceof APICallError && error.isRetryable;
+		}
+	});
 }
 
 //#endregion
@@ -13373,6 +13378,97 @@ async function savePromptSnapshot(schema, aiexDir) {
 	return outputPath;
 }
 
+//#endregion
+//#region src/core/pdf-converter/external.ts
+function applyTemplate(value, context) {
+	return value.replaceAll("{input}", context.input).replaceAll("{outputDir}", context.outputDir).replaceAll("{basename}", context.basename);
+}
+function isError(error) {
+	return error instanceof Error;
+}
+async function pathExists(filePath) {
+	try {
+		await fs.access(filePath);
+		return true;
+	} catch {
+		return false;
+	}
+}
+async function collectMarkdownFiles(dir) {
+	const entries = await fs.readdir(dir, { withFileTypes: true });
+	const files = [];
+	for (const entry of entries) {
+		const entryPath = path.join(dir, entry.name);
+		if (entry.isDirectory()) {
+			files.push(...await collectMarkdownFiles(entryPath));
+			continue;
+		}
+		if (entry.isFile() && entry.name.toLowerCase().endsWith(".md")) files.push(entryPath);
+	}
+	return files.sort();
+}
+async function selectMarkdownFile(outputDir, basename) {
+	const files = await collectMarkdownFiles(outputDir);
+	if (files.length === 0) throw new Error(`External PDF converter did not produce a markdown file in ${outputDir}`);
+	const preferredName = `${basename}.md`.toLowerCase();
+	return files.find((file) => path.basename(file).toLowerCase() === preferredName) ?? files[0];
+}
+function formatCommandError(error, command$1) {
+	if (!isError(error)) return new Error(String(error));
+	const details = [`External PDF converter failed: ${command$1}`];
+	if ("exitCode" in error && typeof error.exitCode === "number") details.push(`exitCode=${error.exitCode}`);
+	if ("signal" in error && error.signal) details.push(`signal=${String(error.signal)}`);
+	if ("stderr" in error && typeof error.stderr === "string" && error.stderr.trim()) details.push(error.stderr.trim());
+	else if (error.message) details.push(error.message);
+	return new Error(details.join("\n"));
+}
+var ExternalCommandPdfConverter = class {
+	name;
+	constructor(name$1, config) {
+		this.config = config;
+		this.name = name$1;
+	}
+	async convert(input, filePath) {
+		const tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "aiex-pdf-"));
+		const outputDir = path.join(tempRoot, "output");
+		await fs.mkdir(outputDir, { recursive: true });
+		const inputPath = filePath ?? path.join(tempRoot, "input.pdf");
+		if (!filePath) await fs.writeFile(inputPath, input);
+		const basename = path.basename(inputPath, path.extname(inputPath));
+		const context = {
+			input: inputPath,
+			outputDir,
+			basename
+		};
+		const args = this.config.args.map((arg) => applyTemplate(arg, context));
+		const timeoutMs = (this.config.timeout ?? 600) * 1e3;
+		try {
+			await execa(this.config.command, args, {
+				shell: false,
+				timeout: timeoutMs,
+				maxBuffer: 1024 * 1024 * 20
+			});
+			const outputPath = this.config.outputFile ? applyTemplate(this.config.outputFile, context) : await selectMarkdownFile(outputDir, basename);
+			if (!await pathExists(outputPath)) throw new Error(`External PDF converter output was not found: ${outputPath}`);
+			return {
+				text: await fs.readFile(outputPath, "utf-8"),
+				pageCount: 0,
+				metadata: {
+					converter: this.name,
+					outputPath
+				}
+			};
+		} catch (error) {
+			throw formatCommandError(error, `${this.config.command} ${args.join(" ")}`);
+		} finally {
+			await fs.rm(tempRoot, {
+				recursive: true,
+				force: true
+			});
+		}
+	}
+};
+
 //#endregion
 //#region src/core/pdf-converter/unpdf.ts
 var UnpdfConverter = class {
@@ -13391,10 +13487,40 @@ var UnpdfConverter = class {
 //#endregion
 //#region src/core/pdf-converter/factory.ts
 const registry = /* @__PURE__ */ new Map();
-function createPdfConverter(type) {
-	const key = type ?? "unpdf";
+var FallbackPdfConverter = class {
+	name;
+	constructor(primary, fallback) {
+		this.primary = primary;
+		this.fallback = fallback;
+		this.name = primary.name;
+	}
+	async convert(input, filePath) {
+		try {
+			return await this.primary.convert(input, filePath);
+		} catch {
+			return await this.fallback.convert(input, filePath);
+		}
+	}
+};
+function withFallback(converter, config) {
+	if (!config.fallbackToUnpdf) return converter;
+	return new FallbackPdfConverter(converter, new UnpdfConverter());
+}
+function createPdfConverter(config) {
+	if (typeof config === "object") {
+		if (config.converter === "mineru") {
+			const mineruConfig = config.mineru ?? DEFAULT_MINERU_CONFIG;
+			return withFallback(new ExternalCommandPdfConverter("mineru", mineruConfig), mineruConfig);
+		}
+		if (config.converter === "external") {
+			if (!config.external) throw new Error("External PDF converter is selected but no external command is configured.");
+			return withFallback(new ExternalCommandPdfConverter("external", config.external), config.external);
+		}
+	}
+	const key = typeof config === "string" ? config : "unpdf";
 	let instance = registry.get(key);
 	if (!instance) {
+		if (key !== "unpdf") throw new Error(`PDF converter "${key}" requires configuration.`);
 		instance = new UnpdfConverter();
 		registry.set(key, instance);
 	}
@@ -13425,7 +13551,6 @@ const SUPPORTED_EXTENSIONS = new Set([
 	"yml"
 ]);
 const JSON_EXT_RE = /\.json$/;
-const PDF_CONVERTER = createPdfConverter();
 async function ensureDatabaseReady(dbPath, schema) {
 	try {
 		await fs.access(dbPath);
@@ -13490,7 +13615,7 @@ async function listSchemas(aiexDir) {
 		return [];
 	}
 }
-async function readExtractFileInput(filePath) {
+async function readExtractFileInput(filePath, aiConfig) {
 	const ext = path.extname(filePath).toLowerCase().replace(".", "");
 	if (FILE_PART_EXTENSIONS.has(ext)) return {
 		text: "",
@@ -13498,7 +13623,7 @@ async function readExtractFileInput(filePath) {
 	};
 	if (ext === "pdf") {
 		const buffer = await fs.readFile(filePath);
-		const result = await PDF_CONVERTER.convert(buffer);
+		const result = await createPdfConverter(aiConfig?.pdf).convert(buffer, filePath);
 		consola.info(`Extracted ${result.pageCount} page(s) from PDF`);
 		return { text: result.text };
 	}
@@ -13581,7 +13706,7 @@ async function extractSingle(aiexDir, config, aiConfig, schemaName, text$1, file
 }
 async function processOneFile(aiexDir, config, aiConfig, schemaName, filePath, modelOverride) {
 	try {
-		const input = await readExtractFileInput(filePath);
+		const input = await readExtractFileInput(filePath, aiConfig);
 		const r = await extractSingle(aiexDir, config, aiConfig, schemaName, input.text, input.filePath, modelOverride, { quiet: false });
 		if (r.success) {
 			consola.success(`Processed: ${path.basename(filePath)}`);
@@ -13739,7 +13864,7 @@ const extractCommand = defineCommand({
 		let text$1 = "";
 		let filePath;
 		if (args.file) try {
-			const input = await readExtractFileInput(args.file);
+			const input = await readExtractFileInput(args.file, aiConfig);
 			text$1 = input.text;
 			filePath = input.filePath;
 		} catch (e) {
@@ -13820,7 +13945,7 @@ async function runInteractive(aiexDir, config, aiConfig, modelOverride) {
 		}
 		const fp = filePathStr;
 		try {
-			const input = await readExtractFileInput(fp);
+			const input = await readExtractFileInput(fp, aiConfig);
 			return (await extractSingle(aiexDir, config, aiConfig, schemaName, input.text, input.filePath, modelOverride)).success;
 		} catch (e) {
 			consola.error(`Cannot read file: ${fp} — ${e instanceof Error ? e.message : String(e)}`);

package/dist/core/schema-sqlite/migrate-helper.mjs

@@ -5,6 +5,7 @@ import process from "node:process";
 import { fileURLToPath } from "node:url";
 import Database from "better-sqlite3";
 import * as esbuild from "esbuild";
+import lockfile from "proper-lockfile";
 
 //#region src/core/schema-sqlite/migration-name.ts
 function sanitizeMigrationName(name) {
@@ -119,35 +120,22 @@ function applyMigrationWithTransaction(dbPath, sqlStatements) {
 	}
 }
 const LOCK_FILE = ".migrate.lock";
-async function acquireLock(aiexDir) {
-	const lockPath = path.join(aiexDir, LOCK_FILE);
+async function acquireMigrationLock(aiexDir) {
 	await fs.mkdir(aiexDir, { recursive: true });
 	try {
-		const fd = await fs.open(lockPath, "wx");
-		await fd.write(`${process.pid}\n${Date.now()}`);
-		await fd.close();
-	} catch (e) {
-		if (e.code === "EEXIST") try {
-			const [pidStr, timestampStr] = (await fs.readFile(lockPath, "utf-8")).split("\n");
-			const lockPid = Number.parseInt(pidStr, 10);
-			const lockTime = Number.parseInt(timestampStr, 10);
-			const lockAge = Date.now() - lockTime;
-			if (lockAge > 3e5) {
-				await fs.unlink(lockPath);
-				return acquireLock(aiexDir);
-			}
-			throw new Error(`Migration is already running (PID ${lockPid}, started ${Math.round(lockAge / 1e3)}s ago). Wait for it to complete or remove ${lockPath} if stale.`);
-		} catch {
-			await fs.unlink(lockPath).catch(() => {});
-			return acquireLock(aiexDir);
-		}
-		throw e;
+		return await lockfile.lock(aiexDir, {
+			lockfilePath: path.join(aiexDir, LOCK_FILE),
+			realpath: false,
+			stale: 3e5,
+			update: 1e4,
+			retries: 0
+		});
+	} catch (error) {
+		const lockPath = path.join(aiexDir, LOCK_FILE);
+		const message = error instanceof Error ? error.message : String(error);
+		throw new Error(`Migration is already running or the lock could not be acquired. Wait for it to complete or remove ${lockPath} if stale. ${message}`);
 	}
 }
-async function releaseLock(aiexDir) {
-	const lockPath = path.join(aiexDir, LOCK_FILE);
-	await fs.unlink(lockPath).catch(() => {});
-}
 async function main() {
 	const args = process.argv.slice(2);
 	const schemaPath = args[0];
@@ -159,8 +147,7 @@ async function main() {
 		process.exit(1);
 	}
 	try {
-		const aiexDir = path.dirname(path.dirname(migrationsPath));
-		await acquireLock(aiexDir);
+		const releaseLock = await acquireMigrationLock(path.dirname(path.dirname(migrationsPath)));
 		try {
 			const exports = await loadSchemaExports(schemaPath);
 			let dbMissing = false;
@@ -188,7 +175,7 @@ async function main() {
 				tag
 			}));
 		} finally {
-			await releaseLock(aiexDir);
+			await releaseLock();
 		}
 	} catch (error) {
 		const message = error instanceof Error ? error.message : String(error);

package/dist/{doctor-collector-D2q6iD_e.mjs → doctor-collector-j2dG7dG1.mjs}

@@ -64,7 +64,7 @@ function doctorDiagnosticsTableRows(d) {
 //#endregion
 //#region package.json
 var name = "aiex-cli";
-var version = "0.0.1-beta.28";
+var version = "0.0.1-beta.29";
 var description = "JSON Schema → SQLite with AI-powered data extraction";
 var package_default = {
 	name,
@@ -137,9 +137,12 @@ var package_default = {
 		"drizzle-orm": "catalog:",
 		"es-toolkit": "catalog:",
 		"esbuild": "catalog:",
+		"execa": "catalog:",
 		"hono": "catalog:",
+		"p-retry": "catalog:",
 		"picocolors": "catalog:",
 		"picomatch": "catalog:",
+		"proper-lockfile": "catalog:",
 		"tsx": "catalog:cli",
 		"unpdf": "catalog:",
 		"update-notifier": "catalog:",
@@ -151,6 +154,7 @@ var package_default = {
 		"@types/better-sqlite3": "catalog:types",
 		"@types/node": "catalog:types",
 		"@types/picomatch": "catalog:",
+		"@types/proper-lockfile": "catalog:",
 		"@types/update-notifier": "catalog:",
 		"@vitest/coverage-v8": "catalog:testing",
 		"eslint": "catalog:cli",
@@ -198,6 +202,22 @@ const PromptConfigSchema = z.object({
 	userTemplate: z.string().min(1)
 });
 const ExtractionConfigSchema = z.object({ outputDir: z.string().min(1) });
+const ExternalPdfConverterConfigSchema = z.object({
+	command: z.string().min(1),
+	args: z.array(z.string()),
+	outputFile: z.string().min(1).optional(),
+	timeout: z.number().int().positive().default(600).optional(),
+	fallbackToUnpdf: z.boolean().optional()
+});
+const PdfConfigSchema = z.object({
+	converter: z.enum([
+		"unpdf",
+		"mineru",
+		"external"
+	]),
+	mineru: ExternalPdfConverterConfigSchema.optional(),
+	external: ExternalPdfConverterConfigSchema.optional()
+});
 const LangfuseConfigSchema = z.object({
 	publicKey: z.string(),
 	secretKey: z.string(),
@@ -207,6 +227,7 @@ const AIConfigSchema = z.object({
 	provider: AIProviderConfigSchema,
 	prompt: PromptConfigSchema,
 	extraction: ExtractionConfigSchema,
+	pdf: PdfConfigSchema.optional(),
 	langfuse: LangfuseConfigSchema.optional()
 });
 
@@ -247,10 +268,26 @@ Extraction requirements:
 {text}`
 };
 const DEFAULT_EXTRACTION_CONFIG = { outputDir: ".aiex/extracted" };
+const DEFAULT_MINERU_CONFIG = {
+	command: "mineru",
+	args: [
+		"-p",
+		"{input}",
+		"-o",
+		"{outputDir}"
+	],
+	timeout: 600,
+	fallbackToUnpdf: true
+};
+const DEFAULT_PDF_CONFIG = {
+	converter: "unpdf",
+	mineru: DEFAULT_MINERU_CONFIG
+};
 const DEFAULT_AI_CONFIG = {
 	provider: DEFAULT_PROVIDER_CONFIG,
 	prompt: DEFAULT_PROMPT_CONFIG,
-	extraction: DEFAULT_EXTRACTION_CONFIG
+	extraction: DEFAULT_EXTRACTION_CONFIG,
+	pdf: DEFAULT_PDF_CONFIG
 };
 
 //#endregion
@@ -274,7 +311,7 @@ async function writeAIConfig(aiexDir, config) {
 	await addToGitignore(aiexDir, CONFIG_FILE_NAME);
 }
 function getDefaultAIConfig() {
-	return { ...DEFAULT_AI_CONFIG };
+	return structuredClone(DEFAULT_AI_CONFIG);
 }
 async function addToGitignore(aiexDir, fileName) {
 	const projectRoot = path.dirname(aiexDir);
@@ -724,4 +761,4 @@ async function collectDoctorDiagnostics(options = {}) {
 }
 
 //#endregion
-export { formatDoctorDiagnosticsJson as C, doctorDiagnosticsTableRows as S, description as _, parseJsonSchema as a, version as b, getDefaultAIConfig as c, DEFAULT_PROMPT_CONFIG as d, PLACEHOLDER_SCHEMA as f, seedConfig as g, createConfig as h, JsonSchemaDefinitionSchema as i, readAIConfig as l, AIConfigSchema as m, createMigrationConfig as n, toSnakeCase as o, PLACEHOLDER_TEXT as p, generateDrizzleConfig as r, generateDrizzleSchema as s, collectDoctorDiagnostics as t, writeAIConfig as u, name as v, buildDoctorDiagnostics as x, package_default as y };
+export { doctorDiagnosticsTableRows as C, buildDoctorDiagnostics as S, seedConfig as _, parseJsonSchema as a, package_default as b, getDefaultAIConfig as c, DEFAULT_MINERU_CONFIG as d, DEFAULT_PROMPT_CONFIG as f, createConfig as g, AIConfigSchema as h, JsonSchemaDefinitionSchema as i, readAIConfig as l, PLACEHOLDER_TEXT as m, createMigrationConfig as n, toSnakeCase as o, PLACEHOLDER_SCHEMA as p, generateDrizzleConfig as r, generateDrizzleSchema as s, collectDoctorDiagnostics as t, writeAIConfig as u, description as v, formatDoctorDiagnosticsJson as w, version as x, name as y };

package/dist/index.mjs

@@ -1,3 +1,3 @@
-import { C as formatDoctorDiagnosticsJson, S as doctorDiagnosticsTableRows, a as parseJsonSchema, i as JsonSchemaDefinitionSchema, n as createMigrationConfig, r as generateDrizzleConfig, s as generateDrizzleSchema, t as collectDoctorDiagnostics, x as buildDoctorDiagnostics } from "./doctor-collector-D2q6iD_e.mjs";
+import { C as doctorDiagnosticsTableRows, S as buildDoctorDiagnostics, a as parseJsonSchema, i as JsonSchemaDefinitionSchema, n as createMigrationConfig, r as generateDrizzleConfig, s as generateDrizzleSchema, t as collectDoctorDiagnostics, w as formatDoctorDiagnosticsJson } from "./doctor-collector-j2dG7dG1.mjs";
 
 export { JsonSchemaDefinitionSchema, buildDoctorDiagnostics, collectDoctorDiagnostics, createMigrationConfig, doctorDiagnosticsTableRows, formatDoctorDiagnosticsJson, generateDrizzleConfig, generateDrizzleSchema, parseJsonSchema };