aiex-cli 0.0.4-beta.5 → 0.0.5-beta.1

package/dist/cli.mjs

@@ -1,4 +1,4 @@
-import { A as formatDoctorDiagnosticsJson, C as seedConfig, D as version, E as package_default, S as createConfig, T as name, _ as DEFAULT_MINERU_CONFIG, a as parseJsonSchema, b as PLACEHOLDER_TEXT, c as recognizeImageText, d as t, f as getDefaultAIConfig, g as DEFAULT_MARKITDOWN_CONFIG, h as DEFAULT_MARKER_CONFIG, i as JsonSchemaDefinitionSchema, k as doctorDiagnosticsTableRows, l as shouldUseImageOcrFallback, m as writeAIConfig, n as createMigrationConfig, o as toSnakeCase, p as readAIConfig, s as generateDrizzleSchema, t as collectDoctorDiagnostics, u as initI18n, v as DEFAULT_PROMPT_CONFIG, w as description, x as AIConfigSchema, y as PLACEHOLDER_SCHEMA } from "./doctor-collector-BtEPFDoa.mjs";
+import { A as doctorDiagnosticsTableRows, C as createConfig, D as package_default, E as name, O as version, S as AIConfigSchema, T as description, _ as DEFAULT_MINERU_API_CONFIG, a as parseJsonSchema, b as PLACEHOLDER_SCHEMA, c as recognizeImageText, d as t, f as getDefaultAIConfig, g as DEFAULT_MARKITDOWN_CONFIG, h as DEFAULT_MARKER_CONFIG, i as JsonSchemaDefinitionSchema, j as formatDoctorDiagnosticsJson, l as shouldUseImageOcrFallback, m as writeAIConfig, n as createMigrationConfig, o as toSnakeCase, p as readAIConfig, s as generateDrizzleSchema, t as collectDoctorDiagnostics, u as initI18n, v as DEFAULT_MINERU_CONFIG, w as seedConfig, x as PLACEHOLDER_TEXT, y as DEFAULT_PROMPT_CONFIG } from "./doctor-collector-GXgIY3DH.mjs";
 import { createRequire } from "node:module";
 import fs from "node:fs/promises";
 import os from "node:os";
@@ -29,6 +29,7 @@ import { Client, extractNotionId } from "@notionhq/client";
 import { execa } from "execa";
 import { glob, globSync } from "tinyglobby";
 import { extractText, getDocumentProxy, getMeta } from "unpdf";
+import AdmZip from "adm-zip";
 import { execFile } from "node:child_process";
 import { promisify } from "node:util";
 import * as chokidar from "chokidar";
@@ -14099,14 +14100,13 @@ var ExternalCommandPdfConverter = class {
 				pageCount,
 				metadata: {
 					converter: this.name,
-					outputPath,
-					...this.config.keepOutput ? { outputDir } : {}
+					outputPath
 				}
 			};
 		} catch (error) {
 			throw formatCommandError(error, `${this.config.command} ${args.join(" ")}`);
 		} finally {
-			if (!this.config.keepOutput) await fs.rm(tempRoot, {
+			await fs.rm(tempRoot, {
 				recursive: true,
 				force: true
 			}).catch(() => {});
@@ -14114,6 +14114,113 @@ var ExternalCommandPdfConverter = class {
 	}
 };
 
+//#endregion
+//#region src/core/pdf-converter/mineru-api.ts
+const TRAILING_SLASH_REGEXP = /\/+$/;
+var MineruApiPdfConverter = class {
+	name = "mineru_api";
+	constructor(config) {
+		this.config = config;
+	}
+	async convert(input, filePath) {
+		const token = this.config.token?.trim();
+		if (!token) throw new Error(t("errors.pdf.mineruApiTokenRequired"));
+		const baseURL = (this.config.baseURL || "https://mineru.net/api/v4").replace(TRAILING_SLASH_REGEXP, "");
+		const modelVersion = this.config.modelVersion || "vlm";
+		const isOcr = this.config.isOcr ?? true;
+		const enableFormula = this.config.enableFormula ?? true;
+		const enableTable = this.config.enableTable ?? true;
+		const fileName = filePath ? path.basename(filePath) : "document.pdf";
+		consola.info("Requesting Mineru upload URL...");
+		const requestUrl = `${baseURL}/file-urls/batch`;
+		const requestPayload = {
+			files: [{
+				name: fileName,
+				data_id: `aiex_${Date.now()}`
+			}],
+			model_version: modelVersion,
+			is_ocr: isOcr,
+			enable_formula: enableFormula,
+			enable_table: enableTable
+		};
+		const response = await fetch(requestUrl, {
+			method: "POST",
+			headers: {
+				"Authorization": `Bearer ${token}`,
+				"Content-Type": "application/json"
+			},
+			body: JSON.stringify(requestPayload)
+		});
+		if (!response.ok) {
+			const text$1 = await response.text().catch(() => "");
+			throw new Error(`Failed to request upload URL: ${response.status} ${response.statusText} ${text$1}`);
+		}
+		const resJson = await response.json();
+		if (resJson.code !== 0) throw new Error(`Mineru API error (file-urls/batch): ${resJson.msg || JSON.stringify(resJson)}`);
+		const batchId = resJson.data?.batch_id;
+		let uploadUrl = "";
+		if (resJson.data?.file_urls && resJson.data.file_urls.length > 0) uploadUrl = resJson.data.file_urls[0];
+		else if (resJson.data?.file_upload_urls && resJson.data.file_upload_urls.length > 0) uploadUrl = resJson.data.file_upload_urls[0].upload_url;
+		if (!uploadUrl || !batchId) throw new Error(`Mineru API did not return upload URLs or batch ID: ${JSON.stringify(resJson)}`);
+		consola.info(`Uploading file to Mineru storage (${(input.byteLength / 1024 / 1024).toFixed(2)} MB)...`);
+		const uploadResponse = await fetch(uploadUrl, {
+			method: "PUT",
+			body: input
+		});
+		if (!uploadResponse.ok) {
+			const text$1 = await uploadResponse.text().catch(() => "");
+			throw new Error(`Failed to upload file to OSS: ${uploadResponse.status} ${uploadResponse.statusText} ${text$1}`);
+		}
+		consola.info(`Mineru task started, polling results (batch_id: ${batchId})...`);
+		const statusUrl = `${baseURL}/extract-results/batch/${batchId}`;
+		const maxPollAttempts = 120;
+		const pollIntervalMs = process.env.NODE_ENV === "test" ? 1 : 5e3;
+		let attempts = 0;
+		let zipUrl = "";
+		let totalPages = 1;
+		while (attempts < maxPollAttempts) {
+			attempts++;
+			await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
+			const pollResponse = await fetch(statusUrl, {
+				method: "GET",
+				headers: { Authorization: `Bearer ${token}` }
+			});
+			if (!pollResponse.ok) {
+				consola.warn(`Poll request failed: ${pollResponse.statusText}. Retrying...`);
+				continue;
+			}
+			const pollJson = await pollResponse.json();
+			if (pollJson.code !== 0) throw new Error(`Mineru API poll error: ${pollJson.msg || JSON.stringify(pollJson)}`);
+			const extractResultList = pollJson.data?.extract_result;
+			if (!extractResultList || !extractResultList.length) throw new Error(`Mineru API did not return extraction results: ${JSON.stringify(pollJson)}`);
+			const result = extractResultList[0];
+			const state = result.state;
+			consola.info(`Mineru parsing state: ${state} (attempt ${attempts}/${maxPollAttempts})`);
+			if (state === "done") {
+				zipUrl = result.full_zip_url;
+				if (result.extract_progress?.total_pages) totalPages = result.extract_progress.total_pages;
+				break;
+			}
+			if (state === "failed") throw new Error(`Mineru extraction failed: ${result.err_msg || "Unknown error"}`);
+		}
+		if (!zipUrl) throw new Error(`Mineru extraction timed out after ${maxPollAttempts * pollIntervalMs / 1e3} seconds`);
+		consola.info("Downloading result ZIP from Mineru...");
+		const zipResponse = await fetch(zipUrl);
+		if (!zipResponse.ok) throw new Error(`Failed to download result zip: ${zipResponse.statusText}`);
+		const arrayBuffer = await zipResponse.arrayBuffer();
+		const zipBuffer = Buffer.from(arrayBuffer);
+		consola.info("Extracting Markdown content...");
+		const entries = new AdmZip(zipBuffer).getEntries();
+		let mdEntry = entries.find((e) => e.entryName === "full.md" || e.entryName.endsWith("full.md"));
+		if (!mdEntry) mdEntry = entries.find((e) => e.entryName.endsWith(".md"));
+		if (!mdEntry) throw new Error("Could not find any Markdown (.md) file inside the Mineru result zip");
+		return {
+			text: mdEntry.getData().toString("utf8"),
+			pageCount: totalPages
+		};
+	}
+};
+
 //#endregion
 //#region src/core/pdf-converter/unpdf.ts
 var UnpdfConverter = class {
@@ -14172,6 +14279,7 @@ function createPdfConverter(config) {
 			const mineruConfig = config.mineru ?? DEFAULT_MINERU_CONFIG;
 			return withFallback(new ExternalCommandPdfConverter("mineru", mineruConfig), mineruConfig);
 		}
+		if (config.converter === "mineru_api") return new MineruApiPdfConverter(config.mineruApi ?? DEFAULT_MINERU_API_CONFIG);
 		if (config.converter === "markitdown") {
 			const markitdownConfig = config.markitdown ?? DEFAULT_MARKITDOWN_CONFIG;
 			return withFallback(new ExternalCommandPdfConverter("markitdown", markitdownConfig), markitdownConfig);
@@ -14182,7 +14290,7 @@ function createPdfConverter(config) {
 		}
 		if (config.converter === "external") {
 			if (!config.external) throw new Error(t("errors.pdf.externalNotConfigured"));
-			return withFallback(new ExternalCommandPdfConverter("external", config.external), config.external);
+			return new ExternalCommandPdfConverter("external", config.external);
 		}
 	}
 	const key = typeof config === "string" ? config : "unpdf";

package/dist/{doctor-collector-BtEPFDoa.mjs → doctor-collector-GXgIY3DH.mjs}

@@ -74,7 +74,7 @@ function doctorDiagnosticsTableRows(d) {
 //#endregion
 //#region package.json
 var name = "aiex-cli";
-var version = "0.0.4-beta.5";
+var version = "0.0.5-beta.1";
 var description = "JSON Schema → SQLite with AI-powered data extraction";
 var package_default = {
 	name,
@@ -141,6 +141,7 @@ var package_default = {
 		"@langfuse/otel": "catalog:",
 		"@notionhq/client": "catalog:",
 		"@opentelemetry/sdk-trace-node": "catalog:",
+		"adm-zip": "catalog:",
 		"ai": "catalog:",
 		"better-sqlite3": "catalog:",
 		"chokidar": "catalog:",
@@ -176,6 +177,7 @@ var package_default = {
 	devDependencies: {
 		"@antfu/eslint-config": "catalog:cli",
 		"@antfu/ni": "catalog:cli",
+		"@types/adm-zip": "catalog:",
 		"@types/better-sqlite3": "catalog:types",
 		"@types/jsonfile": "catalog:",
 		"@types/node": "catalog:types",
@@ -238,21 +240,30 @@ const ImageOcrConfigSchema = z.object({
 });
 const ExternalPdfConverterConfigSchema = z.object({
 	command: z.string().min(1),
-	args: z.array(z.string()),
+	args: z.array(z.string()).min(1).refine((args) => args.some((arg) => arg.includes("{input}")), { message: "args must contain {input} template variable" }),
 	outputFile: z.string().min(1).optional(),
 	timeout: z.number().int().positive().default(600).optional(),
-	fallbackToUnpdf: z.boolean().optional(),
-	keepOutput: z.boolean().optional()
+	fallbackToUnpdf: z.boolean().optional()
+});
+const MineruApiPdfConverterConfigSchema = z.object({
+	token: z.string(),
+	baseURL: z.string().url().optional(),
+	modelVersion: z.string().optional(),
+	isOcr: z.boolean().optional(),
+	enableFormula: z.boolean().optional(),
+	enableTable: z.boolean().optional()
 });
 const PdfConfigSchema = z.object({
 	converter: z.enum([
 		"unpdf",
 		"mineru",
+		"mineru_api",
 		"markitdown",
 		"marker",
 		"external"
 	]),
 	mineru: ExternalPdfConverterConfigSchema.optional(),
+	mineruApi: MineruApiPdfConverterConfigSchema.optional(),
 	markitdown: ExternalPdfConverterConfigSchema.optional(),
 	marker: ExternalPdfConverterConfigSchema.optional(),
 	external: ExternalPdfConverterConfigSchema.optional()
@@ -339,8 +350,7 @@ const DEFAULT_MINERU_CONFIG = {
 		"{outputDir}"
 	],
 	timeout: 600,
-	fallbackToUnpdf: true,
-	keepOutput: true
+	fallbackToUnpdf: true
 };
 const DEFAULT_MARKITDOWN_CONFIG = {
 	command: "markitdown",
@@ -349,10 +359,8 @@ const DEFAULT_MARKITDOWN_CONFIG = {
 		"-o",
 		"{outputDir}/{basename}.md"
 	],
-	outputFile: "{outputDir}/{basename}.md",
 	timeout: 600,
-	fallbackToUnpdf: true,
-	keepOutput: true
+	fallbackToUnpdf: true
 };
 const DEFAULT_MARKER_CONFIG = {
 	command: "marker_single",
@@ -361,14 +369,21 @@ const DEFAULT_MARKER_CONFIG = {
 		"--output_dir",
 		"{outputDir}"
 	],
-	outputFile: "{outputDir}/{basename}/{basename}.md",
 	timeout: 600,
-	fallbackToUnpdf: true,
-	keepOutput: true
+	fallbackToUnpdf: true
+};
+const DEFAULT_MINERU_API_CONFIG = {
+	token: "",
+	baseURL: "https://mineru.net/api/v4",
+	modelVersion: "vlm",
+	isOcr: true,
+	enableFormula: true,
+	enableTable: true
 };
 const DEFAULT_PDF_CONFIG = {
 	converter: "unpdf",
 	mineru: DEFAULT_MINERU_CONFIG,
+	mineruApi: DEFAULT_MINERU_API_CONFIG,
 	markitdown: DEFAULT_MARKITDOWN_CONFIG,
 	marker: DEFAULT_MARKER_CONFIG
 };
@@ -698,7 +713,8 @@ const en = {
 		},
 		pdf: {
 			externalNotConfigured: "External PDF converter is selected but no external command is configured.",
-			converterRequiresConfig: "PDF converter \"{{name}}\" requires configuration."
+			converterRequiresConfig: "PDF converter \"{{name}}\" requires configuration.",
+			mineruApiTokenRequired: "Mineru API Token is not configured. Please configure it in AI Settings."
 		},
 		extractionAudit: {
 			recordNotFound: "Extraction audit record not found: {{id}}",
@@ -940,7 +956,7 @@ async function initI18n(lng) {
 			fallbackLng: "en",
 			resources: {
 				"en": { translation: en },
-				"zh-CN": { translation: await import("./zh-CN-BZihcLn3.mjs").then((m) => m.zhCN) }
+				"zh-CN": { translation: await import("./zh-CN-Qcn0DHFh.mjs").then((m) => m.zhCN) }
 			},
 			interpolation: { escapeValue: false },
 			returnNull: false
@@ -1520,4 +1536,4 @@ async function collectDoctorDiagnostics(options = {}) {
 }
 
 //#endregion
-export { formatDoctorDiagnosticsJson as A, seedConfig as C, version as D, package_default as E, buildDoctorDiagnostics as O, createConfig as S, name as T, DEFAULT_MINERU_CONFIG as _, parseJsonSchema as a, PLACEHOLDER_TEXT as b, recognizeImageText as c, t as d, getDefaultAIConfig as f, DEFAULT_MARKITDOWN_CONFIG as g, DEFAULT_MARKER_CONFIG as h, JsonSchemaDefinitionSchema as i, doctorDiagnosticsTableRows as k, shouldUseImageOcrFallback as l, writeAIConfig as m, createMigrationConfig as n, toSnakeCase as o, readAIConfig as p, generateDrizzleConfig as r, generateDrizzleSchema as s, collectDoctorDiagnostics as t, initI18n as u, DEFAULT_PROMPT_CONFIG as v, description as w, AIConfigSchema as x, PLACEHOLDER_SCHEMA as y };
+export { doctorDiagnosticsTableRows as A, createConfig as C, package_default as D, name as E, version as O, AIConfigSchema as S, description as T, DEFAULT_MINERU_API_CONFIG as _, parseJsonSchema as a, PLACEHOLDER_SCHEMA as b, recognizeImageText as c, t as d, getDefaultAIConfig as f, DEFAULT_MARKITDOWN_CONFIG as g, DEFAULT_MARKER_CONFIG as h, JsonSchemaDefinitionSchema as i, formatDoctorDiagnosticsJson as j, buildDoctorDiagnostics as k, shouldUseImageOcrFallback as l, writeAIConfig as m, createMigrationConfig as n, toSnakeCase as o, readAIConfig as p, generateDrizzleConfig as r, generateDrizzleSchema as s, collectDoctorDiagnostics as t, initI18n as u, DEFAULT_MINERU_CONFIG as v, seedConfig as w, PLACEHOLDER_TEXT as x, DEFAULT_PROMPT_CONFIG as y };

package/dist/index.mjs

@@ -1,3 +1,3 @@
-import { A as formatDoctorDiagnosticsJson, O as buildDoctorDiagnostics, a as parseJsonSchema, i as JsonSchemaDefinitionSchema, k as doctorDiagnosticsTableRows, n as createMigrationConfig, r as generateDrizzleConfig, s as generateDrizzleSchema, t as collectDoctorDiagnostics } from "./doctor-collector-BtEPFDoa.mjs";
+import { A as doctorDiagnosticsTableRows, a as parseJsonSchema, i as JsonSchemaDefinitionSchema, j as formatDoctorDiagnosticsJson, k as buildDoctorDiagnostics, n as createMigrationConfig, r as generateDrizzleConfig, s as generateDrizzleSchema, t as collectDoctorDiagnostics } from "./doctor-collector-GXgIY3DH.mjs";
 
 export { JsonSchemaDefinitionSchema, buildDoctorDiagnostics, collectDoctorDiagnostics, createMigrationConfig, doctorDiagnosticsTableRows, formatDoctorDiagnosticsJson, generateDrizzleConfig, generateDrizzleSchema, parseJsonSchema };