aiex-cli 0.0.5-beta.2 → 0.0.5-beta.4

package/README.md

@@ -202,6 +202,17 @@ aiex completion fish | source
 
 <br>
 
+## 📄 Large Document Processing
+
+When processing very large documents (exceeding `40,000` characters), `aiex` runs an optimized **Pipeline Mode** to handle context window limits and control API costs:
+
+- **Sliding Window & Overlapping Slices**: Splits the document logically at Markdown headings or paragraph boundaries. It uses an overlapping sliding window to ensure contextual continuity at slice boundaries. Active heading hierarchies are tracked and prepended to each chunk as context.
+- **Concurrency Limiting**: To respect strict model rate limits, chunk extractions are processed in parallel with a strict concurrency limit (capped at 2 concurrent requests).
+- **Pre-filtering**: Integrates hybrid search-based pre-filtering to score and select only the most relevant document chunks based on schema queries, preventing unnecessary token usage on unrelated sections.
+- **Recursive Merging**: The final extracted JSON objects from each chunk are recursively merged, concatenating lists and deduplicating primitive fields.
+
+<br>
+
 ## 🔧 AI Configuration
 
 aiex works with any OpenAI-compatible API provider. Configure in the Web UI (AI Settings panel):

package/dist/cli.mjs

@@ -1,4 +1,4 @@
-import { A as doctorDiagnosticsTableRows, C as createConfig, D as package_default, E as name, O as version, S as AIConfigSchema, T as description, _ as DEFAULT_MINERU_API_CONFIG, a as parseJsonSchema, b as PLACEHOLDER_SCHEMA, c as recognizeImageText, d as t, f as getDefaultAIConfig, g as DEFAULT_MARKITDOWN_CONFIG, h as DEFAULT_MARKER_CONFIG, i as JsonSchemaDefinitionSchema, j as formatDoctorDiagnosticsJson, l as shouldUseImageOcrFallback, m as writeAIConfig, n as createMigrationConfig, o as toSnakeCase, p as readAIConfig, s as generateDrizzleSchema, t as collectDoctorDiagnostics, u as initI18n, v as DEFAULT_MINERU_CONFIG, w as seedConfig, x as PLACEHOLDER_TEXT, y as DEFAULT_PROMPT_CONFIG } from "./doctor-collector-DZyLrpqA.mjs";
+import { A as doctorDiagnosticsTableRows, C as createConfig, D as package_default, E as name, O as version, S as AIConfigSchema, T as description, _ as DEFAULT_MINERU_API_CONFIG, a as parseJsonSchema, b as PLACEHOLDER_SCHEMA, c as recognizeImageText, d as t, f as getDefaultAIConfig, g as DEFAULT_MARKITDOWN_CONFIG, h as DEFAULT_MARKER_CONFIG, i as JsonSchemaDefinitionSchema, j as formatDoctorDiagnosticsJson, l as shouldUseImageOcrFallback, m as writeAIConfig, n as createMigrationConfig, o as toSnakeCase, p as readAIConfig, s as generateDrizzleSchema, t as collectDoctorDiagnostics, u as initI18n, v as DEFAULT_MINERU_CONFIG, w as seedConfig, x as PLACEHOLDER_TEXT, y as DEFAULT_PROMPT_CONFIG } from "./doctor-collector-Cv7RArla.mjs";
 import { createRequire } from "node:module";
 import fs from "node:fs/promises";
 import os from "node:os";
@@ -13128,7 +13128,7 @@ function propertyToExtractionSchema(property) {
 	}
 	return { type: nullableType(property.type) };
 }
-function isRecord(value) {
+function isRecord$1(value) {
 	return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 function schemaToExtractionOutputSchema(schema) {
@@ -13166,7 +13166,7 @@ function validatePropertyValue(path$1, property, value, issues) {
 			}
 			return;
 		case "object":
-			if (!isRecord(value)) {
+			if (!isRecord$1(value)) {
 				issues.push(`${path$1}: expected object or null`);
 				return;
 			}
@@ -13189,7 +13189,7 @@ function validateProperties(basePath, properties, data, issues) {
 	}
 }
 function validateExtractedData(schema, data) {
-	if (!isRecord(data)) return {
+	if (!isRecord$1(data)) return {
 		success: false,
 		error: "Extracted data must be a JSON object."
 	};
@@ -13512,6 +13512,161 @@ function insertExtractedData(db, schema, data) {
 	}
 }
 
+//#endregion
+//#region src/core/ai-extraction/json-merger.ts
+function isRecord(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function mergePropertyValue(property, values) {
+	const nonNullValues = values.filter((v) => v !== null && v !== void 0);
+	if (nonNullValues.length === 0) return null;
+	if (property.type === "array") {
+		const concatenated = [];
+		for (const val of nonNullValues) if (Array.isArray(val)) concatenated.push(...val);
+		return concatenated;
+	}
+	if (property.type === "object") {
+		const childProperties = property.properties;
+		if (!childProperties) {
+			const mergedObj$1 = {};
+			for (const val of nonNullValues) if (isRecord(val)) Object.assign(mergedObj$1, val);
+			return mergedObj$1;
+		}
+		const mergedObj = {};
+		for (const [propName, propDef] of Object.entries(childProperties)) mergedObj[propName] = mergePropertyValue(propDef, nonNullValues.map((v) => isRecord(v) ? v[propName] : void 0));
+		return mergedObj;
+	}
+	const bestValue = nonNullValues.find((v) => {
+		if (typeof v === "string") return v.trim() !== "";
+		return true;
+	});
+	return bestValue !== void 0 ? bestValue : null;
+}
+/**
+* Merges structured extraction outputs from multiple document chunks
+* according to the schema properties.
+*/
+function mergeExtractionResults(schema, results) {
+	if (results.length === 0) return {};
+	if (results.length === 1) return results[0];
+	const merged = {};
+	for (const [propName, propDef] of Object.entries(schema.properties)) {
+		if (propDef.primary && propDef.autoIncrement) continue;
+		merged[propName] = mergePropertyValue(propDef, results.map((r) => r[propName]));
+	}
+	return merged;
+}
+
+//#endregion
+//#region src/core/ai-extraction/text-splitter.ts
+const HEADING_RE = /^(#{1,6})\s+(\S.*)$/;
+/**
+* Splits a Markdown document into chunks based on header hierarchy.
+* Keeps tables and list blocks intact by splitting along paragraphs (\n\n)
+* when a section exceeds the maxSize limit.
+*/
+function splitMarkdown(text$1, maxSize = 4e4, overlapSize = 0) {
+	const lines = text$1.split("\n");
+	const chunks = [];
+	let currentHeadings = [];
+	let currentChunkLines = [];
+	let currentSize = 0;
+	let hasNewLines = false;
+	const getMetadata = (headings) => {
+		return {
+			h1: headings[0] || void 0,
+			h2: headings[1] || void 0,
+			h3: headings[2] || void 0,
+			h4: headings[3] || void 0
+		};
+	};
+	const flushChunk = (isHeadingChange = false) => {
+		if (currentChunkLines.length === 0 || !hasNewLines) {
+			currentChunkLines = [];
+			currentSize = 0;
+			hasNewLines = false;
+			return;
+		}
+		const pageContent = currentChunkLines.join("\n");
+		let lastChunkContent = "";
+		if (pageContent.length > maxSize) {
+			const paragraphs = pageContent.split("\n\n");
+			let subLines = [];
+			let subSize = 0;
+			for (const para of paragraphs) {
+				const paraSize = para.length;
+				if (subSize + paraSize > maxSize && subLines.length > 0) {
+					const content = subLines.join("\n\n");
+					chunks.push({
+						pageContent: content,
+						metadata: getMetadata(currentHeadings)
+					});
+					const overlapParas = [];
+					let currentOverlapSize = 0;
+					for (let j = subLines.length - 1; j >= 0; j--) {
+						const p = subLines[j];
+						if (currentOverlapSize + p.length > overlapSize && overlapParas.length > 0) break;
+						overlapParas.unshift(p);
+						currentOverlapSize += p.length + 2;
+					}
+					subLines = [...overlapParas];
+					subSize = currentOverlapSize;
+				}
+				subLines.push(para);
+				subSize += paraSize + 2;
+			}
+			if (subLines.length > 0) {
+				const content = subLines.join("\n\n");
+				chunks.push({
+					pageContent: content,
+					metadata: getMetadata(currentHeadings)
+				});
+				lastChunkContent = content;
+			}
+		} else {
+			chunks.push({
+				pageContent,
+				metadata: getMetadata(currentHeadings)
+			});
+			lastChunkContent = pageContent;
+		}
+		if (!isHeadingChange && lastChunkContent && overlapSize > 0) {
+			const paragraphs = lastChunkContent.split("\n\n");
+			const overlapParas = [];
+			let currentOverlapSize = 0;
+			for (let j = paragraphs.length - 1; j >= 0; j--) {
+				const p = paragraphs[j];
+				if (currentOverlapSize + p.length > overlapSize && overlapParas.length > 0) break;
+				overlapParas.unshift(p);
+				currentOverlapSize += p.length + 2;
+			}
+			const overlapText = overlapParas.join("\n\n");
+			currentChunkLines = overlapText.split("\n");
+			currentSize = overlapText.length;
+		} else {
+			currentChunkLines = [];
+			currentSize = 0;
+		}
+		hasNewLines = false;
+	};
+	for (const line of lines) {
+		const headingMatch = line.match(HEADING_RE);
+		if (headingMatch) {
+			flushChunk(true);
+			const depth = headingMatch[1].length;
+			const title = headingMatch[2].trim();
+			currentHeadings = currentHeadings.slice(0, depth - 1);
+			currentHeadings[depth - 1] = title;
+		}
+		currentChunkLines.push(line);
+		currentSize += line.length + 1;
+		hasNewLines = true;
+		if (currentSize > maxSize) flushChunk(false);
+	}
+	flushChunk(true);
+	return chunks;
+}
+
 //#endregion
 //#region src/core/extraction-audit.ts
 const AUDIT_ID_RE = /^[\w.-]+$/;
@@ -14454,6 +14609,42 @@ async function runBatchExtraction(aiexDir, config, aiConfig, schemaName, dir, gl
 //#endregion
 //#region src/core/extract-runner.ts
 const JSON_EXT_RE$1 = /\.json$/;
+async function limitConcurrency(concurrency, items, fn) {
+	const results = Array.from({ length: items.length });
+	let nextIndex = 0;
+	async function worker() {
+		while (nextIndex < items.length) {
+			const currentIndex = nextIndex++;
+			results[currentIndex] = await fn(items[currentIndex], currentIndex);
+		}
+	}
+	const workers = Array.from({ length: Math.min(concurrency, items.length) }, worker);
+	await Promise.all(workers);
+	return results;
+}
+function getSchemaKeywords(schema) {
+	const keywords = /* @__PURE__ */ new Set();
+	function walk(properties) {
+		if (!properties) return;
+		for (const [name$1, prop] of Object.entries(properties)) {
+			keywords.add(name$1.toLowerCase());
+			const parts = name$1.replace(/([a-z0-9])([A-Z])/g, "$1 $2").split(/[\s._:/\\-]+/g);
+			for (const part of parts) if (part.length > 1) keywords.add(part.toLowerCase());
+			if (prop && typeof prop === "object") {
+				const p = prop;
+				if (typeof p.title === "string") keywords.add(p.title.toLowerCase());
+				if (typeof p.description === "string") {
+					const descParts = p.description.toLowerCase().match(/[\p{L}\p{N}_-]+/gu) ?? [];
+					for (const d of descParts) if (d.length > 2) keywords.add(d);
+				}
+				if (p.type === "object") walk(p.properties);
+				if (p.type === "array" && p.items?.type === "object") walk(p.items.properties);
+			}
+		}
+	}
+	walk(schema.properties);
+	return Array.from(keywords);
+}
 async function ensureDatabaseReady(dbPath, schema) {
 	try {
 		await fs.access(dbPath);
@@ -14525,7 +14716,151 @@ async function extractSingle(aiexDir, config, aiConfig, schemaName, text$1, file
 	}
 	const s = spinner();
 	if (!options?.quiet) s.start(filePath ? t("command.extract.file.extractedFrom", { file: path.basename(filePath) }) : t("command.extract.file.extracting"));
-	const result = await extractStructuredData({
+	const CHUNK_LIMIT = 4e4;
+	let result;
+	if (text$1 && text$1.length > CHUNK_LIMIT) {
+		if (!options?.quiet) consola.info(t("command.extract.file.chunking", {
+			length: text$1.length,
+			limit: CHUNK_LIMIT
+		}));
+		const finalDocs = splitMarkdown(text$1, CHUNK_LIMIT, aiConfig.extraction?.overlapSize ?? 2e3);
+		if (!options?.quiet) consola.info(t("command.extract.file.chunksCount", { count: finalDocs.length }));
+		let processedDocs = finalDocs;
+		if (!!aiConfig.extraction?.preFiltering && finalDocs.length > 1) {
+			const preFilteringLimit = aiConfig.extraction?.preFilteringLimit ?? 5;
+			const keywords = getSchemaKeywords(schemaLoad.schema);
+			const scoredChunks = finalDocs.map((doc, idx) => {
+				if (idx === 0) return {
+					index: idx,
+					score: Number.POSITIVE_INFINITY
+				};
+				let score = 0;
+				const docTextLower = doc.pageContent.toLowerCase();
+				for (const kw of keywords) {
+					let pos = docTextLower.indexOf(kw);
+					while (pos !== -1) {
+						score++;
+						pos = docTextLower.indexOf(kw, pos + kw.length);
+					}
+				}
+				return {
+					index: idx,
+					score
+				};
+			}).slice(1).sort((a, b) => b.score - a.score);
+			const selectedIndices = new Set([0]);
+			let keptCount = 0;
+			for (const sc of scoredChunks) if (sc.score > 0 && keptCount < preFilteringLimit) {
+				selectedIndices.add(sc.index);
+				keptCount++;
+			}
+			processedDocs = finalDocs.filter((_, idx) => selectedIndices.has(idx));
+			if (!options?.quiet) consola.info(t("command.extract.file.preFiltering", {
+				original: finalDocs.length,
+				filtered: processedDocs.length
+			}));
+		}
+		const chunkResults = [];
+		const accumulatedTokens = {
+			prompt: 0,
+			completion: 0,
+			total: 0
+		};
+		let success = true;
+		let errorMsg = "";
+		const extractionTasks = processedDocs.map((doc, i) => {
+			return async () => {
+				if (!success) return;
+				const headings = [];
+				if (doc.metadata) {
+					if (doc.metadata.h1) headings.push(doc.metadata.h1);
+					if (doc.metadata.h2) headings.push(doc.metadata.h2);
+					if (doc.metadata.h3) headings.push(doc.metadata.h3);
+					if (doc.metadata.h4) headings.push(doc.metadata.h4);
+				}
+				let chunkText = doc.pageContent;
+				if (headings.length > 0) chunkText = `> **[Context]** Belong to: ${headings.join(" > ")}\n\n${chunkText}`;
+				const chunkResult = await extractStructuredData({
+					config: aiConfig,
+					schema: schemaLoad.schema,
+					text: chunkText,
+					aiexDir,
+					modelOverride,
+					onRetry(info) {
+						if (!options?.quiet) s.message(t("command.extract.file.extractRetryChunk", {
+							current: i + 1,
+							total: processedDocs.length,
+							code: info.statusCode,
+							delay: info.delayMs / 1e3,
+							attempt: info.attempt,
+							max: info.maxRetries
+						}));
+					}
+				});
+				if (!chunkResult.success) {
+					success = false;
+					errorMsg = chunkResult.error || t("common.unknownError");
+					if (!options?.quiet) {
+						s.stop(t("command.extract.file.extractFailChunk", { current: i + 1 }));
+						consola.error(errorMsg);
+					}
+					return;
+				}
+				if (chunkResult.data) chunkResults.push(chunkResult.data);
+				if (chunkResult.tokensUsed) {
+					accumulatedTokens.prompt += chunkResult.tokensUsed.prompt ?? 0;
+					accumulatedTokens.completion += chunkResult.tokensUsed.completion ?? 0;
+					accumulatedTokens.total += chunkResult.tokensUsed.total ?? 0;
+				}
+			};
+		});
+		const concurrency = Math.min(aiConfig.extraction?.concurrency ?? 2, 2);
+		if (!options?.quiet && processedDocs.length > 0) s.message(t("command.extract.file.extractingChunk", {
+			current: 1,
+			total: processedDocs.length
+		}));
+		try {
+			await limitConcurrency(concurrency, extractionTasks, async (task, idx) => {
+				if (!options?.quiet && success) s.message(t("command.extract.file.extractingChunk", {
+					current: idx + 1,
+					total: processedDocs.length
+				}));
+				await task();
+			});
+		} catch (e) {
+			success = false;
+			errorMsg = e instanceof Error ? e.message : String(e);
+		}
+		if (!success) return {
+			success: false,
+			error: errorMsg
+		};
+		const mergedData = mergeExtractionResults(schemaLoad.schema, chunkResults);
+		const validation = validateExtractedData(schemaLoad.schema, mergedData);
+		if (!validation.success) {
+			const valError = validation.error || "Merged data validation failed";
+			if (!options?.quiet) {
+				s.stop(t("command.extract.file.validationFail"));
+				consola.error(valError);
+			}
+			return {
+				success: false,
+				error: valError
+			};
+		}
+		const outputDir = path.resolve(aiexDir, aiConfig.extraction?.outputDir?.replace(".aiex/", "") ?? "extracted");
+		await fs.mkdir(outputDir, { recursive: true });
+		const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+		const outputFileName = `${schemaLoad.schema.table.name}-${timestamp}.json`;
+		const finalMergedOutputPath = path.join(outputDir, outputFileName);
+		await fs.writeFile(finalMergedOutputPath, JSON.stringify(mergedData, null, 2));
+		result = {
+			success: true,
+			data: mergedData,
+			tokensUsed: accumulatedTokens,
+			outputPath: finalMergedOutputPath
+		};
+	} else result = await extractStructuredData({
 		config: aiConfig,
 		schema: schemaLoad.schema,
 		text: text$1 ?? "",
@@ -14553,6 +14888,11 @@ async function extractSingle(aiexDir, config, aiConfig, schemaName, text$1, file
 	}
 	if (!options?.quiet) s.stop(t("command.extract.file.extractComplete"));
 	if (result.outputPath && !options?.quiet) consola.success(t("command.extract.file.resultSaved", { path: pc.cyan(result.outputPath) }));
+	if (result.evidenceSummary && !options?.quiet) {
+		const summary = result.evidenceSummary;
+		const issueText = summary.issueCount > 0 ? pc.yellow(String(summary.issueCount)) : pc.green("0");
+		consola.info(pc.gray(`Evidence coverage: ${summary.evidenceCount}/${summary.fieldCount} fields, found ${summary.foundCount}, inferred ${summary.inferredCount}, missing ${summary.missingCount}, issues ${issueText}`));
+	}
 	if (result.tokensUsed && !options?.quiet) consola.info(pc.gray(t("command.extract.file.tokenUsage", {
 		prompt: result.tokensUsed.prompt,
 		completion: result.tokensUsed.completion,
@@ -14581,6 +14921,7 @@ async function extractSingle(aiexDir, config, aiConfig, schemaName, text$1, file
 						outputPath: result.outputPath,
 						data: result.data,
 						tablesInserted: insertResult.tablesInserted,
+						evidenceSummary: result.evidenceSummary,
 						tokensUsed: result.tokensUsed
 					};
 				} else {
@@ -14607,6 +14948,7 @@ async function extractSingle(aiexDir, config, aiConfig, schemaName, text$1, file
 		success: true,
 		outputPath: result.outputPath,
 		data: result.data,
+		evidenceSummary: result.evidenceSummary,
 		tokensUsed: result.tokensUsed
 	};
 }
@@ -14719,6 +15061,7 @@ async function runAuditedExtraction(options) {
 				outputName: updated.outputName,
 				tablesInserted: updated.tablesInserted,
 				notionPages: updated.notionPages,
+				evidenceSummary: r.evidenceSummary,
 				tokensUsed: updated.tokensUsed,
 				auditId: updated.id,
 				fileHash
@@ -15846,6 +16189,7 @@ function aiRoutes(config) {
 //#endregion
 //#region src/core/data-service.ts
 const FILE_REGEX = /\.json$/;
+const EVIDENCE_FILE_SUFFIX = ".evidence.json";
 const EXTRACTION_TIMESTAMP_RE = /-\d{4}-\d{2}-\d{2}T/;
 const INTERNAL_ROWID_COLUMN = "__aiex_rowid";
 const TIMESTAMP_CLEANUP = /(\d{2})-(\d{2})-(\d{2})/;
@@ -15861,6 +16205,24 @@ function getAuditNotionStatus(record) {
 	if (record.status === "failed") return "failed";
 	return "not_synced";
 }
+async function readEvidenceSummary(extractedDir, outputName) {
+	const evidencePath = path.join(extractedDir, outputName.replace(FILE_REGEX, EVIDENCE_FILE_SUFFIX));
+	try {
+		const coverage = (await readFile(evidencePath))?.coverage;
+		if (!coverage || typeof coverage !== "object") return void 0;
+		return {
+			path: evidencePath,
+			fieldCount: Number(coverage.fieldCount) || 0,
+			evidenceCount: Number(coverage.evidenceCount) || 0,
+			foundCount: Number(coverage.foundCount) || 0,
+			missingCount: Number(coverage.missingCount) || 0,
+			inferredCount: Number(coverage.inferredCount) || 0,
+			issueCount: Number(coverage.issueCount) || 0
+		};
+	} catch {
+		return;
+	}
+}
 async function getRowExtractionActions(aiexDir, tableName) {
 	const actions = /* @__PURE__ */ new Map();
 	const auditRecords = await listExtractionAuditRecords(aiexDir);
@@ -15888,7 +16250,7 @@ async function listExtractions(config) {
 	const aiexDir = path.dirname(config.schemaPath);
 	const extractedDir = path.join(aiexDir, "extracted");
 	await fs.mkdir(extractedDir, { recursive: true });
-	const jsonFiles = (await fs.readdir(extractedDir)).filter((f) => f.endsWith(".json") && !f.endsWith(".prompt.md"));
+	const jsonFiles = (await fs.readdir(extractedDir)).filter((f) => f.endsWith(".json") && !f.endsWith(".prompt.md") && !f.endsWith(EVIDENCE_FILE_SUFFIX));
 	const auditRecords = await listExtractionAuditRecords(aiexDir);
 	const auditByOutputName = new Map(auditRecords.map((record) => [record.outputName, record]));
 	const records = [];
@@ -15907,6 +16269,7 @@ async function listExtractions(config) {
 				timestamp,
 				fileSize: stat.size,
 				modifiedAt: stat.mtime.toISOString(),
+				evidenceSummary: await readEvidenceSummary(extractedDir, file),
 				notionStatus: notionPages ? "synced" : audit?.status === "failed" ? "failed" : "not_synced",
 				notionPages,
 				notionError: !notionPages && audit?.status === "failed" ? audit.error : void 0
@@ -16086,6 +16449,7 @@ async function retryNotionSync(config, fileName) {
 
 //#endregion
 //#region src/server/routes/data.ts
+const JSON_FILE_SUFFIX_RE = /\.json$/;
 const tableParamSchema = z.object({ name: z.string().regex(/^[a-z][a-z0-9_]*$/) });
 const extractionFileParamSchema = z.object({ name: z.string().regex(/^[\w.-]+\.json$/).refine((name$1) => name$1 === path.basename(name$1) && !name$1.includes("..")) });
 const tableQuerySchema = z.object({
@@ -16138,10 +16502,22 @@ function dataRoutes(config) {
 		const filePath = path.join(extractedDir, name$1);
 		try {
 			const content = await fs.readFile(filePath, "utf-8");
+			const evidencePath = path.join(extractedDir, name$1.replace(JSON_FILE_SUFFIX_RE, ".evidence.json"));
+			let evidenceSummary;
+			try {
+				const evidence = JSON.parse(await fs.readFile(evidencePath, "utf-8"));
+				evidenceSummary = evidence?.coverage ? {
+					...evidence.coverage,
+					path: evidencePath
+				} : void 0;
+			} catch {
+				evidenceSummary = void 0;
+			}
 			return c.json({
 				success: true,
 				content,
-				name: name$1
+				name: name$1,
+				evidenceSummary
 			});
 		} catch {
 			return c.json({ error: t("server.extractionNotFound") }, 404);
@@ -16285,6 +16661,7 @@ function extractRoutes(config) {
 				outputName: result.outputName,
 				tablesInserted: result.tablesInserted,
 				notionPages: result.notionPages,
+				evidenceSummary: result.evidenceSummary,
 				tokensUsed: result.tokensUsed,
 				auditId: result.auditId
 			}, 200);
@@ -16352,6 +16729,7 @@ function extractRoutes(config) {
 			outputName: result.outputName,
 			tablesInserted: result.tablesInserted,
 			notionPages: result.notionPages,
+			evidenceSummary: result.evidenceSummary,
 			tokensUsed: result.tokensUsed,
 			auditId: result.auditId
 		}, 200);

package/dist/{doctor-collector-DZyLrpqA.mjs → doctor-collector-Cv7RArla.mjs}

@@ -74,7 +74,7 @@ function doctorDiagnosticsTableRows(d) {
 //#endregion
 //#region package.json
 var name = "aiex-cli";
-var version = "0.0.5-beta.2";
+var version = "0.0.5-beta.4";
 var description = "JSON Schema → SQLite with AI-powered data extraction";
 var package_default = {
 	name,
@@ -228,7 +228,14 @@ const PromptConfigSchema = z.object({
 	systemTemplate: z.string().min(1),
 	userTemplate: z.string().min(1)
 });
-const ExtractionConfigSchema = z.object({ outputDir: z.string().min(1) });
+const ExtractionConfigSchema = z.object({
+	outputDir: z.string().min(1),
+	mode: z.enum(["pipeline"]).default("pipeline").optional(),
+	concurrency: z.number().int().min(1).optional(),
+	overlapSize: z.number().int().nonnegative().optional(),
+	preFiltering: z.boolean().optional(),
+	preFilteringLimit: z.number().int().min(1).optional()
+});
 const ImageOcrConfigSchema = z.object({
 	ocrFallback: z.enum([
 		"auto",
@@ -335,7 +342,10 @@ Extraction requirements:
 	userTemplate: `Please extract data from the following text:
 {text}`
 };
-const DEFAULT_EXTRACTION_CONFIG = { outputDir: ".aiex/extracted" };
+const DEFAULT_EXTRACTION_CONFIG = {
+	outputDir: ".aiex/extracted",
+	mode: "pipeline"
+};
 const DEFAULT_IMAGE_OCR_CONFIG = {
 	ocrFallback: "auto",
 	ocrLanguages: "en-US, zh-Hans",
@@ -567,6 +577,13 @@ const en = {
 				extractFail: "Extraction failed",
 				extractComplete: "Extraction complete",
 				extractRetry: "API responded with {{code}}, retrying in {{delay}}s ({{attempt}}/{{max}})",
+				chunking: "Input text length ({{length}} chars) exceeds limit ({{limit}} chars). Splitting into chunks...",
+				chunksCount: "Split into {{count}} chunk(s).",
+				preFiltering: "Hybrid pre-filtering: selected {{filtered}} out of {{original}} chunks based on schema relevance.",
+				extractingChunk: "Extracting chunk {{current}}/{{total}}...",
+				extractRetryChunk: "Chunk {{current}}/{{total}} API responded with {{code}}, retrying in {{delay}}s ({{attempt}}/{{max}})",
+				extractFailChunk: "Extraction failed for chunk {{current}}/{{total}}",
+				validationFail: "Merged data validation failed",
 				resultSaved: "Result saved: {{path}}",
 				tokenUsage: "Token usage: prompt={{prompt}}, completion={{completion}}, total={{total}}",
 				insertingDb: "Inserting into database...",
@@ -956,7 +973,7 @@ async function initI18n(lng) {
 			fallbackLng: "en",
 			resources: {
 				"en": { translation: en },
-				"zh-CN": { translation: await import("./zh-CN-Qcn0DHFh.mjs").then((m) => m.zhCN) }
+				"zh-CN": { translation: await import("./zh-CN-CyL-61Ow.mjs").then((m) => m.zhCN) }
 			},
 			interpolation: { escapeValue: false },
 			returnNull: false