aiex-cli 0.0.3-beta.1 → 0.0.3-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { S as version, T as formatDoctorDiagnosticsJson, _ as createConfig, a as parseJsonSchema, b as name, c as getDefaultAIConfig, d as DEFAULT_MARKITDOWN_CONFIG, f as DEFAULT_MINERU_CONFIG, g as AIConfigSchema, h as PLACEHOLDER_TEXT, i as JsonSchemaDefinitionSchema, l as readAIConfig, m as PLACEHOLDER_SCHEMA, n as createMigrationConfig, o as toSnakeCase, p as DEFAULT_PROMPT_CONFIG, s as generateDrizzleSchema, t as collectDoctorDiagnostics, u as writeAIConfig, v as seedConfig, w as doctorDiagnosticsTableRows, x as package_default, y as description } from "./doctor-collector-
|
|
1
|
+
import { S as version, T as formatDoctorDiagnosticsJson, _ as createConfig, a as parseJsonSchema, b as name, c as getDefaultAIConfig, d as DEFAULT_MARKITDOWN_CONFIG, f as DEFAULT_MINERU_CONFIG, g as AIConfigSchema, h as PLACEHOLDER_TEXT, i as JsonSchemaDefinitionSchema, l as readAIConfig, m as PLACEHOLDER_SCHEMA, n as createMigrationConfig, o as toSnakeCase, p as DEFAULT_PROMPT_CONFIG, s as generateDrizzleSchema, t as collectDoctorDiagnostics, u as writeAIConfig, v as seedConfig, w as doctorDiagnosticsTableRows, x as package_default, y as description } from "./doctor-collector-Bnkbl48V.mjs";
|
|
2
2
|
import { createRequire } from "node:module";
|
|
3
3
|
import fs from "node:fs/promises";
|
|
4
4
|
import os from "node:os";
|
|
@@ -13513,6 +13513,73 @@ async function deleteExtractionAuditRecord(aiexDir, id) {
|
|
|
13513
13513
|
return true;
|
|
13514
13514
|
}
|
|
13515
13515
|
|
|
13516
|
+
//#endregion
|
|
13517
|
+
//#region src/core/file-constants.ts
|
|
13518
|
+
const MAX_UPLOAD_SIZE = 150 * 1024 * 1024;
|
|
13519
|
+
const MAX_UPLOAD_SIZE_TEXT = "150MB";
|
|
13520
|
+
const SUPPORTED_FILE_TYPES_TEXT = "images, PDF, text, markdown, CSV, JSON, HTML, XML, YAML";
|
|
13521
|
+
const MISSING_UPLOAD_FILE_TEXT = "Uploaded file is no longer available. Re-run extraction with the original file.";
|
|
13522
|
+
const SUPPORTED_MIME_TYPES = new Set([
|
|
13523
|
+
"image/png",
|
|
13524
|
+
"image/jpeg",
|
|
13525
|
+
"image/gif",
|
|
13526
|
+
"image/webp",
|
|
13527
|
+
"image/bmp",
|
|
13528
|
+
"image/svg+xml",
|
|
13529
|
+
"application/pdf",
|
|
13530
|
+
"text/plain",
|
|
13531
|
+
"text/markdown",
|
|
13532
|
+
"text/csv",
|
|
13533
|
+
"application/json",
|
|
13534
|
+
"text/html",
|
|
13535
|
+
"text/xml",
|
|
13536
|
+
"application/x-yaml",
|
|
13537
|
+
"text/yaml"
|
|
13538
|
+
]);
|
|
13539
|
+
const MIME_TO_EXT = {
|
|
13540
|
+
"image/png": "png",
|
|
13541
|
+
"image/jpeg": "jpg",
|
|
13542
|
+
"image/gif": "gif",
|
|
13543
|
+
"image/webp": "webp",
|
|
13544
|
+
"image/bmp": "bmp",
|
|
13545
|
+
"image/svg+xml": "svg",
|
|
13546
|
+
"application/pdf": "pdf",
|
|
13547
|
+
"text/plain": "txt",
|
|
13548
|
+
"text/markdown": "md",
|
|
13549
|
+
"text/csv": "csv",
|
|
13550
|
+
"application/json": "json",
|
|
13551
|
+
"text/html": "html",
|
|
13552
|
+
"text/xml": "xml",
|
|
13553
|
+
"application/x-yaml": "yaml",
|
|
13554
|
+
"text/yaml": "yaml"
|
|
13555
|
+
};
|
|
13556
|
+
function bytesToMB(bytes) {
|
|
13557
|
+
return bytes / (1024 * 1024);
|
|
13558
|
+
}
|
|
13559
|
+
function getExtensionFromMime(mimeType) {
|
|
13560
|
+
return MIME_TO_EXT[mimeType];
|
|
13561
|
+
}
|
|
13562
|
+
function isAllowedMimeType(mimeType) {
|
|
13563
|
+
return SUPPORTED_MIME_TYPES.has(mimeType);
|
|
13564
|
+
}
|
|
13565
|
+
function unsupportedFileTypeMessage(mimeType) {
|
|
13566
|
+
return `Unsupported file type "${mimeType}". Supported: ${SUPPORTED_FILE_TYPES_TEXT}.`;
|
|
13567
|
+
}
|
|
13568
|
+
function isMissingUploadFileError(error) {
|
|
13569
|
+
return !!error && typeof error === "object" && error.code === "ENOENT";
|
|
13570
|
+
}
|
|
13571
|
+
var FileValidationError = class extends Error {
|
|
13572
|
+
constructor(message) {
|
|
13573
|
+
super(message);
|
|
13574
|
+
this.name = "FileValidationError";
|
|
13575
|
+
}
|
|
13576
|
+
};
|
|
13577
|
+
function validateFileUpload(file) {
|
|
13578
|
+
if (file.size === 0) throw new FileValidationError("Uploaded file is empty");
|
|
13579
|
+
if (file.size > MAX_UPLOAD_SIZE) throw new FileValidationError(`File size (${bytesToMB(file.size).toFixed(1)}MB) exceeds ${MAX_UPLOAD_SIZE_TEXT} limit`);
|
|
13580
|
+
if (!isAllowedMimeType(file.type)) throw new FileValidationError(unsupportedFileTypeMessage(file.type));
|
|
13581
|
+
}
|
|
13582
|
+
|
|
13516
13583
|
//#endregion
|
|
13517
13584
|
//#region src/core/notion-sink.ts
|
|
13518
13585
|
const RICH_TEXT_LIMIT = 2e3;
|
|
@@ -14011,12 +14078,9 @@ async function listSchemas(aiexDir) {
|
|
|
14011
14078
|
return [];
|
|
14012
14079
|
}
|
|
14013
14080
|
}
|
|
14014
|
-
function getFileSizeMB(filePath) {
|
|
14015
|
-
return fs$1.statSync(filePath).size / (1024 * 1024);
|
|
14016
|
-
}
|
|
14017
14081
|
async function readExtractFileInput(filePath, aiConfig) {
|
|
14018
|
-
const
|
|
14019
|
-
if (
|
|
14082
|
+
const stat = fs$1.statSync(filePath);
|
|
14083
|
+
if (stat.size > MAX_UPLOAD_SIZE) throw new Error(`File size (${bytesToMB(stat.size).toFixed(1)}MB) exceeds ${MAX_UPLOAD_SIZE_TEXT} limit: ${filePath}`);
|
|
14020
14084
|
const ext = path.extname(filePath).toLowerCase().replace(".", "");
|
|
14021
14085
|
if (FILE_PART_EXTENSIONS.has(ext)) return {
|
|
14022
14086
|
text: "",
|
|
@@ -14412,6 +14476,10 @@ const retryCommand = defineCommand({
|
|
|
14412
14476
|
}
|
|
14413
14477
|
outro("Done!");
|
|
14414
14478
|
} catch (error) {
|
|
14479
|
+
if (isMissingUploadFileError(error)) {
|
|
14480
|
+
failCommand(MISSING_UPLOAD_FILE_TEXT);
|
|
14481
|
+
return;
|
|
14482
|
+
}
|
|
14415
14483
|
failCommand(error instanceof Error ? error.message : String(error));
|
|
14416
14484
|
}
|
|
14417
14485
|
}
|
|
@@ -14464,7 +14532,7 @@ const extractCommand = defineCommand({
|
|
|
14464
14532
|
file: {
|
|
14465
14533
|
type: "string",
|
|
14466
14534
|
alias: "f",
|
|
14467
|
-
description:
|
|
14535
|
+
description: `File path to extract from. Supported: ${SUPPORTED_FILE_TYPES_TEXT}.`
|
|
14468
14536
|
},
|
|
14469
14537
|
model: {
|
|
14470
14538
|
type: "string",
|
|
@@ -15268,62 +15336,6 @@ function dataRoutes(config) {
|
|
|
15268
15336
|
return app;
|
|
15269
15337
|
}
|
|
15270
15338
|
|
|
15271
|
-
//#endregion
|
|
15272
|
-
//#region src/core/file-constants.ts
|
|
15273
|
-
const MAX_UPLOAD_SIZE = 150 * 1024 * 1024;
|
|
15274
|
-
const MAX_UPLOAD_SIZE_TEXT = "150MB";
|
|
15275
|
-
const SUPPORTED_MIME_TYPES = new Set([
|
|
15276
|
-
"image/png",
|
|
15277
|
-
"image/jpeg",
|
|
15278
|
-
"image/gif",
|
|
15279
|
-
"image/webp",
|
|
15280
|
-
"image/bmp",
|
|
15281
|
-
"image/svg+xml",
|
|
15282
|
-
"application/pdf",
|
|
15283
|
-
"text/plain",
|
|
15284
|
-
"text/markdown",
|
|
15285
|
-
"text/csv",
|
|
15286
|
-
"application/json",
|
|
15287
|
-
"text/html",
|
|
15288
|
-
"text/xml",
|
|
15289
|
-
"application/x-yaml",
|
|
15290
|
-
"text/yaml"
|
|
15291
|
-
]);
|
|
15292
|
-
const MIME_TO_EXT = {
|
|
15293
|
-
"image/png": "png",
|
|
15294
|
-
"image/jpeg": "jpg",
|
|
15295
|
-
"image/gif": "gif",
|
|
15296
|
-
"image/webp": "webp",
|
|
15297
|
-
"image/bmp": "bmp",
|
|
15298
|
-
"image/svg+xml": "svg",
|
|
15299
|
-
"application/pdf": "pdf",
|
|
15300
|
-
"text/plain": "txt",
|
|
15301
|
-
"text/markdown": "md",
|
|
15302
|
-
"text/csv": "csv",
|
|
15303
|
-
"application/json": "json",
|
|
15304
|
-
"text/html": "html",
|
|
15305
|
-
"text/xml": "xml",
|
|
15306
|
-
"application/x-yaml": "yaml",
|
|
15307
|
-
"text/yaml": "yaml"
|
|
15308
|
-
};
|
|
15309
|
-
function getExtensionFromMime(mimeType) {
|
|
15310
|
-
return MIME_TO_EXT[mimeType];
|
|
15311
|
-
}
|
|
15312
|
-
function isAllowedMimeType(mimeType) {
|
|
15313
|
-
return SUPPORTED_MIME_TYPES.has(mimeType);
|
|
15314
|
-
}
|
|
15315
|
-
var FileValidationError = class extends Error {
|
|
15316
|
-
constructor(message) {
|
|
15317
|
-
super(message);
|
|
15318
|
-
this.name = "FileValidationError";
|
|
15319
|
-
}
|
|
15320
|
-
};
|
|
15321
|
-
function validateFileUpload(file) {
|
|
15322
|
-
if (file.size === 0) throw new FileValidationError("Uploaded file is empty");
|
|
15323
|
-
if (file.size > MAX_UPLOAD_SIZE) throw new FileValidationError(`File size (${(file.size / 1024 / 1024).toFixed(1)}MB) exceeds ${MAX_UPLOAD_SIZE_TEXT} limit`);
|
|
15324
|
-
if (!isAllowedMimeType(file.type)) throw new FileValidationError(`Unsupported file type "${file.type}". Supported types: ${[...SUPPORTED_MIME_TYPES].join(", ")}`);
|
|
15325
|
-
}
|
|
15326
|
-
|
|
15327
15339
|
//#endregion
|
|
15328
15340
|
//#region src/server/routes/extract.ts
|
|
15329
15341
|
function getFormString(value) {
|
|
@@ -15340,9 +15352,26 @@ function safeUploadName(name$1) {
|
|
|
15340
15352
|
function safeUploadNameForMime(file) {
|
|
15341
15353
|
const safeName = safeUploadName(file.name);
|
|
15342
15354
|
const ext = getExtensionFromMime(file.type);
|
|
15343
|
-
if (!ext) throw new FileValidationError(
|
|
15355
|
+
if (!ext) throw new FileValidationError(unsupportedFileTypeMessage(file.type));
|
|
15344
15356
|
return `${path.parse(safeName).name || "upload"}.${ext}`;
|
|
15345
15357
|
}
|
|
15358
|
+
function jsonResponse(body, status) {
|
|
15359
|
+
return new Response(JSON.stringify(body), {
|
|
15360
|
+
status,
|
|
15361
|
+
headers: { "content-type": "application/json" }
|
|
15362
|
+
});
|
|
15363
|
+
}
|
|
15364
|
+
async function auditFailureResponse(aiexDir, auditId, error, status) {
|
|
15365
|
+
const record = await updateExtractionAuditRecord(aiexDir, auditId, {
|
|
15366
|
+
status: "failed",
|
|
15367
|
+
error
|
|
15368
|
+
});
|
|
15369
|
+
return jsonResponse({
|
|
15370
|
+
success: false,
|
|
15371
|
+
error: record.error,
|
|
15372
|
+
auditId: record.id
|
|
15373
|
+
}, status);
|
|
15374
|
+
}
|
|
15346
15375
|
async function saveUploadToFile(file, uploadsDir, id) {
|
|
15347
15376
|
validateFileUpload(file);
|
|
15348
15377
|
await fs.mkdir(uploadsDir, { recursive: true });
|
|
@@ -15353,85 +15382,23 @@ async function saveUploadToFile(file, uploadsDir, id) {
|
|
|
15353
15382
|
}
|
|
15354
15383
|
async function executeAuditedExtraction(input) {
|
|
15355
15384
|
const aiConfig = await readAIConfig(input.aiexDir);
|
|
15356
|
-
if (!aiConfig)
|
|
15357
|
-
|
|
15358
|
-
|
|
15359
|
-
error: "AI configuration not found. Configure AI settings first."
|
|
15360
|
-
});
|
|
15361
|
-
return new Response(JSON.stringify({
|
|
15362
|
-
success: false,
|
|
15363
|
-
error: record$1.error,
|
|
15364
|
-
auditId: record$1.id
|
|
15365
|
-
}), {
|
|
15366
|
-
status: 400,
|
|
15367
|
-
headers: { "content-type": "application/json" }
|
|
15368
|
-
});
|
|
15369
|
-
}
|
|
15370
|
-
if (!aiConfig.provider.apiKey) {
|
|
15371
|
-
const record$1 = await updateExtractionAuditRecord(input.aiexDir, input.auditId, {
|
|
15372
|
-
status: "failed",
|
|
15373
|
-
error: "API Key not configured. Configure AI settings first."
|
|
15374
|
-
});
|
|
15375
|
-
return new Response(JSON.stringify({
|
|
15376
|
-
success: false,
|
|
15377
|
-
error: record$1.error,
|
|
15378
|
-
auditId: record$1.id
|
|
15379
|
-
}), {
|
|
15380
|
-
status: 400,
|
|
15381
|
-
headers: { "content-type": "application/json" }
|
|
15382
|
-
});
|
|
15383
|
-
}
|
|
15384
|
-
if (!aiConfig.provider.models?.length) {
|
|
15385
|
-
const record$1 = await updateExtractionAuditRecord(input.aiexDir, input.auditId, {
|
|
15386
|
-
status: "failed",
|
|
15387
|
-
error: "No models configured. Add at least one model in AI Settings."
|
|
15388
|
-
});
|
|
15389
|
-
return new Response(JSON.stringify({
|
|
15390
|
-
success: false,
|
|
15391
|
-
error: record$1.error,
|
|
15392
|
-
auditId: record$1.id
|
|
15393
|
-
}), {
|
|
15394
|
-
status: 400,
|
|
15395
|
-
headers: { "content-type": "application/json" }
|
|
15396
|
-
});
|
|
15397
|
-
}
|
|
15385
|
+
if (!aiConfig) return auditFailureResponse(input.aiexDir, input.auditId, "AI configuration not found. Configure AI settings first.", 400);
|
|
15386
|
+
if (!aiConfig.provider.apiKey) return auditFailureResponse(input.aiexDir, input.auditId, "API Key not configured. Configure AI settings first.", 400);
|
|
15387
|
+
if (!aiConfig.provider.models?.length) return auditFailureResponse(input.aiexDir, input.auditId, "No models configured. Add at least one model in AI Settings.", 400);
|
|
15398
15388
|
const modelOverride = input.modelName ? aiConfig.provider.models.find((model) => model.name === input.modelName) : void 0;
|
|
15399
|
-
if (input.modelName && !modelOverride) {
|
|
15400
|
-
const record$1 = await updateExtractionAuditRecord(input.aiexDir, input.auditId, {
|
|
15401
|
-
status: "failed",
|
|
15402
|
-
error: `Model "${input.modelName}" not found in AI settings`
|
|
15403
|
-
});
|
|
15404
|
-
return new Response(JSON.stringify({
|
|
15405
|
-
success: false,
|
|
15406
|
-
error: record$1.error,
|
|
15407
|
-
auditId: record$1.id
|
|
15408
|
-
}), {
|
|
15409
|
-
status: 400,
|
|
15410
|
-
headers: { "content-type": "application/json" }
|
|
15411
|
-
});
|
|
15412
|
-
}
|
|
15389
|
+
if (input.modelName && !modelOverride) return auditFailureResponse(input.aiexDir, input.auditId, `Model "${input.modelName}" not found in AI settings`, 400);
|
|
15413
15390
|
let inputText = input.text;
|
|
15414
15391
|
let inputFilePath = input.filePath;
|
|
15415
|
-
if (input.filePath) {
|
|
15392
|
+
if (input.filePath) try {
|
|
15416
15393
|
const source = await readExtractFileInput(input.filePath, aiConfig);
|
|
15417
15394
|
inputText = source.text;
|
|
15418
15395
|
inputFilePath = source.filePath;
|
|
15396
|
+
} catch (error) {
|
|
15397
|
+
if (isMissingUploadFileError(error)) return auditFailureResponse(input.aiexDir, input.auditId, MISSING_UPLOAD_FILE_TEXT, 400);
|
|
15398
|
+
throw error;
|
|
15419
15399
|
}
|
|
15420
15400
|
const result = await extractSingle(input.aiexDir, input.config, aiConfig, input.schemaName, inputText, inputFilePath, modelOverride, { quiet: true });
|
|
15421
|
-
if (!result.success)
|
|
15422
|
-
const record$1 = await updateExtractionAuditRecord(input.aiexDir, input.auditId, {
|
|
15423
|
-
status: "failed",
|
|
15424
|
-
error: result.error || "Extraction failed"
|
|
15425
|
-
});
|
|
15426
|
-
return new Response(JSON.stringify({
|
|
15427
|
-
success: false,
|
|
15428
|
-
error: record$1.error,
|
|
15429
|
-
auditId: record$1.id
|
|
15430
|
-
}), {
|
|
15431
|
-
status: 500,
|
|
15432
|
-
headers: { "content-type": "application/json" }
|
|
15433
|
-
});
|
|
15434
|
-
}
|
|
15401
|
+
if (!result.success) return auditFailureResponse(input.aiexDir, input.auditId, result.error || "Extraction failed", 500);
|
|
15435
15402
|
const notionPages = [];
|
|
15436
15403
|
if (aiConfig.notion?.enabled && aiConfig.notion.schemas?.[input.schemaName]?.databaseId?.trim()) try {
|
|
15437
15404
|
if (!result.data || typeof result.data !== "object" || Array.isArray(result.data)) throw new Error("Extraction result is not an object and cannot be written to Notion.");
|
|
@@ -15445,14 +15412,11 @@ async function executeAuditedExtraction(input) {
|
|
|
15445
15412
|
tokensUsed: result.tokensUsed,
|
|
15446
15413
|
error: error instanceof Error ? error.message : String(error)
|
|
15447
15414
|
});
|
|
15448
|
-
return
|
|
15415
|
+
return jsonResponse({
|
|
15449
15416
|
success: false,
|
|
15450
15417
|
error: record$1.error,
|
|
15451
15418
|
auditId: record$1.id
|
|
15452
|
-
}
|
|
15453
|
-
status: 500,
|
|
15454
|
-
headers: { "content-type": "application/json" }
|
|
15455
|
-
});
|
|
15419
|
+
}, 500);
|
|
15456
15420
|
}
|
|
15457
15421
|
const record = await updateExtractionAuditRecord(input.aiexDir, input.auditId, {
|
|
15458
15422
|
status: "succeeded",
|
|
@@ -15462,7 +15426,7 @@ async function executeAuditedExtraction(input) {
|
|
|
15462
15426
|
notionPages: notionPages.length > 0 ? notionPages : void 0,
|
|
15463
15427
|
tokensUsed: result.tokensUsed
|
|
15464
15428
|
});
|
|
15465
|
-
return
|
|
15429
|
+
return jsonResponse({
|
|
15466
15430
|
success: true,
|
|
15467
15431
|
outputPath: record.outputPath,
|
|
15468
15432
|
outputName: record.outputName,
|
|
@@ -15470,10 +15434,7 @@ async function executeAuditedExtraction(input) {
|
|
|
15470
15434
|
notionPages: record.notionPages,
|
|
15471
15435
|
tokensUsed: record.tokensUsed,
|
|
15472
15436
|
auditId: record.id
|
|
15473
|
-
}
|
|
15474
|
-
status: 200,
|
|
15475
|
-
headers: { "content-type": "application/json" }
|
|
15476
|
-
});
|
|
15437
|
+
}, 200);
|
|
15477
15438
|
}
|
|
15478
15439
|
function extractRoutes(config) {
|
|
15479
15440
|
const app = new Hono();
|
|
@@ -15524,7 +15485,8 @@ function extractRoutes(config) {
|
|
|
15524
15485
|
});
|
|
15525
15486
|
return c.json({
|
|
15526
15487
|
success: false,
|
|
15527
|
-
error: e.message
|
|
15488
|
+
error: e.message,
|
|
15489
|
+
auditId: audit.id
|
|
15528
15490
|
}, 400);
|
|
15529
15491
|
}
|
|
15530
15492
|
throw e;
|
|
@@ -65,7 +65,7 @@ function doctorDiagnosticsTableRows(d) {
|
|
|
65
65
|
//#endregion
|
|
66
66
|
//#region package.json
|
|
67
67
|
var name = "aiex-cli";
|
|
68
|
-
var version = "0.0.3-beta.
|
|
68
|
+
var version = "0.0.3-beta.2";
|
|
69
69
|
var description = "JSON Schema → SQLite with AI-powered data extraction";
|
|
70
70
|
var package_default = {
|
|
71
71
|
name,
|
package/dist/index.mjs
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import { C as buildDoctorDiagnostics, T as formatDoctorDiagnosticsJson, a as parseJsonSchema, i as JsonSchemaDefinitionSchema, n as createMigrationConfig, r as generateDrizzleConfig, s as generateDrizzleSchema, t as collectDoctorDiagnostics, w as doctorDiagnosticsTableRows } from "./doctor-collector-
|
|
1
|
+
import { C as buildDoctorDiagnostics, T as formatDoctorDiagnosticsJson, a as parseJsonSchema, i as JsonSchemaDefinitionSchema, n as createMigrationConfig, r as generateDrizzleConfig, s as generateDrizzleSchema, t as collectDoctorDiagnostics, w as doctorDiagnosticsTableRows } from "./doctor-collector-Bnkbl48V.mjs";
|
|
2
2
|
|
|
3
3
|
export { JsonSchemaDefinitionSchema, buildDoctorDiagnostics, collectDoctorDiagnostics, createMigrationConfig, doctorDiagnosticsTableRows, formatDoctorDiagnosticsJson, generateDrizzleConfig, generateDrizzleSchema, parseJsonSchema };
|