aiex-cli 0.0.3 → 0.0.4-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.mjs +1046 -791
- package/dist/{doctor-collector-BVbuw5LY.mjs → doctor-collector-CDg_BdI8.mjs} +556 -10
- package/dist/index.mjs +1 -1
- package/dist/web/assets/AISettings-kEwOynPE.js +272 -0
- package/dist/web/assets/DataBrowser-IlgTMGi0.js +97 -0
- package/dist/web/assets/ExtractionViewer-0F4C26V5.js +1 -0
- package/dist/web/assets/JsonSchemaEditor-Dyl391lX.js +571 -0
- package/dist/web/assets/{api-client-BsgtGnzl.js → api-client-b4ZBXpNH.js} +1 -1
- package/dist/web/assets/{cssMode-CPThwItX.js → cssMode-BM5FOYIl.js} +1 -1
- package/dist/web/assets/dialog-CnZ7jH1l.js +109 -0
- package/dist/web/assets/dist-CElVIpns.js +1 -0
- package/dist/web/assets/{editor.main-BnOkwRFv.js → editor.main-C2Q97Dkk.js} +2 -2
- package/dist/web/assets/{freemarker2-DWDTYVJR.js → freemarker2-BqyJTCTn.js} +1 -1
- package/dist/web/assets/{handlebars-D4DzjGQ7.js → handlebars-DxRJTefg.js} +1 -1
- package/dist/web/assets/{html-DnzhKSoD.js → html-gyvgrapw.js} +1 -1
- package/dist/web/assets/{htmlMode-CR7UKfEH.js → htmlMode-CNjCRwdY.js} +1 -1
- package/dist/web/assets/{index-CPjJbU4i.js → index-C8qUHnqD.js} +38 -38
- package/dist/web/assets/{javascript-D2srszZ8.js → javascript-BK6ufvq6.js} +1 -1
- package/dist/web/assets/{jsonMode-B4jaPYEr.js → jsonMode-m2trGjkO.js} +1 -1
- package/dist/web/assets/{liquid-CIT2Wl_l.js → liquid-BtyuYqQQ.js} +1 -1
- package/dist/web/assets/{mdx-CWLaEOFy.js → mdx-C8K4EvCQ.js} +1 -1
- package/dist/web/assets/{monaco.contribution-DDv5ldfS.js → monaco.contribution-BTr-G8hO.js} +2 -2
- package/dist/web/assets/object-utils-C6FkG7fw.js +1 -0
- package/dist/web/assets/{python-6CGfpCNq.js → python-8dyH1nS_.js} +1 -1
- package/dist/web/assets/{razor-DEMMh3TD.js → razor-DtWMI74k.js} +1 -1
- package/dist/web/assets/textarea-DMpqBhjw.js +522 -0
- package/dist/web/assets/{tsMode-Cm1NtjPs.js → tsMode-Dv8YG-YK.js} +1 -1
- package/dist/web/assets/{typescript-BM9aPEFg.js → typescript-DbClKYS3.js} +1 -1
- package/dist/web/assets/vue-i18n-Du42D0vb.js +931 -0
- package/dist/web/assets/{xml-CoSbvcg5.js → xml-Bb59gjP6.js} +1 -1
- package/dist/web/assets/{yaml-56GOgy8k.js → yaml-DVMb_IfV.js} +1 -1
- package/dist/web/index.html +8 -8
- package/dist/zh-CN-B5QVQVm-.mjs +486 -0
- package/package.json +3 -1
- package/dist/web/assets/AISettings-D6EpB8tt.js +0 -272
- package/dist/web/assets/DataBrowser-N77fBaoa.js +0 -97
- package/dist/web/assets/ExtractionViewer-BSZycwgL.js +0 -1
- package/dist/web/assets/JsonSchemaEditor-DfHs5bc0.js +0 -571
- package/dist/web/assets/button-Cdgr9Igy.js +0 -927
- package/dist/web/assets/dialog-CUkPLPNP.js +0 -109
- package/dist/web/assets/dist-9yHVMqQ0.js +0 -1
- package/dist/web/assets/object-utils-I4gWdSnS.js +0 -1
- package/dist/web/assets/runtime-dom.esm-bundler-ei_N7Xjw.js +0 -1
- package/dist/web/assets/textarea-DEQMRfG8.js +0 -522
- /package/dist/{completions-C3rmTwXZ.mjs → completions-Bh0DOngr.mjs} +0 -0
- /package/dist/web/assets/{abap-Bgec7Keq.js → abap-DiwvWnMr.js} +0 -0
- /package/dist/web/assets/{apex-VBlPwEoQ.js → apex-CmtZjKlf.js} +0 -0
- /package/dist/web/assets/{azcli-DKqrEFBx.js → azcli-DL2My_i-.js} +0 -0
- /package/dist/web/assets/{bat-DdgQWy_0.js → bat-B-nC98wG.js} +0 -0
- /package/dist/web/assets/{bicep-CRMM43EB.js → bicep-Ju5MwOgh.js} +0 -0
- /package/dist/web/assets/{cameligo-UatALtML.js → cameligo-8Eu1TyBr.js} +0 -0
- /package/dist/web/assets/{clojure-D8JU08RA.js → clojure-u-RpMkH3.js} +0 -0
- /package/dist/web/assets/{coffee-C56wu358.js → coffee-CdA7bbTe.js} +0 -0
- /package/dist/web/assets/{cpp-CyZLvhJG.js → cpp-CzNFP8ks.js} +0 -0
- /package/dist/web/assets/{csharp-BJl3ixva.js → csharp-j1LThmcE.js} +0 -0
- /package/dist/web/assets/{csp-CxEKxmO-.js → csp-CLRC61y6.js} +0 -0
- /package/dist/web/assets/{css-B0t_muXd.js → css-r6rC_7P2.js} +0 -0
- /package/dist/web/assets/{cypher-D1hqiMFD.js → cypher-CW08XVUh.js} +0 -0
- /package/dist/web/assets/{dart-Bz550Pyv.js → dart-Cs9aL5T_.js} +0 -0
- /package/dist/web/assets/{dockerfile-CIXgVAuA.js → dockerfile-BWM0M184.js} +0 -0
- /package/dist/web/assets/{ecl-D9qbvZoA.js → ecl-MJJuer5P.js} +0 -0
- /package/dist/web/assets/{editor.api-C8BHpRhn.js → editor.api-nsOUOZde.js} +0 -0
- /package/dist/web/assets/{elixir-b2M38fAy.js → elixir-D2AIuXqn.js} +0 -0
- /package/dist/web/assets/{flow9-Dq1UYMkt.js → flow9-B2H24giC.js} +0 -0
- /package/dist/web/assets/{fsharp-BaeLhgfq.js → fsharp-CFNadkg7.js} +0 -0
- /package/dist/web/assets/{go-Bd-NFKIC.js → go-dSur1iB2.js} +0 -0
- /package/dist/web/assets/{graphql-DZVerJfy.js → graphql-qyhAo11d.js} +0 -0
- /package/dist/web/assets/{hcl-CAVzrZfH.js → hcl-DFzjMyzm.js} +0 -0
- /package/dist/web/assets/{ini-CyXdX58t.js → ini-TdzA8TIl.js} +0 -0
- /package/dist/web/assets/{java-B5pNgvhy.js → java-CSGA9pkE.js} +0 -0
- /package/dist/web/assets/{julia-XRhmV3AN.js → julia-9izz5OsY.js} +0 -0
- /package/dist/web/assets/{kotlin-DOd3J5vr.js → kotlin-DuPK7AtF.js} +0 -0
- /package/dist/web/assets/{less-veZSnyw6.js → less-B8d93iCg.js} +0 -0
- /package/dist/web/assets/{lexon-QWGkuK0H.js → lexon-DWtEIyu7.js} +0 -0
- /package/dist/web/assets/{lua-CYGpjuO5.js → lua-Ciq0OGgt.js} +0 -0
- /package/dist/web/assets/{m3-yNnrZkdc.js → m3-Cki6JWj_.js} +0 -0
- /package/dist/web/assets/{markdown-BCSWEPSX.js → markdown-Cu47xwU0.js} +0 -0
- /package/dist/web/assets/{mips-OpYmcC30.js → mips-BM8ui995.js} +0 -0
- /package/dist/web/assets/{msdax-2oxoTO9Z.js → msdax-DqLio0_c.js} +0 -0
- /package/dist/web/assets/{mysql-5KlC-K_9.js → mysql-v1wbjJOq.js} +0 -0
- /package/dist/web/assets/{objective-c-CcDCgtLx.js → objective-c-CQl3PGSB.js} +0 -0
- /package/dist/web/assets/{pascal-BZGsbaEV.js → pascal-D4iW0ZtD.js} +0 -0
- /package/dist/web/assets/{pascaligo-DtD5qU3G.js → pascaligo-BdC9CZdj.js} +0 -0
- /package/dist/web/assets/{perl-C1jNNS3E.js → perl-BL10m4XD.js} +0 -0
- /package/dist/web/assets/{pgsql-CT0fhiZa.js → pgsql-Be_oqVo3.js} +0 -0
- /package/dist/web/assets/{php-D6DrXoPM.js → php-BtvXSFRI.js} +0 -0
- /package/dist/web/assets/{pla-b3-HN2pF.js → pla-B2vUy15C.js} +0 -0
- /package/dist/web/assets/{postiats-Bin2ApVS.js → postiats-CbmTTfXr.js} +0 -0
- /package/dist/web/assets/{powerquery-7ASnn-ZG.js → powerquery-DszLhJGx.js} +0 -0
- /package/dist/web/assets/{powershell-t4p7sU1H.js → powershell-B0dYktF6.js} +0 -0
- /package/dist/web/assets/{preload-helper-Dd-HcVz_.js → preload-helper-DWTEM3RW.js} +0 -0
- /package/dist/web/assets/{protobuf-BUGeWa_j.js → protobuf-CZvaj1VX.js} +0 -0
- /package/dist/web/assets/{pug-BuKcgC9s.js → pug-CPDx1B3S.js} +0 -0
- /package/dist/web/assets/{qsharp-DxLLX8mo.js → qsharp-CAxMZVjw.js} +0 -0
- /package/dist/web/assets/{r-DMlFgn7A.js → r-8DbbFX2l.js} +0 -0
- /package/dist/web/assets/{redis-cXItkC5u.js → redis-DRWj9MtJ.js} +0 -0
- /package/dist/web/assets/{redshift-BZVbW7HE.js → redshift-C6cElE_5.js} +0 -0
- /package/dist/web/assets/{restructuredtext-BzjxwS8h.js → restructuredtext-W9pS9n3m.js} +0 -0
- /package/dist/web/assets/{ruby-C5nyLV4l.js → ruby-BKnzWnk-.js} +0 -0
- /package/dist/web/assets/{rust-BcmMsHdf.js → rust-YPCclWwe.js} +0 -0
- /package/dist/web/assets/{sb-Dnb1iy6B.js → sb-BgM4DTFb.js} +0 -0
- /package/dist/web/assets/{scala-anMIFYpA.js → scala-fz1OPLMl.js} +0 -0
- /package/dist/web/assets/{scheme-BItQTe08.js → scheme-8Uz1RIbu.js} +0 -0
- /package/dist/web/assets/{scss-BOv51BJ5.js → scss-Djo3IYXr.js} +0 -0
- /package/dist/web/assets/{shell-BsRYRTNN.js → shell-CINF5Tx_.js} +0 -0
- /package/dist/web/assets/{solidity-BtuLgGDx.js → solidity-GgiNEuUm.js} +0 -0
- /package/dist/web/assets/{sophia-B0Vkc5MF.js → sophia-Culj97P9.js} +0 -0
- /package/dist/web/assets/{sparql-B7lvkZQM.js → sparql-C2ZlpxOY.js} +0 -0
- /package/dist/web/assets/{sql-DvP5MpA3.js → sql-BEf5Pg7Y.js} +0 -0
- /package/dist/web/assets/{st-GVUeyB3U.js → st-CT6UUoeH.js} +0 -0
- /package/dist/web/assets/{swift-DSPIoCjm.js → swift-B5g0xTG3.js} +0 -0
- /package/dist/web/assets/{systemverilog-Icj2-k23.js → systemverilog-CEgQz9DR.js} +0 -0
- /package/dist/web/assets/{tcl-Cd8KQcm-.js → tcl-D0qL2L0I.js} +0 -0
- /package/dist/web/assets/{twig-CBHmt8z3.js → twig-BFUAVf1E.js} +0 -0
- /package/dist/web/assets/{typespec-Ckc037mq.js → typespec-CjVVcNKm.js} +0 -0
- /package/dist/web/assets/{vb-B97GW9Wb.js → vb-CZJr-DQz.js} +0 -0
- /package/dist/web/assets/{wgsl-DIKmb3YH.js → wgsl-ivoXUo2e.js} +0 -0
package/dist/cli.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { C as
|
|
1
|
+
import { A as formatDoctorDiagnosticsJson, C as seedConfig, D as version, E as package_default, S as createConfig, T as name, _ as DEFAULT_MINERU_CONFIG, a as parseJsonSchema, b as PLACEHOLDER_TEXT, c as recognizeImageText, d as t, f as getDefaultAIConfig, g as DEFAULT_MARKITDOWN_CONFIG, h as DEFAULT_MARKER_CONFIG, i as JsonSchemaDefinitionSchema, k as doctorDiagnosticsTableRows, l as shouldUseImageOcrFallback, m as writeAIConfig, n as createMigrationConfig, o as toSnakeCase, p as readAIConfig, s as generateDrizzleSchema, t as collectDoctorDiagnostics, u as initI18n, v as DEFAULT_PROMPT_CONFIG, w as description, x as AIConfigSchema, y as PLACEHOLDER_SCHEMA } from "./doctor-collector-CDg_BdI8.mjs";
|
|
2
2
|
import { createRequire } from "node:module";
|
|
3
3
|
import fs from "node:fs/promises";
|
|
4
4
|
import os from "node:os";
|
|
@@ -15,20 +15,20 @@ import fs$1 from "node:fs";
|
|
|
15
15
|
import { intro, isCancel, outro, select, spinner, text } from "@clack/prompts";
|
|
16
16
|
import Database from "better-sqlite3";
|
|
17
17
|
import pc from "picocolors";
|
|
18
|
+
import { Buffer } from "node:buffer";
|
|
18
19
|
import * as XLSX from "xlsx";
|
|
19
|
-
import { glob, globSync } from "tinyglobby";
|
|
20
20
|
import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
|
|
21
|
-
import { LangfuseSpanProcessor } from "@langfuse/otel";
|
|
22
|
-
import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
|
|
23
21
|
import { APICallError, Output, generateText, jsonSchema } from "ai";
|
|
24
|
-
import mime from "mime";
|
|
25
22
|
import pRetry from "p-retry";
|
|
23
|
+
import mime from "mime";
|
|
26
24
|
import { jsonrepair } from "jsonrepair";
|
|
25
|
+
import { LangfuseSpanProcessor } from "@langfuse/otel";
|
|
26
|
+
import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
|
|
27
|
+
import crypto from "node:crypto";
|
|
27
28
|
import { Client, extractNotionId } from "@notionhq/client";
|
|
28
|
-
import { Buffer } from "node:buffer";
|
|
29
29
|
import { execa } from "execa";
|
|
30
|
+
import { glob, globSync } from "tinyglobby";
|
|
30
31
|
import { extractText, getDocumentProxy, getMeta } from "unpdf";
|
|
31
|
-
import crypto from "node:crypto";
|
|
32
32
|
import { execFile } from "node:child_process";
|
|
33
33
|
import { promisify } from "node:util";
|
|
34
34
|
import * as chokidar from "chokidar";
|
|
@@ -156,11 +156,11 @@ function generateCompletionScript(name$1, shell) {
|
|
|
156
156
|
const completionCommand = defineCommand({
|
|
157
157
|
meta: {
|
|
158
158
|
name: "completion",
|
|
159
|
-
description: "
|
|
159
|
+
description: t("command.completion.description")
|
|
160
160
|
},
|
|
161
161
|
args: { shell: {
|
|
162
162
|
type: "positional",
|
|
163
|
-
description: "
|
|
163
|
+
description: t("command.completion.args.shell"),
|
|
164
164
|
required: true
|
|
165
165
|
} },
|
|
166
166
|
async run({ args }) {
|
|
@@ -169,7 +169,7 @@ const completionCommand = defineCommand({
|
|
|
169
169
|
try {
|
|
170
170
|
process.stdout.write(generateCompletionScript(name$1, shell));
|
|
171
171
|
} catch (error) {
|
|
172
|
-
process.stderr.write(
|
|
172
|
+
process.stderr.write(`${t("command.completion.error", { error: error instanceof Error ? error.message : String(error) })}\n`);
|
|
173
173
|
process.exit(1);
|
|
174
174
|
}
|
|
175
175
|
}
|
|
@@ -180,11 +180,11 @@ const completionCommand = defineCommand({
|
|
|
180
180
|
const doctorCommand = defineCommand({
|
|
181
181
|
meta: {
|
|
182
182
|
name: "doctor",
|
|
183
|
-
description: "
|
|
183
|
+
description: t("command.doctor.description")
|
|
184
184
|
},
|
|
185
185
|
args: { json: {
|
|
186
186
|
type: "boolean",
|
|
187
|
-
description: "
|
|
187
|
+
description: t("command.doctor.args.json")
|
|
188
188
|
} },
|
|
189
189
|
async run({ args }) {
|
|
190
190
|
try {
|
|
@@ -194,15 +194,15 @@ const doctorCommand = defineCommand({
|
|
|
194
194
|
return;
|
|
195
195
|
}
|
|
196
196
|
consola.info(`${diagnostics.cli.name} ${diagnostics.cli.version}`);
|
|
197
|
-
const
|
|
198
|
-
head: ["
|
|
197
|
+
const table = new CliTable3({
|
|
198
|
+
head: [t("command.doctor.headers.0"), t("command.doctor.headers.1")],
|
|
199
199
|
colAligns: ["right", "left"],
|
|
200
200
|
style: { compact: true }
|
|
201
201
|
});
|
|
202
|
-
|
|
203
|
-
process.stdout.write(`${
|
|
202
|
+
table.push(...doctorDiagnosticsTableRows(diagnostics));
|
|
203
|
+
process.stdout.write(`${table.toString()}\n`);
|
|
204
204
|
} catch (err) {
|
|
205
|
-
consola.error(
|
|
205
|
+
consola.error(t("command.doctor.diagnosticsFailed", { error: err }));
|
|
206
206
|
}
|
|
207
207
|
}
|
|
208
208
|
});
|
|
@@ -211,10 +211,54 @@ const doctorCommand = defineCommand({
|
|
|
211
211
|
//#region src/commands/utils.ts
|
|
212
212
|
function failCommand(message) {
|
|
213
213
|
if (message) consola.error(message);
|
|
214
|
-
outro("
|
|
214
|
+
outro(t("common.failed"));
|
|
215
215
|
process.exitCode = 1;
|
|
216
216
|
}
|
|
217
217
|
|
|
218
|
+
//#endregion
|
|
219
|
+
//#region src/core/export-manager.ts
|
|
220
|
+
function formatRowsConformingToSchema(rows, columns, schema, format) {
|
|
221
|
+
return rows.map((row) => {
|
|
222
|
+
const newRow = {};
|
|
223
|
+
columns.forEach((col) => {
|
|
224
|
+
const colName = col.name;
|
|
225
|
+
const val = row[colName];
|
|
226
|
+
const type = (schema?.properties?.[colName])?.type || "";
|
|
227
|
+
if (val === null || val === void 0) newRow[colName] = "";
|
|
228
|
+
else if (type === "boolean") if (format === "xlsx") newRow[colName] = val === 1 || val === "1" || val === true;
|
|
229
|
+
else newRow[colName] = val === 1 || val === "1" || val === true ? "true" : "false";
|
|
230
|
+
else if (type === "number" || type === "integer") if (val === "") newRow[colName] = "";
|
|
231
|
+
else {
|
|
232
|
+
const num = Number(val);
|
|
233
|
+
newRow[colName] = Number.isNaN(num) ? val : num;
|
|
234
|
+
}
|
|
235
|
+
else if (typeof val === "object") newRow[colName] = JSON.stringify(val);
|
|
236
|
+
else {
|
|
237
|
+
const dbType = (col.type || "").toLowerCase();
|
|
238
|
+
if ((dbType.includes("int") || dbType.includes("real") || dbType.includes("num") || dbType.includes("double") || dbType.includes("float")) && typeof val === "string" && val !== "") {
|
|
239
|
+
const num = Number(val);
|
|
240
|
+
newRow[colName] = Number.isNaN(num) ? val : num;
|
|
241
|
+
} else newRow[colName] = val;
|
|
242
|
+
}
|
|
243
|
+
});
|
|
244
|
+
return newRow;
|
|
245
|
+
});
|
|
246
|
+
}
|
|
247
|
+
function generateExportBuffer(tableName, formattedRows, columns, format) {
|
|
248
|
+
const ws = XLSX.utils.json_to_sheet(formattedRows, { header: columns.map((col) => col.name) });
|
|
249
|
+
if (format === "xlsx") {
|
|
250
|
+
const wb = XLSX.utils.book_new();
|
|
251
|
+
XLSX.utils.book_append_sheet(wb, ws, tableName.slice(0, 31));
|
|
252
|
+
return XLSX.write(wb, {
|
|
253
|
+
bookType: "xlsx",
|
|
254
|
+
type: "buffer"
|
|
255
|
+
});
|
|
256
|
+
} else {
|
|
257
|
+
const csv = XLSX.utils.sheet_to_csv(ws);
|
|
258
|
+
return Buffer.from("" + csv, "utf8");
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
218
262
|
//#endregion
|
|
219
263
|
//#region src/core/ai-extraction/model-capabilities.json
|
|
220
264
|
var model_capabilities_default = {
|
|
@@ -12814,6 +12858,28 @@ async function withRetry(fn, onRetry, maxRetries = 5) {
|
|
|
12814
12858
|
});
|
|
12815
12859
|
}
|
|
12816
12860
|
|
|
12861
|
+
//#endregion
|
|
12862
|
+
//#region src/core/ai-extraction/file-utils.ts
|
|
12863
|
+
function detectMimeType(filePath) {
|
|
12864
|
+
return mime.getType(filePath) ?? "application/octet-stream";
|
|
12865
|
+
}
|
|
12866
|
+
async function readFilePart(filePath) {
|
|
12867
|
+
const mimeStr = detectMimeType(filePath);
|
|
12868
|
+
const buffer = await fs.readFile(filePath);
|
|
12869
|
+
const name$1 = path.basename(filePath);
|
|
12870
|
+
if (mimeStr.startsWith("image/")) return {
|
|
12871
|
+
type: "image",
|
|
12872
|
+
image: buffer,
|
|
12873
|
+
mimeType: mimeStr
|
|
12874
|
+
};
|
|
12875
|
+
return {
|
|
12876
|
+
type: "file",
|
|
12877
|
+
data: buffer,
|
|
12878
|
+
mediaType: mimeStr,
|
|
12879
|
+
filename: name$1
|
|
12880
|
+
};
|
|
12881
|
+
}
|
|
12882
|
+
|
|
12817
12883
|
//#endregion
|
|
12818
12884
|
//#region src/core/ai-extraction/json-utils.ts
|
|
12819
12885
|
function parseJsonLike(text$1) {
|
|
@@ -12875,15 +12941,18 @@ function filterCompatible(models, inputTokens, outputTokens) {
|
|
|
12875
12941
|
}
|
|
12876
12942
|
function selectModel(input) {
|
|
12877
12943
|
const { models, isImage, fileName, inputTokens, outputTokens } = input;
|
|
12878
|
-
if (models.length === 0) throw new Error("
|
|
12944
|
+
if (models.length === 0) throw new Error(t("errors.ai.noModels"));
|
|
12879
12945
|
let candidates = filterCompatible(models, inputTokens, outputTokens);
|
|
12880
12946
|
if (candidates.length === 0) candidates = models;
|
|
12881
12947
|
if (isImage) {
|
|
12882
12948
|
const visionModel = candidates.find((m) => m.capabilities.vision);
|
|
12883
12949
|
if (!visionModel) {
|
|
12884
12950
|
const hint = fileName ? ` (${fileName})` : "";
|
|
12885
|
-
const msg = inputTokens ?
|
|
12886
|
-
|
|
12951
|
+
const msg = inputTokens ? t("errors.ai.noVisionModelContext", {
|
|
12952
|
+
tokens: inputTokens,
|
|
12953
|
+
hint
|
|
12954
|
+
}) : t("errors.ai.noVisionModel", { hint });
|
|
12955
|
+
throw new Error(msg + t("errors.ai.addSuitableModel"));
|
|
12887
12956
|
}
|
|
12888
12957
|
return {
|
|
12889
12958
|
name: visionModel.name,
|
|
@@ -12990,7 +13059,34 @@ function generatePromptSnapshot(schema, promptConfig = DEFAULT_PROMPT_CONFIG) {
|
|
|
12990
13059
|
}
|
|
12991
13060
|
|
|
12992
13061
|
//#endregion
|
|
12993
|
-
//#region src/core/ai-extraction/
|
|
13062
|
+
//#region src/core/ai-extraction/snapshot.ts
|
|
13063
|
+
const SYSTEM_PROMPT_REGEX = /## System Prompt\n([\s\S]*?)(?=## User Prompt|$)/;
|
|
13064
|
+
const USER_PROMPT_REGEX = /## User Prompt Template\n([\s\S]*)$/;
|
|
13065
|
+
async function loadPromptSnapshot(aiexDir, tableName) {
|
|
13066
|
+
const snapshotPath = path.join(aiexDir, "extracted", `${tableName}.prompt.md`);
|
|
13067
|
+
try {
|
|
13068
|
+
const content = await fs.readFile(snapshotPath, "utf-8");
|
|
13069
|
+
const systemMatch = content.match(SYSTEM_PROMPT_REGEX);
|
|
13070
|
+
const userMatch = content.match(USER_PROMPT_REGEX);
|
|
13071
|
+
if (systemMatch && userMatch) return {
|
|
13072
|
+
system: systemMatch[1].trim(),
|
|
13073
|
+
user: userMatch[1].trim()
|
|
13074
|
+
};
|
|
13075
|
+
} catch {}
|
|
13076
|
+
return null;
|
|
13077
|
+
}
|
|
13078
|
+
async function savePromptSnapshot(schema, aiexDir) {
|
|
13079
|
+
const content = generatePromptSnapshot(schema, (await readAIConfig(aiexDir))?.prompt ?? DEFAULT_PROMPT_CONFIG);
|
|
13080
|
+
const outputDir = path.join(aiexDir, "extracted");
|
|
13081
|
+
await fs.mkdir(outputDir, { recursive: true });
|
|
13082
|
+
const fileName = `${schema.table.name}.prompt.md`;
|
|
13083
|
+
const outputPath = path.join(outputDir, fileName);
|
|
13084
|
+
await fs.writeFile(outputPath, content);
|
|
13085
|
+
return outputPath;
|
|
13086
|
+
}
|
|
13087
|
+
|
|
13088
|
+
//#endregion
|
|
13089
|
+
//#region src/core/ai-extraction/telemetry.ts
|
|
12994
13090
|
let langfuseInitialized = false;
|
|
12995
13091
|
function initLangfuse(config) {
|
|
12996
13092
|
if (!config.langfuse?.publicKey || !config.langfuse.secretKey) return;
|
|
@@ -13007,28 +13103,9 @@ function initLangfuse(config) {
|
|
|
13007
13103
|
console.warn("[Langfuse] Failed to initialize tracing:", e instanceof Error ? e.message : e);
|
|
13008
13104
|
}
|
|
13009
13105
|
}
|
|
13010
|
-
|
|
13011
|
-
|
|
13012
|
-
|
|
13013
|
-
function detectMimeType(filePath) {
|
|
13014
|
-
return mime.getType(filePath) ?? "application/octet-stream";
|
|
13015
|
-
}
|
|
13016
|
-
async function readFilePart(filePath) {
|
|
13017
|
-
const mime$1 = detectMimeType(filePath);
|
|
13018
|
-
const buffer = await fs.readFile(filePath);
|
|
13019
|
-
const name$1 = path.basename(filePath);
|
|
13020
|
-
if (mime$1.startsWith("image/")) return {
|
|
13021
|
-
type: "image",
|
|
13022
|
-
image: buffer,
|
|
13023
|
-
mimeType: mime$1
|
|
13024
|
-
};
|
|
13025
|
-
return {
|
|
13026
|
-
type: "file",
|
|
13027
|
-
data: buffer,
|
|
13028
|
-
mediaType: mime$1,
|
|
13029
|
-
filename: name$1
|
|
13030
|
-
};
|
|
13031
|
-
}
|
|
13106
|
+
|
|
13107
|
+
//#endregion
|
|
13108
|
+
//#region src/core/ai-extraction/validator.ts
|
|
13032
13109
|
function nullableType(type) {
|
|
13033
13110
|
return type === "null" ? ["null"] : [type, "null"];
|
|
13034
13111
|
}
|
|
@@ -13123,24 +13200,15 @@ function validateExtractedData(schema, data) {
|
|
|
13123
13200
|
};
|
|
13124
13201
|
return { success: true };
|
|
13125
13202
|
}
|
|
13126
|
-
|
|
13127
|
-
|
|
13128
|
-
|
|
13129
|
-
|
|
13130
|
-
const systemMatch = content.match(SYSTEM_PROMPT_REGEX);
|
|
13131
|
-
const userMatch = content.match(USER_PROMPT_REGEX);
|
|
13132
|
-
if (systemMatch && userMatch) return {
|
|
13133
|
-
system: systemMatch[1].trim(),
|
|
13134
|
-
user: userMatch[1].trim()
|
|
13135
|
-
};
|
|
13136
|
-
} catch {}
|
|
13137
|
-
return null;
|
|
13138
|
-
}
|
|
13203
|
+
|
|
13204
|
+
//#endregion
|
|
13205
|
+
//#region src/core/ai-extraction/extractor.ts
|
|
13206
|
+
const OPENAI_COMPATIBLE_PROVIDER_NAME = "openai-compatible";
|
|
13139
13207
|
async function extractStructuredData(input) {
|
|
13140
13208
|
const { config, schema, text: text$1, aiexDir, file, modelOverride } = input;
|
|
13141
13209
|
if (!config.provider.apiKey) return {
|
|
13142
13210
|
success: false,
|
|
13143
|
-
error: "
|
|
13211
|
+
error: t("errors.ai.apiKeyMissing")
|
|
13144
13212
|
};
|
|
13145
13213
|
const useFileContent = !!file;
|
|
13146
13214
|
const isImageFile = useFileContent && detectMimeType(file).startsWith("image/");
|
|
@@ -13185,66 +13253,118 @@ async function extractStructuredData(input) {
|
|
|
13185
13253
|
user = generated.user;
|
|
13186
13254
|
}
|
|
13187
13255
|
const outputSchema = jsonSchema(schemaToExtractionOutputSchema(schema));
|
|
13188
|
-
let result;
|
|
13189
13256
|
const timeoutMs = (config.provider.timeout ?? 300) * 1e3;
|
|
13190
|
-
|
|
13191
|
-
|
|
13192
|
-
|
|
13193
|
-
|
|
13194
|
-
|
|
13195
|
-
|
|
13196
|
-
|
|
13197
|
-
|
|
13198
|
-
|
|
13199
|
-
|
|
13200
|
-
|
|
13201
|
-
|
|
13202
|
-
|
|
13203
|
-
|
|
13204
|
-
|
|
13205
|
-
|
|
13206
|
-
|
|
13207
|
-
|
|
13208
|
-
|
|
13209
|
-
|
|
13210
|
-
|
|
13211
|
-
|
|
13212
|
-
|
|
13213
|
-
|
|
13214
|
-
|
|
13215
|
-
|
|
13216
|
-
|
|
13217
|
-
|
|
13218
|
-
|
|
13219
|
-
|
|
13220
|
-
|
|
13257
|
+
let systemPrompt = system;
|
|
13258
|
+
let userPrompt = user;
|
|
13259
|
+
const maxAttempts = 3;
|
|
13260
|
+
let lastError = "";
|
|
13261
|
+
let totalPromptTokens = 0;
|
|
13262
|
+
let totalCompletionTokens = 0;
|
|
13263
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
13264
|
+
let result = null;
|
|
13265
|
+
let data;
|
|
13266
|
+
let parseError;
|
|
13267
|
+
let validationError;
|
|
13268
|
+
try {
|
|
13269
|
+
if (useFileContent) {
|
|
13270
|
+
const filePart = await readFilePart(file);
|
|
13271
|
+
const fileName = filePart.type === "file" ? filePart.filename : path.basename(file);
|
|
13272
|
+
const contentParts = [{
|
|
13273
|
+
type: "text",
|
|
13274
|
+
text: userPrompt.includes(PLACEHOLDER_TEXT) ? userPrompt.replaceAll(PLACEHOLDER_TEXT, text$1 || `Data is contained in the attached file: ${fileName}`) : userPrompt
|
|
13275
|
+
}, filePart];
|
|
13276
|
+
const fileOpts = {
|
|
13277
|
+
model: provider.chatModel(selected.name),
|
|
13278
|
+
system: systemPrompt,
|
|
13279
|
+
messages: [{
|
|
13280
|
+
role: "user",
|
|
13281
|
+
content: contentParts
|
|
13282
|
+
}],
|
|
13283
|
+
abortSignal: AbortSignal.timeout(timeoutMs),
|
|
13284
|
+
maxRetries: 0,
|
|
13285
|
+
experimental_telemetry: { isEnabled: useTelemetry }
|
|
13286
|
+
};
|
|
13287
|
+
if (useStructuredOutput) fileOpts.output = Output.object({ schema: outputSchema });
|
|
13288
|
+
result = await withRetry(() => generateText(fileOpts), input.onRetry);
|
|
13289
|
+
} else {
|
|
13290
|
+
const textOpts = {
|
|
13291
|
+
model: provider.chatModel(selected.name),
|
|
13292
|
+
system: systemPrompt,
|
|
13293
|
+
prompt: userPrompt,
|
|
13294
|
+
abortSignal: AbortSignal.timeout(timeoutMs),
|
|
13295
|
+
maxRetries: 0,
|
|
13296
|
+
experimental_telemetry: { isEnabled: useTelemetry }
|
|
13297
|
+
};
|
|
13298
|
+
if (useStructuredOutput) textOpts.output = Output.object({ schema: outputSchema });
|
|
13299
|
+
result = await withRetry(() => generateText(textOpts), input.onRetry);
|
|
13300
|
+
}
|
|
13301
|
+
if (result.usage) {
|
|
13302
|
+
totalPromptTokens += result.usage.inputTokens ?? 0;
|
|
13303
|
+
totalCompletionTokens += result.usage.outputTokens ?? 0;
|
|
13304
|
+
}
|
|
13305
|
+
if (useStructuredOutput) data = result.output;
|
|
13306
|
+
else try {
|
|
13307
|
+
data = safeParseJSON(result.text);
|
|
13308
|
+
} catch (e) {
|
|
13309
|
+
parseError = e instanceof Error ? e.message : String(e);
|
|
13310
|
+
}
|
|
13311
|
+
} catch (error) {
|
|
13312
|
+
parseError = getErrorMessage(error);
|
|
13313
|
+
}
|
|
13314
|
+
if (!parseError && data !== void 0) {
|
|
13315
|
+
const validation = validateExtractedData(schema, data);
|
|
13316
|
+
if (validation.success) {
|
|
13317
|
+
const outputDir = path.resolve(aiexDir, config.extraction.outputDir.replace(".aiex/", ""));
|
|
13318
|
+
await fs.mkdir(outputDir, { recursive: true });
|
|
13319
|
+
const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
13320
|
+
const outputFileName = `${schema.table.name}-${timestamp}.json`;
|
|
13321
|
+
const outputPath = path.join(outputDir, outputFileName);
|
|
13322
|
+
await writeFile(outputPath, data, {
|
|
13323
|
+
spaces: 2,
|
|
13324
|
+
EOL: "\n"
|
|
13325
|
+
});
|
|
13326
|
+
return {
|
|
13327
|
+
success: true,
|
|
13328
|
+
outputPath,
|
|
13329
|
+
data,
|
|
13330
|
+
tokensUsed: {
|
|
13331
|
+
prompt: totalPromptTokens,
|
|
13332
|
+
completion: totalCompletionTokens,
|
|
13333
|
+
total: totalPromptTokens + totalCompletionTokens
|
|
13334
|
+
}
|
|
13335
|
+
};
|
|
13336
|
+
} else validationError = validation.error;
|
|
13337
|
+
}
|
|
13338
|
+
const errorMsg = parseError || validationError || "Unknown validation error";
|
|
13339
|
+
lastError = errorMsg;
|
|
13340
|
+
if (attempt < maxAttempts) {
|
|
13341
|
+
const invalidJson = data !== void 0 ? JSON.stringify(data, null, 2) : result ? result.text : "";
|
|
13342
|
+
systemPrompt = `You are a precise data correction assistant. Your task is to correct validation errors in a previously generated JSON object to make it comply with the provided JSON Schema.
|
|
13343
|
+
|
|
13344
|
+
CRITICAL RULES:
|
|
13345
|
+
1. Only correct the fields that failed validation.
|
|
13346
|
+
2. Preserve all other correctly extracted fields and their values exactly.
|
|
13347
|
+
3. Return ONLY the corrected JSON object. No explanations, no markdown blocks other than JSON.`;
|
|
13348
|
+
userPrompt = `The JSON data you generated previously failed validation. Please correct it.
|
|
13349
|
+
|
|
13350
|
+
[Original Text]
|
|
13351
|
+
${text$1 || "Data is contained in the attached file."}
|
|
13352
|
+
|
|
13353
|
+
[JSON Schema Definition]
|
|
13354
|
+
${JSON.stringify(schemaToExtractionOutputSchema(schema), null, 2)}
|
|
13355
|
+
|
|
13356
|
+
[Previously Generated Invalid JSON]
|
|
13357
|
+
${invalidJson}
|
|
13358
|
+
|
|
13359
|
+
[Validation Error Details]
|
|
13360
|
+
${errorMsg}
|
|
13361
|
+
|
|
13362
|
+
Please output the corrected JSON object now:`;
|
|
13363
|
+
}
|
|
13221
13364
|
}
|
|
13222
|
-
let data;
|
|
13223
|
-
if (useStructuredOutput) data = result.output;
|
|
13224
|
-
else data = safeParseJSON(result.text);
|
|
13225
|
-
const validation = validateExtractedData(schema, data);
|
|
13226
|
-
if (!validation.success) return {
|
|
13227
|
-
success: false,
|
|
13228
|
-
error: validation.error
|
|
13229
|
-
};
|
|
13230
|
-
const outputDir = path.resolve(aiexDir, config.extraction.outputDir.replace(".aiex/", ""));
|
|
13231
|
-
await fs.mkdir(outputDir, { recursive: true });
|
|
13232
|
-
const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
13233
|
-
const outputFileName = `${schema.table.name}-${timestamp}.json`;
|
|
13234
|
-
const outputPath = path.join(outputDir, outputFileName);
|
|
13235
|
-
await writeFile(outputPath, data, {
|
|
13236
|
-
spaces: 2,
|
|
13237
|
-
EOL: "\n"
|
|
13238
|
-
});
|
|
13239
13365
|
return {
|
|
13240
|
-
success:
|
|
13241
|
-
|
|
13242
|
-
data,
|
|
13243
|
-
tokensUsed: result.usage ? {
|
|
13244
|
-
prompt: result.usage.inputTokens ?? 0,
|
|
13245
|
-
completion: result.usage.outputTokens ?? 0,
|
|
13246
|
-
total: (result.usage.inputTokens ?? 0) + (result.usage.outputTokens ?? 0)
|
|
13247
|
-
} : void 0
|
|
13366
|
+
success: false,
|
|
13367
|
+
error: lastError || "Extraction failed after self-reflection retries"
|
|
13248
13368
|
};
|
|
13249
13369
|
} catch (error) {
|
|
13250
13370
|
return {
|
|
@@ -13345,7 +13465,7 @@ function insertExtractedData(db, schema, data) {
|
|
|
13345
13465
|
const [propName] = propEntry;
|
|
13346
13466
|
const nestedValue = data[propName];
|
|
13347
13467
|
if (nestedValue === null || nestedValue === void 0) continue;
|
|
13348
|
-
const nestedTable = parseResult.tables.find((t) => t.name === revRel.toTable);
|
|
13468
|
+
const nestedTable = parseResult.tables.find((t$1) => t$1.name === revRel.toTable);
|
|
13349
13469
|
if (!nestedTable) continue;
|
|
13350
13470
|
if (revRel.type === "has-one") {
|
|
13351
13471
|
const rowId = insertTableRow({
|
|
@@ -13391,18 +13511,6 @@ function insertExtractedData(db, schema, data) {
|
|
|
13391
13511
|
}
|
|
13392
13512
|
}
|
|
13393
13513
|
|
|
13394
|
-
//#endregion
|
|
13395
|
-
//#region src/core/ai-extraction/snapshot.ts
|
|
13396
|
-
async function savePromptSnapshot(schema, aiexDir) {
|
|
13397
|
-
const content = generatePromptSnapshot(schema, (await readAIConfig(aiexDir))?.prompt ?? DEFAULT_PROMPT_CONFIG);
|
|
13398
|
-
const outputDir = path.join(aiexDir, "extracted");
|
|
13399
|
-
await fs.mkdir(outputDir, { recursive: true });
|
|
13400
|
-
const fileName = `${schema.table.name}.prompt.md`;
|
|
13401
|
-
const outputPath = path.join(outputDir, fileName);
|
|
13402
|
-
await fs.writeFile(outputPath, content);
|
|
13403
|
-
return outputPath;
|
|
13404
|
-
}
|
|
13405
|
-
|
|
13406
13514
|
//#endregion
|
|
13407
13515
|
//#region src/core/extraction-audit.ts
|
|
13408
13516
|
const AUDIT_ID_RE = /^[\w.-]+$/;
|
|
@@ -13440,7 +13548,7 @@ async function createExtractionAuditRecord(aiexDir, input) {
|
|
|
13440
13548
|
}
|
|
13441
13549
|
async function updateExtractionAuditRecord(aiexDir, id, patch) {
|
|
13442
13550
|
const current = await readExtractionAuditRecord(aiexDir, id);
|
|
13443
|
-
if (!current) throw new Error(
|
|
13551
|
+
if (!current) throw new Error(t("errors.extractionAudit.recordNotFound", { id }));
|
|
13444
13552
|
const record = {
|
|
13445
13553
|
...current,
|
|
13446
13554
|
...patch,
|
|
@@ -13473,7 +13581,7 @@ async function markStaleIfNeeded(aiexDir, record) {
|
|
|
13473
13581
|
const staleRecord = {
|
|
13474
13582
|
...record,
|
|
13475
13583
|
status: "stale",
|
|
13476
|
-
error: record.error ?? "
|
|
13584
|
+
error: record.error ?? t("errors.extractionAudit.interrupted"),
|
|
13477
13585
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
13478
13586
|
};
|
|
13479
13587
|
await writeFile(auditPath(aiexDir, staleRecord.id), staleRecord, {
|
|
@@ -13539,70 +13647,18 @@ async function findSucceededAuditByHash(aiexDir, schemaName, fileHash) {
|
|
|
13539
13647
|
}
|
|
13540
13648
|
|
|
13541
13649
|
//#endregion
|
|
13542
|
-
//#region src/
|
|
13543
|
-
|
|
13544
|
-
|
|
13545
|
-
|
|
13546
|
-
|
|
13547
|
-
|
|
13548
|
-
|
|
13549
|
-
|
|
13550
|
-
|
|
13551
|
-
|
|
13552
|
-
|
|
13553
|
-
|
|
13554
|
-
"application/pdf",
|
|
13555
|
-
"text/plain",
|
|
13556
|
-
"text/markdown",
|
|
13557
|
-
"text/csv",
|
|
13558
|
-
"application/json",
|
|
13559
|
-
"text/html",
|
|
13560
|
-
"text/xml",
|
|
13561
|
-
"application/x-yaml",
|
|
13562
|
-
"text/yaml"
|
|
13563
|
-
]);
|
|
13564
|
-
const MIME_TO_EXT = {
|
|
13565
|
-
"image/png": "png",
|
|
13566
|
-
"image/jpeg": "jpg",
|
|
13567
|
-
"image/gif": "gif",
|
|
13568
|
-
"image/webp": "webp",
|
|
13569
|
-
"image/bmp": "bmp",
|
|
13570
|
-
"image/svg+xml": "svg",
|
|
13571
|
-
"application/pdf": "pdf",
|
|
13572
|
-
"text/plain": "txt",
|
|
13573
|
-
"text/markdown": "md",
|
|
13574
|
-
"text/csv": "csv",
|
|
13575
|
-
"application/json": "json",
|
|
13576
|
-
"text/html": "html",
|
|
13577
|
-
"text/xml": "xml",
|
|
13578
|
-
"application/x-yaml": "yaml",
|
|
13579
|
-
"text/yaml": "yaml"
|
|
13580
|
-
};
|
|
13581
|
-
function bytesToMB(bytes) {
|
|
13582
|
-
return bytes / (1024 * 1024);
|
|
13583
|
-
}
|
|
13584
|
-
function getExtensionFromMime(mimeType) {
|
|
13585
|
-
return MIME_TO_EXT[mimeType];
|
|
13586
|
-
}
|
|
13587
|
-
function isAllowedMimeType(mimeType) {
|
|
13588
|
-
return SUPPORTED_MIME_TYPES.has(mimeType);
|
|
13589
|
-
}
|
|
13590
|
-
function unsupportedFileTypeMessage(mimeType) {
|
|
13591
|
-
return `Unsupported file type "${mimeType}". Supported: ${SUPPORTED_FILE_TYPES_TEXT}.`;
|
|
13592
|
-
}
|
|
13593
|
-
function isMissingUploadFileError(error) {
|
|
13594
|
-
return !!error && typeof error === "object" && error.code === "ENOENT";
|
|
13595
|
-
}
|
|
13596
|
-
var FileValidationError = class extends Error {
|
|
13597
|
-
constructor(message) {
|
|
13598
|
-
super(message);
|
|
13599
|
-
this.name = "FileValidationError";
|
|
13600
|
-
}
|
|
13601
|
-
};
|
|
13602
|
-
function validateFileUpload(file) {
|
|
13603
|
-
if (file.size === 0) throw new FileValidationError("Uploaded file is empty");
|
|
13604
|
-
if (file.size > MAX_UPLOAD_SIZE) throw new FileValidationError(`File size (${bytesToMB(file.size).toFixed(1)}MB) exceeds ${MAX_UPLOAD_SIZE_TEXT} limit`);
|
|
13605
|
-
if (!isAllowedMimeType(file.type)) throw new FileValidationError(unsupportedFileTypeMessage(file.type));
|
|
13650
|
+
//#region src/utils/hash.ts
|
|
13651
|
+
/**
|
|
13652
|
+
* Helper to compute SHA-256 hash of a file asynchronously.
|
|
13653
|
+
*/
|
|
13654
|
+
function getFileHash(filePath) {
|
|
13655
|
+
return new Promise((resolve, reject) => {
|
|
13656
|
+
const hash = crypto.createHash("sha256");
|
|
13657
|
+
const stream = fs$1.createReadStream(filePath);
|
|
13658
|
+
stream.on("data", (data) => hash.update(data));
|
|
13659
|
+
stream.on("end", () => resolve(hash.digest("hex")));
|
|
13660
|
+
stream.on("error", (err) => reject(err));
|
|
13661
|
+
});
|
|
13606
13662
|
}
|
|
13607
13663
|
|
|
13608
13664
|
//#endregion
|
|
@@ -13756,7 +13812,7 @@ function firstDataSourceId(database) {
|
|
|
13756
13812
|
}
|
|
13757
13813
|
async function resolveNotionDataSource(notion, inputId) {
|
|
13758
13814
|
const id = parseNotionDatabaseId(inputId);
|
|
13759
|
-
if (!id) throw new Error("
|
|
13815
|
+
if (!id) throw new Error(t("errors.notion.idRequired"));
|
|
13760
13816
|
try {
|
|
13761
13817
|
const dataSource$1 = await notion.dataSources.retrieve({ data_source_id: id });
|
|
13762
13818
|
if (isDataSourceResponse(dataSource$1)) return {
|
|
@@ -13768,9 +13824,9 @@ async function resolveNotionDataSource(notion, inputId) {
|
|
|
13768
13824
|
} catch {}
|
|
13769
13825
|
const database = await notion.databases.retrieve({ database_id: id });
|
|
13770
13826
|
const dataSourceId = firstDataSourceId(database);
|
|
13771
|
-
if (!dataSourceId) throw new Error("
|
|
13827
|
+
if (!dataSourceId) throw new Error(t("errors.notion.noDataSource"));
|
|
13772
13828
|
const dataSource = await notion.dataSources.retrieve({ data_source_id: dataSourceId });
|
|
13773
|
-
if (!isDataSourceResponse(dataSource)) throw new Error("
|
|
13829
|
+
if (!isDataSourceResponse(dataSource)) throw new Error(t("errors.notion.noProperties"));
|
|
13774
13830
|
return {
|
|
13775
13831
|
databaseId: database.id,
|
|
13776
13832
|
dataSourceId: dataSource.id,
|
|
@@ -13779,9 +13835,9 @@ async function resolveNotionDataSource(notion, inputId) {
|
|
|
13779
13835
|
};
|
|
13780
13836
|
}
|
|
13781
13837
|
async function inspectNotionDatabase(input) {
|
|
13782
|
-
if (!input.token.trim()) throw new Error("
|
|
13838
|
+
if (!input.token.trim()) throw new Error(t("errors.notion.tokenRequired"));
|
|
13783
13839
|
const id = parseNotionDatabaseId(input.databaseId);
|
|
13784
|
-
if (!id) throw new Error("
|
|
13840
|
+
if (!id) throw new Error(t("errors.notion.idRequired"));
|
|
13785
13841
|
const resolved = await resolveNotionDataSource(new Client({ auth: input.token }), id);
|
|
13786
13842
|
const databaseProperties = resolved.properties;
|
|
13787
13843
|
const titleProperty = findTitleProperty(databaseProperties) ?? void 0;
|
|
@@ -13797,8 +13853,8 @@ async function inspectNotionDatabase(input) {
|
|
|
13797
13853
|
};
|
|
13798
13854
|
}
|
|
13799
13855
|
function validateNotionConfig(config) {
|
|
13800
|
-
if (!config?.enabled) return "
|
|
13801
|
-
if (!config.token.trim()) return "
|
|
13856
|
+
if (!config?.enabled) return t("errors.notion.notEnabled");
|
|
13857
|
+
if (!config.token.trim()) return t("errors.notion.tokenRequired");
|
|
13802
13858
|
return null;
|
|
13803
13859
|
}
|
|
13804
13860
|
async function writeNotionPage(config, schemaName, data) {
|
|
@@ -13806,8 +13862,8 @@ async function writeNotionPage(config, schemaName, data) {
|
|
|
13806
13862
|
if (configError) throw new Error(configError);
|
|
13807
13863
|
const notionConfig = config;
|
|
13808
13864
|
const schemaConfig = notionConfig.schemas[schemaName];
|
|
13809
|
-
if (!schemaConfig) throw new Error(
|
|
13810
|
-
if (!schemaConfig.databaseId.trim()) throw new Error(
|
|
13865
|
+
if (!schemaConfig) throw new Error(t("errors.notion.noSchemaConfig", { name: schemaName }));
|
|
13866
|
+
if (!schemaConfig.databaseId.trim()) throw new Error(t("errors.notion.noDatabaseId", { name: schemaName }));
|
|
13811
13867
|
const notion = new Client({ auth: notionConfig.token });
|
|
13812
13868
|
const resolved = await resolveNotionDataSource(notion, schemaConfig.databaseId);
|
|
13813
13869
|
const databaseProperties = resolved.properties;
|
|
@@ -13825,7 +13881,7 @@ async function writeNotionPage(config, schemaName, data) {
|
|
|
13825
13881
|
}
|
|
13826
13882
|
const titleProperty = findTitleProperty(databaseProperties, schemaConfig.titleProperty);
|
|
13827
13883
|
if (titleProperty && !properties[titleProperty]) properties[titleProperty] = buildPropertyValue("title", schemaName);
|
|
13828
|
-
if (Object.keys(properties).length === 0) throw new Error("
|
|
13884
|
+
if (Object.keys(properties).length === 0) throw new Error(t("errors.notion.noFieldsMatched"));
|
|
13829
13885
|
return {
|
|
13830
13886
|
pageId: (await notion.pages.create({
|
|
13831
13887
|
parent: resolved.parent,
|
|
@@ -13837,27 +13893,155 @@ async function writeNotionPage(config, schemaName, data) {
|
|
|
13837
13893
|
}
|
|
13838
13894
|
|
|
13839
13895
|
//#endregion
|
|
13840
|
-
//#region src/core/
|
|
13841
|
-
function
|
|
13842
|
-
|
|
13896
|
+
//#region src/core/webhook-sink.ts
|
|
13897
|
+
async function sendWebhook(config, payload) {
|
|
13898
|
+
if (!config || !config.enabled || !config.url) return;
|
|
13899
|
+
const body = JSON.stringify(payload);
|
|
13900
|
+
const headers = {
|
|
13901
|
+
"Content-Type": "application/json",
|
|
13902
|
+
"User-Agent": "aiex-webhook-dispatcher"
|
|
13903
|
+
};
|
|
13904
|
+
if (config.secret) headers["X-Aiex-Signature"] = `sha256=${crypto.createHmac("sha256", config.secret).update(body).digest("hex")}`;
|
|
13905
|
+
const response = await fetch(config.url, {
|
|
13906
|
+
method: "POST",
|
|
13907
|
+
headers,
|
|
13908
|
+
body
|
|
13909
|
+
});
|
|
13910
|
+
if (!response.ok) throw new Error(`Webhook request failed with status: ${response.status} ${response.statusText}`);
|
|
13843
13911
|
}
|
|
13844
|
-
|
|
13845
|
-
|
|
13912
|
+
|
|
13913
|
+
//#endregion
|
|
13914
|
+
//#region src/core/integration/dispatcher.ts
|
|
13915
|
+
async function syncResultToNotion(aiConfig, schemaName, data) {
|
|
13916
|
+
if (!data || typeof data !== "object" || Array.isArray(data)) throw new Error(t("errors.ai.extractionNotObject"));
|
|
13917
|
+
const page = await writeNotionPage(aiConfig.notion, schemaName, data);
|
|
13918
|
+
return [{
|
|
13919
|
+
databaseId: page.databaseId,
|
|
13920
|
+
pageId: page.pageId
|
|
13921
|
+
}];
|
|
13846
13922
|
}
|
|
13847
|
-
|
|
13923
|
+
function shouldSyncNotion(aiConfig, schemaName) {
|
|
13924
|
+
return !!aiConfig.notion?.enabled && !!aiConfig.notion.schemas?.[schemaName]?.databaseId?.trim();
|
|
13925
|
+
}
|
|
13926
|
+
async function triggerWebhook(aiConfig, auditId, schemaName, event, source, data, error, tokensUsed, quiet = false) {
|
|
13927
|
+
if (!aiConfig.webhook?.enabled) return;
|
|
13848
13928
|
try {
|
|
13849
|
-
await
|
|
13850
|
-
|
|
13851
|
-
|
|
13852
|
-
|
|
13929
|
+
await sendWebhook(aiConfig.webhook, {
|
|
13930
|
+
event,
|
|
13931
|
+
schemaName,
|
|
13932
|
+
auditId,
|
|
13933
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
13934
|
+
source: {
|
|
13935
|
+
type: source.type,
|
|
13936
|
+
fileName: source.filePath ? path.basename(source.filePath) : void 0,
|
|
13937
|
+
filePath: source.filePath
|
|
13938
|
+
},
|
|
13939
|
+
data,
|
|
13940
|
+
error,
|
|
13941
|
+
tokensUsed
|
|
13942
|
+
});
|
|
13943
|
+
if (!quiet) consola.success(t("extract.file.webhookSynced"));
|
|
13944
|
+
} catch (err) {
|
|
13945
|
+
if (!quiet) consola.error(t("extract.file.webhookSyncFail", { error: err instanceof Error ? err.message : String(err) }));
|
|
13853
13946
|
}
|
|
13854
13947
|
}
|
|
13855
|
-
|
|
13856
|
-
|
|
13857
|
-
|
|
13858
|
-
|
|
13859
|
-
|
|
13860
|
-
|
|
13948
|
+
|
|
13949
|
+
//#endregion
|
|
13950
|
+
//#region src/core/file-constants.ts
|
|
13951
|
+
const MAX_UPLOAD_SIZE = 30 * 1024 * 1024;
|
|
13952
|
+
const MAX_UPLOAD_SIZE_TEXT = "30MB";
|
|
13953
|
+
const SUPPORTED_FILE_TYPES_TEXT = "images, PDF, text, markdown, CSV, JSON, HTML, XML, YAML";
|
|
13954
|
+
const MISSING_UPLOAD_FILE_TEXT = t("errors.file.missingUpload");
|
|
13955
|
+
const SUPPORTED_MIME_TYPES = new Set([
|
|
13956
|
+
"image/png",
|
|
13957
|
+
"image/jpeg",
|
|
13958
|
+
"image/gif",
|
|
13959
|
+
"image/webp",
|
|
13960
|
+
"image/bmp",
|
|
13961
|
+
"image/svg+xml",
|
|
13962
|
+
"application/pdf",
|
|
13963
|
+
"text/plain",
|
|
13964
|
+
"text/markdown",
|
|
13965
|
+
"text/csv",
|
|
13966
|
+
"application/json",
|
|
13967
|
+
"text/html",
|
|
13968
|
+
"text/xml",
|
|
13969
|
+
"application/x-yaml",
|
|
13970
|
+
"text/yaml"
|
|
13971
|
+
]);
|
|
13972
|
+
const MIME_TO_EXT = {
|
|
13973
|
+
"image/png": "png",
|
|
13974
|
+
"image/jpeg": "jpg",
|
|
13975
|
+
"image/gif": "gif",
|
|
13976
|
+
"image/webp": "webp",
|
|
13977
|
+
"image/bmp": "bmp",
|
|
13978
|
+
"image/svg+xml": "svg",
|
|
13979
|
+
"application/pdf": "pdf",
|
|
13980
|
+
"text/plain": "txt",
|
|
13981
|
+
"text/markdown": "md",
|
|
13982
|
+
"text/csv": "csv",
|
|
13983
|
+
"application/json": "json",
|
|
13984
|
+
"text/html": "html",
|
|
13985
|
+
"text/xml": "xml",
|
|
13986
|
+
"application/x-yaml": "yaml",
|
|
13987
|
+
"text/yaml": "yaml"
|
|
13988
|
+
};
|
|
13989
|
+
function bytesToMB(bytes) {
|
|
13990
|
+
return bytes / (1024 * 1024);
|
|
13991
|
+
}
|
|
13992
|
+
function getExtensionFromMime(mimeType) {
|
|
13993
|
+
return MIME_TO_EXT[mimeType];
|
|
13994
|
+
}
|
|
13995
|
+
function isAllowedMimeType(mimeType) {
|
|
13996
|
+
return SUPPORTED_MIME_TYPES.has(mimeType);
|
|
13997
|
+
}
|
|
13998
|
+
function unsupportedFileTypeMessage(mimeType) {
|
|
13999
|
+
return t("errors.file.unsupportedType", {
|
|
14000
|
+
type: mimeType,
|
|
14001
|
+
supported: SUPPORTED_FILE_TYPES_TEXT
|
|
14002
|
+
});
|
|
14003
|
+
}
|
|
14004
|
+
function isMissingUploadFileError(error) {
|
|
14005
|
+
return !!error && typeof error === "object" && error.code === "ENOENT";
|
|
14006
|
+
}
|
|
14007
|
+
var FileValidationError = class extends Error {
|
|
14008
|
+
constructor(message) {
|
|
14009
|
+
super(message);
|
|
14010
|
+
this.name = "FileValidationError";
|
|
14011
|
+
}
|
|
14012
|
+
};
|
|
14013
|
+
function validateFileUpload(file) {
|
|
14014
|
+
if (file.size === 0) throw new FileValidationError(t("errors.file.empty"));
|
|
14015
|
+
if (file.size > MAX_UPLOAD_SIZE) throw new FileValidationError(t("errors.file.sizeExceeded", {
|
|
14016
|
+
size: bytesToMB(file.size).toFixed(1),
|
|
14017
|
+
limit: MAX_UPLOAD_SIZE_TEXT,
|
|
14018
|
+
file: file.name
|
|
14019
|
+
}));
|
|
14020
|
+
if (!isAllowedMimeType(file.type)) throw new FileValidationError(unsupportedFileTypeMessage(file.type));
|
|
14021
|
+
}
|
|
14022
|
+
|
|
14023
|
+
//#endregion
|
|
14024
|
+
//#region src/core/pdf-converter/external.ts
|
|
14025
|
+
function applyTemplate(value, context) {
|
|
14026
|
+
return value.replaceAll("{input}", context.input).replaceAll("{outputDir}", context.outputDir).replaceAll("{basename}", context.basename);
|
|
14027
|
+
}
|
|
14028
|
+
function isError(error) {
|
|
14029
|
+
return error instanceof Error;
|
|
14030
|
+
}
|
|
14031
|
+
async function pathExists(filePath) {
|
|
14032
|
+
try {
|
|
14033
|
+
await fs.access(filePath);
|
|
14034
|
+
return true;
|
|
14035
|
+
} catch {
|
|
14036
|
+
return false;
|
|
14037
|
+
}
|
|
14038
|
+
}
|
|
14039
|
+
async function collectMarkdownFiles(dir) {
|
|
14040
|
+
return (await glob("**/*.md", {
|
|
14041
|
+
cwd: dir,
|
|
14042
|
+
absolute: true,
|
|
14043
|
+
onlyFiles: true
|
|
14044
|
+
})).sort();
|
|
13861
14045
|
}
|
|
13862
14046
|
async function selectMarkdownFile(outputDir, basename) {
|
|
13863
14047
|
const files = await collectMarkdownFiles(outputDir);
|
|
@@ -13962,7 +14146,10 @@ var FallbackPdfConverter = class {
|
|
|
13962
14146
|
try {
|
|
13963
14147
|
return await this.primary.convert(input, filePath);
|
|
13964
14148
|
} catch (err) {
|
|
13965
|
-
consola.warn(
|
|
14149
|
+
consola.warn(t("command.extract.file.errorProcessing", {
|
|
14150
|
+
name: this.primary.name,
|
|
14151
|
+
error: err instanceof Error ? err.message : String(err)
|
|
14152
|
+
}));
|
|
13966
14153
|
consola.info(`Falling back to ${this.fallback.name}`);
|
|
13967
14154
|
const result = await this.fallback.convert(input, filePath);
|
|
13968
14155
|
return {
|
|
@@ -13994,14 +14181,14 @@ function createPdfConverter(config) {
|
|
|
13994
14181
|
return withFallback(new ExternalCommandPdfConverter("marker", markerConfig), markerConfig);
|
|
13995
14182
|
}
|
|
13996
14183
|
if (config.converter === "external") {
|
|
13997
|
-
if (!config.external) throw new Error("
|
|
14184
|
+
if (!config.external) throw new Error(t("errors.pdf.externalNotConfigured"));
|
|
13998
14185
|
return withFallback(new ExternalCommandPdfConverter("external", config.external), config.external);
|
|
13999
14186
|
}
|
|
14000
14187
|
}
|
|
14001
14188
|
const key = typeof config === "string" ? config : "unpdf";
|
|
14002
14189
|
let instance = registry.get(key);
|
|
14003
14190
|
if (!instance) {
|
|
14004
|
-
if (key !== "unpdf") throw new Error(
|
|
14191
|
+
if (key !== "unpdf") throw new Error(t("errors.pdf.converterRequiresConfig", { name: key }));
|
|
14005
14192
|
instance = new UnpdfConverter();
|
|
14006
14193
|
registry.set(key, instance);
|
|
14007
14194
|
}
|
|
@@ -14009,22 +14196,7 @@ function createPdfConverter(config) {
|
|
|
14009
14196
|
}
|
|
14010
14197
|
|
|
14011
14198
|
//#endregion
|
|
14012
|
-
//#region src/
|
|
14013
|
-
/**
|
|
14014
|
-
* Helper to compute SHA-256 hash of a file asynchronously.
|
|
14015
|
-
*/
|
|
14016
|
-
function getFileHash(filePath) {
|
|
14017
|
-
return new Promise((resolve, reject) => {
|
|
14018
|
-
const hash = crypto.createHash("sha256");
|
|
14019
|
-
const stream = fs$1.createReadStream(filePath);
|
|
14020
|
-
stream.on("data", (data) => hash.update(data));
|
|
14021
|
-
stream.on("end", () => resolve(hash.digest("hex")));
|
|
14022
|
-
stream.on("error", (err) => reject(err));
|
|
14023
|
-
});
|
|
14024
|
-
}
|
|
14025
|
-
|
|
14026
|
-
//#endregion
|
|
14027
|
-
//#region src/core/extract-runner.ts
|
|
14199
|
+
//#region src/core/pdf-converter/orchestrator.ts
|
|
14028
14200
|
const FILE_PART_EXTENSIONS = new Set([
|
|
14029
14201
|
"png",
|
|
14030
14202
|
"jpg",
|
|
@@ -14034,6 +14206,51 @@ const FILE_PART_EXTENSIONS = new Set([
|
|
|
14034
14206
|
"bmp",
|
|
14035
14207
|
"svg"
|
|
14036
14208
|
]);
|
|
14209
|
+
const PDF_EXT_RE = /\.pdf$/i;
|
|
14210
|
+
async function readExtractFileInput(filePath, aiConfig, modelOverride) {
|
|
14211
|
+
const stat = fs$1.statSync(filePath);
|
|
14212
|
+
if (stat.size > MAX_UPLOAD_SIZE) throw new Error(t("errors.file.sizeExceeded", {
|
|
14213
|
+
size: bytesToMB(stat.size).toFixed(1),
|
|
14214
|
+
limit: MAX_UPLOAD_SIZE_TEXT,
|
|
14215
|
+
file: filePath
|
|
14216
|
+
}));
|
|
14217
|
+
const ext = path.extname(filePath).toLowerCase().replace(".", "");
|
|
14218
|
+
if (FILE_PART_EXTENSIONS.has(ext)) {
|
|
14219
|
+
if (shouldUseImageOcrFallback(aiConfig, modelOverride)) {
|
|
14220
|
+
const result = await recognizeImageText(filePath, aiConfig?.image);
|
|
14221
|
+
consola.info(t("extract.file.ocrText", { confidence: (result.confidence * 100).toFixed(1) }));
|
|
14222
|
+
return { text: result.text };
|
|
14223
|
+
}
|
|
14224
|
+
return {
|
|
14225
|
+
text: "",
|
|
14226
|
+
filePath
|
|
14227
|
+
};
|
|
14228
|
+
}
|
|
14229
|
+
if (ext === "pdf") {
|
|
14230
|
+
const buffer = await fs.readFile(filePath);
|
|
14231
|
+
const converter = createPdfConverter(aiConfig?.pdf);
|
|
14232
|
+
const result = await converter.convert(buffer, filePath);
|
|
14233
|
+
if (result.metadata?.fallback === "true") consola.info(t("extract.file.pdfFallback", { count: result.pageCount }));
|
|
14234
|
+
else consola.info(t("extract.file.pdfConverted", {
|
|
14235
|
+
name: converter.name,
|
|
14236
|
+
count: result.pageCount
|
|
14237
|
+
}));
|
|
14238
|
+
const mdPath = filePath.replace(PDF_EXT_RE, ".md");
|
|
14239
|
+
try {
|
|
14240
|
+
await fs.writeFile(mdPath, result.text);
|
|
14241
|
+
consola.info(t("extract.file.markdownSaved", { path: mdPath }));
|
|
14242
|
+
} catch {
|
|
14243
|
+
const fallbackMd = path.join(os.tmpdir(), `${path.basename(filePath, ".pdf")}.md`);
|
|
14244
|
+
await fs.writeFile(fallbackMd, result.text);
|
|
14245
|
+
consola.info(t("extract.file.markdownSaved", { path: fallbackMd }));
|
|
14246
|
+
}
|
|
14247
|
+
return { text: result.text };
|
|
14248
|
+
}
|
|
14249
|
+
return { text: await fs.readFile(filePath, "utf-8") };
|
|
14250
|
+
}
|
|
14251
|
+
|
|
14252
|
+
//#endregion
|
|
14253
|
+
//#region src/core/batch/batch-processor.ts
|
|
14037
14254
|
const SUPPORTED_EXTENSIONS$1 = new Set([
|
|
14038
14255
|
...FILE_PART_EXTENSIONS,
|
|
14039
14256
|
"pdf",
|
|
@@ -14046,50 +14263,114 @@ const SUPPORTED_EXTENSIONS$1 = new Set([
|
|
|
14046
14263
|
"yaml",
|
|
14047
14264
|
"yml"
|
|
14048
14265
|
]);
|
|
14049
|
-
const PDF_EXT_RE = /\.pdf$/i;
|
|
14050
|
-
const JSON_EXT_RE$1 = /\.json$/;
|
|
14051
14266
|
const SUPPORTED_FILE_PATTERN = `*.{${[...SUPPORTED_EXTENSIONS$1].join(",")}}`;
|
|
14052
|
-
|
|
14053
|
-
if (!
|
|
14054
|
-
|
|
14055
|
-
|
|
14056
|
-
|
|
14057
|
-
|
|
14058
|
-
}
|
|
14267
|
+
function listSupportedFiles(dir, pattern) {
|
|
14268
|
+
if (!fs$1.statSync(dir).isDirectory()) throw new Error(t("errors.file.notADirectory", { dir }));
|
|
14269
|
+
return globSync(pattern ?? SUPPORTED_FILE_PATTERN, {
|
|
14270
|
+
cwd: dir,
|
|
14271
|
+
absolute: true,
|
|
14272
|
+
onlyFiles: true
|
|
14273
|
+
}).filter((file) => {
|
|
14274
|
+
const ext = path.extname(file).toLowerCase().replace(".", "");
|
|
14275
|
+
return SUPPORTED_EXTENSIONS$1.has(ext);
|
|
14276
|
+
}).sort();
|
|
14059
14277
|
}
|
|
14060
|
-
function
|
|
14061
|
-
|
|
14278
|
+
async function processOneFile(aiexDir, config, aiConfig, schemaName, filePath, modelOverride, options) {
|
|
14279
|
+
const result = await runAuditedExtraction({
|
|
14280
|
+
aiexDir,
|
|
14281
|
+
config,
|
|
14282
|
+
aiConfig,
|
|
14283
|
+
schemaName,
|
|
14284
|
+
source: {
|
|
14285
|
+
type: "file",
|
|
14286
|
+
filePath
|
|
14287
|
+
},
|
|
14288
|
+
modelOverride,
|
|
14289
|
+
insert: options?.insert,
|
|
14290
|
+
force: options?.force,
|
|
14291
|
+
quiet: false
|
|
14292
|
+
});
|
|
14293
|
+
if (result.success) {
|
|
14294
|
+
if (!result.skipped) consola.success(t("extract.file.processSuccess", { file: path.basename(filePath) }));
|
|
14295
|
+
return true;
|
|
14296
|
+
}
|
|
14297
|
+
return false;
|
|
14298
|
+
}
|
|
14299
|
+
async function runBatchExtraction(aiexDir, config, aiConfig, schemaName, dir, globPattern, modelOverride, options) {
|
|
14300
|
+
consola.info(t("extract.batch.scanning", { dir: pc.cyan(dir) }));
|
|
14301
|
+
let files;
|
|
14302
|
+
try {
|
|
14303
|
+
files = listSupportedFiles(dir, globPattern);
|
|
14304
|
+
} catch {
|
|
14305
|
+
return {
|
|
14306
|
+
ok: false,
|
|
14307
|
+
successCount: 0,
|
|
14308
|
+
failCount: 0,
|
|
14309
|
+
error: t("extract.batch.errors.cannotReadDir", { dir })
|
|
14310
|
+
};
|
|
14311
|
+
}
|
|
14312
|
+
if (files.length === 0) return {
|
|
14313
|
+
ok: false,
|
|
14314
|
+
successCount: 0,
|
|
14315
|
+
failCount: 0,
|
|
14316
|
+
error: t("extract.batch.errors.noSupportedFiles", { dir })
|
|
14317
|
+
};
|
|
14318
|
+
consola.info(t("extract.batch.found", { count: files.length }));
|
|
14319
|
+
let successCount = 0;
|
|
14320
|
+
let failCount = 0;
|
|
14321
|
+
for (let i = 0; i < files.length; i++) {
|
|
14322
|
+
const file = files[i];
|
|
14323
|
+
consola.info(`\n${t("extract.batch.processing", {
|
|
14324
|
+
current: i + 1,
|
|
14325
|
+
total: files.length,
|
|
14326
|
+
file: pc.cyan(path.basename(file))
|
|
14327
|
+
})}`);
|
|
14328
|
+
if (await processOneFile(aiexDir, config, aiConfig, schemaName, file, modelOverride, {
|
|
14329
|
+
insert: options?.insert,
|
|
14330
|
+
force: options?.force
|
|
14331
|
+
})) successCount++;
|
|
14332
|
+
else failCount++;
|
|
14333
|
+
}
|
|
14334
|
+
consola.info(`\n${t("extract.batch.complete", {
|
|
14335
|
+
success: pc.green(successCount),
|
|
14336
|
+
fail: pc.red(failCount),
|
|
14337
|
+
total: files.length
|
|
14338
|
+
})}`);
|
|
14339
|
+
return {
|
|
14340
|
+
ok: true,
|
|
14341
|
+
successCount,
|
|
14342
|
+
failCount
|
|
14343
|
+
};
|
|
14062
14344
|
}
|
|
14345
|
+
|
|
14346
|
+
//#endregion
|
|
14347
|
+
//#region src/core/extract-runner.ts
|
|
14348
|
+
const JSON_EXT_RE$1 = /\.json$/;
|
|
14063
14349
|
async function ensureDatabaseReady(dbPath, schema) {
|
|
14064
14350
|
try {
|
|
14065
14351
|
await fs.access(dbPath);
|
|
14066
14352
|
} catch {
|
|
14067
|
-
return
|
|
14353
|
+
return t("errors.db.notFound", {
|
|
14354
|
+
path: pc.cyan(".aiex/database.db"),
|
|
14355
|
+
cmd: pc.cyan("aiex schema")
|
|
14356
|
+
});
|
|
14068
14357
|
}
|
|
14069
14358
|
try {
|
|
14070
14359
|
const result = parseJsonSchema(schema);
|
|
14071
14360
|
const db = new Database(dbPath);
|
|
14072
14361
|
try {
|
|
14073
|
-
for (const table of result.tables) if (!db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`).get(table.name)) return
|
|
14362
|
+
for (const table of result.tables) if (!db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`).get(table.name)) return t("errors.db.tableNotFound", {
|
|
14363
|
+
name: table.name,
|
|
14364
|
+
cmd: pc.cyan("aiex schema")
|
|
14365
|
+
});
|
|
14074
14366
|
} finally {
|
|
14075
14367
|
db.close();
|
|
14076
14368
|
}
|
|
14077
14369
|
} catch (e) {
|
|
14078
|
-
return
|
|
14370
|
+
return t("errors.db.cannotVerify", { error: e instanceof Error ? e.message : String(e) });
|
|
14079
14371
|
}
|
|
14080
14372
|
return null;
|
|
14081
14373
|
}
|
|
14082
|
-
function listSupportedFiles(dir, pattern) {
|
|
14083
|
-
if (!fs$1.statSync(dir).isDirectory()) throw new Error(`Not a directory: ${dir}`);
|
|
14084
|
-
return globSync(pattern ?? SUPPORTED_FILE_PATTERN, {
|
|
14085
|
-
cwd: dir,
|
|
14086
|
-
absolute: true,
|
|
14087
|
-
onlyFiles: true
|
|
14088
|
-
}).filter((file) => {
|
|
14089
|
-
const ext = path.extname(file).toLowerCase().replace(".", "");
|
|
14090
|
-
return SUPPORTED_EXTENSIONS$1.has(ext);
|
|
14091
|
-
}).sort();
|
|
14092
|
-
}
|
|
14093
14374
|
async function loadSchema(config, schemaName) {
|
|
14094
14375
|
const schemaPath = path.join(config.schemaPath, `${schemaName}.json`);
|
|
14095
14376
|
try {
|
|
@@ -14098,15 +14379,18 @@ async function loadSchema(config, schemaName) {
|
|
|
14098
14379
|
} catch (e) {
|
|
14099
14380
|
if (e instanceof ZodError) return {
|
|
14100
14381
|
schema: null,
|
|
14101
|
-
error:
|
|
14382
|
+
error: t("errors.schema.validationFailed", {
|
|
14383
|
+
name: `${schemaName}.json`,
|
|
14384
|
+
issues: e.issues.map((i) => ` - ${i.path.join(".")}: ${i.message}`).join("\n")
|
|
14385
|
+
})
|
|
14102
14386
|
};
|
|
14103
14387
|
if (e.code === "ENOENT") return {
|
|
14104
14388
|
schema: null,
|
|
14105
|
-
error:
|
|
14389
|
+
error: t("errors.schema.cannotRead", { name: `${schemaName}.json` })
|
|
14106
14390
|
};
|
|
14107
14391
|
if (e instanceof SyntaxError) return {
|
|
14108
14392
|
schema: null,
|
|
14109
|
-
error:
|
|
14393
|
+
error: t("errors.schema.invalidJson", { name: `${schemaName}.json` })
|
|
14110
14394
|
};
|
|
14111
14395
|
return {
|
|
14112
14396
|
schema: null,
|
|
@@ -14122,40 +14406,6 @@ async function listSchemas(aiexDir) {
|
|
|
14122
14406
|
return [];
|
|
14123
14407
|
}
|
|
14124
14408
|
}
|
|
14125
|
-
async function readExtractFileInput(filePath, aiConfig, modelOverride) {
|
|
14126
|
-
const stat = fs$1.statSync(filePath);
|
|
14127
|
-
if (stat.size > MAX_UPLOAD_SIZE) throw new Error(`File size (${bytesToMB(stat.size).toFixed(1)}MB) exceeds ${MAX_UPLOAD_SIZE_TEXT} limit: ${filePath}`);
|
|
14128
|
-
const ext = path.extname(filePath).toLowerCase().replace(".", "");
|
|
14129
|
-
if (FILE_PART_EXTENSIONS.has(ext)) {
|
|
14130
|
-
if (shouldUseImageOcrFallback(aiConfig, modelOverride)) {
|
|
14131
|
-
const result = await recognizeImageText(filePath, aiConfig?.image);
|
|
14132
|
-
consola.info(`Extracted image text via local OCR (confidence: ${(result.confidence * 100).toFixed(1)}%)`);
|
|
14133
|
-
return { text: result.text };
|
|
14134
|
-
}
|
|
14135
|
-
return {
|
|
14136
|
-
text: "",
|
|
14137
|
-
filePath
|
|
14138
|
-
};
|
|
14139
|
-
}
|
|
14140
|
-
if (ext === "pdf") {
|
|
14141
|
-
const buffer = await fs.readFile(filePath);
|
|
14142
|
-
const converter = createPdfConverter(aiConfig?.pdf);
|
|
14143
|
-
const result = await converter.convert(buffer, filePath);
|
|
14144
|
-
if (result.metadata?.fallback === "true") consola.info(`Fell back to unpdf — ${result.pageCount} page(s) extracted`);
|
|
14145
|
-
else consola.info(`Converted PDF via ${converter.name}, ${result.pageCount} page(s)`);
|
|
14146
|
-
const mdPath = filePath.replace(PDF_EXT_RE, ".md");
|
|
14147
|
-
try {
|
|
14148
|
-
await fs.writeFile(mdPath, result.text);
|
|
14149
|
-
consola.info(`Markdown saved: ${mdPath}`);
|
|
14150
|
-
} catch {
|
|
14151
|
-
const fallbackMd = path.join(os.tmpdir(), `${path.basename(filePath, ".pdf")}.md`);
|
|
14152
|
-
await fs.writeFile(fallbackMd, result.text);
|
|
14153
|
-
consola.info(`Markdown saved: ${fallbackMd}`);
|
|
14154
|
-
}
|
|
14155
|
-
return { text: result.text };
|
|
14156
|
-
}
|
|
14157
|
-
return { text: await fs.readFile(filePath, "utf-8") };
|
|
14158
|
-
}
|
|
14159
14409
|
async function extractSingle(aiexDir, config, aiConfig, schemaName, text$1, filePath, modelOverride, options) {
|
|
14160
14410
|
const schemaLoad = await loadSchema(config, schemaName);
|
|
14161
14411
|
if (!schemaLoad.schema) {
|
|
@@ -14166,7 +14416,7 @@ async function extractSingle(aiexDir, config, aiConfig, schemaName, text$1, file
|
|
|
14166
14416
|
};
|
|
14167
14417
|
}
|
|
14168
14418
|
const s = spinner();
|
|
14169
|
-
if (!options?.quiet) s.start(filePath ?
|
|
14419
|
+
if (!options?.quiet) s.start(filePath ? t("extract.file.extractedFrom", { file: path.basename(filePath) }) : t("extract.file.extracting"));
|
|
14170
14420
|
const result = await extractStructuredData({
|
|
14171
14421
|
config: aiConfig,
|
|
14172
14422
|
schema: schemaLoad.schema,
|
|
@@ -14175,28 +14425,37 @@ async function extractSingle(aiexDir, config, aiConfig, schemaName, text$1, file
|
|
|
14175
14425
|
file: filePath,
|
|
14176
14426
|
modelOverride,
|
|
14177
14427
|
onRetry(info) {
|
|
14178
|
-
if (!options?.quiet) s.message(
|
|
14428
|
+
if (!options?.quiet) s.message(t("extract.file.extractRetry", {
|
|
14429
|
+
code: info.statusCode,
|
|
14430
|
+
delay: info.delayMs / 1e3,
|
|
14431
|
+
attempt: info.attempt,
|
|
14432
|
+
max: info.maxRetries
|
|
14433
|
+
}));
|
|
14179
14434
|
}
|
|
14180
14435
|
});
|
|
14181
14436
|
if (!result.success) {
|
|
14182
14437
|
if (!options?.quiet) {
|
|
14183
|
-
s.stop("
|
|
14184
|
-
consola.error(result.error || "
|
|
14438
|
+
s.stop(t("extract.file.extractFail"));
|
|
14439
|
+
consola.error(result.error || t("common.unknownError"));
|
|
14185
14440
|
}
|
|
14186
14441
|
return {
|
|
14187
14442
|
success: false,
|
|
14188
|
-
error: result.error || "
|
|
14443
|
+
error: result.error || t("common.unknownError")
|
|
14189
14444
|
};
|
|
14190
14445
|
}
|
|
14191
|
-
if (!options?.quiet) s.stop("
|
|
14192
|
-
if (result.outputPath && !options?.quiet) consola.success(
|
|
14193
|
-
if (result.tokensUsed && !options?.quiet) consola.info(pc.gray(
|
|
14446
|
+
if (!options?.quiet) s.stop(t("extract.file.extractComplete"));
|
|
14447
|
+
if (result.outputPath && !options?.quiet) consola.success(t("extract.file.resultSaved", { path: pc.cyan(result.outputPath) }));
|
|
14448
|
+
if (result.tokensUsed && !options?.quiet) consola.info(pc.gray(t("extract.file.tokenUsage", {
|
|
14449
|
+
prompt: result.tokensUsed.prompt,
|
|
14450
|
+
completion: result.tokensUsed.completion,
|
|
14451
|
+
total: result.tokensUsed.total
|
|
14452
|
+
})));
|
|
14194
14453
|
if (result.data && options?.insert !== false) {
|
|
14195
14454
|
const s2 = spinner();
|
|
14196
|
-
if (!options?.quiet) s2.start("
|
|
14455
|
+
if (!options?.quiet) s2.start(t("extract.file.insertingDb"));
|
|
14197
14456
|
const dbError = await ensureDatabaseReady(config.databasePath, schemaLoad.schema);
|
|
14198
14457
|
if (dbError) {
|
|
14199
|
-
if (!options?.quiet) s2.stop("
|
|
14458
|
+
if (!options?.quiet) s2.stop(t("extract.file.dbNotReady"));
|
|
14200
14459
|
consola.error(dbError);
|
|
14201
14460
|
return {
|
|
14202
14461
|
success: false,
|
|
@@ -14208,7 +14467,7 @@ async function extractSingle(aiexDir, config, aiConfig, schemaName, text$1, file
|
|
|
14208
14467
|
try {
|
|
14209
14468
|
const insertResult = insertExtractedData(db, schemaLoad.schema, result.data);
|
|
14210
14469
|
if (insertResult.success) {
|
|
14211
|
-
if (!options?.quiet) s2.stop(
|
|
14470
|
+
if (!options?.quiet) s2.stop(t("extract.file.insertedTables", { count: insertResult.tablesInserted.length }));
|
|
14212
14471
|
return {
|
|
14213
14472
|
success: true,
|
|
14214
14473
|
outputPath: result.outputPath,
|
|
@@ -14217,8 +14476,8 @@ async function extractSingle(aiexDir, config, aiConfig, schemaName, text$1, file
|
|
|
14217
14476
|
tokensUsed: result.tokensUsed
|
|
14218
14477
|
};
|
|
14219
14478
|
} else {
|
|
14220
|
-
if (!options?.quiet) s2.stop("
|
|
14221
|
-
consola.error(insertResult.error || "
|
|
14479
|
+
if (!options?.quiet) s2.stop(t("extract.file.dbInsertFail"));
|
|
14480
|
+
consola.error(insertResult.error || t("common.unknownError"));
|
|
14222
14481
|
return {
|
|
14223
14482
|
success: false,
|
|
14224
14483
|
error: insertResult.error
|
|
@@ -14228,7 +14487,7 @@ async function extractSingle(aiexDir, config, aiConfig, schemaName, text$1, file
|
|
|
14228
14487
|
db.close();
|
|
14229
14488
|
}
|
|
14230
14489
|
} catch (e) {
|
|
14231
|
-
if (!options?.quiet) s2.stop("
|
|
14490
|
+
if (!options?.quiet) s2.stop(t("extract.file.dbInsertFail"));
|
|
14232
14491
|
consola.error(e instanceof Error ? e.message : String(e));
|
|
14233
14492
|
return {
|
|
14234
14493
|
success: false,
|
|
@@ -14262,12 +14521,18 @@ async function runAuditedExtraction(options) {
|
|
|
14262
14521
|
try {
|
|
14263
14522
|
fileHash = await getFileHash(source.filePath);
|
|
14264
14523
|
} catch (e) {
|
|
14265
|
-
if (!quiet) consola.warn(
|
|
14524
|
+
if (!quiet) consola.warn(t("extract.file.hashWarning", {
|
|
14525
|
+
file: path.basename(source.filePath),
|
|
14526
|
+
error: e instanceof Error ? e.message : String(e)
|
|
14527
|
+
}));
|
|
14266
14528
|
}
|
|
14267
14529
|
if (fileHash && !isPlainTextFile && !force) {
|
|
14268
14530
|
const existing = await findSucceededAuditByHash(aiexDir, schemaName, fileHash);
|
|
14269
14531
|
if (existing) {
|
|
14270
|
-
if (!quiet) consola.info(
|
|
14532
|
+
if (!quiet) consola.info(t("extract.file.alreadyProcessed", {
|
|
14533
|
+
file: pc.cyan(path.basename(source.filePath)),
|
|
14534
|
+
hash: fileHash.slice(0, 8)
|
|
14535
|
+
}));
|
|
14271
14536
|
return {
|
|
14272
14537
|
success: true,
|
|
14273
14538
|
skipped: true,
|
|
@@ -14312,7 +14577,7 @@ async function runAuditedExtraction(options) {
|
|
|
14312
14577
|
let notionPages;
|
|
14313
14578
|
if (shouldSyncNotion(aiConfig, schemaName)) try {
|
|
14314
14579
|
notionPages = await syncResultToNotion(aiConfig, schemaName, r.data);
|
|
14315
|
-
if (!quiet) consola.success(
|
|
14580
|
+
if (!quiet) consola.success(t("extract.file.notionSynced", { count: notionPages.length }));
|
|
14316
14581
|
} catch (error) {
|
|
14317
14582
|
await updateExtractionAuditRecord(aiexDir, audit.id, {
|
|
14318
14583
|
status: "failed",
|
|
@@ -14322,7 +14587,8 @@ async function runAuditedExtraction(options) {
|
|
|
14322
14587
|
tokensUsed: r.tokensUsed,
|
|
14323
14588
|
error: error instanceof Error ? error.message : String(error)
|
|
14324
14589
|
});
|
|
14325
|
-
if (!quiet) consola.error(
|
|
14590
|
+
if (!quiet) consola.error(t("extract.file.notionSyncFail", { error: error instanceof Error ? error.message : String(error) }));
|
|
14591
|
+
await triggerWebhook(aiConfig, audit.id, schemaName, "extraction.failed", source, r.data, error instanceof Error ? error.message : String(error), r.tokensUsed, quiet);
|
|
14326
14592
|
return {
|
|
14327
14593
|
success: false,
|
|
14328
14594
|
error: error instanceof Error ? error.message : String(error),
|
|
@@ -14338,6 +14604,7 @@ async function runAuditedExtraction(options) {
|
|
|
14338
14604
|
notionPages,
|
|
14339
14605
|
tokensUsed: r.tokensUsed
|
|
14340
14606
|
});
|
|
14607
|
+
await triggerWebhook(aiConfig, audit.id, schemaName, "extraction.success", source, r.data, void 0, r.tokensUsed, quiet);
|
|
14341
14608
|
return {
|
|
14342
14609
|
success: true,
|
|
14343
14610
|
outputPath: updated.outputPath,
|
|
@@ -14353,7 +14620,8 @@ async function runAuditedExtraction(options) {
|
|
|
14353
14620
|
status: "failed",
|
|
14354
14621
|
error: r.error || "Extraction failed"
|
|
14355
14622
|
});
|
|
14356
|
-
if (!quiet) consola.error(
|
|
14623
|
+
if (!quiet) consola.error(t("extract.file.extractionFailed", { error: r.error }));
|
|
14624
|
+
await triggerWebhook(aiConfig, audit.id, schemaName, "extraction.failed", source, void 0, r.error || "Extraction failed", void 0, quiet);
|
|
14357
14625
|
return {
|
|
14358
14626
|
success: false,
|
|
14359
14627
|
error: r.error,
|
|
@@ -14368,8 +14636,12 @@ async function runAuditedExtraction(options) {
|
|
|
14368
14636
|
});
|
|
14369
14637
|
if (!quiet) {
|
|
14370
14638
|
const name$1 = source.type === "file" ? path.basename(source.filePath) : "text input";
|
|
14371
|
-
consola.error(
|
|
14639
|
+
consola.error(t("extract.file.errorProcessing", {
|
|
14640
|
+
name: name$1,
|
|
14641
|
+
error: e instanceof Error ? e.message : String(e)
|
|
14642
|
+
}));
|
|
14372
14643
|
}
|
|
14644
|
+
await triggerWebhook(aiConfig, audit.id, schemaName, "extraction.failed", source, void 0, e instanceof Error ? e.message : String(e), void 0, quiet);
|
|
14373
14645
|
return {
|
|
14374
14646
|
success: false,
|
|
14375
14647
|
error: e instanceof Error ? e.message : String(e),
|
|
@@ -14378,99 +14650,41 @@ async function runAuditedExtraction(options) {
|
|
|
14378
14650
|
};
|
|
14379
14651
|
}
|
|
14380
14652
|
}
|
|
14381
|
-
async function processOneFile(aiexDir, config, aiConfig, schemaName, filePath, modelOverride, options) {
|
|
14382
|
-
const result = await runAuditedExtraction({
|
|
14383
|
-
aiexDir,
|
|
14384
|
-
config,
|
|
14385
|
-
aiConfig,
|
|
14386
|
-
schemaName,
|
|
14387
|
-
source: {
|
|
14388
|
-
type: "file",
|
|
14389
|
-
filePath
|
|
14390
|
-
},
|
|
14391
|
-
modelOverride,
|
|
14392
|
-
insert: options?.insert,
|
|
14393
|
-
force: options?.force,
|
|
14394
|
-
quiet: false
|
|
14395
|
-
});
|
|
14396
|
-
if (result.success) {
|
|
14397
|
-
if (!result.skipped) consola.success(`Processed: ${path.basename(filePath)}`);
|
|
14398
|
-
return true;
|
|
14399
|
-
}
|
|
14400
|
-
return false;
|
|
14401
|
-
}
|
|
14402
|
-
async function runBatchExtraction(aiexDir, config, aiConfig, schemaName, dir, globPattern, modelOverride, options) {
|
|
14403
|
-
consola.info(`Scanning ${pc.cyan(dir)} for supported files...`);
|
|
14404
|
-
let files;
|
|
14405
|
-
try {
|
|
14406
|
-
files = listSupportedFiles(dir, globPattern);
|
|
14407
|
-
} catch {
|
|
14408
|
-
return {
|
|
14409
|
-
ok: false,
|
|
14410
|
-
successCount: 0,
|
|
14411
|
-
failCount: 0,
|
|
14412
|
-
error: `Cannot read directory: ${dir}`
|
|
14413
|
-
};
|
|
14414
|
-
}
|
|
14415
|
-
if (files.length === 0) return {
|
|
14416
|
-
ok: false,
|
|
14417
|
-
successCount: 0,
|
|
14418
|
-
failCount: 0,
|
|
14419
|
-
error: `No supported files found in ${dir}`
|
|
14420
|
-
};
|
|
14421
|
-
consola.info(`Found ${files.length} file(s) to process`);
|
|
14422
|
-
let successCount = 0;
|
|
14423
|
-
let failCount = 0;
|
|
14424
|
-
for (let i = 0; i < files.length; i++) {
|
|
14425
|
-
const file = files[i];
|
|
14426
|
-
consola.info(`\n[${i + 1}/${files.length}] Processing: ${pc.cyan(path.basename(file))}`);
|
|
14427
|
-
if (await processOneFile(aiexDir, config, aiConfig, schemaName, file, modelOverride, {
|
|
14428
|
-
insert: options?.insert,
|
|
14429
|
-
force: options?.force
|
|
14430
|
-
})) successCount++;
|
|
14431
|
-
else failCount++;
|
|
14432
|
-
}
|
|
14433
|
-
consola.info(`\nBatch complete: ${pc.green(`${successCount} succeeded`)}, ${pc.red(`${failCount} failed`)}, ${files.length} total`);
|
|
14434
|
-
return {
|
|
14435
|
-
ok: true,
|
|
14436
|
-
successCount,
|
|
14437
|
-
failCount
|
|
14438
|
-
};
|
|
14439
|
-
}
|
|
14440
14653
|
|
|
14441
14654
|
//#endregion
|
|
14442
14655
|
//#region src/commands/dump.ts
|
|
14443
14656
|
const dumpCommand = defineCommand({
|
|
14444
14657
|
meta: {
|
|
14445
14658
|
name: "dump",
|
|
14446
|
-
description: "
|
|
14659
|
+
description: t("command.dump.description")
|
|
14447
14660
|
},
|
|
14448
14661
|
args: {
|
|
14449
14662
|
table: {
|
|
14450
14663
|
type: "string",
|
|
14451
14664
|
alias: "t",
|
|
14452
|
-
description: "
|
|
14665
|
+
description: t("command.dump.args.table")
|
|
14453
14666
|
},
|
|
14454
14667
|
schema: {
|
|
14455
14668
|
type: "string",
|
|
14456
14669
|
alias: "s",
|
|
14457
|
-
description: "
|
|
14670
|
+
description: t("command.dump.args.schema")
|
|
14458
14671
|
},
|
|
14459
14672
|
format: {
|
|
14460
14673
|
type: "string",
|
|
14461
14674
|
alias: "f",
|
|
14462
|
-
description: "
|
|
14675
|
+
description: t("command.dump.args.format")
|
|
14463
14676
|
},
|
|
14464
14677
|
output: {
|
|
14465
14678
|
type: "string",
|
|
14466
14679
|
alias: "o",
|
|
14467
|
-
description: "
|
|
14680
|
+
description: t("command.dump.args.output")
|
|
14468
14681
|
}
|
|
14469
14682
|
},
|
|
14470
14683
|
async run({ args }) {
|
|
14471
14684
|
intro(pc.inverse(" aiex dump "));
|
|
14685
|
+
await initI18n();
|
|
14472
14686
|
if (!args.table && !args.schema) {
|
|
14473
|
-
failCommand(
|
|
14687
|
+
failCommand(t("command.dump.errors.tableOrSchemaRequired"));
|
|
14474
14688
|
return;
|
|
14475
14689
|
}
|
|
14476
14690
|
const cwd = process.cwd();
|
|
@@ -14481,17 +14695,20 @@ const dumpCommand = defineCommand({
|
|
|
14481
14695
|
if (args.schema) {
|
|
14482
14696
|
const schemaLoad = await loadSchema(config, args.schema);
|
|
14483
14697
|
if (!schemaLoad.schema) {
|
|
14484
|
-
failCommand(schemaLoad.error ||
|
|
14698
|
+
failCommand(schemaLoad.error || t("command.dump.errors.schemaNotFound", { name: args.schema }));
|
|
14485
14699
|
return;
|
|
14486
14700
|
}
|
|
14487
14701
|
schema = schemaLoad.schema;
|
|
14488
14702
|
const tName = schema.table?.name;
|
|
14489
14703
|
if (!tName) {
|
|
14490
|
-
failCommand(
|
|
14704
|
+
failCommand(t("command.dump.errors.noTableName", { name: args.schema }));
|
|
14491
14705
|
return;
|
|
14492
14706
|
}
|
|
14493
14707
|
if (tableName && tableName !== tName) {
|
|
14494
|
-
failCommand(
|
|
14708
|
+
failCommand(t("command.dump.errors.tableMismatch", {
|
|
14709
|
+
table: tableName,
|
|
14710
|
+
schemaTable: tName
|
|
14711
|
+
}));
|
|
14495
14712
|
return;
|
|
14496
14713
|
}
|
|
14497
14714
|
tableName = tName;
|
|
@@ -14518,16 +14735,19 @@ const dumpCommand = defineCommand({
|
|
|
14518
14735
|
}
|
|
14519
14736
|
if (!format) format = "csv";
|
|
14520
14737
|
if (format !== "csv" && format !== "xlsx") {
|
|
14521
|
-
failCommand(
|
|
14738
|
+
failCommand(t("command.dump.errors.unsupportedFormat", { format }));
|
|
14522
14739
|
return;
|
|
14523
14740
|
}
|
|
14524
14741
|
const resolvedOutput = outputPathArg ? path.resolve(outputPathArg) : path.resolve(cwd, `${tableName}.${format}`);
|
|
14525
14742
|
if (!fs$1.existsSync(config.databasePath)) {
|
|
14526
|
-
failCommand(
|
|
14743
|
+
failCommand(t("command.dump.errors.dbNotFound", {
|
|
14744
|
+
path: config.databasePath,
|
|
14745
|
+
cmd: "aiex schema"
|
|
14746
|
+
}));
|
|
14527
14747
|
return;
|
|
14528
14748
|
}
|
|
14529
14749
|
const s = spinner();
|
|
14530
|
-
s.start(
|
|
14750
|
+
s.start(t("command.dump.loading", { name: tableName }));
|
|
14531
14751
|
let columns = [];
|
|
14532
14752
|
let rows = [];
|
|
14533
14753
|
try {
|
|
@@ -14536,8 +14756,11 @@ const dumpCommand = defineCommand({
|
|
|
14536
14756
|
select name from sqlite_master
|
|
14537
14757
|
where type = 'table' and name = ?
|
|
14538
14758
|
`).get(tableName)) {
|
|
14539
|
-
s.stop("
|
|
14540
|
-
failCommand(
|
|
14759
|
+
s.stop(t("command.dump.dbQueryFailed"));
|
|
14760
|
+
failCommand(t("command.dump.errors.tableNotFound", {
|
|
14761
|
+
name: tableName,
|
|
14762
|
+
cmd: "aiex schema"
|
|
14763
|
+
}));
|
|
14541
14764
|
db.close();
|
|
14542
14765
|
return;
|
|
14543
14766
|
}
|
|
@@ -14545,64 +14768,46 @@ const dumpCommand = defineCommand({
|
|
|
14545
14768
|
rows = db.prepare(`select * from ${tableName}`).all();
|
|
14546
14769
|
db.close();
|
|
14547
14770
|
} catch (error) {
|
|
14548
|
-
s.stop("
|
|
14771
|
+
s.stop(t("command.dump.dbQueryFailed"));
|
|
14549
14772
|
failCommand(error instanceof Error ? error.message : String(error));
|
|
14550
14773
|
return;
|
|
14551
14774
|
}
|
|
14552
14775
|
if (rows.length === 0) {
|
|
14553
|
-
s.stop("
|
|
14554
|
-
consola.warn(
|
|
14555
|
-
} else s.stop(
|
|
14776
|
+
s.stop(t("command.dump.emptyTable"));
|
|
14777
|
+
consola.warn(t("command.dump.errors.tableEmpty", { name: tableName }));
|
|
14778
|
+
} else s.stop(t("command.dump.loaded", { count: rows.length }));
|
|
14556
14779
|
const s2 = spinner();
|
|
14557
|
-
s2.start("
|
|
14558
|
-
|
|
14559
|
-
|
|
14560
|
-
|
|
14561
|
-
|
|
14562
|
-
|
|
14563
|
-
|
|
14564
|
-
|
|
14565
|
-
|
|
14566
|
-
|
|
14567
|
-
else if (type === "number" || type === "integer") if (val === "") newRow[colName] = "";
|
|
14568
|
-
else {
|
|
14569
|
-
const num = Number(val);
|
|
14570
|
-
newRow[colName] = Number.isNaN(num) ? val : num;
|
|
14571
|
-
}
|
|
14572
|
-
else if (typeof val === "object") newRow[colName] = JSON.stringify(val);
|
|
14573
|
-
else {
|
|
14574
|
-
const dbType = (col.type || "").toLowerCase();
|
|
14575
|
-
if ((dbType.includes("int") || dbType.includes("real") || dbType.includes("num") || dbType.includes("double") || dbType.includes("float")) && typeof val === "string" && val !== "") {
|
|
14576
|
-
const num = Number(val);
|
|
14577
|
-
newRow[colName] = Number.isNaN(num) ? val : num;
|
|
14578
|
-
} else newRow[colName] = val;
|
|
14579
|
-
}
|
|
14580
|
-
});
|
|
14581
|
-
return newRow;
|
|
14582
|
-
});
|
|
14583
|
-
s2.stop("Data formatted");
|
|
14780
|
+
s2.start(t("command.dump.formatting"));
|
|
14781
|
+
let formattedRows;
|
|
14782
|
+
try {
|
|
14783
|
+
formattedRows = formatRowsConformingToSchema(rows, columns, schema, format);
|
|
14784
|
+
s2.stop(t("command.dump.formatted"));
|
|
14785
|
+
} catch (error) {
|
|
14786
|
+
s2.stop(t("command.dump.dbQueryFailed"));
|
|
14787
|
+
failCommand(error instanceof Error ? error.message : String(error));
|
|
14788
|
+
return;
|
|
14789
|
+
}
|
|
14584
14790
|
const s3 = spinner();
|
|
14585
|
-
s3.start(
|
|
14791
|
+
s3.start(t("command.dump.writing", {
|
|
14792
|
+
format: format.toUpperCase(),
|
|
14793
|
+
path: resolvedOutput
|
|
14794
|
+
}));
|
|
14586
14795
|
try {
|
|
14587
|
-
const
|
|
14796
|
+
const buffer = generateExportBuffer(tableName, formattedRows, columns, format);
|
|
14588
14797
|
const outputDir = path.dirname(resolvedOutput);
|
|
14589
14798
|
if (!fs$1.existsSync(outputDir)) fs$1.mkdirSync(outputDir, { recursive: true });
|
|
14590
|
-
|
|
14591
|
-
|
|
14592
|
-
|
|
14593
|
-
|
|
14594
|
-
|
|
14595
|
-
|
|
14596
|
-
fs$1.writeFileSync(resolvedOutput, "" + csv, "utf8");
|
|
14597
|
-
}
|
|
14598
|
-
s3.stop("Dump completed successfully");
|
|
14599
|
-
consola.success(`Successfully dumped ${rows.length} row(s) to ${pc.cyan(resolvedOutput)}`);
|
|
14799
|
+
fs$1.writeFileSync(resolvedOutput, buffer);
|
|
14800
|
+
s3.stop(t("command.dump.dumpCompleted"));
|
|
14801
|
+
consola.success(t("command.dump.successMsg", {
|
|
14802
|
+
count: rows.length,
|
|
14803
|
+
path: pc.cyan(resolvedOutput)
|
|
14804
|
+
}));
|
|
14600
14805
|
} catch (error) {
|
|
14601
|
-
s3.stop("
|
|
14806
|
+
s3.stop(t("command.dump.fileWriteFailed"));
|
|
14602
14807
|
failCommand(error instanceof Error ? error.message : String(error));
|
|
14603
14808
|
return;
|
|
14604
14809
|
}
|
|
14605
|
-
outro("
|
|
14810
|
+
outro(t("common.done"));
|
|
14606
14811
|
}
|
|
14607
14812
|
});
|
|
14608
14813
|
|
|
@@ -14628,15 +14833,15 @@ function formatSource(source) {
|
|
|
14628
14833
|
async function loadConfiguredAI(aiexDir) {
|
|
14629
14834
|
const aiConfig = await readAIConfig(aiexDir);
|
|
14630
14835
|
if (!aiConfig) {
|
|
14631
|
-
failCommand("
|
|
14836
|
+
failCommand(t("command.extract.errors.noAIConfig", { cmd: "aiex web" }));
|
|
14632
14837
|
return null;
|
|
14633
14838
|
}
|
|
14634
14839
|
if (!aiConfig.provider.apiKey) {
|
|
14635
|
-
failCommand("
|
|
14840
|
+
failCommand(t("command.extract.errors.noApiKey"));
|
|
14636
14841
|
return null;
|
|
14637
14842
|
}
|
|
14638
14843
|
if (!aiConfig.provider.models?.length) {
|
|
14639
|
-
failCommand("
|
|
14844
|
+
failCommand(t("command.extract.errors.noModels"));
|
|
14640
14845
|
return null;
|
|
14641
14846
|
}
|
|
14642
14847
|
return aiConfig;
|
|
@@ -14645,7 +14850,10 @@ function resolveModelOverride(aiConfig, modelName) {
|
|
|
14645
14850
|
if (!modelName) return void 0;
|
|
14646
14851
|
const matched = aiConfig.provider.models.find((m) => m.name === modelName);
|
|
14647
14852
|
if (!matched) {
|
|
14648
|
-
failCommand(
|
|
14853
|
+
failCommand(t("command.extract.errors.modelNotFound", {
|
|
14854
|
+
model: modelName,
|
|
14855
|
+
available: aiConfig.provider.models.map((m) => m.name).join(", ")
|
|
14856
|
+
}));
|
|
14649
14857
|
return null;
|
|
14650
14858
|
}
|
|
14651
14859
|
return matched;
|
|
@@ -14653,13 +14861,13 @@ function resolveModelOverride(aiConfig, modelName) {
|
|
|
14653
14861
|
const historyCommand = defineCommand({
|
|
14654
14862
|
meta: {
|
|
14655
14863
|
name: "history",
|
|
14656
|
-
description: "
|
|
14864
|
+
description: t("command.extract.history.description")
|
|
14657
14865
|
},
|
|
14658
14866
|
async run() {
|
|
14659
14867
|
const config = createMigrationConfig(process.cwd());
|
|
14660
14868
|
const records = await listExtractionAuditRecords(path.dirname(config.schemaPath));
|
|
14661
14869
|
if (records.length === 0) {
|
|
14662
|
-
consola.info("
|
|
14870
|
+
consola.info(t("command.extract.history.empty"));
|
|
14663
14871
|
return;
|
|
14664
14872
|
}
|
|
14665
14873
|
for (const record of records) {
|
|
@@ -14671,22 +14879,22 @@ const historyCommand = defineCommand({
|
|
|
14671
14879
|
const showCommand = defineCommand({
|
|
14672
14880
|
meta: {
|
|
14673
14881
|
name: "show",
|
|
14674
|
-
description: "
|
|
14882
|
+
description: t("command.extract.show.description")
|
|
14675
14883
|
},
|
|
14676
14884
|
args: { id: {
|
|
14677
14885
|
type: "string",
|
|
14678
|
-
description: "
|
|
14886
|
+
description: t("command.extract.show.args.id")
|
|
14679
14887
|
} },
|
|
14680
14888
|
async run({ args }) {
|
|
14681
14889
|
const id = getIdArg(args);
|
|
14682
14890
|
if (!id) {
|
|
14683
|
-
failCommand("
|
|
14891
|
+
failCommand(t("command.extract.history.errors.idRequired"));
|
|
14684
14892
|
return;
|
|
14685
14893
|
}
|
|
14686
14894
|
const config = createMigrationConfig(process.cwd());
|
|
14687
14895
|
const record = await readExtractionAuditRecord(path.dirname(config.schemaPath), id);
|
|
14688
14896
|
if (!record) {
|
|
14689
|
-
failCommand(
|
|
14897
|
+
failCommand(t("command.extract.history.errors.recordNotFound", { id }));
|
|
14690
14898
|
return;
|
|
14691
14899
|
}
|
|
14692
14900
|
consola.info(JSON.stringify(record, null, 2));
|
|
@@ -14695,31 +14903,32 @@ const showCommand = defineCommand({
|
|
|
14695
14903
|
const retryCommand = defineCommand({
|
|
14696
14904
|
meta: {
|
|
14697
14905
|
name: "retry",
|
|
14698
|
-
description: "
|
|
14906
|
+
description: t("command.extract.retry.description")
|
|
14699
14907
|
},
|
|
14700
14908
|
args: {
|
|
14701
14909
|
id: {
|
|
14702
14910
|
type: "string",
|
|
14703
|
-
description: "
|
|
14911
|
+
description: t("command.extract.retry.args.id")
|
|
14704
14912
|
},
|
|
14705
14913
|
noInsert: {
|
|
14706
14914
|
type: "boolean",
|
|
14707
|
-
description: "
|
|
14915
|
+
description: t("command.extract.retry.args.noInsert"),
|
|
14708
14916
|
default: false
|
|
14709
14917
|
}
|
|
14710
14918
|
},
|
|
14711
14919
|
async run({ args }) {
|
|
14712
14920
|
intro(pc.inverse(" aiex extract retry "));
|
|
14921
|
+
await initI18n();
|
|
14713
14922
|
const id = getIdArg(args);
|
|
14714
14923
|
if (!id) {
|
|
14715
|
-
failCommand("
|
|
14924
|
+
failCommand(t("command.extract.history.errors.idRequired"));
|
|
14716
14925
|
return;
|
|
14717
14926
|
}
|
|
14718
14927
|
const config = createMigrationConfig(process.cwd());
|
|
14719
14928
|
const aiexDir = path.dirname(config.schemaPath);
|
|
14720
14929
|
const record = await readExtractionAuditRecord(aiexDir, id);
|
|
14721
14930
|
if (!record) {
|
|
14722
|
-
failCommand(
|
|
14931
|
+
failCommand(t("command.extract.history.errors.recordNotFound", { id }));
|
|
14723
14932
|
return;
|
|
14724
14933
|
}
|
|
14725
14934
|
const aiConfig = await loadConfiguredAI(aiexDir);
|
|
@@ -14742,7 +14951,7 @@ const retryCommand = defineCommand({
|
|
|
14742
14951
|
failCommand(result.error);
|
|
14743
14952
|
return;
|
|
14744
14953
|
}
|
|
14745
|
-
outro("
|
|
14954
|
+
outro(t("common.done"));
|
|
14746
14955
|
} catch (error) {
|
|
14747
14956
|
if (isMissingUploadFileError(error)) {
|
|
14748
14957
|
failCommand(MISSING_UPLOAD_FILE_TEXT);
|
|
@@ -14755,30 +14964,30 @@ const retryCommand = defineCommand({
|
|
|
14755
14964
|
const rmCommand = defineCommand({
|
|
14756
14965
|
meta: {
|
|
14757
14966
|
name: "rm",
|
|
14758
|
-
description: "
|
|
14967
|
+
description: t("command.extract.rm.description")
|
|
14759
14968
|
},
|
|
14760
14969
|
args: { id: {
|
|
14761
14970
|
type: "string",
|
|
14762
|
-
description: "
|
|
14971
|
+
description: t("command.extract.rm.args.id")
|
|
14763
14972
|
} },
|
|
14764
14973
|
async run({ args }) {
|
|
14765
14974
|
const id = getIdArg(args);
|
|
14766
14975
|
if (!id) {
|
|
14767
|
-
failCommand("
|
|
14976
|
+
failCommand(t("command.extract.history.errors.idRequired"));
|
|
14768
14977
|
return;
|
|
14769
14978
|
}
|
|
14770
14979
|
const config = createMigrationConfig(process.cwd());
|
|
14771
14980
|
if (!await deleteExtractionAuditRecord(path.dirname(config.schemaPath), id)) {
|
|
14772
|
-
failCommand(
|
|
14981
|
+
failCommand(t("command.extract.history.errors.recordNotFound", { id }));
|
|
14773
14982
|
return;
|
|
14774
14983
|
}
|
|
14775
|
-
consola.success(
|
|
14984
|
+
consola.success(t("command.extract.history.deleted", { id }));
|
|
14776
14985
|
}
|
|
14777
14986
|
});
|
|
14778
14987
|
const extractCommand = defineCommand({
|
|
14779
14988
|
meta: {
|
|
14780
14989
|
name: "extract",
|
|
14781
|
-
description: "
|
|
14990
|
+
description: t("command.extract.description")
|
|
14782
14991
|
},
|
|
14783
14992
|
subCommands: {
|
|
14784
14993
|
history: historyCommand,
|
|
@@ -14790,46 +14999,47 @@ const extractCommand = defineCommand({
|
|
|
14790
14999
|
schema: {
|
|
14791
15000
|
type: "string",
|
|
14792
15001
|
alias: "s",
|
|
14793
|
-
description: "
|
|
15002
|
+
description: t("command.extract.args.schema")
|
|
14794
15003
|
},
|
|
14795
15004
|
file: {
|
|
14796
15005
|
type: "string",
|
|
14797
15006
|
alias: "f",
|
|
14798
|
-
description:
|
|
15007
|
+
description: t("command.extract.args.file", { types: SUPPORTED_FILE_TYPES_TEXT })
|
|
14799
15008
|
},
|
|
14800
15009
|
model: {
|
|
14801
15010
|
type: "string",
|
|
14802
15011
|
alias: "m",
|
|
14803
|
-
description: "
|
|
15012
|
+
description: t("command.extract.args.model")
|
|
14804
15013
|
},
|
|
14805
15014
|
dir: {
|
|
14806
15015
|
type: "string",
|
|
14807
15016
|
alias: "d",
|
|
14808
|
-
description: "
|
|
15017
|
+
description: t("command.extract.args.dir")
|
|
14809
15018
|
},
|
|
14810
15019
|
glob: {
|
|
14811
15020
|
type: "string",
|
|
14812
15021
|
alias: "g",
|
|
14813
|
-
description: "
|
|
15022
|
+
description: t("command.extract.args.glob")
|
|
14814
15023
|
},
|
|
14815
15024
|
noInsert: {
|
|
14816
15025
|
type: "boolean",
|
|
14817
|
-
description: "
|
|
15026
|
+
description: t("command.extract.args.noInsert"),
|
|
14818
15027
|
default: false
|
|
14819
15028
|
},
|
|
14820
15029
|
force: {
|
|
14821
15030
|
type: "boolean",
|
|
14822
|
-
description: "
|
|
15031
|
+
description: t("command.extract.args.force"),
|
|
14823
15032
|
default: false
|
|
14824
15033
|
}
|
|
14825
15034
|
},
|
|
14826
15035
|
async run({ args, rawArgs }) {
|
|
14827
15036
|
if (isExtractSubCommand(rawArgs)) return;
|
|
14828
15037
|
intro(pc.inverse(" aiex extract "));
|
|
15038
|
+
await initI18n();
|
|
14829
15039
|
const config = createMigrationConfig(process.cwd());
|
|
14830
15040
|
const aiexDir = path.dirname(config.schemaPath);
|
|
14831
15041
|
if (args.dir && args.file) {
|
|
14832
|
-
failCommand("
|
|
15042
|
+
failCommand(t("command.extract.errors.conflictFileDir"));
|
|
14833
15043
|
return;
|
|
14834
15044
|
}
|
|
14835
15045
|
const aiConfig = await loadConfiguredAI(aiexDir);
|
|
@@ -14837,12 +15047,12 @@ const extractCommand = defineCommand({
|
|
|
14837
15047
|
const modelOverride = resolveModelOverride(aiConfig, args.model);
|
|
14838
15048
|
if (modelOverride === null) return;
|
|
14839
15049
|
if (!args.schema && !args.file && !args.dir) {
|
|
14840
|
-
if (await runInteractive(aiexDir, config, aiConfig, modelOverride)) outro("
|
|
15050
|
+
if (await runInteractive(aiexDir, config, aiConfig, modelOverride)) outro(t("common.done"));
|
|
14841
15051
|
return;
|
|
14842
15052
|
}
|
|
14843
15053
|
if (args.dir) {
|
|
14844
15054
|
if (!args.schema) {
|
|
14845
|
-
failCommand(
|
|
15055
|
+
failCommand(t("command.extract.errors.schemaRequiredBatch"));
|
|
14846
15056
|
return;
|
|
14847
15057
|
}
|
|
14848
15058
|
const result$1 = await runBatchExtraction(aiexDir, config, aiConfig, args.schema, args.dir, args.glob, modelOverride, {
|
|
@@ -14854,16 +15064,16 @@ const extractCommand = defineCommand({
|
|
|
14854
15064
|
return;
|
|
14855
15065
|
}
|
|
14856
15066
|
if (result$1.failCount > 0) process.exitCode = 1;
|
|
14857
|
-
if (result$1.failCount > 0) outro(
|
|
14858
|
-
else outro("
|
|
15067
|
+
if (result$1.failCount > 0) outro(t("command.extract.batch.failSummary", { count: result$1.failCount }));
|
|
15068
|
+
else outro(t("common.done"));
|
|
14859
15069
|
return;
|
|
14860
15070
|
}
|
|
14861
15071
|
if (!args.schema) {
|
|
14862
|
-
failCommand("
|
|
15072
|
+
failCommand(t("command.extract.errors.schemaRequiredSingle"));
|
|
14863
15073
|
return;
|
|
14864
15074
|
}
|
|
14865
15075
|
if (!args.file) {
|
|
14866
|
-
failCommand("
|
|
15076
|
+
failCommand(t("command.extract.errors.fileRequiredSingle"));
|
|
14867
15077
|
return;
|
|
14868
15078
|
}
|
|
14869
15079
|
const result = await runAuditedExtraction({
|
|
@@ -14884,51 +15094,54 @@ const extractCommand = defineCommand({
|
|
|
14884
15094
|
failCommand(result.error);
|
|
14885
15095
|
return;
|
|
14886
15096
|
}
|
|
14887
|
-
outro("
|
|
15097
|
+
outro(t("common.done"));
|
|
14888
15098
|
}
|
|
14889
15099
|
});
|
|
14890
15100
|
async function runInteractive(aiexDir, config, aiConfig, modelOverride) {
|
|
14891
15101
|
const schemas = await listSchemas(aiexDir);
|
|
14892
15102
|
if (schemas.length === 0) {
|
|
14893
|
-
failCommand(
|
|
15103
|
+
failCommand(t("command.extract.errors.noSchemas", {
|
|
15104
|
+
path: pc.cyan(".aiex/schema/"),
|
|
15105
|
+
cmd: pc.cyan("aiex web")
|
|
15106
|
+
}));
|
|
14894
15107
|
return false;
|
|
14895
15108
|
}
|
|
14896
15109
|
const schemaName = await select({
|
|
14897
|
-
message: "
|
|
15110
|
+
message: t("command.extract.interactive.selectSchema"),
|
|
14898
15111
|
options: schemas.map((s) => ({
|
|
14899
15112
|
label: s,
|
|
14900
15113
|
value: s
|
|
14901
15114
|
}))
|
|
14902
15115
|
});
|
|
14903
15116
|
if (isCancel(schemaName)) {
|
|
14904
|
-
cancel("
|
|
15117
|
+
cancel(t("common.cancelled"));
|
|
14905
15118
|
return false;
|
|
14906
15119
|
}
|
|
14907
15120
|
const inputSource = await select({
|
|
14908
|
-
message: "
|
|
15121
|
+
message: t("command.extract.interactive.chooseSource"),
|
|
14909
15122
|
options: [{
|
|
14910
|
-
label: "
|
|
15123
|
+
label: t("command.extract.interactive.singleFile"),
|
|
14911
15124
|
value: "file",
|
|
14912
|
-
hint: "
|
|
15125
|
+
hint: t("command.extract.interactive.singleFileHint")
|
|
14913
15126
|
}, {
|
|
14914
|
-
label: "
|
|
15127
|
+
label: t("command.extract.interactive.batchDir"),
|
|
14915
15128
|
value: "dir",
|
|
14916
|
-
hint: "
|
|
15129
|
+
hint: t("command.extract.interactive.batchDirHint")
|
|
14917
15130
|
}]
|
|
14918
15131
|
});
|
|
14919
15132
|
if (isCancel(inputSource)) {
|
|
14920
|
-
cancel("
|
|
15133
|
+
cancel(t("common.cancelled"));
|
|
14921
15134
|
return false;
|
|
14922
15135
|
}
|
|
14923
15136
|
if (inputSource === "file") {
|
|
14924
15137
|
const filePathStr = await text({
|
|
14925
|
-
message: "
|
|
15138
|
+
message: t("command.extract.interactive.enterFilePath"),
|
|
14926
15139
|
validate(value) {
|
|
14927
|
-
if (!value || value.trim().length === 0) return "
|
|
15140
|
+
if (!value || value.trim().length === 0) return t("command.extract.interactive.filePathRequired");
|
|
14928
15141
|
}
|
|
14929
15142
|
});
|
|
14930
15143
|
if (isCancel(filePathStr)) {
|
|
14931
|
-
cancel("
|
|
15144
|
+
cancel(t("common.cancelled"));
|
|
14932
15145
|
return false;
|
|
14933
15146
|
}
|
|
14934
15147
|
return (await runAuditedExtraction({
|
|
@@ -14944,13 +15157,13 @@ async function runInteractive(aiexDir, config, aiConfig, modelOverride) {
|
|
|
14944
15157
|
})).success;
|
|
14945
15158
|
} else if (inputSource === "dir") {
|
|
14946
15159
|
const dirPath = await text({
|
|
14947
|
-
message: "
|
|
15160
|
+
message: t("command.extract.interactive.enterDirPath"),
|
|
14948
15161
|
validate(value) {
|
|
14949
|
-
if (!value || value.trim().length === 0) return "
|
|
15162
|
+
if (!value || value.trim().length === 0) return t("command.extract.interactive.dirPathRequired");
|
|
14950
15163
|
}
|
|
14951
15164
|
});
|
|
14952
15165
|
if (isCancel(dirPath)) {
|
|
14953
|
-
cancel("
|
|
15166
|
+
cancel(t("common.cancelled"));
|
|
14954
15167
|
return false;
|
|
14955
15168
|
}
|
|
14956
15169
|
const result = await runBatchExtraction(aiexDir, config, aiConfig, schemaName, dirPath, void 0, modelOverride);
|
|
@@ -14961,7 +15174,7 @@ async function runInteractive(aiexDir, config, aiConfig, modelOverride) {
|
|
|
14961
15174
|
}
|
|
14962
15175
|
function cancel(msg) {
|
|
14963
15176
|
consola.info(msg);
|
|
14964
|
-
outro("
|
|
15177
|
+
outro(t("common.cancelled"));
|
|
14965
15178
|
process.exitCode = 0;
|
|
14966
15179
|
}
|
|
14967
15180
|
|
|
@@ -15006,18 +15219,18 @@ function parseMigrationOutput(stdout, stderr) {
|
|
|
15006
15219
|
const jsonLine = stdout.trim().split("\n").find((l) => l.startsWith("{") && l.endsWith("}"));
|
|
15007
15220
|
if (!jsonLine) return {
|
|
15008
15221
|
success: false,
|
|
15009
|
-
error: "
|
|
15222
|
+
error: t("errors.schema.migrationHelperInvalidOutput")
|
|
15010
15223
|
};
|
|
15011
15224
|
const result = JSON.parse(jsonLine);
|
|
15012
15225
|
if (!result.success) return {
|
|
15013
15226
|
success: false,
|
|
15014
|
-
error: result.error || "
|
|
15227
|
+
error: result.error || t("errors.schema.migrationFailed")
|
|
15015
15228
|
};
|
|
15016
15229
|
return result;
|
|
15017
15230
|
} catch {
|
|
15018
15231
|
return {
|
|
15019
15232
|
success: false,
|
|
15020
|
-
error: stderr || stdout || "
|
|
15233
|
+
error: stderr || stdout || t("errors.schema.migrationHelperFailed")
|
|
15021
15234
|
};
|
|
15022
15235
|
}
|
|
15023
15236
|
}
|
|
@@ -15046,7 +15259,7 @@ async function runSchemaSync(config, options = {}) {
|
|
|
15046
15259
|
const schemaFiles = await listSchemaFiles(config.schemaPath);
|
|
15047
15260
|
if (schemaFiles.length === 0) return {
|
|
15048
15261
|
success: false,
|
|
15049
|
-
error: "
|
|
15262
|
+
error: t("errors.schema.noFiles"),
|
|
15050
15263
|
warnings: [],
|
|
15051
15264
|
schemaCount: 0,
|
|
15052
15265
|
tables: 0,
|
|
@@ -15085,61 +15298,65 @@ async function runSchemaSync(config, options = {}) {
|
|
|
15085
15298
|
const schemaCommand = defineCommand({
|
|
15086
15299
|
meta: {
|
|
15087
15300
|
name: "schema",
|
|
15088
|
-
description: "
|
|
15301
|
+
description: t("command.schema.description")
|
|
15089
15302
|
},
|
|
15090
15303
|
args: {
|
|
15091
15304
|
generate: {
|
|
15092
15305
|
type: "boolean",
|
|
15093
15306
|
alias: "g",
|
|
15094
|
-
description: "
|
|
15307
|
+
description: t("command.schema.args.generate"),
|
|
15095
15308
|
default: false
|
|
15096
15309
|
},
|
|
15097
15310
|
name: {
|
|
15098
15311
|
type: "string",
|
|
15099
|
-
description: "
|
|
15312
|
+
description: t("command.schema.args.name")
|
|
15100
15313
|
}
|
|
15101
15314
|
},
|
|
15102
15315
|
async run({ args }) {
|
|
15103
15316
|
intro(pc.inverse(" aiex schema "));
|
|
15317
|
+
await initI18n();
|
|
15104
15318
|
const config = createMigrationConfig(process.cwd());
|
|
15105
15319
|
const schemaFiles = await listSchemaFiles(config.schemaPath);
|
|
15106
15320
|
if (schemaFiles.length === 0) {
|
|
15107
|
-
consola.info(
|
|
15108
|
-
failCommand(
|
|
15321
|
+
consola.info(t("command.schema.runWebHint", { cmd: pc.cyan("aiex web") }));
|
|
15322
|
+
failCommand(t("command.schema.noSchemas", { path: pc.cyan(".aiex/schema/") }));
|
|
15109
15323
|
return;
|
|
15110
15324
|
}
|
|
15111
15325
|
const s1 = spinner();
|
|
15112
|
-
s1.start("
|
|
15326
|
+
s1.start(t("command.schema.generating"));
|
|
15113
15327
|
const generated = await generateSchemaFromFiles(schemaFiles, config);
|
|
15114
15328
|
for (const warning of generated.warnings) consola.warn(warning);
|
|
15115
|
-
if (generated.success) consola.success(
|
|
15329
|
+
if (generated.success) consola.success(t("command.schema.generated", {
|
|
15330
|
+
path: pc.cyan(".aiex/drizzle/schema.ts"),
|
|
15331
|
+
count: generated.schemaCount
|
|
15332
|
+
}));
|
|
15116
15333
|
else if (generated.error) consola.error(generated.error);
|
|
15117
|
-
s1.stop(generated.success ? "
|
|
15334
|
+
s1.stop(generated.success ? t("command.schema.generatedOk") : t("command.schema.generatedFail"));
|
|
15118
15335
|
if (!generated.success) {
|
|
15119
|
-
failCommand();
|
|
15336
|
+
failCommand(t("common.failed"));
|
|
15120
15337
|
return;
|
|
15121
15338
|
}
|
|
15122
15339
|
if (args.generate) {
|
|
15123
|
-
outro("
|
|
15340
|
+
outro(t("command.schema.runWithoutGenerate"));
|
|
15124
15341
|
return;
|
|
15125
15342
|
}
|
|
15126
15343
|
const s2 = spinner();
|
|
15127
|
-
s2.start("
|
|
15344
|
+
s2.start(t("command.schema.runningMigrations"));
|
|
15128
15345
|
const migration = await runSchemaMigration(config, args.name);
|
|
15129
15346
|
if (!migration.success) {
|
|
15130
|
-
consola.error("
|
|
15131
|
-
consola.error(migration.error || "
|
|
15132
|
-
} else if (migration.changes === 0) consola.info(pc.gray("
|
|
15347
|
+
consola.error(t("command.schema.migrationFailed"));
|
|
15348
|
+
consola.error(migration.error || t("command.schema.migrationFail"));
|
|
15349
|
+
} else if (migration.changes === 0) consola.info(pc.gray(t("command.schema.noChanges")));
|
|
15133
15350
|
else {
|
|
15134
|
-
consola.success(pc.green("
|
|
15135
|
-
consola.success(pc.green("
|
|
15351
|
+
consola.success(pc.green(t("command.schema.migrationFilesGenerated")));
|
|
15352
|
+
consola.success(pc.green(t("command.schema.databaseMigrated")));
|
|
15136
15353
|
}
|
|
15137
|
-
s2.stop(migration.success ? "
|
|
15354
|
+
s2.stop(migration.success ? t("command.schema.migrationsApplied") : t("command.schema.migrationFail"));
|
|
15138
15355
|
if (!migration.success) {
|
|
15139
|
-
failCommand();
|
|
15356
|
+
failCommand(t("common.failed"));
|
|
15140
15357
|
return;
|
|
15141
15358
|
}
|
|
15142
|
-
outro("
|
|
15359
|
+
outro(t("common.done"));
|
|
15143
15360
|
}
|
|
15144
15361
|
});
|
|
15145
15362
|
|
|
@@ -15208,14 +15425,14 @@ var WatchRegistry = class {
|
|
|
15208
15425
|
};
|
|
15209
15426
|
async function notifySuccess(fileName) {
|
|
15210
15427
|
if (process.platform === "darwin") try {
|
|
15211
|
-
await execa("osascript", ["-e", `display notification "
|
|
15428
|
+
await execa("osascript", ["-e", `display notification "${t("command.watch.notification.success")}" with title "${t("command.watch.notification.successTitle", { file: fileName })}"`]);
|
|
15212
15429
|
await execa("afplay", ["/System/Library/Sounds/Glass.aiff"]);
|
|
15213
15430
|
} catch {}
|
|
15214
15431
|
else process.stdout.write("\x07");
|
|
15215
15432
|
}
|
|
15216
15433
|
async function notifyFailure(fileName, errorMessage) {
|
|
15217
15434
|
if (process.platform === "darwin") try {
|
|
15218
|
-
await execa("osascript", ["-e", `display notification "${errorMessage.replace(/"/g, "\\\"")}" with title "
|
|
15435
|
+
await execa("osascript", ["-e", `display notification "${errorMessage.replace(/"/g, "\\\"")}" with title "${t("command.watch.notification.failTitle", { file: fileName })}"`]);
|
|
15219
15436
|
await execa("afplay", ["/System/Library/Sounds/Basso.aiff"]);
|
|
15220
15437
|
} catch {}
|
|
15221
15438
|
else process.stdout.write("\x07\x07");
|
|
@@ -15227,9 +15444,9 @@ function startWatcher(options) {
|
|
|
15227
15444
|
const registry$2 = new WatchRegistry(aiexDir);
|
|
15228
15445
|
fs$1.mkdirSync(queueDirActive, { recursive: true });
|
|
15229
15446
|
fs$1.mkdirSync(queueDirFailed, { recursive: true });
|
|
15230
|
-
consola.info(pc.green(
|
|
15231
|
-
consola.info(pc.green(
|
|
15232
|
-
if (modelOverride) consola.info(pc.green(
|
|
15447
|
+
consola.info(pc.green(t("command.watch.starting.watchFolder", { dir: pc.cyan(watchDir) })));
|
|
15448
|
+
consola.info(pc.green(t("command.watch.starting.schema", { name: pc.cyan(schemaName) })));
|
|
15449
|
+
if (modelOverride) consola.info(pc.green(t("command.watch.starting.modelOverride", { name: pc.cyan(modelOverride.name) })));
|
|
15233
15450
|
const watcher = chokidar.watch(watchDir, {
|
|
15234
15451
|
persistent: true,
|
|
15235
15452
|
ignoreInitial: false,
|
|
@@ -15245,15 +15462,18 @@ function startWatcher(options) {
|
|
|
15245
15462
|
if (!stat || !stat.isFile()) return;
|
|
15246
15463
|
const ext = path.extname(resolvedPath).toLowerCase().replace(".", "");
|
|
15247
15464
|
if (!SUPPORTED_EXTENSIONS.has(ext)) {
|
|
15248
|
-
consola.warn(
|
|
15465
|
+
consola.warn(t("command.watch.events.skippedUnsupported", { file: path.basename(resolvedPath) }));
|
|
15249
15466
|
return;
|
|
15250
15467
|
}
|
|
15251
15468
|
const fileName = path.basename(resolvedPath);
|
|
15252
|
-
consola.info(
|
|
15469
|
+
consola.info(t("command.watch.events.fileDetected", { file: pc.cyan(fileName) }));
|
|
15253
15470
|
try {
|
|
15254
15471
|
const hash = await getFileHash(resolvedPath);
|
|
15255
15472
|
if (await registry$2.getStatus(hash) === "succeeded") {
|
|
15256
|
-
consola.info(
|
|
15473
|
+
consola.info(t("command.watch.events.alreadyProcessed", {
|
|
15474
|
+
file: pc.cyan(fileName),
|
|
15475
|
+
hash: hash.slice(0, 8)
|
|
15476
|
+
}));
|
|
15257
15477
|
return;
|
|
15258
15478
|
}
|
|
15259
15479
|
const activeQueuePath = path.join(queueDirActive, `${hash}.${ext}`);
|
|
@@ -15262,25 +15482,28 @@ function startWatcher(options) {
|
|
|
15262
15482
|
await registry$2.markSucceeded(hash, resolvedPath);
|
|
15263
15483
|
await fs.rm(activeQueuePath, { force: true }).catch(() => {});
|
|
15264
15484
|
await fs.rm(activeQueuePath.replace(PDF_EXT_REGEXP, ".md"), { force: true }).catch(() => {});
|
|
15265
|
-
consola.success(
|
|
15485
|
+
consola.success(t("command.watch.events.processedSuccess", { file: pc.green(fileName) }));
|
|
15266
15486
|
await notifySuccess(fileName);
|
|
15267
15487
|
} else {
|
|
15268
|
-
const errorMsg = "
|
|
15488
|
+
const errorMsg = t("command.watch.events.extractionFailed");
|
|
15269
15489
|
await registry$2.markFailed(hash, resolvedPath, errorMsg);
|
|
15270
15490
|
const failedQueuePath = path.join(queueDirFailed, `${hash}-${Date.now()}.${ext}`);
|
|
15271
15491
|
await fs.rename(activeQueuePath, failedQueuePath).catch(() => {});
|
|
15272
15492
|
await fs.rm(activeQueuePath.replace(PDF_EXT_REGEXP, ".md"), { force: true }).catch(() => {});
|
|
15273
|
-
consola.error(
|
|
15493
|
+
consola.error(t("command.watch.events.processingFailed", { file: pc.red(fileName) }));
|
|
15274
15494
|
await notifyFailure(fileName, errorMsg);
|
|
15275
15495
|
}
|
|
15276
15496
|
} catch (e) {
|
|
15277
15497
|
const errorMsg = e instanceof Error ? e.message : String(e);
|
|
15278
|
-
consola.error(
|
|
15498
|
+
consola.error(t("command.watch.events.errorProcessing", {
|
|
15499
|
+
file: fileName,
|
|
15500
|
+
error: errorMsg
|
|
15501
|
+
}));
|
|
15279
15502
|
await notifyFailure(fileName, errorMsg);
|
|
15280
15503
|
}
|
|
15281
15504
|
});
|
|
15282
15505
|
watcher.on("error", (error) => {
|
|
15283
|
-
consola.error(
|
|
15506
|
+
consola.error(t("command.watch.events.watcherError", { error: error?.message || String(error) }));
|
|
15284
15507
|
});
|
|
15285
15508
|
return watcher;
|
|
15286
15509
|
}
|
|
@@ -15290,56 +15513,60 @@ function startWatcher(options) {
|
|
|
15290
15513
|
const watchCommand = defineCommand({
|
|
15291
15514
|
meta: {
|
|
15292
15515
|
name: "watch",
|
|
15293
|
-
description: "
|
|
15516
|
+
description: t("command.watch.description")
|
|
15294
15517
|
},
|
|
15295
15518
|
args: {
|
|
15296
15519
|
schema: {
|
|
15297
15520
|
type: "string",
|
|
15298
15521
|
alias: "s",
|
|
15299
|
-
description: "
|
|
15522
|
+
description: t("command.watch.args.schema")
|
|
15300
15523
|
},
|
|
15301
15524
|
dir: {
|
|
15302
15525
|
type: "string",
|
|
15303
15526
|
alias: "d",
|
|
15304
|
-
description: "
|
|
15527
|
+
description: t("command.watch.args.dir")
|
|
15305
15528
|
},
|
|
15306
15529
|
model: {
|
|
15307
15530
|
type: "string",
|
|
15308
15531
|
alias: "m",
|
|
15309
|
-
description: "
|
|
15532
|
+
description: t("command.watch.args.model")
|
|
15310
15533
|
},
|
|
15311
15534
|
noInsert: {
|
|
15312
15535
|
type: "boolean",
|
|
15313
|
-
description: "
|
|
15536
|
+
description: t("command.watch.args.noInsert"),
|
|
15314
15537
|
default: false
|
|
15315
15538
|
}
|
|
15316
15539
|
},
|
|
15317
15540
|
async run({ args }) {
|
|
15318
15541
|
intro(pc.inverse(" aiex watch "));
|
|
15542
|
+
await initI18n();
|
|
15319
15543
|
if (!args.schema) {
|
|
15320
|
-
failCommand(
|
|
15544
|
+
failCommand(t("command.watch.errors.schemaRequired"));
|
|
15321
15545
|
return;
|
|
15322
15546
|
}
|
|
15323
15547
|
if (!args.dir) {
|
|
15324
|
-
failCommand(
|
|
15548
|
+
failCommand(t("command.watch.errors.dirRequired"));
|
|
15325
15549
|
return;
|
|
15326
15550
|
}
|
|
15327
15551
|
const config = createMigrationConfig(process.cwd());
|
|
15328
15552
|
const aiexDir = path.dirname(config.schemaPath);
|
|
15329
15553
|
const schemaLoad = await loadSchema(config, args.schema);
|
|
15330
15554
|
if (!schemaLoad.schema) {
|
|
15331
|
-
failCommand(schemaLoad.error ||
|
|
15555
|
+
failCommand(schemaLoad.error || t("command.watch.errors.schemaNotFound", { name: args.schema }));
|
|
15332
15556
|
return;
|
|
15333
15557
|
}
|
|
15334
15558
|
let watchDirStat;
|
|
15335
15559
|
try {
|
|
15336
15560
|
watchDirStat = fs$1.statSync(args.dir);
|
|
15337
15561
|
} catch (e) {
|
|
15338
|
-
failCommand(
|
|
15562
|
+
failCommand(t("command.watch.errors.dirNotExist", {
|
|
15563
|
+
dir: args.dir,
|
|
15564
|
+
error: e instanceof Error ? e.message : String(e)
|
|
15565
|
+
}));
|
|
15339
15566
|
return;
|
|
15340
15567
|
}
|
|
15341
15568
|
if (!watchDirStat.isDirectory()) {
|
|
15342
|
-
failCommand(
|
|
15569
|
+
failCommand(t("command.watch.errors.notADirectory", { dir: args.dir }));
|
|
15343
15570
|
return;
|
|
15344
15571
|
}
|
|
15345
15572
|
const watchDirAbs = path.resolve(args.dir);
|
|
@@ -15357,14 +15584,14 @@ const watchCommand = defineCommand({
|
|
|
15357
15584
|
insert: !args.noInsert
|
|
15358
15585
|
});
|
|
15359
15586
|
const cleanup = async () => {
|
|
15360
|
-
consola.info("
|
|
15587
|
+
consola.info(t("command.watch.events.stopped"));
|
|
15361
15588
|
await watcher.close();
|
|
15362
|
-
consola.success("
|
|
15589
|
+
consola.success(t("command.watch.events.stoppedOk"));
|
|
15363
15590
|
process.exit(0);
|
|
15364
15591
|
};
|
|
15365
15592
|
process.on("SIGINT", cleanup);
|
|
15366
15593
|
process.on("SIGTERM", cleanup);
|
|
15367
|
-
consola.info("
|
|
15594
|
+
consola.info(t("command.watch.events.pressCtrlC"));
|
|
15368
15595
|
}
|
|
15369
15596
|
});
|
|
15370
15597
|
|
|
@@ -15430,7 +15657,7 @@ function aiRoutes(config) {
|
|
|
15430
15657
|
const schemaName = typeof body.schemaName === "string" ? body.schemaName : "";
|
|
15431
15658
|
if (!schemaName) return c.json({
|
|
15432
15659
|
success: false,
|
|
15433
|
-
error: "
|
|
15660
|
+
error: t("server.schemaRequired")
|
|
15434
15661
|
}, 400);
|
|
15435
15662
|
const result = await inspectNotionDatabase({
|
|
15436
15663
|
token,
|
|
@@ -15455,26 +15682,26 @@ function aiRoutes(config) {
|
|
|
15455
15682
|
const userTpl = body?.prompt?.userTemplate;
|
|
15456
15683
|
if (!systemTpl || !systemTpl.includes("{schema}")) return c.json({
|
|
15457
15684
|
success: false,
|
|
15458
|
-
error: "
|
|
15685
|
+
error: t("server.promptSchemaPlaceholder")
|
|
15459
15686
|
}, 400);
|
|
15460
15687
|
if (!userTpl?.includes("{text}")) return c.json({
|
|
15461
15688
|
success: false,
|
|
15462
|
-
error: "
|
|
15689
|
+
error: t("server.promptTextPlaceholder")
|
|
15463
15690
|
}, 400);
|
|
15464
15691
|
if (!body.provider?.models?.length) return c.json({
|
|
15465
15692
|
success: false,
|
|
15466
|
-
error: "
|
|
15693
|
+
error: t("server.atLeastOneModel")
|
|
15467
15694
|
}, 400);
|
|
15468
15695
|
if (body.notion?.enabled) {
|
|
15469
15696
|
if (!body.notion.token?.trim()) return c.json({
|
|
15470
15697
|
success: false,
|
|
15471
|
-
error: "
|
|
15698
|
+
error: t("server.notionTokenRequired")
|
|
15472
15699
|
}, 400);
|
|
15473
15700
|
for (const [schemaName, schemaConfig] of Object.entries(body.notion.schemas ?? {})) {
|
|
15474
15701
|
if (typeof schemaConfig.databaseId === "string") schemaConfig.databaseId = parseNotionDatabaseId(schemaConfig.databaseId);
|
|
15475
15702
|
if (!schemaConfig.databaseId?.trim()) return c.json({
|
|
15476
15703
|
success: false,
|
|
15477
|
-
error:
|
|
15704
|
+
error: t("server.notionDbIdRequired", { name: schemaName })
|
|
15478
15705
|
}, 400);
|
|
15479
15706
|
}
|
|
15480
15707
|
}
|
|
@@ -15491,26 +15718,17 @@ function aiRoutes(config) {
|
|
|
15491
15718
|
}
|
|
15492
15719
|
|
|
15493
15720
|
//#endregion
|
|
15494
|
-
//#region src/
|
|
15721
|
+
//#region src/core/data-service.ts
|
|
15495
15722
|
const FILE_REGEX = /\.json$/;
|
|
15496
15723
|
const EXTRACTION_TIMESTAMP_RE = /-\d{4}-\d{2}-\d{2}T/;
|
|
15497
15724
|
const INTERNAL_ROWID_COLUMN = "__aiex_rowid";
|
|
15498
15725
|
const TIMESTAMP_CLEANUP = /(\d{2})-(\d{2})-(\d{2})/;
|
|
15499
15726
|
const TIMESTAMP_TZ = /(\d{3})Z/;
|
|
15500
|
-
|
|
15501
|
-
const
|
|
15502
|
-
const
|
|
15503
|
-
|
|
15504
|
-
|
|
15505
|
-
search: z.string().catch(""),
|
|
15506
|
-
sortField: z.string().optional(),
|
|
15507
|
-
sortOrder: z.preprocess((value) => typeof value === "string" ? value.toLowerCase() : value, z.enum(["asc", "desc"]).catch("asc")),
|
|
15508
|
-
all: z.preprocess((value) => value === "true" || value === true, z.boolean().catch(false))
|
|
15509
|
-
});
|
|
15510
|
-
function invalidParamResponse$1(message) {
|
|
15511
|
-
return (result, c) => {
|
|
15512
|
-
if (!result.success) return c.json({ error: message }, 400);
|
|
15513
|
-
};
|
|
15727
|
+
function schemaNameFromExtractionFile(name$1) {
|
|
15728
|
+
const stem = name$1.replace(FILE_REGEX, "");
|
|
15729
|
+
const match = stem.match(EXTRACTION_TIMESTAMP_RE);
|
|
15730
|
+
if (!match || typeof match.index !== "number" || match.index <= 0) return null;
|
|
15731
|
+
return stem.slice(0, match.index);
|
|
15514
15732
|
}
|
|
15515
15733
|
function getAuditNotionStatus(record) {
|
|
15516
15734
|
if (record.notionPages?.length) return "synced";
|
|
@@ -15537,50 +15755,233 @@ async function getRowExtractionActions(aiexDir, tableName) {
|
|
|
15537
15755
|
}
|
|
15538
15756
|
return actions;
|
|
15539
15757
|
}
|
|
15540
|
-
function schemaNameFromExtractionFile(name$1) {
|
|
15541
|
-
const stem = name$1.replace(FILE_REGEX, "");
|
|
15542
|
-
const match = stem.match(EXTRACTION_TIMESTAMP_RE);
|
|
15543
|
-
if (!match || typeof match.index !== "number" || match.index <= 0) return null;
|
|
15544
|
-
return stem.slice(0, match.index);
|
|
15545
|
-
}
|
|
15546
15758
|
function createReadonlyQueryDb(databasePath) {
|
|
15547
15759
|
return new Kysely({ dialect: new SqliteDialect({ database: new Database(databasePath, { readonly: true }) }) });
|
|
15548
15760
|
}
|
|
15761
|
+
async function listExtractions(config) {
|
|
15762
|
+
const aiexDir = path.dirname(config.schemaPath);
|
|
15763
|
+
const extractedDir = path.join(aiexDir, "extracted");
|
|
15764
|
+
await fs.mkdir(extractedDir, { recursive: true });
|
|
15765
|
+
const jsonFiles = (await fs.readdir(extractedDir)).filter((f) => f.endsWith(".json") && !f.endsWith(".prompt.md"));
|
|
15766
|
+
const auditRecords = await listExtractionAuditRecords(aiexDir);
|
|
15767
|
+
const auditByOutputName = new Map(auditRecords.map((record) => [record.outputName, record]));
|
|
15768
|
+
const records = [];
|
|
15769
|
+
for (const file of jsonFiles) {
|
|
15770
|
+
const schemaName = schemaNameFromExtractionFile(file);
|
|
15771
|
+
if (!schemaName) continue;
|
|
15772
|
+
const timestamp = file.replace(FILE_REGEX, "").slice(schemaName.length + 1).replace(/-/g, (d, i) => i === 4 || i === 7 ? "-" : d).replace(TIMESTAMP_CLEANUP, (_, h, m, s) => `${h}:${m}:${s}`).replace(TIMESTAMP_TZ, ".$1Z");
|
|
15773
|
+
const filePath = path.join(extractedDir, file);
|
|
15774
|
+
try {
|
|
15775
|
+
const stat = await fs.stat(filePath);
|
|
15776
|
+
const audit = auditByOutputName.get(file);
|
|
15777
|
+
const notionPages = audit?.notionPages?.length ? audit.notionPages : void 0;
|
|
15778
|
+
records.push({
|
|
15779
|
+
name: file,
|
|
15780
|
+
schemaName,
|
|
15781
|
+
timestamp,
|
|
15782
|
+
fileSize: stat.size,
|
|
15783
|
+
modifiedAt: stat.mtime.toISOString(),
|
|
15784
|
+
notionStatus: notionPages ? "synced" : audit?.status === "failed" ? "failed" : "not_synced",
|
|
15785
|
+
notionPages,
|
|
15786
|
+
notionError: !notionPages && audit?.status === "failed" ? audit.error : void 0
|
|
15787
|
+
});
|
|
15788
|
+
} catch {
|
|
15789
|
+
continue;
|
|
15790
|
+
}
|
|
15791
|
+
}
|
|
15792
|
+
records.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
|
|
15793
|
+
return records;
|
|
15794
|
+
}
|
|
15795
|
+
async function listTables(config) {
|
|
15796
|
+
const schemaDir = config.schemaPath;
|
|
15797
|
+
let schemaFiles = [];
|
|
15798
|
+
try {
|
|
15799
|
+
schemaFiles = (await fs.readdir(schemaDir)).filter((f) => f.endsWith(".json"));
|
|
15800
|
+
} catch {
|
|
15801
|
+
schemaFiles = [];
|
|
15802
|
+
}
|
|
15803
|
+
let db = null;
|
|
15804
|
+
let dbTables = [];
|
|
15805
|
+
try {
|
|
15806
|
+
db = createReadonlyQueryDb(config.databasePath);
|
|
15807
|
+
dbTables = (await sql`
|
|
15808
|
+
select name
|
|
15809
|
+
from sqlite_master
|
|
15810
|
+
where type = 'table' and name not like 'sqlite_%' and name not like '_%'
|
|
15811
|
+
order by name
|
|
15812
|
+
`.execute(db)).rows.map((row) => row.name);
|
|
15813
|
+
} catch {} finally {
|
|
15814
|
+
await db?.destroy();
|
|
15815
|
+
}
|
|
15816
|
+
const tables = [];
|
|
15817
|
+
for (const file of schemaFiles) try {
|
|
15818
|
+
const schema = await readFile(path.join(schemaDir, file));
|
|
15819
|
+
const tableName = schema.table?.name;
|
|
15820
|
+
if (!tableName) continue;
|
|
15821
|
+
tables.push({
|
|
15822
|
+
name: tableName,
|
|
15823
|
+
title: schema.title || tableName,
|
|
15824
|
+
hasData: dbTables.includes(tableName)
|
|
15825
|
+
});
|
|
15826
|
+
} catch {
|
|
15827
|
+
continue;
|
|
15828
|
+
}
|
|
15829
|
+
return tables;
|
|
15830
|
+
}
|
|
15831
|
+
async function getTableData(config, tableName, query) {
|
|
15832
|
+
const { page, pageSize, search, sortField, sortOrder, all } = query;
|
|
15833
|
+
const aiexDir = path.dirname(config.schemaPath);
|
|
15834
|
+
let db;
|
|
15835
|
+
try {
|
|
15836
|
+
db = createReadonlyQueryDb(config.databasePath);
|
|
15837
|
+
} catch {
|
|
15838
|
+
throw new Error(t("server.dbNotFound"));
|
|
15839
|
+
}
|
|
15840
|
+
try {
|
|
15841
|
+
if ((await sql`
|
|
15842
|
+
select name
|
|
15843
|
+
from sqlite_master
|
|
15844
|
+
where type = 'table' and name = ${tableName}
|
|
15845
|
+
`.execute(db)).rows.length === 0) throw new Error(t("server.tableNotFound", { name: tableName }));
|
|
15846
|
+
const columns = (await sql`
|
|
15847
|
+
pragma table_info(${sql.table(tableName)})
|
|
15848
|
+
`.execute(db)).rows.map((col) => ({
|
|
15849
|
+
name: col.name,
|
|
15850
|
+
type: col.type,
|
|
15851
|
+
notNull: !!col.notnull,
|
|
15852
|
+
pk: !!col.pk
|
|
15853
|
+
}));
|
|
15854
|
+
const searchConditions = columns.map((col) => sql`${sql.ref(col.name)} like ${`%${search}%`}`);
|
|
15855
|
+
const searchCondition = search ? sql`where ${sql.join(searchConditions, sql` or `)}` : sql``;
|
|
15856
|
+
const sortColumn = columns.find((col) => col.name === sortField);
|
|
15857
|
+
const orderBy = sortColumn ? sql`order by ${sql.ref(sortColumn.name)} ${sql.raw(sortOrder === "desc" ? "desc" : "asc")}` : sql``;
|
|
15858
|
+
const total = (await sql`
|
|
15859
|
+
select count(*) as count
|
|
15860
|
+
from ${sql.table(tableName)}
|
|
15861
|
+
${searchCondition}
|
|
15862
|
+
`.execute(db)).rows[0]?.count ?? 0;
|
|
15863
|
+
const offset = (page - 1) * pageSize;
|
|
15864
|
+
const totalPages = all ? 1 : Math.max(1, Math.ceil(total / pageSize));
|
|
15865
|
+
const result = all ? await sql`
|
|
15866
|
+
select rowid as ${sql.raw(INTERNAL_ROWID_COLUMN)}, *
|
|
15867
|
+
from ${sql.table(tableName)}
|
|
15868
|
+
${searchCondition}
|
|
15869
|
+
${orderBy}
|
|
15870
|
+
`.execute(db) : await sql`
|
|
15871
|
+
select rowid as ${sql.raw(INTERNAL_ROWID_COLUMN)}, *
|
|
15872
|
+
from ${sql.table(tableName)}
|
|
15873
|
+
${searchCondition}
|
|
15874
|
+
${orderBy}
|
|
15875
|
+
limit ${pageSize}
|
|
15876
|
+
offset ${offset}
|
|
15877
|
+
`.execute(db);
|
|
15878
|
+
const actionsByRowId = await getRowExtractionActions(aiexDir, tableName);
|
|
15879
|
+
const rowActions = Object.fromEntries(result.rows.map((row, index) => {
|
|
15880
|
+
const rowId = row[INTERNAL_ROWID_COLUMN];
|
|
15881
|
+
const action = rowId === null || rowId === void 0 ? void 0 : actionsByRowId.get(String(rowId));
|
|
15882
|
+
return action ? [String(index), action] : null;
|
|
15883
|
+
}).filter((entry) => !!entry));
|
|
15884
|
+
const rows = result.rows.map(({ [INTERNAL_ROWID_COLUMN]: _rowid, ...row }) => row);
|
|
15885
|
+
const schemaDir = config.schemaPath;
|
|
15886
|
+
let schema = null;
|
|
15887
|
+
try {
|
|
15888
|
+
const schemaFiles = (await fs.readdir(schemaDir)).filter((f) => f.endsWith(".json"));
|
|
15889
|
+
for (const file of schemaFiles) {
|
|
15890
|
+
const s = await readFile(path.join(schemaDir, file));
|
|
15891
|
+
if (s.table?.name === tableName) {
|
|
15892
|
+
schema = s;
|
|
15893
|
+
break;
|
|
15894
|
+
}
|
|
15895
|
+
}
|
|
15896
|
+
} catch {}
|
|
15897
|
+
return {
|
|
15898
|
+
columns,
|
|
15899
|
+
rows,
|
|
15900
|
+
rowActions,
|
|
15901
|
+
total,
|
|
15902
|
+
page: all ? 1 : page,
|
|
15903
|
+
pageSize: all ? total : pageSize,
|
|
15904
|
+
totalPages,
|
|
15905
|
+
schema
|
|
15906
|
+
};
|
|
15907
|
+
} finally {
|
|
15908
|
+
await db.destroy();
|
|
15909
|
+
}
|
|
15910
|
+
}
|
|
15911
|
+
async function retryNotionSync(config, fileName) {
|
|
15912
|
+
const aiexDir = path.dirname(config.schemaPath);
|
|
15913
|
+
const extractedDir = path.join(aiexDir, "extracted");
|
|
15914
|
+
const filePath = path.join(extractedDir, fileName);
|
|
15915
|
+
const schemaName = schemaNameFromExtractionFile(fileName);
|
|
15916
|
+
if (!schemaName) throw new Error(t("server.cannotInferSchema"));
|
|
15917
|
+
const aiConfig = await readAIConfig(aiexDir);
|
|
15918
|
+
if (!aiConfig?.notion?.enabled) throw new Error(t("errors.notion.notEnabled"));
|
|
15919
|
+
if (!aiConfig.notion.schemas?.[schemaName]?.databaseId?.trim()) throw new Error(t("errors.notion.noSchemaConfig", { name: schemaName }));
|
|
15920
|
+
try {
|
|
15921
|
+
const data = await readFile(filePath);
|
|
15922
|
+
if (!data || typeof data !== "object" || Array.isArray(data)) throw new Error(t("errors.ai.extractionNotObject"));
|
|
15923
|
+
const page = await writeNotionPage(aiConfig.notion, schemaName, data);
|
|
15924
|
+
const notionPages = [{
|
|
15925
|
+
databaseId: page.databaseId,
|
|
15926
|
+
pageId: page.pageId
|
|
15927
|
+
}];
|
|
15928
|
+
let record = (await listExtractionAuditRecords(aiexDir)).find((record$1) => record$1.outputName === fileName);
|
|
15929
|
+
if (!record) record = await createExtractionAuditRecord(aiexDir, {
|
|
15930
|
+
schemaName,
|
|
15931
|
+
source: {
|
|
15932
|
+
type: "file",
|
|
15933
|
+
filePath,
|
|
15934
|
+
fileName
|
|
15935
|
+
}
|
|
15936
|
+
});
|
|
15937
|
+
if (record) await updateExtractionAuditRecord(aiexDir, record.id, {
|
|
15938
|
+
status: "succeeded",
|
|
15939
|
+
outputPath: filePath,
|
|
15940
|
+
outputName: fileName,
|
|
15941
|
+
notionPages,
|
|
15942
|
+
error: void 0
|
|
15943
|
+
});
|
|
15944
|
+
return {
|
|
15945
|
+
success: true,
|
|
15946
|
+
notionPages
|
|
15947
|
+
};
|
|
15948
|
+
} catch (error) {
|
|
15949
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
15950
|
+
const record = (await listExtractionAuditRecords(aiexDir)).find((record$1) => record$1.outputName === fileName);
|
|
15951
|
+
if (record) await updateExtractionAuditRecord(aiexDir, record.id, {
|
|
15952
|
+
status: "failed",
|
|
15953
|
+
outputPath: filePath,
|
|
15954
|
+
outputName: fileName,
|
|
15955
|
+
error: message
|
|
15956
|
+
});
|
|
15957
|
+
throw error;
|
|
15958
|
+
}
|
|
15959
|
+
}
|
|
15960
|
+
|
|
15961
|
+
//#endregion
|
|
15962
|
+
//#region src/server/routes/data.ts
|
|
15963
|
+
const tableParamSchema = z.object({ name: z.string().regex(/^[a-z][a-z0-9_]*$/) });
|
|
15964
|
+
const extractionFileParamSchema = z.object({ name: z.string().regex(/^[\w.-]+\.json$/).refine((name$1) => name$1 === path.basename(name$1) && !name$1.includes("..")) });
|
|
15965
|
+
const tableQuerySchema = z.object({
|
|
15966
|
+
page: z.coerce.number().int().min(1).catch(1),
|
|
15967
|
+
pageSize: z.coerce.number().int().min(1).max(500).catch(50),
|
|
15968
|
+
search: z.string().catch(""),
|
|
15969
|
+
sortField: z.string().optional(),
|
|
15970
|
+
sortOrder: z.preprocess((value) => typeof value === "string" ? value.toLowerCase() : value, z.enum(["asc", "desc"]).catch("asc")),
|
|
15971
|
+
all: z.preprocess((value) => value === "true" || value === true, z.boolean().catch(false))
|
|
15972
|
+
});
|
|
15973
|
+
function invalidParamResponse$1(message) {
|
|
15974
|
+
return (result, c) => {
|
|
15975
|
+
if (!result.success) return c.json({ error: message }, 400);
|
|
15976
|
+
};
|
|
15977
|
+
}
|
|
15549
15978
|
function dataRoutes(config) {
|
|
15550
15979
|
const app = new Hono();
|
|
15551
15980
|
const aiexDir = path.dirname(config.schemaPath);
|
|
15552
15981
|
const extractedDir = path.join(aiexDir, "extracted");
|
|
15553
15982
|
app.get("/data", async (c) => {
|
|
15554
15983
|
try {
|
|
15555
|
-
|
|
15556
|
-
const jsonFiles = (await fs.readdir(extractedDir)).filter((f) => f.endsWith(".json") && !f.endsWith(".prompt.md"));
|
|
15557
|
-
const auditRecords = await listExtractionAuditRecords(aiexDir);
|
|
15558
|
-
const auditByOutputName = new Map(auditRecords.map((record) => [record.outputName, record]));
|
|
15559
|
-
const records = [];
|
|
15560
|
-
for (const file of jsonFiles) {
|
|
15561
|
-
const schemaName = schemaNameFromExtractionFile(file);
|
|
15562
|
-
if (!schemaName) continue;
|
|
15563
|
-
const timestamp = file.replace(FILE_REGEX, "").slice(schemaName.length + 1).replace(/-/g, (d, i) => i === 4 || i === 7 ? "-" : d).replace(TIMESTAMP_CLEANUP, (_, h, m, s) => `${h}:${m}:${s}`).replace(TIMESTAMP_TZ, ".$1Z");
|
|
15564
|
-
const filePath = path.join(extractedDir, file);
|
|
15565
|
-
try {
|
|
15566
|
-
const stat = await fs.stat(filePath);
|
|
15567
|
-
const audit = auditByOutputName.get(file);
|
|
15568
|
-
const notionPages = audit?.notionPages?.length ? audit.notionPages : void 0;
|
|
15569
|
-
records.push({
|
|
15570
|
-
name: file,
|
|
15571
|
-
schemaName,
|
|
15572
|
-
timestamp,
|
|
15573
|
-
fileSize: stat.size,
|
|
15574
|
-
modifiedAt: stat.mtime.toISOString(),
|
|
15575
|
-
notionStatus: notionPages ? "synced" : audit?.status === "failed" ? "failed" : "not_synced",
|
|
15576
|
-
notionPages,
|
|
15577
|
-
notionError: !notionPages && audit?.status === "failed" ? audit.error : void 0
|
|
15578
|
-
});
|
|
15579
|
-
} catch {
|
|
15580
|
-
continue;
|
|
15581
|
-
}
|
|
15582
|
-
}
|
|
15583
|
-
records.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
|
|
15984
|
+
const records = await listExtractions(config);
|
|
15584
15985
|
return c.json(records);
|
|
15585
15986
|
} catch (error) {
|
|
15586
15987
|
return c.json({ error: error instanceof Error ? error.message : String(error) }, 500);
|
|
@@ -15588,127 +15989,25 @@ function dataRoutes(config) {
|
|
|
15588
15989
|
});
|
|
15589
15990
|
app.get("/data/tables", async (c) => {
|
|
15590
15991
|
try {
|
|
15591
|
-
const
|
|
15592
|
-
let schemaFiles = [];
|
|
15593
|
-
try {
|
|
15594
|
-
schemaFiles = (await fs.readdir(schemaDir)).filter((f) => f.endsWith(".json"));
|
|
15595
|
-
} catch {
|
|
15596
|
-
schemaFiles = [];
|
|
15597
|
-
}
|
|
15598
|
-
let db = null;
|
|
15599
|
-
let dbTables = [];
|
|
15600
|
-
try {
|
|
15601
|
-
db = createReadonlyQueryDb(config.databasePath);
|
|
15602
|
-
dbTables = (await sql`
|
|
15603
|
-
select name
|
|
15604
|
-
from sqlite_master
|
|
15605
|
-
where type = 'table' and name not like 'sqlite_%' and name not like '_%'
|
|
15606
|
-
order by name
|
|
15607
|
-
`.execute(db)).rows.map((row) => row.name);
|
|
15608
|
-
} catch {} finally {
|
|
15609
|
-
await db?.destroy();
|
|
15610
|
-
}
|
|
15611
|
-
const tables = [];
|
|
15612
|
-
for (const file of schemaFiles) try {
|
|
15613
|
-
const schema = await readFile(path.join(schemaDir, file));
|
|
15614
|
-
const tableName = schema.table?.name;
|
|
15615
|
-
if (!tableName) continue;
|
|
15616
|
-
tables.push({
|
|
15617
|
-
name: tableName,
|
|
15618
|
-
title: schema.title || tableName,
|
|
15619
|
-
hasData: dbTables.includes(tableName)
|
|
15620
|
-
});
|
|
15621
|
-
} catch {
|
|
15622
|
-
continue;
|
|
15623
|
-
}
|
|
15992
|
+
const tables = await listTables(config);
|
|
15624
15993
|
return c.json(tables);
|
|
15625
15994
|
} catch (error) {
|
|
15626
15995
|
return c.json({ error: error instanceof Error ? error.message : String(error) }, 500);
|
|
15627
15996
|
}
|
|
15628
15997
|
});
|
|
15629
|
-
app.get("/data/tables/:name", zValidator("param", tableParamSchema, invalidParamResponse$1("
|
|
15998
|
+
app.get("/data/tables/:name", zValidator("param", tableParamSchema, invalidParamResponse$1(t("server.invalidTableName"))), zValidator("query", tableQuerySchema), async (c) => {
|
|
15630
15999
|
const { name: tableName } = c.req.valid("param");
|
|
15631
|
-
const
|
|
15632
|
-
let db;
|
|
15633
|
-
try {
|
|
15634
|
-
db = createReadonlyQueryDb(config.databasePath);
|
|
15635
|
-
} catch {
|
|
15636
|
-
return c.json({ error: "Database not found. Run `aiex schema` first." }, 400);
|
|
15637
|
-
}
|
|
16000
|
+
const query = c.req.valid("query");
|
|
15638
16001
|
try {
|
|
15639
|
-
|
|
15640
|
-
|
|
15641
|
-
from sqlite_master
|
|
15642
|
-
where type = 'table' and name = ${tableName}
|
|
15643
|
-
`.execute(db)).rows.length === 0) return c.json({ error: `Table "${tableName}" not found in database` }, 404);
|
|
15644
|
-
const columns = (await sql`
|
|
15645
|
-
pragma table_info(${sql.table(tableName)})
|
|
15646
|
-
`.execute(db)).rows.map((col) => ({
|
|
15647
|
-
name: col.name,
|
|
15648
|
-
type: col.type,
|
|
15649
|
-
notNull: !!col.notnull,
|
|
15650
|
-
pk: !!col.pk
|
|
15651
|
-
}));
|
|
15652
|
-
const searchConditions = columns.map((col) => sql`${sql.ref(col.name)} like ${`%${search}%`}`);
|
|
15653
|
-
const searchCondition = search ? sql`where ${sql.join(searchConditions, sql` or `)}` : sql``;
|
|
15654
|
-
const sortColumn = columns.find((col) => col.name === sortField);
|
|
15655
|
-
const orderBy = sortColumn ? sql`order by ${sql.ref(sortColumn.name)} ${sql.raw(sortOrder === "desc" ? "desc" : "asc")}` : sql``;
|
|
15656
|
-
const total = (await sql`
|
|
15657
|
-
select count(*) as count
|
|
15658
|
-
from ${sql.table(tableName)}
|
|
15659
|
-
${searchCondition}
|
|
15660
|
-
`.execute(db)).rows[0]?.count ?? 0;
|
|
15661
|
-
const offset = (page - 1) * pageSize;
|
|
15662
|
-
const totalPages = all ? 1 : Math.max(1, Math.ceil(total / pageSize));
|
|
15663
|
-
const result = all ? await sql`
|
|
15664
|
-
select rowid as ${sql.raw(INTERNAL_ROWID_COLUMN)}, *
|
|
15665
|
-
from ${sql.table(tableName)}
|
|
15666
|
-
${searchCondition}
|
|
15667
|
-
${orderBy}
|
|
15668
|
-
`.execute(db) : await sql`
|
|
15669
|
-
select rowid as ${sql.raw(INTERNAL_ROWID_COLUMN)}, *
|
|
15670
|
-
from ${sql.table(tableName)}
|
|
15671
|
-
${searchCondition}
|
|
15672
|
-
${orderBy}
|
|
15673
|
-
limit ${pageSize}
|
|
15674
|
-
offset ${offset}
|
|
15675
|
-
`.execute(db);
|
|
15676
|
-
const actionsByRowId = await getRowExtractionActions(aiexDir, tableName);
|
|
15677
|
-
const rowActions = Object.fromEntries(result.rows.map((row, index) => {
|
|
15678
|
-
const rowId = row[INTERNAL_ROWID_COLUMN];
|
|
15679
|
-
const action = rowId === null || rowId === void 0 ? void 0 : actionsByRowId.get(String(rowId));
|
|
15680
|
-
return action ? [String(index), action] : null;
|
|
15681
|
-
}).filter((entry) => !!entry));
|
|
15682
|
-
const rows = result.rows.map(({ [INTERNAL_ROWID_COLUMN]: _rowid, ...row }) => row);
|
|
15683
|
-
const schemaDir = config.schemaPath;
|
|
15684
|
-
let schema = null;
|
|
15685
|
-
try {
|
|
15686
|
-
const schemaFiles = (await fs.readdir(schemaDir)).filter((f) => f.endsWith(".json"));
|
|
15687
|
-
for (const file of schemaFiles) {
|
|
15688
|
-
const s = await readFile(path.join(schemaDir, file));
|
|
15689
|
-
if (s.table?.name === tableName) {
|
|
15690
|
-
schema = s;
|
|
15691
|
-
break;
|
|
15692
|
-
}
|
|
15693
|
-
}
|
|
15694
|
-
} catch {}
|
|
15695
|
-
return c.json({
|
|
15696
|
-
columns,
|
|
15697
|
-
rows,
|
|
15698
|
-
rowActions,
|
|
15699
|
-
total,
|
|
15700
|
-
page: all ? 1 : page,
|
|
15701
|
-
pageSize: all ? total : pageSize,
|
|
15702
|
-
totalPages,
|
|
15703
|
-
schema
|
|
15704
|
-
});
|
|
16002
|
+
const result = await getTableData(config, tableName, query);
|
|
16003
|
+
return c.json(result);
|
|
15705
16004
|
} catch (error) {
|
|
15706
|
-
|
|
15707
|
-
|
|
15708
|
-
|
|
16005
|
+
const errMessage = error instanceof Error ? error.message : String(error);
|
|
16006
|
+
const status = errMessage.includes("not found") ? 404 : 500;
|
|
16007
|
+
return c.json({ error: errMessage }, status);
|
|
15709
16008
|
}
|
|
15710
16009
|
});
|
|
15711
|
-
app.get("/data/:name", zValidator("param", extractionFileParamSchema, invalidParamResponse$1("
|
|
16010
|
+
app.get("/data/:name", zValidator("param", extractionFileParamSchema, invalidParamResponse$1(t("server.invalidFileName"))), async (c) => {
|
|
15712
16011
|
const { name: name$1 } = c.req.valid("param");
|
|
15713
16012
|
const filePath = path.join(extractedDir, name$1);
|
|
15714
16013
|
try {
|
|
@@ -15719,66 +16018,20 @@ function dataRoutes(config) {
|
|
|
15719
16018
|
name: name$1
|
|
15720
16019
|
});
|
|
15721
16020
|
} catch {
|
|
15722
|
-
return c.json({ error: "
|
|
16021
|
+
return c.json({ error: t("server.extractionNotFound") }, 404);
|
|
15723
16022
|
}
|
|
15724
16023
|
});
|
|
15725
|
-
app.post("/data/:name/notion/retry", zValidator("param", extractionFileParamSchema, invalidParamResponse$1("
|
|
16024
|
+
app.post("/data/:name/notion/retry", zValidator("param", extractionFileParamSchema, invalidParamResponse$1(t("server.invalidFileName"))), async (c) => {
|
|
15726
16025
|
const { name: name$1 } = c.req.valid("param");
|
|
15727
|
-
|
|
15728
|
-
const schemaName = schemaNameFromExtractionFile(name$1);
|
|
15729
|
-
if (!schemaName) return c.json({
|
|
15730
|
-
success: false,
|
|
15731
|
-
error: "Cannot infer schema name from extraction file name"
|
|
15732
|
-
}, 400);
|
|
15733
|
-
const aiConfig = await readAIConfig(aiexDir);
|
|
15734
|
-
if (!aiConfig?.notion?.enabled) return c.json({
|
|
15735
|
-
success: false,
|
|
15736
|
-
error: "Notion export is not enabled. Configure Notion settings first."
|
|
15737
|
-
}, 400);
|
|
15738
|
-
if (!aiConfig.notion.schemas?.[schemaName]?.databaseId?.trim()) return c.json({
|
|
16026
|
+
if (!schemaNameFromExtractionFile(name$1)) return c.json({
|
|
15739
16027
|
success: false,
|
|
15740
|
-
error:
|
|
16028
|
+
error: t("server.cannotInferSchema")
|
|
15741
16029
|
}, 400);
|
|
15742
16030
|
try {
|
|
15743
|
-
const
|
|
15744
|
-
|
|
15745
|
-
success: false,
|
|
15746
|
-
error: "Extraction result is not a JSON object and cannot be written to Notion."
|
|
15747
|
-
}, 400);
|
|
15748
|
-
const page = await writeNotionPage(aiConfig.notion, schemaName, data);
|
|
15749
|
-
const notionPages = [{
|
|
15750
|
-
databaseId: page.databaseId,
|
|
15751
|
-
pageId: page.pageId
|
|
15752
|
-
}];
|
|
15753
|
-
let record = (await listExtractionAuditRecords(aiexDir)).find((record$1) => record$1.outputName === name$1);
|
|
15754
|
-
if (!record) record = await createExtractionAuditRecord(aiexDir, {
|
|
15755
|
-
schemaName,
|
|
15756
|
-
source: {
|
|
15757
|
-
type: "file",
|
|
15758
|
-
filePath,
|
|
15759
|
-
fileName: name$1
|
|
15760
|
-
}
|
|
15761
|
-
});
|
|
15762
|
-
if (record) await updateExtractionAuditRecord(aiexDir, record.id, {
|
|
15763
|
-
status: "succeeded",
|
|
15764
|
-
outputPath: filePath,
|
|
15765
|
-
outputName: name$1,
|
|
15766
|
-
notionPages,
|
|
15767
|
-
error: void 0
|
|
15768
|
-
});
|
|
15769
|
-
return c.json({
|
|
15770
|
-
success: true,
|
|
15771
|
-
notionPages
|
|
15772
|
-
});
|
|
16031
|
+
const result = await retryNotionSync(config, name$1);
|
|
16032
|
+
return c.json(result);
|
|
15773
16033
|
} catch (error) {
|
|
15774
16034
|
const message = error instanceof Error ? error.message : String(error);
|
|
15775
|
-
const record = (await listExtractionAuditRecords(aiexDir)).find((record$1) => record$1.outputName === name$1);
|
|
15776
|
-
if (record) await updateExtractionAuditRecord(aiexDir, record.id, {
|
|
15777
|
-
status: "failed",
|
|
15778
|
-
outputPath: filePath,
|
|
15779
|
-
outputName: name$1,
|
|
15780
|
-
error: message
|
|
15781
|
-
});
|
|
15782
16035
|
return c.json({
|
|
15783
16036
|
success: false,
|
|
15784
16037
|
error: message
|
|
@@ -15837,15 +16090,15 @@ function extractRoutes(config) {
|
|
|
15837
16090
|
const file = getFormFile(body.file);
|
|
15838
16091
|
if (!schemaName) return c.json({
|
|
15839
16092
|
success: false,
|
|
15840
|
-
error: "
|
|
16093
|
+
error: t("server.schemaRequired")
|
|
15841
16094
|
}, 400);
|
|
15842
16095
|
if (!text$1 && !file) return c.json({
|
|
15843
16096
|
success: false,
|
|
15844
|
-
error: "
|
|
16097
|
+
error: t("server.provideTextOrFile")
|
|
15845
16098
|
}, 400);
|
|
15846
16099
|
if (text$1 && file) return c.json({
|
|
15847
16100
|
success: false,
|
|
15848
|
-
error: "
|
|
16101
|
+
error: t("server.conflictTextAndFile")
|
|
15849
16102
|
}, 400);
|
|
15850
16103
|
let source;
|
|
15851
16104
|
if (file) {
|
|
@@ -15871,20 +16124,20 @@ function extractRoutes(config) {
|
|
|
15871
16124
|
const aiConfig = await readAIConfig(aiexDir);
|
|
15872
16125
|
if (!aiConfig) return c.json({
|
|
15873
16126
|
success: false,
|
|
15874
|
-
error: "
|
|
16127
|
+
error: t("server.aiConfigNotFound")
|
|
15875
16128
|
}, 400);
|
|
15876
16129
|
if (!aiConfig.provider.apiKey) return c.json({
|
|
15877
16130
|
success: false,
|
|
15878
|
-
error: "
|
|
16131
|
+
error: t("server.apiKeyNotConfigured")
|
|
15879
16132
|
}, 400);
|
|
15880
16133
|
if (!aiConfig.provider.models?.length) return c.json({
|
|
15881
16134
|
success: false,
|
|
15882
|
-
error: "
|
|
16135
|
+
error: t("server.noModelsConfigured")
|
|
15883
16136
|
}, 400);
|
|
15884
16137
|
const modelOverride = modelName ? aiConfig.provider.models.find((model) => model.name === modelName) : void 0;
|
|
15885
16138
|
if (modelName && !modelOverride) return c.json({
|
|
15886
16139
|
success: false,
|
|
15887
|
-
error:
|
|
16140
|
+
error: t("server.modelNotFound", { name: modelName })
|
|
15888
16141
|
}, 400);
|
|
15889
16142
|
const result = await runAuditedExtraction({
|
|
15890
16143
|
aiexDir,
|
|
@@ -15924,25 +16177,25 @@ function extractRoutes(config) {
|
|
|
15924
16177
|
const original = await readExtractionAuditRecord(aiexDir, c.req.param("id"));
|
|
15925
16178
|
if (!original) return c.json({
|
|
15926
16179
|
success: false,
|
|
15927
|
-
error: "
|
|
16180
|
+
error: t("server.extractionRecordNotFound")
|
|
15928
16181
|
}, 404);
|
|
15929
16182
|
const aiConfig = await readAIConfig(aiexDir);
|
|
15930
16183
|
if (!aiConfig) return c.json({
|
|
15931
16184
|
success: false,
|
|
15932
|
-
error: "
|
|
16185
|
+
error: t("server.aiConfigNotFound")
|
|
15933
16186
|
}, 400);
|
|
15934
16187
|
if (!aiConfig.provider.apiKey) return c.json({
|
|
15935
16188
|
success: false,
|
|
15936
|
-
error: "
|
|
16189
|
+
error: t("server.apiKeyNotConfigured")
|
|
15937
16190
|
}, 400);
|
|
15938
16191
|
if (!aiConfig.provider.models?.length) return c.json({
|
|
15939
16192
|
success: false,
|
|
15940
|
-
error: "
|
|
16193
|
+
error: t("server.noModelsConfigured")
|
|
15941
16194
|
}, 400);
|
|
15942
16195
|
const modelOverride = original.modelName ? aiConfig.provider.models.find((m) => m.name === original.modelName) : void 0;
|
|
15943
16196
|
if (original.modelName && !modelOverride) return c.json({
|
|
15944
16197
|
success: false,
|
|
15945
|
-
error:
|
|
16198
|
+
error: t("server.modelNotFound", { name: original.modelName })
|
|
15946
16199
|
}, 400);
|
|
15947
16200
|
const source = original.source.type === "file" && original.source.filePath ? {
|
|
15948
16201
|
type: "file",
|
|
@@ -15981,7 +16234,7 @@ function extractRoutes(config) {
|
|
|
15981
16234
|
const id = c.req.param("id");
|
|
15982
16235
|
if (!await readExtractionAuditRecord(aiexDir, id)) return c.json({
|
|
15983
16236
|
success: false,
|
|
15984
|
-
error: "
|
|
16237
|
+
error: t("server.extractionRecordNotFound")
|
|
15985
16238
|
}, 404);
|
|
15986
16239
|
await deleteExtractionAuditRecord(aiexDir, id);
|
|
15987
16240
|
return c.json({ success: true });
|
|
@@ -16010,16 +16263,16 @@ function schemaRoutes(config) {
|
|
|
16010
16263
|
const jsonFiles = (await fs.readdir(schemaDir)).filter((f) => f.endsWith(".json"));
|
|
16011
16264
|
return c.json(jsonFiles);
|
|
16012
16265
|
});
|
|
16013
|
-
app.get("/schema/:name", zValidator("param", schemaFileParamSchema, invalidParamResponse("
|
|
16266
|
+
app.get("/schema/:name", zValidator("param", schemaFileParamSchema, invalidParamResponse(t("server.invalidTableName"))), async (c) => {
|
|
16014
16267
|
const { name: name$1 } = c.req.valid("param");
|
|
16015
16268
|
const filePath = path.join(schemaDir, name$1);
|
|
16016
16269
|
try {
|
|
16017
16270
|
return c.json(await readFile(filePath));
|
|
16018
16271
|
} catch {
|
|
16019
|
-
return c.json({ error: "
|
|
16272
|
+
return c.json({ error: t("server.schemaNotFound") }, 404);
|
|
16020
16273
|
}
|
|
16021
16274
|
});
|
|
16022
|
-
app.post("/schema/:name", zValidator("param", schemaFileParamSchema, invalidParamResponse("
|
|
16275
|
+
app.post("/schema/:name", zValidator("param", schemaFileParamSchema, invalidParamResponse(t("server.invalidTableName"))), async (c) => {
|
|
16023
16276
|
const { name: name$1 } = c.req.valid("param");
|
|
16024
16277
|
const filePath = path.join(schemaDir, name$1);
|
|
16025
16278
|
try {
|
|
@@ -16035,10 +16288,10 @@ function schemaRoutes(config) {
|
|
|
16035
16288
|
} catch {}
|
|
16036
16289
|
return c.json({ success: true });
|
|
16037
16290
|
} catch {
|
|
16038
|
-
return c.json({ error: "
|
|
16291
|
+
return c.json({ error: t("server.saveSchemaFailed") }, 500);
|
|
16039
16292
|
}
|
|
16040
16293
|
});
|
|
16041
|
-
app.get("/prompt-snapshot/:name", zValidator("param", tableNameParamSchema, invalidParamResponse("
|
|
16294
|
+
app.get("/prompt-snapshot/:name", zValidator("param", tableNameParamSchema, invalidParamResponse(t("server.invalidTableName"))), async (c) => {
|
|
16042
16295
|
const { name: name$1 } = c.req.valid("param");
|
|
16043
16296
|
const aiexDir = path.dirname(schemaDir);
|
|
16044
16297
|
const snapshotPath = path.join(aiexDir, "extracted", `${name$1}.prompt.md`);
|
|
@@ -16051,11 +16304,11 @@ function schemaRoutes(config) {
|
|
|
16051
16304
|
} catch {
|
|
16052
16305
|
return c.json({
|
|
16053
16306
|
success: false,
|
|
16054
|
-
error: "
|
|
16307
|
+
error: t("server.promptSnapshotNotAvailable")
|
|
16055
16308
|
}, 404);
|
|
16056
16309
|
}
|
|
16057
16310
|
});
|
|
16058
|
-
app.delete("/schema/:name", zValidator("param", schemaFileParamSchema, invalidParamResponse("
|
|
16311
|
+
app.delete("/schema/:name", zValidator("param", schemaFileParamSchema, invalidParamResponse(t("server.invalidTableName"))), async (c) => {
|
|
16059
16312
|
const { name: name$1 } = c.req.valid("param");
|
|
16060
16313
|
const filePath = path.join(schemaDir, name$1);
|
|
16061
16314
|
try {
|
|
@@ -16071,7 +16324,7 @@ function schemaRoutes(config) {
|
|
|
16071
16324
|
await fs.unlink(filePath);
|
|
16072
16325
|
return c.json({ success: true });
|
|
16073
16326
|
} catch {
|
|
16074
|
-
return c.json({ error: "
|
|
16327
|
+
return c.json({ error: t("server.deleteSchemaFailed") }, 500);
|
|
16075
16328
|
}
|
|
16076
16329
|
});
|
|
16077
16330
|
app.post("/migrate", async (c) => {
|
|
@@ -16082,7 +16335,7 @@ function schemaRoutes(config) {
|
|
|
16082
16335
|
const status = result.schemaCount === 0 ? 400 : 500;
|
|
16083
16336
|
return c.json({
|
|
16084
16337
|
success: false,
|
|
16085
|
-
error: result.error || "
|
|
16338
|
+
error: result.error || t("server.migrationFailed")
|
|
16086
16339
|
}, status);
|
|
16087
16340
|
}
|
|
16088
16341
|
return c.json({
|
|
@@ -16123,7 +16376,7 @@ function createApp(config, staticDir) {
|
|
|
16123
16376
|
html = await fs.readFile(`${staticDir}/index.html`, "utf-8");
|
|
16124
16377
|
} catch {
|
|
16125
16378
|
html = `<!DOCTYPE html>
|
|
16126
|
-
<html lang="
|
|
16379
|
+
<html lang="en">
|
|
16127
16380
|
<head>
|
|
16128
16381
|
<meta charset="UTF-8" />
|
|
16129
16382
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
@@ -16173,31 +16426,32 @@ async function startWebServer(input) {
|
|
|
16173
16426
|
const webCommand = defineCommand({
|
|
16174
16427
|
meta: {
|
|
16175
16428
|
name: "web",
|
|
16176
|
-
description: "
|
|
16429
|
+
description: t("command.web.description")
|
|
16177
16430
|
},
|
|
16178
16431
|
args: { port: {
|
|
16179
16432
|
type: "string",
|
|
16180
16433
|
alias: "p",
|
|
16181
|
-
description: "
|
|
16434
|
+
description: t("command.web.args.port"),
|
|
16182
16435
|
default: "13000"
|
|
16183
16436
|
} },
|
|
16184
16437
|
async run({ args }) {
|
|
16438
|
+
await initI18n();
|
|
16185
16439
|
intro(pc.inverse(" aiex web "));
|
|
16186
16440
|
const cwd = process.cwd();
|
|
16187
16441
|
const port = Number(args.port) || 13e3;
|
|
16188
16442
|
const config = createMigrationConfig(cwd);
|
|
16189
16443
|
const s = spinner();
|
|
16190
|
-
s.start("
|
|
16444
|
+
s.start(t("command.web.starting"));
|
|
16191
16445
|
await startWebServer({
|
|
16192
16446
|
config,
|
|
16193
16447
|
port,
|
|
16194
16448
|
onStarted(info) {
|
|
16195
|
-
s.stop(
|
|
16196
|
-
consola.info(
|
|
16197
|
-
consola.info("
|
|
16449
|
+
s.stop(t("command.web.serverRunning", { url: pc.cyan(info.url) }));
|
|
16450
|
+
consola.info(t("command.web.schemaDir", { path: pc.dim(info.schemaPath) }));
|
|
16451
|
+
consola.info(t("command.web.pressCtrlC"));
|
|
16198
16452
|
},
|
|
16199
16453
|
onOpenFailed(url) {
|
|
16200
|
-
consola.warn(
|
|
16454
|
+
consola.warn(t("command.web.browserOpenFailed", { url }));
|
|
16201
16455
|
}
|
|
16202
16456
|
});
|
|
16203
16457
|
}
|
|
@@ -16217,6 +16471,7 @@ const subCommands = {
|
|
|
16217
16471
|
|
|
16218
16472
|
//#endregion
|
|
16219
16473
|
//#region src/cli.ts
|
|
16474
|
+
await initI18n();
|
|
16220
16475
|
seedConfig(createConfig());
|
|
16221
16476
|
updateNotifier({ pkg: package_default }).notify();
|
|
16222
16477
|
process.on("uncaughtException", (error) => {
|
|
@@ -16229,7 +16484,7 @@ process.on("unhandledRejection", (reason) => {
|
|
|
16229
16484
|
process.exit(1);
|
|
16230
16485
|
});
|
|
16231
16486
|
if (process.argv[2] === "_complete") {
|
|
16232
|
-
const { getCompletions } = await import("./completions-
|
|
16487
|
+
const { getCompletions } = await import("./completions-Bh0DOngr.mjs");
|
|
16233
16488
|
const suggestions = getCompletions(subCommands, process.argv.slice(4));
|
|
16234
16489
|
for (const s of suggestions) process.stdout.write(`${s}\n`);
|
|
16235
16490
|
process.exit(0);
|