aiex-cli 0.0.1-beta.8 → 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -25
- package/dist/cli.mjs +912 -572
- package/dist/{completions-ygS1okck.mjs → completions-C3rmTwXZ.mjs} +2 -2
- package/dist/core/schema-sqlite/migrate-helper.mjs +35 -37
- package/dist/{doctor-DxG7uaGR.mjs → doctor-collector-CykRm0fC.mjs} +282 -223
- package/dist/index.d.mts +15 -15
- package/dist/index.mjs +1 -1
- package/dist/table-schema.json +4 -0
- package/dist/web/assets/AISettings-CI6Lgx0p.js +339 -0
- package/dist/web/assets/DataBrowser-CwcTG80-.js +6 -0
- package/dist/web/assets/ExtractionViewer-CsdK1kKK.js +1 -0
- package/dist/web/assets/JsonSchemaEditor-D477lV5a.js +570 -0
- package/dist/web/assets/api-client-D2Y_-4JM.js +1 -0
- package/dist/web/assets/button-Cdgr9Igy.js +927 -0
- package/dist/web/assets/{cssMode-DAbG0CMn.js → cssMode-CPThwItX.js} +1 -1
- package/dist/web/assets/dialog-CUkPLPNP.js +109 -0
- package/dist/web/assets/dist-9yHVMqQ0.js +1 -0
- package/dist/web/assets/{editor.main-BqhfoHxy.js → editor.main-BnOkwRFv.js} +2 -2
- package/dist/web/assets/{freemarker2-B9_5ct2b.js → freemarker2-DWDTYVJR.js} +1 -1
- package/dist/web/assets/{handlebars-TY59WcoQ.js → handlebars-D4DzjGQ7.js} +1 -1
- package/dist/web/assets/{html-CLULsh27.js → html-DnzhKSoD.js} +1 -1
- package/dist/web/assets/{htmlMode-BvG7RNbU.js → htmlMode-CR7UKfEH.js} +1 -1
- package/dist/web/assets/index-C9N8oWt4.css +2 -0
- package/dist/web/assets/{index-BRvFRL-3.js → index-DVDVw-GK.js} +38 -38
- package/dist/web/assets/{javascript-DHrLp6gu.js → javascript-D2srszZ8.js} +1 -1
- package/dist/web/assets/{jsonMode-DBDhdzl1.js → jsonMode-B4jaPYEr.js} +1 -1
- package/dist/web/assets/{liquid-tGeb-nqF.js → liquid-CIT2Wl_l.js} +1 -1
- package/dist/web/assets/{mdx-Cmdz78VU.js → mdx-CWLaEOFy.js} +1 -1
- package/dist/web/assets/{monaco.contribution-CroYPUF5.js → monaco.contribution-DDv5ldfS.js} +2 -2
- package/dist/web/assets/object-utils-I4gWdSnS.js +1 -0
- package/dist/web/assets/{python-Dmfz4iDE.js → python-6CGfpCNq.js} +1 -1
- package/dist/web/assets/{razor-BJicZHJs.js → razor-DEMMh3TD.js} +1 -1
- package/dist/web/assets/runtime-dom.esm-bundler-ei_N7Xjw.js +1 -0
- package/dist/web/assets/select-BGex2SPs.js +439 -0
- package/dist/web/assets/{tsMode-DYqTyE66.js → tsMode-Cm1NtjPs.js} +1 -1
- package/dist/web/assets/{typescript-DLnTe9Hf.js → typescript-BM9aPEFg.js} +1 -1
- package/dist/web/assets/{xml-BIYqLORk.js → xml-CoSbvcg5.js} +1 -1
- package/dist/web/assets/{yaml-BjmulkMX.js → yaml-56GOgy8k.js} +1 -1
- package/dist/web/index.html +10 -8
- package/package.json +16 -1
- package/src/core/schema-sqlite/migrate-helper.ts +32 -46
- package/dist/web/assets/AISettings-D_AFhorO.js +0 -334
- package/dist/web/assets/DataBrowser-rznfVRaV.js +0 -3
- package/dist/web/assets/JsonSchemaEditor-C9iyQs7N.js +0 -929
- package/dist/web/assets/api-client-Dsg4WOM9.js +0 -1
- package/dist/web/assets/button-kTMweGMc.js +0 -927
- package/dist/web/assets/dialog-CWuu7WjI.js +0 -108
- package/dist/web/assets/index-DDFnprdM.css +0 -2
- package/dist/web/assets/lib-C30cIFrm.js +0 -1
- package/dist/web/assets/overlayeventbus-AtOpmI6n.js +0 -80
- package/dist/web/assets/table-schema-mJrrf9qw.js +0 -2
- /package/dist/web/assets/{abap-DrZwwXZX.js → abap-Bgec7Keq.js} +0 -0
- /package/dist/web/assets/{apex-CrCz0btt.js → apex-VBlPwEoQ.js} +0 -0
- /package/dist/web/assets/{azcli-BapzKHay.js → azcli-DKqrEFBx.js} +0 -0
- /package/dist/web/assets/{bat-C_NRAiA1.js → bat-DdgQWy_0.js} +0 -0
- /package/dist/web/assets/{bicep-C7pp2CNk.js → bicep-CRMM43EB.js} +0 -0
- /package/dist/web/assets/{cameligo-BhhK9vxZ.js → cameligo-UatALtML.js} +0 -0
- /package/dist/web/assets/{clojure-D0ujmUyE.js → clojure-D8JU08RA.js} +0 -0
- /package/dist/web/assets/{coffee-DHEl7Jbb.js → coffee-C56wu358.js} +0 -0
- /package/dist/web/assets/{cpp-Iil-3nzZ.js → cpp-CyZLvhJG.js} +0 -0
- /package/dist/web/assets/{csharp-Dh0Ee7SY.js → csharp-BJl3ixva.js} +0 -0
- /package/dist/web/assets/{csp-mwzjw0JL.js → csp-CxEKxmO-.js} +0 -0
- /package/dist/web/assets/{css-COIa8ZTR.js → css-B0t_muXd.js} +0 -0
- /package/dist/web/assets/{cypher-GVc17FC4.js → cypher-D1hqiMFD.js} +0 -0
- /package/dist/web/assets/{dart-phiCaE7_.js → dart-Bz550Pyv.js} +0 -0
- /package/dist/web/assets/{dockerfile-BMaDhdim.js → dockerfile-CIXgVAuA.js} +0 -0
- /package/dist/web/assets/{ecl-Cj47kvqp.js → ecl-D9qbvZoA.js} +0 -0
- /package/dist/web/assets/{editor.api-DLXGyrN1.js → editor.api-C8BHpRhn.js} +0 -0
- /package/dist/web/assets/{elixir-DBbstcE1.js → elixir-b2M38fAy.js} +0 -0
- /package/dist/web/assets/{flow9-ChHb1adO.js → flow9-Dq1UYMkt.js} +0 -0
- /package/dist/web/assets/{fsharp-CMk2OIJN.js → fsharp-BaeLhgfq.js} +0 -0
- /package/dist/web/assets/{go-BrMkuJg0.js → go-Bd-NFKIC.js} +0 -0
- /package/dist/web/assets/{graphql-PSR1UKGv.js → graphql-DZVerJfy.js} +0 -0
- /package/dist/web/assets/{hcl-DAQrbDOW.js → hcl-CAVzrZfH.js} +0 -0
- /package/dist/web/assets/{ini-0TG5BxW0.js → ini-CyXdX58t.js} +0 -0
- /package/dist/web/assets/{java-rgorz17v.js → java-B5pNgvhy.js} +0 -0
- /package/dist/web/assets/{julia-C8VMdHm8.js → julia-XRhmV3AN.js} +0 -0
- /package/dist/web/assets/{kotlin-CllWo3gX.js → kotlin-DOd3J5vr.js} +0 -0
- /package/dist/web/assets/{less-Cgca25AP.js → less-veZSnyw6.js} +0 -0
- /package/dist/web/assets/{lexon-D0GHdBaw.js → lexon-QWGkuK0H.js} +0 -0
- /package/dist/web/assets/{lua-DmRsNG-P.js → lua-CYGpjuO5.js} +0 -0
- /package/dist/web/assets/{m3-BgL5dNKT.js → m3-yNnrZkdc.js} +0 -0
- /package/dist/web/assets/{markdown-BuJfycGS.js → markdown-BCSWEPSX.js} +0 -0
- /package/dist/web/assets/{mips-C9m_93PR.js → mips-OpYmcC30.js} +0 -0
- /package/dist/web/assets/{msdax-CpFHC9OI.js → msdax-2oxoTO9Z.js} +0 -0
- /package/dist/web/assets/{mysql-qFvltsqN.js → mysql-5KlC-K_9.js} +0 -0
- /package/dist/web/assets/{objective-c-Bnmr858J.js → objective-c-CcDCgtLx.js} +0 -0
- /package/dist/web/assets/{pascal-WP0_D5AO.js → pascal-BZGsbaEV.js} +0 -0
- /package/dist/web/assets/{pascaligo-Blom4Rij.js → pascaligo-DtD5qU3G.js} +0 -0
- /package/dist/web/assets/{perl-B-vk8g64.js → perl-C1jNNS3E.js} +0 -0
- /package/dist/web/assets/{pgsql-Cgvz6v67.js → pgsql-CT0fhiZa.js} +0 -0
- /package/dist/web/assets/{php-8a3Lrw9m.js → php-D6DrXoPM.js} +0 -0
- /package/dist/web/assets/{pla-DuFqEZ8V.js → pla-b3-HN2pF.js} +0 -0
- /package/dist/web/assets/{postiats-DkLtSgkp.js → postiats-Bin2ApVS.js} +0 -0
- /package/dist/web/assets/{powerquery-BJ1aNepW.js → powerquery-7ASnn-ZG.js} +0 -0
- /package/dist/web/assets/{powershell-rE98k687.js → powershell-t4p7sU1H.js} +0 -0
- /package/dist/web/assets/{preload-helper-DWTEM3RW.js → preload-helper-Dd-HcVz_.js} +0 -0
- /package/dist/web/assets/{protobuf-CUheFacr.js → protobuf-BUGeWa_j.js} +0 -0
- /package/dist/web/assets/{pug-LDcAMD8w.js → pug-BuKcgC9s.js} +0 -0
- /package/dist/web/assets/{qsharp-IHfqKOfK.js → qsharp-DxLLX8mo.js} +0 -0
- /package/dist/web/assets/{r-D-QApv87.js → r-DMlFgn7A.js} +0 -0
- /package/dist/web/assets/{redis-SXdDyWR9.js → redis-cXItkC5u.js} +0 -0
- /package/dist/web/assets/{redshift-Y6lsCryn.js → redshift-BZVbW7HE.js} +0 -0
- /package/dist/web/assets/{restructuredtext-edObr9a8.js → restructuredtext-BzjxwS8h.js} +0 -0
- /package/dist/web/assets/{ruby-CNnUfF-8.js → ruby-C5nyLV4l.js} +0 -0
- /package/dist/web/assets/{rust-IHUZWzBr.js → rust-BcmMsHdf.js} +0 -0
- /package/dist/web/assets/{sb-DrUvY44N.js → sb-Dnb1iy6B.js} +0 -0
- /package/dist/web/assets/{scala-B4hbXGLM.js → scala-anMIFYpA.js} +0 -0
- /package/dist/web/assets/{scheme-BGrd12j3.js → scheme-BItQTe08.js} +0 -0
- /package/dist/web/assets/{scss-x5G1ES4U.js → scss-BOv51BJ5.js} +0 -0
- /package/dist/web/assets/{shell-DOehe2Y8.js → shell-BsRYRTNN.js} +0 -0
- /package/dist/web/assets/{solidity-BeRvcwWV.js → solidity-BtuLgGDx.js} +0 -0
- /package/dist/web/assets/{sophia-DZbkUNjy.js → sophia-B0Vkc5MF.js} +0 -0
- /package/dist/web/assets/{sparql-B7_oi5-h.js → sparql-B7lvkZQM.js} +0 -0
- /package/dist/web/assets/{sql-CTlsFWVE.js → sql-DvP5MpA3.js} +0 -0
- /package/dist/web/assets/{st-DJVEJdPE.js → st-GVUeyB3U.js} +0 -0
- /package/dist/web/assets/{swift-CwhT3fYa.js → swift-DSPIoCjm.js} +0 -0
- /package/dist/web/assets/{systemverilog-BQN63pkN.js → systemverilog-Icj2-k23.js} +0 -0
- /package/dist/web/assets/{tcl-DqwfpskA.js → tcl-Cd8KQcm-.js} +0 -0
- /package/dist/web/assets/{twig-BiyenUgc.js → twig-CBHmt8z3.js} +0 -0
- /package/dist/web/assets/{typespec-CWOJribt.js → typespec-Ckc037mq.js} +0 -0
- /package/dist/web/assets/{vb-Cq5F87m3.js → vb-B97GW9Wb.js} +0 -0
- /package/dist/web/assets/{wgsl-BAvW2lVr.js → wgsl-DIKmb3YH.js} +0 -0
package/dist/cli.mjs
CHANGED
|
@@ -1,25 +1,40 @@
|
|
|
1
|
-
import { C as doctorDiagnosticsTableRows, a as
|
|
1
|
+
import { C as doctorDiagnosticsTableRows, _ as seedConfig, a as parseJsonSchema, b as package_default, c as getDefaultAIConfig, d as DEFAULT_MINERU_CONFIG, f as DEFAULT_PROMPT_CONFIG, g as createConfig, h as AIConfigSchema, i as JsonSchemaDefinitionSchema, l as readAIConfig, m as PLACEHOLDER_TEXT, n as createMigrationConfig, o as toSnakeCase, p as PLACEHOLDER_SCHEMA, s as generateDrizzleSchema, t as collectDoctorDiagnostics, u as writeAIConfig, v as description, w as formatDoctorDiagnosticsJson, x as version, y as name } from "./doctor-collector-CykRm0fC.mjs";
|
|
2
2
|
import { createRequire } from "node:module";
|
|
3
|
+
import fs from "node:fs/promises";
|
|
4
|
+
import os from "node:os";
|
|
3
5
|
import path from "node:path";
|
|
4
6
|
import process from "node:process";
|
|
7
|
+
import { readFile, writeFile } from "jsonfile";
|
|
8
|
+
import { ZodError, z } from "zod";
|
|
5
9
|
import { fileURLToPath } from "node:url";
|
|
6
|
-
import { ZodError } from "zod";
|
|
7
|
-
import fs from "node:fs/promises";
|
|
8
10
|
import { defineCommand, runMain } from "citty";
|
|
9
11
|
import { consola } from "consola";
|
|
10
12
|
import updateNotifier from "update-notifier";
|
|
11
13
|
import CliTable3 from "cli-table3";
|
|
12
|
-
import { intro, outro, spinner } from "@clack/prompts";
|
|
13
|
-
import Database from "better-sqlite3";
|
|
14
|
+
import { intro, isCancel, outro, select, spinner, text } from "@clack/prompts";
|
|
14
15
|
import pc from "picocolors";
|
|
15
16
|
import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
|
|
17
|
+
import { LangfuseSpanProcessor } from "@langfuse/otel";
|
|
18
|
+
import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
|
|
16
19
|
import { APICallError, Output, generateText, jsonSchema } from "ai";
|
|
17
|
-
import
|
|
20
|
+
import mime from "mime";
|
|
21
|
+
import pRetry from "p-retry";
|
|
22
|
+
import { jsonrepair } from "jsonrepair";
|
|
23
|
+
import fs$1 from "node:fs";
|
|
24
|
+
import Database from "better-sqlite3";
|
|
25
|
+
import { glob, globSync } from "tinyglobby";
|
|
26
|
+
import { execa } from "execa";
|
|
27
|
+
import { extractText, getDocumentProxy, getMeta } from "unpdf";
|
|
28
|
+
import { Buffer } from "node:buffer";
|
|
29
|
+
import { execFile } from "node:child_process";
|
|
18
30
|
import { promisify } from "node:util";
|
|
19
31
|
import { serve } from "@hono/node-server";
|
|
32
|
+
import open from "open";
|
|
20
33
|
import { serveStatic } from "@hono/node-server/serve-static";
|
|
21
34
|
import { Hono } from "hono";
|
|
22
35
|
import { cors } from "hono/cors";
|
|
36
|
+
import { zValidator } from "@hono/zod-validator";
|
|
37
|
+
import { Kysely, SqliteDialect, sql } from "kysely";
|
|
23
38
|
|
|
24
39
|
//#region src/core/schema-sqlite/helpers.ts
|
|
25
40
|
const __filename = fileURLToPath(import.meta.url);
|
|
@@ -96,7 +111,7 @@ function parseAllSchemas(entries) {
|
|
|
96
111
|
}
|
|
97
112
|
|
|
98
113
|
//#endregion
|
|
99
|
-
//#region src/
|
|
114
|
+
//#region src/core/completion-scripts.ts
|
|
100
115
|
function bashScript(name$1) {
|
|
101
116
|
return `# ${name$1} bash completion
|
|
102
117
|
_${name$1}() {
|
|
@@ -123,7 +138,7 @@ function fishScript(name$1) {
|
|
|
123
138
|
complete -c ${name$1} -f -a '(${name$1} _complete (commandline -cp) 2>/dev/null)'
|
|
124
139
|
`;
|
|
125
140
|
}
|
|
126
|
-
function
|
|
141
|
+
function generateCompletionScript(name$1, shell) {
|
|
127
142
|
switch (shell) {
|
|
128
143
|
case "bash": return bashScript(name$1);
|
|
129
144
|
case "zsh": return zshScript(name$1);
|
|
@@ -131,6 +146,9 @@ function generateScript(name$1, shell) {
|
|
|
131
146
|
default: throw new Error(`Unsupported shell: ${shell}. Use bash, zsh, or fish.`);
|
|
132
147
|
}
|
|
133
148
|
}
|
|
149
|
+
|
|
150
|
+
//#endregion
|
|
151
|
+
//#region src/commands/completion.ts
|
|
134
152
|
const completionCommand = defineCommand({
|
|
135
153
|
meta: {
|
|
136
154
|
name: "completion",
|
|
@@ -145,7 +163,7 @@ const completionCommand = defineCommand({
|
|
|
145
163
|
const name$1 = "aiex";
|
|
146
164
|
const shell = args.shell;
|
|
147
165
|
try {
|
|
148
|
-
process.stdout.write(
|
|
166
|
+
process.stdout.write(generateCompletionScript(name$1, shell));
|
|
149
167
|
} catch (error) {
|
|
150
168
|
process.stderr.write(`Error: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
151
169
|
process.exit(1);
|
|
@@ -185,6 +203,14 @@ const doctorCommand = defineCommand({
|
|
|
185
203
|
}
|
|
186
204
|
});
|
|
187
205
|
|
|
206
|
+
//#endregion
|
|
207
|
+
//#region src/commands/utils.ts
|
|
208
|
+
function failCommand(message) {
|
|
209
|
+
if (message) consola.error(message);
|
|
210
|
+
outro("Failed!");
|
|
211
|
+
process.exitCode = 1;
|
|
212
|
+
}
|
|
213
|
+
|
|
188
214
|
//#endregion
|
|
189
215
|
//#region src/core/ai-extraction/model-capabilities.json
|
|
190
216
|
var model_capabilities_default = {
|
|
@@ -12727,26 +12753,26 @@ var model_capabilities_default = {
|
|
|
12727
12753
|
|
|
12728
12754
|
//#endregion
|
|
12729
12755
|
//#region src/core/ai-extraction/model-registry.ts
|
|
12730
|
-
const registry = model_capabilities_default;
|
|
12756
|
+
const registry$1 = model_capabilities_default;
|
|
12731
12757
|
function normalize(name$1) {
|
|
12732
12758
|
return name$1.toLowerCase().replace(/[-_. ]/g, "");
|
|
12733
12759
|
}
|
|
12734
12760
|
const normalizedCache = /* @__PURE__ */ new Map();
|
|
12735
12761
|
function buildNormalizedCache() {
|
|
12736
12762
|
if (normalizedCache.size > 0) return;
|
|
12737
|
-
for (const key of Object.keys(registry)) {
|
|
12763
|
+
for (const key of Object.keys(registry$1)) {
|
|
12738
12764
|
const nk = normalize(key);
|
|
12739
12765
|
if (!normalizedCache.has(nk)) normalizedCache.set(nk, key);
|
|
12740
12766
|
}
|
|
12741
12767
|
}
|
|
12742
12768
|
function lookupModel(name$1) {
|
|
12743
|
-
const exact = registry[name$1];
|
|
12769
|
+
const exact = registry$1[name$1];
|
|
12744
12770
|
if (exact) return { ...exact };
|
|
12745
12771
|
buildNormalizedCache();
|
|
12746
12772
|
const nk = normalize(name$1);
|
|
12747
12773
|
const matched = normalizedCache.get(nk);
|
|
12748
12774
|
if (matched) {
|
|
12749
|
-
const entry = registry[matched];
|
|
12775
|
+
const entry = registry$1[matched];
|
|
12750
12776
|
return entry ? { ...entry } : null;
|
|
12751
12777
|
}
|
|
12752
12778
|
return null;
|
|
@@ -12763,29 +12789,40 @@ function lookupModelCapabilities(modelName) {
|
|
|
12763
12789
|
//#endregion
|
|
12764
12790
|
//#region src/utils/retry.ts
|
|
12765
12791
|
async function withRetry(fn, onRetry, maxRetries = 5) {
|
|
12766
|
-
|
|
12767
|
-
|
|
12768
|
-
|
|
12769
|
-
|
|
12770
|
-
|
|
12771
|
-
|
|
12772
|
-
|
|
12773
|
-
|
|
12774
|
-
|
|
12775
|
-
|
|
12776
|
-
|
|
12777
|
-
|
|
12778
|
-
|
|
12779
|
-
|
|
12780
|
-
|
|
12781
|
-
|
|
12782
|
-
|
|
12792
|
+
return pRetry(async () => fn(), {
|
|
12793
|
+
retries: maxRetries,
|
|
12794
|
+
factor: 2,
|
|
12795
|
+
minTimeout: 1e3,
|
|
12796
|
+
randomize: true,
|
|
12797
|
+
onFailedAttempt({ error, attemptNumber, retriesLeft }) {
|
|
12798
|
+
if (!(error instanceof APICallError) || !error.isRetryable || retriesLeft <= 0) return;
|
|
12799
|
+
const baseDelayMs = 1e3 * 2 ** (attemptNumber - 1);
|
|
12800
|
+
onRetry?.({
|
|
12801
|
+
attempt: attemptNumber,
|
|
12802
|
+
maxRetries,
|
|
12803
|
+
delayMs: baseDelayMs,
|
|
12804
|
+
statusCode: error.statusCode
|
|
12805
|
+
});
|
|
12806
|
+
},
|
|
12807
|
+
shouldRetry({ error }) {
|
|
12808
|
+
return error instanceof APICallError && error.isRetryable;
|
|
12809
|
+
}
|
|
12810
|
+
});
|
|
12783
12811
|
}
|
|
12784
12812
|
|
|
12785
12813
|
//#endregion
|
|
12786
12814
|
//#region src/core/ai-extraction/json-utils.ts
|
|
12787
|
-
function
|
|
12788
|
-
const trimmed = text.trim();
|
|
12815
|
+
function parseJsonLike(text$1) {
|
|
12816
|
+
const trimmed = text$1.trim();
|
|
12817
|
+
try {
|
|
12818
|
+
return JSON.parse(trimmed);
|
|
12819
|
+
} catch {
|
|
12820
|
+
if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) throw new SyntaxError("JSON candidate must start with an object or array");
|
|
12821
|
+
return JSON.parse(jsonrepair(trimmed));
|
|
12822
|
+
}
|
|
12823
|
+
}
|
|
12824
|
+
function stripFences(text$1) {
|
|
12825
|
+
const trimmed = text$1.trim();
|
|
12789
12826
|
if (!trimmed.startsWith("```")) return null;
|
|
12790
12827
|
const endIndex = trimmed.lastIndexOf("```");
|
|
12791
12828
|
if (endIndex <= 3) return null;
|
|
@@ -12794,8 +12831,8 @@ function stripFences(text) {
|
|
|
12794
12831
|
if (firstNewline === -1) return null;
|
|
12795
12832
|
return inside.slice(firstNewline + 1).trim();
|
|
12796
12833
|
}
|
|
12797
|
-
function extractFirstJSON(text) {
|
|
12798
|
-
const trimmed = text.trim();
|
|
12834
|
+
function extractFirstJSON(text$1) {
|
|
12835
|
+
const trimmed = text$1.trim();
|
|
12799
12836
|
const firstBrace = trimmed.indexOf("{");
|
|
12800
12837
|
const firstBracket = trimmed.indexOf("[");
|
|
12801
12838
|
let start = -1;
|
|
@@ -12806,20 +12843,20 @@ function extractFirstJSON(text) {
|
|
|
12806
12843
|
if (end <= start) return null;
|
|
12807
12844
|
return trimmed.slice(start, end);
|
|
12808
12845
|
}
|
|
12809
|
-
function safeParseJSON(text) {
|
|
12810
|
-
const cleaned = text.trim();
|
|
12846
|
+
function safeParseJSON(text$1) {
|
|
12847
|
+
const cleaned = text$1.trim();
|
|
12811
12848
|
try {
|
|
12812
12849
|
return JSON.parse(cleaned);
|
|
12813
12850
|
} catch {}
|
|
12814
12851
|
const fromFence = stripFences(cleaned);
|
|
12815
12852
|
if (fromFence) try {
|
|
12816
|
-
return
|
|
12853
|
+
return parseJsonLike(fromFence);
|
|
12817
12854
|
} catch {}
|
|
12818
12855
|
const extracted = extractFirstJSON(cleaned);
|
|
12819
12856
|
if (extracted) try {
|
|
12820
|
-
return
|
|
12857
|
+
return parseJsonLike(extracted);
|
|
12821
12858
|
} catch {}
|
|
12822
|
-
const truncated = text.length > 200 ? `${text.slice(0, 200)}...` : text;
|
|
12859
|
+
const truncated = text$1.length > 200 ? `${text$1.slice(0, 200)}...` : text$1;
|
|
12823
12860
|
throw new Error(`Failed to parse JSON from model output. Expected a valid JSON object or array but received unparseable text. Raw output: ${truncated}`);
|
|
12824
12861
|
}
|
|
12825
12862
|
|
|
@@ -12907,11 +12944,11 @@ function schemaToDescription(schema) {
|
|
|
12907
12944
|
}
|
|
12908
12945
|
return lines.join("\n");
|
|
12909
12946
|
}
|
|
12910
|
-
function generateExtractionPrompt(schema, text, promptConfig = DEFAULT_PROMPT_CONFIG) {
|
|
12947
|
+
function generateExtractionPrompt(schema, text$1, promptConfig = DEFAULT_PROMPT_CONFIG) {
|
|
12911
12948
|
const schemaDescription = schemaToDescription(schema);
|
|
12912
12949
|
return {
|
|
12913
12950
|
system: promptConfig.systemTemplate.replaceAll(PLACEHOLDER_SCHEMA, schemaDescription),
|
|
12914
|
-
user: promptConfig.userTemplate.replaceAll(PLACEHOLDER_TEXT, text)
|
|
12951
|
+
user: promptConfig.userTemplate.replaceAll(PLACEHOLDER_TEXT, text$1)
|
|
12915
12952
|
};
|
|
12916
12953
|
}
|
|
12917
12954
|
function generatePromptSnapshot(schema, promptConfig = DEFAULT_PROMPT_CONFIG) {
|
|
@@ -12934,39 +12971,40 @@ function generatePromptSnapshot(schema, promptConfig = DEFAULT_PROMPT_CONFIG) {
|
|
|
12934
12971
|
|
|
12935
12972
|
//#endregion
|
|
12936
12973
|
//#region src/core/ai-extraction/extractor.ts
|
|
12974
|
+
let langfuseInitialized = false;
|
|
12975
|
+
function initLangfuse(config) {
|
|
12976
|
+
if (!config.langfuse?.publicKey || !config.langfuse.secretKey) return;
|
|
12977
|
+
if (langfuseInitialized) return;
|
|
12978
|
+
langfuseInitialized = true;
|
|
12979
|
+
try {
|
|
12980
|
+
new NodeTracerProvider({ spanProcessors: [new LangfuseSpanProcessor({
|
|
12981
|
+
publicKey: config.langfuse.publicKey,
|
|
12982
|
+
secretKey: config.langfuse.secretKey,
|
|
12983
|
+
baseUrl: config.langfuse.host || "https://us.cloud.langfuse.com",
|
|
12984
|
+
exportMode: "immediate"
|
|
12985
|
+
})] }).register();
|
|
12986
|
+
} catch (e) {
|
|
12987
|
+
console.warn("[Langfuse] Failed to initialize tracing:", e instanceof Error ? e.message : e);
|
|
12988
|
+
}
|
|
12989
|
+
}
|
|
12937
12990
|
const SYSTEM_PROMPT_REGEX = /## System Prompt\n([\s\S]*?)(?=## User Prompt|$)/;
|
|
12938
12991
|
const USER_PROMPT_REGEX = /## User Prompt Template\n([\s\S]*)$/;
|
|
12939
|
-
const MIME_TYPES = {
|
|
12940
|
-
png: "image/png",
|
|
12941
|
-
jpg: "image/jpeg",
|
|
12942
|
-
jpeg: "image/jpeg",
|
|
12943
|
-
gif: "image/gif",
|
|
12944
|
-
webp: "image/webp",
|
|
12945
|
-
bmp: "image/bmp",
|
|
12946
|
-
svg: "image/svg+xml",
|
|
12947
|
-
pdf: "application/pdf",
|
|
12948
|
-
txt: "text/plain",
|
|
12949
|
-
csv: "text/csv",
|
|
12950
|
-
json: "application/json",
|
|
12951
|
-
md: "text/markdown",
|
|
12952
|
-
html: "text/html"
|
|
12953
|
-
};
|
|
12954
12992
|
function detectMimeType(filePath) {
|
|
12955
|
-
return
|
|
12993
|
+
return mime.getType(filePath) ?? "application/octet-stream";
|
|
12956
12994
|
}
|
|
12957
12995
|
async function readFilePart(filePath) {
|
|
12958
|
-
const mime = detectMimeType(filePath);
|
|
12996
|
+
const mime$1 = detectMimeType(filePath);
|
|
12959
12997
|
const buffer = await fs.readFile(filePath);
|
|
12960
12998
|
const name$1 = path.basename(filePath);
|
|
12961
|
-
if (mime.startsWith("image/")) return {
|
|
12999
|
+
if (mime$1.startsWith("image/")) return {
|
|
12962
13000
|
type: "image",
|
|
12963
13001
|
image: buffer,
|
|
12964
|
-
mimeType: mime
|
|
13002
|
+
mimeType: mime$1
|
|
12965
13003
|
};
|
|
12966
13004
|
return {
|
|
12967
13005
|
type: "file",
|
|
12968
13006
|
data: buffer,
|
|
12969
|
-
mediaType: mime,
|
|
13007
|
+
mediaType: mime$1,
|
|
12970
13008
|
filename: name$1
|
|
12971
13009
|
};
|
|
12972
13010
|
}
|
|
@@ -13078,25 +13116,35 @@ async function loadPromptSnapshot(aiexDir, tableName) {
|
|
|
13078
13116
|
return null;
|
|
13079
13117
|
}
|
|
13080
13118
|
async function extractStructuredData(input) {
|
|
13081
|
-
const { config, schema, text, aiexDir, file, modelOverride } = input;
|
|
13119
|
+
const { config, schema, text: text$1, aiexDir, file, modelOverride } = input;
|
|
13082
13120
|
if (!config.provider.apiKey) return {
|
|
13083
13121
|
success: false,
|
|
13084
13122
|
error: "API Key not configured. Please configure AI settings in the web UI."
|
|
13085
13123
|
};
|
|
13086
13124
|
const useFileContent = !!file;
|
|
13087
13125
|
const isImageFile = useFileContent && detectMimeType(file).startsWith("image/");
|
|
13088
|
-
const inputTokens = text ? Math.ceil(text.length / 2) : void 0;
|
|
13126
|
+
const inputTokens = text$1 ? Math.ceil(text$1.length / 2) : void 0;
|
|
13089
13127
|
const fieldCount = schema.properties ? Object.keys(schema.properties).length : 0;
|
|
13090
13128
|
const outputTokens = fieldCount > 0 ? fieldCount * 80 : void 0;
|
|
13091
|
-
|
|
13092
|
-
|
|
13093
|
-
|
|
13094
|
-
|
|
13095
|
-
|
|
13096
|
-
|
|
13097
|
-
|
|
13129
|
+
let selected;
|
|
13130
|
+
try {
|
|
13131
|
+
selected = modelOverride ?? selectModel({
|
|
13132
|
+
models: config.provider.models,
|
|
13133
|
+
isImage: isImageFile,
|
|
13134
|
+
fileName: file,
|
|
13135
|
+
inputTokens,
|
|
13136
|
+
outputTokens
|
|
13137
|
+
});
|
|
13138
|
+
} catch (e) {
|
|
13139
|
+
return {
|
|
13140
|
+
success: false,
|
|
13141
|
+
error: e.message
|
|
13142
|
+
};
|
|
13143
|
+
}
|
|
13098
13144
|
const useStructuredOutput = selected.capabilities.structuredOutput;
|
|
13145
|
+
const useTelemetry = !!(config.langfuse?.publicKey && config.langfuse.secretKey);
|
|
13099
13146
|
try {
|
|
13147
|
+
if (useTelemetry) initLangfuse(config);
|
|
13100
13148
|
const provider = createOpenAICompatible({
|
|
13101
13149
|
baseURL: config.provider.baseURL,
|
|
13102
13150
|
name: "qwen",
|
|
@@ -13105,7 +13153,7 @@ async function extractStructuredData(input) {
|
|
|
13105
13153
|
let system;
|
|
13106
13154
|
let user;
|
|
13107
13155
|
const snapshot = await loadPromptSnapshot(aiexDir, schema.table.name);
|
|
13108
|
-
const promptText = file ? PLACEHOLDER_TEXT : text;
|
|
13156
|
+
const promptText = file ? PLACEHOLDER_TEXT : text$1;
|
|
13109
13157
|
if (snapshot) {
|
|
13110
13158
|
system = snapshot.system;
|
|
13111
13159
|
user = snapshot.user.replaceAll(PLACEHOLDER_TEXT, promptText);
|
|
@@ -13116,11 +13164,13 @@ async function extractStructuredData(input) {
|
|
|
13116
13164
|
}
|
|
13117
13165
|
const outputSchema = jsonSchema(schemaToExtractionOutputSchema(schema));
|
|
13118
13166
|
let result;
|
|
13167
|
+
const timeoutMs = (config.provider.timeout ?? 300) * 1e3;
|
|
13119
13168
|
if (useFileContent) {
|
|
13120
13169
|
const filePart = await readFilePart(file);
|
|
13170
|
+
const fileName = filePart.type === "file" ? filePart.filename : path.basename(file);
|
|
13121
13171
|
const contentParts = [{
|
|
13122
13172
|
type: "text",
|
|
13123
|
-
text: user.includes(PLACEHOLDER_TEXT) ? user.replaceAll(PLACEHOLDER_TEXT, text || `Data is contained in the attached file: ${
|
|
13173
|
+
text: user.includes(PLACEHOLDER_TEXT) ? user.replaceAll(PLACEHOLDER_TEXT, text$1 || `Data is contained in the attached file: ${fileName}`) : user
|
|
13124
13174
|
}, filePart];
|
|
13125
13175
|
const fileOpts = {
|
|
13126
13176
|
model: provider.chatModel(selected.name),
|
|
@@ -13129,8 +13179,9 @@ async function extractStructuredData(input) {
|
|
|
13129
13179
|
role: "user",
|
|
13130
13180
|
content: contentParts
|
|
13131
13181
|
}],
|
|
13132
|
-
abortSignal: AbortSignal.timeout(
|
|
13133
|
-
maxRetries: 0
|
|
13182
|
+
abortSignal: AbortSignal.timeout(timeoutMs),
|
|
13183
|
+
maxRetries: 0,
|
|
13184
|
+
experimental_telemetry: { isEnabled: useTelemetry }
|
|
13134
13185
|
};
|
|
13135
13186
|
if (useStructuredOutput) fileOpts.output = Output.object({ schema: outputSchema });
|
|
13136
13187
|
result = await withRetry(() => generateText(fileOpts), input.onRetry);
|
|
@@ -13139,8 +13190,9 @@ async function extractStructuredData(input) {
|
|
|
13139
13190
|
model: provider.chatModel(selected.name),
|
|
13140
13191
|
system,
|
|
13141
13192
|
prompt: user,
|
|
13142
|
-
abortSignal: AbortSignal.timeout(
|
|
13143
|
-
maxRetries: 0
|
|
13193
|
+
abortSignal: AbortSignal.timeout(timeoutMs),
|
|
13194
|
+
maxRetries: 0,
|
|
13195
|
+
experimental_telemetry: { isEnabled: useTelemetry }
|
|
13144
13196
|
};
|
|
13145
13197
|
if (useStructuredOutput) textOpts.output = Output.object({ schema: outputSchema });
|
|
13146
13198
|
result = await withRetry(() => generateText(textOpts), input.onRetry);
|
|
@@ -13158,7 +13210,10 @@ async function extractStructuredData(input) {
|
|
|
13158
13210
|
const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
13159
13211
|
const outputFileName = `${schema.table.name}-${timestamp}.json`;
|
|
13160
13212
|
const outputPath = path.join(outputDir, outputFileName);
|
|
13161
|
-
await
|
|
13213
|
+
await writeFile(outputPath, data, {
|
|
13214
|
+
spaces: 2,
|
|
13215
|
+
EOL: "\n"
|
|
13216
|
+
});
|
|
13162
13217
|
return {
|
|
13163
13218
|
success: true,
|
|
13164
13219
|
outputPath,
|
|
@@ -13223,8 +13278,8 @@ function buildInsertSql(table, data) {
|
|
|
13223
13278
|
function insertTableRow({ db, table, data, parentRowId, foreignKeyColumn }) {
|
|
13224
13279
|
const rowData = { ...data };
|
|
13225
13280
|
if (parentRowId !== void 0 && foreignKeyColumn) rowData[foreignKeyColumn] = parentRowId;
|
|
13226
|
-
const { sql, values } = buildInsertSql(table, rowData);
|
|
13227
|
-
const info = db.prepare(sql).run(...values);
|
|
13281
|
+
const { sql: sql$1, values } = buildInsertSql(table, rowData);
|
|
13282
|
+
const info = db.prepare(sql$1).run(...values);
|
|
13228
13283
|
return Number(info.lastInsertRowid);
|
|
13229
13284
|
}
|
|
13230
13285
|
function parseDataByColumns(data, schema, table) {
|
|
@@ -13327,8 +13382,172 @@ async function savePromptSnapshot(schema, aiexDir) {
|
|
|
13327
13382
|
}
|
|
13328
13383
|
|
|
13329
13384
|
//#endregion
|
|
13330
|
-
//#region src/
|
|
13331
|
-
|
|
13385
|
+
//#region src/core/pdf-converter/external.ts
|
|
13386
|
+
function applyTemplate(value, context) {
|
|
13387
|
+
return value.replaceAll("{input}", context.input).replaceAll("{outputDir}", context.outputDir).replaceAll("{basename}", context.basename);
|
|
13388
|
+
}
|
|
13389
|
+
function isError(error) {
|
|
13390
|
+
return error instanceof Error;
|
|
13391
|
+
}
|
|
13392
|
+
async function pathExists(filePath) {
|
|
13393
|
+
try {
|
|
13394
|
+
await fs.access(filePath);
|
|
13395
|
+
return true;
|
|
13396
|
+
} catch {
|
|
13397
|
+
return false;
|
|
13398
|
+
}
|
|
13399
|
+
}
|
|
13400
|
+
async function collectMarkdownFiles(dir) {
|
|
13401
|
+
return (await glob("**/*.md", {
|
|
13402
|
+
cwd: dir,
|
|
13403
|
+
absolute: true,
|
|
13404
|
+
onlyFiles: true
|
|
13405
|
+
})).sort();
|
|
13406
|
+
}
|
|
13407
|
+
async function selectMarkdownFile(outputDir, basename) {
|
|
13408
|
+
const files = await collectMarkdownFiles(outputDir);
|
|
13409
|
+
if (files.length === 0) throw new Error(`External PDF converter did not produce a markdown file in ${outputDir}`);
|
|
13410
|
+
const preferredName = `${basename}.md`.toLowerCase();
|
|
13411
|
+
return files.find((file) => path.basename(file).toLowerCase() === preferredName) ?? files[0];
|
|
13412
|
+
}
|
|
13413
|
+
function formatCommandError(error, command$1) {
|
|
13414
|
+
if (!isError(error)) return new Error(String(error));
|
|
13415
|
+
const details = [`External PDF converter failed: ${command$1}`];
|
|
13416
|
+
if ("exitCode" in error && typeof error.exitCode === "number") details.push(`exitCode=${error.exitCode}`);
|
|
13417
|
+
if ("signal" in error && error.signal) details.push(`signal=${String(error.signal)}`);
|
|
13418
|
+
if ("stderr" in error && typeof error.stderr === "string" && error.stderr.trim()) details.push(error.stderr.trim());
|
|
13419
|
+
else if (error.message) details.push(error.message);
|
|
13420
|
+
return new Error(details.join("\n"));
|
|
13421
|
+
}
|
|
13422
|
+
async function countPdfPages(input) {
|
|
13423
|
+
try {
|
|
13424
|
+
return (await getDocumentProxy(input)).numPages;
|
|
13425
|
+
} catch {
|
|
13426
|
+
return 0;
|
|
13427
|
+
}
|
|
13428
|
+
}
|
|
13429
|
+
var ExternalCommandPdfConverter = class {
|
|
13430
|
+
name;
|
|
13431
|
+
constructor(name$1, config) {
|
|
13432
|
+
this.config = config;
|
|
13433
|
+
this.name = name$1;
|
|
13434
|
+
}
|
|
13435
|
+
async convert(input, filePath) {
|
|
13436
|
+
const tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "aiex-mineru-"));
|
|
13437
|
+
const outputDir = path.join(tempRoot, "output");
|
|
13438
|
+
await fs.mkdir(outputDir, { recursive: true });
|
|
13439
|
+
const inputPath = filePath ?? path.join(tempRoot, "input.pdf");
|
|
13440
|
+
if (!filePath) await fs.writeFile(inputPath, input);
|
|
13441
|
+
const pageCount = await countPdfPages(input);
|
|
13442
|
+
const basename = path.basename(inputPath, path.extname(inputPath));
|
|
13443
|
+
const context = {
|
|
13444
|
+
input: inputPath,
|
|
13445
|
+
outputDir,
|
|
13446
|
+
basename
|
|
13447
|
+
};
|
|
13448
|
+
const args = this.config.args.map((arg) => applyTemplate(arg, context));
|
|
13449
|
+
const timeoutMs = (this.config.timeout ?? 600) * 1e3;
|
|
13450
|
+
try {
|
|
13451
|
+
await execa(this.config.command, args, {
|
|
13452
|
+
shell: false,
|
|
13453
|
+
timeout: timeoutMs,
|
|
13454
|
+
maxBuffer: 1024 * 1024 * 20
|
|
13455
|
+
});
|
|
13456
|
+
const outputPath = this.config.outputFile ? applyTemplate(this.config.outputFile, context) : await selectMarkdownFile(outputDir, basename);
|
|
13457
|
+
if (!await pathExists(outputPath)) throw new Error(`External PDF converter output was not found: ${outputPath}`);
|
|
13458
|
+
return {
|
|
13459
|
+
text: await fs.readFile(outputPath, "utf-8"),
|
|
13460
|
+
pageCount,
|
|
13461
|
+
metadata: {
|
|
13462
|
+
converter: this.name,
|
|
13463
|
+
outputPath,
|
|
13464
|
+
...this.config.keepOutput ? { outputDir } : {}
|
|
13465
|
+
}
|
|
13466
|
+
};
|
|
13467
|
+
} catch (error) {
|
|
13468
|
+
throw formatCommandError(error, `${this.config.command} ${args.join(" ")}`);
|
|
13469
|
+
} finally {
|
|
13470
|
+
if (!this.config.keepOutput) await fs.rm(tempRoot, {
|
|
13471
|
+
recursive: true,
|
|
13472
|
+
force: true
|
|
13473
|
+
}).catch(() => {});
|
|
13474
|
+
}
|
|
13475
|
+
}
|
|
13476
|
+
};
|
|
13477
|
+
|
|
13478
|
+
//#endregion
|
|
13479
|
+
//#region src/core/pdf-converter/unpdf.ts
|
|
13480
|
+
var UnpdfConverter = class {
|
|
13481
|
+
name = "unpdf";
|
|
13482
|
+
async convert(input) {
|
|
13483
|
+
const data = Buffer.isBuffer(input) ? new Uint8Array(input) : input;
|
|
13484
|
+
const [textResult, meta] = await Promise.all([extractText(data, { mergePages: true }), getMeta(data).catch(() => null)]);
|
|
13485
|
+
return {
|
|
13486
|
+
text: textResult.text,
|
|
13487
|
+
pageCount: textResult.totalPages,
|
|
13488
|
+
metadata: {
|
|
13489
|
+
converter: this.name,
|
|
13490
|
+
...meta?.info ? Object.fromEntries(Object.entries(meta.info).map(([k, v]) => [k, String(v)])) : {}
|
|
13491
|
+
}
|
|
13492
|
+
};
|
|
13493
|
+
}
|
|
13494
|
+
};
|
|
13495
|
+
|
|
13496
|
+
//#endregion
|
|
13497
|
+
//#region src/core/pdf-converter/factory.ts
|
|
13498
|
+
const registry = /* @__PURE__ */ new Map();
|
|
13499
|
+
var FallbackPdfConverter = class {
|
|
13500
|
+
name;
|
|
13501
|
+
constructor(primary, fallback) {
|
|
13502
|
+
this.primary = primary;
|
|
13503
|
+
this.fallback = fallback;
|
|
13504
|
+
this.name = primary.name;
|
|
13505
|
+
}
|
|
13506
|
+
async convert(input, filePath) {
|
|
13507
|
+
try {
|
|
13508
|
+
return await this.primary.convert(input, filePath);
|
|
13509
|
+
} catch (err) {
|
|
13510
|
+
consola.warn(`${this.primary.name} failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
13511
|
+
consola.info(`Falling back to ${this.fallback.name}`);
|
|
13512
|
+
const result = await this.fallback.convert(input, filePath);
|
|
13513
|
+
return {
|
|
13514
|
+
...result,
|
|
13515
|
+
metadata: {
|
|
13516
|
+
...result.metadata,
|
|
13517
|
+
fallback: "true"
|
|
13518
|
+
}
|
|
13519
|
+
};
|
|
13520
|
+
}
|
|
13521
|
+
}
|
|
13522
|
+
};
|
|
13523
|
+
function withFallback(converter, config) {
|
|
13524
|
+
if (!config.fallbackToUnpdf) return converter;
|
|
13525
|
+
return new FallbackPdfConverter(converter, new UnpdfConverter());
|
|
13526
|
+
}
|
|
13527
|
+
function createPdfConverter(config) {
|
|
13528
|
+
if (typeof config === "object") {
|
|
13529
|
+
if (config.converter === "mineru") {
|
|
13530
|
+
const mineruConfig = config.mineru ?? DEFAULT_MINERU_CONFIG;
|
|
13531
|
+
return withFallback(new ExternalCommandPdfConverter("mineru", mineruConfig), mineruConfig);
|
|
13532
|
+
}
|
|
13533
|
+
if (config.converter === "external") {
|
|
13534
|
+
if (!config.external) throw new Error("External PDF converter is selected but no external command is configured.");
|
|
13535
|
+
return withFallback(new ExternalCommandPdfConverter("external", config.external), config.external);
|
|
13536
|
+
}
|
|
13537
|
+
}
|
|
13538
|
+
const key = typeof config === "string" ? config : "unpdf";
|
|
13539
|
+
let instance = registry.get(key);
|
|
13540
|
+
if (!instance) {
|
|
13541
|
+
if (key !== "unpdf") throw new Error(`PDF converter "${key}" requires configuration.`);
|
|
13542
|
+
instance = new UnpdfConverter();
|
|
13543
|
+
registry.set(key, instance);
|
|
13544
|
+
}
|
|
13545
|
+
return instance;
|
|
13546
|
+
}
|
|
13547
|
+
|
|
13548
|
+
//#endregion
|
|
13549
|
+
//#region src/core/extract-runner.ts
|
|
13550
|
+
const FILE_PART_EXTENSIONS = new Set([
|
|
13332
13551
|
"png",
|
|
13333
13552
|
"jpg",
|
|
13334
13553
|
"jpeg",
|
|
@@ -13337,12 +13556,21 @@ const IMAGE_EXTENSIONS = new Set([
|
|
|
13337
13556
|
"bmp",
|
|
13338
13557
|
"svg"
|
|
13339
13558
|
]);
|
|
13340
|
-
const
|
|
13341
|
-
|
|
13342
|
-
|
|
13343
|
-
|
|
13344
|
-
|
|
13345
|
-
|
|
13559
|
+
const SUPPORTED_EXTENSIONS = new Set([
|
|
13560
|
+
...FILE_PART_EXTENSIONS,
|
|
13561
|
+
"pdf",
|
|
13562
|
+
"txt",
|
|
13563
|
+
"md",
|
|
13564
|
+
"csv",
|
|
13565
|
+
"json",
|
|
13566
|
+
"html",
|
|
13567
|
+
"xml",
|
|
13568
|
+
"yaml",
|
|
13569
|
+
"yml"
|
|
13570
|
+
]);
|
|
13571
|
+
const PDF_EXT_RE = /\.pdf$/i;
|
|
13572
|
+
const JSON_EXT_RE = /\.json$/;
|
|
13573
|
+
const SUPPORTED_FILE_PATTERN = `*.{${[...SUPPORTED_EXTENSIONS].join(",")}}`;
|
|
13346
13574
|
async function ensureDatabaseReady(dbPath, schema) {
|
|
13347
13575
|
try {
|
|
13348
13576
|
await fs.access(dbPath);
|
|
@@ -13362,17 +13590,213 @@ async function ensureDatabaseReady(dbPath, schema) {
|
|
|
13362
13590
|
}
|
|
13363
13591
|
return null;
|
|
13364
13592
|
}
|
|
13593
|
+
function listSupportedFiles(dir, pattern) {
|
|
13594
|
+
if (!fs$1.statSync(dir).isDirectory()) throw new Error(`Not a directory: ${dir}`);
|
|
13595
|
+
return globSync(pattern ?? SUPPORTED_FILE_PATTERN, {
|
|
13596
|
+
cwd: dir,
|
|
13597
|
+
absolute: true,
|
|
13598
|
+
onlyFiles: true
|
|
13599
|
+
}).filter((file) => {
|
|
13600
|
+
const ext = path.extname(file).toLowerCase().replace(".", "");
|
|
13601
|
+
return SUPPORTED_EXTENSIONS.has(ext);
|
|
13602
|
+
}).sort();
|
|
13603
|
+
}
|
|
13604
|
+
async function loadSchema(config, schemaName) {
|
|
13605
|
+
const schemaPath = path.join(config.schemaPath, `${schemaName}.json`);
|
|
13606
|
+
try {
|
|
13607
|
+
const parsed = await readFile(schemaPath);
|
|
13608
|
+
return { schema: JsonSchemaDefinitionSchema.parse(parsed) };
|
|
13609
|
+
} catch (e) {
|
|
13610
|
+
if (e instanceof ZodError) return {
|
|
13611
|
+
schema: null,
|
|
13612
|
+
error: `Schema validation failed: ${schemaName}.json\n${e.issues.map((i) => ` - ${i.path.join(".")}: ${i.message}`).join("\n")}`
|
|
13613
|
+
};
|
|
13614
|
+
if (e.code === "ENOENT") return {
|
|
13615
|
+
schema: null,
|
|
13616
|
+
error: `Cannot read schema file: ${schemaName}.json`
|
|
13617
|
+
};
|
|
13618
|
+
if (e instanceof SyntaxError) return {
|
|
13619
|
+
schema: null,
|
|
13620
|
+
error: `Invalid JSON in schema file: ${schemaName}.json`
|
|
13621
|
+
};
|
|
13622
|
+
return {
|
|
13623
|
+
schema: null,
|
|
13624
|
+
error: String(e)
|
|
13625
|
+
};
|
|
13626
|
+
}
|
|
13627
|
+
}
|
|
13628
|
+
async function listSchemas(aiexDir) {
|
|
13629
|
+
try {
|
|
13630
|
+
const dir = path.join(aiexDir, "schema");
|
|
13631
|
+
return (await fs.readdir(dir)).filter((f) => f.endsWith(".json")).map((f) => f.replace(JSON_EXT_RE, "")).sort();
|
|
13632
|
+
} catch {
|
|
13633
|
+
return [];
|
|
13634
|
+
}
|
|
13635
|
+
}
|
|
13636
|
+
async function readExtractFileInput(filePath, aiConfig) {
|
|
13637
|
+
const ext = path.extname(filePath).toLowerCase().replace(".", "");
|
|
13638
|
+
if (FILE_PART_EXTENSIONS.has(ext)) return {
|
|
13639
|
+
text: "",
|
|
13640
|
+
filePath
|
|
13641
|
+
};
|
|
13642
|
+
if (ext === "pdf") {
|
|
13643
|
+
const buffer = await fs.readFile(filePath);
|
|
13644
|
+
const converter = createPdfConverter(aiConfig?.pdf);
|
|
13645
|
+
const result = await converter.convert(buffer, filePath);
|
|
13646
|
+
if (result.metadata?.fallback === "true") consola.info(`Fell back to unpdf — ${result.pageCount} page(s) extracted`);
|
|
13647
|
+
else consola.info(`Converted PDF via ${converter.name}, ${result.pageCount} page(s)`);
|
|
13648
|
+
const mdPath = filePath.replace(PDF_EXT_RE, ".md");
|
|
13649
|
+
try {
|
|
13650
|
+
await fs.writeFile(mdPath, result.text);
|
|
13651
|
+
consola.info(`Markdown saved: ${mdPath}`);
|
|
13652
|
+
} catch {
|
|
13653
|
+
const fallbackMd = path.join(os.tmpdir(), `${path.basename(filePath, ".pdf")}.md`);
|
|
13654
|
+
await fs.writeFile(fallbackMd, result.text);
|
|
13655
|
+
consola.info(`Markdown saved: ${fallbackMd}`);
|
|
13656
|
+
}
|
|
13657
|
+
return { text: result.text };
|
|
13658
|
+
}
|
|
13659
|
+
return { text: await fs.readFile(filePath, "utf-8") };
|
|
13660
|
+
}
|
|
13661
|
+
async function extractSingle(aiexDir, config, aiConfig, schemaName, text$1, filePath, modelOverride, options) {
|
|
13662
|
+
const schemaLoad = await loadSchema(config, schemaName);
|
|
13663
|
+
if (!schemaLoad.schema) {
|
|
13664
|
+
if (!options?.quiet) consola.error(schemaLoad.error);
|
|
13665
|
+
return {
|
|
13666
|
+
success: false,
|
|
13667
|
+
error: schemaLoad.error
|
|
13668
|
+
};
|
|
13669
|
+
}
|
|
13670
|
+
const s = spinner();
|
|
13671
|
+
if (!options?.quiet) s.start(filePath ? `Extracting from ${path.basename(filePath)}...` : "Extracting data...");
|
|
13672
|
+
const result = await extractStructuredData({
|
|
13673
|
+
config: aiConfig,
|
|
13674
|
+
schema: schemaLoad.schema,
|
|
13675
|
+
text: text$1 ?? "",
|
|
13676
|
+
aiexDir,
|
|
13677
|
+
file: filePath,
|
|
13678
|
+
modelOverride,
|
|
13679
|
+
onRetry(info) {
|
|
13680
|
+
if (!options?.quiet) s.message(`API responded with ${info.statusCode}, retrying in ${info.delayMs / 1e3}s (${info.attempt}/${info.maxRetries})...`);
|
|
13681
|
+
}
|
|
13682
|
+
});
|
|
13683
|
+
if (!result.success) {
|
|
13684
|
+
if (!options?.quiet) {
|
|
13685
|
+
s.stop("Extraction failed");
|
|
13686
|
+
consola.error(result.error || "Unknown error");
|
|
13687
|
+
}
|
|
13688
|
+
return {
|
|
13689
|
+
success: false,
|
|
13690
|
+
error: result.error || "Unknown error"
|
|
13691
|
+
};
|
|
13692
|
+
}
|
|
13693
|
+
if (!options?.quiet) s.stop("Extraction complete");
|
|
13694
|
+
if (result.outputPath && !options?.quiet) consola.success(`Result saved: ${pc.cyan(result.outputPath)}`);
|
|
13695
|
+
if (result.tokensUsed && !options?.quiet) consola.info(pc.gray(`Token usage: prompt=${result.tokensUsed.prompt}, completion=${result.tokensUsed.completion}, total=${result.tokensUsed.total}`));
|
|
13696
|
+
if (result.data) {
|
|
13697
|
+
const s2 = spinner();
|
|
13698
|
+
if (!options?.quiet) s2.start("Inserting into database...");
|
|
13699
|
+
const dbError = await ensureDatabaseReady(config.databasePath, schemaLoad.schema);
|
|
13700
|
+
if (dbError) {
|
|
13701
|
+
if (!options?.quiet) s2.stop("Database not ready");
|
|
13702
|
+
consola.error(dbError);
|
|
13703
|
+
return {
|
|
13704
|
+
success: false,
|
|
13705
|
+
error: dbError
|
|
13706
|
+
};
|
|
13707
|
+
}
|
|
13708
|
+
try {
|
|
13709
|
+
const db = new Database(config.databasePath);
|
|
13710
|
+
try {
|
|
13711
|
+
const insertResult = insertExtractedData(db, schemaLoad.schema, result.data);
|
|
13712
|
+
if (insertResult.success) {
|
|
13713
|
+
if (!options?.quiet) s2.stop(`Inserted into ${insertResult.tablesInserted.length} table(s)`);
|
|
13714
|
+
} else {
|
|
13715
|
+
if (!options?.quiet) s2.stop("Database insert failed");
|
|
13716
|
+
consola.error(insertResult.error || "Unknown error");
|
|
13717
|
+
return {
|
|
13718
|
+
success: false,
|
|
13719
|
+
error: insertResult.error
|
|
13720
|
+
};
|
|
13721
|
+
}
|
|
13722
|
+
} finally {
|
|
13723
|
+
db.close();
|
|
13724
|
+
}
|
|
13725
|
+
} catch (e) {
|
|
13726
|
+
if (!options?.quiet) s2.stop("Database insert failed");
|
|
13727
|
+
consola.error(e instanceof Error ? e.message : String(e));
|
|
13728
|
+
return {
|
|
13729
|
+
success: false,
|
|
13730
|
+
error: String(e)
|
|
13731
|
+
};
|
|
13732
|
+
}
|
|
13733
|
+
}
|
|
13734
|
+
return { success: true };
|
|
13735
|
+
}
|
|
13736
|
+
async function processOneFile(aiexDir, config, aiConfig, schemaName, filePath, modelOverride) {
|
|
13737
|
+
try {
|
|
13738
|
+
const input = await readExtractFileInput(filePath, aiConfig);
|
|
13739
|
+
const r = await extractSingle(aiexDir, config, aiConfig, schemaName, input.text, input.filePath, modelOverride, { quiet: false });
|
|
13740
|
+
if (r.success) {
|
|
13741
|
+
consola.success(`Processed: ${path.basename(filePath)}`);
|
|
13742
|
+
return true;
|
|
13743
|
+
} else {
|
|
13744
|
+
consola.error(`Failed: ${r.error}`);
|
|
13745
|
+
return false;
|
|
13746
|
+
}
|
|
13747
|
+
} catch (e) {
|
|
13748
|
+
consola.error(`Error processing ${path.basename(filePath)}: ${e instanceof Error ? e.message : String(e)}`);
|
|
13749
|
+
return false;
|
|
13750
|
+
}
|
|
13751
|
+
}
|
|
13752
|
+
async function runBatchExtraction(aiexDir, config, aiConfig, schemaName, dir, globPattern, modelOverride) {
|
|
13753
|
+
consola.info(`Scanning ${pc.cyan(dir)} for supported files...`);
|
|
13754
|
+
let files;
|
|
13755
|
+
try {
|
|
13756
|
+
files = listSupportedFiles(dir, globPattern);
|
|
13757
|
+
} catch {
|
|
13758
|
+
return {
|
|
13759
|
+
ok: false,
|
|
13760
|
+
successCount: 0,
|
|
13761
|
+
failCount: 0,
|
|
13762
|
+
error: `Cannot read directory: ${dir}`
|
|
13763
|
+
};
|
|
13764
|
+
}
|
|
13765
|
+
if (files.length === 0) return {
|
|
13766
|
+
ok: false,
|
|
13767
|
+
successCount: 0,
|
|
13768
|
+
failCount: 0,
|
|
13769
|
+
error: `No supported files found in ${dir}`
|
|
13770
|
+
};
|
|
13771
|
+
consola.info(`Found ${files.length} file(s) to process`);
|
|
13772
|
+
let successCount = 0;
|
|
13773
|
+
let failCount = 0;
|
|
13774
|
+
for (let i = 0; i < files.length; i++) {
|
|
13775
|
+
const file = files[i];
|
|
13776
|
+
consola.info(`\n[${i + 1}/${files.length}] Processing: ${pc.cyan(path.basename(file))}`);
|
|
13777
|
+
if (await processOneFile(aiexDir, config, aiConfig, schemaName, file, modelOverride)) successCount++;
|
|
13778
|
+
else failCount++;
|
|
13779
|
+
}
|
|
13780
|
+
consola.info(`\nBatch complete: ${pc.green(`${successCount} succeeded`)}, ${pc.red(`${failCount} failed`)}, ${files.length} total`);
|
|
13781
|
+
return {
|
|
13782
|
+
ok: true,
|
|
13783
|
+
successCount,
|
|
13784
|
+
failCount
|
|
13785
|
+
};
|
|
13786
|
+
}
|
|
13787
|
+
|
|
13788
|
+
//#endregion
|
|
13789
|
+
//#region src/commands/extract.ts
|
|
13365
13790
|
const extractCommand = defineCommand({
|
|
13366
13791
|
meta: {
|
|
13367
13792
|
name: "extract",
|
|
13368
|
-
description: "Extract structured data from text or
|
|
13793
|
+
description: "Extract structured data from text, images, or PDFs"
|
|
13369
13794
|
},
|
|
13370
13795
|
args: {
|
|
13371
13796
|
schema: {
|
|
13372
13797
|
type: "string",
|
|
13373
13798
|
alias: "s",
|
|
13374
|
-
description: "Schema name (without .json extension)"
|
|
13375
|
-
required: true
|
|
13799
|
+
description: "Schema name (without .json extension)"
|
|
13376
13800
|
},
|
|
13377
13801
|
text: {
|
|
13378
13802
|
type: "string",
|
|
@@ -13382,43 +13806,47 @@ const extractCommand = defineCommand({
|
|
|
13382
13806
|
file: {
|
|
13383
13807
|
type: "string",
|
|
13384
13808
|
alias: "f",
|
|
13385
|
-
description: "File path (
|
|
13809
|
+
description: "File path (image/PDF) to extract from"
|
|
13386
13810
|
},
|
|
13387
13811
|
model: {
|
|
13388
13812
|
type: "string",
|
|
13389
13813
|
alias: "m",
|
|
13390
13814
|
description: "AI model to use for extraction (overrides auto-selection)"
|
|
13391
13815
|
},
|
|
13392
|
-
|
|
13393
|
-
type: "
|
|
13816
|
+
dir: {
|
|
13817
|
+
type: "string",
|
|
13394
13818
|
alias: "d",
|
|
13395
|
-
description: "
|
|
13396
|
-
|
|
13819
|
+
description: "Directory containing files to batch extract"
|
|
13820
|
+
},
|
|
13821
|
+
glob: {
|
|
13822
|
+
type: "string",
|
|
13823
|
+
alias: "g",
|
|
13824
|
+
description: "Glob pattern to filter files in batch mode (e.g. \"*.pdf\")"
|
|
13397
13825
|
}
|
|
13398
13826
|
},
|
|
13399
13827
|
async run({ args }) {
|
|
13400
13828
|
intro(pc.inverse(" aiex extract "));
|
|
13401
13829
|
const config = createMigrationConfig(process.cwd());
|
|
13402
13830
|
const aiexDir = path.dirname(config.schemaPath);
|
|
13403
|
-
if (
|
|
13404
|
-
|
|
13831
|
+
if (args.dir && args.text) {
|
|
13832
|
+
failCommand("Cannot combine -t/--text with -d/--dir");
|
|
13405
13833
|
return;
|
|
13406
13834
|
}
|
|
13407
|
-
if (args.
|
|
13408
|
-
|
|
13835
|
+
if (args.dir && args.file) {
|
|
13836
|
+
failCommand("Cannot combine -f/--file with -d/--dir");
|
|
13409
13837
|
return;
|
|
13410
13838
|
}
|
|
13411
13839
|
const aiConfig = await readAIConfig(aiexDir);
|
|
13412
13840
|
if (!aiConfig) {
|
|
13413
|
-
|
|
13841
|
+
failCommand("AI configuration not found. Please run \"aiex web\" to configure AI settings first");
|
|
13414
13842
|
return;
|
|
13415
13843
|
}
|
|
13416
13844
|
if (!aiConfig.provider.apiKey) {
|
|
13417
|
-
|
|
13845
|
+
failCommand("API Key not configured. Please configure AI settings in the Web interface first");
|
|
13418
13846
|
return;
|
|
13419
13847
|
}
|
|
13420
13848
|
if (!aiConfig.provider.models?.length) {
|
|
13421
|
-
|
|
13849
|
+
failCommand("No models configured. Please add at least one model in AI Settings");
|
|
13422
13850
|
return;
|
|
13423
13851
|
}
|
|
13424
13852
|
let modelOverride;
|
|
@@ -13426,135 +13854,212 @@ const extractCommand = defineCommand({
|
|
|
13426
13854
|
const matched = aiConfig.provider.models.find((m) => m.name === args.model);
|
|
13427
13855
|
if (!matched) {
|
|
13428
13856
|
const available = aiConfig.provider.models.map((m) => m.name).join(", ");
|
|
13429
|
-
|
|
13857
|
+
failCommand(`Model "${args.model}" not found in configuration. Available models: ${available}`);
|
|
13430
13858
|
return;
|
|
13431
13859
|
}
|
|
13432
13860
|
modelOverride = matched;
|
|
13433
13861
|
}
|
|
13434
|
-
|
|
13435
|
-
|
|
13436
|
-
|
|
13437
|
-
|
|
13438
|
-
|
|
13439
|
-
|
|
13440
|
-
|
|
13441
|
-
} catch {
|
|
13442
|
-
fail$1(`Cannot read file: ${args.file}`);
|
|
13862
|
+
if (!args.schema && !args.text && !args.file && !args.dir) {
|
|
13863
|
+
if (await runInteractive(aiexDir, config, aiConfig, modelOverride)) outro("Done!");
|
|
13864
|
+
return;
|
|
13865
|
+
}
|
|
13866
|
+
if (args.dir) {
|
|
13867
|
+
if (!args.schema) {
|
|
13868
|
+
failCommand("Schema name (-s) is required in batch mode");
|
|
13443
13869
|
return;
|
|
13444
13870
|
}
|
|
13445
|
-
|
|
13446
|
-
|
|
13447
|
-
|
|
13448
|
-
|
|
13449
|
-
|
|
13450
|
-
|
|
13451
|
-
|
|
13452
|
-
|
|
13453
|
-
fail$1(`Cannot read schema file: ${schemaName}.json`);
|
|
13871
|
+
const result = await runBatchExtraction(aiexDir, config, aiConfig, args.schema, args.dir, args.glob, modelOverride);
|
|
13872
|
+
if (!result.ok) {
|
|
13873
|
+
failCommand(result.error);
|
|
13874
|
+
return;
|
|
13875
|
+
}
|
|
13876
|
+
if (result.failCount > 0) process.exitCode = 1;
|
|
13877
|
+
if (result.failCount > 0) outro(`Completed with failures (${result.failCount} failed)`);
|
|
13878
|
+
else outro("Done!");
|
|
13454
13879
|
return;
|
|
13455
13880
|
}
|
|
13456
|
-
|
|
13457
|
-
schema
|
|
13881
|
+
if (!args.schema) {
|
|
13882
|
+
failCommand("Please provide a schema name (-s) to extract from");
|
|
13883
|
+
return;
|
|
13884
|
+
}
|
|
13885
|
+
if (!args.text && !args.file) {
|
|
13886
|
+
failCommand("Please provide text (-t) or a file (-f) to extract from");
|
|
13887
|
+
return;
|
|
13888
|
+
}
|
|
13889
|
+
if (args.text && args.file) {
|
|
13890
|
+
failCommand("-t and -f cannot be used together");
|
|
13891
|
+
return;
|
|
13892
|
+
}
|
|
13893
|
+
let text$1 = "";
|
|
13894
|
+
let filePath;
|
|
13895
|
+
if (args.file) try {
|
|
13896
|
+
const input = await readExtractFileInput(args.file, aiConfig);
|
|
13897
|
+
text$1 = input.text;
|
|
13898
|
+
filePath = input.filePath;
|
|
13458
13899
|
} catch (e) {
|
|
13459
|
-
|
|
13460
|
-
consola.error(`Schema validation failed: ${schemaName}.json`);
|
|
13461
|
-
for (const issue of e.issues) consola.error(` - ${issue.path.join(".")}: ${issue.message}`);
|
|
13462
|
-
}
|
|
13463
|
-
fail$1();
|
|
13900
|
+
failCommand(`Cannot read file: ${args.file} — ${e instanceof Error ? e.message : String(e)}`);
|
|
13464
13901
|
return;
|
|
13465
13902
|
}
|
|
13466
|
-
|
|
13467
|
-
|
|
13468
|
-
|
|
13469
|
-
|
|
13470
|
-
|
|
13471
|
-
|
|
13472
|
-
|
|
13473
|
-
|
|
13474
|
-
|
|
13475
|
-
|
|
13476
|
-
|
|
13903
|
+
else if (args.text) text$1 = args.text;
|
|
13904
|
+
if (!(await extractSingle(aiexDir, config, aiConfig, args.schema, text$1, filePath, modelOverride)).success) {
|
|
13905
|
+
failCommand();
|
|
13906
|
+
return;
|
|
13907
|
+
}
|
|
13908
|
+
outro("Done!");
|
|
13909
|
+
}
|
|
13910
|
+
});
|
|
13911
|
+
async function runInteractive(aiexDir, config, aiConfig, modelOverride) {
|
|
13912
|
+
const schemas = await listSchemas(aiexDir);
|
|
13913
|
+
if (schemas.length === 0) {
|
|
13914
|
+
failCommand(`No schema files found in ${pc.cyan(".aiex/schema/")}. Run ${pc.cyan("aiex web")} to create and configure schemas first.`);
|
|
13915
|
+
return false;
|
|
13916
|
+
}
|
|
13917
|
+
const schemaName = await select({
|
|
13918
|
+
message: "Select a schema to extract data for:",
|
|
13919
|
+
options: schemas.map((s) => ({
|
|
13920
|
+
label: s,
|
|
13921
|
+
value: s
|
|
13922
|
+
}))
|
|
13923
|
+
});
|
|
13924
|
+
if (isCancel(schemaName)) {
|
|
13925
|
+
cancel("Cancelled");
|
|
13926
|
+
return false;
|
|
13927
|
+
}
|
|
13928
|
+
const inputSource = await select({
|
|
13929
|
+
message: "Choose input source:",
|
|
13930
|
+
options: [
|
|
13931
|
+
{
|
|
13932
|
+
label: "Text content",
|
|
13933
|
+
value: "text",
|
|
13934
|
+
hint: "Paste or type text directly"
|
|
13935
|
+
},
|
|
13936
|
+
{
|
|
13937
|
+
label: "Single file",
|
|
13938
|
+
value: "file",
|
|
13939
|
+
hint: "Extract from a file (txt, pdf, image)"
|
|
13940
|
+
},
|
|
13941
|
+
{
|
|
13942
|
+
label: "Batch directory",
|
|
13943
|
+
value: "dir",
|
|
13944
|
+
hint: "Extract all supported files in a directory"
|
|
13945
|
+
}
|
|
13946
|
+
]
|
|
13947
|
+
});
|
|
13948
|
+
if (isCancel(inputSource)) {
|
|
13949
|
+
cancel("Cancelled");
|
|
13950
|
+
return false;
|
|
13951
|
+
}
|
|
13952
|
+
if (inputSource === "text") {
|
|
13953
|
+
const textContent = await text({
|
|
13954
|
+
message: "Enter text content to extract:",
|
|
13955
|
+
validate(value) {
|
|
13956
|
+
if (!value || value.trim().length === 0) return "Please enter some text";
|
|
13477
13957
|
}
|
|
13478
13958
|
});
|
|
13479
|
-
if (
|
|
13480
|
-
|
|
13481
|
-
|
|
13482
|
-
return;
|
|
13959
|
+
if (isCancel(textContent)) {
|
|
13960
|
+
cancel("Cancelled");
|
|
13961
|
+
return false;
|
|
13483
13962
|
}
|
|
13484
|
-
|
|
13485
|
-
|
|
13486
|
-
|
|
13487
|
-
|
|
13488
|
-
|
|
13489
|
-
|
|
13490
|
-
const dbError = await ensureDatabaseReady(config.databasePath, schema);
|
|
13491
|
-
if (dbError) {
|
|
13492
|
-
s2.stop("Database not ready");
|
|
13493
|
-
fail$1(dbError);
|
|
13494
|
-
return;
|
|
13963
|
+
return (await extractSingle(aiexDir, config, aiConfig, schemaName, textContent, void 0, modelOverride)).success;
|
|
13964
|
+
} else if (inputSource === "file") {
|
|
13965
|
+
const filePathStr = await text({
|
|
13966
|
+
message: "Enter file path:",
|
|
13967
|
+
validate(value) {
|
|
13968
|
+
if (!value || value.trim().length === 0) return "Please enter a file path";
|
|
13495
13969
|
}
|
|
13496
|
-
|
|
13497
|
-
|
|
13498
|
-
|
|
13499
|
-
|
|
13500
|
-
|
|
13501
|
-
|
|
13502
|
-
|
|
13503
|
-
|
|
13504
|
-
|
|
13505
|
-
|
|
13506
|
-
|
|
13507
|
-
|
|
13508
|
-
|
|
13509
|
-
|
|
13510
|
-
|
|
13511
|
-
|
|
13512
|
-
|
|
13970
|
+
});
|
|
13971
|
+
if (isCancel(filePathStr)) {
|
|
13972
|
+
cancel("Cancelled");
|
|
13973
|
+
return false;
|
|
13974
|
+
}
|
|
13975
|
+
const fp = filePathStr;
|
|
13976
|
+
try {
|
|
13977
|
+
const input = await readExtractFileInput(fp, aiConfig);
|
|
13978
|
+
return (await extractSingle(aiexDir, config, aiConfig, schemaName, input.text, input.filePath, modelOverride)).success;
|
|
13979
|
+
} catch (e) {
|
|
13980
|
+
consola.error(`Cannot read file: ${fp} — ${e instanceof Error ? e.message : String(e)}`);
|
|
13981
|
+
return false;
|
|
13982
|
+
}
|
|
13983
|
+
} else if (inputSource === "dir") {
|
|
13984
|
+
const dirPath = await text({
|
|
13985
|
+
message: "Enter directory path:",
|
|
13986
|
+
validate(value) {
|
|
13987
|
+
if (!value || value.trim().length === 0) return "Please enter a directory path";
|
|
13513
13988
|
}
|
|
13989
|
+
});
|
|
13990
|
+
if (isCancel(dirPath)) {
|
|
13991
|
+
cancel("Cancelled");
|
|
13992
|
+
return false;
|
|
13514
13993
|
}
|
|
13515
|
-
|
|
13994
|
+
const result = await runBatchExtraction(aiexDir, config, aiConfig, schemaName, dirPath, void 0, modelOverride);
|
|
13995
|
+
if (!result.ok) failCommand(result.error);
|
|
13996
|
+
return result.ok && result.failCount === 0;
|
|
13516
13997
|
}
|
|
13517
|
-
|
|
13518
|
-
|
|
13519
|
-
|
|
13520
|
-
|
|
13521
|
-
|
|
13998
|
+
return false;
|
|
13999
|
+
}
|
|
14000
|
+
function cancel(msg) {
|
|
14001
|
+
consola.info(msg);
|
|
14002
|
+
outro("Cancelled");
|
|
14003
|
+
process.exitCode = 0;
|
|
14004
|
+
}
|
|
13522
14005
|
|
|
13523
14006
|
//#endregion
|
|
13524
|
-
//#region src/
|
|
13525
|
-
const execFileAsync
|
|
13526
|
-
function
|
|
13527
|
-
if (message) consola.error(message);
|
|
13528
|
-
outro("Failed!");
|
|
13529
|
-
process.exitCode = 1;
|
|
13530
|
-
}
|
|
13531
|
-
async function writeJsonIfAbsent(filePath, data) {
|
|
14007
|
+
//#region src/core/schema-runner.ts
|
|
14008
|
+
const execFileAsync = promisify(execFile);
|
|
14009
|
+
async function listSchemaFiles(schemaDir) {
|
|
13532
14010
|
try {
|
|
13533
|
-
await fs.
|
|
13534
|
-
|
|
13535
|
-
|
|
13536
|
-
if (error.code === "EEXIST") return "skipped";
|
|
13537
|
-
throw error;
|
|
14011
|
+
return (await fs.readdir(schemaDir)).filter((f) => f.endsWith(".json")).map((f) => path.join(schemaDir, f)).sort();
|
|
14012
|
+
} catch {
|
|
14013
|
+
return [];
|
|
13538
14014
|
}
|
|
13539
14015
|
}
|
|
13540
|
-
async function
|
|
14016
|
+
async function generateSchemaFromFiles(schemaFiles, config) {
|
|
13541
14017
|
const result = parseAllSchemas(await Promise.all(schemaFiles.map(async (filePath) => {
|
|
13542
14018
|
return {
|
|
13543
14019
|
filePath,
|
|
13544
14020
|
content: await fs.readFile(filePath, "utf-8")
|
|
13545
14021
|
};
|
|
13546
14022
|
})));
|
|
13547
|
-
if (!result.success) {
|
|
13548
|
-
|
|
13549
|
-
|
|
13550
|
-
|
|
13551
|
-
|
|
14023
|
+
if (!result.success) return {
|
|
14024
|
+
success: false,
|
|
14025
|
+
error: result.error,
|
|
14026
|
+
warnings: [],
|
|
14027
|
+
schemaCount: schemaFiles.length,
|
|
14028
|
+
tables: 0,
|
|
14029
|
+
relations: 0
|
|
14030
|
+
};
|
|
14031
|
+
const { tables, relations, reverseRelations, warnings, drizzleCode } = result.data;
|
|
13552
14032
|
await fs.mkdir(path.dirname(config.drizzleSchemaPath), { recursive: true });
|
|
13553
|
-
await fs.writeFile(config.drizzleSchemaPath,
|
|
13554
|
-
|
|
13555
|
-
|
|
14033
|
+
await fs.writeFile(config.drizzleSchemaPath, drizzleCode);
|
|
14034
|
+
return {
|
|
14035
|
+
success: true,
|
|
14036
|
+
warnings,
|
|
14037
|
+
schemaCount: schemaFiles.length,
|
|
14038
|
+
tables: tables.length,
|
|
14039
|
+
relations: relations.length + reverseRelations.length
|
|
14040
|
+
};
|
|
13556
14041
|
}
|
|
13557
|
-
|
|
14042
|
+
function parseMigrationOutput(stdout, stderr) {
|
|
14043
|
+
try {
|
|
14044
|
+
const jsonLine = stdout.trim().split("\n").find((l) => l.startsWith("{") && l.endsWith("}"));
|
|
14045
|
+
if (!jsonLine) return {
|
|
14046
|
+
success: false,
|
|
14047
|
+
error: "Migration helper did not return valid output"
|
|
14048
|
+
};
|
|
14049
|
+
const result = JSON.parse(jsonLine);
|
|
14050
|
+
if (!result.success) return {
|
|
14051
|
+
success: false,
|
|
14052
|
+
error: result.error || "Migration failed"
|
|
14053
|
+
};
|
|
14054
|
+
return result;
|
|
14055
|
+
} catch {
|
|
14056
|
+
return {
|
|
14057
|
+
success: false,
|
|
14058
|
+
error: stderr || stdout || "Migration helper failed"
|
|
14059
|
+
};
|
|
14060
|
+
}
|
|
14061
|
+
}
|
|
14062
|
+
async function runSchemaMigration(config, migrationName) {
|
|
13558
14063
|
const helperPath = resolveHelperPath();
|
|
13559
14064
|
const helperArgs = [
|
|
13560
14065
|
resolveTsxPath(),
|
|
@@ -13565,40 +14070,62 @@ async function migrate(config, migrationName) {
|
|
|
13565
14070
|
];
|
|
13566
14071
|
if (migrationName) helperArgs.push(migrationName);
|
|
13567
14072
|
try {
|
|
13568
|
-
const { stdout } = await execFileAsync
|
|
13569
|
-
|
|
13570
|
-
if (!result.success) {
|
|
13571
|
-
consola.error("Failed to generate migration");
|
|
13572
|
-
consola.error(result.error);
|
|
13573
|
-
return false;
|
|
13574
|
-
}
|
|
13575
|
-
if (result.changes === 0) {
|
|
13576
|
-
consola.info(pc.gray("No changes detected"));
|
|
13577
|
-
return true;
|
|
13578
|
-
}
|
|
13579
|
-
consola.success(pc.green("Migration files generated"));
|
|
13580
|
-
consola.success(pc.green("Database migrated"));
|
|
13581
|
-
return true;
|
|
14073
|
+
const { stdout, stderr } = await execFileAsync(process.execPath, helperArgs, { cwd: process.cwd() });
|
|
14074
|
+
return parseMigrationOutput(stdout, stderr);
|
|
13582
14075
|
} catch (error) {
|
|
13583
|
-
consola.error("Failed to generate migration");
|
|
13584
14076
|
const execError = error;
|
|
13585
|
-
|
|
13586
|
-
|
|
13587
|
-
|
|
14077
|
+
return {
|
|
14078
|
+
success: false,
|
|
14079
|
+
error: execError.stderr || execError.stdout || execError.message || String(error)
|
|
14080
|
+
};
|
|
13588
14081
|
}
|
|
13589
14082
|
}
|
|
14083
|
+
async function runSchemaSync(config, options = {}) {
|
|
14084
|
+
const schemaFiles = await listSchemaFiles(config.schemaPath);
|
|
14085
|
+
if (schemaFiles.length === 0) return {
|
|
14086
|
+
success: false,
|
|
14087
|
+
error: "No schema files found",
|
|
14088
|
+
warnings: [],
|
|
14089
|
+
schemaCount: 0,
|
|
14090
|
+
tables: 0,
|
|
14091
|
+
relations: 0
|
|
14092
|
+
};
|
|
14093
|
+
const generated = await generateSchemaFromFiles(schemaFiles, config);
|
|
14094
|
+
if (!generated.success) return {
|
|
14095
|
+
success: false,
|
|
14096
|
+
error: generated.error,
|
|
14097
|
+
warnings: generated.warnings,
|
|
14098
|
+
schemaCount: generated.schemaCount,
|
|
14099
|
+
tables: generated.tables,
|
|
14100
|
+
relations: generated.relations
|
|
14101
|
+
};
|
|
14102
|
+
if (options.generateOnly) return {
|
|
14103
|
+
success: true,
|
|
14104
|
+
warnings: generated.warnings,
|
|
14105
|
+
schemaCount: generated.schemaCount,
|
|
14106
|
+
tables: generated.tables,
|
|
14107
|
+
relations: generated.relations
|
|
14108
|
+
};
|
|
14109
|
+
const migration = await runSchemaMigration(config, options.migrationName);
|
|
14110
|
+
return {
|
|
14111
|
+
success: migration.success,
|
|
14112
|
+
error: migration.error,
|
|
14113
|
+
warnings: generated.warnings,
|
|
14114
|
+
schemaCount: generated.schemaCount,
|
|
14115
|
+
tables: generated.tables,
|
|
14116
|
+
relations: generated.relations,
|
|
14117
|
+
migration
|
|
14118
|
+
};
|
|
14119
|
+
}
|
|
14120
|
+
|
|
14121
|
+
//#endregion
|
|
14122
|
+
//#region src/commands/schema.ts
|
|
13590
14123
|
const schemaCommand = defineCommand({
|
|
13591
14124
|
meta: {
|
|
13592
14125
|
name: "schema",
|
|
13593
14126
|
description: "Sync JSON Schema to SQLite database"
|
|
13594
14127
|
},
|
|
13595
14128
|
args: {
|
|
13596
|
-
init: {
|
|
13597
|
-
type: "boolean",
|
|
13598
|
-
alias: "i",
|
|
13599
|
-
description: "Only initialize .aiex/ directory with example schema",
|
|
13600
|
-
default: false
|
|
13601
|
-
},
|
|
13602
14129
|
generate: {
|
|
13603
14130
|
type: "boolean",
|
|
13604
14131
|
alias: "g",
|
|
@@ -13613,226 +14140,21 @@ const schemaCommand = defineCommand({
|
|
|
13613
14140
|
async run({ args }) {
|
|
13614
14141
|
intro(pc.inverse(" aiex schema "));
|
|
13615
14142
|
const config = createMigrationConfig(process.cwd());
|
|
13616
|
-
|
|
13617
|
-
await fs.mkdir(config.schemaPath, { recursive: true });
|
|
13618
|
-
await fs.mkdir(path.dirname(config.drizzleSchemaPath), { recursive: true });
|
|
13619
|
-
await fs.mkdir(config.migrationsPath, { recursive: true });
|
|
13620
|
-
const userSchema = {
|
|
13621
|
-
$schema: $id,
|
|
13622
|
-
title: "User",
|
|
13623
|
-
type: "object",
|
|
13624
|
-
table: {
|
|
13625
|
-
name: "users",
|
|
13626
|
-
timestamps: true,
|
|
13627
|
-
softDelete: true
|
|
13628
|
-
},
|
|
13629
|
-
properties: {
|
|
13630
|
-
id: {
|
|
13631
|
-
type: "integer",
|
|
13632
|
-
primary: true,
|
|
13633
|
-
autoIncrement: true
|
|
13634
|
-
},
|
|
13635
|
-
email: {
|
|
13636
|
-
type: "string",
|
|
13637
|
-
format: "email",
|
|
13638
|
-
unique: true
|
|
13639
|
-
},
|
|
13640
|
-
username: {
|
|
13641
|
-
type: "string",
|
|
13642
|
-
minLength: 3,
|
|
13643
|
-
maxLength: 50,
|
|
13644
|
-
unique: true
|
|
13645
|
-
},
|
|
13646
|
-
displayName: {
|
|
13647
|
-
type: "string",
|
|
13648
|
-
maxLength: 100
|
|
13649
|
-
},
|
|
13650
|
-
bio: {
|
|
13651
|
-
type: "string",
|
|
13652
|
-
maxLength: 500
|
|
13653
|
-
},
|
|
13654
|
-
avatarUrl: {
|
|
13655
|
-
type: "string",
|
|
13656
|
-
format: "uri"
|
|
13657
|
-
},
|
|
13658
|
-
role: {
|
|
13659
|
-
type: "string",
|
|
13660
|
-
default: "member"
|
|
13661
|
-
},
|
|
13662
|
-
isActive: {
|
|
13663
|
-
type: "boolean",
|
|
13664
|
-
default: true
|
|
13665
|
-
},
|
|
13666
|
-
lastLoginAt: {
|
|
13667
|
-
type: "string",
|
|
13668
|
-
format: "date-time"
|
|
13669
|
-
},
|
|
13670
|
-
profile: {
|
|
13671
|
-
type: "object",
|
|
13672
|
-
drizzle: { mode: "json" },
|
|
13673
|
-
properties: {
|
|
13674
|
-
website: { type: "string" },
|
|
13675
|
-
location: { type: "string" },
|
|
13676
|
-
socialLinks: {
|
|
13677
|
-
type: "array",
|
|
13678
|
-
items: {
|
|
13679
|
-
type: "object",
|
|
13680
|
-
properties: {
|
|
13681
|
-
platform: { type: "string" },
|
|
13682
|
-
url: { type: "string" }
|
|
13683
|
-
}
|
|
13684
|
-
}
|
|
13685
|
-
}
|
|
13686
|
-
}
|
|
13687
|
-
},
|
|
13688
|
-
preferences: {
|
|
13689
|
-
type: "object",
|
|
13690
|
-
nested: {
|
|
13691
|
-
enabled: true,
|
|
13692
|
-
relation: "has-one"
|
|
13693
|
-
},
|
|
13694
|
-
properties: {
|
|
13695
|
-
theme: {
|
|
13696
|
-
type: "string",
|
|
13697
|
-
default: "light"
|
|
13698
|
-
},
|
|
13699
|
-
language: {
|
|
13700
|
-
type: "string",
|
|
13701
|
-
default: "en"
|
|
13702
|
-
},
|
|
13703
|
-
emailNotifications: {
|
|
13704
|
-
type: "boolean",
|
|
13705
|
-
default: true
|
|
13706
|
-
},
|
|
13707
|
-
pushNotifications: {
|
|
13708
|
-
type: "boolean",
|
|
13709
|
-
default: false
|
|
13710
|
-
}
|
|
13711
|
-
}
|
|
13712
|
-
}
|
|
13713
|
-
},
|
|
13714
|
-
required: ["email", "username"]
|
|
13715
|
-
};
|
|
13716
|
-
const postSchema = {
|
|
13717
|
-
$schema: $id,
|
|
13718
|
-
title: "Post",
|
|
13719
|
-
type: "object",
|
|
13720
|
-
table: {
|
|
13721
|
-
name: "posts",
|
|
13722
|
-
timestamps: true,
|
|
13723
|
-
softDelete: true
|
|
13724
|
-
},
|
|
13725
|
-
properties: {
|
|
13726
|
-
id: {
|
|
13727
|
-
type: "integer",
|
|
13728
|
-
primary: true,
|
|
13729
|
-
autoIncrement: true
|
|
13730
|
-
},
|
|
13731
|
-
title: {
|
|
13732
|
-
type: "string",
|
|
13733
|
-
minLength: 5,
|
|
13734
|
-
maxLength: 200
|
|
13735
|
-
},
|
|
13736
|
-
slug: {
|
|
13737
|
-
type: "string",
|
|
13738
|
-
maxLength: 250,
|
|
13739
|
-
unique: true
|
|
13740
|
-
},
|
|
13741
|
-
content: { type: "string" },
|
|
13742
|
-
excerpt: {
|
|
13743
|
-
type: "string",
|
|
13744
|
-
maxLength: 300
|
|
13745
|
-
},
|
|
13746
|
-
authorId: { type: "integer" },
|
|
13747
|
-
status: {
|
|
13748
|
-
type: "string",
|
|
13749
|
-
default: "draft"
|
|
13750
|
-
},
|
|
13751
|
-
viewCount: {
|
|
13752
|
-
type: "integer",
|
|
13753
|
-
default: 0,
|
|
13754
|
-
minimum: 0
|
|
13755
|
-
},
|
|
13756
|
-
likeCount: {
|
|
13757
|
-
type: "integer",
|
|
13758
|
-
default: 0,
|
|
13759
|
-
minimum: 0
|
|
13760
|
-
},
|
|
13761
|
-
publishedAt: {
|
|
13762
|
-
type: "string",
|
|
13763
|
-
format: "date-time"
|
|
13764
|
-
},
|
|
13765
|
-
tags: {
|
|
13766
|
-
type: "array",
|
|
13767
|
-
items: { type: "string" }
|
|
13768
|
-
},
|
|
13769
|
-
metadata: {
|
|
13770
|
-
type: "object",
|
|
13771
|
-
drizzle: { mode: "json" },
|
|
13772
|
-
properties: {
|
|
13773
|
-
featuredImage: { type: "string" },
|
|
13774
|
-
readingTime: { type: "integer" },
|
|
13775
|
-
seoTitle: { type: "string" },
|
|
13776
|
-
seoDescription: { type: "string" }
|
|
13777
|
-
}
|
|
13778
|
-
},
|
|
13779
|
-
comments: {
|
|
13780
|
-
type: "array",
|
|
13781
|
-
items: {
|
|
13782
|
-
type: "object",
|
|
13783
|
-
nested: {
|
|
13784
|
-
enabled: true,
|
|
13785
|
-
relation: "has-many"
|
|
13786
|
-
},
|
|
13787
|
-
properties: {
|
|
13788
|
-
content: {
|
|
13789
|
-
type: "string",
|
|
13790
|
-
minLength: 1,
|
|
13791
|
-
maxLength: 1e3
|
|
13792
|
-
},
|
|
13793
|
-
authorId: { type: "integer" },
|
|
13794
|
-
status: {
|
|
13795
|
-
type: "string",
|
|
13796
|
-
default: "pending"
|
|
13797
|
-
},
|
|
13798
|
-
parentId: { type: "integer" }
|
|
13799
|
-
}
|
|
13800
|
-
}
|
|
13801
|
-
}
|
|
13802
|
-
},
|
|
13803
|
-
required: [
|
|
13804
|
-
"title",
|
|
13805
|
-
"slug",
|
|
13806
|
-
"authorId"
|
|
13807
|
-
]
|
|
13808
|
-
};
|
|
13809
|
-
const userStatus = await writeJsonIfAbsent(path.join(config.schemaPath, "user.json"), userSchema);
|
|
13810
|
-
const postStatus = await writeJsonIfAbsent(path.join(config.schemaPath, "post.json"), postSchema);
|
|
13811
|
-
consola.success(`Initialized ${pc.cyan(".aiex/")} with example schemas`);
|
|
13812
|
-
if (userStatus === "skipped") consola.warn(`${pc.cyan(".aiex/schema/user.json")} already exists, skipped`);
|
|
13813
|
-
if (postStatus === "skipped") consola.warn(`${pc.cyan(".aiex/schema/post.json")} already exists, skipped`);
|
|
13814
|
-
consola.info("Example includes: User (with preferences has-one), Post (with comments has-many)");
|
|
13815
|
-
outro("Run: aiex schema");
|
|
13816
|
-
return;
|
|
13817
|
-
}
|
|
13818
|
-
let schemaFiles;
|
|
13819
|
-
try {
|
|
13820
|
-
schemaFiles = await fs.readdir(config.schemaPath);
|
|
13821
|
-
schemaFiles = schemaFiles.filter((f) => f.endsWith(".json")).map((f) => path.join(config.schemaPath, f));
|
|
13822
|
-
} catch {
|
|
13823
|
-
schemaFiles = [];
|
|
13824
|
-
}
|
|
14143
|
+
const schemaFiles = await listSchemaFiles(config.schemaPath);
|
|
13825
14144
|
if (schemaFiles.length === 0) {
|
|
13826
|
-
consola.info("
|
|
13827
|
-
|
|
14145
|
+
consola.info(`Run ${pc.cyan("aiex web")} to create and configure schemas in the Web UI`);
|
|
14146
|
+
failCommand(`No schema files found in ${pc.cyan(".aiex/schema/")}`);
|
|
13828
14147
|
return;
|
|
13829
14148
|
}
|
|
13830
14149
|
const s1 = spinner();
|
|
13831
14150
|
s1.start("Generating Drizzle schema...");
|
|
13832
|
-
const
|
|
13833
|
-
|
|
13834
|
-
if (
|
|
13835
|
-
|
|
14151
|
+
const generated = await generateSchemaFromFiles(schemaFiles, config);
|
|
14152
|
+
for (const warning of generated.warnings) consola.warn(warning);
|
|
14153
|
+
if (generated.success) consola.success(`Generated ${pc.cyan(".aiex/drizzle/schema.ts")} from ${generated.schemaCount} schema file(s)`);
|
|
14154
|
+
else if (generated.error) consola.error(generated.error);
|
|
14155
|
+
s1.stop(generated.success ? "Schema generated" : "Generation failed");
|
|
14156
|
+
if (!generated.success) {
|
|
14157
|
+
failCommand();
|
|
13836
14158
|
return;
|
|
13837
14159
|
}
|
|
13838
14160
|
if (args.generate) {
|
|
@@ -13841,10 +14163,18 @@ const schemaCommand = defineCommand({
|
|
|
13841
14163
|
}
|
|
13842
14164
|
const s2 = spinner();
|
|
13843
14165
|
s2.start("Running migrations...");
|
|
13844
|
-
const
|
|
13845
|
-
|
|
13846
|
-
|
|
13847
|
-
|
|
14166
|
+
const migration = await runSchemaMigration(config, args.name);
|
|
14167
|
+
if (!migration.success) {
|
|
14168
|
+
consola.error("Failed to generate migration");
|
|
14169
|
+
consola.error(migration.error || "Migration failed");
|
|
14170
|
+
} else if (migration.changes === 0) consola.info(pc.gray("No changes detected"));
|
|
14171
|
+
else {
|
|
14172
|
+
consola.success(pc.green("Migration files generated"));
|
|
14173
|
+
consola.success(pc.green("Database migrated"));
|
|
14174
|
+
}
|
|
14175
|
+
s2.stop(migration.success ? "Migrations applied" : "Migration failed");
|
|
14176
|
+
if (!migration.success) {
|
|
14177
|
+
failCommand();
|
|
13848
14178
|
return;
|
|
13849
14179
|
}
|
|
13850
14180
|
outro("Done!");
|
|
@@ -13868,13 +14198,7 @@ function aiRoutes(config) {
|
|
|
13868
14198
|
}
|
|
13869
14199
|
});
|
|
13870
14200
|
}
|
|
13871
|
-
return c.json(
|
|
13872
|
-
...aiConfig,
|
|
13873
|
-
provider: {
|
|
13874
|
-
...aiConfig.provider,
|
|
13875
|
-
apiKey: maskApiKey(aiConfig.provider.apiKey)
|
|
13876
|
-
}
|
|
13877
|
-
});
|
|
14201
|
+
return c.json(aiConfig);
|
|
13878
14202
|
});
|
|
13879
14203
|
app.post("/ai/registry-lookup", async (c) => {
|
|
13880
14204
|
try {
|
|
@@ -13902,11 +14226,6 @@ function aiRoutes(config) {
|
|
|
13902
14226
|
success: false,
|
|
13903
14227
|
error: "At least one model must be configured"
|
|
13904
14228
|
}, 400);
|
|
13905
|
-
if (body.provider?.apiKey?.startsWith("sk-***")) {
|
|
13906
|
-
const existing = await readAIConfig(aiexDir);
|
|
13907
|
-
if (existing) body.provider.apiKey = existing.provider.apiKey;
|
|
13908
|
-
else body.provider.apiKey = "";
|
|
13909
|
-
}
|
|
13910
14229
|
await writeAIConfig(aiexDir, AIConfigSchema.parse(body));
|
|
13911
14230
|
return c.json({ success: true });
|
|
13912
14231
|
} catch (error) {
|
|
@@ -13922,10 +14241,25 @@ function aiRoutes(config) {
|
|
|
13922
14241
|
//#endregion
|
|
13923
14242
|
//#region src/server/routes/data.ts
|
|
13924
14243
|
const FILE_REGEX = /\.json$/;
|
|
13925
|
-
const EXTRACTION_FILE_RE = /^[\w.-]+\.json$/;
|
|
13926
|
-
const TABLE_NAME_RE$1 = /^[a-z][a-z0-9_]*$/;
|
|
13927
14244
|
const TIMESTAMP_CLEANUP = /(\d{2})-(\d{2})-(\d{2})/;
|
|
13928
14245
|
const TIMESTAMP_TZ = /(\d{3})Z/;
|
|
14246
|
+
const tableParamSchema = z.object({ name: z.string().regex(/^[a-z][a-z0-9_]*$/) });
|
|
14247
|
+
const extractionFileParamSchema = z.object({ name: z.string().regex(/^[\w.-]+\.json$/).refine((name$1) => name$1 === path.basename(name$1) && !name$1.includes("..")) });
|
|
14248
|
+
const tableQuerySchema = z.object({
|
|
14249
|
+
page: z.coerce.number().int().min(1).catch(1),
|
|
14250
|
+
pageSize: z.coerce.number().int().min(1).max(500).catch(50),
|
|
14251
|
+
search: z.string().catch(""),
|
|
14252
|
+
sortField: z.string().optional(),
|
|
14253
|
+
sortOrder: z.preprocess((value) => typeof value === "string" ? value.toLowerCase() : value, z.enum(["asc", "desc"]).catch("asc"))
|
|
14254
|
+
});
|
|
14255
|
+
function invalidParamResponse$1(message) {
|
|
14256
|
+
return (result, c) => {
|
|
14257
|
+
if (!result.success) return c.json({ error: message }, 400);
|
|
14258
|
+
};
|
|
14259
|
+
}
|
|
14260
|
+
function createReadonlyQueryDb(databasePath) {
|
|
14261
|
+
return new Kysely({ dialect: new SqliteDialect({ database: new Database(databasePath, { readonly: true }) }) });
|
|
14262
|
+
}
|
|
13929
14263
|
function dataRoutes(config) {
|
|
13930
14264
|
const app = new Hono();
|
|
13931
14265
|
const aiexDir = path.dirname(config.schemaPath);
|
|
@@ -13973,15 +14307,19 @@ function dataRoutes(config) {
|
|
|
13973
14307
|
let db = null;
|
|
13974
14308
|
let dbTables = [];
|
|
13975
14309
|
try {
|
|
13976
|
-
db =
|
|
13977
|
-
dbTables =
|
|
14310
|
+
db = createReadonlyQueryDb(config.databasePath);
|
|
14311
|
+
dbTables = (await sql`
|
|
14312
|
+
select name
|
|
14313
|
+
from sqlite_master
|
|
14314
|
+
where type = 'table' and name not like 'sqlite_%' and name not like '_%'
|
|
14315
|
+
order by name
|
|
14316
|
+
`.execute(db)).rows.map((row) => row.name);
|
|
13978
14317
|
} catch {} finally {
|
|
13979
|
-
db?.
|
|
14318
|
+
await db?.destroy();
|
|
13980
14319
|
}
|
|
13981
14320
|
const tables = [];
|
|
13982
14321
|
for (const file of schemaFiles) try {
|
|
13983
|
-
const
|
|
13984
|
-
const schema = JSON.parse(content);
|
|
14322
|
+
const schema = await readFile(path.join(schemaDir, file));
|
|
13985
14323
|
const tableName = schema.table?.name;
|
|
13986
14324
|
if (!tableName) continue;
|
|
13987
14325
|
tables.push({
|
|
@@ -13997,46 +14335,64 @@ function dataRoutes(config) {
|
|
|
13997
14335
|
return c.json({ error: error instanceof Error ? error.message : String(error) }, 500);
|
|
13998
14336
|
}
|
|
13999
14337
|
});
|
|
14000
|
-
app.get("/data/tables/:name", async (c) => {
|
|
14001
|
-
const tableName = c.req.
|
|
14002
|
-
|
|
14003
|
-
const sortField = c.req.query("sortField");
|
|
14004
|
-
const sortOrder = c.req.query("sortOrder") || "asc";
|
|
14338
|
+
app.get("/data/tables/:name", zValidator("param", tableParamSchema, invalidParamResponse$1("Invalid table name")), zValidator("query", tableQuerySchema), async (c) => {
|
|
14339
|
+
const { name: tableName } = c.req.valid("param");
|
|
14340
|
+
const { page, pageSize, search, sortField, sortOrder } = c.req.valid("query");
|
|
14005
14341
|
let db;
|
|
14006
14342
|
try {
|
|
14007
|
-
db =
|
|
14343
|
+
db = createReadonlyQueryDb(config.databasePath);
|
|
14008
14344
|
} catch {
|
|
14009
14345
|
return c.json({ error: "Database not found. Run `aiex schema` first." }, 400);
|
|
14010
14346
|
}
|
|
14011
14347
|
try {
|
|
14012
|
-
if (
|
|
14013
|
-
|
|
14014
|
-
|
|
14015
|
-
|
|
14016
|
-
|
|
14348
|
+
if ((await sql`
|
|
14349
|
+
select name
|
|
14350
|
+
from sqlite_master
|
|
14351
|
+
where type = 'table' and name = ${tableName}
|
|
14352
|
+
`.execute(db)).rows.length === 0) return c.json({ error: `Table "${tableName}" not found in database` }, 404);
|
|
14353
|
+
const columns = (await sql`
|
|
14354
|
+
pragma table_info(${sql.table(tableName)})
|
|
14355
|
+
`.execute(db)).rows.map((col) => ({
|
|
14017
14356
|
name: col.name,
|
|
14018
14357
|
type: col.type,
|
|
14019
14358
|
notNull: !!col.notnull,
|
|
14020
14359
|
pk: !!col.pk
|
|
14021
14360
|
}));
|
|
14022
|
-
|
|
14023
|
-
|
|
14024
|
-
const
|
|
14025
|
-
const
|
|
14361
|
+
const searchConditions = columns.map((col) => sql`${sql.ref(col.name)} like ${`%${search}%`}`);
|
|
14362
|
+
const searchCondition = search ? sql`where ${sql.join(searchConditions, sql` or `)}` : sql``;
|
|
14363
|
+
const sortColumn = columns.find((col) => col.name === sortField);
|
|
14364
|
+
const orderBy = sortColumn ? sql`order by ${sql.ref(sortColumn.name)} ${sql.raw(sortOrder === "desc" ? "desc" : "asc")}` : sql``;
|
|
14365
|
+
const total = (await sql`
|
|
14366
|
+
select count(*) as count
|
|
14367
|
+
from ${sql.table(tableName)}
|
|
14368
|
+
${searchCondition}
|
|
14369
|
+
`.execute(db)).rows[0]?.count ?? 0;
|
|
14370
|
+
const offset = (page - 1) * pageSize;
|
|
14371
|
+
const totalPages = Math.max(1, Math.ceil(total / pageSize));
|
|
14372
|
+
const result = await sql`
|
|
14373
|
+
select *
|
|
14374
|
+
from ${sql.table(tableName)}
|
|
14375
|
+
${searchCondition}
|
|
14376
|
+
${orderBy}
|
|
14377
|
+
limit ${pageSize}
|
|
14378
|
+
offset ${offset}
|
|
14379
|
+
`.execute(db);
|
|
14026
14380
|
return c.json({
|
|
14027
14381
|
columns,
|
|
14028
|
-
rows,
|
|
14029
|
-
total
|
|
14382
|
+
rows: result.rows,
|
|
14383
|
+
total,
|
|
14384
|
+
page,
|
|
14385
|
+
pageSize,
|
|
14386
|
+
totalPages
|
|
14030
14387
|
});
|
|
14031
14388
|
} catch (error) {
|
|
14032
14389
|
return c.json({ error: error instanceof Error ? error.message : String(error) }, 500);
|
|
14033
14390
|
} finally {
|
|
14034
|
-
db.
|
|
14391
|
+
await db.destroy();
|
|
14035
14392
|
}
|
|
14036
14393
|
});
|
|
14037
|
-
app.get("/data/:name", async (c) => {
|
|
14038
|
-
const name$1 = c.req.
|
|
14039
|
-
if (name$1 !== path.basename(name$1) || !EXTRACTION_FILE_RE.test(name$1) || name$1.includes("..")) return c.json({ error: "Invalid extraction file name" }, 400);
|
|
14394
|
+
app.get("/data/:name", zValidator("param", extractionFileParamSchema, invalidParamResponse$1("Invalid extraction file name")), async (c) => {
|
|
14395
|
+
const { name: name$1 } = c.req.valid("param");
|
|
14040
14396
|
const filePath = path.join(extractedDir, name$1);
|
|
14041
14397
|
try {
|
|
14042
14398
|
const content = await fs.readFile(filePath, "utf-8");
|
|
@@ -14054,12 +14410,13 @@ function dataRoutes(config) {
|
|
|
14054
14410
|
|
|
14055
14411
|
//#endregion
|
|
14056
14412
|
//#region src/server/routes/schema.ts
|
|
14057
|
-
const
|
|
14058
|
-
const
|
|
14059
|
-
const
|
|
14060
|
-
function
|
|
14061
|
-
|
|
14062
|
-
|
|
14413
|
+
const schemaFileNameSchema = z.string().regex(/^[\w.-]+\.json$/).refine((name$1) => name$1 === path.basename(name$1) && !name$1.includes(".."));
|
|
14414
|
+
const schemaFileParamSchema = z.object({ name: schemaFileNameSchema });
|
|
14415
|
+
const tableNameParamSchema = z.object({ name: z.string().regex(/^[a-z][a-z0-9_]*$/) });
|
|
14416
|
+
function invalidParamResponse(message) {
|
|
14417
|
+
return (result, c) => {
|
|
14418
|
+
if (!result.success) return c.json({ error: message }, 400);
|
|
14419
|
+
};
|
|
14063
14420
|
}
|
|
14064
14421
|
function schemaRoutes(config) {
|
|
14065
14422
|
const app = new Hono();
|
|
@@ -14072,23 +14429,25 @@ function schemaRoutes(config) {
|
|
|
14072
14429
|
const jsonFiles = (await fs.readdir(schemaDir)).filter((f) => f.endsWith(".json"));
|
|
14073
14430
|
return c.json(jsonFiles);
|
|
14074
14431
|
});
|
|
14075
|
-
app.get("/schema/:name", async (c) => {
|
|
14076
|
-
const
|
|
14077
|
-
|
|
14432
|
+
app.get("/schema/:name", zValidator("param", schemaFileParamSchema, invalidParamResponse("Invalid schema file name")), async (c) => {
|
|
14433
|
+
const { name: name$1 } = c.req.valid("param");
|
|
14434
|
+
const filePath = path.join(schemaDir, name$1);
|
|
14078
14435
|
try {
|
|
14079
|
-
|
|
14080
|
-
return c.json(JSON.parse(content));
|
|
14436
|
+
return c.json(await readFile(filePath));
|
|
14081
14437
|
} catch {
|
|
14082
14438
|
return c.json({ error: "Schema not found" }, 404);
|
|
14083
14439
|
}
|
|
14084
14440
|
});
|
|
14085
|
-
app.post("/schema/:name", async (c) => {
|
|
14086
|
-
const
|
|
14087
|
-
|
|
14441
|
+
app.post("/schema/:name", zValidator("param", schemaFileParamSchema, invalidParamResponse("Invalid schema file name")), async (c) => {
|
|
14442
|
+
const { name: name$1 } = c.req.valid("param");
|
|
14443
|
+
const filePath = path.join(schemaDir, name$1);
|
|
14088
14444
|
try {
|
|
14089
14445
|
const body = await c.req.json();
|
|
14090
14446
|
await ensureDir();
|
|
14091
|
-
await
|
|
14447
|
+
await writeFile(filePath, body, {
|
|
14448
|
+
spaces: 2,
|
|
14449
|
+
EOL: "\n"
|
|
14450
|
+
});
|
|
14092
14451
|
const aiexDir = path.dirname(schemaDir);
|
|
14093
14452
|
try {
|
|
14094
14453
|
await savePromptSnapshot(JsonSchemaDefinitionSchema.parse(body), aiexDir);
|
|
@@ -14098,12 +14457,8 @@ function schemaRoutes(config) {
|
|
|
14098
14457
|
return c.json({ error: "Failed to save schema" }, 500);
|
|
14099
14458
|
}
|
|
14100
14459
|
});
|
|
14101
|
-
app.get("/prompt-snapshot/:name", async (c) => {
|
|
14102
|
-
const name$1 = c.req.
|
|
14103
|
-
if (!TABLE_NAME_RE.test(name$1)) return c.json({
|
|
14104
|
-
success: false,
|
|
14105
|
-
error: "Invalid table name"
|
|
14106
|
-
}, 400);
|
|
14460
|
+
app.get("/prompt-snapshot/:name", zValidator("param", tableNameParamSchema, invalidParamResponse("Invalid table name")), async (c) => {
|
|
14461
|
+
const { name: name$1 } = c.req.valid("param");
|
|
14107
14462
|
const aiexDir = path.dirname(schemaDir);
|
|
14108
14463
|
const snapshotPath = path.join(aiexDir, "extracted", `${name$1}.prompt.md`);
|
|
14109
14464
|
try {
|
|
@@ -14119,14 +14474,13 @@ function schemaRoutes(config) {
|
|
|
14119
14474
|
}, 404);
|
|
14120
14475
|
}
|
|
14121
14476
|
});
|
|
14122
|
-
app.delete("/schema/:name", async (c) => {
|
|
14123
|
-
const
|
|
14124
|
-
|
|
14477
|
+
app.delete("/schema/:name", zValidator("param", schemaFileParamSchema, invalidParamResponse("Invalid schema file name")), async (c) => {
|
|
14478
|
+
const { name: name$1 } = c.req.valid("param");
|
|
14479
|
+
const filePath = path.join(schemaDir, name$1);
|
|
14125
14480
|
try {
|
|
14126
14481
|
const aiexDir = path.dirname(schemaDir);
|
|
14127
14482
|
try {
|
|
14128
|
-
const
|
|
14129
|
-
const parsed = JsonSchemaDefinitionSchema.safeParse(JSON.parse(content));
|
|
14483
|
+
const parsed = JsonSchemaDefinitionSchema.safeParse(await readFile(filePath));
|
|
14130
14484
|
if (parsed.success) {
|
|
14131
14485
|
const tableName = parsed.data.table.name;
|
|
14132
14486
|
const snapshotPath = path.join(aiexDir, "extracted", `${tableName}.prompt.md`);
|
|
@@ -14142,59 +14496,21 @@ function schemaRoutes(config) {
|
|
|
14142
14496
|
app.post("/migrate", async (c) => {
|
|
14143
14497
|
try {
|
|
14144
14498
|
await ensureDir();
|
|
14145
|
-
await
|
|
14146
|
-
|
|
14147
|
-
|
|
14148
|
-
success: false,
|
|
14149
|
-
error: "No schema files found"
|
|
14150
|
-
}, 400);
|
|
14151
|
-
const parsedResult = parseAllSchemas(await Promise.all(jsonFiles.map(async (fileName) => {
|
|
14152
|
-
const filePath = path.join(schemaDir, fileName);
|
|
14153
|
-
return {
|
|
14154
|
-
filePath,
|
|
14155
|
-
content: await fs.readFile(filePath, "utf-8")
|
|
14156
|
-
};
|
|
14157
|
-
})));
|
|
14158
|
-
if (!parsedResult.success) return c.json({
|
|
14159
|
-
success: false,
|
|
14160
|
-
error: parsedResult.error
|
|
14161
|
-
}, 400);
|
|
14162
|
-
const { tables, relations, reverseRelations, warnings, drizzleCode } = parsedResult.data;
|
|
14163
|
-
await fs.writeFile(config.drizzleSchemaPath, drizzleCode);
|
|
14164
|
-
const helperPath = resolveHelperPath();
|
|
14165
|
-
const tsxPath = resolveTsxPath();
|
|
14166
|
-
const { stdout, stderr } = await execFileAsync(process.execPath, [
|
|
14167
|
-
tsxPath,
|
|
14168
|
-
helperPath,
|
|
14169
|
-
config.drizzleSchemaPath,
|
|
14170
|
-
config.migrationsPath,
|
|
14171
|
-
config.databasePath
|
|
14172
|
-
], { cwd: process.cwd() });
|
|
14173
|
-
let migrationResult;
|
|
14174
|
-
try {
|
|
14175
|
-
const jsonLine = stdout.trim().split("\n").find((l) => l.startsWith("{") && l.endsWith("}"));
|
|
14176
|
-
if (!jsonLine) return c.json({
|
|
14177
|
-
success: false,
|
|
14178
|
-
error: "Migration helper did not return valid output"
|
|
14179
|
-
}, 500);
|
|
14180
|
-
migrationResult = JSON.parse(jsonLine);
|
|
14181
|
-
} catch {
|
|
14499
|
+
const result = await runSchemaSync(config);
|
|
14500
|
+
if (!result.success) {
|
|
14501
|
+
const status = result.schemaCount === 0 ? 400 : 500;
|
|
14182
14502
|
return c.json({
|
|
14183
14503
|
success: false,
|
|
14184
|
-
error:
|
|
14185
|
-
},
|
|
14504
|
+
error: result.error || "Migration failed"
|
|
14505
|
+
}, status);
|
|
14186
14506
|
}
|
|
14187
|
-
if (!migrationResult.success) return c.json({
|
|
14188
|
-
success: false,
|
|
14189
|
-
error: migrationResult.error || "Migration failed"
|
|
14190
|
-
}, 500);
|
|
14191
14507
|
return c.json({
|
|
14192
14508
|
success: true,
|
|
14193
|
-
changes:
|
|
14194
|
-
tag:
|
|
14195
|
-
tables: tables
|
|
14196
|
-
relations: relations
|
|
14197
|
-
warnings
|
|
14509
|
+
changes: result.migration?.changes ?? 0,
|
|
14510
|
+
tag: result.migration?.tag,
|
|
14511
|
+
tables: result.tables,
|
|
14512
|
+
relations: result.relations,
|
|
14513
|
+
warnings: result.warnings
|
|
14198
14514
|
});
|
|
14199
14515
|
} catch (error) {
|
|
14200
14516
|
return c.json({
|
|
@@ -14242,9 +14558,36 @@ function createApp(config, staticDir) {
|
|
|
14242
14558
|
return app;
|
|
14243
14559
|
}
|
|
14244
14560
|
|
|
14561
|
+
//#endregion
|
|
14562
|
+
//#region src/core/web-runner.ts
|
|
14563
|
+
function resolveWebStaticDir() {
|
|
14564
|
+
return path.join(resolvePackageRoot(), "dist/web");
|
|
14565
|
+
}
|
|
14566
|
+
async function openBrowser(url) {
|
|
14567
|
+
await open(url);
|
|
14568
|
+
}
|
|
14569
|
+
async function startWebServer(input) {
|
|
14570
|
+
const { config, port } = input;
|
|
14571
|
+
const staticDir = input.staticDir ?? resolveWebStaticDir();
|
|
14572
|
+
const url = `http://localhost:${port}`;
|
|
14573
|
+
serve({
|
|
14574
|
+
fetch: createApp(config, staticDir).fetch,
|
|
14575
|
+
port
|
|
14576
|
+
}, () => {
|
|
14577
|
+
input.onStarted?.({
|
|
14578
|
+
url,
|
|
14579
|
+
schemaPath: config.schemaPath
|
|
14580
|
+
});
|
|
14581
|
+
if (input.open === false) return;
|
|
14582
|
+
openBrowser(url).catch(() => {
|
|
14583
|
+
input.onOpenFailed?.(url);
|
|
14584
|
+
});
|
|
14585
|
+
});
|
|
14586
|
+
await new Promise(() => {});
|
|
14587
|
+
}
|
|
14588
|
+
|
|
14245
14589
|
//#endregion
|
|
14246
14590
|
//#region src/commands/web.ts
|
|
14247
|
-
const execAsync = promisify(exec);
|
|
14248
14591
|
const webCommand = defineCommand({
|
|
14249
14592
|
meta: {
|
|
14250
14593
|
name: "web",
|
|
@@ -14261,23 +14604,20 @@ const webCommand = defineCommand({
|
|
|
14261
14604
|
const cwd = process.cwd();
|
|
14262
14605
|
const port = Number(args.port) || 13e3;
|
|
14263
14606
|
const config = createMigrationConfig(cwd);
|
|
14264
|
-
const packageRoot = resolvePackageRoot();
|
|
14265
|
-
const staticDir = path.join(packageRoot, "dist/web");
|
|
14266
14607
|
const s = spinner();
|
|
14267
14608
|
s.start("Starting web server...");
|
|
14268
|
-
|
|
14269
|
-
|
|
14270
|
-
port
|
|
14271
|
-
|
|
14272
|
-
|
|
14273
|
-
|
|
14274
|
-
|
|
14275
|
-
|
|
14276
|
-
|
|
14609
|
+
await startWebServer({
|
|
14610
|
+
config,
|
|
14611
|
+
port,
|
|
14612
|
+
onStarted(info) {
|
|
14613
|
+
s.stop(`Server running at ${pc.cyan(info.url)}`);
|
|
14614
|
+
consola.info(`Schema directory: ${pc.dim(info.schemaPath)}`);
|
|
14615
|
+
consola.info("Press Ctrl+C to stop");
|
|
14616
|
+
},
|
|
14617
|
+
onOpenFailed(url) {
|
|
14277
14618
|
consola.warn(`Could not open browser. Visit ${url} manually.`);
|
|
14278
|
-
}
|
|
14619
|
+
}
|
|
14279
14620
|
});
|
|
14280
|
-
await new Promise(() => {});
|
|
14281
14621
|
}
|
|
14282
14622
|
});
|
|
14283
14623
|
|
|
@@ -14305,7 +14645,7 @@ process.on("unhandledRejection", (reason) => {
|
|
|
14305
14645
|
process.exit(1);
|
|
14306
14646
|
});
|
|
14307
14647
|
if (process.argv[2] === "_complete") {
|
|
14308
|
-
const { getCompletions } = await import("./completions-
|
|
14648
|
+
const { getCompletions } = await import("./completions-C3rmTwXZ.mjs");
|
|
14309
14649
|
const suggestions = getCompletions(subCommands, process.argv.slice(3));
|
|
14310
14650
|
for (const s of suggestions) process.stdout.write(`${s}\n`);
|
|
14311
14651
|
process.exit(0);
|