aiex-cli 0.0.1-beta.9 → 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -25
- package/dist/cli.mjs +912 -567
- package/dist/{completions-ygS1okck.mjs → completions-C3rmTwXZ.mjs} +2 -2
- package/dist/core/schema-sqlite/migrate-helper.mjs +35 -37
- package/dist/{doctor-BiU1lDp-.mjs → doctor-collector-CykRm0fC.mjs} +282 -223
- package/dist/index.d.mts +15 -15
- package/dist/index.mjs +1 -1
- package/dist/table-schema.json +4 -0
- package/dist/web/assets/AISettings-CI6Lgx0p.js +339 -0
- package/dist/web/assets/DataBrowser-CwcTG80-.js +6 -0
- package/dist/web/assets/ExtractionViewer-CsdK1kKK.js +1 -0
- package/dist/web/assets/JsonSchemaEditor-D477lV5a.js +570 -0
- package/dist/web/assets/api-client-D2Y_-4JM.js +1 -0
- package/dist/web/assets/button-Cdgr9Igy.js +927 -0
- package/dist/web/assets/{cssMode-DAbG0CMn.js → cssMode-CPThwItX.js} +1 -1
- package/dist/web/assets/dialog-CUkPLPNP.js +109 -0
- package/dist/web/assets/dist-9yHVMqQ0.js +1 -0
- package/dist/web/assets/{editor.main-BqhfoHxy.js → editor.main-BnOkwRFv.js} +2 -2
- package/dist/web/assets/{freemarker2-B9_5ct2b.js → freemarker2-DWDTYVJR.js} +1 -1
- package/dist/web/assets/{handlebars-TY59WcoQ.js → handlebars-D4DzjGQ7.js} +1 -1
- package/dist/web/assets/{html-CLULsh27.js → html-DnzhKSoD.js} +1 -1
- package/dist/web/assets/{htmlMode-BvG7RNbU.js → htmlMode-CR7UKfEH.js} +1 -1
- package/dist/web/assets/index-C9N8oWt4.css +2 -0
- package/dist/web/assets/{index-CKUXTDYj.js → index-DVDVw-GK.js} +38 -38
- package/dist/web/assets/{javascript-DHrLp6gu.js → javascript-D2srszZ8.js} +1 -1
- package/dist/web/assets/{jsonMode-DBDhdzl1.js → jsonMode-B4jaPYEr.js} +1 -1
- package/dist/web/assets/{liquid-tGeb-nqF.js → liquid-CIT2Wl_l.js} +1 -1
- package/dist/web/assets/{mdx-Cmdz78VU.js → mdx-CWLaEOFy.js} +1 -1
- package/dist/web/assets/{monaco.contribution-CroYPUF5.js → monaco.contribution-DDv5ldfS.js} +2 -2
- package/dist/web/assets/object-utils-I4gWdSnS.js +1 -0
- package/dist/web/assets/{python-Dmfz4iDE.js → python-6CGfpCNq.js} +1 -1
- package/dist/web/assets/{razor-BJicZHJs.js → razor-DEMMh3TD.js} +1 -1
- package/dist/web/assets/runtime-dom.esm-bundler-ei_N7Xjw.js +1 -0
- package/dist/web/assets/select-BGex2SPs.js +439 -0
- package/dist/web/assets/{tsMode-DYqTyE66.js → tsMode-Cm1NtjPs.js} +1 -1
- package/dist/web/assets/{typescript-DLnTe9Hf.js → typescript-BM9aPEFg.js} +1 -1
- package/dist/web/assets/{xml-BIYqLORk.js → xml-CoSbvcg5.js} +1 -1
- package/dist/web/assets/{yaml-BjmulkMX.js → yaml-56GOgy8k.js} +1 -1
- package/dist/web/index.html +10 -8
- package/package.json +16 -1
- package/src/core/schema-sqlite/migrate-helper.ts +32 -46
- package/dist/web/assets/AISettings-DOzonIux.js +0 -334
- package/dist/web/assets/DataBrowser-rznfVRaV.js +0 -3
- package/dist/web/assets/JsonSchemaEditor-C9iyQs7N.js +0 -929
- package/dist/web/assets/api-client-Dsg4WOM9.js +0 -1
- package/dist/web/assets/button-kTMweGMc.js +0 -927
- package/dist/web/assets/dialog-CWuu7WjI.js +0 -108
- package/dist/web/assets/index-DDFnprdM.css +0 -2
- package/dist/web/assets/lib-C30cIFrm.js +0 -1
- package/dist/web/assets/overlayeventbus-AtOpmI6n.js +0 -80
- package/dist/web/assets/table-schema-mJrrf9qw.js +0 -2
- /package/dist/web/assets/{abap-DrZwwXZX.js → abap-Bgec7Keq.js} +0 -0
- /package/dist/web/assets/{apex-CrCz0btt.js → apex-VBlPwEoQ.js} +0 -0
- /package/dist/web/assets/{azcli-BapzKHay.js → azcli-DKqrEFBx.js} +0 -0
- /package/dist/web/assets/{bat-C_NRAiA1.js → bat-DdgQWy_0.js} +0 -0
- /package/dist/web/assets/{bicep-C7pp2CNk.js → bicep-CRMM43EB.js} +0 -0
- /package/dist/web/assets/{cameligo-BhhK9vxZ.js → cameligo-UatALtML.js} +0 -0
- /package/dist/web/assets/{clojure-D0ujmUyE.js → clojure-D8JU08RA.js} +0 -0
- /package/dist/web/assets/{coffee-DHEl7Jbb.js → coffee-C56wu358.js} +0 -0
- /package/dist/web/assets/{cpp-Iil-3nzZ.js → cpp-CyZLvhJG.js} +0 -0
- /package/dist/web/assets/{csharp-Dh0Ee7SY.js → csharp-BJl3ixva.js} +0 -0
- /package/dist/web/assets/{csp-mwzjw0JL.js → csp-CxEKxmO-.js} +0 -0
- /package/dist/web/assets/{css-COIa8ZTR.js → css-B0t_muXd.js} +0 -0
- /package/dist/web/assets/{cypher-GVc17FC4.js → cypher-D1hqiMFD.js} +0 -0
- /package/dist/web/assets/{dart-phiCaE7_.js → dart-Bz550Pyv.js} +0 -0
- /package/dist/web/assets/{dockerfile-BMaDhdim.js → dockerfile-CIXgVAuA.js} +0 -0
- /package/dist/web/assets/{ecl-Cj47kvqp.js → ecl-D9qbvZoA.js} +0 -0
- /package/dist/web/assets/{editor.api-DLXGyrN1.js → editor.api-C8BHpRhn.js} +0 -0
- /package/dist/web/assets/{elixir-DBbstcE1.js → elixir-b2M38fAy.js} +0 -0
- /package/dist/web/assets/{flow9-ChHb1adO.js → flow9-Dq1UYMkt.js} +0 -0
- /package/dist/web/assets/{fsharp-CMk2OIJN.js → fsharp-BaeLhgfq.js} +0 -0
- /package/dist/web/assets/{go-BrMkuJg0.js → go-Bd-NFKIC.js} +0 -0
- /package/dist/web/assets/{graphql-PSR1UKGv.js → graphql-DZVerJfy.js} +0 -0
- /package/dist/web/assets/{hcl-DAQrbDOW.js → hcl-CAVzrZfH.js} +0 -0
- /package/dist/web/assets/{ini-0TG5BxW0.js → ini-CyXdX58t.js} +0 -0
- /package/dist/web/assets/{java-rgorz17v.js → java-B5pNgvhy.js} +0 -0
- /package/dist/web/assets/{julia-C8VMdHm8.js → julia-XRhmV3AN.js} +0 -0
- /package/dist/web/assets/{kotlin-CllWo3gX.js → kotlin-DOd3J5vr.js} +0 -0
- /package/dist/web/assets/{less-Cgca25AP.js → less-veZSnyw6.js} +0 -0
- /package/dist/web/assets/{lexon-D0GHdBaw.js → lexon-QWGkuK0H.js} +0 -0
- /package/dist/web/assets/{lua-DmRsNG-P.js → lua-CYGpjuO5.js} +0 -0
- /package/dist/web/assets/{m3-BgL5dNKT.js → m3-yNnrZkdc.js} +0 -0
- /package/dist/web/assets/{markdown-BuJfycGS.js → markdown-BCSWEPSX.js} +0 -0
- /package/dist/web/assets/{mips-C9m_93PR.js → mips-OpYmcC30.js} +0 -0
- /package/dist/web/assets/{msdax-CpFHC9OI.js → msdax-2oxoTO9Z.js} +0 -0
- /package/dist/web/assets/{mysql-qFvltsqN.js → mysql-5KlC-K_9.js} +0 -0
- /package/dist/web/assets/{objective-c-Bnmr858J.js → objective-c-CcDCgtLx.js} +0 -0
- /package/dist/web/assets/{pascal-WP0_D5AO.js → pascal-BZGsbaEV.js} +0 -0
- /package/dist/web/assets/{pascaligo-Blom4Rij.js → pascaligo-DtD5qU3G.js} +0 -0
- /package/dist/web/assets/{perl-B-vk8g64.js → perl-C1jNNS3E.js} +0 -0
- /package/dist/web/assets/{pgsql-Cgvz6v67.js → pgsql-CT0fhiZa.js} +0 -0
- /package/dist/web/assets/{php-8a3Lrw9m.js → php-D6DrXoPM.js} +0 -0
- /package/dist/web/assets/{pla-DuFqEZ8V.js → pla-b3-HN2pF.js} +0 -0
- /package/dist/web/assets/{postiats-DkLtSgkp.js → postiats-Bin2ApVS.js} +0 -0
- /package/dist/web/assets/{powerquery-BJ1aNepW.js → powerquery-7ASnn-ZG.js} +0 -0
- /package/dist/web/assets/{powershell-rE98k687.js → powershell-t4p7sU1H.js} +0 -0
- /package/dist/web/assets/{preload-helper-DWTEM3RW.js → preload-helper-Dd-HcVz_.js} +0 -0
- /package/dist/web/assets/{protobuf-CUheFacr.js → protobuf-BUGeWa_j.js} +0 -0
- /package/dist/web/assets/{pug-LDcAMD8w.js → pug-BuKcgC9s.js} +0 -0
- /package/dist/web/assets/{qsharp-IHfqKOfK.js → qsharp-DxLLX8mo.js} +0 -0
- /package/dist/web/assets/{r-D-QApv87.js → r-DMlFgn7A.js} +0 -0
- /package/dist/web/assets/{redis-SXdDyWR9.js → redis-cXItkC5u.js} +0 -0
- /package/dist/web/assets/{redshift-Y6lsCryn.js → redshift-BZVbW7HE.js} +0 -0
- /package/dist/web/assets/{restructuredtext-edObr9a8.js → restructuredtext-BzjxwS8h.js} +0 -0
- /package/dist/web/assets/{ruby-CNnUfF-8.js → ruby-C5nyLV4l.js} +0 -0
- /package/dist/web/assets/{rust-IHUZWzBr.js → rust-BcmMsHdf.js} +0 -0
- /package/dist/web/assets/{sb-DrUvY44N.js → sb-Dnb1iy6B.js} +0 -0
- /package/dist/web/assets/{scala-B4hbXGLM.js → scala-anMIFYpA.js} +0 -0
- /package/dist/web/assets/{scheme-BGrd12j3.js → scheme-BItQTe08.js} +0 -0
- /package/dist/web/assets/{scss-x5G1ES4U.js → scss-BOv51BJ5.js} +0 -0
- /package/dist/web/assets/{shell-DOehe2Y8.js → shell-BsRYRTNN.js} +0 -0
- /package/dist/web/assets/{solidity-BeRvcwWV.js → solidity-BtuLgGDx.js} +0 -0
- /package/dist/web/assets/{sophia-DZbkUNjy.js → sophia-B0Vkc5MF.js} +0 -0
- /package/dist/web/assets/{sparql-B7_oi5-h.js → sparql-B7lvkZQM.js} +0 -0
- /package/dist/web/assets/{sql-CTlsFWVE.js → sql-DvP5MpA3.js} +0 -0
- /package/dist/web/assets/{st-DJVEJdPE.js → st-GVUeyB3U.js} +0 -0
- /package/dist/web/assets/{swift-CwhT3fYa.js → swift-DSPIoCjm.js} +0 -0
- /package/dist/web/assets/{systemverilog-BQN63pkN.js → systemverilog-Icj2-k23.js} +0 -0
- /package/dist/web/assets/{tcl-DqwfpskA.js → tcl-Cd8KQcm-.js} +0 -0
- /package/dist/web/assets/{twig-BiyenUgc.js → twig-CBHmt8z3.js} +0 -0
- /package/dist/web/assets/{typespec-CWOJribt.js → typespec-Ckc037mq.js} +0 -0
- /package/dist/web/assets/{vb-Cq5F87m3.js → vb-B97GW9Wb.js} +0 -0
- /package/dist/web/assets/{wgsl-BAvW2lVr.js → wgsl-DIKmb3YH.js} +0 -0
package/dist/cli.mjs
CHANGED
|
@@ -1,25 +1,40 @@
|
|
|
1
|
-
import { C as doctorDiagnosticsTableRows, a as
|
|
1
|
+
import { C as doctorDiagnosticsTableRows, _ as seedConfig, a as parseJsonSchema, b as package_default, c as getDefaultAIConfig, d as DEFAULT_MINERU_CONFIG, f as DEFAULT_PROMPT_CONFIG, g as createConfig, h as AIConfigSchema, i as JsonSchemaDefinitionSchema, l as readAIConfig, m as PLACEHOLDER_TEXT, n as createMigrationConfig, o as toSnakeCase, p as PLACEHOLDER_SCHEMA, s as generateDrizzleSchema, t as collectDoctorDiagnostics, u as writeAIConfig, v as description, w as formatDoctorDiagnosticsJson, x as version, y as name } from "./doctor-collector-CykRm0fC.mjs";
|
|
2
2
|
import { createRequire } from "node:module";
|
|
3
|
+
import fs from "node:fs/promises";
|
|
4
|
+
import os from "node:os";
|
|
3
5
|
import path from "node:path";
|
|
4
6
|
import process from "node:process";
|
|
7
|
+
import { readFile, writeFile } from "jsonfile";
|
|
8
|
+
import { ZodError, z } from "zod";
|
|
5
9
|
import { fileURLToPath } from "node:url";
|
|
6
|
-
import { ZodError } from "zod";
|
|
7
|
-
import fs from "node:fs/promises";
|
|
8
10
|
import { defineCommand, runMain } from "citty";
|
|
9
11
|
import { consola } from "consola";
|
|
10
12
|
import updateNotifier from "update-notifier";
|
|
11
13
|
import CliTable3 from "cli-table3";
|
|
12
|
-
import { intro, outro, spinner } from "@clack/prompts";
|
|
13
|
-
import Database from "better-sqlite3";
|
|
14
|
+
import { intro, isCancel, outro, select, spinner, text } from "@clack/prompts";
|
|
14
15
|
import pc from "picocolors";
|
|
15
16
|
import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
|
|
17
|
+
import { LangfuseSpanProcessor } from "@langfuse/otel";
|
|
18
|
+
import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
|
|
16
19
|
import { APICallError, Output, generateText, jsonSchema } from "ai";
|
|
17
|
-
import
|
|
20
|
+
import mime from "mime";
|
|
21
|
+
import pRetry from "p-retry";
|
|
22
|
+
import { jsonrepair } from "jsonrepair";
|
|
23
|
+
import fs$1 from "node:fs";
|
|
24
|
+
import Database from "better-sqlite3";
|
|
25
|
+
import { glob, globSync } from "tinyglobby";
|
|
26
|
+
import { execa } from "execa";
|
|
27
|
+
import { extractText, getDocumentProxy, getMeta } from "unpdf";
|
|
28
|
+
import { Buffer } from "node:buffer";
|
|
29
|
+
import { execFile } from "node:child_process";
|
|
18
30
|
import { promisify } from "node:util";
|
|
19
31
|
import { serve } from "@hono/node-server";
|
|
32
|
+
import open from "open";
|
|
20
33
|
import { serveStatic } from "@hono/node-server/serve-static";
|
|
21
34
|
import { Hono } from "hono";
|
|
22
35
|
import { cors } from "hono/cors";
|
|
36
|
+
import { zValidator } from "@hono/zod-validator";
|
|
37
|
+
import { Kysely, SqliteDialect, sql } from "kysely";
|
|
23
38
|
|
|
24
39
|
//#region src/core/schema-sqlite/helpers.ts
|
|
25
40
|
const __filename = fileURLToPath(import.meta.url);
|
|
@@ -96,7 +111,7 @@ function parseAllSchemas(entries) {
|
|
|
96
111
|
}
|
|
97
112
|
|
|
98
113
|
//#endregion
|
|
99
|
-
//#region src/
|
|
114
|
+
//#region src/core/completion-scripts.ts
|
|
100
115
|
function bashScript(name$1) {
|
|
101
116
|
return `# ${name$1} bash completion
|
|
102
117
|
_${name$1}() {
|
|
@@ -123,7 +138,7 @@ function fishScript(name$1) {
|
|
|
123
138
|
complete -c ${name$1} -f -a '(${name$1} _complete (commandline -cp) 2>/dev/null)'
|
|
124
139
|
`;
|
|
125
140
|
}
|
|
126
|
-
function
|
|
141
|
+
function generateCompletionScript(name$1, shell) {
|
|
127
142
|
switch (shell) {
|
|
128
143
|
case "bash": return bashScript(name$1);
|
|
129
144
|
case "zsh": return zshScript(name$1);
|
|
@@ -131,6 +146,9 @@ function generateScript(name$1, shell) {
|
|
|
131
146
|
default: throw new Error(`Unsupported shell: ${shell}. Use bash, zsh, or fish.`);
|
|
132
147
|
}
|
|
133
148
|
}
|
|
149
|
+
|
|
150
|
+
//#endregion
|
|
151
|
+
//#region src/commands/completion.ts
|
|
134
152
|
const completionCommand = defineCommand({
|
|
135
153
|
meta: {
|
|
136
154
|
name: "completion",
|
|
@@ -145,7 +163,7 @@ const completionCommand = defineCommand({
|
|
|
145
163
|
const name$1 = "aiex";
|
|
146
164
|
const shell = args.shell;
|
|
147
165
|
try {
|
|
148
|
-
process.stdout.write(
|
|
166
|
+
process.stdout.write(generateCompletionScript(name$1, shell));
|
|
149
167
|
} catch (error) {
|
|
150
168
|
process.stderr.write(`Error: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
151
169
|
process.exit(1);
|
|
@@ -185,6 +203,14 @@ const doctorCommand = defineCommand({
|
|
|
185
203
|
}
|
|
186
204
|
});
|
|
187
205
|
|
|
206
|
+
//#endregion
|
|
207
|
+
//#region src/commands/utils.ts
|
|
208
|
+
function failCommand(message) {
|
|
209
|
+
if (message) consola.error(message);
|
|
210
|
+
outro("Failed!");
|
|
211
|
+
process.exitCode = 1;
|
|
212
|
+
}
|
|
213
|
+
|
|
188
214
|
//#endregion
|
|
189
215
|
//#region src/core/ai-extraction/model-capabilities.json
|
|
190
216
|
var model_capabilities_default = {
|
|
@@ -12727,26 +12753,26 @@ var model_capabilities_default = {
|
|
|
12727
12753
|
|
|
12728
12754
|
//#endregion
|
|
12729
12755
|
//#region src/core/ai-extraction/model-registry.ts
|
|
12730
|
-
const registry = model_capabilities_default;
|
|
12756
|
+
const registry$1 = model_capabilities_default;
|
|
12731
12757
|
function normalize(name$1) {
|
|
12732
12758
|
return name$1.toLowerCase().replace(/[-_. ]/g, "");
|
|
12733
12759
|
}
|
|
12734
12760
|
const normalizedCache = /* @__PURE__ */ new Map();
|
|
12735
12761
|
function buildNormalizedCache() {
|
|
12736
12762
|
if (normalizedCache.size > 0) return;
|
|
12737
|
-
for (const key of Object.keys(registry)) {
|
|
12763
|
+
for (const key of Object.keys(registry$1)) {
|
|
12738
12764
|
const nk = normalize(key);
|
|
12739
12765
|
if (!normalizedCache.has(nk)) normalizedCache.set(nk, key);
|
|
12740
12766
|
}
|
|
12741
12767
|
}
|
|
12742
12768
|
function lookupModel(name$1) {
|
|
12743
|
-
const exact = registry[name$1];
|
|
12769
|
+
const exact = registry$1[name$1];
|
|
12744
12770
|
if (exact) return { ...exact };
|
|
12745
12771
|
buildNormalizedCache();
|
|
12746
12772
|
const nk = normalize(name$1);
|
|
12747
12773
|
const matched = normalizedCache.get(nk);
|
|
12748
12774
|
if (matched) {
|
|
12749
|
-
const entry = registry[matched];
|
|
12775
|
+
const entry = registry$1[matched];
|
|
12750
12776
|
return entry ? { ...entry } : null;
|
|
12751
12777
|
}
|
|
12752
12778
|
return null;
|
|
@@ -12763,29 +12789,40 @@ function lookupModelCapabilities(modelName) {
|
|
|
12763
12789
|
//#endregion
|
|
12764
12790
|
//#region src/utils/retry.ts
|
|
12765
12791
|
async function withRetry(fn, onRetry, maxRetries = 5) {
|
|
12766
|
-
|
|
12767
|
-
|
|
12768
|
-
|
|
12769
|
-
|
|
12770
|
-
|
|
12771
|
-
|
|
12772
|
-
|
|
12773
|
-
|
|
12774
|
-
|
|
12775
|
-
|
|
12776
|
-
|
|
12777
|
-
|
|
12778
|
-
|
|
12779
|
-
|
|
12780
|
-
|
|
12781
|
-
|
|
12782
|
-
|
|
12792
|
+
return pRetry(async () => fn(), {
|
|
12793
|
+
retries: maxRetries,
|
|
12794
|
+
factor: 2,
|
|
12795
|
+
minTimeout: 1e3,
|
|
12796
|
+
randomize: true,
|
|
12797
|
+
onFailedAttempt({ error, attemptNumber, retriesLeft }) {
|
|
12798
|
+
if (!(error instanceof APICallError) || !error.isRetryable || retriesLeft <= 0) return;
|
|
12799
|
+
const baseDelayMs = 1e3 * 2 ** (attemptNumber - 1);
|
|
12800
|
+
onRetry?.({
|
|
12801
|
+
attempt: attemptNumber,
|
|
12802
|
+
maxRetries,
|
|
12803
|
+
delayMs: baseDelayMs,
|
|
12804
|
+
statusCode: error.statusCode
|
|
12805
|
+
});
|
|
12806
|
+
},
|
|
12807
|
+
shouldRetry({ error }) {
|
|
12808
|
+
return error instanceof APICallError && error.isRetryable;
|
|
12809
|
+
}
|
|
12810
|
+
});
|
|
12783
12811
|
}
|
|
12784
12812
|
|
|
12785
12813
|
//#endregion
|
|
12786
12814
|
//#region src/core/ai-extraction/json-utils.ts
|
|
12787
|
-
function
|
|
12788
|
-
const trimmed = text.trim();
|
|
12815
|
+
function parseJsonLike(text$1) {
|
|
12816
|
+
const trimmed = text$1.trim();
|
|
12817
|
+
try {
|
|
12818
|
+
return JSON.parse(trimmed);
|
|
12819
|
+
} catch {
|
|
12820
|
+
if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) throw new SyntaxError("JSON candidate must start with an object or array");
|
|
12821
|
+
return JSON.parse(jsonrepair(trimmed));
|
|
12822
|
+
}
|
|
12823
|
+
}
|
|
12824
|
+
function stripFences(text$1) {
|
|
12825
|
+
const trimmed = text$1.trim();
|
|
12789
12826
|
if (!trimmed.startsWith("```")) return null;
|
|
12790
12827
|
const endIndex = trimmed.lastIndexOf("```");
|
|
12791
12828
|
if (endIndex <= 3) return null;
|
|
@@ -12794,8 +12831,8 @@ function stripFences(text) {
|
|
|
12794
12831
|
if (firstNewline === -1) return null;
|
|
12795
12832
|
return inside.slice(firstNewline + 1).trim();
|
|
12796
12833
|
}
|
|
12797
|
-
function extractFirstJSON(text) {
|
|
12798
|
-
const trimmed = text.trim();
|
|
12834
|
+
function extractFirstJSON(text$1) {
|
|
12835
|
+
const trimmed = text$1.trim();
|
|
12799
12836
|
const firstBrace = trimmed.indexOf("{");
|
|
12800
12837
|
const firstBracket = trimmed.indexOf("[");
|
|
12801
12838
|
let start = -1;
|
|
@@ -12806,20 +12843,20 @@ function extractFirstJSON(text) {
|
|
|
12806
12843
|
if (end <= start) return null;
|
|
12807
12844
|
return trimmed.slice(start, end);
|
|
12808
12845
|
}
|
|
12809
|
-
function safeParseJSON(text) {
|
|
12810
|
-
const cleaned = text.trim();
|
|
12846
|
+
function safeParseJSON(text$1) {
|
|
12847
|
+
const cleaned = text$1.trim();
|
|
12811
12848
|
try {
|
|
12812
12849
|
return JSON.parse(cleaned);
|
|
12813
12850
|
} catch {}
|
|
12814
12851
|
const fromFence = stripFences(cleaned);
|
|
12815
12852
|
if (fromFence) try {
|
|
12816
|
-
return
|
|
12853
|
+
return parseJsonLike(fromFence);
|
|
12817
12854
|
} catch {}
|
|
12818
12855
|
const extracted = extractFirstJSON(cleaned);
|
|
12819
12856
|
if (extracted) try {
|
|
12820
|
-
return
|
|
12857
|
+
return parseJsonLike(extracted);
|
|
12821
12858
|
} catch {}
|
|
12822
|
-
const truncated = text.length > 200 ? `${text.slice(0, 200)}...` : text;
|
|
12859
|
+
const truncated = text$1.length > 200 ? `${text$1.slice(0, 200)}...` : text$1;
|
|
12823
12860
|
throw new Error(`Failed to parse JSON from model output. Expected a valid JSON object or array but received unparseable text. Raw output: ${truncated}`);
|
|
12824
12861
|
}
|
|
12825
12862
|
|
|
@@ -12907,11 +12944,11 @@ function schemaToDescription(schema) {
|
|
|
12907
12944
|
}
|
|
12908
12945
|
return lines.join("\n");
|
|
12909
12946
|
}
|
|
12910
|
-
function generateExtractionPrompt(schema, text, promptConfig = DEFAULT_PROMPT_CONFIG) {
|
|
12947
|
+
function generateExtractionPrompt(schema, text$1, promptConfig = DEFAULT_PROMPT_CONFIG) {
|
|
12911
12948
|
const schemaDescription = schemaToDescription(schema);
|
|
12912
12949
|
return {
|
|
12913
12950
|
system: promptConfig.systemTemplate.replaceAll(PLACEHOLDER_SCHEMA, schemaDescription),
|
|
12914
|
-
user: promptConfig.userTemplate.replaceAll(PLACEHOLDER_TEXT, text)
|
|
12951
|
+
user: promptConfig.userTemplate.replaceAll(PLACEHOLDER_TEXT, text$1)
|
|
12915
12952
|
};
|
|
12916
12953
|
}
|
|
12917
12954
|
function generatePromptSnapshot(schema, promptConfig = DEFAULT_PROMPT_CONFIG) {
|
|
@@ -12934,39 +12971,40 @@ function generatePromptSnapshot(schema, promptConfig = DEFAULT_PROMPT_CONFIG) {
|
|
|
12934
12971
|
|
|
12935
12972
|
//#endregion
|
|
12936
12973
|
//#region src/core/ai-extraction/extractor.ts
|
|
12974
|
+
let langfuseInitialized = false;
|
|
12975
|
+
function initLangfuse(config) {
|
|
12976
|
+
if (!config.langfuse?.publicKey || !config.langfuse.secretKey) return;
|
|
12977
|
+
if (langfuseInitialized) return;
|
|
12978
|
+
langfuseInitialized = true;
|
|
12979
|
+
try {
|
|
12980
|
+
new NodeTracerProvider({ spanProcessors: [new LangfuseSpanProcessor({
|
|
12981
|
+
publicKey: config.langfuse.publicKey,
|
|
12982
|
+
secretKey: config.langfuse.secretKey,
|
|
12983
|
+
baseUrl: config.langfuse.host || "https://us.cloud.langfuse.com",
|
|
12984
|
+
exportMode: "immediate"
|
|
12985
|
+
})] }).register();
|
|
12986
|
+
} catch (e) {
|
|
12987
|
+
console.warn("[Langfuse] Failed to initialize tracing:", e instanceof Error ? e.message : e);
|
|
12988
|
+
}
|
|
12989
|
+
}
|
|
12937
12990
|
const SYSTEM_PROMPT_REGEX = /## System Prompt\n([\s\S]*?)(?=## User Prompt|$)/;
|
|
12938
12991
|
const USER_PROMPT_REGEX = /## User Prompt Template\n([\s\S]*)$/;
|
|
12939
|
-
const MIME_TYPES = {
|
|
12940
|
-
png: "image/png",
|
|
12941
|
-
jpg: "image/jpeg",
|
|
12942
|
-
jpeg: "image/jpeg",
|
|
12943
|
-
gif: "image/gif",
|
|
12944
|
-
webp: "image/webp",
|
|
12945
|
-
bmp: "image/bmp",
|
|
12946
|
-
svg: "image/svg+xml",
|
|
12947
|
-
pdf: "application/pdf",
|
|
12948
|
-
txt: "text/plain",
|
|
12949
|
-
csv: "text/csv",
|
|
12950
|
-
json: "application/json",
|
|
12951
|
-
md: "text/markdown",
|
|
12952
|
-
html: "text/html"
|
|
12953
|
-
};
|
|
12954
12992
|
function detectMimeType(filePath) {
|
|
12955
|
-
return
|
|
12993
|
+
return mime.getType(filePath) ?? "application/octet-stream";
|
|
12956
12994
|
}
|
|
12957
12995
|
async function readFilePart(filePath) {
|
|
12958
|
-
const mime = detectMimeType(filePath);
|
|
12996
|
+
const mime$1 = detectMimeType(filePath);
|
|
12959
12997
|
const buffer = await fs.readFile(filePath);
|
|
12960
12998
|
const name$1 = path.basename(filePath);
|
|
12961
|
-
if (mime.startsWith("image/")) return {
|
|
12999
|
+
if (mime$1.startsWith("image/")) return {
|
|
12962
13000
|
type: "image",
|
|
12963
13001
|
image: buffer,
|
|
12964
|
-
mimeType: mime
|
|
13002
|
+
mimeType: mime$1
|
|
12965
13003
|
};
|
|
12966
13004
|
return {
|
|
12967
13005
|
type: "file",
|
|
12968
13006
|
data: buffer,
|
|
12969
|
-
mediaType: mime,
|
|
13007
|
+
mediaType: mime$1,
|
|
12970
13008
|
filename: name$1
|
|
12971
13009
|
};
|
|
12972
13010
|
}
|
|
@@ -13078,25 +13116,35 @@ async function loadPromptSnapshot(aiexDir, tableName) {
|
|
|
13078
13116
|
return null;
|
|
13079
13117
|
}
|
|
13080
13118
|
async function extractStructuredData(input) {
|
|
13081
|
-
const { config, schema, text, aiexDir, file, modelOverride } = input;
|
|
13119
|
+
const { config, schema, text: text$1, aiexDir, file, modelOverride } = input;
|
|
13082
13120
|
if (!config.provider.apiKey) return {
|
|
13083
13121
|
success: false,
|
|
13084
13122
|
error: "API Key not configured. Please configure AI settings in the web UI."
|
|
13085
13123
|
};
|
|
13086
13124
|
const useFileContent = !!file;
|
|
13087
13125
|
const isImageFile = useFileContent && detectMimeType(file).startsWith("image/");
|
|
13088
|
-
const inputTokens = text ? Math.ceil(text.length / 2) : void 0;
|
|
13126
|
+
const inputTokens = text$1 ? Math.ceil(text$1.length / 2) : void 0;
|
|
13089
13127
|
const fieldCount = schema.properties ? Object.keys(schema.properties).length : 0;
|
|
13090
13128
|
const outputTokens = fieldCount > 0 ? fieldCount * 80 : void 0;
|
|
13091
|
-
|
|
13092
|
-
|
|
13093
|
-
|
|
13094
|
-
|
|
13095
|
-
|
|
13096
|
-
|
|
13097
|
-
|
|
13129
|
+
let selected;
|
|
13130
|
+
try {
|
|
13131
|
+
selected = modelOverride ?? selectModel({
|
|
13132
|
+
models: config.provider.models,
|
|
13133
|
+
isImage: isImageFile,
|
|
13134
|
+
fileName: file,
|
|
13135
|
+
inputTokens,
|
|
13136
|
+
outputTokens
|
|
13137
|
+
});
|
|
13138
|
+
} catch (e) {
|
|
13139
|
+
return {
|
|
13140
|
+
success: false,
|
|
13141
|
+
error: e.message
|
|
13142
|
+
};
|
|
13143
|
+
}
|
|
13098
13144
|
const useStructuredOutput = selected.capabilities.structuredOutput;
|
|
13145
|
+
const useTelemetry = !!(config.langfuse?.publicKey && config.langfuse.secretKey);
|
|
13099
13146
|
try {
|
|
13147
|
+
if (useTelemetry) initLangfuse(config);
|
|
13100
13148
|
const provider = createOpenAICompatible({
|
|
13101
13149
|
baseURL: config.provider.baseURL,
|
|
13102
13150
|
name: "qwen",
|
|
@@ -13105,7 +13153,7 @@ async function extractStructuredData(input) {
|
|
|
13105
13153
|
let system;
|
|
13106
13154
|
let user;
|
|
13107
13155
|
const snapshot = await loadPromptSnapshot(aiexDir, schema.table.name);
|
|
13108
|
-
const promptText = file ? PLACEHOLDER_TEXT : text;
|
|
13156
|
+
const promptText = file ? PLACEHOLDER_TEXT : text$1;
|
|
13109
13157
|
if (snapshot) {
|
|
13110
13158
|
system = snapshot.system;
|
|
13111
13159
|
user = snapshot.user.replaceAll(PLACEHOLDER_TEXT, promptText);
|
|
@@ -13116,12 +13164,13 @@ async function extractStructuredData(input) {
|
|
|
13116
13164
|
}
|
|
13117
13165
|
const outputSchema = jsonSchema(schemaToExtractionOutputSchema(schema));
|
|
13118
13166
|
let result;
|
|
13167
|
+
const timeoutMs = (config.provider.timeout ?? 300) * 1e3;
|
|
13119
13168
|
if (useFileContent) {
|
|
13120
13169
|
const filePart = await readFilePart(file);
|
|
13121
13170
|
const fileName = filePart.type === "file" ? filePart.filename : path.basename(file);
|
|
13122
13171
|
const contentParts = [{
|
|
13123
13172
|
type: "text",
|
|
13124
|
-
text: user.includes(PLACEHOLDER_TEXT) ? user.replaceAll(PLACEHOLDER_TEXT, text || `Data is contained in the attached file: ${fileName}`) : user
|
|
13173
|
+
text: user.includes(PLACEHOLDER_TEXT) ? user.replaceAll(PLACEHOLDER_TEXT, text$1 || `Data is contained in the attached file: ${fileName}`) : user
|
|
13125
13174
|
}, filePart];
|
|
13126
13175
|
const fileOpts = {
|
|
13127
13176
|
model: provider.chatModel(selected.name),
|
|
@@ -13130,8 +13179,9 @@ async function extractStructuredData(input) {
|
|
|
13130
13179
|
role: "user",
|
|
13131
13180
|
content: contentParts
|
|
13132
13181
|
}],
|
|
13133
|
-
abortSignal: AbortSignal.timeout(
|
|
13134
|
-
maxRetries: 0
|
|
13182
|
+
abortSignal: AbortSignal.timeout(timeoutMs),
|
|
13183
|
+
maxRetries: 0,
|
|
13184
|
+
experimental_telemetry: { isEnabled: useTelemetry }
|
|
13135
13185
|
};
|
|
13136
13186
|
if (useStructuredOutput) fileOpts.output = Output.object({ schema: outputSchema });
|
|
13137
13187
|
result = await withRetry(() => generateText(fileOpts), input.onRetry);
|
|
@@ -13140,8 +13190,9 @@ async function extractStructuredData(input) {
|
|
|
13140
13190
|
model: provider.chatModel(selected.name),
|
|
13141
13191
|
system,
|
|
13142
13192
|
prompt: user,
|
|
13143
|
-
abortSignal: AbortSignal.timeout(
|
|
13144
|
-
maxRetries: 0
|
|
13193
|
+
abortSignal: AbortSignal.timeout(timeoutMs),
|
|
13194
|
+
maxRetries: 0,
|
|
13195
|
+
experimental_telemetry: { isEnabled: useTelemetry }
|
|
13145
13196
|
};
|
|
13146
13197
|
if (useStructuredOutput) textOpts.output = Output.object({ schema: outputSchema });
|
|
13147
13198
|
result = await withRetry(() => generateText(textOpts), input.onRetry);
|
|
@@ -13159,7 +13210,10 @@ async function extractStructuredData(input) {
|
|
|
13159
13210
|
const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
13160
13211
|
const outputFileName = `${schema.table.name}-${timestamp}.json`;
|
|
13161
13212
|
const outputPath = path.join(outputDir, outputFileName);
|
|
13162
|
-
await
|
|
13213
|
+
await writeFile(outputPath, data, {
|
|
13214
|
+
spaces: 2,
|
|
13215
|
+
EOL: "\n"
|
|
13216
|
+
});
|
|
13163
13217
|
return {
|
|
13164
13218
|
success: true,
|
|
13165
13219
|
outputPath,
|
|
@@ -13224,8 +13278,8 @@ function buildInsertSql(table, data) {
|
|
|
13224
13278
|
function insertTableRow({ db, table, data, parentRowId, foreignKeyColumn }) {
|
|
13225
13279
|
const rowData = { ...data };
|
|
13226
13280
|
if (parentRowId !== void 0 && foreignKeyColumn) rowData[foreignKeyColumn] = parentRowId;
|
|
13227
|
-
const { sql, values } = buildInsertSql(table, rowData);
|
|
13228
|
-
const info = db.prepare(sql).run(...values);
|
|
13281
|
+
const { sql: sql$1, values } = buildInsertSql(table, rowData);
|
|
13282
|
+
const info = db.prepare(sql$1).run(...values);
|
|
13229
13283
|
return Number(info.lastInsertRowid);
|
|
13230
13284
|
}
|
|
13231
13285
|
function parseDataByColumns(data, schema, table) {
|
|
@@ -13328,7 +13382,171 @@ async function savePromptSnapshot(schema, aiexDir) {
|
|
|
13328
13382
|
}
|
|
13329
13383
|
|
|
13330
13384
|
//#endregion
|
|
13331
|
-
//#region src/
|
|
13385
|
+
//#region src/core/pdf-converter/external.ts
|
|
13386
|
+
function applyTemplate(value, context) {
|
|
13387
|
+
return value.replaceAll("{input}", context.input).replaceAll("{outputDir}", context.outputDir).replaceAll("{basename}", context.basename);
|
|
13388
|
+
}
|
|
13389
|
+
function isError(error) {
|
|
13390
|
+
return error instanceof Error;
|
|
13391
|
+
}
|
|
13392
|
+
async function pathExists(filePath) {
|
|
13393
|
+
try {
|
|
13394
|
+
await fs.access(filePath);
|
|
13395
|
+
return true;
|
|
13396
|
+
} catch {
|
|
13397
|
+
return false;
|
|
13398
|
+
}
|
|
13399
|
+
}
|
|
13400
|
+
async function collectMarkdownFiles(dir) {
|
|
13401
|
+
return (await glob("**/*.md", {
|
|
13402
|
+
cwd: dir,
|
|
13403
|
+
absolute: true,
|
|
13404
|
+
onlyFiles: true
|
|
13405
|
+
})).sort();
|
|
13406
|
+
}
|
|
13407
|
+
async function selectMarkdownFile(outputDir, basename) {
|
|
13408
|
+
const files = await collectMarkdownFiles(outputDir);
|
|
13409
|
+
if (files.length === 0) throw new Error(`External PDF converter did not produce a markdown file in ${outputDir}`);
|
|
13410
|
+
const preferredName = `${basename}.md`.toLowerCase();
|
|
13411
|
+
return files.find((file) => path.basename(file).toLowerCase() === preferredName) ?? files[0];
|
|
13412
|
+
}
|
|
13413
|
+
function formatCommandError(error, command$1) {
|
|
13414
|
+
if (!isError(error)) return new Error(String(error));
|
|
13415
|
+
const details = [`External PDF converter failed: ${command$1}`];
|
|
13416
|
+
if ("exitCode" in error && typeof error.exitCode === "number") details.push(`exitCode=${error.exitCode}`);
|
|
13417
|
+
if ("signal" in error && error.signal) details.push(`signal=${String(error.signal)}`);
|
|
13418
|
+
if ("stderr" in error && typeof error.stderr === "string" && error.stderr.trim()) details.push(error.stderr.trim());
|
|
13419
|
+
else if (error.message) details.push(error.message);
|
|
13420
|
+
return new Error(details.join("\n"));
|
|
13421
|
+
}
|
|
13422
|
+
async function countPdfPages(input) {
|
|
13423
|
+
try {
|
|
13424
|
+
return (await getDocumentProxy(input)).numPages;
|
|
13425
|
+
} catch {
|
|
13426
|
+
return 0;
|
|
13427
|
+
}
|
|
13428
|
+
}
|
|
13429
|
+
var ExternalCommandPdfConverter = class {
|
|
13430
|
+
name;
|
|
13431
|
+
constructor(name$1, config) {
|
|
13432
|
+
this.config = config;
|
|
13433
|
+
this.name = name$1;
|
|
13434
|
+
}
|
|
13435
|
+
async convert(input, filePath) {
|
|
13436
|
+
const tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "aiex-mineru-"));
|
|
13437
|
+
const outputDir = path.join(tempRoot, "output");
|
|
13438
|
+
await fs.mkdir(outputDir, { recursive: true });
|
|
13439
|
+
const inputPath = filePath ?? path.join(tempRoot, "input.pdf");
|
|
13440
|
+
if (!filePath) await fs.writeFile(inputPath, input);
|
|
13441
|
+
const pageCount = await countPdfPages(input);
|
|
13442
|
+
const basename = path.basename(inputPath, path.extname(inputPath));
|
|
13443
|
+
const context = {
|
|
13444
|
+
input: inputPath,
|
|
13445
|
+
outputDir,
|
|
13446
|
+
basename
|
|
13447
|
+
};
|
|
13448
|
+
const args = this.config.args.map((arg) => applyTemplate(arg, context));
|
|
13449
|
+
const timeoutMs = (this.config.timeout ?? 600) * 1e3;
|
|
13450
|
+
try {
|
|
13451
|
+
await execa(this.config.command, args, {
|
|
13452
|
+
shell: false,
|
|
13453
|
+
timeout: timeoutMs,
|
|
13454
|
+
maxBuffer: 1024 * 1024 * 20
|
|
13455
|
+
});
|
|
13456
|
+
const outputPath = this.config.outputFile ? applyTemplate(this.config.outputFile, context) : await selectMarkdownFile(outputDir, basename);
|
|
13457
|
+
if (!await pathExists(outputPath)) throw new Error(`External PDF converter output was not found: ${outputPath}`);
|
|
13458
|
+
return {
|
|
13459
|
+
text: await fs.readFile(outputPath, "utf-8"),
|
|
13460
|
+
pageCount,
|
|
13461
|
+
metadata: {
|
|
13462
|
+
converter: this.name,
|
|
13463
|
+
outputPath,
|
|
13464
|
+
...this.config.keepOutput ? { outputDir } : {}
|
|
13465
|
+
}
|
|
13466
|
+
};
|
|
13467
|
+
} catch (error) {
|
|
13468
|
+
throw formatCommandError(error, `${this.config.command} ${args.join(" ")}`);
|
|
13469
|
+
} finally {
|
|
13470
|
+
if (!this.config.keepOutput) await fs.rm(tempRoot, {
|
|
13471
|
+
recursive: true,
|
|
13472
|
+
force: true
|
|
13473
|
+
}).catch(() => {});
|
|
13474
|
+
}
|
|
13475
|
+
}
|
|
13476
|
+
};
|
|
13477
|
+
|
|
13478
|
+
//#endregion
|
|
13479
|
+
//#region src/core/pdf-converter/unpdf.ts
|
|
13480
|
+
var UnpdfConverter = class {
|
|
13481
|
+
name = "unpdf";
|
|
13482
|
+
async convert(input) {
|
|
13483
|
+
const data = Buffer.isBuffer(input) ? new Uint8Array(input) : input;
|
|
13484
|
+
const [textResult, meta] = await Promise.all([extractText(data, { mergePages: true }), getMeta(data).catch(() => null)]);
|
|
13485
|
+
return {
|
|
13486
|
+
text: textResult.text,
|
|
13487
|
+
pageCount: textResult.totalPages,
|
|
13488
|
+
metadata: {
|
|
13489
|
+
converter: this.name,
|
|
13490
|
+
...meta?.info ? Object.fromEntries(Object.entries(meta.info).map(([k, v]) => [k, String(v)])) : {}
|
|
13491
|
+
}
|
|
13492
|
+
};
|
|
13493
|
+
}
|
|
13494
|
+
};
|
|
13495
|
+
|
|
13496
|
+
//#endregion
|
|
13497
|
+
//#region src/core/pdf-converter/factory.ts
|
|
13498
|
+
const registry = /* @__PURE__ */ new Map();
|
|
13499
|
+
var FallbackPdfConverter = class {
|
|
13500
|
+
name;
|
|
13501
|
+
constructor(primary, fallback) {
|
|
13502
|
+
this.primary = primary;
|
|
13503
|
+
this.fallback = fallback;
|
|
13504
|
+
this.name = primary.name;
|
|
13505
|
+
}
|
|
13506
|
+
async convert(input, filePath) {
|
|
13507
|
+
try {
|
|
13508
|
+
return await this.primary.convert(input, filePath);
|
|
13509
|
+
} catch (err) {
|
|
13510
|
+
consola.warn(`${this.primary.name} failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
13511
|
+
consola.info(`Falling back to ${this.fallback.name}`);
|
|
13512
|
+
const result = await this.fallback.convert(input, filePath);
|
|
13513
|
+
return {
|
|
13514
|
+
...result,
|
|
13515
|
+
metadata: {
|
|
13516
|
+
...result.metadata,
|
|
13517
|
+
fallback: "true"
|
|
13518
|
+
}
|
|
13519
|
+
};
|
|
13520
|
+
}
|
|
13521
|
+
}
|
|
13522
|
+
};
|
|
13523
|
+
function withFallback(converter, config) {
|
|
13524
|
+
if (!config.fallbackToUnpdf) return converter;
|
|
13525
|
+
return new FallbackPdfConverter(converter, new UnpdfConverter());
|
|
13526
|
+
}
|
|
13527
|
+
function createPdfConverter(config) {
|
|
13528
|
+
if (typeof config === "object") {
|
|
13529
|
+
if (config.converter === "mineru") {
|
|
13530
|
+
const mineruConfig = config.mineru ?? DEFAULT_MINERU_CONFIG;
|
|
13531
|
+
return withFallback(new ExternalCommandPdfConverter("mineru", mineruConfig), mineruConfig);
|
|
13532
|
+
}
|
|
13533
|
+
if (config.converter === "external") {
|
|
13534
|
+
if (!config.external) throw new Error("External PDF converter is selected but no external command is configured.");
|
|
13535
|
+
return withFallback(new ExternalCommandPdfConverter("external", config.external), config.external);
|
|
13536
|
+
}
|
|
13537
|
+
}
|
|
13538
|
+
const key = typeof config === "string" ? config : "unpdf";
|
|
13539
|
+
let instance = registry.get(key);
|
|
13540
|
+
if (!instance) {
|
|
13541
|
+
if (key !== "unpdf") throw new Error(`PDF converter "${key}" requires configuration.`);
|
|
13542
|
+
instance = new UnpdfConverter();
|
|
13543
|
+
registry.set(key, instance);
|
|
13544
|
+
}
|
|
13545
|
+
return instance;
|
|
13546
|
+
}
|
|
13547
|
+
|
|
13548
|
+
//#endregion
|
|
13549
|
+
//#region src/core/extract-runner.ts
|
|
13332
13550
|
const FILE_PART_EXTENSIONS = new Set([
|
|
13333
13551
|
"png",
|
|
13334
13552
|
"jpg",
|
|
@@ -13338,11 +13556,21 @@ const FILE_PART_EXTENSIONS = new Set([
|
|
|
13338
13556
|
"bmp",
|
|
13339
13557
|
"svg"
|
|
13340
13558
|
]);
|
|
13341
|
-
|
|
13342
|
-
|
|
13343
|
-
|
|
13344
|
-
|
|
13345
|
-
|
|
13559
|
+
const SUPPORTED_EXTENSIONS = new Set([
|
|
13560
|
+
...FILE_PART_EXTENSIONS,
|
|
13561
|
+
"pdf",
|
|
13562
|
+
"txt",
|
|
13563
|
+
"md",
|
|
13564
|
+
"csv",
|
|
13565
|
+
"json",
|
|
13566
|
+
"html",
|
|
13567
|
+
"xml",
|
|
13568
|
+
"yaml",
|
|
13569
|
+
"yml"
|
|
13570
|
+
]);
|
|
13571
|
+
const PDF_EXT_RE = /\.pdf$/i;
|
|
13572
|
+
const JSON_EXT_RE = /\.json$/;
|
|
13573
|
+
const SUPPORTED_FILE_PATTERN = `*.{${[...SUPPORTED_EXTENSIONS].join(",")}}`;
|
|
13346
13574
|
async function ensureDatabaseReady(dbPath, schema) {
|
|
13347
13575
|
try {
|
|
13348
13576
|
await fs.access(dbPath);
|
|
@@ -13362,17 +13590,213 @@ async function ensureDatabaseReady(dbPath, schema) {
|
|
|
13362
13590
|
}
|
|
13363
13591
|
return null;
|
|
13364
13592
|
}
|
|
13593
|
+
function listSupportedFiles(dir, pattern) {
|
|
13594
|
+
if (!fs$1.statSync(dir).isDirectory()) throw new Error(`Not a directory: ${dir}`);
|
|
13595
|
+
return globSync(pattern ?? SUPPORTED_FILE_PATTERN, {
|
|
13596
|
+
cwd: dir,
|
|
13597
|
+
absolute: true,
|
|
13598
|
+
onlyFiles: true
|
|
13599
|
+
}).filter((file) => {
|
|
13600
|
+
const ext = path.extname(file).toLowerCase().replace(".", "");
|
|
13601
|
+
return SUPPORTED_EXTENSIONS.has(ext);
|
|
13602
|
+
}).sort();
|
|
13603
|
+
}
|
|
13604
|
+
async function loadSchema(config, schemaName) {
|
|
13605
|
+
const schemaPath = path.join(config.schemaPath, `${schemaName}.json`);
|
|
13606
|
+
try {
|
|
13607
|
+
const parsed = await readFile(schemaPath);
|
|
13608
|
+
return { schema: JsonSchemaDefinitionSchema.parse(parsed) };
|
|
13609
|
+
} catch (e) {
|
|
13610
|
+
if (e instanceof ZodError) return {
|
|
13611
|
+
schema: null,
|
|
13612
|
+
error: `Schema validation failed: ${schemaName}.json\n${e.issues.map((i) => ` - ${i.path.join(".")}: ${i.message}`).join("\n")}`
|
|
13613
|
+
};
|
|
13614
|
+
if (e.code === "ENOENT") return {
|
|
13615
|
+
schema: null,
|
|
13616
|
+
error: `Cannot read schema file: ${schemaName}.json`
|
|
13617
|
+
};
|
|
13618
|
+
if (e instanceof SyntaxError) return {
|
|
13619
|
+
schema: null,
|
|
13620
|
+
error: `Invalid JSON in schema file: ${schemaName}.json`
|
|
13621
|
+
};
|
|
13622
|
+
return {
|
|
13623
|
+
schema: null,
|
|
13624
|
+
error: String(e)
|
|
13625
|
+
};
|
|
13626
|
+
}
|
|
13627
|
+
}
|
|
13628
|
+
async function listSchemas(aiexDir) {
|
|
13629
|
+
try {
|
|
13630
|
+
const dir = path.join(aiexDir, "schema");
|
|
13631
|
+
return (await fs.readdir(dir)).filter((f) => f.endsWith(".json")).map((f) => f.replace(JSON_EXT_RE, "")).sort();
|
|
13632
|
+
} catch {
|
|
13633
|
+
return [];
|
|
13634
|
+
}
|
|
13635
|
+
}
|
|
13636
|
+
async function readExtractFileInput(filePath, aiConfig) {
|
|
13637
|
+
const ext = path.extname(filePath).toLowerCase().replace(".", "");
|
|
13638
|
+
if (FILE_PART_EXTENSIONS.has(ext)) return {
|
|
13639
|
+
text: "",
|
|
13640
|
+
filePath
|
|
13641
|
+
};
|
|
13642
|
+
if (ext === "pdf") {
|
|
13643
|
+
const buffer = await fs.readFile(filePath);
|
|
13644
|
+
const converter = createPdfConverter(aiConfig?.pdf);
|
|
13645
|
+
const result = await converter.convert(buffer, filePath);
|
|
13646
|
+
if (result.metadata?.fallback === "true") consola.info(`Fell back to unpdf — ${result.pageCount} page(s) extracted`);
|
|
13647
|
+
else consola.info(`Converted PDF via ${converter.name}, ${result.pageCount} page(s)`);
|
|
13648
|
+
const mdPath = filePath.replace(PDF_EXT_RE, ".md");
|
|
13649
|
+
try {
|
|
13650
|
+
await fs.writeFile(mdPath, result.text);
|
|
13651
|
+
consola.info(`Markdown saved: ${mdPath}`);
|
|
13652
|
+
} catch {
|
|
13653
|
+
const fallbackMd = path.join(os.tmpdir(), `${path.basename(filePath, ".pdf")}.md`);
|
|
13654
|
+
await fs.writeFile(fallbackMd, result.text);
|
|
13655
|
+
consola.info(`Markdown saved: ${fallbackMd}`);
|
|
13656
|
+
}
|
|
13657
|
+
return { text: result.text };
|
|
13658
|
+
}
|
|
13659
|
+
return { text: await fs.readFile(filePath, "utf-8") };
|
|
13660
|
+
}
|
|
13661
|
+
async function extractSingle(aiexDir, config, aiConfig, schemaName, text$1, filePath, modelOverride, options) {
|
|
13662
|
+
const schemaLoad = await loadSchema(config, schemaName);
|
|
13663
|
+
if (!schemaLoad.schema) {
|
|
13664
|
+
if (!options?.quiet) consola.error(schemaLoad.error);
|
|
13665
|
+
return {
|
|
13666
|
+
success: false,
|
|
13667
|
+
error: schemaLoad.error
|
|
13668
|
+
};
|
|
13669
|
+
}
|
|
13670
|
+
const s = spinner();
|
|
13671
|
+
if (!options?.quiet) s.start(filePath ? `Extracting from ${path.basename(filePath)}...` : "Extracting data...");
|
|
13672
|
+
const result = await extractStructuredData({
|
|
13673
|
+
config: aiConfig,
|
|
13674
|
+
schema: schemaLoad.schema,
|
|
13675
|
+
text: text$1 ?? "",
|
|
13676
|
+
aiexDir,
|
|
13677
|
+
file: filePath,
|
|
13678
|
+
modelOverride,
|
|
13679
|
+
onRetry(info) {
|
|
13680
|
+
if (!options?.quiet) s.message(`API responded with ${info.statusCode}, retrying in ${info.delayMs / 1e3}s (${info.attempt}/${info.maxRetries})...`);
|
|
13681
|
+
}
|
|
13682
|
+
});
|
|
13683
|
+
if (!result.success) {
|
|
13684
|
+
if (!options?.quiet) {
|
|
13685
|
+
s.stop("Extraction failed");
|
|
13686
|
+
consola.error(result.error || "Unknown error");
|
|
13687
|
+
}
|
|
13688
|
+
return {
|
|
13689
|
+
success: false,
|
|
13690
|
+
error: result.error || "Unknown error"
|
|
13691
|
+
};
|
|
13692
|
+
}
|
|
13693
|
+
if (!options?.quiet) s.stop("Extraction complete");
|
|
13694
|
+
if (result.outputPath && !options?.quiet) consola.success(`Result saved: ${pc.cyan(result.outputPath)}`);
|
|
13695
|
+
if (result.tokensUsed && !options?.quiet) consola.info(pc.gray(`Token usage: prompt=${result.tokensUsed.prompt}, completion=${result.tokensUsed.completion}, total=${result.tokensUsed.total}`));
|
|
13696
|
+
if (result.data) {
|
|
13697
|
+
const s2 = spinner();
|
|
13698
|
+
if (!options?.quiet) s2.start("Inserting into database...");
|
|
13699
|
+
const dbError = await ensureDatabaseReady(config.databasePath, schemaLoad.schema);
|
|
13700
|
+
if (dbError) {
|
|
13701
|
+
if (!options?.quiet) s2.stop("Database not ready");
|
|
13702
|
+
consola.error(dbError);
|
|
13703
|
+
return {
|
|
13704
|
+
success: false,
|
|
13705
|
+
error: dbError
|
|
13706
|
+
};
|
|
13707
|
+
}
|
|
13708
|
+
try {
|
|
13709
|
+
const db = new Database(config.databasePath);
|
|
13710
|
+
try {
|
|
13711
|
+
const insertResult = insertExtractedData(db, schemaLoad.schema, result.data);
|
|
13712
|
+
if (insertResult.success) {
|
|
13713
|
+
if (!options?.quiet) s2.stop(`Inserted into ${insertResult.tablesInserted.length} table(s)`);
|
|
13714
|
+
} else {
|
|
13715
|
+
if (!options?.quiet) s2.stop("Database insert failed");
|
|
13716
|
+
consola.error(insertResult.error || "Unknown error");
|
|
13717
|
+
return {
|
|
13718
|
+
success: false,
|
|
13719
|
+
error: insertResult.error
|
|
13720
|
+
};
|
|
13721
|
+
}
|
|
13722
|
+
} finally {
|
|
13723
|
+
db.close();
|
|
13724
|
+
}
|
|
13725
|
+
} catch (e) {
|
|
13726
|
+
if (!options?.quiet) s2.stop("Database insert failed");
|
|
13727
|
+
consola.error(e instanceof Error ? e.message : String(e));
|
|
13728
|
+
return {
|
|
13729
|
+
success: false,
|
|
13730
|
+
error: String(e)
|
|
13731
|
+
};
|
|
13732
|
+
}
|
|
13733
|
+
}
|
|
13734
|
+
return { success: true };
|
|
13735
|
+
}
|
|
13736
|
+
async function processOneFile(aiexDir, config, aiConfig, schemaName, filePath, modelOverride) {
|
|
13737
|
+
try {
|
|
13738
|
+
const input = await readExtractFileInput(filePath, aiConfig);
|
|
13739
|
+
const r = await extractSingle(aiexDir, config, aiConfig, schemaName, input.text, input.filePath, modelOverride, { quiet: false });
|
|
13740
|
+
if (r.success) {
|
|
13741
|
+
consola.success(`Processed: ${path.basename(filePath)}`);
|
|
13742
|
+
return true;
|
|
13743
|
+
} else {
|
|
13744
|
+
consola.error(`Failed: ${r.error}`);
|
|
13745
|
+
return false;
|
|
13746
|
+
}
|
|
13747
|
+
} catch (e) {
|
|
13748
|
+
consola.error(`Error processing ${path.basename(filePath)}: ${e instanceof Error ? e.message : String(e)}`);
|
|
13749
|
+
return false;
|
|
13750
|
+
}
|
|
13751
|
+
}
|
|
13752
|
+
async function runBatchExtraction(aiexDir, config, aiConfig, schemaName, dir, globPattern, modelOverride) {
|
|
13753
|
+
consola.info(`Scanning ${pc.cyan(dir)} for supported files...`);
|
|
13754
|
+
let files;
|
|
13755
|
+
try {
|
|
13756
|
+
files = listSupportedFiles(dir, globPattern);
|
|
13757
|
+
} catch {
|
|
13758
|
+
return {
|
|
13759
|
+
ok: false,
|
|
13760
|
+
successCount: 0,
|
|
13761
|
+
failCount: 0,
|
|
13762
|
+
error: `Cannot read directory: ${dir}`
|
|
13763
|
+
};
|
|
13764
|
+
}
|
|
13765
|
+
if (files.length === 0) return {
|
|
13766
|
+
ok: false,
|
|
13767
|
+
successCount: 0,
|
|
13768
|
+
failCount: 0,
|
|
13769
|
+
error: `No supported files found in ${dir}`
|
|
13770
|
+
};
|
|
13771
|
+
consola.info(`Found ${files.length} file(s) to process`);
|
|
13772
|
+
let successCount = 0;
|
|
13773
|
+
let failCount = 0;
|
|
13774
|
+
for (let i = 0; i < files.length; i++) {
|
|
13775
|
+
const file = files[i];
|
|
13776
|
+
consola.info(`\n[${i + 1}/${files.length}] Processing: ${pc.cyan(path.basename(file))}`);
|
|
13777
|
+
if (await processOneFile(aiexDir, config, aiConfig, schemaName, file, modelOverride)) successCount++;
|
|
13778
|
+
else failCount++;
|
|
13779
|
+
}
|
|
13780
|
+
consola.info(`\nBatch complete: ${pc.green(`${successCount} succeeded`)}, ${pc.red(`${failCount} failed`)}, ${files.length} total`);
|
|
13781
|
+
return {
|
|
13782
|
+
ok: true,
|
|
13783
|
+
successCount,
|
|
13784
|
+
failCount
|
|
13785
|
+
};
|
|
13786
|
+
}
|
|
13787
|
+
|
|
13788
|
+
//#endregion
|
|
13789
|
+
//#region src/commands/extract.ts
|
|
13365
13790
|
const extractCommand = defineCommand({
|
|
13366
13791
|
meta: {
|
|
13367
13792
|
name: "extract",
|
|
13368
|
-
description: "Extract structured data from text or
|
|
13793
|
+
description: "Extract structured data from text, images, or PDFs"
|
|
13369
13794
|
},
|
|
13370
13795
|
args: {
|
|
13371
13796
|
schema: {
|
|
13372
13797
|
type: "string",
|
|
13373
13798
|
alias: "s",
|
|
13374
|
-
description: "Schema name (without .json extension)"
|
|
13375
|
-
required: true
|
|
13799
|
+
description: "Schema name (without .json extension)"
|
|
13376
13800
|
},
|
|
13377
13801
|
text: {
|
|
13378
13802
|
type: "string",
|
|
@@ -13382,43 +13806,47 @@ const extractCommand = defineCommand({
|
|
|
13382
13806
|
file: {
|
|
13383
13807
|
type: "string",
|
|
13384
13808
|
alias: "f",
|
|
13385
|
-
description: "File path (image) to extract from"
|
|
13809
|
+
description: "File path (image/PDF) to extract from"
|
|
13386
13810
|
},
|
|
13387
13811
|
model: {
|
|
13388
13812
|
type: "string",
|
|
13389
13813
|
alias: "m",
|
|
13390
13814
|
description: "AI model to use for extraction (overrides auto-selection)"
|
|
13391
13815
|
},
|
|
13392
|
-
|
|
13393
|
-
type: "
|
|
13816
|
+
dir: {
|
|
13817
|
+
type: "string",
|
|
13394
13818
|
alias: "d",
|
|
13395
|
-
description: "
|
|
13396
|
-
|
|
13819
|
+
description: "Directory containing files to batch extract"
|
|
13820
|
+
},
|
|
13821
|
+
glob: {
|
|
13822
|
+
type: "string",
|
|
13823
|
+
alias: "g",
|
|
13824
|
+
description: "Glob pattern to filter files in batch mode (e.g. \"*.pdf\")"
|
|
13397
13825
|
}
|
|
13398
13826
|
},
|
|
13399
13827
|
async run({ args }) {
|
|
13400
13828
|
intro(pc.inverse(" aiex extract "));
|
|
13401
13829
|
const config = createMigrationConfig(process.cwd());
|
|
13402
13830
|
const aiexDir = path.dirname(config.schemaPath);
|
|
13403
|
-
if (
|
|
13404
|
-
|
|
13831
|
+
if (args.dir && args.text) {
|
|
13832
|
+
failCommand("Cannot combine -t/--text with -d/--dir");
|
|
13405
13833
|
return;
|
|
13406
13834
|
}
|
|
13407
|
-
if (args.
|
|
13408
|
-
|
|
13835
|
+
if (args.dir && args.file) {
|
|
13836
|
+
failCommand("Cannot combine -f/--file with -d/--dir");
|
|
13409
13837
|
return;
|
|
13410
13838
|
}
|
|
13411
13839
|
const aiConfig = await readAIConfig(aiexDir);
|
|
13412
13840
|
if (!aiConfig) {
|
|
13413
|
-
|
|
13841
|
+
failCommand("AI configuration not found. Please run \"aiex web\" to configure AI settings first");
|
|
13414
13842
|
return;
|
|
13415
13843
|
}
|
|
13416
13844
|
if (!aiConfig.provider.apiKey) {
|
|
13417
|
-
|
|
13845
|
+
failCommand("API Key not configured. Please configure AI settings in the Web interface first");
|
|
13418
13846
|
return;
|
|
13419
13847
|
}
|
|
13420
13848
|
if (!aiConfig.provider.models?.length) {
|
|
13421
|
-
|
|
13849
|
+
failCommand("No models configured. Please add at least one model in AI Settings");
|
|
13422
13850
|
return;
|
|
13423
13851
|
}
|
|
13424
13852
|
let modelOverride;
|
|
@@ -13426,130 +13854,212 @@ const extractCommand = defineCommand({
|
|
|
13426
13854
|
const matched = aiConfig.provider.models.find((m) => m.name === args.model);
|
|
13427
13855
|
if (!matched) {
|
|
13428
13856
|
const available = aiConfig.provider.models.map((m) => m.name).join(", ");
|
|
13429
|
-
|
|
13857
|
+
failCommand(`Model "${args.model}" not found in configuration. Available models: ${available}`);
|
|
13430
13858
|
return;
|
|
13431
13859
|
}
|
|
13432
13860
|
modelOverride = matched;
|
|
13433
13861
|
}
|
|
13434
|
-
|
|
13435
|
-
|
|
13436
|
-
if (args.file) {
|
|
13437
|
-
const ext = path.extname(args.file).toLowerCase().replace(".", "");
|
|
13438
|
-
if (FILE_PART_EXTENSIONS.has(ext)) filePath = args.file;
|
|
13439
|
-
else text = await fs.readFile(args.file, "utf-8");
|
|
13440
|
-
} else if (args.text) text = args.text;
|
|
13441
|
-
const schemaName = args.schema;
|
|
13442
|
-
const schemaPath = path.join(config.schemaPath, `${schemaName}.json`);
|
|
13443
|
-
let schema;
|
|
13444
|
-
try {
|
|
13445
|
-
const content = await fs.readFile(schemaPath, "utf-8");
|
|
13446
|
-
schema = JSON.parse(content);
|
|
13447
|
-
} catch {
|
|
13448
|
-
fail$1(`Cannot read schema file: ${schemaName}.json`);
|
|
13862
|
+
if (!args.schema && !args.text && !args.file && !args.dir) {
|
|
13863
|
+
if (await runInteractive(aiexDir, config, aiConfig, modelOverride)) outro("Done!");
|
|
13449
13864
|
return;
|
|
13450
13865
|
}
|
|
13451
|
-
|
|
13452
|
-
|
|
13453
|
-
|
|
13454
|
-
|
|
13455
|
-
|
|
13456
|
-
|
|
13866
|
+
if (args.dir) {
|
|
13867
|
+
if (!args.schema) {
|
|
13868
|
+
failCommand("Schema name (-s) is required in batch mode");
|
|
13869
|
+
return;
|
|
13870
|
+
}
|
|
13871
|
+
const result = await runBatchExtraction(aiexDir, config, aiConfig, args.schema, args.dir, args.glob, modelOverride);
|
|
13872
|
+
if (!result.ok) {
|
|
13873
|
+
failCommand(result.error);
|
|
13874
|
+
return;
|
|
13457
13875
|
}
|
|
13458
|
-
|
|
13876
|
+
if (result.failCount > 0) process.exitCode = 1;
|
|
13877
|
+
if (result.failCount > 0) outro(`Completed with failures (${result.failCount} failed)`);
|
|
13878
|
+
else outro("Done!");
|
|
13459
13879
|
return;
|
|
13460
13880
|
}
|
|
13461
|
-
|
|
13462
|
-
|
|
13463
|
-
|
|
13464
|
-
|
|
13465
|
-
|
|
13466
|
-
text
|
|
13467
|
-
|
|
13468
|
-
|
|
13469
|
-
|
|
13470
|
-
|
|
13471
|
-
|
|
13881
|
+
if (!args.schema) {
|
|
13882
|
+
failCommand("Please provide a schema name (-s) to extract from");
|
|
13883
|
+
return;
|
|
13884
|
+
}
|
|
13885
|
+
if (!args.text && !args.file) {
|
|
13886
|
+
failCommand("Please provide text (-t) or a file (-f) to extract from");
|
|
13887
|
+
return;
|
|
13888
|
+
}
|
|
13889
|
+
if (args.text && args.file) {
|
|
13890
|
+
failCommand("-t and -f cannot be used together");
|
|
13891
|
+
return;
|
|
13892
|
+
}
|
|
13893
|
+
let text$1 = "";
|
|
13894
|
+
let filePath;
|
|
13895
|
+
if (args.file) try {
|
|
13896
|
+
const input = await readExtractFileInput(args.file, aiConfig);
|
|
13897
|
+
text$1 = input.text;
|
|
13898
|
+
filePath = input.filePath;
|
|
13899
|
+
} catch (e) {
|
|
13900
|
+
failCommand(`Cannot read file: ${args.file} — ${e instanceof Error ? e.message : String(e)}`);
|
|
13901
|
+
return;
|
|
13902
|
+
}
|
|
13903
|
+
else if (args.text) text$1 = args.text;
|
|
13904
|
+
if (!(await extractSingle(aiexDir, config, aiConfig, args.schema, text$1, filePath, modelOverride)).success) {
|
|
13905
|
+
failCommand();
|
|
13906
|
+
return;
|
|
13907
|
+
}
|
|
13908
|
+
outro("Done!");
|
|
13909
|
+
}
|
|
13910
|
+
});
|
|
13911
|
+
async function runInteractive(aiexDir, config, aiConfig, modelOverride) {
|
|
13912
|
+
const schemas = await listSchemas(aiexDir);
|
|
13913
|
+
if (schemas.length === 0) {
|
|
13914
|
+
failCommand(`No schema files found in ${pc.cyan(".aiex/schema/")}. Run ${pc.cyan("aiex web")} to create and configure schemas first.`);
|
|
13915
|
+
return false;
|
|
13916
|
+
}
|
|
13917
|
+
const schemaName = await select({
|
|
13918
|
+
message: "Select a schema to extract data for:",
|
|
13919
|
+
options: schemas.map((s) => ({
|
|
13920
|
+
label: s,
|
|
13921
|
+
value: s
|
|
13922
|
+
}))
|
|
13923
|
+
});
|
|
13924
|
+
if (isCancel(schemaName)) {
|
|
13925
|
+
cancel("Cancelled");
|
|
13926
|
+
return false;
|
|
13927
|
+
}
|
|
13928
|
+
const inputSource = await select({
|
|
13929
|
+
message: "Choose input source:",
|
|
13930
|
+
options: [
|
|
13931
|
+
{
|
|
13932
|
+
label: "Text content",
|
|
13933
|
+
value: "text",
|
|
13934
|
+
hint: "Paste or type text directly"
|
|
13935
|
+
},
|
|
13936
|
+
{
|
|
13937
|
+
label: "Single file",
|
|
13938
|
+
value: "file",
|
|
13939
|
+
hint: "Extract from a file (txt, pdf, image)"
|
|
13940
|
+
},
|
|
13941
|
+
{
|
|
13942
|
+
label: "Batch directory",
|
|
13943
|
+
value: "dir",
|
|
13944
|
+
hint: "Extract all supported files in a directory"
|
|
13945
|
+
}
|
|
13946
|
+
]
|
|
13947
|
+
});
|
|
13948
|
+
if (isCancel(inputSource)) {
|
|
13949
|
+
cancel("Cancelled");
|
|
13950
|
+
return false;
|
|
13951
|
+
}
|
|
13952
|
+
if (inputSource === "text") {
|
|
13953
|
+
const textContent = await text({
|
|
13954
|
+
message: "Enter text content to extract:",
|
|
13955
|
+
validate(value) {
|
|
13956
|
+
if (!value || value.trim().length === 0) return "Please enter some text";
|
|
13472
13957
|
}
|
|
13473
13958
|
});
|
|
13474
|
-
if (
|
|
13475
|
-
|
|
13476
|
-
|
|
13477
|
-
return;
|
|
13959
|
+
if (isCancel(textContent)) {
|
|
13960
|
+
cancel("Cancelled");
|
|
13961
|
+
return false;
|
|
13478
13962
|
}
|
|
13479
|
-
|
|
13480
|
-
|
|
13481
|
-
|
|
13482
|
-
|
|
13483
|
-
|
|
13484
|
-
|
|
13485
|
-
const dbError = await ensureDatabaseReady(config.databasePath, schema);
|
|
13486
|
-
if (dbError) {
|
|
13487
|
-
s2.stop("Database not ready");
|
|
13488
|
-
fail$1(dbError);
|
|
13489
|
-
return;
|
|
13963
|
+
return (await extractSingle(aiexDir, config, aiConfig, schemaName, textContent, void 0, modelOverride)).success;
|
|
13964
|
+
} else if (inputSource === "file") {
|
|
13965
|
+
const filePathStr = await text({
|
|
13966
|
+
message: "Enter file path:",
|
|
13967
|
+
validate(value) {
|
|
13968
|
+
if (!value || value.trim().length === 0) return "Please enter a file path";
|
|
13490
13969
|
}
|
|
13491
|
-
|
|
13492
|
-
|
|
13493
|
-
|
|
13494
|
-
|
|
13495
|
-
|
|
13496
|
-
|
|
13497
|
-
|
|
13498
|
-
|
|
13499
|
-
|
|
13500
|
-
|
|
13501
|
-
|
|
13502
|
-
|
|
13503
|
-
|
|
13504
|
-
|
|
13505
|
-
|
|
13506
|
-
|
|
13507
|
-
|
|
13970
|
+
});
|
|
13971
|
+
if (isCancel(filePathStr)) {
|
|
13972
|
+
cancel("Cancelled");
|
|
13973
|
+
return false;
|
|
13974
|
+
}
|
|
13975
|
+
const fp = filePathStr;
|
|
13976
|
+
try {
|
|
13977
|
+
const input = await readExtractFileInput(fp, aiConfig);
|
|
13978
|
+
return (await extractSingle(aiexDir, config, aiConfig, schemaName, input.text, input.filePath, modelOverride)).success;
|
|
13979
|
+
} catch (e) {
|
|
13980
|
+
consola.error(`Cannot read file: ${fp} — ${e instanceof Error ? e.message : String(e)}`);
|
|
13981
|
+
return false;
|
|
13982
|
+
}
|
|
13983
|
+
} else if (inputSource === "dir") {
|
|
13984
|
+
const dirPath = await text({
|
|
13985
|
+
message: "Enter directory path:",
|
|
13986
|
+
validate(value) {
|
|
13987
|
+
if (!value || value.trim().length === 0) return "Please enter a directory path";
|
|
13508
13988
|
}
|
|
13989
|
+
});
|
|
13990
|
+
if (isCancel(dirPath)) {
|
|
13991
|
+
cancel("Cancelled");
|
|
13992
|
+
return false;
|
|
13509
13993
|
}
|
|
13510
|
-
|
|
13994
|
+
const result = await runBatchExtraction(aiexDir, config, aiConfig, schemaName, dirPath, void 0, modelOverride);
|
|
13995
|
+
if (!result.ok) failCommand(result.error);
|
|
13996
|
+
return result.ok && result.failCount === 0;
|
|
13511
13997
|
}
|
|
13512
|
-
|
|
13513
|
-
|
|
13514
|
-
|
|
13515
|
-
|
|
13516
|
-
|
|
13998
|
+
return false;
|
|
13999
|
+
}
|
|
14000
|
+
function cancel(msg) {
|
|
14001
|
+
consola.info(msg);
|
|
14002
|
+
outro("Cancelled");
|
|
14003
|
+
process.exitCode = 0;
|
|
14004
|
+
}
|
|
13517
14005
|
|
|
13518
14006
|
//#endregion
|
|
13519
|
-
//#region src/
|
|
13520
|
-
const execFileAsync
|
|
13521
|
-
function
|
|
13522
|
-
if (message) consola.error(message);
|
|
13523
|
-
outro("Failed!");
|
|
13524
|
-
process.exitCode = 1;
|
|
13525
|
-
}
|
|
13526
|
-
async function writeJsonIfAbsent(filePath, data) {
|
|
14007
|
+
//#region src/core/schema-runner.ts
|
|
14008
|
+
const execFileAsync = promisify(execFile);
|
|
14009
|
+
async function listSchemaFiles(schemaDir) {
|
|
13527
14010
|
try {
|
|
13528
|
-
await fs.
|
|
13529
|
-
|
|
13530
|
-
|
|
13531
|
-
if (error.code === "EEXIST") return "skipped";
|
|
13532
|
-
throw error;
|
|
14011
|
+
return (await fs.readdir(schemaDir)).filter((f) => f.endsWith(".json")).map((f) => path.join(schemaDir, f)).sort();
|
|
14012
|
+
} catch {
|
|
14013
|
+
return [];
|
|
13533
14014
|
}
|
|
13534
14015
|
}
|
|
13535
|
-
async function
|
|
14016
|
+
async function generateSchemaFromFiles(schemaFiles, config) {
|
|
13536
14017
|
const result = parseAllSchemas(await Promise.all(schemaFiles.map(async (filePath) => {
|
|
13537
14018
|
return {
|
|
13538
14019
|
filePath,
|
|
13539
14020
|
content: await fs.readFile(filePath, "utf-8")
|
|
13540
14021
|
};
|
|
13541
14022
|
})));
|
|
13542
|
-
if (!result.success) {
|
|
13543
|
-
|
|
13544
|
-
|
|
13545
|
-
|
|
13546
|
-
|
|
14023
|
+
if (!result.success) return {
|
|
14024
|
+
success: false,
|
|
14025
|
+
error: result.error,
|
|
14026
|
+
warnings: [],
|
|
14027
|
+
schemaCount: schemaFiles.length,
|
|
14028
|
+
tables: 0,
|
|
14029
|
+
relations: 0
|
|
14030
|
+
};
|
|
14031
|
+
const { tables, relations, reverseRelations, warnings, drizzleCode } = result.data;
|
|
13547
14032
|
await fs.mkdir(path.dirname(config.drizzleSchemaPath), { recursive: true });
|
|
13548
|
-
await fs.writeFile(config.drizzleSchemaPath,
|
|
13549
|
-
|
|
13550
|
-
|
|
14033
|
+
await fs.writeFile(config.drizzleSchemaPath, drizzleCode);
|
|
14034
|
+
return {
|
|
14035
|
+
success: true,
|
|
14036
|
+
warnings,
|
|
14037
|
+
schemaCount: schemaFiles.length,
|
|
14038
|
+
tables: tables.length,
|
|
14039
|
+
relations: relations.length + reverseRelations.length
|
|
14040
|
+
};
|
|
13551
14041
|
}
|
|
13552
|
-
|
|
14042
|
+
function parseMigrationOutput(stdout, stderr) {
|
|
14043
|
+
try {
|
|
14044
|
+
const jsonLine = stdout.trim().split("\n").find((l) => l.startsWith("{") && l.endsWith("}"));
|
|
14045
|
+
if (!jsonLine) return {
|
|
14046
|
+
success: false,
|
|
14047
|
+
error: "Migration helper did not return valid output"
|
|
14048
|
+
};
|
|
14049
|
+
const result = JSON.parse(jsonLine);
|
|
14050
|
+
if (!result.success) return {
|
|
14051
|
+
success: false,
|
|
14052
|
+
error: result.error || "Migration failed"
|
|
14053
|
+
};
|
|
14054
|
+
return result;
|
|
14055
|
+
} catch {
|
|
14056
|
+
return {
|
|
14057
|
+
success: false,
|
|
14058
|
+
error: stderr || stdout || "Migration helper failed"
|
|
14059
|
+
};
|
|
14060
|
+
}
|
|
14061
|
+
}
|
|
14062
|
+
async function runSchemaMigration(config, migrationName) {
|
|
13553
14063
|
const helperPath = resolveHelperPath();
|
|
13554
14064
|
const helperArgs = [
|
|
13555
14065
|
resolveTsxPath(),
|
|
@@ -13560,40 +14070,62 @@ async function migrate(config, migrationName) {
|
|
|
13560
14070
|
];
|
|
13561
14071
|
if (migrationName) helperArgs.push(migrationName);
|
|
13562
14072
|
try {
|
|
13563
|
-
const { stdout } = await execFileAsync
|
|
13564
|
-
|
|
13565
|
-
if (!result.success) {
|
|
13566
|
-
consola.error("Failed to generate migration");
|
|
13567
|
-
consola.error(result.error);
|
|
13568
|
-
return false;
|
|
13569
|
-
}
|
|
13570
|
-
if (result.changes === 0) {
|
|
13571
|
-
consola.info(pc.gray("No changes detected"));
|
|
13572
|
-
return true;
|
|
13573
|
-
}
|
|
13574
|
-
consola.success(pc.green("Migration files generated"));
|
|
13575
|
-
consola.success(pc.green("Database migrated"));
|
|
13576
|
-
return true;
|
|
14073
|
+
const { stdout, stderr } = await execFileAsync(process.execPath, helperArgs, { cwd: process.cwd() });
|
|
14074
|
+
return parseMigrationOutput(stdout, stderr);
|
|
13577
14075
|
} catch (error) {
|
|
13578
|
-
consola.error("Failed to generate migration");
|
|
13579
14076
|
const execError = error;
|
|
13580
|
-
|
|
13581
|
-
|
|
13582
|
-
|
|
14077
|
+
return {
|
|
14078
|
+
success: false,
|
|
14079
|
+
error: execError.stderr || execError.stdout || execError.message || String(error)
|
|
14080
|
+
};
|
|
13583
14081
|
}
|
|
13584
14082
|
}
|
|
14083
|
+
async function runSchemaSync(config, options = {}) {
|
|
14084
|
+
const schemaFiles = await listSchemaFiles(config.schemaPath);
|
|
14085
|
+
if (schemaFiles.length === 0) return {
|
|
14086
|
+
success: false,
|
|
14087
|
+
error: "No schema files found",
|
|
14088
|
+
warnings: [],
|
|
14089
|
+
schemaCount: 0,
|
|
14090
|
+
tables: 0,
|
|
14091
|
+
relations: 0
|
|
14092
|
+
};
|
|
14093
|
+
const generated = await generateSchemaFromFiles(schemaFiles, config);
|
|
14094
|
+
if (!generated.success) return {
|
|
14095
|
+
success: false,
|
|
14096
|
+
error: generated.error,
|
|
14097
|
+
warnings: generated.warnings,
|
|
14098
|
+
schemaCount: generated.schemaCount,
|
|
14099
|
+
tables: generated.tables,
|
|
14100
|
+
relations: generated.relations
|
|
14101
|
+
};
|
|
14102
|
+
if (options.generateOnly) return {
|
|
14103
|
+
success: true,
|
|
14104
|
+
warnings: generated.warnings,
|
|
14105
|
+
schemaCount: generated.schemaCount,
|
|
14106
|
+
tables: generated.tables,
|
|
14107
|
+
relations: generated.relations
|
|
14108
|
+
};
|
|
14109
|
+
const migration = await runSchemaMigration(config, options.migrationName);
|
|
14110
|
+
return {
|
|
14111
|
+
success: migration.success,
|
|
14112
|
+
error: migration.error,
|
|
14113
|
+
warnings: generated.warnings,
|
|
14114
|
+
schemaCount: generated.schemaCount,
|
|
14115
|
+
tables: generated.tables,
|
|
14116
|
+
relations: generated.relations,
|
|
14117
|
+
migration
|
|
14118
|
+
};
|
|
14119
|
+
}
|
|
14120
|
+
|
|
14121
|
+
//#endregion
|
|
14122
|
+
//#region src/commands/schema.ts
|
|
13585
14123
|
const schemaCommand = defineCommand({
|
|
13586
14124
|
meta: {
|
|
13587
14125
|
name: "schema",
|
|
13588
14126
|
description: "Sync JSON Schema to SQLite database"
|
|
13589
14127
|
},
|
|
13590
14128
|
args: {
|
|
13591
|
-
init: {
|
|
13592
|
-
type: "boolean",
|
|
13593
|
-
alias: "i",
|
|
13594
|
-
description: "Only initialize .aiex/ directory with example schema",
|
|
13595
|
-
default: false
|
|
13596
|
-
},
|
|
13597
14129
|
generate: {
|
|
13598
14130
|
type: "boolean",
|
|
13599
14131
|
alias: "g",
|
|
@@ -13608,226 +14140,21 @@ const schemaCommand = defineCommand({
|
|
|
13608
14140
|
async run({ args }) {
|
|
13609
14141
|
intro(pc.inverse(" aiex schema "));
|
|
13610
14142
|
const config = createMigrationConfig(process.cwd());
|
|
13611
|
-
|
|
13612
|
-
await fs.mkdir(config.schemaPath, { recursive: true });
|
|
13613
|
-
await fs.mkdir(path.dirname(config.drizzleSchemaPath), { recursive: true });
|
|
13614
|
-
await fs.mkdir(config.migrationsPath, { recursive: true });
|
|
13615
|
-
const userSchema = {
|
|
13616
|
-
$schema: $id,
|
|
13617
|
-
title: "User",
|
|
13618
|
-
type: "object",
|
|
13619
|
-
table: {
|
|
13620
|
-
name: "users",
|
|
13621
|
-
timestamps: true,
|
|
13622
|
-
softDelete: true
|
|
13623
|
-
},
|
|
13624
|
-
properties: {
|
|
13625
|
-
id: {
|
|
13626
|
-
type: "integer",
|
|
13627
|
-
primary: true,
|
|
13628
|
-
autoIncrement: true
|
|
13629
|
-
},
|
|
13630
|
-
email: {
|
|
13631
|
-
type: "string",
|
|
13632
|
-
format: "email",
|
|
13633
|
-
unique: true
|
|
13634
|
-
},
|
|
13635
|
-
username: {
|
|
13636
|
-
type: "string",
|
|
13637
|
-
minLength: 3,
|
|
13638
|
-
maxLength: 50,
|
|
13639
|
-
unique: true
|
|
13640
|
-
},
|
|
13641
|
-
displayName: {
|
|
13642
|
-
type: "string",
|
|
13643
|
-
maxLength: 100
|
|
13644
|
-
},
|
|
13645
|
-
bio: {
|
|
13646
|
-
type: "string",
|
|
13647
|
-
maxLength: 500
|
|
13648
|
-
},
|
|
13649
|
-
avatarUrl: {
|
|
13650
|
-
type: "string",
|
|
13651
|
-
format: "uri"
|
|
13652
|
-
},
|
|
13653
|
-
role: {
|
|
13654
|
-
type: "string",
|
|
13655
|
-
default: "member"
|
|
13656
|
-
},
|
|
13657
|
-
isActive: {
|
|
13658
|
-
type: "boolean",
|
|
13659
|
-
default: true
|
|
13660
|
-
},
|
|
13661
|
-
lastLoginAt: {
|
|
13662
|
-
type: "string",
|
|
13663
|
-
format: "date-time"
|
|
13664
|
-
},
|
|
13665
|
-
profile: {
|
|
13666
|
-
type: "object",
|
|
13667
|
-
drizzle: { mode: "json" },
|
|
13668
|
-
properties: {
|
|
13669
|
-
website: { type: "string" },
|
|
13670
|
-
location: { type: "string" },
|
|
13671
|
-
socialLinks: {
|
|
13672
|
-
type: "array",
|
|
13673
|
-
items: {
|
|
13674
|
-
type: "object",
|
|
13675
|
-
properties: {
|
|
13676
|
-
platform: { type: "string" },
|
|
13677
|
-
url: { type: "string" }
|
|
13678
|
-
}
|
|
13679
|
-
}
|
|
13680
|
-
}
|
|
13681
|
-
}
|
|
13682
|
-
},
|
|
13683
|
-
preferences: {
|
|
13684
|
-
type: "object",
|
|
13685
|
-
nested: {
|
|
13686
|
-
enabled: true,
|
|
13687
|
-
relation: "has-one"
|
|
13688
|
-
},
|
|
13689
|
-
properties: {
|
|
13690
|
-
theme: {
|
|
13691
|
-
type: "string",
|
|
13692
|
-
default: "light"
|
|
13693
|
-
},
|
|
13694
|
-
language: {
|
|
13695
|
-
type: "string",
|
|
13696
|
-
default: "en"
|
|
13697
|
-
},
|
|
13698
|
-
emailNotifications: {
|
|
13699
|
-
type: "boolean",
|
|
13700
|
-
default: true
|
|
13701
|
-
},
|
|
13702
|
-
pushNotifications: {
|
|
13703
|
-
type: "boolean",
|
|
13704
|
-
default: false
|
|
13705
|
-
}
|
|
13706
|
-
}
|
|
13707
|
-
}
|
|
13708
|
-
},
|
|
13709
|
-
required: ["email", "username"]
|
|
13710
|
-
};
|
|
13711
|
-
const postSchema = {
|
|
13712
|
-
$schema: $id,
|
|
13713
|
-
title: "Post",
|
|
13714
|
-
type: "object",
|
|
13715
|
-
table: {
|
|
13716
|
-
name: "posts",
|
|
13717
|
-
timestamps: true,
|
|
13718
|
-
softDelete: true
|
|
13719
|
-
},
|
|
13720
|
-
properties: {
|
|
13721
|
-
id: {
|
|
13722
|
-
type: "integer",
|
|
13723
|
-
primary: true,
|
|
13724
|
-
autoIncrement: true
|
|
13725
|
-
},
|
|
13726
|
-
title: {
|
|
13727
|
-
type: "string",
|
|
13728
|
-
minLength: 5,
|
|
13729
|
-
maxLength: 200
|
|
13730
|
-
},
|
|
13731
|
-
slug: {
|
|
13732
|
-
type: "string",
|
|
13733
|
-
maxLength: 250,
|
|
13734
|
-
unique: true
|
|
13735
|
-
},
|
|
13736
|
-
content: { type: "string" },
|
|
13737
|
-
excerpt: {
|
|
13738
|
-
type: "string",
|
|
13739
|
-
maxLength: 300
|
|
13740
|
-
},
|
|
13741
|
-
authorId: { type: "integer" },
|
|
13742
|
-
status: {
|
|
13743
|
-
type: "string",
|
|
13744
|
-
default: "draft"
|
|
13745
|
-
},
|
|
13746
|
-
viewCount: {
|
|
13747
|
-
type: "integer",
|
|
13748
|
-
default: 0,
|
|
13749
|
-
minimum: 0
|
|
13750
|
-
},
|
|
13751
|
-
likeCount: {
|
|
13752
|
-
type: "integer",
|
|
13753
|
-
default: 0,
|
|
13754
|
-
minimum: 0
|
|
13755
|
-
},
|
|
13756
|
-
publishedAt: {
|
|
13757
|
-
type: "string",
|
|
13758
|
-
format: "date-time"
|
|
13759
|
-
},
|
|
13760
|
-
tags: {
|
|
13761
|
-
type: "array",
|
|
13762
|
-
items: { type: "string" }
|
|
13763
|
-
},
|
|
13764
|
-
metadata: {
|
|
13765
|
-
type: "object",
|
|
13766
|
-
drizzle: { mode: "json" },
|
|
13767
|
-
properties: {
|
|
13768
|
-
featuredImage: { type: "string" },
|
|
13769
|
-
readingTime: { type: "integer" },
|
|
13770
|
-
seoTitle: { type: "string" },
|
|
13771
|
-
seoDescription: { type: "string" }
|
|
13772
|
-
}
|
|
13773
|
-
},
|
|
13774
|
-
comments: {
|
|
13775
|
-
type: "array",
|
|
13776
|
-
items: {
|
|
13777
|
-
type: "object",
|
|
13778
|
-
nested: {
|
|
13779
|
-
enabled: true,
|
|
13780
|
-
relation: "has-many"
|
|
13781
|
-
},
|
|
13782
|
-
properties: {
|
|
13783
|
-
content: {
|
|
13784
|
-
type: "string",
|
|
13785
|
-
minLength: 1,
|
|
13786
|
-
maxLength: 1e3
|
|
13787
|
-
},
|
|
13788
|
-
authorId: { type: "integer" },
|
|
13789
|
-
status: {
|
|
13790
|
-
type: "string",
|
|
13791
|
-
default: "pending"
|
|
13792
|
-
},
|
|
13793
|
-
parentId: { type: "integer" }
|
|
13794
|
-
}
|
|
13795
|
-
}
|
|
13796
|
-
}
|
|
13797
|
-
},
|
|
13798
|
-
required: [
|
|
13799
|
-
"title",
|
|
13800
|
-
"slug",
|
|
13801
|
-
"authorId"
|
|
13802
|
-
]
|
|
13803
|
-
};
|
|
13804
|
-
const userStatus = await writeJsonIfAbsent(path.join(config.schemaPath, "user.json"), userSchema);
|
|
13805
|
-
const postStatus = await writeJsonIfAbsent(path.join(config.schemaPath, "post.json"), postSchema);
|
|
13806
|
-
consola.success(`Initialized ${pc.cyan(".aiex/")} with example schemas`);
|
|
13807
|
-
if (userStatus === "skipped") consola.warn(`${pc.cyan(".aiex/schema/user.json")} already exists, skipped`);
|
|
13808
|
-
if (postStatus === "skipped") consola.warn(`${pc.cyan(".aiex/schema/post.json")} already exists, skipped`);
|
|
13809
|
-
consola.info("Example includes: User (with preferences has-one), Post (with comments has-many)");
|
|
13810
|
-
outro("Run: aiex schema");
|
|
13811
|
-
return;
|
|
13812
|
-
}
|
|
13813
|
-
let schemaFiles;
|
|
13814
|
-
try {
|
|
13815
|
-
schemaFiles = await fs.readdir(config.schemaPath);
|
|
13816
|
-
schemaFiles = schemaFiles.filter((f) => f.endsWith(".json")).map((f) => path.join(config.schemaPath, f));
|
|
13817
|
-
} catch {
|
|
13818
|
-
schemaFiles = [];
|
|
13819
|
-
}
|
|
14143
|
+
const schemaFiles = await listSchemaFiles(config.schemaPath);
|
|
13820
14144
|
if (schemaFiles.length === 0) {
|
|
13821
|
-
consola.info("
|
|
13822
|
-
|
|
14145
|
+
consola.info(`Run ${pc.cyan("aiex web")} to create and configure schemas in the Web UI`);
|
|
14146
|
+
failCommand(`No schema files found in ${pc.cyan(".aiex/schema/")}`);
|
|
13823
14147
|
return;
|
|
13824
14148
|
}
|
|
13825
14149
|
const s1 = spinner();
|
|
13826
14150
|
s1.start("Generating Drizzle schema...");
|
|
13827
|
-
const
|
|
13828
|
-
|
|
13829
|
-
if (
|
|
13830
|
-
|
|
14151
|
+
const generated = await generateSchemaFromFiles(schemaFiles, config);
|
|
14152
|
+
for (const warning of generated.warnings) consola.warn(warning);
|
|
14153
|
+
if (generated.success) consola.success(`Generated ${pc.cyan(".aiex/drizzle/schema.ts")} from ${generated.schemaCount} schema file(s)`);
|
|
14154
|
+
else if (generated.error) consola.error(generated.error);
|
|
14155
|
+
s1.stop(generated.success ? "Schema generated" : "Generation failed");
|
|
14156
|
+
if (!generated.success) {
|
|
14157
|
+
failCommand();
|
|
13831
14158
|
return;
|
|
13832
14159
|
}
|
|
13833
14160
|
if (args.generate) {
|
|
@@ -13836,10 +14163,18 @@ const schemaCommand = defineCommand({
|
|
|
13836
14163
|
}
|
|
13837
14164
|
const s2 = spinner();
|
|
13838
14165
|
s2.start("Running migrations...");
|
|
13839
|
-
const
|
|
13840
|
-
|
|
13841
|
-
|
|
13842
|
-
|
|
14166
|
+
const migration = await runSchemaMigration(config, args.name);
|
|
14167
|
+
if (!migration.success) {
|
|
14168
|
+
consola.error("Failed to generate migration");
|
|
14169
|
+
consola.error(migration.error || "Migration failed");
|
|
14170
|
+
} else if (migration.changes === 0) consola.info(pc.gray("No changes detected"));
|
|
14171
|
+
else {
|
|
14172
|
+
consola.success(pc.green("Migration files generated"));
|
|
14173
|
+
consola.success(pc.green("Database migrated"));
|
|
14174
|
+
}
|
|
14175
|
+
s2.stop(migration.success ? "Migrations applied" : "Migration failed");
|
|
14176
|
+
if (!migration.success) {
|
|
14177
|
+
failCommand();
|
|
13843
14178
|
return;
|
|
13844
14179
|
}
|
|
13845
14180
|
outro("Done!");
|
|
@@ -13863,13 +14198,7 @@ function aiRoutes(config) {
|
|
|
13863
14198
|
}
|
|
13864
14199
|
});
|
|
13865
14200
|
}
|
|
13866
|
-
return c.json(
|
|
13867
|
-
...aiConfig,
|
|
13868
|
-
provider: {
|
|
13869
|
-
...aiConfig.provider,
|
|
13870
|
-
apiKey: maskApiKey(aiConfig.provider.apiKey)
|
|
13871
|
-
}
|
|
13872
|
-
});
|
|
14201
|
+
return c.json(aiConfig);
|
|
13873
14202
|
});
|
|
13874
14203
|
app.post("/ai/registry-lookup", async (c) => {
|
|
13875
14204
|
try {
|
|
@@ -13897,11 +14226,6 @@ function aiRoutes(config) {
|
|
|
13897
14226
|
success: false,
|
|
13898
14227
|
error: "At least one model must be configured"
|
|
13899
14228
|
}, 400);
|
|
13900
|
-
if (body.provider?.apiKey?.startsWith("sk-***")) {
|
|
13901
|
-
const existing = await readAIConfig(aiexDir);
|
|
13902
|
-
if (existing) body.provider.apiKey = existing.provider.apiKey;
|
|
13903
|
-
else body.provider.apiKey = "";
|
|
13904
|
-
}
|
|
13905
14229
|
await writeAIConfig(aiexDir, AIConfigSchema.parse(body));
|
|
13906
14230
|
return c.json({ success: true });
|
|
13907
14231
|
} catch (error) {
|
|
@@ -13917,10 +14241,25 @@ function aiRoutes(config) {
|
|
|
13917
14241
|
//#endregion
|
|
13918
14242
|
//#region src/server/routes/data.ts
|
|
13919
14243
|
const FILE_REGEX = /\.json$/;
|
|
13920
|
-
const EXTRACTION_FILE_RE = /^[\w.-]+\.json$/;
|
|
13921
|
-
const TABLE_NAME_RE$1 = /^[a-z][a-z0-9_]*$/;
|
|
13922
14244
|
const TIMESTAMP_CLEANUP = /(\d{2})-(\d{2})-(\d{2})/;
|
|
13923
14245
|
const TIMESTAMP_TZ = /(\d{3})Z/;
|
|
14246
|
+
const tableParamSchema = z.object({ name: z.string().regex(/^[a-z][a-z0-9_]*$/) });
|
|
14247
|
+
const extractionFileParamSchema = z.object({ name: z.string().regex(/^[\w.-]+\.json$/).refine((name$1) => name$1 === path.basename(name$1) && !name$1.includes("..")) });
|
|
14248
|
+
const tableQuerySchema = z.object({
|
|
14249
|
+
page: z.coerce.number().int().min(1).catch(1),
|
|
14250
|
+
pageSize: z.coerce.number().int().min(1).max(500).catch(50),
|
|
14251
|
+
search: z.string().catch(""),
|
|
14252
|
+
sortField: z.string().optional(),
|
|
14253
|
+
sortOrder: z.preprocess((value) => typeof value === "string" ? value.toLowerCase() : value, z.enum(["asc", "desc"]).catch("asc"))
|
|
14254
|
+
});
|
|
14255
|
+
function invalidParamResponse$1(message) {
|
|
14256
|
+
return (result, c) => {
|
|
14257
|
+
if (!result.success) return c.json({ error: message }, 400);
|
|
14258
|
+
};
|
|
14259
|
+
}
|
|
14260
|
+
function createReadonlyQueryDb(databasePath) {
|
|
14261
|
+
return new Kysely({ dialect: new SqliteDialect({ database: new Database(databasePath, { readonly: true }) }) });
|
|
14262
|
+
}
|
|
13924
14263
|
function dataRoutes(config) {
|
|
13925
14264
|
const app = new Hono();
|
|
13926
14265
|
const aiexDir = path.dirname(config.schemaPath);
|
|
@@ -13968,15 +14307,19 @@ function dataRoutes(config) {
|
|
|
13968
14307
|
let db = null;
|
|
13969
14308
|
let dbTables = [];
|
|
13970
14309
|
try {
|
|
13971
|
-
db =
|
|
13972
|
-
dbTables =
|
|
14310
|
+
db = createReadonlyQueryDb(config.databasePath);
|
|
14311
|
+
dbTables = (await sql`
|
|
14312
|
+
select name
|
|
14313
|
+
from sqlite_master
|
|
14314
|
+
where type = 'table' and name not like 'sqlite_%' and name not like '_%'
|
|
14315
|
+
order by name
|
|
14316
|
+
`.execute(db)).rows.map((row) => row.name);
|
|
13973
14317
|
} catch {} finally {
|
|
13974
|
-
db?.
|
|
14318
|
+
await db?.destroy();
|
|
13975
14319
|
}
|
|
13976
14320
|
const tables = [];
|
|
13977
14321
|
for (const file of schemaFiles) try {
|
|
13978
|
-
const
|
|
13979
|
-
const schema = JSON.parse(content);
|
|
14322
|
+
const schema = await readFile(path.join(schemaDir, file));
|
|
13980
14323
|
const tableName = schema.table?.name;
|
|
13981
14324
|
if (!tableName) continue;
|
|
13982
14325
|
tables.push({
|
|
@@ -13992,46 +14335,64 @@ function dataRoutes(config) {
|
|
|
13992
14335
|
return c.json({ error: error instanceof Error ? error.message : String(error) }, 500);
|
|
13993
14336
|
}
|
|
13994
14337
|
});
|
|
13995
|
-
app.get("/data/tables/:name", async (c) => {
|
|
13996
|
-
const tableName = c.req.
|
|
13997
|
-
|
|
13998
|
-
const sortField = c.req.query("sortField");
|
|
13999
|
-
const sortOrder = c.req.query("sortOrder") || "asc";
|
|
14338
|
+
app.get("/data/tables/:name", zValidator("param", tableParamSchema, invalidParamResponse$1("Invalid table name")), zValidator("query", tableQuerySchema), async (c) => {
|
|
14339
|
+
const { name: tableName } = c.req.valid("param");
|
|
14340
|
+
const { page, pageSize, search, sortField, sortOrder } = c.req.valid("query");
|
|
14000
14341
|
let db;
|
|
14001
14342
|
try {
|
|
14002
|
-
db =
|
|
14343
|
+
db = createReadonlyQueryDb(config.databasePath);
|
|
14003
14344
|
} catch {
|
|
14004
14345
|
return c.json({ error: "Database not found. Run `aiex schema` first." }, 400);
|
|
14005
14346
|
}
|
|
14006
14347
|
try {
|
|
14007
|
-
if (
|
|
14008
|
-
|
|
14009
|
-
|
|
14010
|
-
|
|
14011
|
-
|
|
14348
|
+
if ((await sql`
|
|
14349
|
+
select name
|
|
14350
|
+
from sqlite_master
|
|
14351
|
+
where type = 'table' and name = ${tableName}
|
|
14352
|
+
`.execute(db)).rows.length === 0) return c.json({ error: `Table "${tableName}" not found in database` }, 404);
|
|
14353
|
+
const columns = (await sql`
|
|
14354
|
+
pragma table_info(${sql.table(tableName)})
|
|
14355
|
+
`.execute(db)).rows.map((col) => ({
|
|
14012
14356
|
name: col.name,
|
|
14013
14357
|
type: col.type,
|
|
14014
14358
|
notNull: !!col.notnull,
|
|
14015
14359
|
pk: !!col.pk
|
|
14016
14360
|
}));
|
|
14017
|
-
|
|
14018
|
-
|
|
14019
|
-
const
|
|
14020
|
-
const
|
|
14361
|
+
const searchConditions = columns.map((col) => sql`${sql.ref(col.name)} like ${`%${search}%`}`);
|
|
14362
|
+
const searchCondition = search ? sql`where ${sql.join(searchConditions, sql` or `)}` : sql``;
|
|
14363
|
+
const sortColumn = columns.find((col) => col.name === sortField);
|
|
14364
|
+
const orderBy = sortColumn ? sql`order by ${sql.ref(sortColumn.name)} ${sql.raw(sortOrder === "desc" ? "desc" : "asc")}` : sql``;
|
|
14365
|
+
const total = (await sql`
|
|
14366
|
+
select count(*) as count
|
|
14367
|
+
from ${sql.table(tableName)}
|
|
14368
|
+
${searchCondition}
|
|
14369
|
+
`.execute(db)).rows[0]?.count ?? 0;
|
|
14370
|
+
const offset = (page - 1) * pageSize;
|
|
14371
|
+
const totalPages = Math.max(1, Math.ceil(total / pageSize));
|
|
14372
|
+
const result = await sql`
|
|
14373
|
+
select *
|
|
14374
|
+
from ${sql.table(tableName)}
|
|
14375
|
+
${searchCondition}
|
|
14376
|
+
${orderBy}
|
|
14377
|
+
limit ${pageSize}
|
|
14378
|
+
offset ${offset}
|
|
14379
|
+
`.execute(db);
|
|
14021
14380
|
return c.json({
|
|
14022
14381
|
columns,
|
|
14023
|
-
rows,
|
|
14024
|
-
total
|
|
14382
|
+
rows: result.rows,
|
|
14383
|
+
total,
|
|
14384
|
+
page,
|
|
14385
|
+
pageSize,
|
|
14386
|
+
totalPages
|
|
14025
14387
|
});
|
|
14026
14388
|
} catch (error) {
|
|
14027
14389
|
return c.json({ error: error instanceof Error ? error.message : String(error) }, 500);
|
|
14028
14390
|
} finally {
|
|
14029
|
-
db.
|
|
14391
|
+
await db.destroy();
|
|
14030
14392
|
}
|
|
14031
14393
|
});
|
|
14032
|
-
app.get("/data/:name", async (c) => {
|
|
14033
|
-
const name$1 = c.req.
|
|
14034
|
-
if (name$1 !== path.basename(name$1) || !EXTRACTION_FILE_RE.test(name$1) || name$1.includes("..")) return c.json({ error: "Invalid extraction file name" }, 400);
|
|
14394
|
+
app.get("/data/:name", zValidator("param", extractionFileParamSchema, invalidParamResponse$1("Invalid extraction file name")), async (c) => {
|
|
14395
|
+
const { name: name$1 } = c.req.valid("param");
|
|
14035
14396
|
const filePath = path.join(extractedDir, name$1);
|
|
14036
14397
|
try {
|
|
14037
14398
|
const content = await fs.readFile(filePath, "utf-8");
|
|
@@ -14049,12 +14410,13 @@ function dataRoutes(config) {
|
|
|
14049
14410
|
|
|
14050
14411
|
//#endregion
|
|
14051
14412
|
//#region src/server/routes/schema.ts
|
|
14052
|
-
const
|
|
14053
|
-
const
|
|
14054
|
-
const
|
|
14055
|
-
function
|
|
14056
|
-
|
|
14057
|
-
|
|
14413
|
+
const schemaFileNameSchema = z.string().regex(/^[\w.-]+\.json$/).refine((name$1) => name$1 === path.basename(name$1) && !name$1.includes(".."));
|
|
14414
|
+
const schemaFileParamSchema = z.object({ name: schemaFileNameSchema });
|
|
14415
|
+
const tableNameParamSchema = z.object({ name: z.string().regex(/^[a-z][a-z0-9_]*$/) });
|
|
14416
|
+
function invalidParamResponse(message) {
|
|
14417
|
+
return (result, c) => {
|
|
14418
|
+
if (!result.success) return c.json({ error: message }, 400);
|
|
14419
|
+
};
|
|
14058
14420
|
}
|
|
14059
14421
|
function schemaRoutes(config) {
|
|
14060
14422
|
const app = new Hono();
|
|
@@ -14067,23 +14429,25 @@ function schemaRoutes(config) {
|
|
|
14067
14429
|
const jsonFiles = (await fs.readdir(schemaDir)).filter((f) => f.endsWith(".json"));
|
|
14068
14430
|
return c.json(jsonFiles);
|
|
14069
14431
|
});
|
|
14070
|
-
app.get("/schema/:name", async (c) => {
|
|
14071
|
-
const
|
|
14072
|
-
|
|
14432
|
+
app.get("/schema/:name", zValidator("param", schemaFileParamSchema, invalidParamResponse("Invalid schema file name")), async (c) => {
|
|
14433
|
+
const { name: name$1 } = c.req.valid("param");
|
|
14434
|
+
const filePath = path.join(schemaDir, name$1);
|
|
14073
14435
|
try {
|
|
14074
|
-
|
|
14075
|
-
return c.json(JSON.parse(content));
|
|
14436
|
+
return c.json(await readFile(filePath));
|
|
14076
14437
|
} catch {
|
|
14077
14438
|
return c.json({ error: "Schema not found" }, 404);
|
|
14078
14439
|
}
|
|
14079
14440
|
});
|
|
14080
|
-
app.post("/schema/:name", async (c) => {
|
|
14081
|
-
const
|
|
14082
|
-
|
|
14441
|
+
app.post("/schema/:name", zValidator("param", schemaFileParamSchema, invalidParamResponse("Invalid schema file name")), async (c) => {
|
|
14442
|
+
const { name: name$1 } = c.req.valid("param");
|
|
14443
|
+
const filePath = path.join(schemaDir, name$1);
|
|
14083
14444
|
try {
|
|
14084
14445
|
const body = await c.req.json();
|
|
14085
14446
|
await ensureDir();
|
|
14086
|
-
await
|
|
14447
|
+
await writeFile(filePath, body, {
|
|
14448
|
+
spaces: 2,
|
|
14449
|
+
EOL: "\n"
|
|
14450
|
+
});
|
|
14087
14451
|
const aiexDir = path.dirname(schemaDir);
|
|
14088
14452
|
try {
|
|
14089
14453
|
await savePromptSnapshot(JsonSchemaDefinitionSchema.parse(body), aiexDir);
|
|
@@ -14093,12 +14457,8 @@ function schemaRoutes(config) {
|
|
|
14093
14457
|
return c.json({ error: "Failed to save schema" }, 500);
|
|
14094
14458
|
}
|
|
14095
14459
|
});
|
|
14096
|
-
app.get("/prompt-snapshot/:name", async (c) => {
|
|
14097
|
-
const name$1 = c.req.
|
|
14098
|
-
if (!TABLE_NAME_RE.test(name$1)) return c.json({
|
|
14099
|
-
success: false,
|
|
14100
|
-
error: "Invalid table name"
|
|
14101
|
-
}, 400);
|
|
14460
|
+
app.get("/prompt-snapshot/:name", zValidator("param", tableNameParamSchema, invalidParamResponse("Invalid table name")), async (c) => {
|
|
14461
|
+
const { name: name$1 } = c.req.valid("param");
|
|
14102
14462
|
const aiexDir = path.dirname(schemaDir);
|
|
14103
14463
|
const snapshotPath = path.join(aiexDir, "extracted", `${name$1}.prompt.md`);
|
|
14104
14464
|
try {
|
|
@@ -14114,14 +14474,13 @@ function schemaRoutes(config) {
|
|
|
14114
14474
|
}, 404);
|
|
14115
14475
|
}
|
|
14116
14476
|
});
|
|
14117
|
-
app.delete("/schema/:name", async (c) => {
|
|
14118
|
-
const
|
|
14119
|
-
|
|
14477
|
+
app.delete("/schema/:name", zValidator("param", schemaFileParamSchema, invalidParamResponse("Invalid schema file name")), async (c) => {
|
|
14478
|
+
const { name: name$1 } = c.req.valid("param");
|
|
14479
|
+
const filePath = path.join(schemaDir, name$1);
|
|
14120
14480
|
try {
|
|
14121
14481
|
const aiexDir = path.dirname(schemaDir);
|
|
14122
14482
|
try {
|
|
14123
|
-
const
|
|
14124
|
-
const parsed = JsonSchemaDefinitionSchema.safeParse(JSON.parse(content));
|
|
14483
|
+
const parsed = JsonSchemaDefinitionSchema.safeParse(await readFile(filePath));
|
|
14125
14484
|
if (parsed.success) {
|
|
14126
14485
|
const tableName = parsed.data.table.name;
|
|
14127
14486
|
const snapshotPath = path.join(aiexDir, "extracted", `${tableName}.prompt.md`);
|
|
@@ -14137,59 +14496,21 @@ function schemaRoutes(config) {
|
|
|
14137
14496
|
app.post("/migrate", async (c) => {
|
|
14138
14497
|
try {
|
|
14139
14498
|
await ensureDir();
|
|
14140
|
-
await
|
|
14141
|
-
|
|
14142
|
-
|
|
14143
|
-
success: false,
|
|
14144
|
-
error: "No schema files found"
|
|
14145
|
-
}, 400);
|
|
14146
|
-
const parsedResult = parseAllSchemas(await Promise.all(jsonFiles.map(async (fileName) => {
|
|
14147
|
-
const filePath = path.join(schemaDir, fileName);
|
|
14148
|
-
return {
|
|
14149
|
-
filePath,
|
|
14150
|
-
content: await fs.readFile(filePath, "utf-8")
|
|
14151
|
-
};
|
|
14152
|
-
})));
|
|
14153
|
-
if (!parsedResult.success) return c.json({
|
|
14154
|
-
success: false,
|
|
14155
|
-
error: parsedResult.error
|
|
14156
|
-
}, 400);
|
|
14157
|
-
const { tables, relations, reverseRelations, warnings, drizzleCode } = parsedResult.data;
|
|
14158
|
-
await fs.writeFile(config.drizzleSchemaPath, drizzleCode);
|
|
14159
|
-
const helperPath = resolveHelperPath();
|
|
14160
|
-
const tsxPath = resolveTsxPath();
|
|
14161
|
-
const { stdout, stderr } = await execFileAsync(process.execPath, [
|
|
14162
|
-
tsxPath,
|
|
14163
|
-
helperPath,
|
|
14164
|
-
config.drizzleSchemaPath,
|
|
14165
|
-
config.migrationsPath,
|
|
14166
|
-
config.databasePath
|
|
14167
|
-
], { cwd: process.cwd() });
|
|
14168
|
-
let migrationResult;
|
|
14169
|
-
try {
|
|
14170
|
-
const jsonLine = stdout.trim().split("\n").find((l) => l.startsWith("{") && l.endsWith("}"));
|
|
14171
|
-
if (!jsonLine) return c.json({
|
|
14172
|
-
success: false,
|
|
14173
|
-
error: "Migration helper did not return valid output"
|
|
14174
|
-
}, 500);
|
|
14175
|
-
migrationResult = JSON.parse(jsonLine);
|
|
14176
|
-
} catch {
|
|
14499
|
+
const result = await runSchemaSync(config);
|
|
14500
|
+
if (!result.success) {
|
|
14501
|
+
const status = result.schemaCount === 0 ? 400 : 500;
|
|
14177
14502
|
return c.json({
|
|
14178
14503
|
success: false,
|
|
14179
|
-
error:
|
|
14180
|
-
},
|
|
14504
|
+
error: result.error || "Migration failed"
|
|
14505
|
+
}, status);
|
|
14181
14506
|
}
|
|
14182
|
-
if (!migrationResult.success) return c.json({
|
|
14183
|
-
success: false,
|
|
14184
|
-
error: migrationResult.error || "Migration failed"
|
|
14185
|
-
}, 500);
|
|
14186
14507
|
return c.json({
|
|
14187
14508
|
success: true,
|
|
14188
|
-
changes:
|
|
14189
|
-
tag:
|
|
14190
|
-
tables: tables
|
|
14191
|
-
relations: relations
|
|
14192
|
-
warnings
|
|
14509
|
+
changes: result.migration?.changes ?? 0,
|
|
14510
|
+
tag: result.migration?.tag,
|
|
14511
|
+
tables: result.tables,
|
|
14512
|
+
relations: result.relations,
|
|
14513
|
+
warnings: result.warnings
|
|
14193
14514
|
});
|
|
14194
14515
|
} catch (error) {
|
|
14195
14516
|
return c.json({
|
|
@@ -14237,9 +14558,36 @@ function createApp(config, staticDir) {
|
|
|
14237
14558
|
return app;
|
|
14238
14559
|
}
|
|
14239
14560
|
|
|
14561
|
+
//#endregion
|
|
14562
|
+
//#region src/core/web-runner.ts
|
|
14563
|
+
function resolveWebStaticDir() {
|
|
14564
|
+
return path.join(resolvePackageRoot(), "dist/web");
|
|
14565
|
+
}
|
|
14566
|
+
async function openBrowser(url) {
|
|
14567
|
+
await open(url);
|
|
14568
|
+
}
|
|
14569
|
+
async function startWebServer(input) {
|
|
14570
|
+
const { config, port } = input;
|
|
14571
|
+
const staticDir = input.staticDir ?? resolveWebStaticDir();
|
|
14572
|
+
const url = `http://localhost:${port}`;
|
|
14573
|
+
serve({
|
|
14574
|
+
fetch: createApp(config, staticDir).fetch,
|
|
14575
|
+
port
|
|
14576
|
+
}, () => {
|
|
14577
|
+
input.onStarted?.({
|
|
14578
|
+
url,
|
|
14579
|
+
schemaPath: config.schemaPath
|
|
14580
|
+
});
|
|
14581
|
+
if (input.open === false) return;
|
|
14582
|
+
openBrowser(url).catch(() => {
|
|
14583
|
+
input.onOpenFailed?.(url);
|
|
14584
|
+
});
|
|
14585
|
+
});
|
|
14586
|
+
await new Promise(() => {});
|
|
14587
|
+
}
|
|
14588
|
+
|
|
14240
14589
|
//#endregion
|
|
14241
14590
|
//#region src/commands/web.ts
|
|
14242
|
-
const execAsync = promisify(exec);
|
|
14243
14591
|
const webCommand = defineCommand({
|
|
14244
14592
|
meta: {
|
|
14245
14593
|
name: "web",
|
|
@@ -14256,23 +14604,20 @@ const webCommand = defineCommand({
|
|
|
14256
14604
|
const cwd = process.cwd();
|
|
14257
14605
|
const port = Number(args.port) || 13e3;
|
|
14258
14606
|
const config = createMigrationConfig(cwd);
|
|
14259
|
-
const packageRoot = resolvePackageRoot();
|
|
14260
|
-
const staticDir = path.join(packageRoot, "dist/web");
|
|
14261
14607
|
const s = spinner();
|
|
14262
14608
|
s.start("Starting web server...");
|
|
14263
|
-
|
|
14264
|
-
|
|
14265
|
-
port
|
|
14266
|
-
|
|
14267
|
-
|
|
14268
|
-
|
|
14269
|
-
|
|
14270
|
-
|
|
14271
|
-
|
|
14609
|
+
await startWebServer({
|
|
14610
|
+
config,
|
|
14611
|
+
port,
|
|
14612
|
+
onStarted(info) {
|
|
14613
|
+
s.stop(`Server running at ${pc.cyan(info.url)}`);
|
|
14614
|
+
consola.info(`Schema directory: ${pc.dim(info.schemaPath)}`);
|
|
14615
|
+
consola.info("Press Ctrl+C to stop");
|
|
14616
|
+
},
|
|
14617
|
+
onOpenFailed(url) {
|
|
14272
14618
|
consola.warn(`Could not open browser. Visit ${url} manually.`);
|
|
14273
|
-
}
|
|
14619
|
+
}
|
|
14274
14620
|
});
|
|
14275
|
-
await new Promise(() => {});
|
|
14276
14621
|
}
|
|
14277
14622
|
});
|
|
14278
14623
|
|
|
@@ -14300,7 +14645,7 @@ process.on("unhandledRejection", (reason) => {
|
|
|
14300
14645
|
process.exit(1);
|
|
14301
14646
|
});
|
|
14302
14647
|
if (process.argv[2] === "_complete") {
|
|
14303
|
-
const { getCompletions } = await import("./completions-
|
|
14648
|
+
const { getCompletions } = await import("./completions-C3rmTwXZ.mjs");
|
|
14304
14649
|
const suggestions = getCompletions(subCommands, process.argv.slice(3));
|
|
14305
14650
|
for (const s of suggestions) process.stdout.write(`${s}\n`);
|
|
14306
14651
|
process.exit(0);
|