@deepagents/text2sql 0.38.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +166 -72
- package/dist/bin/sql.js +699 -0
- package/dist/index.d.ts +2 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1989 -1898
- package/dist/index.js.map +4 -4
- package/dist/lib/adapter-index.d.ts +35 -0
- package/dist/lib/adapter-index.d.ts.map +1 -0
- package/dist/lib/adapters/adapter.d.ts +1 -0
- package/dist/lib/adapters/adapter.d.ts.map +1 -1
- package/dist/lib/adapters/bigquery/index.js +24 -13
- package/dist/lib/adapters/bigquery/index.js.map +2 -2
- package/dist/lib/adapters/groundings/context.d.ts +2 -2
- package/dist/lib/adapters/groundings/context.d.ts.map +1 -1
- package/dist/lib/adapters/groundings/index.js +20 -9
- package/dist/lib/adapters/groundings/index.js.map +2 -2
- package/dist/lib/adapters/mysql/index.js +23 -12
- package/dist/lib/adapters/mysql/index.js.map +2 -2
- package/dist/lib/adapters/postgres/index.js +31 -18
- package/dist/lib/adapters/postgres/index.js.map +3 -3
- package/dist/lib/adapters/postgres/postgres.d.ts.map +1 -1
- package/dist/lib/adapters/spreadsheet/index.js +20 -9
- package/dist/lib/adapters/spreadsheet/index.js.map +2 -2
- package/dist/lib/adapters/sqlite/index.js +23 -12
- package/dist/lib/adapters/sqlite/index.js.map +2 -2
- package/dist/lib/adapters/sqlserver/index.js +31 -18
- package/dist/lib/adapters/sqlserver/index.js.map +3 -3
- package/dist/lib/adapters/sqlserver/sqlserver.d.ts.map +1 -1
- package/dist/lib/agents/question.agent.d.ts.map +1 -1
- package/dist/lib/agents/sql-transform-plugins.d.ts +3 -3
- package/dist/lib/agents/sql-transform-plugins.d.ts.map +1 -1
- package/dist/lib/agents/sql.agent.d.ts.map +1 -1
- package/dist/lib/instructions.d.ts +1 -1
- package/dist/lib/instructions.d.ts.map +1 -1
- package/dist/lib/sql-hooks.d.ts +15 -0
- package/dist/lib/sql-hooks.d.ts.map +1 -0
- package/dist/lib/sql-meta-hook.d.ts +20 -0
- package/dist/lib/sql-meta-hook.d.ts.map +1 -0
- package/dist/lib/sql-repair.d.ts +2 -0
- package/dist/lib/sql-repair.d.ts.map +1 -0
- package/dist/lib/sql-transform.d.ts +2 -0
- package/dist/lib/sql-transform.d.ts.map +1 -0
- package/dist/lib/sql.d.ts +5 -37
- package/dist/lib/sql.d.ts.map +1 -1
- package/dist/lib/synthesis/extractors/base-contextual-extractor.d.ts.map +1 -1
- package/dist/lib/synthesis/extractors/segmented-context-extractor.d.ts.map +1 -1
- package/dist/lib/synthesis/extractors/sql-extractor.d.ts.map +1 -1
- package/dist/lib/synthesis/index.js +59 -2
- package/dist/lib/synthesis/index.js.map +4 -4
- package/dist/lib/synthesis/synthesizers/breadth-evolver.d.ts.map +1 -1
- package/dist/lib/synthesis/synthesizers/depth-evolver.d.ts.map +1 -1
- package/dist/lib/synthesis/synthesizers/persona-generator.d.ts.map +1 -1
- package/package.json +13 -9
- package/dist/lib/agents/sql-command.d.ts +0 -11
- package/dist/lib/agents/sql-command.d.ts.map +0 -1
- package/dist/lib/demo.d.ts +0 -2
- package/dist/lib/demo.d.ts.map +0 -1
- package/dist/lib/sandbox.d.ts +0 -21
- package/dist/lib/sandbox.d.ts.map +0 -1
package/dist/index.js
CHANGED
|
@@ -163,6 +163,17 @@ function relationship(input) {
|
|
|
163
163
|
}
|
|
164
164
|
|
|
165
165
|
// packages/text2sql/src/lib/adapters/groundings/context.ts
|
|
166
|
+
var noopProgress = () => {
|
|
167
|
+
};
|
|
168
|
+
function timestampProgress(onProgress) {
|
|
169
|
+
if (!onProgress) return noopProgress;
|
|
170
|
+
return (progress) => {
|
|
171
|
+
onProgress({
|
|
172
|
+
...progress,
|
|
173
|
+
timestampMs: progress.timestampMs ?? Date.now()
|
|
174
|
+
});
|
|
175
|
+
};
|
|
176
|
+
}
|
|
166
177
|
function createGroundingContext(options = {}) {
|
|
167
178
|
return {
|
|
168
179
|
tables: [],
|
|
@@ -170,7 +181,7 @@ function createGroundingContext(options = {}) {
|
|
|
170
181
|
relationships: [],
|
|
171
182
|
info: void 0,
|
|
172
183
|
cache: /* @__PURE__ */ new Map(),
|
|
173
|
-
onProgress: options.onProgress
|
|
184
|
+
onProgress: timestampProgress(options.onProgress)
|
|
174
185
|
};
|
|
175
186
|
}
|
|
176
187
|
|
|
@@ -317,7 +328,7 @@ var Adapter = class {
|
|
|
317
328
|
for (const fn of this.grounding) {
|
|
318
329
|
const grounding = fn(this);
|
|
319
330
|
if (grounding.phase) {
|
|
320
|
-
ctx.onProgress
|
|
331
|
+
ctx.onProgress({
|
|
321
332
|
type: "phase:start",
|
|
322
333
|
phase: grounding.phase,
|
|
323
334
|
message: `Starting ${grounding.name} introspection...`
|
|
@@ -327,7 +338,7 @@ var Adapter = class {
|
|
|
327
338
|
await grounding.execute(ctx);
|
|
328
339
|
} finally {
|
|
329
340
|
if (grounding.phase) {
|
|
330
|
-
ctx.onProgress
|
|
341
|
+
ctx.onProgress({
|
|
331
342
|
type: "phase:end",
|
|
332
343
|
phase: grounding.phase,
|
|
333
344
|
message: `Finished ${grounding.name} introspection.`
|
|
@@ -735,1748 +746,1074 @@ function getTablesWithRelated(allTables, relationships, filter) {
|
|
|
735
746
|
return Array.from(result);
|
|
736
747
|
}
|
|
737
748
|
|
|
738
|
-
// packages/text2sql/src/lib/agents/sql
|
|
739
|
-
import {
|
|
749
|
+
// packages/text2sql/src/lib/agents/sql.agent.ts
|
|
750
|
+
import { groq } from "@ai-sdk/groq";
|
|
740
751
|
import {
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
752
|
+
APICallError,
|
|
753
|
+
JSONParseError,
|
|
754
|
+
NoContentGeneratedError,
|
|
755
|
+
NoObjectGeneratedError,
|
|
756
|
+
NoOutputGeneratedError,
|
|
757
|
+
TypeValidationError,
|
|
758
|
+
defaultSettingsMiddleware,
|
|
759
|
+
wrapLanguageModel
|
|
760
|
+
} from "ai";
|
|
761
|
+
import dedent from "dedent";
|
|
762
|
+
import pRetry from "p-retry";
|
|
763
|
+
import z from "zod";
|
|
764
|
+
import "@deepagents/agent";
|
|
765
|
+
import {
|
|
766
|
+
ContextEngine,
|
|
767
|
+
InMemoryContextStore,
|
|
768
|
+
example,
|
|
769
|
+
fragment,
|
|
770
|
+
guardrail,
|
|
771
|
+
hint,
|
|
772
|
+
persona,
|
|
773
|
+
policy,
|
|
774
|
+
structuredOutput,
|
|
775
|
+
user,
|
|
776
|
+
workflow
|
|
745
777
|
} from "@deepagents/context";
|
|
746
|
-
var
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
778
|
+
var RETRY_TEMPERATURES = [0, 0.2, 0.3];
|
|
779
|
+
var SQL_AGENT_ROLE = "Expert SQL query generator.";
|
|
780
|
+
var SQL_AGENT_OBJECTIVE = "Generate precise SQL grounded in provided schema.";
|
|
781
|
+
var FORMATTER_SCOPE_DIALECTS = /* @__PURE__ */ new Map([
|
|
782
|
+
["sqlite", "sqlite"],
|
|
783
|
+
["postgresql", "postgresql"],
|
|
784
|
+
["bigquery", "bigquery"],
|
|
785
|
+
["transactsql", "transactsql"],
|
|
786
|
+
["mysql", "mysql"]
|
|
787
|
+
]);
|
|
788
|
+
var SQL_AGENT_POLICIES = [
|
|
789
|
+
fragment(
|
|
790
|
+
"schema_mapping",
|
|
791
|
+
policy({
|
|
792
|
+
rule: "Translate natural language into precise SQL grounded in available schema entities."
|
|
793
|
+
}),
|
|
794
|
+
hint("Preserve schema spelling exactly, including typos in column names.")
|
|
795
|
+
),
|
|
796
|
+
fragment(
|
|
797
|
+
"projection_minimality",
|
|
798
|
+
policy({
|
|
799
|
+
rule: "Return only columns requested by the question; do not add helper columns unless explicitly requested."
|
|
800
|
+
}),
|
|
801
|
+
policy({
|
|
802
|
+
rule: 'For requests of the form "X sorted/ordered by Y", project X only unless Y is explicitly requested as an output field.'
|
|
803
|
+
}),
|
|
804
|
+
policy({
|
|
805
|
+
rule: "Prefer selecting schema columns directly without derived expressions when direct selection answers the request."
|
|
806
|
+
}),
|
|
807
|
+
hint(
|
|
808
|
+
"Do not include ORDER BY, GROUP BY, or JOIN helper columns in SELECT output unless the question explicitly asks for them."
|
|
809
|
+
),
|
|
810
|
+
policy({
|
|
811
|
+
rule: "Use DISTINCT only when uniqueness is explicitly requested (for example distinct/unique/different/no duplicates)."
|
|
812
|
+
}),
|
|
813
|
+
hint(
|
|
814
|
+
'Do not infer DISTINCT from generic wording such as "some", plural nouns, or entity-set phrasing; for transactional/attendance-style tables, default to raw rows unless uniqueness is explicitly requested.'
|
|
815
|
+
)
|
|
816
|
+
),
|
|
817
|
+
fragment(
|
|
818
|
+
"date_transform_safety",
|
|
819
|
+
policy({
|
|
820
|
+
rule: "Do not assume VARCHAR/TEXT values are parseable dates. Avoid date extraction functions on text columns by default."
|
|
821
|
+
}),
|
|
822
|
+
policy({
|
|
823
|
+
rule: "Use date-part extraction only when both conditions hold: the question explicitly asks for transformation and schema values require transformation to produce that unit."
|
|
824
|
+
}),
|
|
825
|
+
hint(
|
|
826
|
+
"Do not apply SUBSTR, STRFTIME, DATE_PART, YEAR, or similar extraction functions unless the question explicitly asks for transformation and schema values require it."
|
|
827
|
+
),
|
|
828
|
+
hint(
|
|
829
|
+
"If a column already represents the requested concept (for example a stored year-like value), use the column as-is."
|
|
830
|
+
)
|
|
831
|
+
),
|
|
832
|
+
fragment(
|
|
833
|
+
"sql_minimality",
|
|
834
|
+
guardrail({
|
|
835
|
+
rule: "Never hallucinate tables or columns.",
|
|
836
|
+
reason: "Schema fidelity is required.",
|
|
837
|
+
action: "Use only available schema entities."
|
|
838
|
+
}),
|
|
839
|
+
guardrail({
|
|
840
|
+
rule: "Avoid unnecessary transformations and derived projections.",
|
|
841
|
+
reason: "Extra transformations frequently change semantics and reduce correctness.",
|
|
842
|
+
action: "Do not add date parsing, substring extraction, or derived columns unless explicitly required by the question or schema."
|
|
843
|
+
})
|
|
844
|
+
),
|
|
845
|
+
fragment(
|
|
846
|
+
"preflight_checklist",
|
|
847
|
+
workflow({
|
|
848
|
+
task: "Final SQL preflight before returning output",
|
|
849
|
+
steps: [
|
|
850
|
+
"Verify selected columns match the question and remove unrequested helper projections.",
|
|
851
|
+
"If aggregate values are used only for ranking/filtering, keep them out of SELECT unless explicitly requested.",
|
|
852
|
+
"Prefer raw schema columns over derived expressions when raw columns already satisfy the request.",
|
|
853
|
+
"If a candidate query uses STRFTIME, SUBSTR, DATE_PART, YEAR, or similar extraction on text-like columns, remove that transformation unless explicitly required by the question.",
|
|
854
|
+
"Return only schema-grounded SQL using existing tables and columns."
|
|
855
|
+
]
|
|
856
|
+
})
|
|
857
|
+
),
|
|
858
|
+
fragment(
|
|
859
|
+
"set_semantics",
|
|
860
|
+
policy({
|
|
861
|
+
rule: "For questions asking where both condition A and condition B hold over an attribute, compute the intersection of qualifying sets for that attribute."
|
|
862
|
+
}),
|
|
863
|
+
policy({
|
|
864
|
+
rule: "Do not force the same entity instance to satisfy both conditions unless the question explicitly requests the same person/row/entity."
|
|
865
|
+
}),
|
|
866
|
+
hint(
|
|
867
|
+
"Prefer INTERSECT (or logically equivalent set-based shape) over requiring the same physical row/entity to satisfy both conditions unless explicitly requested."
|
|
868
|
+
),
|
|
869
|
+
hint(
|
|
870
|
+
"When two conditions describe different row groups whose shared attribute is requested, build each group separately and intersect the attribute values."
|
|
871
|
+
),
|
|
872
|
+
hint(
|
|
873
|
+
"Do not collapse cross-group conditions into a single-row AND predicate when the intent is shared values across groups."
|
|
874
|
+
),
|
|
875
|
+
policy({
|
|
876
|
+
rule: "If two predicates on the same field cannot both be true for one row, do not combine them with AND; use set operations across separate filtered subsets when shared values are requested."
|
|
877
|
+
})
|
|
878
|
+
),
|
|
879
|
+
fragment(
|
|
880
|
+
"predicate_column_alignment",
|
|
881
|
+
policy({
|
|
882
|
+
rule: "Match literal values to semantically compatible columns. Do not compare descriptive names to identifier columns."
|
|
883
|
+
}),
|
|
884
|
+
hint(
|
|
885
|
+
"When a filter value is a descriptive label (for example a department name), join through the lookup table and filter on its name/title column, not on *_id columns."
|
|
886
|
+
),
|
|
887
|
+
hint(
|
|
888
|
+
"When relation roles are explicit in wording (for example host/home/source/destination), prefer foreign keys with matching role qualifiers over generic similarly named columns."
|
|
889
|
+
),
|
|
890
|
+
policy({
|
|
891
|
+
rule: "When multiple foreign-key candidates exist, select the column whose qualifier best matches the relationship described in the question."
|
|
892
|
+
}),
|
|
893
|
+
policy({
|
|
894
|
+
rule: "For hosting/held semantics, prefer host_* relationship columns when available over generic *_id alternatives."
|
|
895
|
+
}),
|
|
896
|
+
hint(
|
|
897
|
+
'Interpret wording like "held/hosted a competition or event" as a hosting relationship and map to host_* foreign keys when present.'
|
|
898
|
+
),
|
|
899
|
+
policy({
|
|
900
|
+
rule: "Do not compare descriptive labels or names to *_id columns; join to the table containing the descriptive field and filter there."
|
|
901
|
+
}),
|
|
902
|
+
policy({
|
|
903
|
+
rule: "Keep numeric identifiers unquoted when used as numeric equality filters unless schema indicates text identifiers."
|
|
904
|
+
}),
|
|
905
|
+
policy({
|
|
906
|
+
rule: "When filtering by a descriptive label value and a related table exposes a corresponding *_name or title column, join to that table and filter on the descriptive column."
|
|
907
|
+
})
|
|
908
|
+
),
|
|
909
|
+
fragment(
|
|
910
|
+
"ordering_semantics",
|
|
911
|
+
policy({
|
|
912
|
+
rule: "Respect explicit sort direction terms. If direction is not specified, use ascending order unless a superlative intent (most/least/highest/lowest) implies direction."
|
|
913
|
+
}),
|
|
914
|
+
policy({
|
|
915
|
+
rule: "When ranking categories by frequency, use COUNT for ordering but keep output focused on requested category fields unless counts are explicitly requested."
|
|
916
|
+
}),
|
|
917
|
+
policy({
|
|
918
|
+
rule: "Do not use DESC unless descending direction is explicit or a superlative intent requires descending ranking."
|
|
919
|
+
}),
|
|
920
|
+
policy({
|
|
921
|
+
rule: 'For "most common/frequent <attribute>" requests, return the attribute value(s) only; use counts only for ordering/filtering unless the question explicitly asks to return counts.'
|
|
922
|
+
}),
|
|
923
|
+
hint(
|
|
924
|
+
'Use DESC with LIMIT 1 for "most/highest/largest"; use ASC with LIMIT 1 for "least/lowest/smallest".'
|
|
925
|
+
)
|
|
926
|
+
),
|
|
927
|
+
fragment(
|
|
928
|
+
"negative_membership_queries",
|
|
929
|
+
policy({
|
|
930
|
+
rule: "For requests asking entities that did not participate/host/appear in related records, prefer NOT IN or NOT EXISTS against the related foreign-key set."
|
|
931
|
+
}),
|
|
932
|
+
hint(
|
|
933
|
+
"Map role-bearing relationship columns carefully (for example host_* foreign keys for hosting relationships) instead of generic IDs when role wording is explicit."
|
|
934
|
+
),
|
|
935
|
+
hint(
|
|
936
|
+
'For "never had/never exceeded" conditions over history tables, exclude entities via NOT IN/NOT EXISTS against the disqualifying entity-id set (often built with GROUP BY/HAVING MAX(...)).'
|
|
937
|
+
)
|
|
938
|
+
),
|
|
939
|
+
fragment(
|
|
940
|
+
"join_completeness",
|
|
941
|
+
policy({
|
|
942
|
+
rule: "Preserve entity-restricting joins implied by the question. Do not widen results by querying only a broader attribute table when a subset entity table is available."
|
|
943
|
+
}),
|
|
944
|
+
policy({
|
|
945
|
+
rule: "If an entity term in the question maps to a table, keep that table in query scope and join to attribute tables rather than dropping the entity table."
|
|
946
|
+
}),
|
|
947
|
+
hint(
|
|
948
|
+
"If the question targets a specific entity group, include that entity table and its join conditions even when selected columns come from a related table."
|
|
949
|
+
),
|
|
950
|
+
hint(
|
|
951
|
+
"When the question names an entity type and a relation table links to that entity via *_id, include the entity table in scope instead of counting only relation rows."
|
|
952
|
+
),
|
|
953
|
+
hint(
|
|
954
|
+
"Prefer INNER JOIN by default; use LEFT JOIN only when the question explicitly requests including unmatched rows or zero-related entities."
|
|
955
|
+
)
|
|
956
|
+
),
|
|
957
|
+
fragment(
|
|
958
|
+
"aggregation_exactness",
|
|
959
|
+
policy({
|
|
960
|
+
rule: "Preserve requested aggregation semantics exactly: use COUNT(*) by default for total rows, use COUNT(DISTINCT ...) only when uniqueness is explicitly requested, and group by stable entity keys when computing per-entity aggregates."
|
|
961
|
+
}),
|
|
962
|
+
policy({
|
|
963
|
+
rule: "For questions asking which entity has lowest/highest average of a metric, compute AVG(metric) per entity (GROUP BY entity) and rank those aggregates."
|
|
964
|
+
}),
|
|
965
|
+
hint(
|
|
966
|
+
'For "how many <entities>" questions over relation records, default to COUNT(*) on qualifying rows unless explicit uniqueness language is present.'
|
|
967
|
+
)
|
|
968
|
+
),
|
|
969
|
+
fragment(
|
|
970
|
+
"query_shape_examples",
|
|
971
|
+
example({
|
|
972
|
+
question: "List categories ordered by how many records belong to each category.",
|
|
973
|
+
answer: "SELECT category FROM records GROUP BY category ORDER BY COUNT(*)"
|
|
974
|
+
}),
|
|
975
|
+
example({
|
|
976
|
+
question: "Show labels shared by rows with metric > 100 and rows with metric < 10.",
|
|
977
|
+
answer: "SELECT label FROM records WHERE metric > 100 INTERSECT SELECT label FROM records WHERE metric < 10"
|
|
978
|
+
}),
|
|
979
|
+
example({
|
|
980
|
+
question: "List locations that have not hosted any event.",
|
|
981
|
+
answer: "SELECT location_name FROM locations WHERE location_id NOT IN (SELECT host_location_id FROM events)"
|
|
982
|
+
}),
|
|
983
|
+
example({
|
|
984
|
+
question: "List the most common category across records.",
|
|
985
|
+
answer: "SELECT category FROM records GROUP BY category ORDER BY COUNT(*) DESC LIMIT 1"
|
|
986
|
+
})
|
|
987
|
+
)
|
|
988
|
+
];
|
|
989
|
+
function extractSql(output) {
|
|
990
|
+
const match = output.match(/```sql\n?([\s\S]*?)```/);
|
|
991
|
+
return match ? match[1].trim() : output.trim();
|
|
757
992
|
}
|
|
758
|
-
function
|
|
759
|
-
const
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
};
|
|
993
|
+
function scopeDialectFor(adapter) {
|
|
994
|
+
const dialect = FORMATTER_SCOPE_DIALECTS.get(adapter.formatterLanguage);
|
|
995
|
+
if (!dialect) {
|
|
996
|
+
throw new TypeError(
|
|
997
|
+
`No scope dialect mapping for formatter language "${adapter.formatterLanguage}".`
|
|
998
|
+
);
|
|
765
999
|
}
|
|
766
|
-
|
|
767
|
-
if (!adapter) {
|
|
768
|
-
return {
|
|
769
|
-
error: `sql ${subcommand}: unknown database "${name}". Available: ${available}`
|
|
770
|
-
};
|
|
771
|
-
}
|
|
772
|
-
const sql = rest.join(" ").trim();
|
|
773
|
-
if (!sql) {
|
|
774
|
-
return { error: `sql ${subcommand}: no query provided` };
|
|
775
|
-
}
|
|
776
|
-
return { adapter, sql };
|
|
1000
|
+
return dialect;
|
|
777
1001
|
}
|
|
778
|
-
function
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
1002
|
+
function fencedMarkdownParseError(sql, adapter) {
|
|
1003
|
+
return new SQLValidationError(
|
|
1004
|
+
JSON.stringify(
|
|
1005
|
+
buildScopeParseErrorPayload(
|
|
1006
|
+
sql,
|
|
1007
|
+
scopeDialectFor(adapter),
|
|
1008
|
+
new Error("SQL response was fenced markdown, not executable SQL.")
|
|
1009
|
+
)
|
|
1010
|
+
)
|
|
1011
|
+
);
|
|
1012
|
+
}
|
|
1013
|
+
async function toSql(options) {
|
|
1014
|
+
const { maxRetries = 3 } = options;
|
|
1015
|
+
return withRetry(
|
|
1016
|
+
async (attemptNumber, errors, attempts) => {
|
|
1017
|
+
const context = new ContextEngine({
|
|
1018
|
+
store: new InMemoryContextStore(),
|
|
1019
|
+
chatId: `sql-gen-${crypto.randomUUID()}`,
|
|
1020
|
+
userId: "system"
|
|
1021
|
+
});
|
|
1022
|
+
context.set(
|
|
1023
|
+
persona({
|
|
1024
|
+
name: "Freya",
|
|
1025
|
+
role: SQL_AGENT_ROLE,
|
|
1026
|
+
objective: SQL_AGENT_OBJECTIVE
|
|
1027
|
+
// role: `You are a data science expert that provides well-reasoned and detailed responses.`,
|
|
1028
|
+
// objective: `Your task is to understand the schema and generate a valid SQL query to answer the question. You first think about the reasoning process as an internal monologue and then provide the user with the answer.`,
|
|
1029
|
+
}),
|
|
1030
|
+
...SQL_AGENT_POLICIES,
|
|
1031
|
+
...options.fragments
|
|
1032
|
+
);
|
|
1033
|
+
if (errors.length) {
|
|
1034
|
+
const lastError = errors.at(-1);
|
|
1035
|
+
context.set(
|
|
1036
|
+
user(dedent`
|
|
1037
|
+
Answer the following question with the SQL code. Use the piece of evidence and base your answer on the database schema.
|
|
1038
|
+
Given the question, the evidence and the database schema, return the SQL script that addresses the question.
|
|
1039
|
+
|
|
1040
|
+
Question: ${options.input}
|
|
1041
|
+
`),
|
|
1042
|
+
UnanswerableSQLError.isInstance(lastError) ? user(
|
|
1043
|
+
`<retry_instruction>Your previous response marked the task as unanswerable. Re-evaluate using best-effort schema mapping. If the core intent is answerable with existing tables/columns, return SQL. Return error only when required core intent cannot be mapped without inventing schema elements.</retry_instruction>`
|
|
1044
|
+
) : user(
|
|
1045
|
+
`<validation_error>Your previous SQL query had the following error: ${lastError?.message}. Please fix the query.</validation_error>`
|
|
1046
|
+
)
|
|
1047
|
+
);
|
|
1048
|
+
} else {
|
|
1049
|
+
context.set(
|
|
1050
|
+
user(dedent`
|
|
1051
|
+
Answer the following question with the SQL code. Use the piece of evidence and base your answer on the database schema.
|
|
1052
|
+
Given the question, the evidence and the database schema, return the SQL script that addresses the question.
|
|
1053
|
+
|
|
1054
|
+
Question: ${options.input}
|
|
1055
|
+
`)
|
|
1056
|
+
);
|
|
1057
|
+
}
|
|
1058
|
+
const temperature = RETRY_TEMPERATURES[attemptNumber - 1] ?? RETRY_TEMPERATURES[RETRY_TEMPERATURES.length - 1];
|
|
1059
|
+
const baseModel = options.model ?? groq("openai/gpt-oss-20b");
|
|
1060
|
+
const model = wrapLanguageModel({
|
|
1061
|
+
model: baseModel,
|
|
1062
|
+
middleware: defaultSettingsMiddleware({ settings: { temperature } })
|
|
1063
|
+
});
|
|
1064
|
+
const sqlOutput = structuredOutput({
|
|
1065
|
+
model,
|
|
1066
|
+
context,
|
|
1067
|
+
schema: z.object({
|
|
1068
|
+
result: z.union([
|
|
1069
|
+
z.object({
|
|
1070
|
+
sql: z.string().describe("The SQL query that answers the question"),
|
|
1071
|
+
reasoning: z.string().describe("The reasoning steps taken to generate the SQL")
|
|
1072
|
+
}),
|
|
1073
|
+
z.object({
|
|
1074
|
+
error: z.string().describe(
|
|
1075
|
+
"Error message explaining why the question cannot be answered with the given schema"
|
|
1076
|
+
)
|
|
1077
|
+
})
|
|
1078
|
+
])
|
|
1079
|
+
})
|
|
1080
|
+
});
|
|
1081
|
+
const { result: output } = await sqlOutput.generate();
|
|
1082
|
+
const finalizeSql = async (rawSql) => {
|
|
1083
|
+
const extractedSql = extractSql(rawSql);
|
|
1084
|
+
const errorsOrUndefined = errors.length ? errors.map(formatErrorMessage) : void 0;
|
|
1085
|
+
if (extractedSql === "") {
|
|
833
1086
|
return {
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
1087
|
+
attempts,
|
|
1088
|
+
sql: "",
|
|
1089
|
+
errors: errorsOrUndefined
|
|
837
1090
|
};
|
|
838
1091
|
}
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
usage: 'validate <db> "SELECT ..."',
|
|
843
|
-
description: "Validate query syntax against <db>",
|
|
844
|
-
repair: repairDbNameAndQuotedArg,
|
|
845
|
-
handler: async (args) => {
|
|
846
|
-
const meta = useBashMeta();
|
|
847
|
-
const resolved = resolveAdapter(adapters, "validate", args);
|
|
848
|
-
if ("error" in resolved) {
|
|
849
|
-
return { stdout: "", stderr: resolved.error, exitCode: 1 };
|
|
1092
|
+
const sql = options.adapter.format(extractedSql);
|
|
1093
|
+
if (rawSql.trimStart().startsWith("```") && extractedSql === rawSql.trim()) {
|
|
1094
|
+
throw fencedMarkdownParseError(sql, options.adapter);
|
|
850
1095
|
}
|
|
851
|
-
const
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
const syntaxError = await adapter.validate(query);
|
|
855
|
-
if (syntaxError) {
|
|
856
|
-
return {
|
|
857
|
-
stdout: "",
|
|
858
|
-
stderr: `sql validate: ${syntaxError}`,
|
|
859
|
-
exitCode: 1
|
|
860
|
-
};
|
|
1096
|
+
const validationError = await options.adapter.validate(sql);
|
|
1097
|
+
if (validationError) {
|
|
1098
|
+
throw new SQLValidationError(validationError);
|
|
861
1099
|
}
|
|
862
1100
|
return {
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
1101
|
+
attempts,
|
|
1102
|
+
sql,
|
|
1103
|
+
errors: errorsOrUndefined
|
|
866
1104
|
};
|
|
1105
|
+
};
|
|
1106
|
+
if ("error" in output) {
|
|
1107
|
+
context.set(
|
|
1108
|
+
user(
|
|
1109
|
+
"<best_effort_fallback>Do not return unanswerable. Produce the best valid SQL query that answers the core intent using only available schema entities.</best_effort_fallback>"
|
|
1110
|
+
)
|
|
1111
|
+
);
|
|
1112
|
+
const forcedSqlOutput = structuredOutput({
|
|
1113
|
+
model,
|
|
1114
|
+
context,
|
|
1115
|
+
schema: z.object({
|
|
1116
|
+
sql: z.string().describe(
|
|
1117
|
+
"Best-effort SQL query that answers the core intent using only available schema entities."
|
|
1118
|
+
),
|
|
1119
|
+
reasoning: z.string().describe("Reasoning steps for best-effort schema mapping.")
|
|
1120
|
+
})
|
|
1121
|
+
});
|
|
1122
|
+
try {
|
|
1123
|
+
const forced = await forcedSqlOutput.generate();
|
|
1124
|
+
return await finalizeSql(forced.sql);
|
|
1125
|
+
} catch (error) {
|
|
1126
|
+
if (SQLValidationError.isInstance(error) || APICallError.isInstance(error) || JSONParseError.isInstance(error) || TypeValidationError.isInstance(error) || NoObjectGeneratedError.isInstance(error) || NoOutputGeneratedError.isInstance(error) || NoContentGeneratedError.isInstance(error)) {
|
|
1127
|
+
throw error;
|
|
1128
|
+
}
|
|
1129
|
+
throw new UnanswerableSQLError(output.error);
|
|
1130
|
+
}
|
|
867
1131
|
}
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
return { command, repair };
|
|
873
|
-
}
|
|
874
|
-
|
|
875
|
-
// packages/text2sql/src/lib/agents/sql-transform-plugins.ts
|
|
876
|
-
import { parse, serialize } from "just-bash";
|
|
877
|
-
import { createHash } from "node:crypto";
|
|
878
|
-
import * as path from "node:path";
|
|
879
|
-
import { BashException, asStaticWordText } from "@deepagents/context";
|
|
880
|
-
function isScriptNode(value) {
|
|
881
|
-
if (typeof value !== "object" || value === null) {
|
|
882
|
-
return false;
|
|
883
|
-
}
|
|
884
|
-
const node = value;
|
|
885
|
-
return node.type === "Script" && Array.isArray(node.statements);
|
|
886
|
-
}
|
|
887
|
-
var BLOCKED_DB_CLIENT_COMMANDS = /* @__PURE__ */ new Set([
|
|
888
|
-
"psql",
|
|
889
|
-
"sqlite3",
|
|
890
|
-
"mysql",
|
|
891
|
-
"duckdb"
|
|
892
|
-
]);
|
|
893
|
-
var BLOCKED_RAW_SQL_COMMANDS = /* @__PURE__ */ new Set(["select", "with"]);
|
|
894
|
-
var ALLOWED_SQL_PROXY_SUBCOMMANDS = /* @__PURE__ */ new Set(["run", "validate"]);
|
|
895
|
-
var SHELL_INTERPRETER_COMMANDS = /* @__PURE__ */ new Set([
|
|
896
|
-
"bash",
|
|
897
|
-
"sh",
|
|
898
|
-
"zsh",
|
|
899
|
-
"dash",
|
|
900
|
-
"ksh"
|
|
901
|
-
]);
|
|
902
|
-
var WRAPPER_COMMANDS = /* @__PURE__ */ new Set(["env", "command", "eval"]);
|
|
903
|
-
var SQL_PROXY_ENFORCEMENT_MESSAGE = [
|
|
904
|
-
"Direct database querying through bash is blocked.",
|
|
905
|
-
"Use SQL proxy commands in this order:",
|
|
906
|
-
'1) sql validate <db> "SELECT ..."',
|
|
907
|
-
'2) sql run <db> "SELECT ..."'
|
|
908
|
-
].join("\n");
|
|
909
|
-
var SqlProxyViolationError = class extends BashException {
|
|
910
|
-
constructor() {
|
|
911
|
-
super(SQL_PROXY_ENFORCEMENT_MESSAGE);
|
|
912
|
-
this.name = "SqlProxyViolationError";
|
|
913
|
-
}
|
|
914
|
-
format() {
|
|
915
|
-
return { stdout: "", stderr: `${this.message}
|
|
916
|
-
`, exitCode: 1 };
|
|
917
|
-
}
|
|
918
|
-
};
|
|
919
|
-
function cloneInspectionContext(context) {
|
|
920
|
-
return {
|
|
921
|
-
functionDefinitions: new Map(context.functionDefinitions),
|
|
922
|
-
callStack: new Set(context.callStack)
|
|
923
|
-
};
|
|
924
|
-
}
|
|
925
|
-
function scriptContainsBlockedCommand(script, context, mode = "blocked-only") {
|
|
926
|
-
return statementsContainBlockedCommand(script.statements, context, mode);
|
|
927
|
-
}
|
|
928
|
-
function statementsContainBlockedCommand(statements, context, mode) {
|
|
929
|
-
for (const statement of statements) {
|
|
930
|
-
if (statementContainsBlockedCommand(statement, context, mode)) {
|
|
931
|
-
return true;
|
|
932
|
-
}
|
|
933
|
-
}
|
|
934
|
-
return false;
|
|
935
|
-
}
|
|
936
|
-
function statementContainsBlockedCommand(statement, context, mode) {
|
|
937
|
-
for (const pipeline of statement.pipelines) {
|
|
938
|
-
if (pipelineContainsBlockedCommand(pipeline, context, mode)) {
|
|
939
|
-
return true;
|
|
940
|
-
}
|
|
941
|
-
}
|
|
942
|
-
return false;
|
|
1132
|
+
return await finalizeSql(output.sql);
|
|
1133
|
+
},
|
|
1134
|
+
{ retries: maxRetries - 1 }
|
|
1135
|
+
);
|
|
943
1136
|
}
|
|
944
|
-
function
|
|
945
|
-
|
|
946
|
-
if (
|
|
947
|
-
|
|
948
|
-
continue;
|
|
949
|
-
}
|
|
950
|
-
if (commandContainsBlockedCommand(command, context, mode, {
|
|
951
|
-
stdinFromPipe: index2 > 0
|
|
952
|
-
})) {
|
|
953
|
-
return true;
|
|
1137
|
+
function formatErrorMessage(error) {
|
|
1138
|
+
if (APICallError.isInstance(error)) {
|
|
1139
|
+
if (error.message.startsWith("Failed to validate JSON")) {
|
|
1140
|
+
return `Schema validation failed: ${error.message}`;
|
|
954
1141
|
}
|
|
1142
|
+
return error.message;
|
|
955
1143
|
}
|
|
956
|
-
|
|
957
|
-
}
|
|
958
|
-
function stringCommandContainsBlockedCommand(command, context, mode = "blocked-only") {
|
|
959
|
-
let script;
|
|
960
|
-
try {
|
|
961
|
-
script = parse(command);
|
|
962
|
-
} catch {
|
|
963
|
-
return false;
|
|
1144
|
+
if (SQLValidationError.isInstance(error)) {
|
|
1145
|
+
return `SQL Validation Error: ${error.message}`;
|
|
964
1146
|
}
|
|
965
|
-
return
|
|
966
|
-
script,
|
|
967
|
-
cloneInspectionContext(context),
|
|
968
|
-
mode
|
|
969
|
-
);
|
|
1147
|
+
return error.message;
|
|
970
1148
|
}
|
|
971
|
-
function
|
|
972
|
-
if (!
|
|
1149
|
+
function isModelUnavailableError(error) {
|
|
1150
|
+
if (!APICallError.isInstance(error)) {
|
|
973
1151
|
return false;
|
|
974
1152
|
}
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
);
|
|
980
|
-
}
|
|
981
|
-
function wordPartContainsBlockedCommand(parts, context, mode) {
|
|
982
|
-
for (const part of parts) {
|
|
983
|
-
if (partContainsBlockedCommand(part, context, mode)) {
|
|
984
|
-
return true;
|
|
985
|
-
}
|
|
986
|
-
}
|
|
987
|
-
return false;
|
|
1153
|
+
const message = error.message.toLowerCase();
|
|
1154
|
+
const responseBody = (error.responseBody ?? "").toLowerCase();
|
|
1155
|
+
const is404ModelError = error.statusCode === 404 && (message.includes("model") || responseBody.includes("model_not_found"));
|
|
1156
|
+
const errorCode = typeof error.data === "object" && error.data !== null && "error" in error.data && typeof error.data.error === "object" && error.data.error !== null && "code" in error.data.error && typeof error.data.error.code === "string" ? error.data.error.code.toLowerCase() : void 0;
|
|
1157
|
+
return is404ModelError || errorCode === "model_not_found" || responseBody.includes('"code":"model_not_found"') || message.includes("model") && message.includes("does not exist or you do not have access to it");
|
|
988
1158
|
}
|
|
989
|
-
function
|
|
990
|
-
const
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
if (type === "ArithCommandSubst" && typeof node.command === "string") {
|
|
1002
|
-
return stringCommandContainsBlockedCommand(node.command, context, mode);
|
|
1003
|
-
}
|
|
1004
|
-
for (const value of Object.values(node)) {
|
|
1005
|
-
if (Array.isArray(value)) {
|
|
1006
|
-
for (const item of value) {
|
|
1007
|
-
if (typeof item === "object" && item !== null) {
|
|
1008
|
-
if (partContainsBlockedCommand(
|
|
1009
|
-
item,
|
|
1010
|
-
context,
|
|
1011
|
-
mode
|
|
1012
|
-
)) {
|
|
1013
|
-
return true;
|
|
1014
|
-
}
|
|
1159
|
+
async function withRetry(computation, options = { retries: 3 }) {
|
|
1160
|
+
const errors = [];
|
|
1161
|
+
let attempts = 0;
|
|
1162
|
+
return pRetry(
|
|
1163
|
+
(attemptNumber) => {
|
|
1164
|
+
return computation(attemptNumber, errors, ++attempts);
|
|
1165
|
+
},
|
|
1166
|
+
{
|
|
1167
|
+
retries: options.retries,
|
|
1168
|
+
shouldRetry: (context) => {
|
|
1169
|
+
if (UnanswerableSQLError.isInstance(context.error)) {
|
|
1170
|
+
return false;
|
|
1015
1171
|
}
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1172
|
+
if (isModelUnavailableError(context.error)) {
|
|
1173
|
+
return false;
|
|
1174
|
+
}
|
|
1175
|
+
if (SQLValidationError.isInstance(context.error)) {
|
|
1176
|
+
return true;
|
|
1177
|
+
}
|
|
1178
|
+
console.log({
|
|
1179
|
+
NoObjectGeneratedError: NoObjectGeneratedError.isInstance(
|
|
1180
|
+
context.error
|
|
1181
|
+
),
|
|
1182
|
+
NoOutputGeneratedError: NoOutputGeneratedError.isInstance(
|
|
1183
|
+
context.error
|
|
1184
|
+
),
|
|
1185
|
+
APICallError: APICallError.isInstance(context.error),
|
|
1186
|
+
JSONParseError: JSONParseError.isInstance(context.error),
|
|
1187
|
+
TypeValidationError: TypeValidationError.isInstance(context.error),
|
|
1188
|
+
NoContentGeneratedError: NoContentGeneratedError.isInstance(
|
|
1189
|
+
context.error
|
|
1190
|
+
)
|
|
1191
|
+
});
|
|
1192
|
+
return APICallError.isInstance(context.error) || JSONParseError.isInstance(context.error) || TypeValidationError.isInstance(context.error) || NoObjectGeneratedError.isInstance(context.error) || NoOutputGeneratedError.isInstance(context.error) || NoContentGeneratedError.isInstance(context.error);
|
|
1193
|
+
},
|
|
1194
|
+
onFailedAttempt(context) {
|
|
1195
|
+
errors.push(context.error);
|
|
1026
1196
|
}
|
|
1027
1197
|
}
|
|
1028
|
-
}
|
|
1029
|
-
return false;
|
|
1030
|
-
}
|
|
1031
|
-
function functionInvocationContainsBlockedCommand(functionName, context, mode) {
|
|
1032
|
-
const definition = context.functionDefinitions.get(functionName);
|
|
1033
|
-
if (!definition) {
|
|
1034
|
-
return false;
|
|
1035
|
-
}
|
|
1036
|
-
if (context.callStack.has(functionName)) {
|
|
1037
|
-
return false;
|
|
1038
|
-
}
|
|
1039
|
-
const invocationContext = cloneInspectionContext(context);
|
|
1040
|
-
invocationContext.callStack.add(functionName);
|
|
1041
|
-
return commandContainsBlockedCommand(
|
|
1042
|
-
definition.body,
|
|
1043
|
-
invocationContext,
|
|
1044
|
-
mode,
|
|
1045
|
-
{ stdinFromPipe: false }
|
|
1046
1198
|
);
|
|
1047
1199
|
}
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1200
|
+
|
|
1201
|
+
// packages/text2sql/src/lib/agents/suggestions.agents.ts
|
|
1202
|
+
import { groq as groq2 } from "@ai-sdk/groq";
|
|
1203
|
+
import dedent2 from "dedent";
|
|
1204
|
+
import z2 from "zod";
|
|
1205
|
+
import { agent, thirdPersonPrompt } from "@deepagents/agent";
|
|
1206
|
+
var suggestionsAgent = agent({
|
|
1207
|
+
name: "text2sql-suggestions",
|
|
1208
|
+
model: groq2("openai/gpt-oss-20b"),
|
|
1209
|
+
output: z2.object({
|
|
1210
|
+
suggestions: z2.array(
|
|
1211
|
+
z2.object({
|
|
1212
|
+
question: z2.string().describe("A complex, high-impact business question."),
|
|
1213
|
+
sql: z2.string().describe("The SQL statement needed to answer the question."),
|
|
1214
|
+
businessValue: z2.string().describe("Why the question matters to stakeholders.")
|
|
1215
|
+
})
|
|
1216
|
+
).min(1).max(5).describe("A set of up to two advanced question + SQL pairs.")
|
|
1217
|
+
}),
|
|
1218
|
+
prompt: (state) => {
|
|
1219
|
+
return dedent2`
|
|
1220
|
+
${thirdPersonPrompt()}
|
|
1221
|
+
|
|
1222
|
+
<identity>
|
|
1223
|
+
You are a senior analytics strategist who proposes ambitious business questions
|
|
1224
|
+
and drafts the SQL needed to answer them. You specialize in identifying ideas
|
|
1225
|
+
that combine multiple tables, apply segmentation or time analysis, and surface
|
|
1226
|
+
metrics that drive executive decisions.
|
|
1227
|
+
</identity>
|
|
1228
|
+
|
|
1229
|
+
|
|
1230
|
+
<instructions>
|
|
1231
|
+
- Recommend one or two UNIQUE questions that go beyond simple counts or listings.
|
|
1232
|
+
- Favor questions that require joins, aggregates, time comparisons, cohort analysis,
|
|
1233
|
+
or window functions.
|
|
1234
|
+
- For each question, explain the business reason stakeholders care about it.
|
|
1235
|
+
- Provide the complete SQL query that could answer the question in the given schema.
|
|
1236
|
+
- Keep result sets scoped with LIMIT clauses (max 50 rows) when returning raw rows.
|
|
1237
|
+
- Ensure table/column names match the provided schema exactly.
|
|
1238
|
+
- Use columns marked [LowCardinality: ...] to identify meaningful categorical filters or segmentations.
|
|
1239
|
+
- Leverage table [rows / size] hints to determine whether to aggregate (large tables) or inspect detailed data (tiny tables).
|
|
1240
|
+
- Reference PK/Indexed annotations and the Indexes list to recommend queries that use efficient join/filter paths.
|
|
1241
|
+
- Column annotations may expose ranges/null percentages—use them to suggest realistic thresholds or quality checks.
|
|
1242
|
+
- Consult <relationship_examples> to anchor your recommendations in the actual join paths between tables.
|
|
1243
|
+
- Output only information grounded in the schema/context provided.
|
|
1244
|
+
</instructions>
|
|
1245
|
+
|
|
1246
|
+
<response-format>
|
|
1247
|
+
Return valid JSON that satisfies the defined output schema.
|
|
1248
|
+
</response-format>
|
|
1249
|
+
`;
|
|
1059
1250
|
}
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1251
|
+
});
|
|
1252
|
+
|
|
1253
|
+
// packages/text2sql/src/lib/adapter-index.ts
|
|
1254
|
+
import { fragment as fragment2 } from "@deepagents/context";
|
|
1255
|
+
|
|
1256
|
+
// packages/text2sql/src/lib/file-cache.ts
|
|
1257
|
+
import { createHash } from "node:crypto";
|
|
1258
|
+
import { existsSync } from "node:fs";
|
|
1259
|
+
import { readFile, writeFile } from "node:fs/promises";
|
|
1260
|
+
import { tmpdir } from "node:os";
|
|
1261
|
+
import path from "node:path";
|
|
1262
|
+
var FileCache = class {
|
|
1263
|
+
path;
|
|
1264
|
+
constructor(watermark, extension = ".txt") {
|
|
1265
|
+
const hash2 = createHash("md5").update(watermark).digest("hex");
|
|
1266
|
+
this.path = path.join(tmpdir(), `text2sql-${hash2}${extension}`);
|
|
1063
1267
|
}
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
return false;
|
|
1268
|
+
async get() {
|
|
1269
|
+
if (existsSync(this.path)) {
|
|
1270
|
+
return readFile(this.path, "utf-8");
|
|
1068
1271
|
}
|
|
1272
|
+
return null;
|
|
1069
1273
|
}
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
function isEnvAssignmentToken(token) {
|
|
1073
|
-
const separatorIndex = token.indexOf("=");
|
|
1074
|
-
if (separatorIndex <= 0) {
|
|
1075
|
-
return false;
|
|
1274
|
+
set(content) {
|
|
1275
|
+
return writeFile(this.path, content, "utf-8");
|
|
1076
1276
|
}
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
return {
|
|
1082
|
-
valid: false,
|
|
1083
|
-
hasCommandFlag: false,
|
|
1084
|
-
hasStdinFlag: false,
|
|
1085
|
-
consumesNextArg: false
|
|
1086
|
-
};
|
|
1277
|
+
};
|
|
1278
|
+
var JsonCache = class extends FileCache {
|
|
1279
|
+
constructor(watermark) {
|
|
1280
|
+
super(watermark, ".json");
|
|
1087
1281
|
}
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
const char = option[index2];
|
|
1093
|
-
if (!isAsciiLetter(char)) {
|
|
1094
|
-
return {
|
|
1095
|
-
valid: false,
|
|
1096
|
-
hasCommandFlag: false,
|
|
1097
|
-
hasStdinFlag: false,
|
|
1098
|
-
consumesNextArg: false
|
|
1099
|
-
};
|
|
1100
|
-
}
|
|
1101
|
-
if (char === "c") {
|
|
1102
|
-
hasCommandFlag = true;
|
|
1103
|
-
} else if (char === "s") {
|
|
1104
|
-
hasStdinFlag = true;
|
|
1105
|
-
} else if (char === "O" || char === "o") {
|
|
1106
|
-
consumesNextArg = true;
|
|
1282
|
+
async read() {
|
|
1283
|
+
const content = await this.get();
|
|
1284
|
+
if (content) {
|
|
1285
|
+
return JSON.parse(content);
|
|
1107
1286
|
}
|
|
1287
|
+
return null;
|
|
1108
1288
|
}
|
|
1109
|
-
|
|
1289
|
+
write(data) {
|
|
1290
|
+
return this.set(JSON.stringify(data));
|
|
1291
|
+
}
|
|
1292
|
+
};
|
|
1293
|
+
|
|
1294
|
+
// packages/text2sql/src/lib/adapter-index.ts
|
|
1295
|
+
var TEXT2SQL_INDEX_PROGRESS_CHUNK = "data-text2sql-index-progress";
|
|
1296
|
+
var noopProgressHandler = () => {
|
|
1297
|
+
};
|
|
1298
|
+
function timestampProgressHandler(onProgress) {
|
|
1299
|
+
if (!onProgress) return noopProgressHandler;
|
|
1300
|
+
return (event) => {
|
|
1301
|
+
onProgress({
|
|
1302
|
+
...event,
|
|
1303
|
+
timestampMs: event.timestampMs ?? Date.now()
|
|
1304
|
+
});
|
|
1305
|
+
};
|
|
1110
1306
|
}
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
if (token === "--") {
|
|
1120
|
-
if (index2 + 1 >= args.length) {
|
|
1121
|
-
break;
|
|
1122
|
-
}
|
|
1123
|
-
return {
|
|
1124
|
-
kind: "script",
|
|
1125
|
-
payload: asStaticWordText(args[index2 + 1])
|
|
1126
|
-
};
|
|
1307
|
+
var AdapterIndexer = class {
|
|
1308
|
+
#adapters;
|
|
1309
|
+
#version;
|
|
1310
|
+
#cacheKey;
|
|
1311
|
+
constructor(options) {
|
|
1312
|
+
const adapterNames = Object.keys(options.adapters);
|
|
1313
|
+
if (adapterNames.length === 0) {
|
|
1314
|
+
throw new Error("AdapterIndexer requires at least one adapter");
|
|
1127
1315
|
}
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1316
|
+
validateAdapterNames(adapterNames);
|
|
1317
|
+
this.#adapters = options.adapters;
|
|
1318
|
+
this.#version = options.version;
|
|
1319
|
+
this.#cacheKey = options.cacheKey;
|
|
1320
|
+
}
|
|
1321
|
+
async index(options = {}) {
|
|
1322
|
+
const progress = timestampProgressHandler(options.onProgress);
|
|
1323
|
+
const entries = this.#resolveIndexEntries(options.adapterNames);
|
|
1324
|
+
progress({
|
|
1325
|
+
type: "index:start",
|
|
1326
|
+
message: `Indexing ${entries.length} adapter${entries.length === 1 ? "" : "s"}...`,
|
|
1327
|
+
current: 0,
|
|
1328
|
+
total: entries.length
|
|
1329
|
+
});
|
|
1330
|
+
const settled = await Promise.allSettled(
|
|
1331
|
+
entries.map(async ([name, adapter]) => {
|
|
1332
|
+
const schema = await this.#indexAdapter(name, adapter, progress);
|
|
1333
|
+
return fragment2(name, ...schema);
|
|
1334
|
+
})
|
|
1335
|
+
);
|
|
1336
|
+
const failed = settled.find((result) => result.status === "rejected");
|
|
1337
|
+
if (failed) {
|
|
1338
|
+
progress({
|
|
1339
|
+
type: "index:error",
|
|
1340
|
+
message: failed.reason instanceof Error ? failed.reason.message : String(failed.reason)
|
|
1341
|
+
});
|
|
1342
|
+
throw failed.reason;
|
|
1133
1343
|
}
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
payload: token.slice("--command=".length)
|
|
1138
|
-
};
|
|
1139
|
-
}
|
|
1140
|
-
if (token.startsWith("--")) {
|
|
1141
|
-
if (token.includes("=")) {
|
|
1142
|
-
continue;
|
|
1143
|
-
}
|
|
1144
|
-
if (longOptionsWithValue.has(token)) {
|
|
1145
|
-
if (index2 + 1 >= args.length) {
|
|
1146
|
-
return { kind: "unknown", payload: null };
|
|
1147
|
-
}
|
|
1148
|
-
index2 += 1;
|
|
1344
|
+
const wrapped = settled.map((result) => {
|
|
1345
|
+
if (result.status === "rejected") {
|
|
1346
|
+
throw result.reason;
|
|
1149
1347
|
}
|
|
1150
|
-
|
|
1348
|
+
return result.value;
|
|
1349
|
+
});
|
|
1350
|
+
progress({
|
|
1351
|
+
type: "index:end",
|
|
1352
|
+
message: "Finished indexing adapters.",
|
|
1353
|
+
current: entries.length,
|
|
1354
|
+
total: entries.length
|
|
1355
|
+
});
|
|
1356
|
+
return wrapped;
|
|
1357
|
+
}
|
|
1358
|
+
async indexAdapter(name, options = {}) {
|
|
1359
|
+
const adapter = this.#adapters[name];
|
|
1360
|
+
if (!adapter) {
|
|
1361
|
+
const available = Object.keys(this.#adapters).join(", ");
|
|
1362
|
+
throw new Error(`unknown adapter "${name}". Available: ${available}`);
|
|
1151
1363
|
}
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
}
|
|
1163
|
-
|
|
1164
|
-
|
|
1364
|
+
return this.#indexAdapter(
|
|
1365
|
+
name,
|
|
1366
|
+
adapter,
|
|
1367
|
+
timestampProgressHandler(options.onProgress)
|
|
1368
|
+
);
|
|
1369
|
+
}
|
|
1370
|
+
async #indexAdapter(name, adapter, progress) {
|
|
1371
|
+
progress({
|
|
1372
|
+
type: "adapter:start",
|
|
1373
|
+
adapter: name,
|
|
1374
|
+
message: `Indexing adapter "${name}"...`
|
|
1375
|
+
});
|
|
1376
|
+
const cacheKey = this.#adapterCacheKey(name);
|
|
1377
|
+
const cache = cacheKey ? new JsonCache(cacheKey) : void 0;
|
|
1378
|
+
try {
|
|
1379
|
+
const cached = await cache?.read();
|
|
1380
|
+
if (cached) {
|
|
1381
|
+
progress({
|
|
1382
|
+
type: "adapter:cache-hit",
|
|
1383
|
+
adapter: name,
|
|
1384
|
+
message: `Using cached index for adapter "${name}".`,
|
|
1385
|
+
cached: true
|
|
1386
|
+
});
|
|
1387
|
+
progress({
|
|
1388
|
+
type: "adapter:end",
|
|
1389
|
+
adapter: name,
|
|
1390
|
+
message: `Finished indexing adapter "${name}".`,
|
|
1391
|
+
cached: true
|
|
1392
|
+
});
|
|
1393
|
+
return cached;
|
|
1165
1394
|
}
|
|
1166
|
-
if (
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1395
|
+
if (cache) {
|
|
1396
|
+
progress({
|
|
1397
|
+
type: "adapter:cache-miss",
|
|
1398
|
+
adapter: name,
|
|
1399
|
+
message: `No cached index for adapter "${name}".`,
|
|
1400
|
+
cached: false
|
|
1401
|
+
});
|
|
1171
1402
|
}
|
|
1172
|
-
|
|
1403
|
+
const ctx = createGroundingContext({
|
|
1404
|
+
onProgress: (event) => progress(adapterProgressEvent(name, event))
|
|
1405
|
+
});
|
|
1406
|
+
const fragments = await adapter.introspect(ctx);
|
|
1407
|
+
await cache?.write(fragments);
|
|
1408
|
+
progress({
|
|
1409
|
+
type: "adapter:end",
|
|
1410
|
+
adapter: name,
|
|
1411
|
+
message: `Finished indexing adapter "${name}".`,
|
|
1412
|
+
cached: false
|
|
1413
|
+
});
|
|
1414
|
+
return fragments;
|
|
1415
|
+
} catch (error) {
|
|
1416
|
+
const reason = errorMessage(error);
|
|
1417
|
+
progress({
|
|
1418
|
+
type: "adapter:error",
|
|
1419
|
+
adapter: name,
|
|
1420
|
+
message: `Failed indexing adapter "${name}": ${reason}`
|
|
1421
|
+
});
|
|
1422
|
+
throw new Error(`introspecting adapter "${name}": ${reason}`, {
|
|
1423
|
+
cause: error
|
|
1424
|
+
});
|
|
1173
1425
|
}
|
|
1174
|
-
return {
|
|
1175
|
-
kind: "script",
|
|
1176
|
-
payload: token
|
|
1177
|
-
};
|
|
1178
1426
|
}
|
|
1179
|
-
|
|
1180
|
-
|
|
1427
|
+
#resolveIndexEntries(adapterNames) {
|
|
1428
|
+
const availableNames = Object.keys(this.#adapters);
|
|
1429
|
+
const requestedNames = adapterNames && adapterNames.length > 0 ? [...new Set(adapterNames)] : availableNames;
|
|
1430
|
+
const available = availableNames.join(", ");
|
|
1431
|
+
return requestedNames.map((name) => {
|
|
1432
|
+
const adapter = this.#adapters[name];
|
|
1433
|
+
if (!adapter) {
|
|
1434
|
+
throw new Error(`unknown adapter "${name}". Available: ${available}`);
|
|
1435
|
+
}
|
|
1436
|
+
return [name, adapter];
|
|
1437
|
+
});
|
|
1181
1438
|
}
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
for (const redirection of redirections) {
|
|
1187
|
-
if (redirection.target.type !== "HereDoc") {
|
|
1188
|
-
continue;
|
|
1189
|
-
}
|
|
1190
|
-
if (!redirection.target.content) {
|
|
1191
|
-
payloads.push("");
|
|
1192
|
-
continue;
|
|
1439
|
+
#adapterCacheKey(name) {
|
|
1440
|
+
const configured = this.#cacheKey?.(name);
|
|
1441
|
+
if (configured !== void 0) {
|
|
1442
|
+
return configured;
|
|
1193
1443
|
}
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
return { hasHereDoc: true, payload: null };
|
|
1444
|
+
if (this.#version) {
|
|
1445
|
+
return `introspection-${this.#version}-${name}`;
|
|
1197
1446
|
}
|
|
1198
|
-
|
|
1199
|
-
}
|
|
1200
|
-
if (payloads.length === 0) {
|
|
1201
|
-
return { hasHereDoc: false, payload: null };
|
|
1447
|
+
return void 0;
|
|
1202
1448
|
}
|
|
1203
|
-
|
|
1449
|
+
};
|
|
1450
|
+
function adapterProgressEvent(adapter, progress) {
|
|
1451
|
+
return {
|
|
1452
|
+
type: progress.type,
|
|
1453
|
+
adapter,
|
|
1454
|
+
phase: progress.phase,
|
|
1455
|
+
table: progress.table,
|
|
1456
|
+
message: progress.message,
|
|
1457
|
+
current: progress.current,
|
|
1458
|
+
total: progress.total,
|
|
1459
|
+
cached: false,
|
|
1460
|
+
timestampMs: progress.timestampMs
|
|
1461
|
+
};
|
|
1204
1462
|
}
|
|
1205
|
-
function
|
|
1206
|
-
|
|
1207
|
-
for (const word of words) {
|
|
1208
|
-
const token = asStaticWordText(word);
|
|
1209
|
-
if (token == null) {
|
|
1210
|
-
return null;
|
|
1211
|
-
}
|
|
1212
|
-
tokens.push(token);
|
|
1213
|
-
}
|
|
1214
|
-
return tokens.join(" ");
|
|
1463
|
+
function errorMessage(error) {
|
|
1464
|
+
return error instanceof Error ? error.message : String(error);
|
|
1215
1465
|
}
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1466
|
+
|
|
1467
|
+
// packages/text2sql/src/lib/checkpoint.ts
|
|
1468
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
1469
|
+
import { existsSync as existsSync2, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
1470
|
+
import pLimit from "p-limit";
|
|
1471
|
+
var Checkpoint = class _Checkpoint {
|
|
1472
|
+
points;
|
|
1473
|
+
path;
|
|
1474
|
+
configHash;
|
|
1475
|
+
constructor(path6, configHash, points) {
|
|
1476
|
+
this.points = points;
|
|
1477
|
+
this.path = path6;
|
|
1478
|
+
this.configHash = configHash;
|
|
1479
|
+
}
|
|
1480
|
+
/**
|
|
1481
|
+
* Load checkpoint from file, or return empty checkpoint if none exists.
|
|
1482
|
+
* Handles corrupted files and config changes gracefully.
|
|
1483
|
+
*/
|
|
1484
|
+
static async load(options) {
|
|
1485
|
+
const { path: path6, configHash } = options;
|
|
1486
|
+
if (existsSync2(path6)) {
|
|
1487
|
+
try {
|
|
1488
|
+
const content = readFileSync(path6, "utf-8");
|
|
1489
|
+
const file = JSON.parse(content);
|
|
1490
|
+
if (configHash && file.configHash && file.configHash !== configHash) {
|
|
1491
|
+
console.log("\u26A0 Config changed, starting fresh");
|
|
1492
|
+
return new _Checkpoint(path6, configHash, {});
|
|
1493
|
+
}
|
|
1494
|
+
const points = file.points ?? {};
|
|
1495
|
+
const totalEntries = Object.values(points).reduce(
|
|
1496
|
+
(sum, p) => sum + p.entries.length,
|
|
1497
|
+
0
|
|
1498
|
+
);
|
|
1499
|
+
console.log(`\u2713 Resuming from checkpoint (${totalEntries} entries)`);
|
|
1500
|
+
return new _Checkpoint(path6, configHash, points);
|
|
1501
|
+
} catch {
|
|
1502
|
+
console.log("\u26A0 Checkpoint corrupted, starting fresh");
|
|
1503
|
+
return new _Checkpoint(path6, configHash, {});
|
|
1230
1504
|
}
|
|
1231
|
-
index2 += 2;
|
|
1232
|
-
continue;
|
|
1233
|
-
}
|
|
1234
|
-
if (token.startsWith("--unset=") || token.startsWith("--chdir=")) {
|
|
1235
|
-
index2 += 1;
|
|
1236
|
-
continue;
|
|
1237
|
-
}
|
|
1238
|
-
if (token.startsWith("-") && token !== "-" && !isEnvAssignmentToken(token)) {
|
|
1239
|
-
index2 += 1;
|
|
1240
|
-
continue;
|
|
1241
|
-
}
|
|
1242
|
-
if (isEnvAssignmentToken(token)) {
|
|
1243
|
-
index2 += 1;
|
|
1244
|
-
continue;
|
|
1245
1505
|
}
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
if (index2 >= args.length) {
|
|
1249
|
-
return { kind: "none" };
|
|
1506
|
+
console.log("Starting new checkpoint");
|
|
1507
|
+
return new _Checkpoint(path6, configHash, {});
|
|
1250
1508
|
}
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
if (token === "-v" || token === "-V") {
|
|
1270
|
-
lookupOnly = true;
|
|
1271
|
-
index2 += 1;
|
|
1272
|
-
continue;
|
|
1273
|
-
}
|
|
1274
|
-
if (token.startsWith("-") && token !== "-") {
|
|
1275
|
-
index2 += 1;
|
|
1276
|
-
continue;
|
|
1277
|
-
}
|
|
1278
|
-
break;
|
|
1509
|
+
/**
|
|
1510
|
+
* Run a single computation with checkpointing.
|
|
1511
|
+
* If already completed, returns cached value.
|
|
1512
|
+
*
|
|
1513
|
+
* @param key - Unique identifier for this computation
|
|
1514
|
+
* @param computation - Async function that produces the value
|
|
1515
|
+
* @param codec - Optional codec for encoding/decoding non-primitive values
|
|
1516
|
+
*/
|
|
1517
|
+
async run(key, computation, codec) {
|
|
1518
|
+
const point = this.point(key);
|
|
1519
|
+
return point.through(
|
|
1520
|
+
"single",
|
|
1521
|
+
async () => {
|
|
1522
|
+
const result = await computation();
|
|
1523
|
+
return codec ? codec.encode(result) : result;
|
|
1524
|
+
},
|
|
1525
|
+
codec
|
|
1526
|
+
);
|
|
1279
1527
|
}
|
|
1280
|
-
|
|
1281
|
-
|
|
1528
|
+
/**
|
|
1529
|
+
* Create a resumable checkpoint point for iterative operations.
|
|
1530
|
+
*
|
|
1531
|
+
* @param step - Unique identifier for this checkpoint point
|
|
1532
|
+
*/
|
|
1533
|
+
point(step) {
|
|
1534
|
+
this.points[step] ??= { committed: false, entries: [] };
|
|
1535
|
+
return new Point(this.points[step], () => this.save());
|
|
1282
1536
|
}
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
case "While":
|
|
1317
|
-
case "Until":
|
|
1318
|
-
return statementsContainBlockedCommand(
|
|
1319
|
-
command.condition,
|
|
1320
|
-
cloneInspectionContext(context),
|
|
1321
|
-
mode
|
|
1322
|
-
) || statementsContainBlockedCommand(
|
|
1323
|
-
command.body,
|
|
1324
|
-
cloneInspectionContext(context),
|
|
1325
|
-
mode
|
|
1326
|
-
);
|
|
1327
|
-
case "Case":
|
|
1328
|
-
return command.items.some(
|
|
1329
|
-
(item) => statementsContainBlockedCommand(
|
|
1330
|
-
item.body,
|
|
1331
|
-
cloneInspectionContext(context),
|
|
1332
|
-
mode
|
|
1333
|
-
)
|
|
1334
|
-
);
|
|
1335
|
-
case "Subshell":
|
|
1336
|
-
case "Group":
|
|
1337
|
-
return statementsContainBlockedCommand(
|
|
1338
|
-
command.body,
|
|
1339
|
-
cloneInspectionContext(context),
|
|
1340
|
-
mode
|
|
1341
|
-
);
|
|
1342
|
-
case "FunctionDef":
|
|
1343
|
-
return false;
|
|
1344
|
-
case "ArithmeticCommand":
|
|
1345
|
-
case "ConditionalCommand":
|
|
1346
|
-
return false;
|
|
1347
|
-
default: {
|
|
1348
|
-
const _unreachable = command;
|
|
1349
|
-
return _unreachable;
|
|
1537
|
+
/**
|
|
1538
|
+
* Process each input with automatic checkpointing and concurrency.
|
|
1539
|
+
*
|
|
1540
|
+
* @param step - Unique identifier for this checkpoint
|
|
1541
|
+
* @param inputs - Items to process
|
|
1542
|
+
* @param process - Function to process each input
|
|
1543
|
+
* @param options - Optional settings like concurrency
|
|
1544
|
+
* @returns All outputs (use `.flat()` if outputs are arrays)
|
|
1545
|
+
*/
|
|
1546
|
+
async each(step, inputs, process, options) {
|
|
1547
|
+
const point = this.point(step);
|
|
1548
|
+
const limit = pLimit(options?.concurrency ?? 1);
|
|
1549
|
+
const inputArray = Array.from(inputs);
|
|
1550
|
+
await Promise.all(
|
|
1551
|
+
inputArray.map(
|
|
1552
|
+
(input) => limit(() => point.through(input, () => process(input)))
|
|
1553
|
+
)
|
|
1554
|
+
);
|
|
1555
|
+
await point.commit();
|
|
1556
|
+
return point.values();
|
|
1557
|
+
}
|
|
1558
|
+
/**
|
|
1559
|
+
* Get clean output from all completed points.
|
|
1560
|
+
* Single-entry points return the value directly, multi-entry return arrays.
|
|
1561
|
+
*/
|
|
1562
|
+
getOutput() {
|
|
1563
|
+
const output = {};
|
|
1564
|
+
for (const [key, pointData] of Object.entries(this.points)) {
|
|
1565
|
+
if (pointData.entries.length === 1) {
|
|
1566
|
+
output[key] = pointData.entries[0].output;
|
|
1567
|
+
} else {
|
|
1568
|
+
output[key] = pointData.entries.map((e) => e.output);
|
|
1569
|
+
}
|
|
1350
1570
|
}
|
|
1571
|
+
return output;
|
|
1351
1572
|
}
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
return true;
|
|
1573
|
+
/** Get the file path where checkpoint is stored */
|
|
1574
|
+
getPath() {
|
|
1575
|
+
return this.path;
|
|
1356
1576
|
}
|
|
1357
|
-
|
|
1358
|
-
|
|
1577
|
+
async save() {
|
|
1578
|
+
const file = {
|
|
1579
|
+
configHash: this.configHash,
|
|
1580
|
+
points: this.points
|
|
1581
|
+
};
|
|
1582
|
+
const content = JSON.stringify(file, null, 2);
|
|
1583
|
+
const tempPath = `${this.path}.tmp`;
|
|
1584
|
+
writeFileSync(tempPath, content);
|
|
1585
|
+
renameSync(tempPath, this.path);
|
|
1359
1586
|
}
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1587
|
+
};
|
|
1588
|
+
function hash(value) {
|
|
1589
|
+
return createHash2("md5").update(JSON.stringify(value)).digest("hex");
|
|
1590
|
+
}
|
|
1591
|
+
var Point = class {
|
|
1592
|
+
#cache;
|
|
1593
|
+
data;
|
|
1594
|
+
persist;
|
|
1595
|
+
constructor(data, persist) {
|
|
1596
|
+
this.#cache = new Map(
|
|
1597
|
+
data.entries.map((e) => [e.inputHash, e.output])
|
|
1598
|
+
);
|
|
1599
|
+
this.data = data;
|
|
1600
|
+
this.persist = persist;
|
|
1366
1601
|
}
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
return wordContainsBlockedCommand(
|
|
1377
|
-
redirection.target.content,
|
|
1378
|
-
context,
|
|
1379
|
-
mode
|
|
1380
|
-
);
|
|
1602
|
+
/**
|
|
1603
|
+
* Execute computation if input wasn't processed before.
|
|
1604
|
+
* Returns cached output if input hash exists, otherwise executes, saves, and returns.
|
|
1605
|
+
*/
|
|
1606
|
+
async through(input, compute, codec) {
|
|
1607
|
+
const inputHash = hash(input);
|
|
1608
|
+
if (this.#cache.has(inputHash)) {
|
|
1609
|
+
const cached = this.#cache.get(inputHash);
|
|
1610
|
+
return codec ? codec.decode(cached) : cached;
|
|
1381
1611
|
}
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1612
|
+
const output = await compute();
|
|
1613
|
+
this.data.entries.push({ inputHash, output });
|
|
1614
|
+
this.#cache.set(inputHash, output);
|
|
1615
|
+
await this.persist();
|
|
1616
|
+
return codec ? codec.decode(output) : output;
|
|
1385
1617
|
}
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1618
|
+
/** Mark this point as complete. */
|
|
1619
|
+
async commit() {
|
|
1620
|
+
this.data.committed = true;
|
|
1621
|
+
await this.persist();
|
|
1389
1622
|
}
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
return
|
|
1623
|
+
/** Check if this point has been committed. */
|
|
1624
|
+
isCommitted() {
|
|
1625
|
+
return this.data.committed;
|
|
1393
1626
|
}
|
|
1394
|
-
|
|
1395
|
-
|
|
1627
|
+
/** Get all outputs from this point. */
|
|
1628
|
+
values() {
|
|
1629
|
+
return this.data.entries.map((e) => e.output);
|
|
1396
1630
|
}
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1631
|
+
};
|
|
1632
|
+
function hashConfig(config) {
|
|
1633
|
+
return createHash2("md5").update(JSON.stringify(config)).digest("hex");
|
|
1634
|
+
}
|
|
1635
|
+
|
|
1636
|
+
// packages/text2sql/src/lib/fs/sqlite/sqlite-fs.ts
|
|
1637
|
+
import * as path2 from "node:path";
|
|
1638
|
+
import { DatabaseSync } from "node:sqlite";
|
|
1639
|
+
|
|
1640
|
+
// packages/text2sql/src/lib/fs/sqlite/ddl.sqlite-fs.sql
|
|
1641
|
+
var ddl_sqlite_fs_default = "-- SQLite-based filesystem schema for artifact storage\n-- Tables: fs_entries (file/directory metadata), fs_chunks (file content)\n\n-- Performance PRAGMAs (session-level, run on each connection)\nPRAGMA journal_mode = WAL;\nPRAGMA synchronous = NORMAL;\nPRAGMA cache_size = -64000;\nPRAGMA temp_store = MEMORY;\nPRAGMA mmap_size = 268435456;\n\n-- Integrity\nPRAGMA foreign_keys = ON;\n\n-- Filesystem entries table (files, directories, symlinks)\nCREATE TABLE IF NOT EXISTS fs_entries (\n path TEXT PRIMARY KEY, -- Normalized absolute path (e.g., '/results/uuid.json')\n type TEXT NOT NULL, -- 'file' | 'directory' | 'symlink'\n mode INTEGER NOT NULL, -- Unix permissions (e.g., 0o644 = 420)\n size INTEGER NOT NULL, -- File size in bytes (0 for directories)\n mtime INTEGER NOT NULL, -- Modified time (milliseconds since epoch)\n symlinkTarget TEXT -- Target path for symlinks (NULL for files/dirs)\n);\n\nCREATE INDEX IF NOT EXISTS idx_fs_entries_type ON fs_entries(type);\n\n-- File content chunks table (for handling large files)\n-- Files are split into 1MB chunks to avoid SQLite BLOB performance issues\nCREATE TABLE IF NOT EXISTS fs_chunks (\n path TEXT NOT NULL, -- Reference to fs_entries.path\n chunkIndex INTEGER NOT NULL, -- 0-based chunk sequence\n data BLOB NOT NULL, -- Chunk content (up to 1MB default)\n PRIMARY KEY (path, chunkIndex),\n FOREIGN KEY (path) REFERENCES fs_entries(path) ON DELETE CASCADE ON UPDATE CASCADE\n);\n";
|
|
1642
|
+
|
|
1643
|
+
// packages/text2sql/src/lib/fs/sqlite/sqlite-fs.ts
|
|
1644
|
+
var SqliteFs = class {
|
|
1645
|
+
#db;
|
|
1646
|
+
#statements = /* @__PURE__ */ new Map();
|
|
1647
|
+
#chunkSize;
|
|
1648
|
+
#root;
|
|
1649
|
+
constructor(options) {
|
|
1650
|
+
this.#chunkSize = options.chunkSize ?? 1024 * 1024;
|
|
1651
|
+
const normalizedRoot = this.#normalizeRoot(options.root);
|
|
1652
|
+
this.#root = normalizedRoot === "/" ? "" : normalizedRoot;
|
|
1653
|
+
this.#db = new DatabaseSync(options.dbPath);
|
|
1654
|
+
this.#db.exec(ddl_sqlite_fs_default);
|
|
1655
|
+
const rootSlashExists = this.#stmt(
|
|
1656
|
+
"SELECT 1 FROM fs_entries WHERE path = ?"
|
|
1657
|
+
).get("/");
|
|
1658
|
+
if (!rootSlashExists) {
|
|
1659
|
+
this.#stmt(
|
|
1660
|
+
`INSERT INTO fs_entries (path, type, mode, size, mtime)
|
|
1661
|
+
VALUES ('/', 'directory', 493, 0, ?)`
|
|
1662
|
+
).run(Date.now());
|
|
1401
1663
|
}
|
|
1402
|
-
if (
|
|
1403
|
-
|
|
1664
|
+
if (this.#root) {
|
|
1665
|
+
this.#createParentDirs(this.#root);
|
|
1666
|
+
const rootExists = this.#stmt(
|
|
1667
|
+
"SELECT 1 FROM fs_entries WHERE path = ?"
|
|
1668
|
+
).get(this.#root);
|
|
1669
|
+
if (!rootExists) {
|
|
1670
|
+
this.#stmt(
|
|
1671
|
+
`INSERT INTO fs_entries (path, type, mode, size, mtime)
|
|
1672
|
+
VALUES (?, 'directory', 493, 0, ?)`
|
|
1673
|
+
).run(this.#root, Date.now());
|
|
1674
|
+
}
|
|
1404
1675
|
}
|
|
1405
|
-
return !ALLOWED_SQL_PROXY_SUBCOMMANDS.has(subcommand);
|
|
1406
1676
|
}
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1677
|
+
/**
|
|
1678
|
+
* Create parent directories for a path (used during initialization).
|
|
1679
|
+
* Creates all segments EXCEPT the last one (the path itself).
|
|
1680
|
+
*/
|
|
1681
|
+
#createParentDirs(p) {
|
|
1682
|
+
const segments = p.split("/").filter(Boolean);
|
|
1683
|
+
let currentPath = "/";
|
|
1684
|
+
for (let i = 0; i < segments.length - 1; i++) {
|
|
1685
|
+
currentPath = path2.posix.join(currentPath, segments[i]);
|
|
1686
|
+
const exists = this.#stmt("SELECT 1 FROM fs_entries WHERE path = ?").get(
|
|
1687
|
+
currentPath
|
|
1688
|
+
);
|
|
1689
|
+
if (!exists) {
|
|
1690
|
+
this.#stmt(
|
|
1691
|
+
`INSERT INTO fs_entries (path, type, mode, size, mtime)
|
|
1692
|
+
VALUES (?, 'directory', 493, 0, ?)`
|
|
1693
|
+
).run(currentPath, Date.now());
|
|
1694
|
+
}
|
|
1410
1695
|
}
|
|
1411
|
-
|
|
1412
|
-
|
|
1696
|
+
}
|
|
1697
|
+
#stmt(sql) {
|
|
1698
|
+
let stmt = this.#statements.get(sql);
|
|
1699
|
+
if (!stmt) {
|
|
1700
|
+
stmt = this.#db.prepare(sql);
|
|
1701
|
+
this.#statements.set(sql, stmt);
|
|
1413
1702
|
}
|
|
1414
|
-
return
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
"block-all-sql",
|
|
1423
|
-
options
|
|
1424
|
-
);
|
|
1425
|
-
};
|
|
1426
|
-
if (WRAPPER_COMMANDS.has(normalizedName)) {
|
|
1427
|
-
if (normalizedName === "env") {
|
|
1428
|
-
return inspectWrappedCommand(resolveEnvWrapperCommand(command.args));
|
|
1429
|
-
}
|
|
1430
|
-
if (normalizedName === "command") {
|
|
1431
|
-
return inspectWrappedCommand(resolveCommandWrapperCommand(command.args));
|
|
1432
|
-
}
|
|
1433
|
-
const evalScript = joinStaticWords(command.args);
|
|
1434
|
-
if (evalScript == null) {
|
|
1435
|
-
return true;
|
|
1703
|
+
return stmt;
|
|
1704
|
+
}
|
|
1705
|
+
#normalizeRoot(root) {
|
|
1706
|
+
return path2.posix.resolve("/", root.trim());
|
|
1707
|
+
}
|
|
1708
|
+
#prefixPath(p) {
|
|
1709
|
+
if (!this.#root) {
|
|
1710
|
+
return p;
|
|
1436
1711
|
}
|
|
1437
|
-
if (
|
|
1438
|
-
return
|
|
1712
|
+
if (p === "/") {
|
|
1713
|
+
return this.#root;
|
|
1439
1714
|
}
|
|
1440
|
-
return
|
|
1441
|
-
evalScript,
|
|
1442
|
-
context,
|
|
1443
|
-
"block-all-sql"
|
|
1444
|
-
);
|
|
1715
|
+
return path2.posix.join(this.#root, p);
|
|
1445
1716
|
}
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
return true;
|
|
1450
|
-
}
|
|
1451
|
-
if (shellInvocation.kind === "command") {
|
|
1452
|
-
if (!shellInvocation.payload) {
|
|
1453
|
-
return true;
|
|
1454
|
-
}
|
|
1455
|
-
if (stringCommandContainsBlockedCommand(
|
|
1456
|
-
shellInvocation.payload,
|
|
1457
|
-
context,
|
|
1458
|
-
"block-all-sql"
|
|
1459
|
-
)) {
|
|
1460
|
-
return true;
|
|
1461
|
-
}
|
|
1462
|
-
return false;
|
|
1717
|
+
#unprefixPath(p) {
|
|
1718
|
+
if (!this.#root) {
|
|
1719
|
+
return p;
|
|
1463
1720
|
}
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
if (hereDoc.payload == null) {
|
|
1467
|
-
return true;
|
|
1468
|
-
}
|
|
1469
|
-
if (hereDoc.payload.trim().length > 0 && stringCommandContainsBlockedCommand(
|
|
1470
|
-
hereDoc.payload,
|
|
1471
|
-
context,
|
|
1472
|
-
"block-all-sql"
|
|
1473
|
-
)) {
|
|
1474
|
-
return true;
|
|
1475
|
-
}
|
|
1721
|
+
if (p === this.#root) {
|
|
1722
|
+
return "/";
|
|
1476
1723
|
}
|
|
1477
|
-
if (
|
|
1478
|
-
return
|
|
1724
|
+
if (p.startsWith(this.#root + "/")) {
|
|
1725
|
+
return p.slice(this.#root.length) || "/";
|
|
1479
1726
|
}
|
|
1480
|
-
|
|
1481
|
-
|
|
1727
|
+
return p;
|
|
1728
|
+
}
|
|
1729
|
+
#useTransaction(fn) {
|
|
1730
|
+
this.#db.exec("BEGIN TRANSACTION");
|
|
1731
|
+
try {
|
|
1732
|
+
const result = fn();
|
|
1733
|
+
this.#db.exec("COMMIT");
|
|
1734
|
+
return result;
|
|
1735
|
+
} catch (error) {
|
|
1736
|
+
this.#db.exec("ROLLBACK");
|
|
1737
|
+
throw error;
|
|
1482
1738
|
}
|
|
1483
1739
|
}
|
|
1484
|
-
|
|
1485
|
-
return
|
|
1740
|
+
#normalizePath(p) {
|
|
1741
|
+
return path2.posix.resolve("/", p);
|
|
1486
1742
|
}
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
name = "sql-proxy-enforcement";
|
|
1491
|
-
transform(context) {
|
|
1492
|
-
const blocked = scriptContainsBlockedCommand(context.ast, {
|
|
1493
|
-
functionDefinitions: /* @__PURE__ */ new Map(),
|
|
1494
|
-
callStack: /* @__PURE__ */ new Set()
|
|
1495
|
-
});
|
|
1496
|
-
if (blocked) {
|
|
1497
|
-
throw new SqlProxyViolationError();
|
|
1498
|
-
}
|
|
1499
|
-
return {
|
|
1500
|
-
ast: context.ast,
|
|
1501
|
-
metadata: { inspected: true }
|
|
1502
|
-
};
|
|
1743
|
+
#dirname(p) {
|
|
1744
|
+
const dir = path2.posix.dirname(p);
|
|
1745
|
+
return dir === "" ? "/" : dir;
|
|
1503
1746
|
}
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
if (
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1747
|
+
#ensureParentExists(filePath) {
|
|
1748
|
+
const parent = this.#dirname(filePath);
|
|
1749
|
+
const rootPath = this.#root || "/";
|
|
1750
|
+
if (parent === rootPath || parent === "/") return;
|
|
1751
|
+
const entry = this.#stmt("SELECT type FROM fs_entries WHERE path = ?").get(
|
|
1752
|
+
parent
|
|
1753
|
+
);
|
|
1754
|
+
if (!entry) {
|
|
1755
|
+
this.#ensureParentExists(parent);
|
|
1756
|
+
this.#stmt(
|
|
1757
|
+
`INSERT INTO fs_entries (path, type, mode, size, mtime)
|
|
1758
|
+
VALUES (?, 'directory', 493, 0, ?)`
|
|
1759
|
+
).run(parent, Date.now());
|
|
1760
|
+
} else if (entry.type !== "directory") {
|
|
1761
|
+
throw new Error(`mkdir: parent is not a directory: ${parent}`);
|
|
1516
1762
|
}
|
|
1517
1763
|
}
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
if (!isValidAdapterName(dbName)) return false;
|
|
1529
|
-
const sqlArgs = cmd.args.slice(2);
|
|
1530
|
-
return sqlArgs.some(
|
|
1531
|
-
(arg) => wordPartsContainBacktickSubstitution(
|
|
1532
|
-
arg.parts
|
|
1533
|
-
)
|
|
1534
|
-
);
|
|
1535
|
-
}
|
|
1536
|
-
function extractWordPartText(parts) {
|
|
1537
|
-
let text = "";
|
|
1538
|
-
for (const part of parts) {
|
|
1539
|
-
const type = part.type;
|
|
1540
|
-
if (type === "Literal" || type === "SingleQuoted" || type === "Escaped") {
|
|
1541
|
-
text += part.value;
|
|
1542
|
-
} else if (type === "DoubleQuoted" && Array.isArray(part.parts)) {
|
|
1543
|
-
text += extractWordPartText(part.parts);
|
|
1544
|
-
} else if (type === "CommandSubstitution" && part.legacy === true) {
|
|
1545
|
-
text += "`" + serialize(part.body).trim() + "`";
|
|
1764
|
+
#writeChunks(filePath, content) {
|
|
1765
|
+
this.#stmt("DELETE FROM fs_chunks WHERE path = ?").run(filePath);
|
|
1766
|
+
for (let i = 0; i < content.length; i += this.#chunkSize) {
|
|
1767
|
+
const chunk = content.slice(
|
|
1768
|
+
i,
|
|
1769
|
+
Math.min(i + this.#chunkSize, content.length)
|
|
1770
|
+
);
|
|
1771
|
+
this.#stmt(
|
|
1772
|
+
"INSERT INTO fs_chunks (path, chunkIndex, data) VALUES (?, ?, ?)"
|
|
1773
|
+
).run(filePath, Math.floor(i / this.#chunkSize), chunk);
|
|
1546
1774
|
}
|
|
1547
1775
|
}
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
(
|
|
1553
|
-
|
|
1554
|
-
)
|
|
1555
|
-
).join(" ");
|
|
1556
|
-
}
|
|
1557
|
-
function rewriteSqlBacktickCommand(cmd, ast) {
|
|
1558
|
-
const subcommand = asStaticWordText(cmd.args[0]);
|
|
1559
|
-
const dbName = asStaticWordText(cmd.args[1]);
|
|
1560
|
-
const sqlArgs = cmd.args.slice(2);
|
|
1561
|
-
const unquotedSql = extractSqlText(sqlArgs);
|
|
1562
|
-
const hash2 = createHash("sha256").update(serialize(ast)).digest("hex").slice(0, 12);
|
|
1563
|
-
const sqlPath = `/tmp/sql-inline-${subcommand}-${hash2}.sql`;
|
|
1564
|
-
const heredocTag = `SQL_${hash2.toUpperCase()}`;
|
|
1565
|
-
const groupScript = [
|
|
1566
|
-
`{ cat > ${sqlPath} <<'${heredocTag}'`,
|
|
1567
|
-
unquotedSql,
|
|
1568
|
-
heredocTag,
|
|
1569
|
-
`sql ${subcommand} ${dbName} "$(cat ${sqlPath})"; }`
|
|
1570
|
-
].join("\n");
|
|
1571
|
-
const groupAst = parse(groupScript);
|
|
1572
|
-
return groupAst.statements[0].pipelines[0].commands[0];
|
|
1573
|
-
}
|
|
1574
|
-
function rewriteBackticksInCommands(commands, ast) {
|
|
1575
|
-
let rewritten = false;
|
|
1576
|
-
for (let i = 0; i < commands.length; i++) {
|
|
1577
|
-
const cmd = commands[i];
|
|
1578
|
-
if (isSqlBacktickCommand(cmd)) {
|
|
1579
|
-
commands[i] = rewriteSqlBacktickCommand(cmd, ast);
|
|
1580
|
-
rewritten = true;
|
|
1581
|
-
continue;
|
|
1776
|
+
#readChunks(filePath) {
|
|
1777
|
+
const rows = this.#stmt(
|
|
1778
|
+
"SELECT data FROM fs_chunks WHERE path = ? ORDER BY chunkIndex"
|
|
1779
|
+
).all(filePath);
|
|
1780
|
+
if (rows.length === 0) {
|
|
1781
|
+
return new Uint8Array(0);
|
|
1582
1782
|
}
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
if (cmd.elseBody && rewriteBackticksInStatements(cmd.elseBody, ast)) {
|
|
1590
|
-
rewritten = true;
|
|
1591
|
-
}
|
|
1592
|
-
} else if (cmd.type === "While" || cmd.type === "Until") {
|
|
1593
|
-
if (rewriteBackticksInStatements(cmd.condition, ast)) rewritten = true;
|
|
1594
|
-
if (rewriteBackticksInStatements(cmd.body, ast)) rewritten = true;
|
|
1595
|
-
} else if (cmd.type === "For" || cmd.type === "CStyleFor") {
|
|
1596
|
-
if (rewriteBackticksInStatements(cmd.body, ast)) rewritten = true;
|
|
1597
|
-
} else if (cmd.type === "Case") {
|
|
1598
|
-
for (const item of cmd.items) {
|
|
1599
|
-
if (rewriteBackticksInStatements(item.body, ast)) rewritten = true;
|
|
1600
|
-
}
|
|
1601
|
-
} else if (cmd.type === "Subshell" || cmd.type === "Group") {
|
|
1602
|
-
if (rewriteBackticksInStatements(cmd.body, ast)) rewritten = true;
|
|
1783
|
+
const totalSize = rows.reduce((sum, row) => sum + row.data.length, 0);
|
|
1784
|
+
const result = new Uint8Array(totalSize);
|
|
1785
|
+
let offset = 0;
|
|
1786
|
+
for (const row of rows) {
|
|
1787
|
+
result.set(row.data, offset);
|
|
1788
|
+
offset += row.data.length;
|
|
1603
1789
|
}
|
|
1790
|
+
return result;
|
|
1604
1791
|
}
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
}
|
|
1792
|
+
#resolveSymlink(p, seen = /* @__PURE__ */ new Set()) {
|
|
1793
|
+
if (seen.has(p)) {
|
|
1794
|
+
throw new Error(`readFile: circular symlink: ${p}`);
|
|
1795
|
+
}
|
|
1796
|
+
const entry = this.#stmt(
|
|
1797
|
+
"SELECT type, symlinkTarget FROM fs_entries WHERE path = ?"
|
|
1798
|
+
).get(p);
|
|
1799
|
+
if (!entry) {
|
|
1800
|
+
throw new Error(`ENOENT: no such file or directory: ${p}`);
|
|
1801
|
+
}
|
|
1802
|
+
if (entry.type !== "symlink") {
|
|
1803
|
+
return p;
|
|
1614
1804
|
}
|
|
1805
|
+
seen.add(p);
|
|
1806
|
+
const target = this.#normalizePath(
|
|
1807
|
+
path2.posix.resolve(this.#dirname(p), entry.symlinkTarget)
|
|
1808
|
+
);
|
|
1809
|
+
return this.#resolveSymlink(target, seen);
|
|
1615
1810
|
}
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
const rewritten = rewriteBackticksInStatements(ast.statements, ast);
|
|
1623
|
-
return { ast, metadata: { rewritten } };
|
|
1624
|
-
}
|
|
1625
|
-
};
|
|
1626
|
-
|
|
1627
|
-
// packages/text2sql/src/lib/agents/sql.agent.ts
|
|
1628
|
-
import { groq } from "@ai-sdk/groq";
|
|
1629
|
-
import {
|
|
1630
|
-
APICallError,
|
|
1631
|
-
JSONParseError,
|
|
1632
|
-
NoContentGeneratedError,
|
|
1633
|
-
NoObjectGeneratedError,
|
|
1634
|
-
NoOutputGeneratedError,
|
|
1635
|
-
TypeValidationError,
|
|
1636
|
-
defaultSettingsMiddleware,
|
|
1637
|
-
wrapLanguageModel
|
|
1638
|
-
} from "ai";
|
|
1639
|
-
import dedent from "dedent";
|
|
1640
|
-
import pRetry from "p-retry";
|
|
1641
|
-
import z from "zod";
|
|
1642
|
-
import "@deepagents/agent";
|
|
1643
|
-
import {
|
|
1644
|
-
ContextEngine,
|
|
1645
|
-
InMemoryContextStore,
|
|
1646
|
-
example,
|
|
1647
|
-
fragment,
|
|
1648
|
-
guardrail,
|
|
1649
|
-
hint,
|
|
1650
|
-
persona,
|
|
1651
|
-
policy,
|
|
1652
|
-
structuredOutput,
|
|
1653
|
-
user,
|
|
1654
|
-
workflow
|
|
1655
|
-
} from "@deepagents/context";
|
|
1656
|
-
var RETRY_TEMPERATURES = [0, 0.2, 0.3];
|
|
1657
|
-
var SQL_AGENT_ROLE = "Expert SQL query generator.";
|
|
1658
|
-
var SQL_AGENT_OBJECTIVE = "Generate precise SQL grounded in provided schema.";
|
|
1659
|
-
var SQL_AGENT_POLICIES = [
|
|
1660
|
-
fragment(
|
|
1661
|
-
"schema_mapping",
|
|
1662
|
-
policy({
|
|
1663
|
-
rule: "Translate natural language into precise SQL grounded in available schema entities."
|
|
1664
|
-
}),
|
|
1665
|
-
hint("Preserve schema spelling exactly, including typos in column names.")
|
|
1666
|
-
),
|
|
1667
|
-
fragment(
|
|
1668
|
-
"projection_minimality",
|
|
1669
|
-
policy({
|
|
1670
|
-
rule: "Return only columns requested by the question; do not add helper columns unless explicitly requested."
|
|
1671
|
-
}),
|
|
1672
|
-
policy({
|
|
1673
|
-
rule: 'For requests of the form "X sorted/ordered by Y", project X only unless Y is explicitly requested as an output field.'
|
|
1674
|
-
}),
|
|
1675
|
-
policy({
|
|
1676
|
-
rule: "Prefer selecting schema columns directly without derived expressions when direct selection answers the request."
|
|
1677
|
-
}),
|
|
1678
|
-
hint(
|
|
1679
|
-
"Do not include ORDER BY, GROUP BY, or JOIN helper columns in SELECT output unless the question explicitly asks for them."
|
|
1680
|
-
),
|
|
1681
|
-
policy({
|
|
1682
|
-
rule: "Use DISTINCT only when uniqueness is explicitly requested (for example distinct/unique/different/no duplicates)."
|
|
1683
|
-
}),
|
|
1684
|
-
hint(
|
|
1685
|
-
'Do not infer DISTINCT from generic wording such as "some", plural nouns, or entity-set phrasing; for transactional/attendance-style tables, default to raw rows unless uniqueness is explicitly requested.'
|
|
1686
|
-
)
|
|
1687
|
-
),
|
|
1688
|
-
fragment(
|
|
1689
|
-
"date_transform_safety",
|
|
1690
|
-
policy({
|
|
1691
|
-
rule: "Do not assume VARCHAR/TEXT values are parseable dates. Avoid date extraction functions on text columns by default."
|
|
1692
|
-
}),
|
|
1693
|
-
policy({
|
|
1694
|
-
rule: "Use date-part extraction only when both conditions hold: the question explicitly asks for transformation and schema values require transformation to produce that unit."
|
|
1695
|
-
}),
|
|
1696
|
-
hint(
|
|
1697
|
-
"Do not apply SUBSTR, STRFTIME, DATE_PART, YEAR, or similar extraction functions unless the question explicitly asks for transformation and schema values require it."
|
|
1698
|
-
),
|
|
1699
|
-
hint(
|
|
1700
|
-
"If a column already represents the requested concept (for example a stored year-like value), use the column as-is."
|
|
1701
|
-
)
|
|
1702
|
-
),
|
|
1703
|
-
fragment(
|
|
1704
|
-
"sql_minimality",
|
|
1705
|
-
guardrail({
|
|
1706
|
-
rule: "Never hallucinate tables or columns.",
|
|
1707
|
-
reason: "Schema fidelity is required.",
|
|
1708
|
-
action: "Use only available schema entities."
|
|
1709
|
-
}),
|
|
1710
|
-
guardrail({
|
|
1711
|
-
rule: "Avoid unnecessary transformations and derived projections.",
|
|
1712
|
-
reason: "Extra transformations frequently change semantics and reduce correctness.",
|
|
1713
|
-
action: "Do not add date parsing, substring extraction, or derived columns unless explicitly required by the question or schema."
|
|
1714
|
-
})
|
|
1715
|
-
),
|
|
1716
|
-
fragment(
|
|
1717
|
-
"preflight_checklist",
|
|
1718
|
-
workflow({
|
|
1719
|
-
task: "Final SQL preflight before returning output",
|
|
1720
|
-
steps: [
|
|
1721
|
-
"Verify selected columns match the question and remove unrequested helper projections.",
|
|
1722
|
-
"If aggregate values are used only for ranking/filtering, keep them out of SELECT unless explicitly requested.",
|
|
1723
|
-
"Prefer raw schema columns over derived expressions when raw columns already satisfy the request.",
|
|
1724
|
-
"If a candidate query uses STRFTIME, SUBSTR, DATE_PART, YEAR, or similar extraction on text-like columns, remove that transformation unless explicitly required by the question.",
|
|
1725
|
-
"Return only schema-grounded SQL using existing tables and columns."
|
|
1726
|
-
]
|
|
1727
|
-
})
|
|
1728
|
-
),
|
|
1729
|
-
fragment(
|
|
1730
|
-
"set_semantics",
|
|
1731
|
-
policy({
|
|
1732
|
-
rule: "For questions asking where both condition A and condition B hold over an attribute, compute the intersection of qualifying sets for that attribute."
|
|
1733
|
-
}),
|
|
1734
|
-
policy({
|
|
1735
|
-
rule: "Do not force the same entity instance to satisfy both conditions unless the question explicitly requests the same person/row/entity."
|
|
1736
|
-
}),
|
|
1737
|
-
hint(
|
|
1738
|
-
"Prefer INTERSECT (or logically equivalent set-based shape) over requiring the same physical row/entity to satisfy both conditions unless explicitly requested."
|
|
1739
|
-
),
|
|
1740
|
-
hint(
|
|
1741
|
-
"When two conditions describe different row groups whose shared attribute is requested, build each group separately and intersect the attribute values."
|
|
1742
|
-
),
|
|
1743
|
-
hint(
|
|
1744
|
-
"Do not collapse cross-group conditions into a single-row AND predicate when the intent is shared values across groups."
|
|
1745
|
-
),
|
|
1746
|
-
policy({
|
|
1747
|
-
rule: "If two predicates on the same field cannot both be true for one row, do not combine them with AND; use set operations across separate filtered subsets when shared values are requested."
|
|
1748
|
-
})
|
|
1749
|
-
),
|
|
1750
|
-
fragment(
|
|
1751
|
-
"predicate_column_alignment",
|
|
1752
|
-
policy({
|
|
1753
|
-
rule: "Match literal values to semantically compatible columns. Do not compare descriptive names to identifier columns."
|
|
1754
|
-
}),
|
|
1755
|
-
hint(
|
|
1756
|
-
"When a filter value is a descriptive label (for example a department name), join through the lookup table and filter on its name/title column, not on *_id columns."
|
|
1757
|
-
),
|
|
1758
|
-
hint(
|
|
1759
|
-
"When relation roles are explicit in wording (for example host/home/source/destination), prefer foreign keys with matching role qualifiers over generic similarly named columns."
|
|
1760
|
-
),
|
|
1761
|
-
policy({
|
|
1762
|
-
rule: "When multiple foreign-key candidates exist, select the column whose qualifier best matches the relationship described in the question."
|
|
1763
|
-
}),
|
|
1764
|
-
policy({
|
|
1765
|
-
rule: "For hosting/held semantics, prefer host_* relationship columns when available over generic *_id alternatives."
|
|
1766
|
-
}),
|
|
1767
|
-
hint(
|
|
1768
|
-
'Interpret wording like "held/hosted a competition or event" as a hosting relationship and map to host_* foreign keys when present.'
|
|
1769
|
-
),
|
|
1770
|
-
policy({
|
|
1771
|
-
rule: "Do not compare descriptive labels or names to *_id columns; join to the table containing the descriptive field and filter there."
|
|
1772
|
-
}),
|
|
1773
|
-
policy({
|
|
1774
|
-
rule: "Keep numeric identifiers unquoted when used as numeric equality filters unless schema indicates text identifiers."
|
|
1775
|
-
}),
|
|
1776
|
-
policy({
|
|
1777
|
-
rule: "When filtering by a descriptive label value and a related table exposes a corresponding *_name or title column, join to that table and filter on the descriptive column."
|
|
1778
|
-
})
|
|
1779
|
-
),
|
|
1780
|
-
fragment(
|
|
1781
|
-
"ordering_semantics",
|
|
1782
|
-
policy({
|
|
1783
|
-
rule: "Respect explicit sort direction terms. If direction is not specified, use ascending order unless a superlative intent (most/least/highest/lowest) implies direction."
|
|
1784
|
-
}),
|
|
1785
|
-
policy({
|
|
1786
|
-
rule: "When ranking categories by frequency, use COUNT for ordering but keep output focused on requested category fields unless counts are explicitly requested."
|
|
1787
|
-
}),
|
|
1788
|
-
policy({
|
|
1789
|
-
rule: "Do not use DESC unless descending direction is explicit or a superlative intent requires descending ranking."
|
|
1790
|
-
}),
|
|
1791
|
-
policy({
|
|
1792
|
-
rule: 'For "most common/frequent <attribute>" requests, return the attribute value(s) only; use counts only for ordering/filtering unless the question explicitly asks to return counts.'
|
|
1793
|
-
}),
|
|
1794
|
-
hint(
|
|
1795
|
-
'Use DESC with LIMIT 1 for "most/highest/largest"; use ASC with LIMIT 1 for "least/lowest/smallest".'
|
|
1796
|
-
)
|
|
1797
|
-
),
|
|
1798
|
-
fragment(
|
|
1799
|
-
"negative_membership_queries",
|
|
1800
|
-
policy({
|
|
1801
|
-
rule: "For requests asking entities that did not participate/host/appear in related records, prefer NOT IN or NOT EXISTS against the related foreign-key set."
|
|
1802
|
-
}),
|
|
1803
|
-
hint(
|
|
1804
|
-
"Map role-bearing relationship columns carefully (for example host_* foreign keys for hosting relationships) instead of generic IDs when role wording is explicit."
|
|
1805
|
-
),
|
|
1806
|
-
hint(
|
|
1807
|
-
'For "never had/never exceeded" conditions over history tables, exclude entities via NOT IN/NOT EXISTS against the disqualifying entity-id set (often built with GROUP BY/HAVING MAX(...)).'
|
|
1808
|
-
)
|
|
1809
|
-
),
|
|
1810
|
-
fragment(
|
|
1811
|
-
"join_completeness",
|
|
1812
|
-
policy({
|
|
1813
|
-
rule: "Preserve entity-restricting joins implied by the question. Do not widen results by querying only a broader attribute table when a subset entity table is available."
|
|
1814
|
-
}),
|
|
1815
|
-
policy({
|
|
1816
|
-
rule: "If an entity term in the question maps to a table, keep that table in query scope and join to attribute tables rather than dropping the entity table."
|
|
1817
|
-
}),
|
|
1818
|
-
hint(
|
|
1819
|
-
"If the question targets a specific entity group, include that entity table and its join conditions even when selected columns come from a related table."
|
|
1820
|
-
),
|
|
1821
|
-
hint(
|
|
1822
|
-
"When the question names an entity type and a relation table links to that entity via *_id, include the entity table in scope instead of counting only relation rows."
|
|
1823
|
-
),
|
|
1824
|
-
hint(
|
|
1825
|
-
"Prefer INNER JOIN by default; use LEFT JOIN only when the question explicitly requests including unmatched rows or zero-related entities."
|
|
1826
|
-
)
|
|
1827
|
-
),
|
|
1828
|
-
fragment(
|
|
1829
|
-
"aggregation_exactness",
|
|
1830
|
-
policy({
|
|
1831
|
-
rule: "Preserve requested aggregation semantics exactly: use COUNT(*) by default for total rows, use COUNT(DISTINCT ...) only when uniqueness is explicitly requested, and group by stable entity keys when computing per-entity aggregates."
|
|
1832
|
-
}),
|
|
1833
|
-
policy({
|
|
1834
|
-
rule: "For questions asking which entity has lowest/highest average of a metric, compute AVG(metric) per entity (GROUP BY entity) and rank those aggregates."
|
|
1835
|
-
}),
|
|
1836
|
-
hint(
|
|
1837
|
-
'For "how many <entities>" questions over relation records, default to COUNT(*) on qualifying rows unless explicit uniqueness language is present.'
|
|
1838
|
-
)
|
|
1839
|
-
),
|
|
1840
|
-
fragment(
|
|
1841
|
-
"query_shape_examples",
|
|
1842
|
-
example({
|
|
1843
|
-
question: "List categories ordered by how many records belong to each category.",
|
|
1844
|
-
answer: "SELECT category FROM records GROUP BY category ORDER BY COUNT(*)"
|
|
1845
|
-
}),
|
|
1846
|
-
example({
|
|
1847
|
-
question: "Show labels shared by rows with metric > 100 and rows with metric < 10.",
|
|
1848
|
-
answer: "SELECT label FROM records WHERE metric > 100 INTERSECT SELECT label FROM records WHERE metric < 10"
|
|
1849
|
-
}),
|
|
1850
|
-
example({
|
|
1851
|
-
question: "List locations that have not hosted any event.",
|
|
1852
|
-
answer: "SELECT location_name FROM locations WHERE location_id NOT IN (SELECT host_location_id FROM events)"
|
|
1853
|
-
}),
|
|
1854
|
-
example({
|
|
1855
|
-
question: "List the most common category across records.",
|
|
1856
|
-
answer: "SELECT category FROM records GROUP BY category ORDER BY COUNT(*) DESC LIMIT 1"
|
|
1857
|
-
})
|
|
1858
|
-
)
|
|
1859
|
-
];
|
|
1860
|
-
function extractSql(output) {
|
|
1861
|
-
const match = output.match(/```sql\n?([\s\S]*?)```/);
|
|
1862
|
-
return match ? match[1].trim() : output.trim();
|
|
1863
|
-
}
|
|
1864
|
-
async function toSql(options) {
|
|
1865
|
-
const { maxRetries = 3 } = options;
|
|
1866
|
-
return withRetry(
|
|
1867
|
-
async (attemptNumber, errors, attempts) => {
|
|
1868
|
-
const context = new ContextEngine({
|
|
1869
|
-
store: new InMemoryContextStore(),
|
|
1870
|
-
chatId: `sql-gen-${crypto.randomUUID()}`,
|
|
1871
|
-
userId: "system"
|
|
1872
|
-
});
|
|
1873
|
-
context.set(
|
|
1874
|
-
persona({
|
|
1875
|
-
name: "Freya",
|
|
1876
|
-
role: SQL_AGENT_ROLE,
|
|
1877
|
-
objective: SQL_AGENT_OBJECTIVE
|
|
1878
|
-
// role: `You are a data science expert that provides well-reasoned and detailed responses.`,
|
|
1879
|
-
// objective: `Your task is to understand the schema and generate a valid SQL query to answer the question. You first think about the reasoning process as an internal monologue and then provide the user with the answer.`,
|
|
1880
|
-
}),
|
|
1881
|
-
...SQL_AGENT_POLICIES,
|
|
1882
|
-
...options.fragments
|
|
1883
|
-
);
|
|
1884
|
-
if (errors.length) {
|
|
1885
|
-
const lastError = errors.at(-1);
|
|
1886
|
-
context.set(
|
|
1887
|
-
user(dedent`
|
|
1888
|
-
Answer the following question with the SQL code. Use the piece of evidence and base your answer on the database schema.
|
|
1889
|
-
Given the question, the evidence and the database schema, return the SQL script that addresses the question.
|
|
1890
|
-
|
|
1891
|
-
Question: ${options.input}
|
|
1892
|
-
`),
|
|
1893
|
-
UnanswerableSQLError.isInstance(lastError) ? user(
|
|
1894
|
-
`<retry_instruction>Your previous response marked the task as unanswerable. Re-evaluate using best-effort schema mapping. If the core intent is answerable with existing tables/columns, return SQL. Return error only when required core intent cannot be mapped without inventing schema elements.</retry_instruction>`
|
|
1895
|
-
) : user(
|
|
1896
|
-
`<validation_error>Your previous SQL query had the following error: ${lastError?.message}. Please fix the query.</validation_error>`
|
|
1897
|
-
)
|
|
1898
|
-
);
|
|
1899
|
-
} else {
|
|
1900
|
-
context.set(
|
|
1901
|
-
user(dedent`
|
|
1902
|
-
Answer the following question with the SQL code. Use the piece of evidence and base your answer on the database schema.
|
|
1903
|
-
Given the question, the evidence and the database schema, return the SQL script that addresses the question.
|
|
1904
|
-
|
|
1905
|
-
Question: ${options.input}
|
|
1906
|
-
`)
|
|
1907
|
-
);
|
|
1908
|
-
}
|
|
1909
|
-
const temperature = RETRY_TEMPERATURES[attemptNumber - 1] ?? RETRY_TEMPERATURES[RETRY_TEMPERATURES.length - 1];
|
|
1910
|
-
const baseModel = options.model ?? groq("openai/gpt-oss-20b");
|
|
1911
|
-
const model = wrapLanguageModel({
|
|
1912
|
-
model: baseModel,
|
|
1913
|
-
middleware: defaultSettingsMiddleware({ settings: { temperature } })
|
|
1914
|
-
});
|
|
1915
|
-
const sqlOutput = structuredOutput({
|
|
1916
|
-
model,
|
|
1917
|
-
context,
|
|
1918
|
-
schema: z.object({
|
|
1919
|
-
result: z.union([
|
|
1920
|
-
z.object({
|
|
1921
|
-
sql: z.string().describe("The SQL query that answers the question"),
|
|
1922
|
-
reasoning: z.string().describe("The reasoning steps taken to generate the SQL")
|
|
1923
|
-
}),
|
|
1924
|
-
z.object({
|
|
1925
|
-
error: z.string().describe(
|
|
1926
|
-
"Error message explaining why the question cannot be answered with the given schema"
|
|
1927
|
-
)
|
|
1928
|
-
})
|
|
1929
|
-
])
|
|
1930
|
-
})
|
|
1931
|
-
});
|
|
1932
|
-
const { result: output } = await sqlOutput.generate();
|
|
1933
|
-
const finalizeSql = async (rawSql) => {
|
|
1934
|
-
const sql = options.adapter.format(extractSql(rawSql));
|
|
1935
|
-
const validationError = await options.adapter.validate(sql);
|
|
1936
|
-
if (validationError) {
|
|
1937
|
-
throw new SQLValidationError(validationError);
|
|
1938
|
-
}
|
|
1939
|
-
return {
|
|
1940
|
-
attempts,
|
|
1941
|
-
sql,
|
|
1942
|
-
errors: errors.length ? errors.map(formatErrorMessage) : void 0
|
|
1943
|
-
};
|
|
1944
|
-
};
|
|
1945
|
-
if ("error" in output) {
|
|
1946
|
-
context.set(
|
|
1947
|
-
user(
|
|
1948
|
-
"<best_effort_fallback>Do not return unanswerable. Produce the best valid SQL query that answers the core intent using only available schema entities.</best_effort_fallback>"
|
|
1949
|
-
)
|
|
1950
|
-
);
|
|
1951
|
-
const forcedSqlOutput = structuredOutput({
|
|
1952
|
-
model,
|
|
1953
|
-
context,
|
|
1954
|
-
schema: z.object({
|
|
1955
|
-
sql: z.string().describe(
|
|
1956
|
-
"Best-effort SQL query that answers the core intent using only available schema entities."
|
|
1957
|
-
),
|
|
1958
|
-
reasoning: z.string().describe("Reasoning steps for best-effort schema mapping.")
|
|
1959
|
-
})
|
|
1960
|
-
});
|
|
1961
|
-
try {
|
|
1962
|
-
const forced = await forcedSqlOutput.generate();
|
|
1963
|
-
return await finalizeSql(forced.sql);
|
|
1964
|
-
} catch (error) {
|
|
1965
|
-
if (SQLValidationError.isInstance(error) || APICallError.isInstance(error) || JSONParseError.isInstance(error) || TypeValidationError.isInstance(error) || NoObjectGeneratedError.isInstance(error) || NoOutputGeneratedError.isInstance(error) || NoContentGeneratedError.isInstance(error)) {
|
|
1966
|
-
throw error;
|
|
1967
|
-
}
|
|
1968
|
-
throw new UnanswerableSQLError(output.error);
|
|
1969
|
-
}
|
|
1970
|
-
}
|
|
1971
|
-
return await finalizeSql(output.sql);
|
|
1972
|
-
},
|
|
1973
|
-
{ retries: maxRetries - 1 }
|
|
1974
|
-
);
|
|
1975
|
-
}
|
|
1976
|
-
function formatErrorMessage(error) {
|
|
1977
|
-
if (APICallError.isInstance(error)) {
|
|
1978
|
-
if (error.message.startsWith("Failed to validate JSON")) {
|
|
1979
|
-
return `Schema validation failed: ${error.message}`;
|
|
1980
|
-
}
|
|
1981
|
-
return error.message;
|
|
1982
|
-
}
|
|
1983
|
-
if (SQLValidationError.isInstance(error)) {
|
|
1984
|
-
return `SQL Validation Error: ${error.message}`;
|
|
1985
|
-
}
|
|
1986
|
-
return error.message;
|
|
1987
|
-
}
|
|
1988
|
-
function isModelUnavailableError(error) {
|
|
1989
|
-
if (!APICallError.isInstance(error)) {
|
|
1990
|
-
return false;
|
|
1991
|
-
}
|
|
1992
|
-
const message = error.message.toLowerCase();
|
|
1993
|
-
const responseBody = (error.responseBody ?? "").toLowerCase();
|
|
1994
|
-
const is404ModelError = error.statusCode === 404 && (message.includes("model") || responseBody.includes("model_not_found"));
|
|
1995
|
-
const errorCode = typeof error.data === "object" && error.data !== null && "error" in error.data && typeof error.data.error === "object" && error.data.error !== null && "code" in error.data.error && typeof error.data.error.code === "string" ? error.data.error.code.toLowerCase() : void 0;
|
|
1996
|
-
return is404ModelError || errorCode === "model_not_found" || responseBody.includes('"code":"model_not_found"') || message.includes("model") && message.includes("does not exist or you do not have access to it");
|
|
1997
|
-
}
|
|
1998
|
-
async function withRetry(computation, options = { retries: 3 }) {
|
|
1999
|
-
const errors = [];
|
|
2000
|
-
let attempts = 0;
|
|
2001
|
-
return pRetry(
|
|
2002
|
-
(attemptNumber) => {
|
|
2003
|
-
return computation(attemptNumber, errors, ++attempts);
|
|
2004
|
-
},
|
|
2005
|
-
{
|
|
2006
|
-
retries: options.retries,
|
|
2007
|
-
shouldRetry: (context) => {
|
|
2008
|
-
if (UnanswerableSQLError.isInstance(context.error)) {
|
|
2009
|
-
return false;
|
|
2010
|
-
}
|
|
2011
|
-
if (isModelUnavailableError(context.error)) {
|
|
2012
|
-
return false;
|
|
2013
|
-
}
|
|
2014
|
-
if (SQLValidationError.isInstance(context.error)) {
|
|
2015
|
-
return true;
|
|
2016
|
-
}
|
|
2017
|
-
console.log({
|
|
2018
|
-
NoObjectGeneratedError: NoObjectGeneratedError.isInstance(
|
|
2019
|
-
context.error
|
|
2020
|
-
),
|
|
2021
|
-
NoOutputGeneratedError: NoOutputGeneratedError.isInstance(
|
|
2022
|
-
context.error
|
|
2023
|
-
),
|
|
2024
|
-
APICallError: APICallError.isInstance(context.error),
|
|
2025
|
-
JSONParseError: JSONParseError.isInstance(context.error),
|
|
2026
|
-
TypeValidationError: TypeValidationError.isInstance(context.error),
|
|
2027
|
-
NoContentGeneratedError: NoContentGeneratedError.isInstance(
|
|
2028
|
-
context.error
|
|
2029
|
-
)
|
|
2030
|
-
});
|
|
2031
|
-
return APICallError.isInstance(context.error) || JSONParseError.isInstance(context.error) || TypeValidationError.isInstance(context.error) || NoObjectGeneratedError.isInstance(context.error) || NoOutputGeneratedError.isInstance(context.error) || NoContentGeneratedError.isInstance(context.error);
|
|
2032
|
-
},
|
|
2033
|
-
onFailedAttempt(context) {
|
|
2034
|
-
errors.push(context.error);
|
|
2035
|
-
}
|
|
2036
|
-
}
|
|
2037
|
-
);
|
|
2038
|
-
}
|
|
2039
|
-
|
|
2040
|
-
// packages/text2sql/src/lib/agents/suggestions.agents.ts
|
|
2041
|
-
import { groq as groq2 } from "@ai-sdk/groq";
|
|
2042
|
-
import dedent2 from "dedent";
|
|
2043
|
-
import z2 from "zod";
|
|
2044
|
-
import { agent, thirdPersonPrompt } from "@deepagents/agent";
|
|
2045
|
-
var suggestionsAgent = agent({
|
|
2046
|
-
name: "text2sql-suggestions",
|
|
2047
|
-
model: groq2("openai/gpt-oss-20b"),
|
|
2048
|
-
output: z2.object({
|
|
2049
|
-
suggestions: z2.array(
|
|
2050
|
-
z2.object({
|
|
2051
|
-
question: z2.string().describe("A complex, high-impact business question."),
|
|
2052
|
-
sql: z2.string().describe("The SQL statement needed to answer the question."),
|
|
2053
|
-
businessValue: z2.string().describe("Why the question matters to stakeholders.")
|
|
2054
|
-
})
|
|
2055
|
-
).min(1).max(5).describe("A set of up to two advanced question + SQL pairs.")
|
|
2056
|
-
}),
|
|
2057
|
-
prompt: (state) => {
|
|
2058
|
-
return dedent2`
|
|
2059
|
-
${thirdPersonPrompt()}
|
|
2060
|
-
|
|
2061
|
-
<identity>
|
|
2062
|
-
You are a senior analytics strategist who proposes ambitious business questions
|
|
2063
|
-
and drafts the SQL needed to answer them. You specialize in identifying ideas
|
|
2064
|
-
that combine multiple tables, apply segmentation or time analysis, and surface
|
|
2065
|
-
metrics that drive executive decisions.
|
|
2066
|
-
</identity>
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
<instructions>
|
|
2070
|
-
- Recommend one or two UNIQUE questions that go beyond simple counts or listings.
|
|
2071
|
-
- Favor questions that require joins, aggregates, time comparisons, cohort analysis,
|
|
2072
|
-
or window functions.
|
|
2073
|
-
- For each question, explain the business reason stakeholders care about it.
|
|
2074
|
-
- Provide the complete SQL query that could answer the question in the given schema.
|
|
2075
|
-
- Keep result sets scoped with LIMIT clauses (max 50 rows) when returning raw rows.
|
|
2076
|
-
- Ensure table/column names match the provided schema exactly.
|
|
2077
|
-
- Use columns marked [LowCardinality: ...] to identify meaningful categorical filters or segmentations.
|
|
2078
|
-
- Leverage table [rows / size] hints to determine whether to aggregate (large tables) or inspect detailed data (tiny tables).
|
|
2079
|
-
- Reference PK/Indexed annotations and the Indexes list to recommend queries that use efficient join/filter paths.
|
|
2080
|
-
- Column annotations may expose ranges/null percentages—use them to suggest realistic thresholds or quality checks.
|
|
2081
|
-
- Consult <relationship_examples> to anchor your recommendations in the actual join paths between tables.
|
|
2082
|
-
- Output only information grounded in the schema/context provided.
|
|
2083
|
-
</instructions>
|
|
2084
|
-
|
|
2085
|
-
<response-format>
|
|
2086
|
-
Return valid JSON that satisfies the defined output schema.
|
|
2087
|
-
</response-format>
|
|
2088
|
-
`;
|
|
2089
|
-
}
|
|
2090
|
-
});
|
|
2091
|
-
|
|
2092
|
-
// packages/text2sql/src/lib/checkpoint.ts
|
|
2093
|
-
import { createHash as createHash2 } from "node:crypto";
|
|
2094
|
-
import { existsSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
2095
|
-
import pLimit from "p-limit";
|
|
2096
|
-
var Checkpoint = class _Checkpoint {
|
|
2097
|
-
points;
|
|
2098
|
-
path;
|
|
2099
|
-
configHash;
|
|
2100
|
-
constructor(path6, configHash, points) {
|
|
2101
|
-
this.points = points;
|
|
2102
|
-
this.path = path6;
|
|
2103
|
-
this.configHash = configHash;
|
|
2104
|
-
}
|
|
2105
|
-
/**
|
|
2106
|
-
* Load checkpoint from file, or return empty checkpoint if none exists.
|
|
2107
|
-
* Handles corrupted files and config changes gracefully.
|
|
2108
|
-
*/
|
|
2109
|
-
static async load(options) {
|
|
2110
|
-
const { path: path6, configHash } = options;
|
|
2111
|
-
if (existsSync(path6)) {
|
|
2112
|
-
try {
|
|
2113
|
-
const content = readFileSync(path6, "utf-8");
|
|
2114
|
-
const file = JSON.parse(content);
|
|
2115
|
-
if (configHash && file.configHash && file.configHash !== configHash) {
|
|
2116
|
-
console.log("\u26A0 Config changed, starting fresh");
|
|
2117
|
-
return new _Checkpoint(path6, configHash, {});
|
|
2118
|
-
}
|
|
2119
|
-
const points = file.points ?? {};
|
|
2120
|
-
const totalEntries = Object.values(points).reduce(
|
|
2121
|
-
(sum, p) => sum + p.entries.length,
|
|
2122
|
-
0
|
|
2123
|
-
);
|
|
2124
|
-
console.log(`\u2713 Resuming from checkpoint (${totalEntries} entries)`);
|
|
2125
|
-
return new _Checkpoint(path6, configHash, points);
|
|
2126
|
-
} catch {
|
|
2127
|
-
console.log("\u26A0 Checkpoint corrupted, starting fresh");
|
|
2128
|
-
return new _Checkpoint(path6, configHash, {});
|
|
2129
|
-
}
|
|
2130
|
-
}
|
|
2131
|
-
console.log("Starting new checkpoint");
|
|
2132
|
-
return new _Checkpoint(path6, configHash, {});
|
|
2133
|
-
}
|
|
2134
|
-
/**
|
|
2135
|
-
* Run a single computation with checkpointing.
|
|
2136
|
-
* If already completed, returns cached value.
|
|
2137
|
-
*
|
|
2138
|
-
* @param key - Unique identifier for this computation
|
|
2139
|
-
* @param computation - Async function that produces the value
|
|
2140
|
-
* @param codec - Optional codec for encoding/decoding non-primitive values
|
|
2141
|
-
*/
|
|
2142
|
-
async run(key, computation, codec) {
|
|
2143
|
-
const point = this.point(key);
|
|
2144
|
-
return point.through(
|
|
2145
|
-
"single",
|
|
2146
|
-
async () => {
|
|
2147
|
-
const result = await computation();
|
|
2148
|
-
return codec ? codec.encode(result) : result;
|
|
2149
|
-
},
|
|
2150
|
-
codec
|
|
2151
|
-
);
|
|
2152
|
-
}
|
|
2153
|
-
/**
|
|
2154
|
-
* Create a resumable checkpoint point for iterative operations.
|
|
2155
|
-
*
|
|
2156
|
-
* @param step - Unique identifier for this checkpoint point
|
|
2157
|
-
*/
|
|
2158
|
-
point(step) {
|
|
2159
|
-
this.points[step] ??= { committed: false, entries: [] };
|
|
2160
|
-
return new Point(this.points[step], () => this.save());
|
|
2161
|
-
}
|
|
2162
|
-
/**
|
|
2163
|
-
* Process each input with automatic checkpointing and concurrency.
|
|
2164
|
-
*
|
|
2165
|
-
* @param step - Unique identifier for this checkpoint
|
|
2166
|
-
* @param inputs - Items to process
|
|
2167
|
-
* @param process - Function to process each input
|
|
2168
|
-
* @param options - Optional settings like concurrency
|
|
2169
|
-
* @returns All outputs (use `.flat()` if outputs are arrays)
|
|
2170
|
-
*/
|
|
2171
|
-
async each(step, inputs, process, options) {
|
|
2172
|
-
const point = this.point(step);
|
|
2173
|
-
const limit = pLimit(options?.concurrency ?? 1);
|
|
2174
|
-
const inputArray = Array.from(inputs);
|
|
2175
|
-
await Promise.all(
|
|
2176
|
-
inputArray.map(
|
|
2177
|
-
(input) => limit(() => point.through(input, () => process(input)))
|
|
2178
|
-
)
|
|
2179
|
-
);
|
|
2180
|
-
await point.commit();
|
|
2181
|
-
return point.values();
|
|
2182
|
-
}
|
|
2183
|
-
/**
|
|
2184
|
-
* Get clean output from all completed points.
|
|
2185
|
-
* Single-entry points return the value directly, multi-entry return arrays.
|
|
2186
|
-
*/
|
|
2187
|
-
getOutput() {
|
|
2188
|
-
const output = {};
|
|
2189
|
-
for (const [key, pointData] of Object.entries(this.points)) {
|
|
2190
|
-
if (pointData.entries.length === 1) {
|
|
2191
|
-
output[key] = pointData.entries[0].output;
|
|
2192
|
-
} else {
|
|
2193
|
-
output[key] = pointData.entries.map((e) => e.output);
|
|
2194
|
-
}
|
|
2195
|
-
}
|
|
2196
|
-
return output;
|
|
2197
|
-
}
|
|
2198
|
-
/** Get the file path where checkpoint is stored */
|
|
2199
|
-
getPath() {
|
|
2200
|
-
return this.path;
|
|
2201
|
-
}
|
|
2202
|
-
async save() {
|
|
2203
|
-
const file = {
|
|
2204
|
-
configHash: this.configHash,
|
|
2205
|
-
points: this.points
|
|
2206
|
-
};
|
|
2207
|
-
const content = JSON.stringify(file, null, 2);
|
|
2208
|
-
const tempPath = `${this.path}.tmp`;
|
|
2209
|
-
writeFileSync(tempPath, content);
|
|
2210
|
-
renameSync(tempPath, this.path);
|
|
2211
|
-
}
|
|
2212
|
-
};
|
|
2213
|
-
function hash(value) {
|
|
2214
|
-
return createHash2("md5").update(JSON.stringify(value)).digest("hex");
|
|
2215
|
-
}
|
|
2216
|
-
var Point = class {
|
|
2217
|
-
#cache;
|
|
2218
|
-
data;
|
|
2219
|
-
persist;
|
|
2220
|
-
constructor(data, persist) {
|
|
2221
|
-
this.#cache = new Map(
|
|
2222
|
-
data.entries.map((e) => [e.inputHash, e.output])
|
|
2223
|
-
);
|
|
2224
|
-
this.data = data;
|
|
2225
|
-
this.persist = persist;
|
|
2226
|
-
}
|
|
2227
|
-
/**
|
|
2228
|
-
* Execute computation if input wasn't processed before.
|
|
2229
|
-
* Returns cached output if input hash exists, otherwise executes, saves, and returns.
|
|
2230
|
-
*/
|
|
2231
|
-
async through(input, compute, codec) {
|
|
2232
|
-
const inputHash = hash(input);
|
|
2233
|
-
if (this.#cache.has(inputHash)) {
|
|
2234
|
-
const cached = this.#cache.get(inputHash);
|
|
2235
|
-
return codec ? codec.decode(cached) : cached;
|
|
2236
|
-
}
|
|
2237
|
-
const output = await compute();
|
|
2238
|
-
this.data.entries.push({ inputHash, output });
|
|
2239
|
-
this.#cache.set(inputHash, output);
|
|
2240
|
-
await this.persist();
|
|
2241
|
-
return codec ? codec.decode(output) : output;
|
|
2242
|
-
}
|
|
2243
|
-
/** Mark this point as complete. */
|
|
2244
|
-
async commit() {
|
|
2245
|
-
this.data.committed = true;
|
|
2246
|
-
await this.persist();
|
|
2247
|
-
}
|
|
2248
|
-
/** Check if this point has been committed. */
|
|
2249
|
-
isCommitted() {
|
|
2250
|
-
return this.data.committed;
|
|
2251
|
-
}
|
|
2252
|
-
/** Get all outputs from this point. */
|
|
2253
|
-
values() {
|
|
2254
|
-
return this.data.entries.map((e) => e.output);
|
|
2255
|
-
}
|
|
2256
|
-
};
|
|
2257
|
-
function hashConfig(config) {
|
|
2258
|
-
return createHash2("md5").update(JSON.stringify(config)).digest("hex");
|
|
2259
|
-
}
|
|
2260
|
-
|
|
2261
|
-
// packages/text2sql/src/lib/file-cache.ts
|
|
2262
|
-
import { createHash as createHash3 } from "node:crypto";
|
|
2263
|
-
import { existsSync as existsSync2 } from "node:fs";
|
|
2264
|
-
import { readFile, writeFile } from "node:fs/promises";
|
|
2265
|
-
import { tmpdir } from "node:os";
|
|
2266
|
-
import path2 from "node:path";
|
|
2267
|
-
var FileCache = class {
|
|
2268
|
-
path;
|
|
2269
|
-
constructor(watermark, extension = ".txt") {
|
|
2270
|
-
const hash2 = createHash3("md5").update(watermark).digest("hex");
|
|
2271
|
-
this.path = path2.join(tmpdir(), `text2sql-${hash2}${extension}`);
|
|
2272
|
-
}
|
|
2273
|
-
async get() {
|
|
2274
|
-
if (existsSync2(this.path)) {
|
|
2275
|
-
return readFile(this.path, "utf-8");
|
|
2276
|
-
}
|
|
2277
|
-
return null;
|
|
2278
|
-
}
|
|
2279
|
-
set(content) {
|
|
2280
|
-
return writeFile(this.path, content, "utf-8");
|
|
2281
|
-
}
|
|
2282
|
-
};
|
|
2283
|
-
var JsonCache = class extends FileCache {
|
|
2284
|
-
constructor(watermark) {
|
|
2285
|
-
super(watermark, ".json");
|
|
2286
|
-
}
|
|
2287
|
-
async read() {
|
|
2288
|
-
const content = await this.get();
|
|
2289
|
-
if (content) {
|
|
2290
|
-
return JSON.parse(content);
|
|
2291
|
-
}
|
|
2292
|
-
return null;
|
|
2293
|
-
}
|
|
2294
|
-
write(data) {
|
|
2295
|
-
return this.set(JSON.stringify(data));
|
|
2296
|
-
}
|
|
2297
|
-
};
|
|
2298
|
-
|
|
2299
|
-
// packages/text2sql/src/lib/fs/sqlite/sqlite-fs.ts
|
|
2300
|
-
import * as path3 from "node:path";
|
|
2301
|
-
import { DatabaseSync } from "node:sqlite";
|
|
2302
|
-
|
|
2303
|
-
// packages/text2sql/src/lib/fs/sqlite/ddl.sqlite-fs.sql
|
|
2304
|
-
var ddl_sqlite_fs_default = "-- SQLite-based filesystem schema for artifact storage\n-- Tables: fs_entries (file/directory metadata), fs_chunks (file content)\n\n-- Performance PRAGMAs (session-level, run on each connection)\nPRAGMA journal_mode = WAL;\nPRAGMA synchronous = NORMAL;\nPRAGMA cache_size = -64000;\nPRAGMA temp_store = MEMORY;\nPRAGMA mmap_size = 268435456;\n\n-- Integrity\nPRAGMA foreign_keys = ON;\n\n-- Filesystem entries table (files, directories, symlinks)\nCREATE TABLE IF NOT EXISTS fs_entries (\n path TEXT PRIMARY KEY, -- Normalized absolute path (e.g., '/results/uuid.json')\n type TEXT NOT NULL, -- 'file' | 'directory' | 'symlink'\n mode INTEGER NOT NULL, -- Unix permissions (e.g., 0o644 = 420)\n size INTEGER NOT NULL, -- File size in bytes (0 for directories)\n mtime INTEGER NOT NULL, -- Modified time (milliseconds since epoch)\n symlinkTarget TEXT -- Target path for symlinks (NULL for files/dirs)\n);\n\nCREATE INDEX IF NOT EXISTS idx_fs_entries_type ON fs_entries(type);\n\n-- File content chunks table (for handling large files)\n-- Files are split into 1MB chunks to avoid SQLite BLOB performance issues\nCREATE TABLE IF NOT EXISTS fs_chunks (\n path TEXT NOT NULL, -- Reference to fs_entries.path\n chunkIndex INTEGER NOT NULL, -- 0-based chunk sequence\n data BLOB NOT NULL, -- Chunk content (up to 1MB default)\n PRIMARY KEY (path, chunkIndex),\n FOREIGN KEY (path) REFERENCES fs_entries(path) ON DELETE CASCADE ON UPDATE CASCADE\n);\n";
|
|
2305
|
-
|
|
2306
|
-
// packages/text2sql/src/lib/fs/sqlite/sqlite-fs.ts
|
|
2307
|
-
var SqliteFs = class {
|
|
2308
|
-
#db;
|
|
2309
|
-
#statements = /* @__PURE__ */ new Map();
|
|
2310
|
-
#chunkSize;
|
|
2311
|
-
#root;
|
|
2312
|
-
constructor(options) {
|
|
2313
|
-
this.#chunkSize = options.chunkSize ?? 1024 * 1024;
|
|
2314
|
-
const normalizedRoot = this.#normalizeRoot(options.root);
|
|
2315
|
-
this.#root = normalizedRoot === "/" ? "" : normalizedRoot;
|
|
2316
|
-
this.#db = new DatabaseSync(options.dbPath);
|
|
2317
|
-
this.#db.exec(ddl_sqlite_fs_default);
|
|
2318
|
-
const rootSlashExists = this.#stmt(
|
|
2319
|
-
"SELECT 1 FROM fs_entries WHERE path = ?"
|
|
2320
|
-
).get("/");
|
|
2321
|
-
if (!rootSlashExists) {
|
|
2322
|
-
this.#stmt(
|
|
2323
|
-
`INSERT INTO fs_entries (path, type, mode, size, mtime)
|
|
2324
|
-
VALUES ('/', 'directory', 493, 0, ?)`
|
|
2325
|
-
).run(Date.now());
|
|
2326
|
-
}
|
|
2327
|
-
if (this.#root) {
|
|
2328
|
-
this.#createParentDirs(this.#root);
|
|
2329
|
-
const rootExists = this.#stmt(
|
|
2330
|
-
"SELECT 1 FROM fs_entries WHERE path = ?"
|
|
2331
|
-
).get(this.#root);
|
|
2332
|
-
if (!rootExists) {
|
|
2333
|
-
this.#stmt(
|
|
2334
|
-
`INSERT INTO fs_entries (path, type, mode, size, mtime)
|
|
2335
|
-
VALUES (?, 'directory', 493, 0, ?)`
|
|
2336
|
-
).run(this.#root, Date.now());
|
|
2337
|
-
}
|
|
2338
|
-
}
|
|
2339
|
-
}
|
|
2340
|
-
/**
|
|
2341
|
-
* Create parent directories for a path (used during initialization).
|
|
2342
|
-
* Creates all segments EXCEPT the last one (the path itself).
|
|
2343
|
-
*/
|
|
2344
|
-
#createParentDirs(p) {
|
|
2345
|
-
const segments = p.split("/").filter(Boolean);
|
|
2346
|
-
let currentPath = "/";
|
|
2347
|
-
for (let i = 0; i < segments.length - 1; i++) {
|
|
2348
|
-
currentPath = path3.posix.join(currentPath, segments[i]);
|
|
2349
|
-
const exists = this.#stmt("SELECT 1 FROM fs_entries WHERE path = ?").get(
|
|
2350
|
-
currentPath
|
|
2351
|
-
);
|
|
2352
|
-
if (!exists) {
|
|
2353
|
-
this.#stmt(
|
|
2354
|
-
`INSERT INTO fs_entries (path, type, mode, size, mtime)
|
|
2355
|
-
VALUES (?, 'directory', 493, 0, ?)`
|
|
2356
|
-
).run(currentPath, Date.now());
|
|
2357
|
-
}
|
|
2358
|
-
}
|
|
2359
|
-
}
|
|
2360
|
-
#stmt(sql) {
|
|
2361
|
-
let stmt = this.#statements.get(sql);
|
|
2362
|
-
if (!stmt) {
|
|
2363
|
-
stmt = this.#db.prepare(sql);
|
|
2364
|
-
this.#statements.set(sql, stmt);
|
|
2365
|
-
}
|
|
2366
|
-
return stmt;
|
|
2367
|
-
}
|
|
2368
|
-
#normalizeRoot(root) {
|
|
2369
|
-
return path3.posix.resolve("/", root.trim());
|
|
2370
|
-
}
|
|
2371
|
-
#prefixPath(p) {
|
|
2372
|
-
if (!this.#root) {
|
|
2373
|
-
return p;
|
|
2374
|
-
}
|
|
2375
|
-
if (p === "/") {
|
|
2376
|
-
return this.#root;
|
|
2377
|
-
}
|
|
2378
|
-
return path3.posix.join(this.#root, p);
|
|
2379
|
-
}
|
|
2380
|
-
#unprefixPath(p) {
|
|
2381
|
-
if (!this.#root) {
|
|
2382
|
-
return p;
|
|
2383
|
-
}
|
|
2384
|
-
if (p === this.#root) {
|
|
2385
|
-
return "/";
|
|
2386
|
-
}
|
|
2387
|
-
if (p.startsWith(this.#root + "/")) {
|
|
2388
|
-
return p.slice(this.#root.length) || "/";
|
|
2389
|
-
}
|
|
2390
|
-
return p;
|
|
2391
|
-
}
|
|
2392
|
-
#useTransaction(fn) {
|
|
2393
|
-
this.#db.exec("BEGIN TRANSACTION");
|
|
2394
|
-
try {
|
|
2395
|
-
const result = fn();
|
|
2396
|
-
this.#db.exec("COMMIT");
|
|
2397
|
-
return result;
|
|
2398
|
-
} catch (error) {
|
|
2399
|
-
this.#db.exec("ROLLBACK");
|
|
2400
|
-
throw error;
|
|
2401
|
-
}
|
|
2402
|
-
}
|
|
2403
|
-
#normalizePath(p) {
|
|
2404
|
-
return path3.posix.resolve("/", p);
|
|
2405
|
-
}
|
|
2406
|
-
#dirname(p) {
|
|
2407
|
-
const dir = path3.posix.dirname(p);
|
|
2408
|
-
return dir === "" ? "/" : dir;
|
|
2409
|
-
}
|
|
2410
|
-
#ensureParentExists(filePath) {
|
|
2411
|
-
const parent = this.#dirname(filePath);
|
|
2412
|
-
const rootPath = this.#root || "/";
|
|
2413
|
-
if (parent === rootPath || parent === "/") return;
|
|
2414
|
-
const entry = this.#stmt("SELECT type FROM fs_entries WHERE path = ?").get(
|
|
2415
|
-
parent
|
|
2416
|
-
);
|
|
2417
|
-
if (!entry) {
|
|
2418
|
-
this.#ensureParentExists(parent);
|
|
2419
|
-
this.#stmt(
|
|
2420
|
-
`INSERT INTO fs_entries (path, type, mode, size, mtime)
|
|
2421
|
-
VALUES (?, 'directory', 493, 0, ?)`
|
|
2422
|
-
).run(parent, Date.now());
|
|
2423
|
-
} else if (entry.type !== "directory") {
|
|
2424
|
-
throw new Error(`mkdir: parent is not a directory: ${parent}`);
|
|
2425
|
-
}
|
|
2426
|
-
}
|
|
2427
|
-
#writeChunks(filePath, content) {
|
|
2428
|
-
this.#stmt("DELETE FROM fs_chunks WHERE path = ?").run(filePath);
|
|
2429
|
-
for (let i = 0; i < content.length; i += this.#chunkSize) {
|
|
2430
|
-
const chunk = content.slice(
|
|
2431
|
-
i,
|
|
2432
|
-
Math.min(i + this.#chunkSize, content.length)
|
|
2433
|
-
);
|
|
2434
|
-
this.#stmt(
|
|
2435
|
-
"INSERT INTO fs_chunks (path, chunkIndex, data) VALUES (?, ?, ?)"
|
|
2436
|
-
).run(filePath, Math.floor(i / this.#chunkSize), chunk);
|
|
2437
|
-
}
|
|
2438
|
-
}
|
|
2439
|
-
#readChunks(filePath) {
|
|
2440
|
-
const rows = this.#stmt(
|
|
2441
|
-
"SELECT data FROM fs_chunks WHERE path = ? ORDER BY chunkIndex"
|
|
2442
|
-
).all(filePath);
|
|
2443
|
-
if (rows.length === 0) {
|
|
2444
|
-
return new Uint8Array(0);
|
|
2445
|
-
}
|
|
2446
|
-
const totalSize = rows.reduce((sum, row) => sum + row.data.length, 0);
|
|
2447
|
-
const result = new Uint8Array(totalSize);
|
|
2448
|
-
let offset = 0;
|
|
2449
|
-
for (const row of rows) {
|
|
2450
|
-
result.set(row.data, offset);
|
|
2451
|
-
offset += row.data.length;
|
|
2452
|
-
}
|
|
2453
|
-
return result;
|
|
2454
|
-
}
|
|
2455
|
-
#resolveSymlink(p, seen = /* @__PURE__ */ new Set()) {
|
|
2456
|
-
if (seen.has(p)) {
|
|
2457
|
-
throw new Error(`readFile: circular symlink: ${p}`);
|
|
2458
|
-
}
|
|
2459
|
-
const entry = this.#stmt(
|
|
2460
|
-
"SELECT type, symlinkTarget FROM fs_entries WHERE path = ?"
|
|
2461
|
-
).get(p);
|
|
2462
|
-
if (!entry) {
|
|
2463
|
-
throw new Error(`ENOENT: no such file or directory: ${p}`);
|
|
2464
|
-
}
|
|
2465
|
-
if (entry.type !== "symlink") {
|
|
2466
|
-
return p;
|
|
2467
|
-
}
|
|
2468
|
-
seen.add(p);
|
|
2469
|
-
const target = this.#normalizePath(
|
|
2470
|
-
path3.posix.resolve(this.#dirname(p), entry.symlinkTarget)
|
|
2471
|
-
);
|
|
2472
|
-
return this.#resolveSymlink(target, seen);
|
|
2473
|
-
}
|
|
2474
|
-
#toUint8Array(content, encoding) {
|
|
2475
|
-
if (content instanceof Uint8Array) {
|
|
2476
|
-
return content;
|
|
2477
|
-
}
|
|
2478
|
-
const enc = encoding ?? "utf8";
|
|
2479
|
-
return new Uint8Array(Buffer.from(content, enc));
|
|
1811
|
+
#toUint8Array(content, encoding) {
|
|
1812
|
+
if (content instanceof Uint8Array) {
|
|
1813
|
+
return content;
|
|
1814
|
+
}
|
|
1815
|
+
const enc = encoding ?? "utf8";
|
|
1816
|
+
return new Uint8Array(Buffer.from(content, enc));
|
|
2480
1817
|
}
|
|
2481
1818
|
// ============================================================================
|
|
2482
1819
|
// IFileSystem Implementation
|
|
@@ -2619,11 +1956,11 @@ var SqliteFs = class {
|
|
|
2619
1956
|
this.#useTransaction(() => {
|
|
2620
1957
|
if (options?.recursive) {
|
|
2621
1958
|
const rootPath = this.#root || "/";
|
|
2622
|
-
const relativePath =
|
|
1959
|
+
const relativePath = path2.posix.relative(rootPath, prefixed);
|
|
2623
1960
|
const segments = relativePath.split("/").filter(Boolean);
|
|
2624
1961
|
let currentPath = rootPath;
|
|
2625
1962
|
for (const segment of segments) {
|
|
2626
|
-
currentPath =
|
|
1963
|
+
currentPath = path2.posix.join(currentPath, segment);
|
|
2627
1964
|
const exists = this.#stmt(
|
|
2628
1965
|
"SELECT type FROM fs_entries WHERE path = ?"
|
|
2629
1966
|
).get(currentPath);
|
|
@@ -2674,7 +2011,7 @@ var SqliteFs = class {
|
|
|
2674
2011
|
AND path != ?
|
|
2675
2012
|
AND path NOT LIKE ? || '%/%'`
|
|
2676
2013
|
).all(prefix, resolved, prefix);
|
|
2677
|
-
return rows.map((row) =>
|
|
2014
|
+
return rows.map((row) => path2.posix.basename(row.path));
|
|
2678
2015
|
}
|
|
2679
2016
|
async readdirWithFileTypes(dirPath) {
|
|
2680
2017
|
const normalized = this.#normalizePath(dirPath);
|
|
@@ -2697,7 +2034,7 @@ var SqliteFs = class {
|
|
|
2697
2034
|
AND path NOT LIKE ? || '%/%'`
|
|
2698
2035
|
).all(prefix, resolved, prefix);
|
|
2699
2036
|
return rows.map((row) => ({
|
|
2700
|
-
name:
|
|
2037
|
+
name: path2.posix.basename(row.path),
|
|
2701
2038
|
isFile: row.type === "file",
|
|
2702
2039
|
isDirectory: row.type === "directory",
|
|
2703
2040
|
isSymbolicLink: row.type === "symlink"
|
|
@@ -2752,8 +2089,8 @@ var SqliteFs = class {
|
|
|
2752
2089
|
`SELECT * FROM fs_entries WHERE path = ? OR path LIKE ? || '/%'`
|
|
2753
2090
|
).all(srcPrefixed, srcPrefixed);
|
|
2754
2091
|
for (const entry of allEntries) {
|
|
2755
|
-
const relativePath =
|
|
2756
|
-
const newPath =
|
|
2092
|
+
const relativePath = path2.posix.relative(srcPrefixed, entry.path);
|
|
2093
|
+
const newPath = path2.posix.join(destPrefixed, relativePath);
|
|
2757
2094
|
this.#stmt(
|
|
2758
2095
|
`INSERT OR REPLACE INTO fs_entries (path, type, mode, size, mtime, symlinkTarget)
|
|
2759
2096
|
VALUES (?, ?, ?, ?, ?, ?)`
|
|
@@ -2820,8 +2157,8 @@ var SqliteFs = class {
|
|
|
2820
2157
|
`SELECT * FROM fs_entries WHERE path = ? OR path LIKE ? || '/%' ORDER BY path DESC`
|
|
2821
2158
|
).all(srcPrefixed, srcPrefixed);
|
|
2822
2159
|
for (const entry of [...allEntries].reverse()) {
|
|
2823
|
-
const relativePath =
|
|
2824
|
-
const newPath =
|
|
2160
|
+
const relativePath = path2.posix.relative(srcPrefixed, entry.path);
|
|
2161
|
+
const newPath = path2.posix.join(destPrefixed, relativePath);
|
|
2825
2162
|
this.#stmt(
|
|
2826
2163
|
`INSERT INTO fs_entries (path, type, mode, size, mtime, symlinkTarget)
|
|
2827
2164
|
VALUES (?, ?, ?, ?, ?, ?)`
|
|
@@ -2874,7 +2211,7 @@ var SqliteFs = class {
|
|
|
2874
2211
|
});
|
|
2875
2212
|
}
|
|
2876
2213
|
resolvePath(base, relativePath) {
|
|
2877
|
-
return
|
|
2214
|
+
return path2.posix.resolve(base, relativePath);
|
|
2878
2215
|
}
|
|
2879
2216
|
getAllPaths() {
|
|
2880
2217
|
const rows = this.#stmt(
|
|
@@ -2986,7 +2323,7 @@ var SqliteFs = class {
|
|
|
2986
2323
|
|
|
2987
2324
|
// packages/text2sql/src/lib/fs/mssql/mssql-fs.ts
|
|
2988
2325
|
import { createRequire } from "node:module";
|
|
2989
|
-
import * as
|
|
2326
|
+
import * as path3 from "node:path";
|
|
2990
2327
|
|
|
2991
2328
|
// packages/text2sql/src/lib/fs/mssql/ddl.mssql-fs.ts
|
|
2992
2329
|
function mssqlFsDDL(schema) {
|
|
@@ -3112,7 +2449,7 @@ var MssqlFs = class _MssqlFs {
|
|
|
3112
2449
|
const segments = p.split("/").filter(Boolean);
|
|
3113
2450
|
let currentPath = "/";
|
|
3114
2451
|
for (let i = 0; i < segments.length - 1; i++) {
|
|
3115
|
-
currentPath =
|
|
2452
|
+
currentPath = path3.posix.join(currentPath, segments[i]);
|
|
3116
2453
|
const exists = await this.#rawQuery(
|
|
3117
2454
|
`SELECT CASE WHEN EXISTS(SELECT 1 FROM ${this.#t("fs_entries")} WHERE path = @p0) THEN 1 ELSE 0 END as [exists]`,
|
|
3118
2455
|
[currentPath]
|
|
@@ -3164,7 +2501,7 @@ var MssqlFs = class _MssqlFs {
|
|
|
3164
2501
|
}
|
|
3165
2502
|
}
|
|
3166
2503
|
#normalizeRoot(root) {
|
|
3167
|
-
return
|
|
2504
|
+
return path3.posix.resolve("/", root.trim());
|
|
3168
2505
|
}
|
|
3169
2506
|
#prefixPath(p) {
|
|
3170
2507
|
if (!this.#root) {
|
|
@@ -3173,7 +2510,7 @@ var MssqlFs = class _MssqlFs {
|
|
|
3173
2510
|
if (p === "/") {
|
|
3174
2511
|
return this.#root;
|
|
3175
2512
|
}
|
|
3176
|
-
return
|
|
2513
|
+
return path3.posix.join(this.#root, p);
|
|
3177
2514
|
}
|
|
3178
2515
|
#unprefixPath(p) {
|
|
3179
2516
|
if (!this.#root) {
|
|
@@ -3188,10 +2525,10 @@ var MssqlFs = class _MssqlFs {
|
|
|
3188
2525
|
return p;
|
|
3189
2526
|
}
|
|
3190
2527
|
#normalizePath(p) {
|
|
3191
|
-
return
|
|
2528
|
+
return path3.posix.resolve("/", p);
|
|
3192
2529
|
}
|
|
3193
2530
|
#dirname(p) {
|
|
3194
|
-
const dir =
|
|
2531
|
+
const dir = path3.posix.dirname(p);
|
|
3195
2532
|
return dir === "" ? "/" : dir;
|
|
3196
2533
|
}
|
|
3197
2534
|
async #ensureParentExists(filePath, transaction) {
|
|
@@ -3280,7 +2617,7 @@ var MssqlFs = class _MssqlFs {
|
|
|
3280
2617
|
}
|
|
3281
2618
|
seen.add(p);
|
|
3282
2619
|
const target = this.#normalizePath(
|
|
3283
|
-
|
|
2620
|
+
path3.posix.resolve(this.#dirname(p), entry.symlinkTarget)
|
|
3284
2621
|
);
|
|
3285
2622
|
return this.#resolveSymlink(target, seen);
|
|
3286
2623
|
}
|
|
@@ -3463,11 +2800,11 @@ var MssqlFs = class _MssqlFs {
|
|
|
3463
2800
|
await this.#useTransaction(async (transaction) => {
|
|
3464
2801
|
if (options?.recursive) {
|
|
3465
2802
|
const rootPath = this.#root || "/";
|
|
3466
|
-
const relativePath =
|
|
2803
|
+
const relativePath = path3.posix.relative(rootPath, prefixed);
|
|
3467
2804
|
const segments = relativePath.split("/").filter(Boolean);
|
|
3468
2805
|
let currentPath = rootPath;
|
|
3469
2806
|
for (const segment of segments) {
|
|
3470
|
-
currentPath =
|
|
2807
|
+
currentPath = path3.posix.join(currentPath, segment);
|
|
3471
2808
|
const checkReq = transaction.request();
|
|
3472
2809
|
checkReq.input("p0", currentPath);
|
|
3473
2810
|
const result = await checkReq.query(
|
|
@@ -3531,7 +2868,7 @@ var MssqlFs = class _MssqlFs {
|
|
|
3531
2868
|
AND path NOT LIKE @p0 + '%/%'`,
|
|
3532
2869
|
[prefix, resolved]
|
|
3533
2870
|
);
|
|
3534
|
-
return rows.map((row) =>
|
|
2871
|
+
return rows.map((row) => path3.posix.basename(row.path));
|
|
3535
2872
|
}
|
|
3536
2873
|
async readdirWithFileTypes(dirPath) {
|
|
3537
2874
|
const normalized = this.#normalizePath(dirPath);
|
|
@@ -3557,7 +2894,7 @@ var MssqlFs = class _MssqlFs {
|
|
|
3557
2894
|
[prefix, resolved]
|
|
3558
2895
|
);
|
|
3559
2896
|
return rows.map((row) => ({
|
|
3560
|
-
name:
|
|
2897
|
+
name: path3.posix.basename(row.path),
|
|
3561
2898
|
isFile: row.type === "file",
|
|
3562
2899
|
isDirectory: row.type === "directory",
|
|
3563
2900
|
isSymbolicLink: row.type === "symlink"
|
|
@@ -3626,8 +2963,8 @@ var MssqlFs = class _MssqlFs {
|
|
|
3626
2963
|
`SELECT * FROM ${this.#t("fs_entries")} WHERE path = @p0 OR path LIKE @p0 + '/%'`
|
|
3627
2964
|
);
|
|
3628
2965
|
for (const entry of allEntriesResult.recordset) {
|
|
3629
|
-
const relativePath =
|
|
3630
|
-
const newPath =
|
|
2966
|
+
const relativePath = path3.posix.relative(srcPrefixed, entry.path);
|
|
2967
|
+
const newPath = path3.posix.join(destPrefixed, relativePath);
|
|
3631
2968
|
const insertReq = transaction.request();
|
|
3632
2969
|
insertReq.input("p0", newPath);
|
|
3633
2970
|
insertReq.input("p1", entry.type);
|
|
@@ -3736,8 +3073,8 @@ var MssqlFs = class _MssqlFs {
|
|
|
3736
3073
|
`DELETE FROM ${this.#t("fs_entries")} WHERE path = @dp0 OR path LIKE @dp0 + '/%'`
|
|
3737
3074
|
);
|
|
3738
3075
|
for (const entry of [...allEntriesResult.recordset].reverse()) {
|
|
3739
|
-
const relativePath =
|
|
3740
|
-
const newPath =
|
|
3076
|
+
const relativePath = path3.posix.relative(srcPrefixed, entry.path);
|
|
3077
|
+
const newPath = path3.posix.join(destPrefixed, relativePath);
|
|
3741
3078
|
const insertReq = transaction.request();
|
|
3742
3079
|
insertReq.input("p0", newPath);
|
|
3743
3080
|
insertReq.input("p1", entry.type);
|
|
@@ -3830,7 +3167,7 @@ var MssqlFs = class _MssqlFs {
|
|
|
3830
3167
|
});
|
|
3831
3168
|
}
|
|
3832
3169
|
resolvePath(base, relativePath) {
|
|
3833
|
-
return
|
|
3170
|
+
return path3.posix.resolve(base, relativePath);
|
|
3834
3171
|
}
|
|
3835
3172
|
getAllPaths() {
|
|
3836
3173
|
throw new Error(
|
|
@@ -3971,7 +3308,7 @@ var MssqlFs = class _MssqlFs {
|
|
|
3971
3308
|
|
|
3972
3309
|
// packages/text2sql/src/lib/fs/postgres/postgres-fs.ts
|
|
3973
3310
|
import { createRequire as createRequire2 } from "node:module";
|
|
3974
|
-
import * as
|
|
3311
|
+
import * as path4 from "node:path";
|
|
3975
3312
|
|
|
3976
3313
|
// packages/text2sql/src/lib/fs/postgres/ddl.postgres-fs.ts
|
|
3977
3314
|
function postgresFsDDL(schema) {
|
|
@@ -4076,7 +3413,7 @@ var PostgresFs = class _PostgresFs {
|
|
|
4076
3413
|
const segments = p.split("/").filter(Boolean);
|
|
4077
3414
|
let currentPath = "/";
|
|
4078
3415
|
for (let i = 0; i < segments.length - 1; i++) {
|
|
4079
|
-
currentPath =
|
|
3416
|
+
currentPath = path4.posix.join(currentPath, segments[i]);
|
|
4080
3417
|
const exists = await this.#rawQuery(
|
|
4081
3418
|
`SELECT EXISTS(SELECT 1 FROM ${this.#t("fs_entries")} WHERE path = $1) AS exists`,
|
|
4082
3419
|
[currentPath]
|
|
@@ -4121,7 +3458,7 @@ var PostgresFs = class _PostgresFs {
|
|
|
4121
3458
|
}
|
|
4122
3459
|
}
|
|
4123
3460
|
#normalizeRoot(root) {
|
|
4124
|
-
return
|
|
3461
|
+
return path4.posix.resolve("/", root.trim());
|
|
4125
3462
|
}
|
|
4126
3463
|
#prefixPath(p) {
|
|
4127
3464
|
if (!this.#root) {
|
|
@@ -4130,7 +3467,7 @@ var PostgresFs = class _PostgresFs {
|
|
|
4130
3467
|
if (p === "/") {
|
|
4131
3468
|
return this.#root;
|
|
4132
3469
|
}
|
|
4133
|
-
return
|
|
3470
|
+
return path4.posix.join(this.#root, p);
|
|
4134
3471
|
}
|
|
4135
3472
|
#unprefixPath(p) {
|
|
4136
3473
|
if (!this.#root) {
|
|
@@ -4145,10 +3482,10 @@ var PostgresFs = class _PostgresFs {
|
|
|
4145
3482
|
return p;
|
|
4146
3483
|
}
|
|
4147
3484
|
#normalizePath(p) {
|
|
4148
|
-
return
|
|
3485
|
+
return path4.posix.resolve("/", p);
|
|
4149
3486
|
}
|
|
4150
3487
|
#dirname(p) {
|
|
4151
|
-
const dir =
|
|
3488
|
+
const dir = path4.posix.dirname(p);
|
|
4152
3489
|
return dir === "" ? "/" : dir;
|
|
4153
3490
|
}
|
|
4154
3491
|
async #ensureParentExists(filePath, client) {
|
|
@@ -4228,7 +3565,7 @@ var PostgresFs = class _PostgresFs {
|
|
|
4228
3565
|
}
|
|
4229
3566
|
seen.add(p);
|
|
4230
3567
|
const target = this.#normalizePath(
|
|
4231
|
-
|
|
3568
|
+
path4.posix.resolve(this.#dirname(p), entry.symlink_target)
|
|
4232
3569
|
);
|
|
4233
3570
|
return this.#resolveSymlink(target, seen);
|
|
4234
3571
|
}
|
|
@@ -4394,11 +3731,11 @@ var PostgresFs = class _PostgresFs {
|
|
|
4394
3731
|
await this.#useTransaction(async (client) => {
|
|
4395
3732
|
if (options?.recursive) {
|
|
4396
3733
|
const rootPath = this.#root || "/";
|
|
4397
|
-
const relativePath =
|
|
3734
|
+
const relativePath = path4.posix.relative(rootPath, prefixed);
|
|
4398
3735
|
const segments = relativePath.split("/").filter(Boolean);
|
|
4399
3736
|
let currentPath = rootPath;
|
|
4400
3737
|
for (const segment of segments) {
|
|
4401
|
-
currentPath =
|
|
3738
|
+
currentPath = path4.posix.join(currentPath, segment);
|
|
4402
3739
|
const result = await client.query(
|
|
4403
3740
|
`SELECT type FROM ${this.#t("fs_entries")} WHERE path = $1`,
|
|
4404
3741
|
[currentPath]
|
|
@@ -4456,7 +3793,7 @@ var PostgresFs = class _PostgresFs {
|
|
|
4456
3793
|
AND path NOT LIKE $1 || '%/%'`,
|
|
4457
3794
|
[prefix, resolved]
|
|
4458
3795
|
);
|
|
4459
|
-
return rows.map((row) =>
|
|
3796
|
+
return rows.map((row) => path4.posix.basename(row.path));
|
|
4460
3797
|
}
|
|
4461
3798
|
async readdirWithFileTypes(dirPath) {
|
|
4462
3799
|
const normalized = this.#normalizePath(dirPath);
|
|
@@ -4482,7 +3819,7 @@ var PostgresFs = class _PostgresFs {
|
|
|
4482
3819
|
[prefix, resolved]
|
|
4483
3820
|
);
|
|
4484
3821
|
return rows.map((row) => ({
|
|
4485
|
-
name:
|
|
3822
|
+
name: path4.posix.basename(row.path),
|
|
4486
3823
|
isFile: row.type === "file",
|
|
4487
3824
|
isDirectory: row.type === "directory",
|
|
4488
3825
|
isSymbolicLink: row.type === "symlink"
|
|
@@ -4547,8 +3884,8 @@ var PostgresFs = class _PostgresFs {
|
|
|
4547
3884
|
[srcPrefixed]
|
|
4548
3885
|
);
|
|
4549
3886
|
for (const entry of allEntriesResult.rows) {
|
|
4550
|
-
const relativePath =
|
|
4551
|
-
const newPath =
|
|
3887
|
+
const relativePath = path4.posix.relative(srcPrefixed, entry.path);
|
|
3888
|
+
const newPath = path4.posix.join(destPrefixed, relativePath);
|
|
4552
3889
|
await client.query(
|
|
4553
3890
|
`INSERT INTO ${this.#t("fs_entries")} (path, type, mode, size, mtime, symlink_target)
|
|
4554
3891
|
VALUES ($1, $2, $3, $4, $5, $6)
|
|
@@ -4637,8 +3974,8 @@ var PostgresFs = class _PostgresFs {
|
|
|
4637
3974
|
[destPrefixed]
|
|
4638
3975
|
);
|
|
4639
3976
|
for (const entry of [...allEntriesResult.rows].reverse()) {
|
|
4640
|
-
const relativePath =
|
|
4641
|
-
const newPath =
|
|
3977
|
+
const relativePath = path4.posix.relative(srcPrefixed, entry.path);
|
|
3978
|
+
const newPath = path4.posix.join(destPrefixed, relativePath);
|
|
4642
3979
|
await client.query(
|
|
4643
3980
|
`INSERT INTO ${this.#t("fs_entries")} (path, type, mode, size, mtime, symlink_target)
|
|
4644
3981
|
VALUES ($1, $2, $3, $4, $5, $6)
|
|
@@ -4711,7 +4048,7 @@ var PostgresFs = class _PostgresFs {
|
|
|
4711
4048
|
});
|
|
4712
4049
|
}
|
|
4713
4050
|
resolvePath(base, relativePath) {
|
|
4714
|
-
return
|
|
4051
|
+
return path4.posix.resolve(base, relativePath);
|
|
4715
4052
|
}
|
|
4716
4053
|
getAllPaths() {
|
|
4717
4054
|
throw new Error(
|
|
@@ -4930,7 +4267,7 @@ import {
|
|
|
4930
4267
|
clarification,
|
|
4931
4268
|
example as example2,
|
|
4932
4269
|
explain,
|
|
4933
|
-
fragment as
|
|
4270
|
+
fragment as fragment3,
|
|
4934
4271
|
guardrail as guardrail2,
|
|
4935
4272
|
hint as hint2,
|
|
4936
4273
|
policy as policy2,
|
|
@@ -4939,12 +4276,12 @@ import {
|
|
|
4939
4276
|
styleGuide,
|
|
4940
4277
|
workflow as workflow2
|
|
4941
4278
|
} from "@deepagents/context";
|
|
4942
|
-
function
|
|
4279
|
+
function instructions() {
|
|
4943
4280
|
const baseTeachings = [
|
|
4944
4281
|
// Include the meta-cognitive reasoning framework
|
|
4945
4282
|
...reasoningFramework(),
|
|
4946
4283
|
// Prerequisite policies (must do X before Y)
|
|
4947
|
-
|
|
4284
|
+
fragment3(
|
|
4948
4285
|
"prerequisite_policies",
|
|
4949
4286
|
policy2({
|
|
4950
4287
|
rule: "YOU MUST inspect schema structure and available tables",
|
|
@@ -4968,7 +4305,7 @@ function guidelines() {
|
|
|
4968
4305
|
})
|
|
4969
4306
|
),
|
|
4970
4307
|
// Few-shot: Applying reasoning principles
|
|
4971
|
-
|
|
4308
|
+
fragment3(
|
|
4972
4309
|
"reasoning_examples",
|
|
4973
4310
|
example2({
|
|
4974
4311
|
question: "Show me sales last month",
|
|
@@ -5008,7 +4345,7 @@ Action: Ask user: "Top by what metric\u2014total revenue, number of orders, or m
|
|
|
5008
4345
|
})
|
|
5009
4346
|
),
|
|
5010
4347
|
// Schema adherence - consolidated into clear rules
|
|
5011
|
-
|
|
4348
|
+
fragment3(
|
|
5012
4349
|
"schema_adherence",
|
|
5013
4350
|
guardrail2({
|
|
5014
4351
|
rule: "Use only tables and columns that exist in the schema.",
|
|
@@ -5021,7 +4358,7 @@ Action: Ask user: "Top by what metric\u2014total revenue, number of orders, or m
|
|
|
5021
4358
|
therefore: 'Use listing queries for "show" requests, aggregate queries for "count" or "total", and use canonical schema values verbatim in filters.'
|
|
5022
4359
|
})
|
|
5023
4360
|
),
|
|
5024
|
-
|
|
4361
|
+
fragment3(
|
|
5025
4362
|
"column_statistics",
|
|
5026
4363
|
explain({
|
|
5027
4364
|
concept: "nDistinct in column stats",
|
|
@@ -5042,7 +4379,7 @@ Action: Ask user: "Top by what metric\u2014total revenue, number of orders, or m
|
|
|
5042
4379
|
"Use JOINs based on schema relationships. Favor PK/indexed columns; follow relationship metadata for direction and cardinality."
|
|
5043
4380
|
),
|
|
5044
4381
|
// Aggregations - explain the concepts
|
|
5045
|
-
|
|
4382
|
+
fragment3(
|
|
5046
4383
|
"aggregations",
|
|
5047
4384
|
hint2(
|
|
5048
4385
|
"Apply COUNT, SUM, AVG when the question implies summarization. Use window functions for ranking, running totals, or row comparisons."
|
|
@@ -5054,7 +4391,7 @@ Action: Ask user: "Top by what metric\u2014total revenue, number of orders, or m
|
|
|
5054
4391
|
})
|
|
5055
4392
|
),
|
|
5056
4393
|
// Query semantics - explain concepts and document quirks
|
|
5057
|
-
|
|
4394
|
+
fragment3(
|
|
5058
4395
|
"query_interpretation",
|
|
5059
4396
|
explain({
|
|
5060
4397
|
concept: "threshold language",
|
|
@@ -5082,7 +4419,7 @@ Action: Ask user: "Top by what metric\u2014total revenue, number of orders, or m
|
|
|
5082
4419
|
prefer: "Concise, business-friendly summaries with key comparisons and helpful follow-ups."
|
|
5083
4420
|
}),
|
|
5084
4421
|
// Safety guardrails - consolidated
|
|
5085
|
-
|
|
4422
|
+
fragment3(
|
|
5086
4423
|
"query_safety",
|
|
5087
4424
|
guardrail2({
|
|
5088
4425
|
rule: "Generate only valid, executable SELECT/WITH statements.",
|
|
@@ -5158,7 +4495,7 @@ Action: Ask user: "Top by what metric\u2014total revenue, number of orders, or m
|
|
|
5158
4495
|
],
|
|
5159
4496
|
notes: "If reference is ambiguous, ask which previous result or entity the user means."
|
|
5160
4497
|
}),
|
|
5161
|
-
|
|
4498
|
+
fragment3(
|
|
5162
4499
|
"bash_tool_usage",
|
|
5163
4500
|
workflow2({
|
|
5164
4501
|
task: "Query execution",
|
|
@@ -5172,7 +4509,7 @@ Action: Ask user: "Top by what metric\u2014total revenue, number of orders, or m
|
|
|
5172
4509
|
}),
|
|
5173
4510
|
guardrail2({
|
|
5174
4511
|
rule: "Do not attempt SQL access through non-bash tools.",
|
|
5175
|
-
reason: "SQL access is only available through the
|
|
4512
|
+
reason: "SQL access is only available through the bash tool.",
|
|
5176
4513
|
action: 'Use `sql run <db> "SELECT ..."` and `sql validate <db> "SELECT ..."` through bash, where `<db>` is the configured database name.'
|
|
5177
4514
|
}),
|
|
5178
4515
|
explain({
|
|
@@ -5181,7 +4518,11 @@ Action: Ask user: "Top by what metric\u2014total revenue, number of orders, or m
|
|
|
5181
4518
|
therefore: "Use the returned column names to build precise jq queries against the output file."
|
|
5182
4519
|
}),
|
|
5183
4520
|
quirk({
|
|
5184
|
-
issue: "
|
|
4521
|
+
issue: "The SQL command is a real CLI installed inside the sandbox, so shell quoting rules apply before SQL validation runs.",
|
|
4522
|
+
workaround: 'Keep the whole SQL query in one quoted argument. If quoting is difficult, write the query to a file and call `sql validate <db> "$(cat query.sql)"` or `sql run <db> "$(cat query.sql)"`.'
|
|
4523
|
+
}),
|
|
4524
|
+
quirk({
|
|
4525
|
+
issue: "SQL result artifacts live inside the sandbox, so you cannot access underlying SQL files directly.",
|
|
5185
4526
|
workaround: "Treat the returned result path as the artifact to inspect, rather than trying to access SQL files themselves."
|
|
5186
4527
|
}),
|
|
5187
4528
|
quirk({
|
|
@@ -5189,7 +4530,7 @@ Action: Ask user: "Top by what metric\u2014total revenue, number of orders, or m
|
|
|
5189
4530
|
workaround: "Read stderr first and classify the failure before retrying or changing the query."
|
|
5190
4531
|
})
|
|
5191
4532
|
),
|
|
5192
|
-
|
|
4533
|
+
fragment3(
|
|
5193
4534
|
"clarifications",
|
|
5194
4535
|
guardrail2({
|
|
5195
4536
|
rule: "Do not invent an answer when the available schema, results, or user request are insufficient to determine it.",
|
|
@@ -5207,58 +4548,952 @@ Action: Ask user: "Top by what metric\u2014total revenue, number of orders, or m
|
|
|
5207
4548
|
reason: "Prevents large scans and irrelevant results."
|
|
5208
4549
|
})
|
|
5209
4550
|
)
|
|
5210
|
-
];
|
|
5211
|
-
return baseTeachings;
|
|
4551
|
+
];
|
|
4552
|
+
return baseTeachings;
|
|
4553
|
+
}
|
|
4554
|
+
|
|
4555
|
+
// packages/text2sql/src/lib/sql-meta-hook.ts
|
|
4556
|
+
import { parse } from "just-bash";
|
|
4557
|
+
import { asStaticWordText, useBashMeta } from "@deepagents/context";
|
|
4558
|
+
var SQL_VALIDATE_REMINDER = 'Always run `sql validate <db> "..."` before `sql run <db> "..."` to catch syntax errors early.';
|
|
4559
|
+
function createSqlMetaHook({ adapters }) {
|
|
4560
|
+
const pendingByCommand = /* @__PURE__ */ new Map();
|
|
4561
|
+
return {
|
|
4562
|
+
track({ repaired, transformed }) {
|
|
4563
|
+
const invocation = parseSqlQueryInvocation(repaired);
|
|
4564
|
+
if (!invocation) return;
|
|
4565
|
+
const pending = pendingByCommand.get(transformed) ?? [];
|
|
4566
|
+
pending.push(invocation);
|
|
4567
|
+
pendingByCommand.set(transformed, pending);
|
|
4568
|
+
},
|
|
4569
|
+
onAfterBashCall({ command }) {
|
|
4570
|
+
const pending = pendingByCommand.get(command);
|
|
4571
|
+
const invocation = pending?.shift();
|
|
4572
|
+
if (pending?.length === 0) pendingByCommand.delete(command);
|
|
4573
|
+
if (!invocation) return void 0;
|
|
4574
|
+
if (invocation.subcommand === "run") {
|
|
4575
|
+
useBashMeta()?.setReminder(SQL_VALIDATE_REMINDER);
|
|
4576
|
+
}
|
|
4577
|
+
const adapter = adapters[invocation.dbName];
|
|
4578
|
+
if (!adapter) return void 0;
|
|
4579
|
+
try {
|
|
4580
|
+
useBashMeta()?.setHidden({
|
|
4581
|
+
formattedSql: adapter.format(invocation.sql)
|
|
4582
|
+
});
|
|
4583
|
+
} catch {
|
|
4584
|
+
}
|
|
4585
|
+
return void 0;
|
|
4586
|
+
}
|
|
4587
|
+
};
|
|
4588
|
+
}
|
|
4589
|
+
function parseSqlQueryInvocation(command) {
|
|
4590
|
+
const simpleCommand = parseSingleSimpleCommand(command);
|
|
4591
|
+
if (!simpleCommand) return null;
|
|
4592
|
+
if (asStaticWordText(simpleCommand.name) !== "sql") return null;
|
|
4593
|
+
if (simpleCommand.args.length < 3) return null;
|
|
4594
|
+
const subcommand = asStaticWordText(simpleCommand.args[0]);
|
|
4595
|
+
if (subcommand !== "run" && subcommand !== "validate") return null;
|
|
4596
|
+
const dbName = asStaticWordText(simpleCommand.args[1]);
|
|
4597
|
+
if (!dbName) return null;
|
|
4598
|
+
const sql = joinSqlWords(simpleCommand.args.slice(2));
|
|
4599
|
+
if (!sql) return null;
|
|
4600
|
+
return { subcommand, dbName, sql };
|
|
4601
|
+
}
|
|
4602
|
+
function parseSingleSimpleCommand(command) {
|
|
4603
|
+
const normalized = command.trim();
|
|
4604
|
+
if (!normalized) return null;
|
|
4605
|
+
try {
|
|
4606
|
+
const script = parse(normalized);
|
|
4607
|
+
if (script.statements.length !== 1) return null;
|
|
4608
|
+
const statement = script.statements[0];
|
|
4609
|
+
if (statement.background || statement.operators.length > 0 || statement.pipelines.length !== 1) {
|
|
4610
|
+
return null;
|
|
4611
|
+
}
|
|
4612
|
+
const pipeline = statement.pipelines[0];
|
|
4613
|
+
if (pipeline.negated || pipeline.timed || pipeline.commands.length !== 1) {
|
|
4614
|
+
return null;
|
|
4615
|
+
}
|
|
4616
|
+
const commandNode = pipeline.commands[0];
|
|
4617
|
+
if (commandNode.type !== "SimpleCommand") return null;
|
|
4618
|
+
if (commandNode.assignments.length > 0 || commandNode.redirections.length > 0 || !commandNode.name) {
|
|
4619
|
+
return null;
|
|
4620
|
+
}
|
|
4621
|
+
return commandNode;
|
|
4622
|
+
} catch {
|
|
4623
|
+
return null;
|
|
4624
|
+
}
|
|
4625
|
+
}
|
|
4626
|
+
function joinSqlWords(words) {
|
|
4627
|
+
const values = [];
|
|
4628
|
+
for (const word of words) {
|
|
4629
|
+
const value = asStaticWordText(word, {
|
|
4630
|
+
preserveLegacyBackticks: true
|
|
4631
|
+
});
|
|
4632
|
+
if (value == null) return null;
|
|
4633
|
+
values.push(value);
|
|
4634
|
+
}
|
|
4635
|
+
const sql = values.join(" ").trim();
|
|
4636
|
+
return sql || null;
|
|
4637
|
+
}
|
|
4638
|
+
|
|
4639
|
+
// packages/text2sql/src/lib/sql-repair.ts
|
|
4640
|
+
import {
|
|
4641
|
+
buildSubcommandRepair,
|
|
4642
|
+
repairQuotedArg
|
|
4643
|
+
} from "@deepagents/context";
|
|
4644
|
+
function repairDbNameAndQuery(rawArgs) {
|
|
4645
|
+
const trimmed = rawArgs.trim();
|
|
4646
|
+
if (!trimmed) return null;
|
|
4647
|
+
const firstSpace = trimmed.search(/\s/);
|
|
4648
|
+
if (firstSpace === -1) return null;
|
|
4649
|
+
const dbName = trimmed.slice(0, firstSpace);
|
|
4650
|
+
const rest = trimmed.slice(firstSpace + 1);
|
|
4651
|
+
const repaired = repairQuotedArg(rest);
|
|
4652
|
+
if (repaired == null) return null;
|
|
4653
|
+
return `${dbName} ${repaired}`;
|
|
4654
|
+
}
|
|
4655
|
+
var inertHandler = () => ({ stdout: "", stderr: "", exitCode: 0 });
|
|
4656
|
+
var repair = buildSubcommandRepair("sql", {
|
|
4657
|
+
run: {
|
|
4658
|
+
usage: 'run <db> "SELECT ..."',
|
|
4659
|
+
description: "Execute query against <db> and store results",
|
|
4660
|
+
repair: repairDbNameAndQuery,
|
|
4661
|
+
handler: inertHandler
|
|
4662
|
+
},
|
|
4663
|
+
validate: {
|
|
4664
|
+
usage: 'validate <db> "SELECT ..."',
|
|
4665
|
+
description: "Validate query syntax against <db>",
|
|
4666
|
+
repair: repairDbNameAndQuery,
|
|
4667
|
+
handler: inertHandler
|
|
4668
|
+
}
|
|
4669
|
+
});
|
|
4670
|
+
function repairSqlCommand(command) {
|
|
4671
|
+
return repair(command);
|
|
4672
|
+
}
|
|
4673
|
+
|
|
4674
|
+
// packages/text2sql/src/lib/sql-transform.ts
|
|
4675
|
+
import { BashTransformPipeline } from "just-bash";
|
|
4676
|
+
|
|
4677
|
+
// packages/text2sql/src/lib/agents/sql-transform-plugins.ts
|
|
4678
|
+
import { parse as parse2, serialize } from "just-bash";
|
|
4679
|
+
import { createHash as createHash3 } from "node:crypto";
|
|
4680
|
+
import * as path5 from "node:path";
|
|
4681
|
+
import { BashException, asStaticWordText as asStaticWordText2 } from "@deepagents/context";
|
|
4682
|
+
function isScriptNode(value) {
|
|
4683
|
+
if (typeof value !== "object" || value === null) {
|
|
4684
|
+
return false;
|
|
4685
|
+
}
|
|
4686
|
+
const node = value;
|
|
4687
|
+
return node.type === "Script" && Array.isArray(node.statements);
|
|
4688
|
+
}
|
|
4689
|
+
var BLOCKED_DB_CLIENT_COMMANDS = /* @__PURE__ */ new Set([
|
|
4690
|
+
"psql",
|
|
4691
|
+
"sqlite3",
|
|
4692
|
+
"mysql",
|
|
4693
|
+
"duckdb"
|
|
4694
|
+
]);
|
|
4695
|
+
var BLOCKED_RAW_SQL_COMMANDS = /* @__PURE__ */ new Set(["select", "with"]);
|
|
4696
|
+
var ALLOWED_SQL_PROXY_SUBCOMMANDS = /* @__PURE__ */ new Set(["run", "validate"]);
|
|
4697
|
+
var SHELL_INTERPRETER_COMMANDS = /* @__PURE__ */ new Set([
|
|
4698
|
+
"bash",
|
|
4699
|
+
"sh",
|
|
4700
|
+
"zsh",
|
|
4701
|
+
"dash",
|
|
4702
|
+
"ksh"
|
|
4703
|
+
]);
|
|
4704
|
+
var WRAPPER_COMMANDS = /* @__PURE__ */ new Set(["env", "command", "eval"]);
|
|
4705
|
+
var SQL_PROXY_ENFORCEMENT_MESSAGE = [
|
|
4706
|
+
"Direct database querying through bash is blocked.",
|
|
4707
|
+
"Use SQL proxy commands in this order:",
|
|
4708
|
+
'1) sql validate <db> "SELECT ..."',
|
|
4709
|
+
'2) sql run <db> "SELECT ..."'
|
|
4710
|
+
].join("\n");
|
|
4711
|
+
var SqlProxyViolationError = class extends BashException {
|
|
4712
|
+
constructor() {
|
|
4713
|
+
super(SQL_PROXY_ENFORCEMENT_MESSAGE);
|
|
4714
|
+
this.name = "SqlProxyViolationError";
|
|
4715
|
+
}
|
|
4716
|
+
format() {
|
|
4717
|
+
return { stdout: "", stderr: `${this.message}
|
|
4718
|
+
`, exitCode: 1 };
|
|
4719
|
+
}
|
|
4720
|
+
};
|
|
4721
|
+
function cloneInspectionContext(context) {
|
|
4722
|
+
return {
|
|
4723
|
+
functionDefinitions: new Map(context.functionDefinitions),
|
|
4724
|
+
callStack: new Set(context.callStack)
|
|
4725
|
+
};
|
|
4726
|
+
}
|
|
4727
|
+
function scriptContainsBlockedCommand(script, context, mode = "blocked-only") {
|
|
4728
|
+
return statementsContainBlockedCommand(script.statements, context, mode);
|
|
4729
|
+
}
|
|
4730
|
+
function statementsContainBlockedCommand(statements, context, mode) {
|
|
4731
|
+
for (const statement of statements) {
|
|
4732
|
+
if (statementContainsBlockedCommand(statement, context, mode)) {
|
|
4733
|
+
return true;
|
|
4734
|
+
}
|
|
4735
|
+
}
|
|
4736
|
+
return false;
|
|
4737
|
+
}
|
|
4738
|
+
function statementContainsBlockedCommand(statement, context, mode) {
|
|
4739
|
+
for (const pipeline of statement.pipelines) {
|
|
4740
|
+
if (pipelineContainsBlockedCommand(pipeline, context, mode)) {
|
|
4741
|
+
return true;
|
|
4742
|
+
}
|
|
4743
|
+
}
|
|
4744
|
+
return false;
|
|
4745
|
+
}
|
|
4746
|
+
function pipelineContainsBlockedCommand(pipeline, context, mode) {
|
|
4747
|
+
for (const [index2, command] of pipeline.commands.entries()) {
|
|
4748
|
+
if (command.type === "FunctionDef") {
|
|
4749
|
+
context.functionDefinitions.set(command.name, command);
|
|
4750
|
+
continue;
|
|
4751
|
+
}
|
|
4752
|
+
if (commandContainsBlockedCommand(command, context, mode, {
|
|
4753
|
+
stdinFromPipe: index2 > 0
|
|
4754
|
+
})) {
|
|
4755
|
+
return true;
|
|
4756
|
+
}
|
|
4757
|
+
}
|
|
4758
|
+
return false;
|
|
4759
|
+
}
|
|
4760
|
+
function stringCommandContainsBlockedCommand(command, context, mode = "blocked-only") {
|
|
4761
|
+
let script;
|
|
4762
|
+
try {
|
|
4763
|
+
script = parse2(command);
|
|
4764
|
+
} catch {
|
|
4765
|
+
return false;
|
|
4766
|
+
}
|
|
4767
|
+
return scriptContainsBlockedCommand(
|
|
4768
|
+
script,
|
|
4769
|
+
cloneInspectionContext(context),
|
|
4770
|
+
mode
|
|
4771
|
+
);
|
|
4772
|
+
}
|
|
4773
|
+
function wordContainsBlockedCommand(word, context, mode) {
|
|
4774
|
+
if (!word) {
|
|
4775
|
+
return false;
|
|
4776
|
+
}
|
|
4777
|
+
return wordPartContainsBlockedCommand(
|
|
4778
|
+
word.parts,
|
|
4779
|
+
context,
|
|
4780
|
+
mode
|
|
4781
|
+
);
|
|
4782
|
+
}
|
|
4783
|
+
function wordPartContainsBlockedCommand(parts, context, mode) {
|
|
4784
|
+
for (const part of parts) {
|
|
4785
|
+
if (partContainsBlockedCommand(part, context, mode)) {
|
|
4786
|
+
return true;
|
|
4787
|
+
}
|
|
4788
|
+
}
|
|
4789
|
+
return false;
|
|
4790
|
+
}
|
|
4791
|
+
function partContainsBlockedCommand(node, context, mode) {
|
|
4792
|
+
const type = node.type;
|
|
4793
|
+
if (type === "CommandSubstitution" || type === "ProcessSubstitution") {
|
|
4794
|
+
if (isScriptNode(node.body)) {
|
|
4795
|
+
return scriptContainsBlockedCommand(
|
|
4796
|
+
node.body,
|
|
4797
|
+
cloneInspectionContext(context),
|
|
4798
|
+
mode
|
|
4799
|
+
);
|
|
4800
|
+
}
|
|
4801
|
+
return false;
|
|
4802
|
+
}
|
|
4803
|
+
if (type === "ArithCommandSubst" && typeof node.command === "string") {
|
|
4804
|
+
return stringCommandContainsBlockedCommand(node.command, context, mode);
|
|
4805
|
+
}
|
|
4806
|
+
for (const value of Object.values(node)) {
|
|
4807
|
+
if (Array.isArray(value)) {
|
|
4808
|
+
for (const item of value) {
|
|
4809
|
+
if (typeof item === "object" && item !== null) {
|
|
4810
|
+
if (partContainsBlockedCommand(
|
|
4811
|
+
item,
|
|
4812
|
+
context,
|
|
4813
|
+
mode
|
|
4814
|
+
)) {
|
|
4815
|
+
return true;
|
|
4816
|
+
}
|
|
4817
|
+
}
|
|
4818
|
+
}
|
|
4819
|
+
continue;
|
|
4820
|
+
}
|
|
4821
|
+
if (typeof value === "object" && value !== null) {
|
|
4822
|
+
if (partContainsBlockedCommand(
|
|
4823
|
+
value,
|
|
4824
|
+
context,
|
|
4825
|
+
mode
|
|
4826
|
+
)) {
|
|
4827
|
+
return true;
|
|
4828
|
+
}
|
|
4829
|
+
}
|
|
4830
|
+
}
|
|
4831
|
+
return false;
|
|
4832
|
+
}
|
|
4833
|
+
function functionInvocationContainsBlockedCommand(functionName, context, mode) {
|
|
4834
|
+
const definition = context.functionDefinitions.get(functionName);
|
|
4835
|
+
if (!definition) {
|
|
4836
|
+
return false;
|
|
4837
|
+
}
|
|
4838
|
+
if (context.callStack.has(functionName)) {
|
|
4839
|
+
return false;
|
|
4840
|
+
}
|
|
4841
|
+
const invocationContext = cloneInspectionContext(context);
|
|
4842
|
+
invocationContext.callStack.add(functionName);
|
|
4843
|
+
return commandContainsBlockedCommand(
|
|
4844
|
+
definition.body,
|
|
4845
|
+
invocationContext,
|
|
4846
|
+
mode,
|
|
4847
|
+
{ stdinFromPipe: false }
|
|
4848
|
+
);
|
|
4849
|
+
}
|
|
4850
|
+
function isAsciiLetter(character) {
|
|
4851
|
+
const charCode = character.charCodeAt(0);
|
|
4852
|
+
return charCode >= 65 && charCode <= 90 || charCode >= 97 && charCode <= 122;
|
|
4853
|
+
}
|
|
4854
|
+
function isAsciiDigit(character) {
|
|
4855
|
+
const charCode = character.charCodeAt(0);
|
|
4856
|
+
return charCode >= 48 && charCode <= 57;
|
|
4857
|
+
}
|
|
4858
|
+
function isValidEnvVariableName(name) {
|
|
4859
|
+
if (!name) {
|
|
4860
|
+
return false;
|
|
4861
|
+
}
|
|
4862
|
+
const firstChar = name[0];
|
|
4863
|
+
if (!(isAsciiLetter(firstChar) || firstChar === "_")) {
|
|
4864
|
+
return false;
|
|
4865
|
+
}
|
|
4866
|
+
for (let index2 = 1; index2 < name.length; index2 += 1) {
|
|
4867
|
+
const char = name[index2];
|
|
4868
|
+
if (!(isAsciiLetter(char) || isAsciiDigit(char) || char === "_")) {
|
|
4869
|
+
return false;
|
|
4870
|
+
}
|
|
4871
|
+
}
|
|
4872
|
+
return true;
|
|
4873
|
+
}
|
|
4874
|
+
function isEnvAssignmentToken(token) {
|
|
4875
|
+
const separatorIndex = token.indexOf("=");
|
|
4876
|
+
if (separatorIndex <= 0) {
|
|
4877
|
+
return false;
|
|
4878
|
+
}
|
|
4879
|
+
return isValidEnvVariableName(token.slice(0, separatorIndex));
|
|
4880
|
+
}
|
|
4881
|
+
function parseShortOptionCluster(option) {
|
|
4882
|
+
if (!option.startsWith("-") || option.startsWith("--") || option.length <= 1) {
|
|
4883
|
+
return {
|
|
4884
|
+
valid: false,
|
|
4885
|
+
hasCommandFlag: false,
|
|
4886
|
+
hasStdinFlag: false,
|
|
4887
|
+
consumesNextArg: false
|
|
4888
|
+
};
|
|
4889
|
+
}
|
|
4890
|
+
let hasCommandFlag = false;
|
|
4891
|
+
let hasStdinFlag = false;
|
|
4892
|
+
let consumesNextArg = false;
|
|
4893
|
+
for (let index2 = 1; index2 < option.length; index2 += 1) {
|
|
4894
|
+
const char = option[index2];
|
|
4895
|
+
if (!isAsciiLetter(char)) {
|
|
4896
|
+
return {
|
|
4897
|
+
valid: false,
|
|
4898
|
+
hasCommandFlag: false,
|
|
4899
|
+
hasStdinFlag: false,
|
|
4900
|
+
consumesNextArg: false
|
|
4901
|
+
};
|
|
4902
|
+
}
|
|
4903
|
+
if (char === "c") {
|
|
4904
|
+
hasCommandFlag = true;
|
|
4905
|
+
} else if (char === "s") {
|
|
4906
|
+
hasStdinFlag = true;
|
|
4907
|
+
} else if (char === "O" || char === "o") {
|
|
4908
|
+
consumesNextArg = true;
|
|
4909
|
+
}
|
|
4910
|
+
}
|
|
4911
|
+
return { valid: true, hasCommandFlag, hasStdinFlag, consumesNextArg };
|
|
4912
|
+
}
|
|
4913
|
+
function getShellInvocationDescriptor(args) {
|
|
4914
|
+
let readsFromStdin = false;
|
|
4915
|
+
const longOptionsWithValue = /* @__PURE__ */ new Set(["--rcfile", "--init-file"]);
|
|
4916
|
+
for (let index2 = 0; index2 < args.length; index2 += 1) {
|
|
4917
|
+
const token = asStaticWordText2(args[index2]);
|
|
4918
|
+
if (token == null) {
|
|
4919
|
+
return { kind: "unknown", payload: null };
|
|
4920
|
+
}
|
|
4921
|
+
if (token === "--") {
|
|
4922
|
+
if (index2 + 1 >= args.length) {
|
|
4923
|
+
break;
|
|
4924
|
+
}
|
|
4925
|
+
return {
|
|
4926
|
+
kind: "script",
|
|
4927
|
+
payload: asStaticWordText2(args[index2 + 1])
|
|
4928
|
+
};
|
|
4929
|
+
}
|
|
4930
|
+
if (token === "--command") {
|
|
4931
|
+
return {
|
|
4932
|
+
kind: "command",
|
|
4933
|
+
payload: asStaticWordText2(args[index2 + 1])
|
|
4934
|
+
};
|
|
4935
|
+
}
|
|
4936
|
+
if (token.startsWith("--command=")) {
|
|
4937
|
+
return {
|
|
4938
|
+
kind: "command",
|
|
4939
|
+
payload: token.slice("--command=".length)
|
|
4940
|
+
};
|
|
4941
|
+
}
|
|
4942
|
+
if (token.startsWith("--")) {
|
|
4943
|
+
if (token.includes("=")) {
|
|
4944
|
+
continue;
|
|
4945
|
+
}
|
|
4946
|
+
if (longOptionsWithValue.has(token)) {
|
|
4947
|
+
if (index2 + 1 >= args.length) {
|
|
4948
|
+
return { kind: "unknown", payload: null };
|
|
4949
|
+
}
|
|
4950
|
+
index2 += 1;
|
|
4951
|
+
}
|
|
4952
|
+
continue;
|
|
4953
|
+
}
|
|
4954
|
+
if (token.startsWith("-") && !token.startsWith("--")) {
|
|
4955
|
+
const parsed = parseShortOptionCluster(token);
|
|
4956
|
+
if (!parsed.valid) {
|
|
4957
|
+
return { kind: "unknown", payload: null };
|
|
4958
|
+
}
|
|
4959
|
+
if (parsed.hasCommandFlag) {
|
|
4960
|
+
return {
|
|
4961
|
+
kind: "command",
|
|
4962
|
+
payload: asStaticWordText2(args[index2 + 1])
|
|
4963
|
+
};
|
|
4964
|
+
}
|
|
4965
|
+
if (parsed.hasStdinFlag) {
|
|
4966
|
+
readsFromStdin = true;
|
|
4967
|
+
}
|
|
4968
|
+
if (parsed.consumesNextArg) {
|
|
4969
|
+
if (index2 + 1 >= args.length) {
|
|
4970
|
+
return { kind: "unknown", payload: null };
|
|
4971
|
+
}
|
|
4972
|
+
index2 += 1;
|
|
4973
|
+
}
|
|
4974
|
+
continue;
|
|
4975
|
+
}
|
|
4976
|
+
return {
|
|
4977
|
+
kind: "script",
|
|
4978
|
+
payload: token
|
|
4979
|
+
};
|
|
4980
|
+
}
|
|
4981
|
+
if (readsFromStdin) {
|
|
4982
|
+
return { kind: "stdin", payload: null };
|
|
4983
|
+
}
|
|
4984
|
+
return { kind: "none", payload: null };
|
|
4985
|
+
}
|
|
4986
|
+
function getHereDocPayload(redirections) {
|
|
4987
|
+
const payloads = [];
|
|
4988
|
+
for (const redirection of redirections) {
|
|
4989
|
+
if (redirection.target.type !== "HereDoc") {
|
|
4990
|
+
continue;
|
|
4991
|
+
}
|
|
4992
|
+
if (!redirection.target.content) {
|
|
4993
|
+
payloads.push("");
|
|
4994
|
+
continue;
|
|
4995
|
+
}
|
|
4996
|
+
const payload = asStaticWordText2(redirection.target.content);
|
|
4997
|
+
if (payload == null) {
|
|
4998
|
+
return { hasHereDoc: true, payload: null };
|
|
4999
|
+
}
|
|
5000
|
+
payloads.push(payload);
|
|
5001
|
+
}
|
|
5002
|
+
if (payloads.length === 0) {
|
|
5003
|
+
return { hasHereDoc: false, payload: null };
|
|
5004
|
+
}
|
|
5005
|
+
return { hasHereDoc: true, payload: payloads.join("\n") };
|
|
5006
|
+
}
|
|
5007
|
+
function joinStaticWords(words) {
|
|
5008
|
+
const tokens = [];
|
|
5009
|
+
for (const word of words) {
|
|
5010
|
+
const token = asStaticWordText2(word);
|
|
5011
|
+
if (token == null) {
|
|
5012
|
+
return null;
|
|
5013
|
+
}
|
|
5014
|
+
tokens.push(token);
|
|
5015
|
+
}
|
|
5016
|
+
return tokens.join(" ");
|
|
5017
|
+
}
|
|
5018
|
+
function resolveEnvWrapperCommand(args) {
|
|
5019
|
+
let index2 = 0;
|
|
5020
|
+
while (index2 < args.length) {
|
|
5021
|
+
const token = asStaticWordText2(args[index2]);
|
|
5022
|
+
if (token == null) {
|
|
5023
|
+
return { kind: "unknown" };
|
|
5024
|
+
}
|
|
5025
|
+
if (token === "--") {
|
|
5026
|
+
index2 += 1;
|
|
5027
|
+
break;
|
|
5028
|
+
}
|
|
5029
|
+
if (token === "-u" || token === "--unset" || token === "--chdir") {
|
|
5030
|
+
if (index2 + 1 >= args.length) {
|
|
5031
|
+
return { kind: "unknown" };
|
|
5032
|
+
}
|
|
5033
|
+
index2 += 2;
|
|
5034
|
+
continue;
|
|
5035
|
+
}
|
|
5036
|
+
if (token.startsWith("--unset=") || token.startsWith("--chdir=")) {
|
|
5037
|
+
index2 += 1;
|
|
5038
|
+
continue;
|
|
5039
|
+
}
|
|
5040
|
+
if (token.startsWith("-") && token !== "-" && !isEnvAssignmentToken(token)) {
|
|
5041
|
+
index2 += 1;
|
|
5042
|
+
continue;
|
|
5043
|
+
}
|
|
5044
|
+
if (isEnvAssignmentToken(token)) {
|
|
5045
|
+
index2 += 1;
|
|
5046
|
+
continue;
|
|
5047
|
+
}
|
|
5048
|
+
break;
|
|
5049
|
+
}
|
|
5050
|
+
if (index2 >= args.length) {
|
|
5051
|
+
return { kind: "none" };
|
|
5052
|
+
}
|
|
5053
|
+
return {
|
|
5054
|
+
kind: "resolved",
|
|
5055
|
+
name: args[index2],
|
|
5056
|
+
args: args.slice(index2 + 1)
|
|
5057
|
+
};
|
|
5058
|
+
}
|
|
5059
|
+
function resolveCommandWrapperCommand(args) {
|
|
5060
|
+
let index2 = 0;
|
|
5061
|
+
let lookupOnly = false;
|
|
5062
|
+
while (index2 < args.length) {
|
|
5063
|
+
const token = asStaticWordText2(args[index2]);
|
|
5064
|
+
if (token == null) {
|
|
5065
|
+
return { kind: "unknown" };
|
|
5066
|
+
}
|
|
5067
|
+
if (token === "--") {
|
|
5068
|
+
index2 += 1;
|
|
5069
|
+
break;
|
|
5070
|
+
}
|
|
5071
|
+
if (token === "-v" || token === "-V") {
|
|
5072
|
+
lookupOnly = true;
|
|
5073
|
+
index2 += 1;
|
|
5074
|
+
continue;
|
|
5075
|
+
}
|
|
5076
|
+
if (token.startsWith("-") && token !== "-") {
|
|
5077
|
+
index2 += 1;
|
|
5078
|
+
continue;
|
|
5079
|
+
}
|
|
5080
|
+
break;
|
|
5081
|
+
}
|
|
5082
|
+
if (lookupOnly || index2 >= args.length) {
|
|
5083
|
+
return { kind: "none" };
|
|
5084
|
+
}
|
|
5085
|
+
return {
|
|
5086
|
+
kind: "resolved",
|
|
5087
|
+
name: args[index2],
|
|
5088
|
+
args: args.slice(index2 + 1)
|
|
5089
|
+
};
|
|
5090
|
+
}
|
|
5091
|
+
function commandContainsBlockedCommand(command, context, mode, options = { stdinFromPipe: false }) {
|
|
5092
|
+
switch (command.type) {
|
|
5093
|
+
case "SimpleCommand":
|
|
5094
|
+
return isBlockedSimpleCommand(command, context, mode, options);
|
|
5095
|
+
case "If":
|
|
5096
|
+
return command.clauses.some(
|
|
5097
|
+
(clause) => statementsContainBlockedCommand(
|
|
5098
|
+
clause.condition,
|
|
5099
|
+
cloneInspectionContext(context),
|
|
5100
|
+
mode
|
|
5101
|
+
) || statementsContainBlockedCommand(
|
|
5102
|
+
clause.body,
|
|
5103
|
+
cloneInspectionContext(context),
|
|
5104
|
+
mode
|
|
5105
|
+
)
|
|
5106
|
+
) || (command.elseBody ? statementsContainBlockedCommand(
|
|
5107
|
+
command.elseBody,
|
|
5108
|
+
cloneInspectionContext(context),
|
|
5109
|
+
mode
|
|
5110
|
+
) : false);
|
|
5111
|
+
case "For":
|
|
5112
|
+
case "CStyleFor":
|
|
5113
|
+
return statementsContainBlockedCommand(
|
|
5114
|
+
command.body,
|
|
5115
|
+
cloneInspectionContext(context),
|
|
5116
|
+
mode
|
|
5117
|
+
);
|
|
5118
|
+
case "While":
|
|
5119
|
+
case "Until":
|
|
5120
|
+
return statementsContainBlockedCommand(
|
|
5121
|
+
command.condition,
|
|
5122
|
+
cloneInspectionContext(context),
|
|
5123
|
+
mode
|
|
5124
|
+
) || statementsContainBlockedCommand(
|
|
5125
|
+
command.body,
|
|
5126
|
+
cloneInspectionContext(context),
|
|
5127
|
+
mode
|
|
5128
|
+
);
|
|
5129
|
+
case "Case":
|
|
5130
|
+
return command.items.some(
|
|
5131
|
+
(item) => statementsContainBlockedCommand(
|
|
5132
|
+
item.body,
|
|
5133
|
+
cloneInspectionContext(context),
|
|
5134
|
+
mode
|
|
5135
|
+
)
|
|
5136
|
+
);
|
|
5137
|
+
case "Subshell":
|
|
5138
|
+
case "Group":
|
|
5139
|
+
return statementsContainBlockedCommand(
|
|
5140
|
+
command.body,
|
|
5141
|
+
cloneInspectionContext(context),
|
|
5142
|
+
mode
|
|
5143
|
+
);
|
|
5144
|
+
case "FunctionDef":
|
|
5145
|
+
return false;
|
|
5146
|
+
case "ArithmeticCommand":
|
|
5147
|
+
case "ConditionalCommand":
|
|
5148
|
+
return false;
|
|
5149
|
+
default: {
|
|
5150
|
+
const _unreachable = command;
|
|
5151
|
+
return _unreachable;
|
|
5152
|
+
}
|
|
5153
|
+
}
|
|
5154
|
+
}
|
|
5155
|
+
function isBlockedSimpleCommand(command, context, mode, options) {
|
|
5156
|
+
if (wordContainsBlockedCommand(command.name, context, mode)) {
|
|
5157
|
+
return true;
|
|
5158
|
+
}
|
|
5159
|
+
if (command.args.some((arg) => wordContainsBlockedCommand(arg, context, mode))) {
|
|
5160
|
+
return true;
|
|
5161
|
+
}
|
|
5162
|
+
if (command.assignments.some(
|
|
5163
|
+
(assignment) => wordContainsBlockedCommand(assignment.value, context, mode) || (assignment.array?.some(
|
|
5164
|
+
(value) => wordContainsBlockedCommand(value, context, mode)
|
|
5165
|
+
) ?? false)
|
|
5166
|
+
)) {
|
|
5167
|
+
return true;
|
|
5168
|
+
}
|
|
5169
|
+
if (command.redirections.some((redirection) => {
|
|
5170
|
+
if (redirection.target.type === "Word") {
|
|
5171
|
+
return wordContainsBlockedCommand(
|
|
5172
|
+
redirection.target,
|
|
5173
|
+
context,
|
|
5174
|
+
mode
|
|
5175
|
+
);
|
|
5176
|
+
}
|
|
5177
|
+
if (redirection.target.type === "HereDoc" && redirection.target.content) {
|
|
5178
|
+
return wordContainsBlockedCommand(
|
|
5179
|
+
redirection.target.content,
|
|
5180
|
+
context,
|
|
5181
|
+
mode
|
|
5182
|
+
);
|
|
5183
|
+
}
|
|
5184
|
+
return false;
|
|
5185
|
+
})) {
|
|
5186
|
+
return true;
|
|
5187
|
+
}
|
|
5188
|
+
const commandName = asStaticWordText2(command.name);
|
|
5189
|
+
if (!commandName) {
|
|
5190
|
+
return false;
|
|
5191
|
+
}
|
|
5192
|
+
const normalizedName = path5.posix.basename(commandName).toLowerCase();
|
|
5193
|
+
if (BLOCKED_DB_CLIENT_COMMANDS.has(normalizedName)) {
|
|
5194
|
+
return true;
|
|
5195
|
+
}
|
|
5196
|
+
if (BLOCKED_RAW_SQL_COMMANDS.has(normalizedName)) {
|
|
5197
|
+
return true;
|
|
5198
|
+
}
|
|
5199
|
+
if (normalizedName === "sql") {
|
|
5200
|
+
const subcommand = asStaticWordText2(command.args[0])?.toLowerCase();
|
|
5201
|
+
if (!subcommand) {
|
|
5202
|
+
return true;
|
|
5203
|
+
}
|
|
5204
|
+
if (mode === "block-all-sql") {
|
|
5205
|
+
return true;
|
|
5206
|
+
}
|
|
5207
|
+
return !ALLOWED_SQL_PROXY_SUBCOMMANDS.has(subcommand);
|
|
5208
|
+
}
|
|
5209
|
+
const inspectWrappedCommand = (resolved) => {
|
|
5210
|
+
if (resolved.kind === "none") {
|
|
5211
|
+
return false;
|
|
5212
|
+
}
|
|
5213
|
+
if (resolved.kind === "unknown" || !resolved.name || !resolved.args) {
|
|
5214
|
+
return true;
|
|
5215
|
+
}
|
|
5216
|
+
return isBlockedSimpleCommand(
|
|
5217
|
+
{
|
|
5218
|
+
name: resolved.name,
|
|
5219
|
+
args: resolved.args,
|
|
5220
|
+
assignments: [],
|
|
5221
|
+
redirections: []
|
|
5222
|
+
},
|
|
5223
|
+
context,
|
|
5224
|
+
"block-all-sql",
|
|
5225
|
+
options
|
|
5226
|
+
);
|
|
5227
|
+
};
|
|
5228
|
+
if (WRAPPER_COMMANDS.has(normalizedName)) {
|
|
5229
|
+
if (normalizedName === "env") {
|
|
5230
|
+
return inspectWrappedCommand(resolveEnvWrapperCommand(command.args));
|
|
5231
|
+
}
|
|
5232
|
+
if (normalizedName === "command") {
|
|
5233
|
+
return inspectWrappedCommand(resolveCommandWrapperCommand(command.args));
|
|
5234
|
+
}
|
|
5235
|
+
const evalScript = joinStaticWords(command.args);
|
|
5236
|
+
if (evalScript == null) {
|
|
5237
|
+
return true;
|
|
5238
|
+
}
|
|
5239
|
+
if (!evalScript.trim()) {
|
|
5240
|
+
return false;
|
|
5241
|
+
}
|
|
5242
|
+
return stringCommandContainsBlockedCommand(
|
|
5243
|
+
evalScript,
|
|
5244
|
+
context,
|
|
5245
|
+
"block-all-sql"
|
|
5246
|
+
);
|
|
5247
|
+
}
|
|
5248
|
+
if (SHELL_INTERPRETER_COMMANDS.has(normalizedName)) {
|
|
5249
|
+
const shellInvocation = getShellInvocationDescriptor(command.args);
|
|
5250
|
+
if (shellInvocation.kind === "unknown") {
|
|
5251
|
+
return true;
|
|
5252
|
+
}
|
|
5253
|
+
if (shellInvocation.kind === "command") {
|
|
5254
|
+
if (!shellInvocation.payload) {
|
|
5255
|
+
return true;
|
|
5256
|
+
}
|
|
5257
|
+
if (stringCommandContainsBlockedCommand(
|
|
5258
|
+
shellInvocation.payload,
|
|
5259
|
+
context,
|
|
5260
|
+
"block-all-sql"
|
|
5261
|
+
)) {
|
|
5262
|
+
return true;
|
|
5263
|
+
}
|
|
5264
|
+
return false;
|
|
5265
|
+
}
|
|
5266
|
+
const hereDoc = getHereDocPayload(command.redirections);
|
|
5267
|
+
if (hereDoc.hasHereDoc) {
|
|
5268
|
+
if (hereDoc.payload == null) {
|
|
5269
|
+
return true;
|
|
5270
|
+
}
|
|
5271
|
+
if (hereDoc.payload.trim().length > 0 && stringCommandContainsBlockedCommand(
|
|
5272
|
+
hereDoc.payload,
|
|
5273
|
+
context,
|
|
5274
|
+
"block-all-sql"
|
|
5275
|
+
)) {
|
|
5276
|
+
return true;
|
|
5277
|
+
}
|
|
5278
|
+
}
|
|
5279
|
+
if (shellInvocation.kind === "script") {
|
|
5280
|
+
return true;
|
|
5281
|
+
}
|
|
5282
|
+
if (options.stdinFromPipe || shellInvocation.kind === "stdin") {
|
|
5283
|
+
return !hereDoc.hasHereDoc;
|
|
5284
|
+
}
|
|
5285
|
+
}
|
|
5286
|
+
if (functionInvocationContainsBlockedCommand(commandName, context, mode)) {
|
|
5287
|
+
return true;
|
|
5288
|
+
}
|
|
5289
|
+
return false;
|
|
5290
|
+
}
|
|
5291
|
+
var SqlProxyEnforcementPlugin = class {
|
|
5292
|
+
name = "sql-proxy-enforcement";
|
|
5293
|
+
transform(context) {
|
|
5294
|
+
const blocked = scriptContainsBlockedCommand(context.ast, {
|
|
5295
|
+
functionDefinitions: /* @__PURE__ */ new Map(),
|
|
5296
|
+
callStack: /* @__PURE__ */ new Set()
|
|
5297
|
+
});
|
|
5298
|
+
if (blocked) {
|
|
5299
|
+
throw new SqlProxyViolationError();
|
|
5300
|
+
}
|
|
5301
|
+
return {
|
|
5302
|
+
ast: context.ast,
|
|
5303
|
+
metadata: { inspected: true }
|
|
5304
|
+
};
|
|
5305
|
+
}
|
|
5306
|
+
};
|
|
5307
|
+
function wordPartsContainExpansion(parts) {
|
|
5308
|
+
for (const part of parts) {
|
|
5309
|
+
if (part.type === "CommandSubstitution" || part.type === "ParameterExpansion" || part.type === "ArithmeticExpansion") {
|
|
5310
|
+
return true;
|
|
5311
|
+
}
|
|
5312
|
+
if (part.type === "DoubleQuoted" && Array.isArray(part.parts)) {
|
|
5313
|
+
if (wordPartsContainExpansion(part.parts)) {
|
|
5314
|
+
return true;
|
|
5315
|
+
}
|
|
5316
|
+
}
|
|
5317
|
+
}
|
|
5318
|
+
return false;
|
|
5319
|
+
}
|
|
5320
|
+
function isSqlCommandRequiringRewrite(cmd) {
|
|
5321
|
+
if (cmd.type !== "SimpleCommand") return false;
|
|
5322
|
+
if (cmd.assignments.length > 0 || cmd.redirections.length > 0) return false;
|
|
5323
|
+
if (asStaticWordText2(cmd.name) !== "sql") return false;
|
|
5324
|
+
if (cmd.args.length < 3) return false;
|
|
5325
|
+
const subcommand = asStaticWordText2(cmd.args[0]);
|
|
5326
|
+
if (subcommand !== "validate" && subcommand !== "run") return false;
|
|
5327
|
+
const dbName = asStaticWordText2(cmd.args[1]);
|
|
5328
|
+
if (!isValidAdapterName(dbName)) return false;
|
|
5329
|
+
const sqlArgs = cmd.args.slice(2);
|
|
5330
|
+
if (sqlArgs.length > 1) return true;
|
|
5331
|
+
return sqlArgs.some(
|
|
5332
|
+
(arg) => wordPartsContainExpansion(
|
|
5333
|
+
arg.parts
|
|
5334
|
+
)
|
|
5335
|
+
);
|
|
5336
|
+
}
|
|
5337
|
+
function extractWordPartText(parts) {
|
|
5338
|
+
let text = "";
|
|
5339
|
+
for (const part of parts) {
|
|
5340
|
+
const type = part.type;
|
|
5341
|
+
if (type === "Literal" || type === "SingleQuoted" || type === "Escaped") {
|
|
5342
|
+
text += part.value;
|
|
5343
|
+
} else if (type === "DoubleQuoted" && Array.isArray(part.parts)) {
|
|
5344
|
+
text += extractWordPartText(part.parts);
|
|
5345
|
+
} else if (type === "CommandSubstitution" && part.legacy === true) {
|
|
5346
|
+
text += "`" + serialize(part.body).trim() + "`";
|
|
5347
|
+
} else if (type === "CommandSubstitution") {
|
|
5348
|
+
text += "$(" + serialize(part.body).trim() + ")";
|
|
5349
|
+
} else if (type === "ParameterExpansion") {
|
|
5350
|
+
text += emitParameterExpansion(part);
|
|
5351
|
+
} else if (type === "ArithmeticExpansion") {
|
|
5352
|
+
const expression = part.expression;
|
|
5353
|
+
text += "$((" + (expression?.originalText ?? "") + "))";
|
|
5354
|
+
}
|
|
5355
|
+
}
|
|
5356
|
+
return text;
|
|
5357
|
+
}
|
|
5358
|
+
function emitParameterExpansion(part) {
|
|
5359
|
+
const parameter = part.parameter;
|
|
5360
|
+
const operation = part.operation;
|
|
5361
|
+
if (!operation) return "$" + parameter;
|
|
5362
|
+
if (operation.type === "DefaultValue" && operation.word) {
|
|
5363
|
+
const fallback = extractWordPartText(operation.word.parts);
|
|
5364
|
+
const separator = operation.checkEmpty ? ":-" : "-";
|
|
5365
|
+
return "${" + parameter + separator + fallback + "}";
|
|
5366
|
+
}
|
|
5367
|
+
return "${" + parameter + "}";
|
|
5368
|
+
}
|
|
5369
|
+
function extractSqlText(args) {
|
|
5370
|
+
return args.map(
|
|
5371
|
+
(arg) => extractWordPartText(
|
|
5372
|
+
arg.parts
|
|
5373
|
+
)
|
|
5374
|
+
).join(" ");
|
|
5375
|
+
}
|
|
5376
|
+
function rewriteSqlCommand(cmd, ast) {
|
|
5377
|
+
const subcommand = asStaticWordText2(cmd.args[0]);
|
|
5378
|
+
const dbName = asStaticWordText2(cmd.args[1]);
|
|
5379
|
+
const sqlArgs = cmd.args.slice(2);
|
|
5380
|
+
const unquotedSql = extractSqlText(sqlArgs);
|
|
5381
|
+
const hash2 = createHash3("sha256").update(serialize(ast)).digest("hex").slice(0, 12);
|
|
5382
|
+
const sqlPath = `/tmp/sql-inline-${subcommand}-${hash2}.sql`;
|
|
5383
|
+
const heredocTag = `SQL_${hash2.toUpperCase()}`;
|
|
5384
|
+
const groupScript = [
|
|
5385
|
+
`{ cat > ${sqlPath} <<'${heredocTag}'`,
|
|
5386
|
+
unquotedSql,
|
|
5387
|
+
heredocTag,
|
|
5388
|
+
`sql ${subcommand} ${dbName} "$(cat ${sqlPath})"; }`
|
|
5389
|
+
].join("\n");
|
|
5390
|
+
const groupAst = parse2(groupScript);
|
|
5391
|
+
return groupAst.statements[0].pipelines[0].commands[0];
|
|
5392
|
+
}
|
|
5393
|
+
function rewriteSqlCommandsInList(commands, ast) {
|
|
5394
|
+
let rewritten = false;
|
|
5395
|
+
for (let i = 0; i < commands.length; i++) {
|
|
5396
|
+
const cmd = commands[i];
|
|
5397
|
+
if (isSqlCommandRequiringRewrite(cmd)) {
|
|
5398
|
+
commands[i] = rewriteSqlCommand(cmd, ast);
|
|
5399
|
+
rewritten = true;
|
|
5400
|
+
continue;
|
|
5401
|
+
}
|
|
5402
|
+
if (cmd.type === "If") {
|
|
5403
|
+
for (const clause of cmd.clauses) {
|
|
5404
|
+
if (rewriteSqlCommandsInStatements(clause.condition, ast))
|
|
5405
|
+
rewritten = true;
|
|
5406
|
+
if (rewriteSqlCommandsInStatements(clause.body, ast)) rewritten = true;
|
|
5407
|
+
}
|
|
5408
|
+
if (cmd.elseBody && rewriteSqlCommandsInStatements(cmd.elseBody, ast)) {
|
|
5409
|
+
rewritten = true;
|
|
5410
|
+
}
|
|
5411
|
+
} else if (cmd.type === "While" || cmd.type === "Until") {
|
|
5412
|
+
if (rewriteSqlCommandsInStatements(cmd.condition, ast)) rewritten = true;
|
|
5413
|
+
if (rewriteSqlCommandsInStatements(cmd.body, ast)) rewritten = true;
|
|
5414
|
+
} else if (cmd.type === "For" || cmd.type === "CStyleFor") {
|
|
5415
|
+
if (rewriteSqlCommandsInStatements(cmd.body, ast)) rewritten = true;
|
|
5416
|
+
} else if (cmd.type === "Case") {
|
|
5417
|
+
for (const item of cmd.items) {
|
|
5418
|
+
if (rewriteSqlCommandsInStatements(item.body, ast)) rewritten = true;
|
|
5419
|
+
}
|
|
5420
|
+
} else if (cmd.type === "Subshell" || cmd.type === "Group") {
|
|
5421
|
+
if (rewriteSqlCommandsInStatements(cmd.body, ast)) rewritten = true;
|
|
5422
|
+
}
|
|
5423
|
+
}
|
|
5424
|
+
return rewritten;
|
|
5425
|
+
}
|
|
5426
|
+
function rewriteSqlCommandsInStatements(statements, ast) {
|
|
5427
|
+
let rewritten = false;
|
|
5428
|
+
for (const statement of statements) {
|
|
5429
|
+
for (const pipeline of statement.pipelines) {
|
|
5430
|
+
if (rewriteSqlCommandsInList(pipeline.commands, ast)) {
|
|
5431
|
+
rewritten = true;
|
|
5432
|
+
}
|
|
5433
|
+
}
|
|
5434
|
+
}
|
|
5435
|
+
return rewritten;
|
|
5436
|
+
}
|
|
5437
|
+
var SqlCommandRewritePlugin = class {
|
|
5438
|
+
name = "sql-command-rewrite";
|
|
5439
|
+
transform(context) {
|
|
5440
|
+
const ast = context.ast;
|
|
5441
|
+
const rewritten = rewriteSqlCommandsInStatements(ast.statements, ast);
|
|
5442
|
+
return { ast, metadata: { rewritten } };
|
|
5443
|
+
}
|
|
5444
|
+
};
|
|
5445
|
+
|
|
5446
|
+
// packages/text2sql/src/lib/sql-transform.ts
|
|
5447
|
+
var sqlTransformPipeline = new BashTransformPipeline().use(new SqlCommandRewritePlugin()).use(new SqlProxyEnforcementPlugin());
|
|
5448
|
+
function transformSqlCommand(command) {
|
|
5449
|
+
try {
|
|
5450
|
+
return sqlTransformPipeline.transform(command).script;
|
|
5451
|
+
} catch (error) {
|
|
5452
|
+
if (error instanceof SqlProxyViolationError) {
|
|
5453
|
+
return blockedSqlProxyCommand(error.message);
|
|
5454
|
+
}
|
|
5455
|
+
return command;
|
|
5456
|
+
}
|
|
5457
|
+
}
|
|
5458
|
+
function blockedSqlProxyCommand(message) {
|
|
5459
|
+
return `{ printf '%s\\n' ${shellQuote(message)} >&2; exit 1; }`;
|
|
5460
|
+
}
|
|
5461
|
+
function shellQuote(value) {
|
|
5462
|
+
return `'${value.replace(/'/g, "'\\''")}'`;
|
|
5212
5463
|
}
|
|
5213
5464
|
|
|
5214
|
-
// packages/text2sql/src/lib/
|
|
5215
|
-
function
|
|
5216
|
-
|
|
5217
|
-
|
|
5218
|
-
|
|
5219
|
-
}
|
|
5220
|
-
validateAdapterNames(adapterNames);
|
|
5221
|
-
const { command, repair } = createSqlCommand(adapters, options);
|
|
5465
|
+
// packages/text2sql/src/lib/sql-hooks.ts
|
|
5466
|
+
function createSqlCommandHooks({
|
|
5467
|
+
adapters
|
|
5468
|
+
}) {
|
|
5469
|
+
const meta = createSqlMetaHook({ adapters });
|
|
5222
5470
|
return {
|
|
5223
|
-
|
|
5224
|
-
|
|
5225
|
-
|
|
5471
|
+
onBeforeBashCall: ({ command }) => {
|
|
5472
|
+
const repaired = repairSqlCommand(command);
|
|
5473
|
+
const transformed = transformSqlCommand(repaired);
|
|
5474
|
+
meta.track({ repaired, transformed });
|
|
5475
|
+
return { command: transformed };
|
|
5476
|
+
},
|
|
5477
|
+
onAfterBashCall: meta.onAfterBashCall
|
|
5226
5478
|
};
|
|
5227
5479
|
}
|
|
5228
5480
|
|
|
5229
5481
|
// packages/text2sql/src/lib/sql.ts
|
|
5230
|
-
import {
|
|
5231
|
-
createUIMessageStream
|
|
5232
|
-
} from "ai";
|
|
5233
5482
|
import "@deepagents/agent";
|
|
5234
|
-
import {
|
|
5235
|
-
agent as agent2,
|
|
5236
|
-
chat,
|
|
5237
|
-
errorRecoveryGuardrail,
|
|
5238
|
-
fragment as fragment3
|
|
5239
|
-
} from "@deepagents/context";
|
|
5240
|
-
var TEXT2SQL_INDEX_PROGRESS_CHUNK = "data-text2sql-index-progress";
|
|
5241
5483
|
var Text2Sql = class {
|
|
5242
5484
|
#config;
|
|
5243
|
-
#
|
|
5485
|
+
#indexer;
|
|
5244
5486
|
constructor(config) {
|
|
5245
5487
|
const adapterNames = Object.keys(config.adapters);
|
|
5246
5488
|
if (adapterNames.length === 0) {
|
|
5247
5489
|
throw new Error("Text2Sql requires at least one adapter");
|
|
5248
5490
|
}
|
|
5249
5491
|
validateAdapterNames(adapterNames);
|
|
5250
|
-
this.#config =
|
|
5251
|
-
|
|
5252
|
-
|
|
5253
|
-
|
|
5254
|
-
|
|
5255
|
-
Object.keys(config.adapters).map((name) => [
|
|
5256
|
-
name,
|
|
5257
|
-
new JsonCache(
|
|
5258
|
-
`introspection-${config.version}-${name}`
|
|
5259
|
-
)
|
|
5260
|
-
])
|
|
5261
|
-
);
|
|
5492
|
+
this.#config = config;
|
|
5493
|
+
this.#indexer = new AdapterIndexer({
|
|
5494
|
+
adapters: config.adapters,
|
|
5495
|
+
version: config.version
|
|
5496
|
+
});
|
|
5262
5497
|
}
|
|
5263
5498
|
#requireAdapter(name) {
|
|
5264
5499
|
const adapter = this.#config.adapters[name];
|
|
@@ -5270,7 +5505,7 @@ var Text2Sql = class {
|
|
|
5270
5505
|
}
|
|
5271
5506
|
async toSql(input, adapterName) {
|
|
5272
5507
|
const adapter = this.#requireAdapter(adapterName);
|
|
5273
|
-
const fragments = await this.#indexAdapter(adapterName
|
|
5508
|
+
const fragments = await this.#indexer.indexAdapter(adapterName);
|
|
5274
5509
|
const result = await toSql({
|
|
5275
5510
|
input,
|
|
5276
5511
|
adapter,
|
|
@@ -5279,158 +5514,15 @@ var Text2Sql = class {
|
|
|
5279
5514
|
});
|
|
5280
5515
|
return result.sql;
|
|
5281
5516
|
}
|
|
5282
|
-
async index() {
|
|
5283
|
-
return this.#index();
|
|
5284
|
-
}
|
|
5285
|
-
async #index(onProgress) {
|
|
5286
|
-
const entries = Object.entries(this.#config.adapters);
|
|
5287
|
-
onProgress?.({
|
|
5288
|
-
type: "index:start",
|
|
5289
|
-
message: `Indexing ${entries.length} adapter${entries.length === 1 ? "" : "s"}...`,
|
|
5290
|
-
current: 0,
|
|
5291
|
-
total: entries.length
|
|
5292
|
-
});
|
|
5293
|
-
const settled = await Promise.allSettled(
|
|
5294
|
-
entries.map(async ([name, adapter]) => {
|
|
5295
|
-
const schema = await this.#indexAdapter(name, adapter, onProgress);
|
|
5296
|
-
return fragment3(name, ...schema);
|
|
5297
|
-
})
|
|
5298
|
-
);
|
|
5299
|
-
const failed = settled.find((result) => result.status === "rejected");
|
|
5300
|
-
if (failed) {
|
|
5301
|
-
onProgress?.({
|
|
5302
|
-
type: "index:error",
|
|
5303
|
-
message: failed.reason instanceof Error ? failed.reason.message : String(failed.reason)
|
|
5304
|
-
});
|
|
5305
|
-
throw failed.reason;
|
|
5306
|
-
}
|
|
5307
|
-
const wrapped = settled.map((result) => {
|
|
5308
|
-
if (result.status === "rejected") {
|
|
5309
|
-
throw result.reason;
|
|
5310
|
-
}
|
|
5311
|
-
return result.value;
|
|
5312
|
-
});
|
|
5313
|
-
onProgress?.({
|
|
5314
|
-
type: "index:end",
|
|
5315
|
-
message: "Finished indexing adapters.",
|
|
5316
|
-
current: entries.length,
|
|
5317
|
-
total: entries.length
|
|
5318
|
-
});
|
|
5319
|
-
return wrapped;
|
|
5320
|
-
}
|
|
5321
|
-
async #indexAdapter(name, adapter, onProgress) {
|
|
5322
|
-
onProgress?.({
|
|
5323
|
-
type: "adapter:start",
|
|
5324
|
-
adapter: name,
|
|
5325
|
-
message: `Indexing adapter "${name}"...`
|
|
5326
|
-
});
|
|
5327
|
-
const cache = this.#introspection.get(name);
|
|
5328
|
-
if (!cache) {
|
|
5329
|
-
throw new Error(`no introspection cache registered for "${name}"`);
|
|
5330
|
-
}
|
|
5331
|
-
try {
|
|
5332
|
-
const cached = await cache.read();
|
|
5333
|
-
if (cached) {
|
|
5334
|
-
onProgress?.({
|
|
5335
|
-
type: "adapter:cache-hit",
|
|
5336
|
-
adapter: name,
|
|
5337
|
-
message: `Using cached index for adapter "${name}".`,
|
|
5338
|
-
cached: true
|
|
5339
|
-
});
|
|
5340
|
-
onProgress?.({
|
|
5341
|
-
type: "adapter:end",
|
|
5342
|
-
adapter: name,
|
|
5343
|
-
message: `Finished indexing adapter "${name}".`,
|
|
5344
|
-
cached: true
|
|
5345
|
-
});
|
|
5346
|
-
return cached;
|
|
5347
|
-
}
|
|
5348
|
-
onProgress?.({
|
|
5349
|
-
type: "adapter:cache-miss",
|
|
5350
|
-
adapter: name,
|
|
5351
|
-
message: `No cached index for adapter "${name}".`,
|
|
5352
|
-
cached: false
|
|
5353
|
-
});
|
|
5354
|
-
const ctx = createGroundingContext({
|
|
5355
|
-
onProgress: (progress) => onProgress?.(this.#adapterProgressEvent(name, progress))
|
|
5356
|
-
});
|
|
5357
|
-
const fragments = await adapter.introspect(ctx);
|
|
5358
|
-
await cache.write(fragments);
|
|
5359
|
-
onProgress?.({
|
|
5360
|
-
type: "adapter:end",
|
|
5361
|
-
adapter: name,
|
|
5362
|
-
message: `Finished indexing adapter "${name}".`,
|
|
5363
|
-
cached: false
|
|
5364
|
-
});
|
|
5365
|
-
return fragments;
|
|
5366
|
-
} catch (error) {
|
|
5367
|
-
const reason = error instanceof Error ? error.message : String(error);
|
|
5368
|
-
onProgress?.({
|
|
5369
|
-
type: "adapter:error",
|
|
5370
|
-
adapter: name,
|
|
5371
|
-
message: `Failed indexing adapter "${name}": ${reason}`
|
|
5372
|
-
});
|
|
5373
|
-
throw new Error(`introspecting adapter "${name}": ${reason}`, {
|
|
5374
|
-
cause: error
|
|
5375
|
-
});
|
|
5376
|
-
}
|
|
5377
|
-
}
|
|
5378
|
-
#adapterProgressEvent(adapter, progress) {
|
|
5379
|
-
return {
|
|
5380
|
-
type: progress.type,
|
|
5381
|
-
adapter,
|
|
5382
|
-
phase: progress.phase,
|
|
5383
|
-
table: progress.table,
|
|
5384
|
-
message: progress.message,
|
|
5385
|
-
current: progress.current,
|
|
5386
|
-
total: progress.total,
|
|
5387
|
-
cached: false
|
|
5388
|
-
};
|
|
5389
|
-
}
|
|
5390
5517
|
async toPairs(adapterName, factory) {
|
|
5391
5518
|
const adapter = this.#requireAdapter(adapterName);
|
|
5392
5519
|
const producer = factory(adapter);
|
|
5393
5520
|
return producer.toPairs();
|
|
5394
5521
|
}
|
|
5395
|
-
async chat(options) {
|
|
5396
|
-
return createUIMessageStream({
|
|
5397
|
-
execute: async ({ writer }) => {
|
|
5398
|
-
const context = this.#config.context;
|
|
5399
|
-
const head = await context.headMessage();
|
|
5400
|
-
if (!head || head.name !== "assistant") {
|
|
5401
|
-
throw new Error(
|
|
5402
|
-
"text2sql.chat: expected head to be an assistant message \u2014 call context.continue() before chat()"
|
|
5403
|
-
);
|
|
5404
|
-
}
|
|
5405
|
-
writer.write({ type: "start", messageId: head.id });
|
|
5406
|
-
const progress = (event) => {
|
|
5407
|
-
writer.write({
|
|
5408
|
-
type: TEXT2SQL_INDEX_PROGRESS_CHUNK,
|
|
5409
|
-
data: event
|
|
5410
|
-
});
|
|
5411
|
-
};
|
|
5412
|
-
context.set(...guidelines(), ...await this.#index(progress));
|
|
5413
|
-
const chatAgent = agent2({
|
|
5414
|
-
name: "text2sql",
|
|
5415
|
-
sandbox: this.#config.sandbox,
|
|
5416
|
-
model: this.#config.model,
|
|
5417
|
-
context,
|
|
5418
|
-
tools: this.#config.tools,
|
|
5419
|
-
guardrails: [errorRecoveryGuardrail],
|
|
5420
|
-
maxGuardrailRetries: 3
|
|
5421
|
-
});
|
|
5422
|
-
const chatStream = await chat(chatAgent, {
|
|
5423
|
-
abortSignal: options?.abortSignal,
|
|
5424
|
-
generateTitle: options?.generateTitle,
|
|
5425
|
-
transform: this.#config.transform
|
|
5426
|
-
});
|
|
5427
|
-
writer.merge(chatStream);
|
|
5428
|
-
}
|
|
5429
|
-
});
|
|
5430
|
-
}
|
|
5431
5522
|
};
|
|
5432
5523
|
export {
|
|
5433
5524
|
Adapter,
|
|
5525
|
+
AdapterIndexer,
|
|
5434
5526
|
Checkpoint,
|
|
5435
5527
|
FileCache,
|
|
5436
5528
|
JsonCache,
|
|
@@ -5441,9 +5533,6 @@ export {
|
|
|
5441
5533
|
SQLScopeError,
|
|
5442
5534
|
SQLValidationError,
|
|
5443
5535
|
ScopedFs,
|
|
5444
|
-
SqlBacktickRewritePlugin,
|
|
5445
|
-
SqlProxyEnforcementPlugin,
|
|
5446
|
-
SqlProxyViolationError,
|
|
5447
5536
|
SqliteFs,
|
|
5448
5537
|
TEXT2SQL_INDEX_PROGRESS_CHUNK,
|
|
5449
5538
|
Text2Sql,
|
|
@@ -5451,21 +5540,23 @@ export {
|
|
|
5451
5540
|
applyTablesFilter,
|
|
5452
5541
|
column,
|
|
5453
5542
|
constraint,
|
|
5454
|
-
|
|
5543
|
+
createSqlCommandHooks,
|
|
5544
|
+
createSqlMetaHook,
|
|
5455
5545
|
dialectInfo,
|
|
5456
5546
|
filterRelationshipsByTables,
|
|
5457
5547
|
filterTablesByName,
|
|
5458
5548
|
getTablesWithRelated,
|
|
5459
|
-
guidelines,
|
|
5460
5549
|
hashConfig,
|
|
5461
5550
|
index,
|
|
5551
|
+
instructions,
|
|
5462
5552
|
isValidAdapterName,
|
|
5463
5553
|
matchesFilter,
|
|
5464
5554
|
relationship,
|
|
5465
|
-
|
|
5555
|
+
repairSqlCommand,
|
|
5466
5556
|
suggestionsAgent,
|
|
5467
5557
|
table,
|
|
5468
5558
|
toSql,
|
|
5559
|
+
transformSqlCommand,
|
|
5469
5560
|
validateAdapterNames,
|
|
5470
5561
|
view
|
|
5471
5562
|
};
|