@deepagents/text2sql 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +22 -17
- package/dist/index.js.map +3 -3
- package/dist/lib/adapters/bigquery/bigquery-fk.d.ts +25 -0
- package/dist/lib/adapters/bigquery/bigquery-fk.d.ts.map +1 -0
- package/dist/lib/adapters/bigquery/constraint.bigquery.grounding.d.ts +3 -1
- package/dist/lib/adapters/bigquery/constraint.bigquery.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/bigquery/index.d.ts.map +1 -1
- package/dist/lib/adapters/bigquery/index.js +356 -201
- package/dist/lib/adapters/bigquery/index.js.map +4 -4
- package/dist/lib/adapters/bigquery/indexes.bigquery.grounding.d.ts +6 -7
- package/dist/lib/adapters/bigquery/indexes.bigquery.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/bigquery/info.bigquery.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/bigquery/row-count.bigquery.grounding.d.ts +5 -7
- package/dist/lib/adapters/bigquery/row-count.bigquery.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/bigquery/table.bigquery.grounding.d.ts +2 -0
- package/dist/lib/adapters/bigquery/table.bigquery.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/bigquery/view.bigquery.grounding.d.ts +2 -2
- package/dist/lib/adapters/bigquery/view.bigquery.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/groundings/context.d.ts +2 -0
- package/dist/lib/adapters/groundings/context.d.ts.map +1 -1
- package/dist/lib/adapters/groundings/index.js +2 -1
- package/dist/lib/adapters/groundings/index.js.map +2 -2
- package/dist/lib/adapters/mysql/index.js +2 -1
- package/dist/lib/adapters/mysql/index.js.map +2 -2
- package/dist/lib/adapters/postgres/index.js +2 -1
- package/dist/lib/adapters/postgres/index.js.map +2 -2
- package/dist/lib/adapters/spreadsheet/index.js +2 -1
- package/dist/lib/adapters/spreadsheet/index.js.map +2 -2
- package/dist/lib/adapters/sqlite/index.js +2 -1
- package/dist/lib/adapters/sqlite/index.js.map +2 -2
- package/dist/lib/adapters/sqlserver/index.js +2 -1
- package/dist/lib/adapters/sqlserver/index.js.map +2 -2
- package/dist/lib/sql.d.ts +3 -3
- package/dist/lib/sql.d.ts.map +1 -1
- package/dist/lib/synthesis/index.js +1 -1
- package/dist/lib/synthesis/index.js.map +1 -1
- package/package.json +8 -7
|
@@ -98,7 +98,8 @@ function createGroundingContext() {
|
|
|
98
98
|
tables: [],
|
|
99
99
|
views: [],
|
|
100
100
|
relationships: [],
|
|
101
|
-
info: void 0
|
|
101
|
+
info: void 0,
|
|
102
|
+
cache: /* @__PURE__ */ new Map()
|
|
102
103
|
};
|
|
103
104
|
}
|
|
104
105
|
|
|
@@ -737,43 +738,152 @@ var BigQuery = class extends Adapter {
|
|
|
737
738
|
}
|
|
738
739
|
};
|
|
739
740
|
|
|
741
|
+
// packages/text2sql/src/lib/adapters/bigquery/bigquery-fk.ts
|
|
742
|
+
var FK_CACHE_PREFIX = "fk:";
|
|
743
|
+
async function resolveForeignKey(adapter, constraintDataset, constraintName, childColumns, cache) {
|
|
744
|
+
const cacheKey = `${FK_CACHE_PREFIX}${constraintDataset}:${constraintName}`;
|
|
745
|
+
if (cache?.has(cacheKey)) {
|
|
746
|
+
const cached = cache.get(cacheKey);
|
|
747
|
+
if (!cached) return void 0;
|
|
748
|
+
return {
|
|
749
|
+
...cached,
|
|
750
|
+
childColumns: [...childColumns].sort((a, b) => a.ordinal - b.ordinal).map((c) => c.column)
|
|
751
|
+
};
|
|
752
|
+
}
|
|
753
|
+
const result = await resolveFK(
|
|
754
|
+
adapter,
|
|
755
|
+
constraintDataset,
|
|
756
|
+
constraintName,
|
|
757
|
+
childColumns
|
|
758
|
+
);
|
|
759
|
+
cache?.set(cacheKey, result);
|
|
760
|
+
return result;
|
|
761
|
+
}
|
|
762
|
+
async function resolveFK(adapter, constraintDataset, constraintName, childColumns) {
|
|
763
|
+
const refRows = await adapter.runQuery(`
|
|
764
|
+
SELECT DISTINCT table_schema, table_name
|
|
765
|
+
FROM ${adapter.infoSchemaView(constraintDataset, "CONSTRAINT_COLUMN_USAGE")}
|
|
766
|
+
WHERE constraint_name = '${adapter.escapeString(constraintName)}'
|
|
767
|
+
`);
|
|
768
|
+
const referenced = refRows.find((r) => r.table_schema && r.table_name);
|
|
769
|
+
if (!referenced?.table_schema || !referenced.table_name) {
|
|
770
|
+
return void 0;
|
|
771
|
+
}
|
|
772
|
+
const referencedDataset = referenced.table_schema;
|
|
773
|
+
const referencedTable = referenced.table_name;
|
|
774
|
+
if (!adapter.isDatasetAllowed(referencedDataset)) {
|
|
775
|
+
return void 0;
|
|
776
|
+
}
|
|
777
|
+
const pkConstraintRows = await adapter.runQuery(`
|
|
778
|
+
SELECT constraint_name
|
|
779
|
+
FROM ${adapter.infoSchemaView(referencedDataset, "TABLE_CONSTRAINTS")}
|
|
780
|
+
WHERE constraint_type = 'PRIMARY KEY'
|
|
781
|
+
AND table_name = '${adapter.escapeString(referencedTable)}'
|
|
782
|
+
LIMIT 1
|
|
783
|
+
`);
|
|
784
|
+
const pkConstraintName = pkConstraintRows[0]?.constraint_name;
|
|
785
|
+
if (!pkConstraintName) return void 0;
|
|
786
|
+
const pkColumnRows = await adapter.runQuery(`
|
|
787
|
+
SELECT column_name, ordinal_position
|
|
788
|
+
FROM ${adapter.infoSchemaView(referencedDataset, "KEY_COLUMN_USAGE")}
|
|
789
|
+
WHERE constraint_name = '${adapter.escapeString(pkConstraintName)}'
|
|
790
|
+
AND table_name = '${adapter.escapeString(referencedTable)}'
|
|
791
|
+
ORDER BY ordinal_position
|
|
792
|
+
`);
|
|
793
|
+
const pkByOrdinal = /* @__PURE__ */ new Map();
|
|
794
|
+
for (const row of pkColumnRows) {
|
|
795
|
+
if (!row.column_name || row.ordinal_position == null) continue;
|
|
796
|
+
pkByOrdinal.set(row.ordinal_position, row.column_name);
|
|
797
|
+
}
|
|
798
|
+
const ordered = [...childColumns].sort((a, b) => a.ordinal - b.ordinal);
|
|
799
|
+
return {
|
|
800
|
+
referencedDataset,
|
|
801
|
+
referencedTable,
|
|
802
|
+
referencedColumns: ordered.map((c) => {
|
|
803
|
+
const pkOrdinal = c.pkOrdinal ?? c.ordinal;
|
|
804
|
+
return pkByOrdinal.get(pkOrdinal) ?? "unknown";
|
|
805
|
+
}),
|
|
806
|
+
childColumns: ordered.map((c) => c.column)
|
|
807
|
+
};
|
|
808
|
+
}
|
|
809
|
+
|
|
740
810
|
// packages/text2sql/src/lib/adapters/bigquery/constraint.bigquery.grounding.ts
|
|
741
811
|
var BigQueryConstraintGrounding = class extends ConstraintGrounding {
|
|
742
812
|
#adapter;
|
|
813
|
+
#cache;
|
|
743
814
|
constructor(adapter, config = {}) {
|
|
744
815
|
super(config);
|
|
745
816
|
this.#adapter = adapter;
|
|
746
817
|
}
|
|
747
|
-
async
|
|
748
|
-
|
|
749
|
-
const
|
|
750
|
-
const
|
|
751
|
-
|
|
818
|
+
async execute(ctx) {
|
|
819
|
+
this.#cache = ctx.cache;
|
|
820
|
+
const byDataset = /* @__PURE__ */ new Map();
|
|
821
|
+
for (const table2 of ctx.tables) {
|
|
822
|
+
const { schema: dataset } = this.#adapter.parseTableName(table2.name);
|
|
823
|
+
const list = byDataset.get(dataset) ?? [];
|
|
824
|
+
list.push(table2);
|
|
825
|
+
byDataset.set(dataset, list);
|
|
826
|
+
}
|
|
827
|
+
for (const [dataset, tables2] of byDataset) {
|
|
828
|
+
try {
|
|
829
|
+
await this.#batchConstraints(dataset, tables2);
|
|
830
|
+
} catch (error) {
|
|
831
|
+
console.warn(
|
|
832
|
+
"Error collecting constraints for dataset",
|
|
833
|
+
dataset,
|
|
834
|
+
error
|
|
835
|
+
);
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
async #batchConstraints(dataset, tables2) {
|
|
840
|
+
const tableNames = tables2.map(
|
|
841
|
+
(t) => this.#adapter.parseTableName(t.name).table
|
|
842
|
+
);
|
|
843
|
+
const inList = tableNames.map((n) => `'${this.#adapter.escapeString(n)}'`).join(", ");
|
|
844
|
+
const constraintsByTable = /* @__PURE__ */ new Map();
|
|
845
|
+
for (const name of tableNames) {
|
|
846
|
+
constraintsByTable.set(name, []);
|
|
847
|
+
}
|
|
848
|
+
await this.#batchColumnMetadata(dataset, inList, constraintsByTable);
|
|
849
|
+
await this.#batchKeyConstraints(dataset, inList, constraintsByTable);
|
|
850
|
+
for (const table2 of tables2) {
|
|
851
|
+
const rawName = this.#adapter.parseTableName(table2.name).table;
|
|
852
|
+
table2.constraints = constraintsByTable.get(rawName) ?? [];
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
async #batchColumnMetadata(dataset, inList, constraintsByTable) {
|
|
856
|
+
const rows = await this.#adapter.runQuery(`
|
|
857
|
+
SELECT table_name, column_name, is_nullable, column_default
|
|
752
858
|
FROM ${this.#adapter.infoSchemaView(dataset, "COLUMNS")}
|
|
753
|
-
WHERE table_name
|
|
754
|
-
ORDER BY ordinal_position
|
|
859
|
+
WHERE table_name IN (${inList})
|
|
860
|
+
ORDER BY table_name, ordinal_position
|
|
755
861
|
`);
|
|
756
|
-
for (const row of
|
|
757
|
-
|
|
758
|
-
|
|
862
|
+
for (const row of rows) {
|
|
863
|
+
if (!row.table_name || !row.column_name) continue;
|
|
864
|
+
const constraints2 = constraintsByTable.get(row.table_name);
|
|
865
|
+
if (!constraints2) continue;
|
|
759
866
|
if ((row.is_nullable ?? "").toUpperCase() === "NO") {
|
|
760
867
|
constraints2.push({
|
|
761
|
-
name: `${
|
|
868
|
+
name: `${dataset}.${row.table_name}.${row.column_name}.NOT_NULL`,
|
|
762
869
|
type: "NOT_NULL",
|
|
763
|
-
columns: [
|
|
870
|
+
columns: [row.column_name]
|
|
764
871
|
});
|
|
765
872
|
}
|
|
766
873
|
if (row.column_default != null && row.column_default !== "") {
|
|
767
874
|
constraints2.push({
|
|
768
|
-
name: `${
|
|
875
|
+
name: `${dataset}.${row.table_name}.${row.column_name}.DEFAULT`,
|
|
769
876
|
type: "DEFAULT",
|
|
770
|
-
columns: [
|
|
877
|
+
columns: [row.column_name],
|
|
771
878
|
defaultValue: row.column_default
|
|
772
879
|
});
|
|
773
880
|
}
|
|
774
881
|
}
|
|
775
|
-
|
|
882
|
+
}
|
|
883
|
+
async #batchKeyConstraints(dataset, inList, constraintsByTable) {
|
|
884
|
+
const rows = await this.#adapter.runQuery(`
|
|
776
885
|
SELECT
|
|
886
|
+
tc.table_name,
|
|
777
887
|
tc.constraint_name,
|
|
778
888
|
tc.constraint_type,
|
|
779
889
|
kcu.column_name,
|
|
@@ -783,100 +893,72 @@ var BigQueryConstraintGrounding = class extends ConstraintGrounding {
|
|
|
783
893
|
JOIN ${this.#adapter.infoSchemaView(dataset, "KEY_COLUMN_USAGE")} AS kcu
|
|
784
894
|
ON tc.constraint_name = kcu.constraint_name
|
|
785
895
|
AND tc.constraint_schema = kcu.constraint_schema
|
|
786
|
-
WHERE tc.table_name
|
|
896
|
+
WHERE tc.table_name IN (${inList})
|
|
787
897
|
AND tc.constraint_type IN ('PRIMARY KEY', 'FOREIGN KEY')
|
|
788
|
-
ORDER BY tc.constraint_name, kcu.ordinal_position
|
|
898
|
+
ORDER BY tc.table_name, tc.constraint_name, kcu.ordinal_position
|
|
789
899
|
`);
|
|
790
|
-
const
|
|
791
|
-
const
|
|
792
|
-
for (const row of
|
|
793
|
-
if (!row.constraint_name || !row.column_name) continue;
|
|
900
|
+
const pkByTable = /* @__PURE__ */ new Map();
|
|
901
|
+
const fkByTable = /* @__PURE__ */ new Map();
|
|
902
|
+
for (const row of rows) {
|
|
903
|
+
if (!row.table_name || !row.constraint_name || !row.column_name) continue;
|
|
794
904
|
const type = (row.constraint_type ?? "").toUpperCase();
|
|
795
905
|
if (type === "PRIMARY KEY") {
|
|
796
|
-
const
|
|
906
|
+
const tableMap = pkByTable.get(row.table_name) ?? /* @__PURE__ */ new Map();
|
|
907
|
+
const cols = tableMap.get(row.constraint_name) ?? [];
|
|
797
908
|
cols.push(row.column_name);
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
}
|
|
801
|
-
|
|
802
|
-
const cols =
|
|
909
|
+
tableMap.set(row.constraint_name, cols);
|
|
910
|
+
pkByTable.set(row.table_name, tableMap);
|
|
911
|
+
} else if (type === "FOREIGN KEY") {
|
|
912
|
+
const tableMap = fkByTable.get(row.table_name) ?? /* @__PURE__ */ new Map();
|
|
913
|
+
const cols = tableMap.get(row.constraint_name) ?? [];
|
|
803
914
|
cols.push({
|
|
804
915
|
column: row.column_name,
|
|
805
916
|
ordinal: row.ordinal_position ?? 0,
|
|
806
917
|
pkOrdinal: row.position_in_unique_constraint
|
|
807
918
|
});
|
|
808
|
-
|
|
919
|
+
tableMap.set(row.constraint_name, cols);
|
|
920
|
+
fkByTable.set(row.table_name, tableMap);
|
|
809
921
|
}
|
|
810
922
|
}
|
|
811
|
-
for (const [
|
|
812
|
-
constraints2.
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
columns: cols
|
|
816
|
-
}
|
|
923
|
+
for (const [tableName, pkMap] of pkByTable) {
|
|
924
|
+
const constraints2 = constraintsByTable.get(tableName);
|
|
925
|
+
if (!constraints2) continue;
|
|
926
|
+
for (const [name, cols] of pkMap) {
|
|
927
|
+
constraints2.push({ name, type: "PRIMARY_KEY", columns: cols });
|
|
928
|
+
}
|
|
817
929
|
}
|
|
818
|
-
for (const [
|
|
819
|
-
const
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
930
|
+
for (const [tableName, fkMap] of fkByTable) {
|
|
931
|
+
const constraints2 = constraintsByTable.get(tableName);
|
|
932
|
+
if (!constraints2) continue;
|
|
933
|
+
for (const [constraintName, childColumns] of fkMap) {
|
|
934
|
+
try {
|
|
935
|
+
const resolution = await resolveForeignKey(
|
|
936
|
+
this.#adapter,
|
|
937
|
+
dataset,
|
|
938
|
+
constraintName,
|
|
939
|
+
childColumns,
|
|
940
|
+
this.#cache
|
|
941
|
+
);
|
|
942
|
+
if (resolution) {
|
|
943
|
+
constraints2.push({
|
|
944
|
+
name: constraintName,
|
|
945
|
+
type: "FOREIGN_KEY",
|
|
946
|
+
columns: resolution.childColumns,
|
|
947
|
+
referencedTable: `${resolution.referencedDataset}.${resolution.referencedTable}`,
|
|
948
|
+
referencedColumns: resolution.referencedColumns
|
|
949
|
+
});
|
|
950
|
+
}
|
|
951
|
+
} catch (err) {
|
|
952
|
+
console.warn(
|
|
953
|
+
`Failed to resolve FK constraint ${constraintName}:`,
|
|
954
|
+
err
|
|
955
|
+
);
|
|
956
|
+
}
|
|
957
|
+
}
|
|
826
958
|
}
|
|
827
|
-
return constraints2;
|
|
828
959
|
}
|
|
829
|
-
async
|
|
830
|
-
|
|
831
|
-
SELECT DISTINCT table_schema, table_name
|
|
832
|
-
FROM ${this.#adapter.infoSchemaView(args.constraintDataset, "CONSTRAINT_COLUMN_USAGE")}
|
|
833
|
-
WHERE constraint_name = '${this.#adapter.escapeString(args.constraintName)}'
|
|
834
|
-
`);
|
|
835
|
-
const referenced = refRows.find((r) => r.table_schema && r.table_name);
|
|
836
|
-
if (!referenced?.table_schema || !referenced.table_name) {
|
|
837
|
-
return void 0;
|
|
838
|
-
}
|
|
839
|
-
const referencedDataset = referenced.table_schema;
|
|
840
|
-
const referencedTable = referenced.table_name;
|
|
841
|
-
if (!this.#adapter.isDatasetAllowed(referencedDataset)) {
|
|
842
|
-
return void 0;
|
|
843
|
-
}
|
|
844
|
-
const pkConstraintRows = await this.#adapter.runQuery(`
|
|
845
|
-
SELECT constraint_name
|
|
846
|
-
FROM ${this.#adapter.infoSchemaView(referencedDataset, "TABLE_CONSTRAINTS")}
|
|
847
|
-
WHERE constraint_type = 'PRIMARY KEY'
|
|
848
|
-
AND table_name = '${this.#adapter.escapeString(referencedTable)}'
|
|
849
|
-
LIMIT 1
|
|
850
|
-
`);
|
|
851
|
-
const pkConstraintName = pkConstraintRows[0]?.constraint_name;
|
|
852
|
-
if (!pkConstraintName) return void 0;
|
|
853
|
-
const pkColumns = await this.#adapter.runQuery(`
|
|
854
|
-
SELECT column_name, ordinal_position
|
|
855
|
-
FROM ${this.#adapter.infoSchemaView(referencedDataset, "KEY_COLUMN_USAGE")}
|
|
856
|
-
WHERE constraint_name = '${this.#adapter.escapeString(pkConstraintName)}'
|
|
857
|
-
AND table_name = '${this.#adapter.escapeString(referencedTable)}'
|
|
858
|
-
ORDER BY ordinal_position
|
|
859
|
-
`);
|
|
860
|
-
const pkByOrdinal = /* @__PURE__ */ new Map();
|
|
861
|
-
for (const row of pkColumns) {
|
|
862
|
-
if (!row.column_name || row.ordinal_position == null) continue;
|
|
863
|
-
pkByOrdinal.set(row.ordinal_position, row.column_name);
|
|
864
|
-
}
|
|
865
|
-
const orderedChild = [...args.childColumns].sort(
|
|
866
|
-
(a, b) => a.ordinal - b.ordinal
|
|
867
|
-
);
|
|
868
|
-
const columns = orderedChild.map((c) => c.column);
|
|
869
|
-
const referencedColumns = orderedChild.map((c) => {
|
|
870
|
-
const pkOrdinal = c.pkOrdinal ?? c.ordinal;
|
|
871
|
-
return pkByOrdinal.get(pkOrdinal) ?? "unknown";
|
|
872
|
-
});
|
|
873
|
-
return {
|
|
874
|
-
name: args.constraintName,
|
|
875
|
-
type: "FOREIGN_KEY",
|
|
876
|
-
columns,
|
|
877
|
-
referencedTable: `${referencedDataset}.${referencedTable}`,
|
|
878
|
-
referencedColumns
|
|
879
|
-
};
|
|
960
|
+
async getConstraints(_tableName) {
|
|
961
|
+
return [];
|
|
880
962
|
}
|
|
881
963
|
};
|
|
882
964
|
|
|
@@ -887,46 +969,82 @@ var BigQueryIndexesGrounding = class extends IndexesGrounding {
|
|
|
887
969
|
super(config);
|
|
888
970
|
this.#adapter = adapter;
|
|
889
971
|
}
|
|
890
|
-
async
|
|
891
|
-
const
|
|
972
|
+
async execute(ctx) {
|
|
973
|
+
const byDataset = /* @__PURE__ */ new Map();
|
|
974
|
+
for (const table2 of ctx.tables) {
|
|
975
|
+
const { schema: dataset } = this.#adapter.parseTableName(table2.name);
|
|
976
|
+
const list = byDataset.get(dataset) ?? [];
|
|
977
|
+
list.push(table2);
|
|
978
|
+
byDataset.set(dataset, list);
|
|
979
|
+
}
|
|
980
|
+
for (const [dataset, tables2] of byDataset) {
|
|
981
|
+
try {
|
|
982
|
+
await this.#batchIndexes(dataset, tables2);
|
|
983
|
+
} catch {
|
|
984
|
+
}
|
|
985
|
+
}
|
|
986
|
+
}
|
|
987
|
+
async #batchIndexes(dataset, tables2) {
|
|
988
|
+
const tableNames = tables2.map(
|
|
989
|
+
(t) => this.#adapter.parseTableName(t.name).table
|
|
990
|
+
);
|
|
991
|
+
const inList = tableNames.map((n) => `'${this.#adapter.escapeString(n)}'`).join(", ");
|
|
892
992
|
const rows = await this.#adapter.runQuery(`
|
|
893
|
-
SELECT column_name, is_partitioning_column, clustering_ordinal_position
|
|
993
|
+
SELECT table_name, column_name, is_partitioning_column, clustering_ordinal_position
|
|
894
994
|
FROM ${this.#adapter.infoSchemaView(dataset, "COLUMNS")}
|
|
895
|
-
WHERE table_name
|
|
995
|
+
WHERE table_name IN (${inList})
|
|
896
996
|
AND (is_partitioning_column = 'YES' OR clustering_ordinal_position IS NOT NULL)
|
|
897
|
-
ORDER BY clustering_ordinal_position
|
|
997
|
+
ORDER BY table_name, clustering_ordinal_position
|
|
898
998
|
`);
|
|
899
|
-
const
|
|
900
|
-
const clusteringColumns = [];
|
|
999
|
+
const byTable = /* @__PURE__ */ new Map();
|
|
901
1000
|
for (const row of rows) {
|
|
902
|
-
if (!row.column_name) continue;
|
|
1001
|
+
if (!row.table_name || !row.column_name) continue;
|
|
1002
|
+
const entry = byTable.get(row.table_name) ?? {
|
|
1003
|
+
partition: [],
|
|
1004
|
+
clustering: []
|
|
1005
|
+
};
|
|
903
1006
|
if ((row.is_partitioning_column ?? "").toUpperCase() === "YES") {
|
|
904
|
-
|
|
1007
|
+
entry.partition.push(row.column_name);
|
|
905
1008
|
}
|
|
906
1009
|
if (row.clustering_ordinal_position != null) {
|
|
907
|
-
|
|
1010
|
+
entry.clustering.push({
|
|
908
1011
|
name: row.column_name,
|
|
909
1012
|
pos: row.clustering_ordinal_position
|
|
910
1013
|
});
|
|
911
1014
|
}
|
|
1015
|
+
byTable.set(row.table_name, entry);
|
|
1016
|
+
}
|
|
1017
|
+
for (const table2 of tables2) {
|
|
1018
|
+
const rawName = this.#adapter.parseTableName(table2.name).table;
|
|
1019
|
+
const entry = byTable.get(rawName);
|
|
1020
|
+
if (!entry) continue;
|
|
1021
|
+
const indexes2 = [];
|
|
1022
|
+
if (entry.partition.length > 0) {
|
|
1023
|
+
indexes2.push({
|
|
1024
|
+
name: `${rawName}_partition`,
|
|
1025
|
+
columns: entry.partition,
|
|
1026
|
+
type: "PARTITION"
|
|
1027
|
+
});
|
|
1028
|
+
}
|
|
1029
|
+
if (entry.clustering.length > 0) {
|
|
1030
|
+
entry.clustering.sort((a, b) => a.pos - b.pos);
|
|
1031
|
+
indexes2.push({
|
|
1032
|
+
name: `${rawName}_clustering`,
|
|
1033
|
+
columns: entry.clustering.map((c) => c.name),
|
|
1034
|
+
type: "CLUSTERING"
|
|
1035
|
+
});
|
|
1036
|
+
}
|
|
1037
|
+
table2.indexes = indexes2;
|
|
1038
|
+
for (const idx of indexes2) {
|
|
1039
|
+
for (const colName of idx.columns) {
|
|
1040
|
+
const column2 = table2.columns.find((c) => c.name === colName);
|
|
1041
|
+
if (column2) column2.isIndexed = true;
|
|
1042
|
+
}
|
|
1043
|
+
}
|
|
912
1044
|
}
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
name: `${table2}_partition`,
|
|
917
|
-
columns: partitionColumns,
|
|
918
|
-
type: "PARTITION"
|
|
919
|
-
});
|
|
920
|
-
}
|
|
921
|
-
if (clusteringColumns.length > 0) {
|
|
922
|
-
clusteringColumns.sort((a, b) => a.pos - b.pos);
|
|
923
|
-
indexes2.push({
|
|
924
|
-
name: `${table2}_clustering`,
|
|
925
|
-
columns: clusteringColumns.map((c) => c.name),
|
|
926
|
-
type: "CLUSTERING"
|
|
927
|
-
});
|
|
928
|
-
}
|
|
929
|
-
return indexes2;
|
|
1045
|
+
}
|
|
1046
|
+
async getIndexes(_tableName) {
|
|
1047
|
+
return [];
|
|
930
1048
|
}
|
|
931
1049
|
};
|
|
932
1050
|
|
|
@@ -938,13 +1056,14 @@ var BigQueryInfoGrounding = class extends InfoGrounding {
|
|
|
938
1056
|
this.#adapter = adapter;
|
|
939
1057
|
}
|
|
940
1058
|
async collectInfo() {
|
|
1059
|
+
const qualifiedTable = this.#adapter.projectId ? "project.dataset.table" : "dataset.table";
|
|
941
1060
|
return {
|
|
942
1061
|
dialect: "bigquery",
|
|
943
1062
|
database: this.#adapter.projectId,
|
|
944
1063
|
details: {
|
|
945
1064
|
identifierQuote: "`",
|
|
946
1065
|
identifiers: {
|
|
947
|
-
qualifiedTable
|
|
1066
|
+
qualifiedTable,
|
|
948
1067
|
nestedFieldPath: "col.path.to.field"
|
|
949
1068
|
},
|
|
950
1069
|
parameters: {
|
|
@@ -963,26 +1082,99 @@ var BigQueryRowCountGrounding = class extends RowCountGrounding {
|
|
|
963
1082
|
super(config);
|
|
964
1083
|
this.#adapter = adapter;
|
|
965
1084
|
}
|
|
966
|
-
async
|
|
967
|
-
const
|
|
1085
|
+
async execute(ctx) {
|
|
1086
|
+
const byDataset = /* @__PURE__ */ new Map();
|
|
1087
|
+
for (const table2 of ctx.tables) {
|
|
1088
|
+
const { schema: dataset } = this.#adapter.parseTableName(table2.name);
|
|
1089
|
+
const list = byDataset.get(dataset) ?? [];
|
|
1090
|
+
list.push(table2);
|
|
1091
|
+
byDataset.set(dataset, list);
|
|
1092
|
+
}
|
|
1093
|
+
for (const [dataset, tables2] of byDataset) {
|
|
1094
|
+
const tableNames = tables2.map(
|
|
1095
|
+
(t) => this.#adapter.parseTableName(t.name).table
|
|
1096
|
+
);
|
|
1097
|
+
const counts = await this.#fetchRowCounts(dataset, tableNames);
|
|
1098
|
+
for (const table2 of tables2) {
|
|
1099
|
+
const rawName = this.#adapter.parseTableName(table2.name).table;
|
|
1100
|
+
const count = counts.get(rawName);
|
|
1101
|
+
if (count != null) {
|
|
1102
|
+
table2.rowCount = count;
|
|
1103
|
+
table2.sizeHint = this.#classifyRowCount(count);
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
async #fetchRowCounts(dataset, tableNames) {
|
|
1109
|
+
const inList = tableNames.map((n) => `'${this.#adapter.escapeString(n)}'`).join(", ");
|
|
1110
|
+
try {
|
|
1111
|
+
return await this.#fromTableStorage(dataset, inList);
|
|
1112
|
+
} catch {
|
|
1113
|
+
}
|
|
1114
|
+
try {
|
|
1115
|
+
return await this.#fromLegacyTables(dataset, inList);
|
|
1116
|
+
} catch {
|
|
1117
|
+
}
|
|
1118
|
+
return /* @__PURE__ */ new Map();
|
|
1119
|
+
}
|
|
1120
|
+
async #fromTableStorage(dataset, inList) {
|
|
968
1121
|
const rows = await this.#adapter.runQuery(`
|
|
969
|
-
SELECT total_rows
|
|
1122
|
+
SELECT table_name, total_rows
|
|
970
1123
|
FROM ${this.#adapter.infoSchemaView(dataset, "TABLE_STORAGE")}
|
|
971
|
-
WHERE table_name
|
|
972
|
-
LIMIT 1
|
|
1124
|
+
WHERE table_name IN (${inList})
|
|
973
1125
|
`);
|
|
974
|
-
const
|
|
975
|
-
|
|
1126
|
+
const result = /* @__PURE__ */ new Map();
|
|
1127
|
+
for (const row of rows) {
|
|
1128
|
+
if (!row.table_name) continue;
|
|
1129
|
+
const count = this.#adapter.toNumber(row.total_rows);
|
|
1130
|
+
if (count != null) result.set(row.table_name, count);
|
|
1131
|
+
}
|
|
1132
|
+
return result;
|
|
1133
|
+
}
|
|
1134
|
+
async #fromLegacyTables(dataset, inList) {
|
|
1135
|
+
const projectPrefix = this.#adapter.projectId ? `\`${this.#adapter.projectId}\`.` : "";
|
|
1136
|
+
const rows = await this.#adapter.runQuery(`
|
|
1137
|
+
SELECT table_id AS table_name, row_count
|
|
1138
|
+
FROM ${projectPrefix}\`${dataset}\`.__TABLES__
|
|
1139
|
+
WHERE table_id IN (${inList})
|
|
1140
|
+
`);
|
|
1141
|
+
const result = /* @__PURE__ */ new Map();
|
|
1142
|
+
for (const row of rows) {
|
|
1143
|
+
if (!row.table_name) continue;
|
|
1144
|
+
const count = this.#adapter.toNumber(row.row_count);
|
|
1145
|
+
if (count != null) result.set(row.table_name, count);
|
|
1146
|
+
}
|
|
1147
|
+
return result;
|
|
1148
|
+
}
|
|
1149
|
+
#classifyRowCount(count) {
|
|
1150
|
+
if (count < 100) return "tiny";
|
|
1151
|
+
if (count < 1e3) return "small";
|
|
1152
|
+
if (count < 1e4) return "medium";
|
|
1153
|
+
if (count < 1e5) return "large";
|
|
1154
|
+
return "huge";
|
|
1155
|
+
}
|
|
1156
|
+
async getRowCount(_tableName) {
|
|
1157
|
+
return void 0;
|
|
976
1158
|
}
|
|
977
1159
|
};
|
|
978
1160
|
|
|
979
1161
|
// packages/text2sql/src/lib/adapters/bigquery/table.bigquery.grounding.ts
|
|
980
1162
|
var BigQueryTableGrounding = class extends TableGrounding {
|
|
981
1163
|
#adapter;
|
|
1164
|
+
#cache;
|
|
982
1165
|
constructor(adapter, config = {}) {
|
|
983
1166
|
super(config);
|
|
984
1167
|
this.#adapter = adapter;
|
|
985
1168
|
}
|
|
1169
|
+
async execute(ctx) {
|
|
1170
|
+
this.#cache = ctx.cache;
|
|
1171
|
+
await super.execute(ctx);
|
|
1172
|
+
ctx.tables = ctx.tables.filter((t) => t.columns.length > 0);
|
|
1173
|
+
const tableNames = new Set(ctx.tables.map((t) => t.name));
|
|
1174
|
+
ctx.relationships = ctx.relationships.filter(
|
|
1175
|
+
(r) => tableNames.has(r.table) && tableNames.has(r.referenced_table)
|
|
1176
|
+
);
|
|
1177
|
+
}
|
|
986
1178
|
async applyFilter() {
|
|
987
1179
|
const names = await super.applyFilter();
|
|
988
1180
|
return names.filter((name) => this.#isTableInScope(name));
|
|
@@ -1064,14 +1256,21 @@ var BigQueryTableGrounding = class extends TableGrounding {
|
|
|
1064
1256
|
}
|
|
1065
1257
|
const rels = [];
|
|
1066
1258
|
for (const [constraintName, columns] of byConstraint.entries()) {
|
|
1067
|
-
const
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
childTable: table2,
|
|
1259
|
+
const resolution = await resolveForeignKey(
|
|
1260
|
+
this.#adapter,
|
|
1261
|
+
dataset,
|
|
1071
1262
|
constraintName,
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1263
|
+
columns,
|
|
1264
|
+
this.#cache
|
|
1265
|
+
);
|
|
1266
|
+
if (resolution) {
|
|
1267
|
+
rels.push({
|
|
1268
|
+
table: `${dataset}.${table2}`,
|
|
1269
|
+
from: resolution.childColumns,
|
|
1270
|
+
referenced_table: `${resolution.referencedDataset}.${resolution.referencedTable}`,
|
|
1271
|
+
to: resolution.referencedColumns
|
|
1272
|
+
});
|
|
1273
|
+
}
|
|
1075
1274
|
}
|
|
1076
1275
|
return rels;
|
|
1077
1276
|
}
|
|
@@ -1087,22 +1286,18 @@ var BigQueryTableGrounding = class extends TableGrounding {
|
|
|
1087
1286
|
`);
|
|
1088
1287
|
for (const row of rows) {
|
|
1089
1288
|
if (!row.constraint_name) continue;
|
|
1090
|
-
const rel = await this.#
|
|
1289
|
+
const rel = await this.#resolveIncomingRelationship(
|
|
1091
1290
|
constraintDataset,
|
|
1092
|
-
row.constraint_name
|
|
1291
|
+
row.constraint_name,
|
|
1292
|
+
referencedDataset,
|
|
1293
|
+
referencedTable
|
|
1093
1294
|
);
|
|
1094
|
-
if (rel
|
|
1095
|
-
rels.push(rel);
|
|
1096
|
-
}
|
|
1295
|
+
if (rel) rels.push(rel);
|
|
1097
1296
|
}
|
|
1098
1297
|
}
|
|
1099
1298
|
return rels;
|
|
1100
1299
|
}
|
|
1101
|
-
#
|
|
1102
|
-
const { schema } = this.#adapter.parseTableName(tableName);
|
|
1103
|
-
return this.#adapter.isDatasetAllowed(schema);
|
|
1104
|
-
}
|
|
1105
|
-
async #buildForeignKeyRelationshipFromConstraintName(constraintDataset, constraintName) {
|
|
1300
|
+
async #resolveIncomingRelationship(constraintDataset, constraintName, expectedReferencedDataset, expectedReferencedTable) {
|
|
1106
1301
|
const keyRows = await this.#adapter.runQuery(`
|
|
1107
1302
|
SELECT
|
|
1108
1303
|
kcu.constraint_name,
|
|
@@ -1126,67 +1321,27 @@ var BigQueryTableGrounding = class extends TableGrounding {
|
|
|
1126
1321
|
ordinal: r.ordinal_position ?? 0,
|
|
1127
1322
|
pkOrdinal: r.position_in_unique_constraint
|
|
1128
1323
|
}));
|
|
1129
|
-
|
|
1324
|
+
const resolution = await resolveForeignKey(
|
|
1325
|
+
this.#adapter,
|
|
1130
1326
|
constraintDataset,
|
|
1131
|
-
childDataset: constraintDataset,
|
|
1132
|
-
childTable,
|
|
1133
1327
|
constraintName,
|
|
1134
|
-
childColumns
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
const refTableRows = await this.#adapter.runQuery(`
|
|
1139
|
-
SELECT DISTINCT table_schema, table_name
|
|
1140
|
-
FROM ${this.#adapter.infoSchemaView(args.constraintDataset, "CONSTRAINT_COLUMN_USAGE")}
|
|
1141
|
-
WHERE constraint_name = '${this.#adapter.escapeString(args.constraintName)}'
|
|
1142
|
-
`);
|
|
1143
|
-
const referenced = refTableRows.find((r) => r.table_schema && r.table_name);
|
|
1144
|
-
if (!referenced?.table_schema || !referenced.table_name) {
|
|
1145
|
-
return void 0;
|
|
1146
|
-
}
|
|
1147
|
-
const referencedDataset = referenced.table_schema;
|
|
1148
|
-
const referencedTable = referenced.table_name;
|
|
1149
|
-
if (!this.#adapter.isDatasetAllowed(referencedDataset)) {
|
|
1150
|
-
return void 0;
|
|
1151
|
-
}
|
|
1152
|
-
const pkConstraintRows = await this.#adapter.runQuery(`
|
|
1153
|
-
SELECT constraint_name
|
|
1154
|
-
FROM ${this.#adapter.infoSchemaView(referencedDataset, "TABLE_CONSTRAINTS")}
|
|
1155
|
-
WHERE constraint_type = 'PRIMARY KEY'
|
|
1156
|
-
AND table_name = '${this.#adapter.escapeString(referencedTable)}'
|
|
1157
|
-
LIMIT 1
|
|
1158
|
-
`);
|
|
1159
|
-
const pkConstraintName = pkConstraintRows[0]?.constraint_name;
|
|
1160
|
-
if (!pkConstraintName) {
|
|
1328
|
+
childColumns,
|
|
1329
|
+
this.#cache
|
|
1330
|
+
);
|
|
1331
|
+
if (!resolution || resolution.referencedDataset !== expectedReferencedDataset || resolution.referencedTable !== expectedReferencedTable) {
|
|
1161
1332
|
return void 0;
|
|
1162
1333
|
}
|
|
1163
|
-
const pkColumnRows = await this.#adapter.runQuery(`
|
|
1164
|
-
SELECT column_name, ordinal_position
|
|
1165
|
-
FROM ${this.#adapter.infoSchemaView(referencedDataset, "KEY_COLUMN_USAGE")}
|
|
1166
|
-
WHERE constraint_name = '${this.#adapter.escapeString(pkConstraintName)}'
|
|
1167
|
-
AND table_name = '${this.#adapter.escapeString(referencedTable)}'
|
|
1168
|
-
ORDER BY ordinal_position
|
|
1169
|
-
`);
|
|
1170
|
-
const pkByOrdinal = /* @__PURE__ */ new Map();
|
|
1171
|
-
for (const row of pkColumnRows) {
|
|
1172
|
-
if (!row.column_name || row.ordinal_position == null) continue;
|
|
1173
|
-
pkByOrdinal.set(row.ordinal_position, row.column_name);
|
|
1174
|
-
}
|
|
1175
|
-
const orderedChild = [...args.childColumns].sort(
|
|
1176
|
-
(a, b) => a.ordinal - b.ordinal
|
|
1177
|
-
);
|
|
1178
|
-
const from = orderedChild.map((c) => c.column);
|
|
1179
|
-
const to = orderedChild.map((c) => {
|
|
1180
|
-
const pkOrdinal = c.pkOrdinal ?? c.ordinal;
|
|
1181
|
-
return pkByOrdinal.get(pkOrdinal) ?? "unknown";
|
|
1182
|
-
});
|
|
1183
1334
|
return {
|
|
1184
|
-
table: `${
|
|
1185
|
-
from,
|
|
1186
|
-
referenced_table: `${referencedDataset}.${referencedTable}`,
|
|
1187
|
-
to
|
|
1335
|
+
table: `${constraintDataset}.${childTable}`,
|
|
1336
|
+
from: resolution.childColumns,
|
|
1337
|
+
referenced_table: `${resolution.referencedDataset}.${resolution.referencedTable}`,
|
|
1338
|
+
to: resolution.referencedColumns
|
|
1188
1339
|
};
|
|
1189
1340
|
}
|
|
1341
|
+
#isTableInScope(tableName) {
|
|
1342
|
+
const { schema } = this.#adapter.parseTableName(tableName);
|
|
1343
|
+
return this.#adapter.isDatasetAllowed(schema);
|
|
1344
|
+
}
|
|
1190
1345
|
};
|
|
1191
1346
|
|
|
1192
1347
|
// packages/text2sql/src/lib/adapters/groundings/view.grounding.ts
|