@quillsql/node 0.3.6 → 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/pgtypes.js +2785 -0
- package/dist/db/BigQuery.js +189 -0
- package/dist/db/{CachedPools.js → CachedConnection.js} +11 -23
- package/dist/db/DatabaseHelper.js +187 -0
- package/dist/db/Mysql.js +187 -0
- package/dist/db/Postgres.js +156 -0
- package/dist/db/Snowflake.js +179 -0
- package/dist/index.js +47 -18
- package/dist/index.uspec.js +3 -2
- package/dist/models/Client.js +2 -0
- package/dist/utils/RunQueryProcesses.js +4 -4
- package/dist/utils/textProcessing.js +17 -0
- package/examples/node-server/app.ts +6 -8
- package/package.json +5 -1
- package/src/assets/pgtypes.ts +2782 -0
- package/src/db/BigQuery.ts +201 -0
- package/src/db/{CachedPools.ts → CachedConnection.ts} +25 -21
- package/src/db/DatabaseHelper.ts +340 -0
- package/src/db/Mysql.ts +209 -0
- package/src/db/Postgres.ts +178 -0
- package/src/db/Snowflake.ts +191 -0
- package/src/index.ts +69 -18
- package/src/index.uspec.ts +9 -2
- package/src/models/Client.ts +29 -0
- package/src/models/Quill.ts +0 -6
- package/src/utils/RunQueryProcesses.ts +5 -5
- package/src/utils/textProcessing.ts +13 -0
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.getSchemaColumnInfoBigQuery = exports.getForeignKeysBigQuery = exports.formatBigQueryConfig = exports.getColumnsByTableBigQuery = exports.getTablesBySchemaBigQuery = exports.getSchemaBigQuery = exports.runQueryBigQuery = exports.connectToBigQuery = void 0;
|
|
13
|
+
const bigquery_1 = require("@google-cloud/bigquery");
|
|
14
|
+
const textProcessing_1 = require("../utils/textProcessing");
|
|
15
|
+
function connectToBigQuery(config) {
|
|
16
|
+
return new bigquery_1.BigQuery(config);
|
|
17
|
+
}
|
|
18
|
+
exports.connectToBigQuery = connectToBigQuery;
|
|
19
|
+
function runQueryBigQuery(sql, bigQuery) {
|
|
20
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
21
|
+
const rows = yield bigQuery.query(sql);
|
|
22
|
+
if (!rows[0] || rows[0].length === 0)
|
|
23
|
+
return { fields: [], rows: [] };
|
|
24
|
+
const typedRows = rows[0];
|
|
25
|
+
const fields = Object.keys(typedRows[0]).map((name) => ({
|
|
26
|
+
name,
|
|
27
|
+
dataTypeID: 1043,
|
|
28
|
+
}));
|
|
29
|
+
fields.forEach((field) => {
|
|
30
|
+
typedRows.some((row) => {
|
|
31
|
+
if (row[field.name] === null)
|
|
32
|
+
return false;
|
|
33
|
+
field.dataTypeID = inferType(row[field.name]);
|
|
34
|
+
return true;
|
|
35
|
+
});
|
|
36
|
+
});
|
|
37
|
+
return {
|
|
38
|
+
fields: fields,
|
|
39
|
+
rows: typedRows,
|
|
40
|
+
};
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
exports.runQueryBigQuery = runQueryBigQuery;
|
|
44
|
+
function getSchemaBigQuery(bigQuery) {
|
|
45
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
46
|
+
const [datasets] = yield bigQuery.getDatasets();
|
|
47
|
+
const definedDatasets = datasets.map((dataset) => dataset.id);
|
|
48
|
+
const filtered = [];
|
|
49
|
+
definedDatasets.forEach((dataset) => {
|
|
50
|
+
if (dataset !== undefined) {
|
|
51
|
+
filtered.push(dataset);
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
return filtered;
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
exports.getSchemaBigQuery = getSchemaBigQuery;
|
|
58
|
+
function getTablesBySchemaBigQuery(bigQuery, schemaName) {
|
|
59
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
60
|
+
const sql = `SELECT table_name FROM ${schemaName}.INFORMATION_SCHEMA.TABLES WHERE table_type = 'BASE TABLE'`;
|
|
61
|
+
const rows = yield bigQuery.query(sql);
|
|
62
|
+
return rows[0].map((row) => row.table_name);
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
exports.getTablesBySchemaBigQuery = getTablesBySchemaBigQuery;
|
|
66
|
+
function getColumnsByTableBigQuery(bigQuery, schemaName, tableName) {
|
|
67
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
68
|
+
const sql = `SELECT column_name FROM ${schemaName}.INFORMATION_SCHEMA.COLUMNS WHERE table_name = '${tableName}'`;
|
|
69
|
+
const rows = yield bigQuery.query(sql);
|
|
70
|
+
return rows[0].map((row) => row.column_name);
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
exports.getColumnsByTableBigQuery = getColumnsByTableBigQuery;
|
|
74
|
+
function formatBigQueryConfig(connectionString) {
|
|
75
|
+
const jsonStartIndex = connectionString.indexOf("{");
|
|
76
|
+
if (jsonStartIndex === -1) {
|
|
77
|
+
throw new Error("Invalid input string. No JSON data found.");
|
|
78
|
+
}
|
|
79
|
+
const datasetName = connectionString.substring(0, jsonStartIndex).trim();
|
|
80
|
+
const jsonString = connectionString.substring(jsonStartIndex);
|
|
81
|
+
try {
|
|
82
|
+
const serviceAccount = JSON.parse(jsonString);
|
|
83
|
+
// Validate if required fields are present
|
|
84
|
+
if (!serviceAccount.project_id || !serviceAccount.private_key) {
|
|
85
|
+
throw new Error("Invalid service account JSON. Required fields are missing.");
|
|
86
|
+
}
|
|
87
|
+
return {
|
|
88
|
+
datasetName,
|
|
89
|
+
projectId: serviceAccount.project_id,
|
|
90
|
+
credentials: serviceAccount,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
catch (error) {
|
|
94
|
+
throw new Error("Failed to parse JSON string: " + error);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
exports.formatBigQueryConfig = formatBigQueryConfig;
|
|
98
|
+
function getForeignKeysBigQuery(connection, schemaName, tableName, primaryKey) {
|
|
99
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
100
|
+
const depluralizedTableName = (0, textProcessing_1.depluralize)(tableName);
|
|
101
|
+
let sql = `SELECT column_name FROM ${schemaName}.INFORMATION_SCHEMA.COLUMNS
|
|
102
|
+
WHERE table_name != '${tableName}'
|
|
103
|
+
and (column_name = '${primaryKey}'
|
|
104
|
+
or column_name = '${depluralizedTableName}_${primaryKey}'
|
|
105
|
+
or column_name = '${depluralizedTableName}${(0, textProcessing_1.capitalize)(primaryKey)}')`;
|
|
106
|
+
const results = yield runQueryBigQuery(sql, connection);
|
|
107
|
+
let foreignKeysString = results.rows.map((key) => {
|
|
108
|
+
return key.column_name;
|
|
109
|
+
});
|
|
110
|
+
foreignKeysString = foreignKeysString.filter((key) => key !== "id" && key !== "_id_");
|
|
111
|
+
foreignKeysString = [...new Set(foreignKeysString)];
|
|
112
|
+
if (foreignKeysString.length === 0) {
|
|
113
|
+
sql = `SELECT column_name FROM ${schemaName}.INFORMATION_SCHEMA.COLUMNS
|
|
114
|
+
WHERE table_name != '${tableName}'
|
|
115
|
+
and (column_name like '${depluralizedTableName}%'
|
|
116
|
+
or column_name like '%_id'
|
|
117
|
+
or column_name like '%Id'
|
|
118
|
+
or column_name like '%_${primaryKey}'
|
|
119
|
+
or column_name like '%${(0, textProcessing_1.capitalize)(primaryKey)}')`;
|
|
120
|
+
const results = yield runQueryBigQuery(sql, connection);
|
|
121
|
+
foreignKeysString = results.rows.map((key) => {
|
|
122
|
+
return key.column_name;
|
|
123
|
+
});
|
|
124
|
+
foreignKeysString = [...new Set(foreignKeysString)];
|
|
125
|
+
}
|
|
126
|
+
return foreignKeysString;
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
exports.getForeignKeysBigQuery = getForeignKeysBigQuery;
|
|
130
|
+
function getSchemaColumnInfoBigQuery(connection, schemaName, tableNames) {
|
|
131
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
132
|
+
const allColumns = yield Promise.all(tableNames.map((tableName) => __awaiter(this, void 0, void 0, function* () {
|
|
133
|
+
const query = `
|
|
134
|
+
SELECT column_name as columnName, data_type as dataType
|
|
135
|
+
FROM ${schemaName}.INFORMATION_SCHEMA.COLUMNS
|
|
136
|
+
WHERE table_name = '${tableName}'
|
|
137
|
+
ORDER BY ordinal_position;
|
|
138
|
+
`;
|
|
139
|
+
const results = yield runQueryBigQuery(query, connection);
|
|
140
|
+
return {
|
|
141
|
+
tableName,
|
|
142
|
+
displayName: tableName,
|
|
143
|
+
columns: results.rows.map((row) => ({
|
|
144
|
+
columnName: row.columnName,
|
|
145
|
+
displayName: row.columnName,
|
|
146
|
+
dataTypeId: inferType(convertBigQueryTypeToPostgresOID(row.dataType)),
|
|
147
|
+
fieldType: row.dataType,
|
|
148
|
+
})),
|
|
149
|
+
};
|
|
150
|
+
})));
|
|
151
|
+
return allColumns;
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
exports.getSchemaColumnInfoBigQuery = getSchemaColumnInfoBigQuery;
|
|
155
|
+
function convertBigQueryTypeToPostgresOID(type) {
|
|
156
|
+
const typeToOidMap = {
|
|
157
|
+
VARCHAR: 1043,
|
|
158
|
+
INTEGER: 23,
|
|
159
|
+
FLOAT: 700,
|
|
160
|
+
TIMESTAMP: 1114,
|
|
161
|
+
DATE: 1082,
|
|
162
|
+
};
|
|
163
|
+
const postgresType = typeToOidMap[type.toUpperCase()] || "VARCHAR"; // Default to 'text' if the type is not recognized
|
|
164
|
+
return typeToOidMap[postgresType] || 1043; // Default to OID for 'text' if the type is not recognized
|
|
165
|
+
}
|
|
166
|
+
function inferType(elem) {
|
|
167
|
+
if (typeof elem === "number") {
|
|
168
|
+
// Check if the number is a float or an integer
|
|
169
|
+
return Number.isInteger(elem) ? 23 : 700; // 23: integer, 700: real
|
|
170
|
+
}
|
|
171
|
+
if (typeof elem === "string") {
|
|
172
|
+
// Attempt to infer date, time, and timestamp formats
|
|
173
|
+
// Date in YYYY-MM-DD format
|
|
174
|
+
if (/^\d{4}-\d{2}-\d{2}$/.test(elem))
|
|
175
|
+
return 1082; // date
|
|
176
|
+
// Timestamp in YYYY-MM-DDTHH:MM:SS[.fraction] format
|
|
177
|
+
if (/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?$/.test(elem))
|
|
178
|
+
return 1114; // timestamp without timezone
|
|
179
|
+
// Timestamp in YYYY-MM-DDTHH:MM:SS[.fraction]Z format
|
|
180
|
+
if (/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z$/.test(elem))
|
|
181
|
+
return 1184; // timestamp with timezone
|
|
182
|
+
// Time in HH:MM:SS format
|
|
183
|
+
if (/^\d{2}:\d{2}:\d{2}$/.test(elem))
|
|
184
|
+
return 1083; // time
|
|
185
|
+
return 1043; // varchar
|
|
186
|
+
}
|
|
187
|
+
// Add more specific cases or different data types as needed
|
|
188
|
+
return 1043; // default or unknown type
|
|
189
|
+
}
|
|
@@ -9,10 +9,10 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
9
9
|
});
|
|
10
10
|
};
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.
|
|
13
|
-
const pg_1 = require("pg");
|
|
12
|
+
exports.CachedConnection = void 0;
|
|
14
13
|
const redis_1 = require("redis");
|
|
15
14
|
const Error_1 = require("../utils/Error");
|
|
15
|
+
const DatabaseHelper_1 = require("./DatabaseHelper");
|
|
16
16
|
class PgError extends Error {
|
|
17
17
|
// Add other properties if needed
|
|
18
18
|
constructor(detail, hint, position) {
|
|
@@ -24,10 +24,11 @@ class PgError extends Error {
|
|
|
24
24
|
}
|
|
25
25
|
/** The TTL for new cache entries (default: 1h) */
|
|
26
26
|
const DEFAULT_CACHE_TTL = 24 * 60 * 60;
|
|
27
|
-
class
|
|
28
|
-
constructor(config, cacheConfig = {}) {
|
|
27
|
+
class CachedConnection {
|
|
28
|
+
constructor(databaseType, config, cacheConfig = {}) {
|
|
29
29
|
var _a;
|
|
30
|
-
this.
|
|
30
|
+
this.databaseType = databaseType;
|
|
31
|
+
this.pool = (0, DatabaseHelper_1.connectToDatabase)(databaseType, config);
|
|
31
32
|
this.ttl = (_a = cacheConfig === null || cacheConfig === void 0 ? void 0 : cacheConfig.ttl) !== null && _a !== void 0 ? _a : DEFAULT_CACHE_TTL;
|
|
32
33
|
this.cache = this.getCache(cacheConfig);
|
|
33
34
|
}
|
|
@@ -35,14 +36,7 @@ class CachedPool {
|
|
|
35
36
|
return __awaiter(this, void 0, void 0, function* () {
|
|
36
37
|
try {
|
|
37
38
|
if (!this.cache) {
|
|
38
|
-
|
|
39
|
-
return {
|
|
40
|
-
fields: results.fields.map((field) => ({
|
|
41
|
-
name: field.name,
|
|
42
|
-
dataTypeID: field.dataTypeID,
|
|
43
|
-
})),
|
|
44
|
-
rows: results.rows,
|
|
45
|
-
};
|
|
39
|
+
return yield (0, DatabaseHelper_1.runQueryByDatabase)(this.databaseType, this.pool, text);
|
|
46
40
|
}
|
|
47
41
|
const key = `${this.orgId}:${text}`;
|
|
48
42
|
const cachedResult = yield this.cache.get(key);
|
|
@@ -50,16 +44,10 @@ class CachedPool {
|
|
|
50
44
|
return JSON.parse(cachedResult);
|
|
51
45
|
}
|
|
52
46
|
else {
|
|
53
|
-
const newResult = yield this.pool
|
|
47
|
+
const newResult = yield (0, DatabaseHelper_1.runQueryByDatabase)(this.databaseType, this.pool, text);
|
|
54
48
|
const newResultString = JSON.stringify(newResult);
|
|
55
49
|
yield this.cache.set(key, newResultString, "EX", DEFAULT_CACHE_TTL);
|
|
56
|
-
return
|
|
57
|
-
fields: newResult.fields.map((field) => ({
|
|
58
|
-
name: field.name,
|
|
59
|
-
dataTypeID: field.dataTypeID,
|
|
60
|
-
})),
|
|
61
|
-
rows: newResult.rows,
|
|
62
|
-
};
|
|
50
|
+
return newResult;
|
|
63
51
|
}
|
|
64
52
|
}
|
|
65
53
|
catch (err) {
|
|
@@ -84,8 +72,8 @@ class CachedPool {
|
|
|
84
72
|
}
|
|
85
73
|
close() {
|
|
86
74
|
return __awaiter(this, void 0, void 0, function* () {
|
|
87
|
-
yield this.pool
|
|
75
|
+
yield (0, DatabaseHelper_1.disconnectFromDatabase)(this.databaseType, this.pool);
|
|
88
76
|
});
|
|
89
77
|
}
|
|
90
78
|
}
|
|
91
|
-
exports.
|
|
79
|
+
exports.CachedConnection = CachedConnection;
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.getColumnInfoBySchemaByDatabase = exports.getForiegnKeysByDatabase = exports.getColumnsByTableByDatabase = exports.getTablesBySchemaByDatabase = exports.getSchemasByDatabase = exports.disconnectFromDatabase = exports.runQueryByDatabase = exports.connectToDatabase = exports.getDatabaseCredentials = exports.DatabaseType = void 0;
|
|
13
|
+
const Postgres_1 = require("./Postgres");
|
|
14
|
+
const Snowflake_1 = require("./Snowflake");
|
|
15
|
+
const BigQuery_1 = require("./BigQuery");
|
|
16
|
+
const Mysql_1 = require("./Mysql");
|
|
17
|
+
var DatabaseType;
|
|
18
|
+
(function (DatabaseType) {
|
|
19
|
+
DatabaseType["postgres"] = "postgres";
|
|
20
|
+
DatabaseType["postgresql"] = "PostgreSQL";
|
|
21
|
+
DatabaseType["snowflake"] = "Snowflake";
|
|
22
|
+
DatabaseType["bigquery"] = "BigQuery";
|
|
23
|
+
DatabaseType["mysql"] = "MySQL";
|
|
24
|
+
})(DatabaseType || (exports.DatabaseType = DatabaseType = {}));
|
|
25
|
+
function getDatabaseCredentials(databaseType, connectionString) {
|
|
26
|
+
switch (databaseType) {
|
|
27
|
+
case "postgres":
|
|
28
|
+
return (0, Postgres_1.formatPostgresConfig)(connectionString);
|
|
29
|
+
case "PostgreSQL":
|
|
30
|
+
return (0, Postgres_1.formatPostgresConfig)(connectionString);
|
|
31
|
+
case "Snowflake":
|
|
32
|
+
return (0, Snowflake_1.formatSnowflakeConfig)(connectionString);
|
|
33
|
+
case "BigQuery":
|
|
34
|
+
return (0, BigQuery_1.formatBigQueryConfig)(connectionString);
|
|
35
|
+
case "MySQL":
|
|
36
|
+
return (0, Mysql_1.formatMysqlConfig)(connectionString);
|
|
37
|
+
default:
|
|
38
|
+
return undefined;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
exports.getDatabaseCredentials = getDatabaseCredentials;
|
|
42
|
+
function connectToDatabase(databaseType, config) {
|
|
43
|
+
switch (databaseType) {
|
|
44
|
+
case "postgres":
|
|
45
|
+
return (0, Postgres_1.connectToPostgres)(config);
|
|
46
|
+
case "PostgreSQL":
|
|
47
|
+
return (0, Postgres_1.connectToPostgres)(config);
|
|
48
|
+
case "Snowflake":
|
|
49
|
+
return (0, Snowflake_1.connectToSnowflake)(config);
|
|
50
|
+
case "BigQuery":
|
|
51
|
+
return (0, BigQuery_1.connectToBigQuery)(config);
|
|
52
|
+
case "MySQL":
|
|
53
|
+
return (0, Mysql_1.connectToMysql)(config);
|
|
54
|
+
default:
|
|
55
|
+
return (0, Postgres_1.connectToPostgres)(config);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
exports.connectToDatabase = connectToDatabase;
|
|
59
|
+
function runQueryByDatabase(databaseType, connection, sql) {
|
|
60
|
+
switch (databaseType) {
|
|
61
|
+
case "postgres":
|
|
62
|
+
return (0, Postgres_1.runQueryPostgres)(sql, connection);
|
|
63
|
+
case "PostgreSQL":
|
|
64
|
+
return (0, Postgres_1.runQueryPostgres)(sql, connection);
|
|
65
|
+
case "Snowflake":
|
|
66
|
+
return (0, Snowflake_1.runQuerySnowflake)(sql, connection);
|
|
67
|
+
case "BigQuery":
|
|
68
|
+
return (0, BigQuery_1.runQueryBigQuery)(sql, connection);
|
|
69
|
+
case "MySQL":
|
|
70
|
+
return (0, Mysql_1.runQueryMysql)(sql, connection);
|
|
71
|
+
default:
|
|
72
|
+
return undefined;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
exports.runQueryByDatabase = runQueryByDatabase;
|
|
76
|
+
function disconnectFromDatabase(databaseType, database) {
|
|
77
|
+
switch (databaseType) {
|
|
78
|
+
case "postgres":
|
|
79
|
+
return (0, Postgres_1.disconnectFromPostgres)(database);
|
|
80
|
+
case "PostgreSQL":
|
|
81
|
+
return (0, Postgres_1.disconnectFromPostgres)(database);
|
|
82
|
+
case "Snowflake":
|
|
83
|
+
return (0, Snowflake_1.disconnectFromSnowflake)(database);
|
|
84
|
+
case "BigQuery":
|
|
85
|
+
return; // BigQuery does not need to be disconnected
|
|
86
|
+
case "MySQL":
|
|
87
|
+
return (0, Mysql_1.disconnectFromMysql)(database);
|
|
88
|
+
default:
|
|
89
|
+
return undefined;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
exports.disconnectFromDatabase = disconnectFromDatabase;
|
|
93
|
+
function getSchemasByDatabase(databaseType, connection) {
|
|
94
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
95
|
+
switch (databaseType) {
|
|
96
|
+
case "postgres":
|
|
97
|
+
return (0, Postgres_1.getSchemasPostgres)(connection);
|
|
98
|
+
case "PostgreSQL":
|
|
99
|
+
return (0, Postgres_1.getSchemasPostgres)(connection);
|
|
100
|
+
case "Snowflake":
|
|
101
|
+
return (0, Snowflake_1.getSchemasSnowflake)(connection);
|
|
102
|
+
case "BigQuery":
|
|
103
|
+
return (0, BigQuery_1.getSchemaBigQuery)(connection);
|
|
104
|
+
case "MySQL":
|
|
105
|
+
return (0, Mysql_1.getSchemasMysql)(connection);
|
|
106
|
+
default:
|
|
107
|
+
return undefined;
|
|
108
|
+
}
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
exports.getSchemasByDatabase = getSchemasByDatabase;
|
|
112
|
+
// INFORMATION SCHEMA SELECTS
|
|
113
|
+
function getTablesBySchemaByDatabase(databaseType, connection, schemaName) {
|
|
114
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
115
|
+
switch (databaseType) {
|
|
116
|
+
case "postgres":
|
|
117
|
+
return (0, Postgres_1.getTablesBySchemaPostgres)(connection, schemaName);
|
|
118
|
+
case "PostgreSQL":
|
|
119
|
+
return (0, Postgres_1.getTablesBySchemaPostgres)(connection, schemaName);
|
|
120
|
+
case "Snowflake":
|
|
121
|
+
return (0, Snowflake_1.getTablesBySchemaSnowflake)(connection, schemaName);
|
|
122
|
+
case "BigQuery":
|
|
123
|
+
return (0, BigQuery_1.getTablesBySchemaBigQuery)(connection, schemaName);
|
|
124
|
+
case "MySQL":
|
|
125
|
+
return (0, Mysql_1.getTablesBySchemaMysql)(connection, schemaName);
|
|
126
|
+
default:
|
|
127
|
+
return undefined;
|
|
128
|
+
}
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
exports.getTablesBySchemaByDatabase = getTablesBySchemaByDatabase;
|
|
132
|
+
// INFORMATION SCHEMA SELECTS
|
|
133
|
+
function getColumnsByTableByDatabase(databaseType, connection, schemaName, tableName) {
|
|
134
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
135
|
+
switch (databaseType) {
|
|
136
|
+
case "postgres":
|
|
137
|
+
return (0, Postgres_1.getColumnsByTablePostgres)(connection, schemaName, tableName);
|
|
138
|
+
case "PostgreSQL":
|
|
139
|
+
return (0, Postgres_1.getColumnsByTablePostgres)(connection, schemaName, tableName);
|
|
140
|
+
case "Snowflake":
|
|
141
|
+
return (0, Snowflake_1.getColumnsByTableSnowflake)(connection, schemaName, tableName);
|
|
142
|
+
case "BigQuery":
|
|
143
|
+
return (0, BigQuery_1.getColumnsByTableBigQuery)(connection, schemaName, tableName);
|
|
144
|
+
case "MySQL":
|
|
145
|
+
return (0, Mysql_1.getColumnsByTableMysql)(connection, schemaName, tableName);
|
|
146
|
+
default:
|
|
147
|
+
return undefined;
|
|
148
|
+
}
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
exports.getColumnsByTableByDatabase = getColumnsByTableByDatabase;
|
|
152
|
+
function getForiegnKeysByDatabase(databaseType, connection, schemaName, tableName, primaryKey) {
|
|
153
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
154
|
+
switch (databaseType) {
|
|
155
|
+
case "postgres":
|
|
156
|
+
return (0, Postgres_1.getForeignKeysPostgres)(connection, schemaName, tableName, primaryKey);
|
|
157
|
+
case "PostgreSQL":
|
|
158
|
+
return (0, Postgres_1.getForeignKeysPostgres)(connection, schemaName, tableName, primaryKey);
|
|
159
|
+
case "Snowflake":
|
|
160
|
+
return (0, Snowflake_1.getForeignKeysSnowflake)(connection, schemaName, tableName, primaryKey);
|
|
161
|
+
case "BigQuery":
|
|
162
|
+
return (0, BigQuery_1.getForeignKeysBigQuery)(connection, schemaName, tableName, primaryKey);
|
|
163
|
+
case "MySQL":
|
|
164
|
+
return (0, Mysql_1.getForeignKeysMysql)(connection, schemaName, tableName, primaryKey);
|
|
165
|
+
default:
|
|
166
|
+
return undefined;
|
|
167
|
+
}
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
exports.getForiegnKeysByDatabase = getForiegnKeysByDatabase;
|
|
171
|
+
function getColumnInfoBySchemaByDatabase(databaseType, connection, schemaName, tables) {
|
|
172
|
+
switch (databaseType) {
|
|
173
|
+
case "postgres":
|
|
174
|
+
return (0, Postgres_1.getSchemaColumnInfoPostgress)(connection, schemaName, tables);
|
|
175
|
+
case "PostgreSQL":
|
|
176
|
+
return (0, Postgres_1.getSchemaColumnInfoPostgress)(connection, schemaName, tables);
|
|
177
|
+
case "Snowflake":
|
|
178
|
+
return (0, Snowflake_1.getSchemaColumnInfoSnowflake)(connection, schemaName, tables);
|
|
179
|
+
case "BigQuery":
|
|
180
|
+
return (0, BigQuery_1.getSchemaColumnInfoBigQuery)(connection, schemaName, tables);
|
|
181
|
+
case "MySQL":
|
|
182
|
+
return (0, Mysql_1.getSchemaColumnInfoMysql)(connection, schemaName, tables);
|
|
183
|
+
default:
|
|
184
|
+
return undefined;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
exports.getColumnInfoBySchemaByDatabase = getColumnInfoBySchemaByDatabase;
|
package/dist/db/Mysql.js
ADDED
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
+
};
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.getSchemaColumnInfoMysql = exports.getForeignKeysMysql = exports.getColumnsByTableMysql = exports.getTablesBySchemaMysql = exports.getSchemasMysql = exports.runQueryMysql = exports.disconnectFromMysql = exports.connectToMysql = exports.formatMysqlConfig = void 0;
|
|
16
|
+
const mysql2_1 = __importDefault(require("mysql2"));
|
|
17
|
+
const url_1 = __importDefault(require("url"));
|
|
18
|
+
const textProcessing_1 = require("../utils/textProcessing");
|
|
19
|
+
function formatMysqlConfig(connectionString) {
|
|
20
|
+
const parsedUrl = url_1.default.parse(connectionString);
|
|
21
|
+
const [user, password] = (parsedUrl.auth || "").split(":");
|
|
22
|
+
return {
|
|
23
|
+
host: parsedUrl.hostname || "",
|
|
24
|
+
user: user || "",
|
|
25
|
+
password: password || "",
|
|
26
|
+
database: (parsedUrl.pathname || "").slice(1),
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
exports.formatMysqlConfig = formatMysqlConfig;
|
|
30
|
+
function connectToMysql(config) {
|
|
31
|
+
const pool = mysql2_1.default.createPool(Object.assign(Object.assign({}, config), { waitForConnections: true, connectionLimit: 10, queueLimit: 0 }));
|
|
32
|
+
return pool;
|
|
33
|
+
}
|
|
34
|
+
exports.connectToMysql = connectToMysql;
|
|
35
|
+
function disconnectFromMysql(connection) {
|
|
36
|
+
connection.end();
|
|
37
|
+
}
|
|
38
|
+
exports.disconnectFromMysql = disconnectFromMysql;
|
|
39
|
+
function runQueryMysql(sql, connection) {
|
|
40
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
41
|
+
const result = yield new Promise((resolve, reject) => {
|
|
42
|
+
connection.query(sql, (error, results, fields) => {
|
|
43
|
+
if (error) {
|
|
44
|
+
reject(error);
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
const mappedFields = fields
|
|
48
|
+
? fields.map((field) => ({
|
|
49
|
+
name: field.name,
|
|
50
|
+
dataTypeID: mysqlDataTypeIdToPostgresType(field.type || 1043), // Provide a default value for dataTypeID
|
|
51
|
+
}))
|
|
52
|
+
: [];
|
|
53
|
+
const processRows = Array.isArray(results)
|
|
54
|
+
? results.map((row) => {
|
|
55
|
+
return JSON.parse(JSON.stringify(row));
|
|
56
|
+
})
|
|
57
|
+
: [];
|
|
58
|
+
resolve({ fields: mappedFields, rows: processRows });
|
|
59
|
+
});
|
|
60
|
+
});
|
|
61
|
+
return result;
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
exports.runQueryMysql = runQueryMysql;
|
|
65
|
+
function getSchemasMysql(connection) {
|
|
66
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
67
|
+
const sql = `SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA
|
|
68
|
+
WHERE schema_name != 'information_schema'
|
|
69
|
+
AND schema_name != 'performance_schema'
|
|
70
|
+
and schema_name != 'sys';`;
|
|
71
|
+
const results = yield runQueryMysql(sql, connection);
|
|
72
|
+
return results.rows.map((row) => row.SCHEMA_NAME);
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
exports.getSchemasMysql = getSchemasMysql;
|
|
76
|
+
function getTablesBySchemaMysql(connection, schemaName) {
|
|
77
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
78
|
+
const sql = `SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '${schemaName}'`;
|
|
79
|
+
const results = yield runQueryMysql(sql, connection);
|
|
80
|
+
return results.rows.map((row) => row.TABLE_NAME);
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
exports.getTablesBySchemaMysql = getTablesBySchemaMysql;
|
|
84
|
+
function getColumnsByTableMysql(connection, schemaName, tableName) {
|
|
85
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
86
|
+
const sql = `SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '${schemaName}' AND TABLE_NAME = '${tableName}'`;
|
|
87
|
+
const results = yield runQueryMysql(sql, connection);
|
|
88
|
+
return results.rows.map((row) => row.COLUMN_NAME);
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
exports.getColumnsByTableMysql = getColumnsByTableMysql;
|
|
92
|
+
function getForeignKeysMysql(connection, schemaName, tableName, primaryKey) {
|
|
93
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
94
|
+
const depluralizedTableName = (0, textProcessing_1.depluralize)(tableName);
|
|
95
|
+
let sql = `SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS
|
|
96
|
+
WHERE TABLE_SCHEMA = '${schemaName}'
|
|
97
|
+
and table_name != '${tableName}'
|
|
98
|
+
and (column_name = '${primaryKey}'
|
|
99
|
+
or column_name = '${depluralizedTableName}\\_${primaryKey}'
|
|
100
|
+
or column_name = '${depluralizedTableName}${(0, textProcessing_1.capitalize)(primaryKey)}' )`;
|
|
101
|
+
const results = yield runQueryMysql(sql, connection);
|
|
102
|
+
let foreignKeysString = results.rows.map((key) => {
|
|
103
|
+
return key.COLUMN_NAME;
|
|
104
|
+
});
|
|
105
|
+
// remove any foreignKeyStrings that are just 'id'
|
|
106
|
+
foreignKeysString = foreignKeysString.filter((key) => key !== "id" && key !== "_id_");
|
|
107
|
+
foreignKeysString = [...new Set(foreignKeysString)];
|
|
108
|
+
if (foreignKeysString.length === 0) {
|
|
109
|
+
sql = `SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS
|
|
110
|
+
WHERE TABLE_SCHEMA = '${schemaName}'
|
|
111
|
+
and table_name != '${tableName}'
|
|
112
|
+
and (column_name like '${depluralizedTableName}%'
|
|
113
|
+
or column_name like '%\\_id'
|
|
114
|
+
or column_name like '%Id'
|
|
115
|
+
or column_name like '%\\_${primaryKey}'
|
|
116
|
+
or column_name like '%${(0, textProcessing_1.capitalize)(primaryKey)}')`;
|
|
117
|
+
const results = yield runQueryMysql(sql, connection);
|
|
118
|
+
foreignKeysString = results.rows.map((key) => {
|
|
119
|
+
return key.COLUMN_NAME;
|
|
120
|
+
});
|
|
121
|
+
foreignKeysString = [...new Set(foreignKeysString)];
|
|
122
|
+
}
|
|
123
|
+
return foreignKeysString;
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
exports.getForeignKeysMysql = getForeignKeysMysql;
|
|
127
|
+
function getSchemaColumnInfoMysql(connection, schemaName, tableNames) {
|
|
128
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
129
|
+
const allColumns = yield Promise.all(tableNames.map((tableName) => __awaiter(this, void 0, void 0, function* () {
|
|
130
|
+
const query = `
|
|
131
|
+
SELECT COLUMN_NAME as columnName,
|
|
132
|
+
DATA_TYPE as dataType FROM INFORMATION_SCHEMA.COLUMNS
|
|
133
|
+
WHERE TABLE_SCHEMA = '${schemaName}'
|
|
134
|
+
AND TABLE_NAME = '${tableName}'
|
|
135
|
+
`;
|
|
136
|
+
const results = yield runQueryMysql(query, connection);
|
|
137
|
+
return {
|
|
138
|
+
tableName,
|
|
139
|
+
displayName: tableName,
|
|
140
|
+
columns: results.rows.map((row) => ({
|
|
141
|
+
columnName: row.columnName,
|
|
142
|
+
displayName: row.columnName,
|
|
143
|
+
dataTypeId: mysqlTextDataTypeToPostgresOID(row.dataType),
|
|
144
|
+
fieldType: row.dataType,
|
|
145
|
+
})),
|
|
146
|
+
};
|
|
147
|
+
})));
|
|
148
|
+
return allColumns;
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
exports.getSchemaColumnInfoMysql = getSchemaColumnInfoMysql;
|
|
152
|
+
function mysqlTextDataTypeToPostgresOID(type) {
|
|
153
|
+
switch (type) {
|
|
154
|
+
case "bigint": // int
|
|
155
|
+
return 23;
|
|
156
|
+
case "tinyint": // int
|
|
157
|
+
return 23;
|
|
158
|
+
case "float": // float
|
|
159
|
+
return 700;
|
|
160
|
+
case "varchar": // varchar
|
|
161
|
+
return 1043;
|
|
162
|
+
case "timestamp": // date
|
|
163
|
+
return 1082;
|
|
164
|
+
default: // varchar
|
|
165
|
+
return 1043;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
function mysqlDataTypeIdToPostgresType(type) {
|
|
169
|
+
switch (type) {
|
|
170
|
+
case 8: // int
|
|
171
|
+
return 23;
|
|
172
|
+
case 3: // int
|
|
173
|
+
return 23;
|
|
174
|
+
case 2: // int
|
|
175
|
+
return 23;
|
|
176
|
+
case 5: // float
|
|
177
|
+
return 700;
|
|
178
|
+
case 253: // varchar
|
|
179
|
+
return 1043;
|
|
180
|
+
case 7: // date
|
|
181
|
+
return 1082;
|
|
182
|
+
case 7: // date
|
|
183
|
+
return 1082;
|
|
184
|
+
default: // varchar
|
|
185
|
+
return 1043;
|
|
186
|
+
}
|
|
187
|
+
}
|