@quillsql/node 0.3.8 → 0.4.1

package/dist/utils/RunQueryProcesses.js

@@ -9,22 +9,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.mapQueries = exports.removeFields = exports.getTableSchema = void 0;
-function getTableSchema(queryResults, targetConnection) {
-    return __awaiter(this, void 0, void 0, function* () {
-        const typesQuery = yield targetConnection.query("select typname, oid, typarray from pg_type order by oid;");
-        const schema = queryResults[0].fields.map((field) => {
-            return {
-                fieldType: typesQuery.rows.filter((type) => field.dataTypeID === type.oid)[0].typname,
-                name: field.name,
-                displayName: field.name,
-                isVisible: true,
-            };
-        });
-        return schema;
-    });
-}
-exports.getTableSchema = getTableSchema;
+exports.mapQueries = exports.removeFields = void 0;
 function removeFields(queryResults, fieldsToRemove) {
     const fields = queryResults.fields.filter((field) => fieldsToRemove.includes(field.name));
     const rows = queryResults.map((row) => {
@@ -35,7 +20,7 @@ function removeFields(queryResults, fieldsToRemove) {
     });
 }
 exports.removeFields = removeFields;
-function mapQueries(queries, arrayToMap, targetConnection) {
+function mapQueries(queries, targetConnection) {
     return __awaiter(this, void 0, void 0, function* () {
         const mappedArray = [];
         for (let i = 0; i < queries.length; i++) {

package/dist/utils/schemaConversion.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.convertTypeToPostgres = void 0;
+const pgtypes_1 = require("../assets/pgtypes");
+function convertTypeToPostgres(data_type_id) {
+    var _a;
+    const type = pgtypes_1.PG_TYPES.find((type) => data_type_id === type.oid)
+        ? (_a = pgtypes_1.PG_TYPES.find((type) => data_type_id === type.oid)) === null || _a === void 0 ? void 0 : _a.typname
+        : undefined;
+    if (!type) {
+        return "varchar";
+    }
+    return type;
+}
+exports.convertTypeToPostgres = convertTypeToPostgres;

package/examples/node-server/app.ts

@@ -4,11 +4,45 @@ import express from "express";
 import cors from "cors";
 
 const app = express();
-const port = 3001;
+const port = 3000;
+
+// postgresqlConfigExample {
+//   connectionString: process.env.DB_URL,
+//   ssl: {
+//     rejectUnauthorized: false,
+//   },
+// };
+
+// bigqueryConfigExample {
+//   projectId: process.env.PROJECT_ID,
+//   credentials: serviceAccount // You need to make the serviceAccount JSON
+// }
+
+// snowflakeConfigExample {
+//   account: process.env.ACCOUNT,
+//   username: process.env.USERNAME,
+//   password: process.env.PASSWORD,
+//   warehouse: process.env.WAREHOUSE,
+//   database: process.env.DATABASE,
+// }
+
+// mysqlConfigExample {
+//   host: process.env.HOST,
+//   user: process.env.USER,
+//   password: process.env.PASSWORD,
+//   database: process.env.DATABASE,
+// }
 
 const quill = require("../../src/index")({
-  privateKey: process.env.QUILL_PRIVATE_KEY,
+  privateKey: process.env.PRIVATE_KEY,
   databaseConnectionString: process.env.DB_URL,
+  // databaseConfig: {
+  //   connectionString: process.env.DB_URL,
+  //   ssl: {
+  //     rejectUnauthorized: false,
+  //   },
+  // },
+  databaseType: process.env.DB_TYPE,
 });
 
 app.use(cors());
@@ -19,7 +53,6 @@ app.get("/", (req, res) => {
 });
 
 app.post("/quill", async (req, res) => {
-  const organizationId = req.body.orgId;
   const { metadata } = req.body;
   const result = await quill.query({
     orgId: metadata.orgId,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@quillsql/node",
-  "version": "0.3.8",
+  "version": "0.4.1",
   "description": "Quill's client SDK for node backends.",
   "main": "dist/index.js",
   "module": "./dist/index.js",
@@ -36,4 +36,4 @@
     "snowflake-sdk": "^1.9.3",
     "ts-jest": "^29.1.1"
   }
-}
+}

package/src/db/BigQuery.ts

@@ -9,6 +9,35 @@ export interface BigQueryConfig {
   credentials: any;
 }
 
+export function formatBigQueryConfig(connectionString: string): BigQueryConfig {
+  const jsonStartIndex = connectionString.indexOf("{");
+  if (jsonStartIndex === -1) {
+    throw new Error("Invalid input string. No JSON data found.");
+  }
+
+  const datasetName = connectionString.substring(0, jsonStartIndex).trim();
+  const jsonString = connectionString.substring(jsonStartIndex);
+
+  try {
+    const serviceAccount = JSON.parse(jsonString);
+
+    // Validate if required fields are present
+    if (!serviceAccount.project_id || !serviceAccount.private_key) {
+      throw new Error(
+        "Invalid service account JSON. Required fields are missing."
+      );
+    }
+
+    return {
+      datasetName,
+      projectId: serviceAccount.project_id,
+      credentials: serviceAccount,
+    };
+  } catch (error) {
+    throw new Error("Failed to parse JSON string: " + error);
+  }
+}
+
 export function connectToBigQuery(config: BigQueryConfig) {
   return new BigQuery(config);
 }
@@ -51,11 +80,18 @@ export async function getSchemaBigQuery(bigQuery: BigQuery): Promise<string[]> {
 
 export async function getTablesBySchemaBigQuery(
   bigQuery: BigQuery,
-  schemaName: string
-): Promise<string[]> {
-  const sql = `SELECT table_name FROM ${schemaName}.INFORMATION_SCHEMA.TABLES WHERE table_type = 'BASE TABLE'`;
-  const rows = await bigQuery.query(sql);
-  return rows[0].map((row: any) => row.table_name);
+  schemaNames: string[]
+): Promise<{ tableName: string; schemaName: string }[]> {
+  const allColumns = await Promise.all(
+    schemaNames.map(async (schema) => {
+      const sql = `SELECT table_name FROM ${schema}.INFORMATION_SCHEMA.TABLES WHERE table_type = 'BASE TABLE'`;
+      const rows = await bigQuery.query(sql);
+      return rows[0].map((row) => {
+        return { tableName: row.table_name, schemaName: schema };
+      });
+    })
+  );
+  return allColumns.flat();
 }
 
 export async function getColumnsByTableBigQuery(
@@ -68,35 +104,6 @@ export async function getColumnsByTableBigQuery(
   return rows[0].map((row: any) => row.column_name);
 }
 
-export function formatBigQueryConfig(connectionString: string): BigQueryConfig {
-  const jsonStartIndex = connectionString.indexOf("{");
-  if (jsonStartIndex === -1) {
-    throw new Error("Invalid input string. No JSON data found.");
-  }
-
-  const datasetName = connectionString.substring(0, jsonStartIndex).trim();
-  const jsonString = connectionString.substring(jsonStartIndex);
-
-  try {
-    const serviceAccount = JSON.parse(jsonString);
-
-    // Validate if required fields are present
-    if (!serviceAccount.project_id || !serviceAccount.private_key) {
-      throw new Error(
-        "Invalid service account JSON. Required fields are missing."
-      );
-    }
-
-    return {
-      datasetName,
-      projectId: serviceAccount.project_id,
-      credentials: serviceAccount,
-    };
-  } catch (error) {
-    throw new Error("Failed to parse JSON string: " + error);
-  }
-}
-
 export async function getForeignKeysBigQuery(
   connection: BigQuery,
   schemaName: string,
@@ -137,26 +144,26 @@ export async function getForeignKeysBigQuery(
 export async function getSchemaColumnInfoBigQuery(
   connection: BigQuery,
   schemaName: string,
-  tableNames: string[]
+  tableNames: { tableName: string; schemaName: string }[]
 ): Promise<
-  { tableName: string; columns: { columnName: string; dataTypeId: number }[] }[]
+  { tableName: string; columns: { columnName: string; dataTypeID: number }[] }[]
 > {
   const allColumns = await Promise.all(
     tableNames.map(async (tableName) => {
       const query = `
       SELECT column_name as columnName, data_type as dataType
-      FROM ${schemaName}.INFORMATION_SCHEMA.COLUMNS
-      WHERE table_name = '${tableName}'
+      FROM ${tableName.schemaName}.INFORMATION_SCHEMA.COLUMNS
+      WHERE table_name = '${tableName.tableName}'
       ORDER BY ordinal_position;
     `;
       const results = await runQueryBigQuery(query, connection);
       return {
-        tableName,
-        displayName: tableName,
+        tableName: `${tableName.schemaName}.${tableName.tableName}`,
+        displayName: `${tableName.schemaName}.${tableName.tableName}`,
         columns: results.rows.map((row: any) => ({
           columnName: row.columnName,
           displayName: row.columnName,
-          dataTypeId: inferType(convertBigQueryTypeToPostgresOID(row.dataType)),
+          dataTypeID: convertBigQueryTypeToPostgresOID(row.dataType),
           fieldType: row.dataType,
         })),
       };
@@ -183,10 +190,22 @@ function inferType(elem: any) {
     // Check if the number is a float or an integer
     return Number.isInteger(elem) ? 23 : 700; // 23: integer, 700: real
   }
+  // BIG QUERY places data into objects as well
+  if (typeof elem === "object") {
+    if (/^\d{4}-\d{2}-\d{2}$/.test(elem.value)) return 1082; // YYYY-MM-DD format
+    if (/^\d{2}\/\d{2}\/\d{2,4}$/.test(elem.value)) return 1082; // MM\DD\YYYY or MM\DD\YY format
+    if (/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?$/.test(elem.value))
+      return 1114; // YYYY-MM-DDTHH:MM:SS[.fraction] format
+    if (/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z$/.test(elem.value))
+      return 1184; // YYYY-MM-DDTHH:MM:SS[.fraction]Z format
+    if (/^\d{2}:\d{2}:\d{2}$/.test(elem.value)) return 1083; // HH:MM:SS format
+  }
   if (typeof elem === "string") {
     // Attempt to infer date, time, and timestamp formats
     // Date in YYYY-MM-DD format
     if (/^\d{4}-\d{2}-\d{2}$/.test(elem)) return 1082; // date
+    // Date in MM\DD\YYYY or MM\DD\YY format
+    if (/^\d{2}\/\d{2}\/\d{2,4}$/.test(elem)) return 1082; // date
     // Timestamp in YYYY-MM-DDTHH:MM:SS[.fraction] format
     if (/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?$/.test(elem)) return 1114; // timestamp without timezone
     // Timestamp in YYYY-MM-DDTHH:MM:SS[.fraction]Z format

package/src/db/DatabaseHelper.ts

@@ -53,10 +53,10 @@ import {
 
 export enum DatabaseType {
   postgres = "postgres",
-  postgresql = "PostgreSQL",
-  snowflake = "Snowflake",
-  bigquery = "BigQuery",
-  mysql = "MySQL",
+  postgresql = "postgresql",
+  snowflake = "snowflake",
+  bigquery = "bigquery",
+  mysql = "mysql",
 }
 
 // export all database connections types
@@ -80,16 +80,16 @@ export function getDatabaseCredentials(
   | BigQueryConfig
   | MysqlConnectionConfig
   | undefined {
-  switch (databaseType) {
+  switch (databaseType.toLowerCase()) {
     case "postgres":
       return formatPostgresConfig(connectionString);
-    case "PostgreSQL":
+    case "postgresql":
       return formatPostgresConfig(connectionString);
-    case "Snowflake":
+    case "snowflake":
       return formatSnowflakeConfig(connectionString);
-    case "BigQuery":
+    case "bigquery":
       return formatBigQueryConfig(connectionString);
-    case "MySQL":
+    case "mysql":
       return formatMysqlConfig(connectionString);
     default:
       return undefined;
@@ -104,16 +104,16 @@ export function connectToDatabase(
     | BigQueryConfig
     | MysqlConnectionConfig
 ): DatabaseConnection {
-  switch (databaseType) {
+  switch (databaseType.toLowerCase()) {
     case "postgres":
       return connectToPostgres(config as PostgresConnectionConfig);
-    case "PostgreSQL":
+    case "postgresql":
       return connectToPostgres(config as PostgresConnectionConfig);
-    case "Snowflake":
+    case "snowflake":
       return connectToSnowflake(config as SnowflakeConnectionConfig);
-    case "BigQuery":
+    case "bigquery":
       return connectToBigQuery(config as BigQueryConfig);
-    case "MySQL":
+    case "mysql":
       return connectToMysql(config as MysqlConnectionConfig);
     default:
       return connectToPostgres(config as PostgresConnectionConfig);
@@ -125,16 +125,16 @@ export function runQueryByDatabase(
   connection: DatabaseConnection,
   sql: string
 ): Promise<QuillQueryResults> | undefined {
-  switch (databaseType) {
+  switch (databaseType.toLowerCase()) {
     case "postgres":
       return runQueryPostgres(sql, connection as Pool);
-    case "PostgreSQL":
+    case "postgresql":
       return runQueryPostgres(sql, connection as Pool);
-    case "Snowflake":
+    case "snowflake":
       return runQuerySnowflake(sql, connection as snowflake.Connection);
-    case "BigQuery":
+    case "bigquery":
       return runQueryBigQuery(sql, connection as BigQuery);
-    case "MySQL":
+    case "mysql":
       return runQueryMysql(sql, connection as MysqlPool);
     default:
       return undefined;
@@ -145,16 +145,16 @@ export function disconnectFromDatabase(
   databaseType: DatabaseType,
   database: DatabaseConnection
 ) {
-  switch (databaseType) {
+  switch (databaseType.toLowerCase()) {
     case "postgres":
       return disconnectFromPostgres(database as Pool);
-    case "PostgreSQL":
+    case "postgresql":
       return disconnectFromPostgres(database as Pool);
-    case "Snowflake":
+    case "snowflake":
       return disconnectFromSnowflake(database as snowflake.Connection);
-    case "BigQuery":
+    case "bigquery":
       return; // BigQuery does not need to be disconnected
-    case "MySQL":
+    case "mysql":
       return disconnectFromMysql(database as MysqlPool);
     default:
       return undefined;
@@ -165,16 +165,16 @@ export async function getSchemasByDatabase(
   databaseType: DatabaseType,
   connection: DatabaseConnection
 ): Promise<string[] | undefined> {
-  switch (databaseType) {
+  switch (databaseType.toLowerCase()) {
     case "postgres":
       return getSchemasPostgres(connection as Pool);
-    case "PostgreSQL":
+    case "postgresql":
       return getSchemasPostgres(connection as Pool);
-    case "Snowflake":
+    case "snowflake":
       return getSchemasSnowflake(connection as snowflake.Connection);
-    case "BigQuery":
+    case "bigquery":
       return getSchemaBigQuery(connection as BigQuery);
-    case "MySQL":
+    case "mysql":
       return getSchemasMysql(connection as MysqlPool);
     default:
       return undefined;
@@ -185,22 +185,34 @@ export async function getSchemasByDatabase(
 export async function getTablesBySchemaByDatabase(
   databaseType: DatabaseType,
   connection: DatabaseConnection,
-  schemaName: string
-): Promise<string[] | undefined> {
-  switch (databaseType) {
+  schemaName: string | string[]
+): Promise<string[] | { tableName: string; schemaName: string }[] | undefined> {
+  switch (databaseType.toLowerCase()) {
     case "postgres":
-      return getTablesBySchemaPostgres(connection as Pool, schemaName);
-    case "PostgreSQL":
-      return getTablesBySchemaPostgres(connection as Pool, schemaName);
-    case "Snowflake":
+      return getTablesBySchemaPostgres(
+        connection as Pool,
+        schemaName as string[]
+      );
+    case "postgresql":
+      return getTablesBySchemaPostgres(
+        connection as Pool,
+        schemaName as string[]
+      );
+    case "snowflake":
       return getTablesBySchemaSnowflake(
         connection as snowflake.Connection,
-        schemaName
+        schemaName as string[]
+      );
+    case "bigquery":
+      return getTablesBySchemaBigQuery(
+        connection as BigQuery,
+        schemaName as string[]
+      );
+    case "mysql":
+      return getTablesBySchemaMysql(
+        connection as MysqlPool,
+        schemaName as string[]
       );
-    case "BigQuery":
-      return getTablesBySchemaBigQuery(connection as BigQuery, schemaName);
-    case "MySQL":
-      return getTablesBySchemaMysql(connection as MysqlPool, schemaName);
     default:
       return undefined;
   }
@@ -213,32 +225,32 @@ export async function getColumnsByTableByDatabase(
   schemaName: string,
   tableName: string
 ): Promise<string[] | undefined> {
-  switch (databaseType) {
+  switch (databaseType.toLowerCase()) {
     case "postgres":
       return getColumnsByTablePostgres(
         connection as Pool,
         schemaName,
         tableName
       );
-    case "PostgreSQL":
+    case "postgresql":
       return getColumnsByTablePostgres(
         connection as Pool,
         schemaName,
         tableName
       );
-    case "Snowflake":
+    case "snowflake":
       return getColumnsByTableSnowflake(
         connection as snowflake.Connection,
         schemaName,
         tableName
       );
-    case "BigQuery":
+    case "bigquery":
       return getColumnsByTableBigQuery(
         connection as BigQuery,
         schemaName,
         tableName
       );
-    case "MySQL":
+    case "mysql":
       return getColumnsByTableMysql(
         connection as MysqlPool,
         schemaName,
@@ -256,7 +268,7 @@ export async function getForiegnKeysByDatabase(
   tableName: string,
   primaryKey: string
 ): Promise<string[] | undefined> {
-  switch (databaseType) {
+  switch (databaseType.toLowerCase()) {
     case "postgres":
       return getForeignKeysPostgres(
         connection as Pool,
@@ -264,28 +276,28 @@ export async function getForiegnKeysByDatabase(
         tableName,
         primaryKey
       );
-    case "PostgreSQL":
+    case "postgresql":
       return getForeignKeysPostgres(
         connection as Pool,
         schemaName,
         tableName,
         primaryKey
       );
-    case "Snowflake":
+    case "snowflake":
       return getForeignKeysSnowflake(
         connection as snowflake.Connection,
         schemaName,
         tableName,
         primaryKey
       );
-    case "BigQuery":
+    case "bigquery":
       return getForeignKeysBigQuery(
         connection as BigQuery,
         schemaName,
         tableName,
         primaryKey
       );
-    case "MySQL":
+    case "mysql":
       return getForeignKeysMysql(
         connection as MysqlPool,
         schemaName,
@@ -301,38 +313,38 @@ export function getColumnInfoBySchemaByDatabase(
   databaseType: DatabaseType,
   connection: DatabaseConnection,
   schemaName: string,
-  tables: string[]
+  tables: string[] | { tableName: string; schemaName: string }[]
 ) {
-  switch (databaseType) {
+  switch (databaseType.toLowerCase()) {
     case "postgres":
       return getSchemaColumnInfoPostgress(
         connection as Pool,
         schemaName,
-        tables
+        tables as { tableName: string; schemaName: string }[]
       );
-    case "PostgreSQL":
+    case "postgresql":
       return getSchemaColumnInfoPostgress(
         connection as Pool,
         schemaName,
-        tables
+        tables as { tableName: string; schemaName: string }[]
       );
-    case "Snowflake":
+    case "snowflake":
       return getSchemaColumnInfoSnowflake(
         connection as snowflake.Connection,
         schemaName,
-        tables
+        tables as { tableName: string; schemaName: string }[]
       );
-    case "BigQuery":
+    case "bigquery":
       return getSchemaColumnInfoBigQuery(
         connection as BigQuery,
         schemaName,
-        tables
+        tables as { tableName: string; schemaName: string }[]
       );
-    case "MySQL":
+    case "mysql":
       return getSchemaColumnInfoMysql(
         connection as MysqlPool,
         schemaName,
-        tables
+        tables as { tableName: string; schemaName: string }[]
       );
     default:
       return undefined;

package/src/db/Mysql.ts

@@ -82,11 +82,18 @@ export async function getSchemasMysql(
 
 export async function getTablesBySchemaMysql(
   connection: MysqlPool,
-  schemaName: string
-): Promise<string[]> {
-  const sql = `SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '${schemaName}'`;
-  const results = await runQueryMysql(sql, connection);
-  return results.rows.map((row) => row.TABLE_NAME);
+  schemaNames: string[]
+): Promise<{ tableName: string; schemaName: string }[]> {
+  const allColumns = await Promise.all(
+    schemaNames.map(async (schema) => {
+      const sql = `SELECT TABLE_NAME, TABLE_SCHEMA FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '${schema}'`;
+      const results = await runQueryMysql(sql, connection);
+      return results.rows.map((row) => {
+        return { tableName: row.TABLE_NAME, schemaName: row.TABLE_SCHEMA };
+      });
+    })
+  );
+  return allColumns.flat();
 }
 
 export async function getColumnsByTableMysql(
@@ -142,26 +149,26 @@ export async function getForeignKeysMysql(
 export async function getSchemaColumnInfoMysql(
   connection: MysqlPool,
   schemaName: string,
-  tableNames: string[]
+  tableNames:  { tableName: string; schemaName: string }[]
 ): Promise<
-  { tableName: string; columns: { columnName: string; dataTypeId: number }[] }[]
+  { tableName: string; columns: { columnName: string; dataTypeID: number }[] }[]
 > {
   const allColumns = await Promise.all(
     tableNames.map(async (tableName) => {
       const query = `
       SELECT COLUMN_NAME as columnName, 
       DATA_TYPE as dataType FROM INFORMATION_SCHEMA.COLUMNS 
-      WHERE TABLE_SCHEMA = '${schemaName}' 
-      AND TABLE_NAME = '${tableName}'
+      WHERE TABLE_SCHEMA = '${tableName.schemaName}' 
+      AND TABLE_NAME = '${tableName.tableName}'
     `;
       const results = await runQueryMysql(query, connection);
       return {
-        tableName,
-        displayName: tableName,
+        tableName: `${tableName.schemaName}.${tableName.tableName}`,
+        displayName: `${tableName.schemaName}.${tableName.tableName}`,
         columns: results.rows.map((row: any) => ({
           columnName: row.columnName,
           displayName: row.columnName,
-          dataTypeId: mysqlTextDataTypeToPostgresOID(row.dataType),
+          dataTypeID: mysqlTextDataTypeToPostgresOID(row.dataType),
           fieldType: row.dataType,
         })),
       };