@quillsql/node 0.5.9 → 0.6.0

package/src/db/BigQuery.ts

@@ -1,4 +1,3 @@
-import { Client } from "../models/Client";
 import { BigQuery } from "@google-cloud/bigquery";
 import { QuillQueryResults } from "./DatabaseHelper";
 import { capitalize, depluralize } from "../utils/textProcessing";
@@ -24,7 +23,7 @@ export function formatBigQueryConfig(connectionString: string): BigQueryConfig {
     // Validate if required fields are present
     if (!serviceAccount.project_id || !serviceAccount.private_key) {
       throw new Error(
-        "Invalid service account JSON. Required fields are missing."
+        "Invalid service account JSON. Required fields are missing.",
       );
     }
 
@@ -44,7 +43,7 @@ export function connectToBigQuery(config: BigQueryConfig) {
 
 export async function runQueryBigQuery(
   sql: string,
-  bigQuery: BigQuery
+  bigQuery: BigQuery,
 ): Promise<QuillQueryResults> {
   const rows = await bigQuery.query(sql);
   if (!rows[0] || rows[0].length === 0) return { fields: [], rows: [] };
@@ -80,7 +79,7 @@ export async function getSchemaBigQuery(bigQuery: BigQuery): Promise<string[]> {
 
 export async function getTablesBySchemaBigQuery(
   bigQuery: BigQuery,
-  schemaNames: string[]
+  schemaNames: string[],
 ): Promise<{ tableName: string; schemaName: string }[]> {
   const allColumns = await Promise.all(
     schemaNames.map(async (schema) => {
@@ -89,7 +88,7 @@ export async function getTablesBySchemaBigQuery(
       return rows[0].map((row) => {
         return { tableName: row.table_name, schemaName: schema };
       });
-    })
+    }),
   );
   return allColumns.flat();
 }
@@ -97,7 +96,7 @@ export async function getTablesBySchemaBigQuery(
 export async function getColumnsByTableBigQuery(
   bigQuery: BigQuery,
   schemaName: string,
-  tableName: string
+  tableName: string,
 ): Promise<string[]> {
   const sql = `SELECT column_name FROM ${schemaName}.INFORMATION_SCHEMA.COLUMNS WHERE table_name = '${tableName}'`;
   const rows = await bigQuery.query(sql);
@@ -108,7 +107,7 @@ export async function getForeignKeysBigQuery(
   connection: BigQuery,
   schemaName: string,
   tableName: string,
-  primaryKey: string
+  primaryKey: string,
 ): Promise<string[]> {
   const depluralizedTableName = depluralize(tableName);
   let sql = `SELECT column_name FROM ${schemaName}.INFORMATION_SCHEMA.COLUMNS 
@@ -121,7 +120,7 @@ export async function getForeignKeysBigQuery(
     return key.column_name;
   });
   foreignKeysString = foreignKeysString.filter(
-    (key) => key !== "id" && key !== "_id_"
+    (key) => key !== "id" && key !== "_id_",
   );
   foreignKeysString = [...new Set(foreignKeysString)];
   if (foreignKeysString.length === 0) {
@@ -144,7 +143,7 @@ export async function getForeignKeysBigQuery(
 export async function getSchemaColumnInfoBigQuery(
   connection: BigQuery,
   schemaName: string,
-  tableNames: { tableName: string; schemaName: string }[]
+  tableNames: { tableName: string; schemaName: string }[],
 ): Promise<
   { tableName: string; columns: { columnName: string; dataTypeID: number }[] }[]
 > {
@@ -167,7 +166,7 @@ export async function getSchemaColumnInfoBigQuery(
           fieldType: row.dataType,
         })),
       };
-    })
+    }),
   );
   return allColumns;
 }
@@ -179,6 +178,7 @@ function convertBigQueryTypeToPostgresOID(type: string): number {
     FLOAT: 700,
     TIMESTAMP: 1114,
     DATE: 1082,
+    BOOL: 16,
   };
 
   const postgresType = typeToOidMap[type.toUpperCase()] || "VARCHAR"; // Default to 'text' if the type is not recognized

package/src/db/CachedConnection.ts

@@ -1,8 +1,7 @@
-import { Pool } from "pg";
 import { Mapable, CacheCredentials } from "../models/Cache";
 import { QuillConfig } from "../models/Quill";
 import { createClient } from "redis";
-import { isSuperset } from "../utils/Error";
+import { isSuperset, PgError } from "../utils/Error";
 import {
   DatabaseConnection,
   connectToDatabase,
@@ -10,20 +9,6 @@ import {
   runQueryByDatabase,
 } from "./DatabaseHelper";
 
-class PgError extends Error {
-  code?: string;
-  detail?: string;
-  hint?: string;
-  position?: string;
-  // Add other properties if needed
-  constructor(detail?: string, hint?: string, position?: string) {
-    super();
-    this.detail = detail;
-    this.hint = hint;
-    this.position = position;
-  }
-}
-
 /** The TTL for new cache entries (default: 1h) */
 const DEFAULT_CACHE_TTL = 24 * 60 * 60;
 
@@ -34,6 +19,8 @@ export class CachedConnection {
   public ttl: number;
   public cache: Mapable | null;
   private config: any;
+  private activeQueries: number = 0;
+  private readonly MAX_ACTIVE_QUERIES: number = 0;
 
   constructor(
     databaseType: "postgresql" | "snowflake" | "bigquery" | "mysql",
@@ -47,8 +34,9 @@ export class CachedConnection {
     this.cache = this.getCache(cacheConfig);
   }
 
-  public async query(text: string, values?: any[]): Promise<any> {
+  public async query(text: string): Promise<any> {
     try {
+      this.activeQueries++;
       this.pool = this.getPool();
       if (!this.cache) {
         return await runQueryByDatabase(this.databaseType, this.pool, text);
@@ -70,6 +58,7 @@ export class CachedConnection {
     } catch (err) {
       if (isSuperset(err, PgError)) {
         throw new PgError(
+          (err as any).message,
           (err as any).detail,
           (err as any).hint,
           (err as any).position,
@@ -78,8 +67,11 @@ export class CachedConnection {
         throw new Error(err.message);
       }
     } finally {
-      if (this.databaseType.toLowerCase() === "mysql") {
-        this.close();
+      this.activeQueries--;
+      if (this.activeQueries <= this.MAX_ACTIVE_QUERIES) {
+        if (this.databaseType.toLowerCase() === "mysql") {
+          this.close();
+        }
       }
     }
   }

package/src/db/DatabaseHelper.ts

@@ -1,6 +1,5 @@
 import { Pool } from "pg";
 import snowflake from "snowflake-sdk";
-import { Client } from "../models/Client";
 import { Pool as MysqlPool } from "mysql2";
 import { BigQuery } from "@google-cloud/bigquery";
 import {
@@ -65,7 +64,7 @@ export interface QuillQueryResults {
 
 export function getDatabaseCredentials(
   databaseType: "postgresql" | "snowflake" | "bigquery" | "mysql",
-  connectionString: string
+  connectionString: string,
 ):
   | PostgresConnectionConfig
   | SnowflakeConnectionConfig
@@ -94,7 +93,7 @@ export function connectToDatabase(
     | PostgresConnectionConfig
     | SnowflakeConnectionConfig
     | BigQueryConfig
-    | MysqlConnectionConfig
+    | MysqlConnectionConfig,
 ): DatabaseConnection {
   switch (databaseType.toLowerCase()) {
     case "postgres":
@@ -115,7 +114,7 @@ export function connectToDatabase(
 export function runQueryByDatabase(
   databaseType: "postgresql" | "snowflake" | "bigquery" | "mysql",
   connection: DatabaseConnection,
-  sql: string
+  sql: string,
 ): Promise<QuillQueryResults> | undefined {
   switch (databaseType.toLowerCase()) {
     case "postgres":
@@ -135,7 +134,7 @@ export function runQueryByDatabase(
 
 export function disconnectFromDatabase(
   databaseType: "postgresql" | "snowflake" | "bigquery" | "mysql",
-  database: DatabaseConnection
+  database: DatabaseConnection,
 ) {
   switch (databaseType.toLowerCase()) {
     case "postgres":
@@ -155,7 +154,7 @@ export function disconnectFromDatabase(
 
 export async function getSchemasByDatabase(
   databaseType: "postgresql" | "snowflake" | "bigquery" | "mysql",
-  connection: DatabaseConnection
+  connection: DatabaseConnection,
 ): Promise<string[] | undefined> {
   switch (databaseType.toLowerCase()) {
     case "postgres":
@@ -177,33 +176,33 @@ export async function getSchemasByDatabase(
 export async function getTablesBySchemaByDatabase(
   databaseType: "postgresql" | "snowflake" | "bigquery" | "mysql",
   connection: DatabaseConnection,
-  schemaName: string | string[]
+  schemaName: string | string[],
 ): Promise<string[] | { tableName: string; schemaName: string }[] | undefined> {
   switch (databaseType.toLowerCase()) {
     case "postgres":
       return getTablesBySchemaPostgres(
         connection as Pool,
-        schemaName as string[]
+        schemaName as string[],
       );
     case "postgresql":
       return getTablesBySchemaPostgres(
         connection as Pool,
-        schemaName as string[]
+        schemaName as string[],
       );
     case "snowflake":
       return getTablesBySchemaSnowflake(
         connection as snowflake.Connection,
-        schemaName as string[]
+        schemaName as string[],
       );
     case "bigquery":
       return getTablesBySchemaBigQuery(
         connection as BigQuery,
-        schemaName as string[]
+        schemaName as string[],
       );
     case "mysql":
       return getTablesBySchemaMysql(
         connection as MysqlPool,
-        schemaName as string[]
+        schemaName as string[],
       );
     default:
       return undefined;
@@ -215,38 +214,38 @@ export async function getColumnsByTableByDatabase(
   databaseType: "postgresql" | "snowflake" | "bigquery" | "mysql",
   connection: DatabaseConnection,
   schemaName: string,
-  tableName: string
+  tableName: string,
 ): Promise<string[] | undefined> {
   switch (databaseType.toLowerCase()) {
     case "postgres":
       return getColumnsByTablePostgres(
         connection as Pool,
         schemaName,
-        tableName
+        tableName,
       );
     case "postgresql":
       return getColumnsByTablePostgres(
         connection as Pool,
         schemaName,
-        tableName
+        tableName,
       );
     case "snowflake":
       return getColumnsByTableSnowflake(
         connection as snowflake.Connection,
         schemaName,
-        tableName
+        tableName,
       );
     case "bigquery":
       return getColumnsByTableBigQuery(
         connection as BigQuery,
         schemaName,
-        tableName
+        tableName,
       );
     case "mysql":
       return getColumnsByTableMysql(
         connection as MysqlPool,
         schemaName,
-        tableName
+        tableName,
       );
     default:
       return undefined;
@@ -258,7 +257,7 @@ export async function getForiegnKeysByDatabase(
   connection: DatabaseConnection,
   schemaName: string,
   tableName: string,
-  primaryKey: string
+  primaryKey: string,
 ): Promise<string[] | undefined> {
   switch (databaseType.toLowerCase()) {
     case "postgres":
@@ -266,35 +265,35 @@ export async function getForiegnKeysByDatabase(
         connection as Pool,
         schemaName,
         tableName,
-        primaryKey
+        primaryKey,
       );
     case "postgresql":
       return getForeignKeysPostgres(
         connection as Pool,
         schemaName,
         tableName,
-        primaryKey
+        primaryKey,
       );
     case "snowflake":
       return getForeignKeysSnowflake(
         connection as snowflake.Connection,
         schemaName,
         tableName,
-        primaryKey
+        primaryKey,
       );
     case "bigquery":
       return getForeignKeysBigQuery(
         connection as BigQuery,
         schemaName,
         tableName,
-        primaryKey
+        primaryKey,
       );
     case "mysql":
       return getForeignKeysMysql(
         connection as MysqlPool,
         schemaName,
         tableName,
-        primaryKey
+        primaryKey,
       );
     default:
       return undefined;
@@ -305,38 +304,38 @@ export function getColumnInfoBySchemaByDatabase(
   databaseType: "postgresql" | "snowflake" | "bigquery" | "mysql",
   connection: DatabaseConnection,
   schemaName: string,
-  tables: string[] | { tableName: string; schemaName: string }[]
+  tables: string[] | { tableName: string; schemaName: string }[],
 ) {
   switch (databaseType.toLowerCase()) {
     case "postgres":
       return getSchemaColumnInfoPostgress(
         connection as Pool,
         schemaName,
-        tables as { tableName: string; schemaName: string }[]
+        tables as { tableName: string; schemaName: string }[],
       );
     case "postgresql":
       return getSchemaColumnInfoPostgress(
         connection as Pool,
         schemaName,
-        tables as { tableName: string; schemaName: string }[]
+        tables as { tableName: string; schemaName: string }[],
       );
     case "snowflake":
       return getSchemaColumnInfoSnowflake(
         connection as snowflake.Connection,
         schemaName,
-        tables as { tableName: string; schemaName: string }[]
+        tables as { tableName: string; schemaName: string }[],
       );
     case "bigquery":
       return getSchemaColumnInfoBigQuery(
         connection as BigQuery,
         schemaName,
-        tables as { tableName: string; schemaName: string }[]
+        tables as { tableName: string; schemaName: string }[],
       );
     case "mysql":
       return getSchemaColumnInfoMysql(
         connection as MysqlPool,
         schemaName,
-        tables as { tableName: string; schemaName: string }[]
+        tables as { tableName: string; schemaName: string }[],
       );
     default:
       return undefined;

package/src/db/Mysql.ts

@@ -1,5 +1,3 @@
-import { connect } from "http2";
-import { Client } from "../models/Client";
 import mysql, { Pool as MysqlPool } from "mysql2";
 import { QuillQueryResults } from "./DatabaseHelper";
 import url from "url";

package/src/db/Postgres.ts

@@ -1,10 +1,8 @@
 import { Pool } from "pg";
-import { Client } from "../models/Client";
-import { disconnect } from "process";
+// import { Client } from "../models/Client";
 import { QuillQueryResults } from "./DatabaseHelper";
 import { capitalize, depluralize } from "../utils/textProcessing";
 import { PG_TYPES } from "../assets/pgtypes";
-import { run } from "node:test";
 
 export type PostgresConnectionConfig = {
   connectionString: string;
@@ -26,7 +24,7 @@ export function disconnectFromPostgres(pool: Pool) {
 
 export async function runQueryPostgres(
   sql: string,
-  pool: Pool
+  pool: Pool,
 ): Promise<QuillQueryResults> {
   const results = await pool.query(sql);
   return {
@@ -47,7 +45,7 @@ export async function getSchemasPostgres(pool: Pool): Promise<string[]> {
 
 export async function getTablesBySchemaPostgres(
   pool: Pool,
-  schemaNames: string[]
+  schemaNames: string[],
 ): Promise<{ tableName: string; schemaName: string }[]> {
   const allColumns = await Promise.all(
     schemaNames.map(async (schema) => {
@@ -56,7 +54,7 @@ export async function getTablesBySchemaPostgres(
       return results.rows.map((row) => {
         return { tableName: row.table_name, schemaName: row.table_schema };
       });
-    })
+    }),
   );
   return allColumns.flat();
 }
@@ -64,7 +62,7 @@ export async function getTablesBySchemaPostgres(
 export async function getColumnsByTablePostgres(
   pool: Pool,
   schemaName: string,
-  tableName: string
+  tableName: string,
 ): Promise<string[]> {
   const sql = `SELECT column_name FROM information_schema.columns WHERE table_schema = '${schemaName}' and table_name = '${tableName}'`;
   const results = await runQueryPostgres(sql, pool);
@@ -75,7 +73,7 @@ export async function getForeignKeysPostgres(
   pool: Pool,
   schemaName: string,
   tableName: string,
-  primaryKey: string
+  primaryKey: string,
 ): Promise<string[]> {
   const depluralizedTableName = depluralize(tableName);
   let sql = `SELECT column_name FROM information_schema.columns 
@@ -89,7 +87,7 @@ export async function getForeignKeysPostgres(
     return key.column_name;
   });
   foreignKeysString = foreignKeysString.filter(
-    (key) => key !== "id" && key !== "_id_"
+    (key) => key !== "id" && key !== "_id_",
   );
   foreignKeysString = [...new Set(foreignKeysString)];
   if (foreignKeysString.length === 0) {
@@ -113,7 +111,7 @@ export async function getForeignKeysPostgres(
 export async function getSchemaColumnInfoPostgress(
   pool: Pool,
   schemaName: string,
-  tableNames: { tableName: string; schemaName: string }[]
+  tableNames: { tableName: string; schemaName: string }[],
 ): Promise<
   { tableName: string; columns: { columnName: string; dataTypeID: number }[] }[]
 > {
@@ -145,43 +143,43 @@ export async function getSchemaColumnInfoPostgress(
           };
         }),
       };
-    })
+    }),
   );
   return allColumns;
 }
 
 export function formatPostgresConfig(
-  connectionString: string
+  connectionString: string,
 ): PostgresConnectionConfig {
   return { connectionString, ssl: { rejectUnauthorized: false } };
 }
 
 // CURRENTLY UNUSED BUT MAYBE USEFUL IN THE FUTURE
-function getSslConfig(client: Client):
-  | {
-      rejectUnauthorized: false;
-      ca?: string;
-      key?: string;
-      cert?: string;
-    }
-  | undefined {
-  if (!client.useSsl) {
-    return undefined;
-  }
-  if (client.serverCa && client.clientKey && client.clientCert) {
-    return {
-      rejectUnauthorized: false,
-      ca: client.serverCa,
-      key: client.clientKey,
-      cert: client.clientCert,
-    };
-  }
-  if (client.serverCa) {
-    return {
-      rejectUnauthorized: false,
-      ca: client.serverCa,
-    };
-  }
-  // if using ssl with no certificates
-  return { rejectUnauthorized: false };
-}
+// function getSslConfig(client: Client):
+//   | {
+//       rejectUnauthorized: false;
+//       ca?: string;
+//       key?: string;
+//       cert?: string;
+//     }
+//   | undefined {
+//   if (!client.useSsl) {
+//     return undefined;
+//   }
+//   if (client.serverCa && client.clientKey && client.clientCert) {
+//     return {
+//       rejectUnauthorized: false,
+//       ca: client.serverCa,
+//       key: client.clientKey,
+//       cert: client.clientCert,
+//     };
+//   }
+//   if (client.serverCa) {
+//     return {
+//       rejectUnauthorized: false,
+//       ca: client.serverCa,
+//     };
+//   }
+//   // if using ssl with no certificates
+//   return { rejectUnauthorized: false };
+// }

package/src/db/Snowflake.ts

@@ -1,5 +1,4 @@
 import snowflake from "snowflake-sdk";
-import { Client } from "../models/Client";
 import { QuillQueryResults } from "./DatabaseHelper";
 import { capitalize, depluralize } from "../utils/textProcessing";
 
@@ -31,7 +30,7 @@ export type SnowflakeConnectionConfig = {
 
 export async function runQuerySnowflake(
   sql: string,
-  connection: snowflake.Connection
+  connection: snowflake.Connection,
 ): Promise<QuillQueryResults> {
   const results = await new Promise((resolve, reject) => {
     connection.execute({
@@ -56,7 +55,7 @@ export async function runQuerySnowflake(
 }
 
 export async function getSchemasSnowflake(
-  connection: snowflake.Connection
+  connection: snowflake.Connection,
 ): Promise<string[]> {
   const sql = `SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA
     WHERE SCHEMA_NAME != 'INFORMATION_SCHEMA'`;
@@ -66,7 +65,7 @@ export async function getSchemasSnowflake(
 
 export async function getTablesBySchemaSnowflake(
   connection: snowflake.Connection,
-  schemaNames: string[]
+  schemaNames: string[],
 ): Promise<{ tableName: string; schemaName: string }[]> {
   const allColumns = await Promise.all(
     schemaNames.map(async (schema) => {
@@ -80,7 +79,7 @@ export async function getTablesBySchemaSnowflake(
       return results.rows.map((row) => {
         return { tableName: row.tableName, schemaName: row.schemaName };
       });
-    })
+    }),
   );
   return allColumns.flat();
 }
@@ -88,7 +87,7 @@ export async function getTablesBySchemaSnowflake(
 export async function getColumnsByTableSnowflake(
   connection: snowflake.Connection,
   schemaName: string,
-  tableName: string
+  tableName: string,
 ): Promise<string[]> {
   const sql = `SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '${schemaName}' AND TABLE_NAME = '${tableName}'`;
   const results = await runQuerySnowflake(sql, connection);
@@ -96,7 +95,7 @@ export async function getColumnsByTableSnowflake(
 }
 
 export function formatSnowflakeConfig(
-  connectionString: string
+  connectionString: string,
 ): SnowflakeConnectionConfig {
   const parsed = new URL(connectionString);
   return {
@@ -109,7 +108,7 @@ export function formatSnowflakeConfig(
 }
 
 export function connectToSnowflake(
-  config: SnowflakeConnectionConfig
+  config: SnowflakeConnectionConfig,
 ): snowflake.Connection {
   const connection = snowflake.createConnection({
     ...config,
@@ -124,9 +123,9 @@ export function connectToSnowflake(
 }
 
 export async function disconnectFromSnowflake(
-  connection: snowflake.Connection
+  connection: snowflake.Connection,
 ) {
-  connection.destroy((err, conn) => {
+  connection.destroy((err) => {
     if (err) {
       console.error(`Failed to disconnect from Snowflake: ${err.message}`);
     }
@@ -137,9 +136,9 @@ export async function getForeignKeysSnowflake(
   connection: snowflake.Connection,
   schemaName: string,
   tableName: string,
-  primaryKey: string
+  primaryKey: string,
 ): Promise<string[]> {
-  let depluralizedTableName = depluralize(tableName);
+  const depluralizedTableName = depluralize(tableName);
   let sql = `SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS 
     WHERE TABLE_SCHEMA = '${schemaName}' 
     AND TABLE_NAME = '${schemaName}' 
@@ -152,7 +151,7 @@ export async function getForeignKeysSnowflake(
     return key.COLUMN_NAME;
   });
   foreignKeysString = foreignKeysString.filter(
-    (key) => key !== "id" && key !== "_id_"
+    (key) => key !== "id" && key !== "_id_",
   );
   foreignKeysString = [...new Set(foreignKeysString)];
   if (foreignKeysString.length === 0) {
@@ -177,7 +176,7 @@ export async function getForeignKeysSnowflake(
 export async function getSchemaColumnInfoSnowflake(
   connection: snowflake.Connection,
   schemaName: string,
-  tableNames: { tableName: string; schemaName: string }[]
+  tableNames: { tableName: string; schemaName: string }[],
 ) {
   const allColumns = await Promise.all(
     tableNames.map(async (tableName) => {
@@ -200,7 +199,7 @@ export async function getSchemaColumnInfoSnowflake(
           };
         }),
       };
-    })
+    }),
   );
   return allColumns;
 }