@quillsql/node 0.5.8 → 0.6.0

package/src/db/BigQuery.ts

@@ -1,4 +1,3 @@
-import { Client } from "../models/Client";
 import { BigQuery } from "@google-cloud/bigquery";
 import { QuillQueryResults } from "./DatabaseHelper";
 import { capitalize, depluralize } from "../utils/textProcessing";
@@ -24,7 +23,7 @@ export function formatBigQueryConfig(connectionString: string): BigQueryConfig {
     // Validate if required fields are present
     if (!serviceAccount.project_id || !serviceAccount.private_key) {
       throw new Error(
-        "Invalid service account JSON. Required fields are missing."
+        "Invalid service account JSON. Required fields are missing.",
       );
     }
 
@@ -44,7 +43,7 @@ export function connectToBigQuery(config: BigQueryConfig) {
 
 export async function runQueryBigQuery(
   sql: string,
-  bigQuery: BigQuery
+  bigQuery: BigQuery,
 ): Promise<QuillQueryResults> {
   const rows = await bigQuery.query(sql);
   if (!rows[0] || rows[0].length === 0) return { fields: [], rows: [] };
@@ -80,7 +79,7 @@ export async function getSchemaBigQuery(bigQuery: BigQuery): Promise<string[]> {
 
 export async function getTablesBySchemaBigQuery(
   bigQuery: BigQuery,
-  schemaNames: string[]
+  schemaNames: string[],
 ): Promise<{ tableName: string; schemaName: string }[]> {
   const allColumns = await Promise.all(
     schemaNames.map(async (schema) => {
@@ -89,7 +88,7 @@ export async function getTablesBySchemaBigQuery(
       return rows[0].map((row) => {
         return { tableName: row.table_name, schemaName: schema };
       });
-    })
+    }),
   );
   return allColumns.flat();
 }
@@ -97,7 +96,7 @@ export async function getTablesBySchemaBigQuery(
 export async function getColumnsByTableBigQuery(
   bigQuery: BigQuery,
   schemaName: string,
-  tableName: string
+  tableName: string,
 ): Promise<string[]> {
   const sql = `SELECT column_name FROM ${schemaName}.INFORMATION_SCHEMA.COLUMNS WHERE table_name = '${tableName}'`;
   const rows = await bigQuery.query(sql);
@@ -108,7 +107,7 @@ export async function getForeignKeysBigQuery(
   connection: BigQuery,
   schemaName: string,
   tableName: string,
-  primaryKey: string
+  primaryKey: string,
 ): Promise<string[]> {
   const depluralizedTableName = depluralize(tableName);
   let sql = `SELECT column_name FROM ${schemaName}.INFORMATION_SCHEMA.COLUMNS 
@@ -121,7 +120,7 @@ export async function getForeignKeysBigQuery(
     return key.column_name;
   });
   foreignKeysString = foreignKeysString.filter(
-    (key) => key !== "id" && key !== "_id_"
+    (key) => key !== "id" && key !== "_id_",
   );
   foreignKeysString = [...new Set(foreignKeysString)];
   if (foreignKeysString.length === 0) {
@@ -144,7 +143,7 @@ export async function getForeignKeysBigQuery(
 export async function getSchemaColumnInfoBigQuery(
   connection: BigQuery,
   schemaName: string,
-  tableNames: { tableName: string; schemaName: string }[]
+  tableNames: { tableName: string; schemaName: string }[],
 ): Promise<
   { tableName: string; columns: { columnName: string; dataTypeID: number }[] }[]
 > {
@@ -167,7 +166,7 @@ export async function getSchemaColumnInfoBigQuery(
           fieldType: row.dataType,
         })),
       };
-    })
+    }),
   );
   return allColumns;
 }
@@ -179,6 +178,7 @@ function convertBigQueryTypeToPostgresOID(type: string): number {
     FLOAT: 700,
     TIMESTAMP: 1114,
     DATE: 1082,
+    BOOL: 16,
   };
 
   const postgresType = typeToOidMap[type.toUpperCase()] || "VARCHAR"; // Default to 'text' if the type is not recognized

package/src/db/CachedConnection.ts

@@ -1,8 +1,7 @@
-import { Pool } from "pg";
 import { Mapable, CacheCredentials } from "../models/Cache";
 import { QuillConfig } from "../models/Quill";
 import { createClient } from "redis";
-import { isSuperset } from "../utils/Error";
+import { isSuperset, PgError } from "../utils/Error";
 import {
   DatabaseConnection,
   connectToDatabase,
@@ -10,43 +9,35 @@ import {
   runQueryByDatabase,
 } from "./DatabaseHelper";
 
-class PgError extends Error {
-  code?: string;
-  detail?: string;
-  hint?: string;
-  position?: string;
-  // Add other properties if needed
-  constructor(detail?: string, hint?: string, position?: string) {
-    super();
-    this.detail = detail;
-    this.hint = hint;
-    this.position = position;
-  }
-}
-
 /** The TTL for new cache entries (default: 1h) */
 const DEFAULT_CACHE_TTL = 24 * 60 * 60;
 
 export class CachedConnection {
   public databaseType: "postgresql" | "snowflake" | "bigquery" | "mysql";
-  public pool: DatabaseConnection;
+  public pool: DatabaseConnection | null;
   public orgId: any;
   public ttl: number;
   public cache: Mapable | null;
+  private config: any;
+  private activeQueries: number = 0;
+  private readonly MAX_ACTIVE_QUERIES: number = 0;
 
   constructor(
     databaseType: "postgresql" | "snowflake" | "bigquery" | "mysql",
     config: any,
-    cacheConfig: Partial<CacheCredentials> = {}
+    cacheConfig: Partial<CacheCredentials> = {},
   ) {
     this.databaseType = databaseType;
     this.pool = connectToDatabase(databaseType, config);
+    this.config = config;
     this.ttl = cacheConfig?.ttl ?? DEFAULT_CACHE_TTL;
     this.cache = this.getCache(cacheConfig);
   }
 
-  public async query(text: string, values?: any[]): Promise<any> {
+  public async query(text: string): Promise<any> {
     try {
+      this.activeQueries++;
+      this.pool = this.getPool();
       if (!this.cache) {
         return await runQueryByDatabase(this.databaseType, this.pool, text);
       }
@@ -58,7 +49,7 @@ export class CachedConnection {
         const newResult = await runQueryByDatabase(
           this.databaseType,
           this.pool,
-          text
+          text,
         );
         const newResultString: string = JSON.stringify(newResult);
         await this.cache.set(key, newResultString, "EX", DEFAULT_CACHE_TTL);
@@ -67,13 +58,21 @@ export class CachedConnection {
     } catch (err) {
       if (isSuperset(err, PgError)) {
         throw new PgError(
+          (err as any).message,
           (err as any).detail,
           (err as any).hint,
-          (err as any).position
+          (err as any).position,
         );
       } else if (err instanceof Error) {
         throw new Error(err.message);
       }
+    } finally {
+      this.activeQueries--;
+      if (this.activeQueries <= this.MAX_ACTIVE_QUERIES) {
+        if (this.databaseType.toLowerCase() === "mysql") {
+          this.close();
+        }
+      }
     }
   }
 
@@ -89,14 +88,24 @@ export class CachedConnection {
   }: QuillConfig["cache"]): Mapable | null {
     if (cacheType === "redis" || cacheType === "rediss") {
       const redisURL = `${cacheType}://${username}:${password}@${host}:${port}`;
-      const client =  createClient({ url: redisURL });
+      const client = createClient({ url: redisURL });
       client.connect();
       return client as Mapable;
     }
     return null;
   }
 
+  public getPool() {
+    if (!this.pool) {
+      this.pool = connectToDatabase(this.databaseType, this.config);
+    }
+    return this.pool;
+  }
+
   async close() {
-    await disconnectFromDatabase(this.databaseType, this.pool);
+    if (this.pool) {
+      disconnectFromDatabase(this.databaseType, this.pool);
+      this.pool = null;
+    }
   }
 }

package/src/db/DatabaseHelper.ts

@@ -1,6 +1,5 @@
 import { Pool } from "pg";
 import snowflake from "snowflake-sdk";
-import { Client } from "../models/Client";
 import { Pool as MysqlPool } from "mysql2";
 import { BigQuery } from "@google-cloud/bigquery";
 import {
@@ -65,7 +64,7 @@ export interface QuillQueryResults {
 
 export function getDatabaseCredentials(
   databaseType: "postgresql" | "snowflake" | "bigquery" | "mysql",
-  connectionString: string
+  connectionString: string,
 ):
   | PostgresConnectionConfig
   | SnowflakeConnectionConfig
@@ -94,7 +93,7 @@ export function connectToDatabase(
     | PostgresConnectionConfig
     | SnowflakeConnectionConfig
     | BigQueryConfig
-    | MysqlConnectionConfig
+    | MysqlConnectionConfig,
 ): DatabaseConnection {
   switch (databaseType.toLowerCase()) {
     case "postgres":
@@ -115,7 +114,7 @@ export function connectToDatabase(
 export function runQueryByDatabase(
   databaseType: "postgresql" | "snowflake" | "bigquery" | "mysql",
   connection: DatabaseConnection,
-  sql: string
+  sql: string,
 ): Promise<QuillQueryResults> | undefined {
   switch (databaseType.toLowerCase()) {
     case "postgres":
@@ -135,7 +134,7 @@ export function runQueryByDatabase(
 
 export function disconnectFromDatabase(
   databaseType: "postgresql" | "snowflake" | "bigquery" | "mysql",
-  database: DatabaseConnection
+  database: DatabaseConnection,
 ) {
   switch (databaseType.toLowerCase()) {
     case "postgres":
@@ -155,7 +154,7 @@ export function disconnectFromDatabase(
 
 export async function getSchemasByDatabase(
   databaseType: "postgresql" | "snowflake" | "bigquery" | "mysql",
-  connection: DatabaseConnection
+  connection: DatabaseConnection,
 ): Promise<string[] | undefined> {
   switch (databaseType.toLowerCase()) {
     case "postgres":
@@ -177,33 +176,33 @@ export async function getSchemasByDatabase(
 export async function getTablesBySchemaByDatabase(
   databaseType: "postgresql" | "snowflake" | "bigquery" | "mysql",
   connection: DatabaseConnection,
-  schemaName: string | string[]
+  schemaName: string | string[],
 ): Promise<string[] | { tableName: string; schemaName: string }[] | undefined> {
   switch (databaseType.toLowerCase()) {
     case "postgres":
       return getTablesBySchemaPostgres(
         connection as Pool,
-        schemaName as string[]
+        schemaName as string[],
       );
     case "postgresql":
       return getTablesBySchemaPostgres(
         connection as Pool,
-        schemaName as string[]
+        schemaName as string[],
       );
     case "snowflake":
       return getTablesBySchemaSnowflake(
         connection as snowflake.Connection,
-        schemaName as string[]
+        schemaName as string[],
       );
     case "bigquery":
       return getTablesBySchemaBigQuery(
         connection as BigQuery,
-        schemaName as string[]
+        schemaName as string[],
       );
     case "mysql":
       return getTablesBySchemaMysql(
         connection as MysqlPool,
-        schemaName as string[]
+        schemaName as string[],
       );
     default:
       return undefined;
@@ -215,38 +214,38 @@ export async function getColumnsByTableByDatabase(
   databaseType: "postgresql" | "snowflake" | "bigquery" | "mysql",
   connection: DatabaseConnection,
   schemaName: string,
-  tableName: string
+  tableName: string,
 ): Promise<string[] | undefined> {
   switch (databaseType.toLowerCase()) {
     case "postgres":
       return getColumnsByTablePostgres(
         connection as Pool,
         schemaName,
-        tableName
+        tableName,
       );
     case "postgresql":
       return getColumnsByTablePostgres(
         connection as Pool,
         schemaName,
-        tableName
+        tableName,
       );
     case "snowflake":
       return getColumnsByTableSnowflake(
         connection as snowflake.Connection,
         schemaName,
-        tableName
+        tableName,
       );
     case "bigquery":
       return getColumnsByTableBigQuery(
         connection as BigQuery,
         schemaName,
-        tableName
+        tableName,
       );
     case "mysql":
       return getColumnsByTableMysql(
         connection as MysqlPool,
         schemaName,
-        tableName
+        tableName,
       );
     default:
       return undefined;
@@ -258,7 +257,7 @@ export async function getForiegnKeysByDatabase(
   connection: DatabaseConnection,
   schemaName: string,
   tableName: string,
-  primaryKey: string
+  primaryKey: string,
 ): Promise<string[] | undefined> {
   switch (databaseType.toLowerCase()) {
     case "postgres":
@@ -266,35 +265,35 @@ export async function getForiegnKeysByDatabase(
         connection as Pool,
         schemaName,
         tableName,
-        primaryKey
+        primaryKey,
       );
     case "postgresql":
       return getForeignKeysPostgres(
         connection as Pool,
         schemaName,
         tableName,
-        primaryKey
+        primaryKey,
       );
     case "snowflake":
       return getForeignKeysSnowflake(
         connection as snowflake.Connection,
         schemaName,
         tableName,
-        primaryKey
+        primaryKey,
       );
     case "bigquery":
       return getForeignKeysBigQuery(
         connection as BigQuery,
         schemaName,
         tableName,
-        primaryKey
+        primaryKey,
       );
     case "mysql":
       return getForeignKeysMysql(
         connection as MysqlPool,
         schemaName,
         tableName,
-        primaryKey
+        primaryKey,
       );
     default:
       return undefined;
@@ -305,38 +304,38 @@ export function getColumnInfoBySchemaByDatabase(
   databaseType: "postgresql" | "snowflake" | "bigquery" | "mysql",
   connection: DatabaseConnection,
   schemaName: string,
-  tables: string[] | { tableName: string; schemaName: string }[]
+  tables: string[] | { tableName: string; schemaName: string }[],
 ) {
   switch (databaseType.toLowerCase()) {
     case "postgres":
       return getSchemaColumnInfoPostgress(
         connection as Pool,
         schemaName,
-        tables as { tableName: string; schemaName: string }[]
+        tables as { tableName: string; schemaName: string }[],
       );
     case "postgresql":
       return getSchemaColumnInfoPostgress(
         connection as Pool,
         schemaName,
-        tables as { tableName: string; schemaName: string }[]
+        tables as { tableName: string; schemaName: string }[],
       );
     case "snowflake":
       return getSchemaColumnInfoSnowflake(
         connection as snowflake.Connection,
         schemaName,
-        tables as { tableName: string; schemaName: string }[]
+        tables as { tableName: string; schemaName: string }[],
       );
     case "bigquery":
       return getSchemaColumnInfoBigQuery(
         connection as BigQuery,
         schemaName,
-        tables as { tableName: string; schemaName: string }[]
+        tables as { tableName: string; schemaName: string }[],
       );
     case "mysql":
       return getSchemaColumnInfoMysql(
         connection as MysqlPool,
         schemaName,
-        tables as { tableName: string; schemaName: string }[]
+        tables as { tableName: string; schemaName: string }[],
       );
     default:
       return undefined;

package/src/db/Mysql.ts

@@ -1,5 +1,3 @@
-import { connect } from "http2";
-import { Client } from "../models/Client";
 import mysql, { Pool as MysqlPool } from "mysql2";
 import { QuillQueryResults } from "./DatabaseHelper";
 import url from "url";
@@ -14,7 +12,7 @@ export interface MysqlConnectionConfig {
 }
 
 export function formatMysqlConfig(
-  connectionString: string
+  connectionString: string,
 ): MysqlConnectionConfig {
   const parsedUrl = url.parse(connectionString);
   const [user, password] = (parsedUrl.auth || "").split(":");
@@ -31,7 +29,7 @@ export function connectToMysql(config: MysqlConnectionConfig): MysqlPool {
   const pool = mysql.createPool({
     ...config,
     waitForConnections: true,
-    connectionLimit: 10,
+    connectionLimit: 128,
     queueLimit: 0,
   });
   return pool;
@@ -43,7 +41,7 @@ export function disconnectFromMysql(connection: MysqlPool) {
 
 export async function runQueryMysql(
   sql: string,
-  connection: MysqlPool
+  connection: MysqlPool,
 ): Promise<QuillQueryResults> {
   const result: QuillQueryResults = await new Promise((resolve, reject) => {
     connection.query(sql, (error, results, fields) => {
@@ -72,7 +70,7 @@ export async function runQueryMysql(
 }
 
 export async function getSchemasMysql(
-  connection: MysqlPool
+  connection: MysqlPool,
 ): Promise<string[]> {
   const sql = `SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA
     WHERE schema_name != 'information_schema' 
@@ -84,7 +82,7 @@ export async function getSchemasMysql(
 
 export async function getTablesBySchemaMysql(
   connection: MysqlPool,
-  schemaNames: string[]
+  schemaNames: string[],
 ): Promise<{ tableName: string; schemaName: string }[]> {
   const allColumns = await Promise.all(
     schemaNames.map(async (schema) => {
@@ -93,7 +91,7 @@ export async function getTablesBySchemaMysql(
       return results.rows.map((row) => {
         return { tableName: row.TABLE_NAME, schemaName: row.TABLE_SCHEMA };
       });
-    })
+    }),
   );
   return allColumns.flat();
 }
@@ -101,7 +99,7 @@ export async function getTablesBySchemaMysql(
 export async function getColumnsByTableMysql(
   connection: MysqlPool,
   schemaName: string,
-  tableName: string
+  tableName: string,
 ): Promise<string[]> {
   const sql = `SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '${schemaName}' AND TABLE_NAME = '${tableName}'`;
   const results = await runQueryMysql(sql, connection);
@@ -112,7 +110,7 @@ export async function getForeignKeysMysql(
   connection: MysqlPool,
   schemaName: string,
   tableName: string,
-  primaryKey: string
+  primaryKey: string,
 ): Promise<string[]> {
   const depluralizedTableName = depluralize(tableName);
   let sql = `SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS 
@@ -127,7 +125,7 @@ export async function getForeignKeysMysql(
   });
   // remove any foreignKeyStrings that are just 'id'
   foreignKeysString = foreignKeysString.filter(
-    (key) => key !== "id" && key !== "_id_"
+    (key) => key !== "id" && key !== "_id_",
   );
   foreignKeysString = [...new Set(foreignKeysString)];
   if (foreignKeysString.length === 0) {
@@ -151,7 +149,7 @@ export async function getForeignKeysMysql(
 export async function getSchemaColumnInfoMysql(
   connection: MysqlPool,
   schemaName: string,
-  tableNames: { tableName: string; schemaName: string }[]
+  tableNames: { tableName: string; schemaName: string }[],
 ): Promise<
   { tableName: string; columns: { columnName: string; dataTypeID: number }[] }[]
 > {
@@ -174,7 +172,7 @@ export async function getSchemaColumnInfoMysql(
           fieldType: row.dataType,
         })),
       };
-    })
+    }),
   );
   return allColumns;
 }
@@ -191,6 +189,10 @@ function mysqlTextDataTypeToPostgresOID(type: string): number {
       return 1043;
     case "timestamp": // date
       return 1082;
+    case "date": // date
+      return 1082;
+    case "datetime": // date
+      return 1082;
     default: // varchar
       return 1043;
   }
@@ -210,7 +212,9 @@ function mysqlDataTypeIdToPostgresType(type: number): number {
       return 1043;
     case 7: // date
       return 1082;
-    case 7: // date
+    case 10: // date
+      return 1082;
+    case 12: // date
       return 1082;
     default: // varchar
       return 1043;