@quillsql/node 0.3.7 → 0.4.0

package/src/db/Mysql.ts

@@ -0,0 +1,216 @@
+import { connect } from "http2";
+import { Client } from "../models/Client";
+import mysql, { Pool as MysqlPool } from "mysql2";
+import { QuillQueryResults } from "./DatabaseHelper";
+import url from "url";
+import { capitalize, depluralize } from "../utils/textProcessing";
+
+export interface MysqlConnectionConfig {
+  host: string;
+  user: string;
+  password: string;
+  database: string;
+}
+
+export function formatMysqlConfig(
+  connectionString: string
+): MysqlConnectionConfig {
+  const parsedUrl = url.parse(connectionString);
+  const [user, password] = (parsedUrl.auth || "").split(":");
+  return {
+    host: parsedUrl.hostname || "",
+    user: user || "",
+    password: password || "",
+    database: (parsedUrl.pathname || "").slice(1),
+  };
+}
+
+export function connectToMysql(config: MysqlConnectionConfig): MysqlPool {
+  const pool = mysql.createPool({
+    ...config,
+    waitForConnections: true,
+    connectionLimit: 10,
+    queueLimit: 0,
+  });
+  return pool;
+}
+
+export function disconnectFromMysql(connection: MysqlPool) {
+  connection.end();
+}
+
+export async function runQueryMysql(
+  sql: string,
+  connection: MysqlPool
+): Promise<QuillQueryResults> {
+  const result: QuillQueryResults = await new Promise((resolve, reject) => {
+    connection.query(sql, (error, results, fields) => {
+      if (error) {
+        reject(error);
+        return;
+      }
+
+      const mappedFields = fields
+        ? fields.map((field) => ({
+            name: field.name,
+            dataTypeID: mysqlDataTypeIdToPostgresType(field.type || 1043), // Provide a default value for dataTypeID
+          }))
+        : [];
+
+      const processRows = Array.isArray(results)
+        ? results.map((row: any) => {
+            return JSON.parse(JSON.stringify(row));
+          })
+        : [];
+
+      resolve({ fields: mappedFields, rows: processRows });
+    });
+  });
+  return result;
+}
+
+export async function getSchemasMysql(
+  connection: MysqlPool
+): Promise<string[]> {
+  const sql = `SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA
+    WHERE schema_name != 'information_schema' 
+    AND schema_name != 'performance_schema'
+    and schema_name != 'sys';`;
+  const results = await runQueryMysql(sql, connection);
+  return results.rows.map((row) => row.SCHEMA_NAME);
+}
+
+export async function getTablesBySchemaMysql(
+  connection: MysqlPool,
+  schemaNames: string[]
+): Promise<{ tableName: string; schemaName: string }[]> {
+  const allColumns = await Promise.all(
+    schemaNames.map(async (schema) => {
+      const sql = `SELECT TABLE_NAME, TABLE_SCHEMA FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '${schema}'`;
+      const results = await runQueryMysql(sql, connection);
+      return results.rows.map((row) => {
+        return { tableName: row.TABLE_NAME, schemaName: row.TABLE_SCHEMA };
+      });
+    })
+  );
+  return allColumns.flat();
+}
+
+export async function getColumnsByTableMysql(
+  connection: MysqlPool,
+  schemaName: string,
+  tableName: string
+): Promise<string[]> {
+  const sql = `SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '${schemaName}' AND TABLE_NAME = '${tableName}'`;
+  const results = await runQueryMysql(sql, connection);
+  return results.rows.map((row) => row.COLUMN_NAME);
+}
+
+export async function getForeignKeysMysql(
+  connection: MysqlPool,
+  schemaName: string,
+  tableName: string,
+  primaryKey: string
+): Promise<string[]> {
+  const depluralizedTableName = depluralize(tableName);
+  let sql = `SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS 
+    WHERE TABLE_SCHEMA = '${schemaName}' 
+    and table_name != '${tableName}' 
+    and (column_name = '${primaryKey}' 
+    or column_name = '${depluralizedTableName}\\_${primaryKey}' 
+    or column_name = '${depluralizedTableName}${capitalize(primaryKey)}' )`;
+  const results = await runQueryMysql(sql, connection);
+  let foreignKeysString = results.rows.map((key) => {
+    return key.COLUMN_NAME;
+  });
+  // remove any foreignKeyStrings that are just 'id'
+  foreignKeysString = foreignKeysString.filter(
+    (key) => key !== "id" && key !== "_id_"
+  );
+  foreignKeysString = [...new Set(foreignKeysString)];
+  if (foreignKeysString.length === 0) {
+    sql = `SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS 
+        WHERE TABLE_SCHEMA = '${schemaName}' 
+        and table_name != '${tableName}' 
+        and (column_name like '${depluralizedTableName}%' 
+        or column_name like '%\\_id' 
+        or column_name like '%Id'
+        or column_name like '%\\_${primaryKey}' 
+        or column_name like '%${capitalize(primaryKey)}')`;
+    const results = await runQueryMysql(sql, connection);
+    foreignKeysString = results.rows.map((key) => {
+      return key.COLUMN_NAME;
+    });
+    foreignKeysString = [...new Set(foreignKeysString)];
+  }
+  return foreignKeysString;
+}
+
+export async function getSchemaColumnInfoMysql(
+  connection: MysqlPool,
+  schemaName: string,
+  tableNames:  { tableName: string; schemaName: string }[]
+): Promise<
+  { tableName: string; columns: { columnName: string; dataTypeID: number }[] }[]
+> {
+  const allColumns = await Promise.all(
+    tableNames.map(async (tableName) => {
+      const query = `
+      SELECT COLUMN_NAME as columnName, 
+      DATA_TYPE as dataType FROM INFORMATION_SCHEMA.COLUMNS 
+      WHERE TABLE_SCHEMA = '${tableName.schemaName}' 
+      AND TABLE_NAME = '${tableName.tableName}'
+    `;
+      const results = await runQueryMysql(query, connection);
+      return {
+        tableName: `${tableName.schemaName}.${tableName.tableName}`,
+        displayName: `${tableName.schemaName}.${tableName.tableName}`,
+        columns: results.rows.map((row: any) => ({
+          columnName: row.columnName,
+          displayName: row.columnName,
+          dataTypeID: mysqlTextDataTypeToPostgresOID(row.dataType),
+          fieldType: row.dataType,
+        })),
+      };
+    })
+  );
+  return allColumns;
+}
+
+function mysqlTextDataTypeToPostgresOID(type: string): number {
+  switch (type) {
+    case "bigint": // int
+      return 23;
+    case "tinyint": // int
+      return 23;
+    case "float": // float
+      return 700;
+    case "varchar": // varchar
+      return 1043;
+    case "timestamp": // date
+      return 1082;
+    default: // varchar
+      return 1043;
+  }
+}
+
+function mysqlDataTypeIdToPostgresType(type: number): number {
+  switch (type) {
+    case 8: // int
+      return 23;
+    case 3: // int
+      return 23;
+    case 2: // int
+      return 23;
+    case 5: // float
+      return 700;
+    case 253: // varchar
+      return 1043;
+    case 7: // date
+      return 1082;
+    case 7: // date
+      return 1082;
+    default: // varchar
+      return 1043;
+  }
+}

package/src/db/Postgres.ts

@@ -0,0 +1,187 @@
+import { Pool } from "pg";
+import { Client } from "../models/Client";
+import { disconnect } from "process";
+import { QuillQueryResults } from "./DatabaseHelper";
+import { capitalize, depluralize } from "../utils/textProcessing";
+import { PG_TYPES } from "../assets/pgtypes";
+import { run } from "node:test";
+
+export type PostgresConnectionConfig = {
+  connectionString: string;
+  ssl?: {
+    rejectUnauthorized: false;
+    ca?: string;
+    key?: string;
+    cert?: string;
+  };
+};
+
+export function connectToPostgres(config: PostgresConnectionConfig): Pool {
+  return new Pool(config);
+}
+
+export function disconnectFromPostgres(pool: Pool) {
+  pool.end();
+}
+
+export async function runQueryPostgres(
+  sql: string,
+  pool: Pool
+): Promise<QuillQueryResults> {
+  const results = await pool.query(sql);
+  return {
+    fields: results.fields.map((field: any) => ({
+      name: field.name,
+      dataTypeID: field.dataTypeID,
+    })),
+    rows: results.rows,
+  };
+}
+
+export async function getSchemasPostgres(pool: Pool): Promise<string[]> {
+  const sql = `SELECT schema_name FROM information_schema.schemata 
+    WHERE schema_name NOT LIKE 'pg_%' AND schema_name != 'information_schema';`;
+  const results = await runQueryPostgres(sql, pool);
+  return results.rows.map((row) => row.schema_name);
+}
+
+export async function getTablesBySchemaPostgres(
+  pool: Pool,
+  schemaNames: string[]
+): Promise<{ tableName: string; schemaName: string }[]> {
+  const allColumns = await Promise.all(
+    schemaNames.map(async (schema) => {
+      const sql = `SELECT table_name, table_schema FROM information_schema.tables WHERE table_schema = '${schema}'`;
+      const results = await runQueryPostgres(sql, pool);
+      return results.rows.map((row) => {
+        return { tableName: row.table_name, schemaName: row.table_schema };
+      });
+    })
+  );
+  return allColumns.flat();
+}
+
+export async function getColumnsByTablePostgres(
+  pool: Pool,
+  schemaName: string,
+  tableName: string
+): Promise<string[]> {
+  const sql = `SELECT column_name FROM information_schema.columns WHERE table_schema = '${schemaName}' and table_name = '${tableName}'`;
+  const results = await runQueryPostgres(sql, pool);
+  return results.rows.map((row) => row.column_name);
+}
+
+export async function getForeignKeysPostgres(
+  pool: Pool,
+  schemaName: string,
+  tableName: string,
+  primaryKey: string
+): Promise<string[]> {
+  const depluralizedTableName = depluralize(tableName);
+  let sql = `SELECT column_name FROM information_schema.columns 
+  WHERE table_schema = '${schemaName}' 
+  and table_name != '${tableName}' 
+  and (column_name = '${primaryKey}' 
+    or column_name = '${depluralizedTableName}_${primaryKey}' 
+    or column_name = '${depluralizedTableName}${capitalize(primaryKey)}')`;
+  const results = await runQueryPostgres(sql, pool);
+  let foreignKeysString = results.rows.map((key) => {
+    return key.column_name;
+  });
+  foreignKeysString = foreignKeysString.filter(
+    (key) => key !== "id" && key !== "_id_"
+  );
+  foreignKeysString = [...new Set(foreignKeysString)];
+  if (foreignKeysString.length === 0) {
+    sql = `SELECT column_name FROM information_schema.columns 
+      WHERE table_schema = '${schemaName}' 
+      and table_name != '${tableName}' 
+      and (column_name like '${tableName}%' 
+      or column_name like '%\\_id' 
+      or column_name like '%Id' 
+      or column_name like '%\\_${primaryKey}' 
+      or column_name like '%${capitalize(primaryKey)}')`;
+    const results = await runQueryPostgres(sql, pool);
+    foreignKeysString = results.rows.map((key) => {
+      return key.column_name;
+    });
+    foreignKeysString = [...new Set(foreignKeysString)];
+  }
+  return foreignKeysString;
+}
+
+export async function getSchemaColumnInfoPostgress(
+  pool: Pool,
+  schemaName: string,
+  tableNames: { tableName: string; schemaName: string }[]
+): Promise<
+  { tableName: string; columns: { columnName: string; dataTypeID: number }[] }[]
+> {
+  const allColumns = await Promise.all(
+    tableNames.map(async (tableName) => {
+      const query = `
+      SELECT column_name as "columnName", udt_name as "fieldType"
+      FROM information_schema.columns
+      WHERE table_schema = '${tableName.schemaName}' 
+      AND table_name = '${tableName.tableName}'
+      ORDER BY ordinal_position;
+    `;
+      const results = await runQueryPostgres(query, pool);
+      return {
+        tableName: `${tableName.schemaName}.${tableName.tableName}`,
+        displayName: `${tableName.schemaName}.${tableName.tableName}`,
+        columns: results.rows.map((row) => {
+          let pgType = PG_TYPES.find((pgType) => {
+            return pgType.typname === row.fieldType;
+          })?.oid;
+          if (!pgType) {
+            pgType = 1043;
+          }
+          return {
+            columnName: row.columnName,
+            displayName: row.columnName,
+            dataTypeID: pgType,
+            fieldType: row.fieldType,
+          };
+        }),
+      };
+    })
+  );
+  return allColumns;
+}
+
+export function formatPostgresConfig(
+  connectionString: string
+): PostgresConnectionConfig {
+  return { connectionString, ssl: { rejectUnauthorized: false } };
+}
+
+// CURRENTLY UNUSED BUT MAYBE USEFUL IN THE FUTURE
+function getSslConfig(client: Client):
+  | {
+      rejectUnauthorized: false;
+      ca?: string;
+      key?: string;
+      cert?: string;
+    }
+  | undefined {
+  if (!client.useSsl) {
+    return undefined;
+  }
+  if (client.serverCa && client.clientKey && client.clientCert) {
+    return {
+      rejectUnauthorized: false,
+      ca: client.serverCa,
+      key: client.clientKey,
+      cert: client.clientCert,
+    };
+  }
+  if (client.serverCa) {
+    return {
+      rejectUnauthorized: false,
+      ca: client.serverCa,
+    };
+  }
+  // if using ssl with no certificates
+  return { rejectUnauthorized: false };
+}

package/src/db/Snowflake.ts

@@ -0,0 +1,203 @@
+import snowflake from "snowflake-sdk";
+import { Client } from "../models/Client";
+import { QuillQueryResults } from "./DatabaseHelper";
+import { capitalize, depluralize } from "../utils/textProcessing";
+
+const POSTGRES_SNOWFLAKE_MAP: { [type: string]: number } = {
+  BOOLEAN: 16,
+  FIXED: 1700, // DECIMAL or NUMERIC
+  REAL: 700, // FLOAT4
+  DOUBLE: 701, // FLOAT8
+  TEXT: 25, // TEXT in PostgreSQL
+  DATE: 1082,
+  DATETIME: 1184, // TIMESTAMP in PostgreSQL
+  TIME: 1083,
+  TIMESTAMP_LTZ: 1184,
+  TIMESTAMP_NTZ: 1184,
+  TIMESTAMP_TZ: 1184,
+  VARIANT: 114, // JSONB in PostgreSQL
+  OBJECT: 114, // JSONB, as an equivalent for structured JSON
+  ARRAY: 1009, // TEXT[], assuming most common case
+  BINARY: 17, // BYTEA
+};
+
+export type SnowflakeConnectionConfig = {
+  account: string;
+  username: string;
+  password: string;
+  database: string;
+  warehouse: string;
+};
+
+export async function runQuerySnowflake(
+  sql: string,
+  connection: snowflake.Connection
+): Promise<QuillQueryResults> {
+  const results = await new Promise((resolve, reject) => {
+    connection.execute({
+      sqlText: sql,
+      complete: (err, stmt, rows) => {
+        if (err) {
+          reject(err);
+          return { success: false, message: err.message };
+        } else {
+          resolve({
+            rows,
+            fields: stmt.getColumns().map((col) => ({
+              name: col.getName(),
+              dataTypeID: POSTGRES_SNOWFLAKE_MAP[col.getType().toUpperCase()],
+            })),
+          });
+        }
+      },
+    });
+  });
+  return results as QuillQueryResults;
+}
+
+export async function getSchemasSnowflake(
+  connection: snowflake.Connection
+): Promise<string[]> {
+  const sql = `SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA
+    WHERE SCHEMA_NAME != 'INFORMATION_SCHEMA'`;
+  const results = await runQuerySnowflake(sql, connection);
+  return results.rows.map((row) => row.SCHEMA_NAME);
+}
+
+export async function getTablesBySchemaSnowflake(
+  connection: snowflake.Connection,
+  schemaNames: string[]
+): Promise<{ tableName: string; schemaName: string }[]> {
+  const allColumns = await Promise.all(
+    schemaNames.map(async (schema) => {
+      const query = `SELECT 
+      TABLE_NAME as "tableName",
+      TABLE_SCHEMA as "schemaName"
+      FROM INFORMATION_SCHEMA.TABLES
+      WHERE  TABLE_SCHEMA = '${schema}';
+    `;
+      const results = await runQuerySnowflake(query, connection);
+      return results.rows.map((row) => {
+        return { tableName: row.tableName, schemaName: row.schemaName };
+      });
+    })
+  );
+  return allColumns.flat();
+}
+
+export async function getColumnsByTableSnowflake(
+  connection: snowflake.Connection,
+  schemaName: string,
+  tableName: string
+): Promise<string[]> {
+  const sql = `SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '${schemaName}' AND TABLE_NAME = '${tableName}'`;
+  const results = await runQuerySnowflake(sql, connection);
+  return results.rows.map((row) => row.COLUMN_NAME);
+}
+
+export function formatSnowflakeConfig(
+  connectionString: string
+): SnowflakeConnectionConfig {
+  const parsed = new URL(connectionString);
+  return {
+    account: parsed.hostname,
+    username: parsed.username,
+    password: parsed.password,
+    database: parsed.pathname.split("/")[1],
+    warehouse: parsed.pathname.split("/")[2],
+  };
+}
+
+export function connectToSnowflake(
+  config: SnowflakeConnectionConfig
+): snowflake.Connection {
+  const connection = snowflake.createConnection(config);
+  connection.connect((err) => {
+    if (err) {
+      console.error(`Failed to connect to Snowflake: ${err.message}`);
+    }
+  });
+  return connection;
+}
+
+export async function disconnectFromSnowflake(
+  connection: snowflake.Connection
+) {
+  connection.destroy((err, conn) => {
+    if (err) {
+      console.error(`Failed to disconnect from Snowflake: ${err.message}`);
+    }
+  });
+}
+
+export async function getForeignKeysSnowflake(
+  connection: snowflake.Connection,
+  schemaName: string,
+  tableName: string,
+  primaryKey: string
+): Promise<string[]> {
+  let depluralizedTableName = depluralize(tableName);
+  let sql = `SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS 
+    WHERE TABLE_SCHEMA = '${schemaName}' 
+    AND TABLE_NAME = '${schemaName}' 
+    and TABLE_NAME != '${tableName}' 
+    and (COLUMN_NAME = '${primaryKey}' 
+    or COLUMN_NAME = '${depluralizedTableName}_${primaryKey}' 
+    or COLUMN_NAME = '${depluralizedTableName}${capitalize(primaryKey)}')`;
+  const results = await runQuerySnowflake(sql, connection);
+  let foreignKeysString = results.rows.map((key) => {
+    return key.COLUMN_NAME;
+  });
+  foreignKeysString = foreignKeysString.filter(
+    (key) => key !== "id" && key !== "_id_"
+  );
+  foreignKeysString = [...new Set(foreignKeysString)];
+  if (foreignKeysString.length === 0) {
+    sql = `SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS 
+      WHERE TABLE_SCHEMA = '${schemaName}' 
+      AND TABLE_NAME = '${schemaName}' 
+      and TABLE_NAME != '${tableName}' 
+      and (COLUMN_NAME like '${depluralizedTableName}%' 
+        or column_name like '%\\_id' 
+        or column_name like '%Id'
+        or COLUMN_NAME like '%\\_${primaryKey}' 
+        or COLUMN_NAME like '%${capitalize(primaryKey)}')`;
+    const results = await runQuerySnowflake(sql, connection);
+    foreignKeysString = results.rows.map((key) => {
+      return key.COLUMN_NAME;
+    });
+    foreignKeysString = [...new Set(foreignKeysString)];
+  }
+  return foreignKeysString;
+}
+
+export async function getSchemaColumnInfoSnowflake(
+  connection: snowflake.Connection,
+  schemaName: string,
+  tableNames: { tableName: string; schemaName: string }[]
+) {
+  const allColumns = await Promise.all(
+    tableNames.map(async (tableName) => {
+      const query = `SELECT 
+      COLUMN_NAME as "columnName", DATA_TYPE as "dataType"
+      FROM INFORMATION_SCHEMA.COLUMNS
+      WHERE TABLE_SCHEMA = '${tableName.schemaName}' AND TABLE_NAME = '${tableName.tableName}';
+    `;
+      const results = await runQuerySnowflake(query, connection);
+      return {
+        tableName: `${tableName.schemaName}.${tableName.tableName}`,
+        displayName: `${tableName.schemaName}.${tableName.tableName}`,
+        columns: results.rows.map((row) => {
+          const postgresType = POSTGRES_SNOWFLAKE_MAP[row.dataType];
+          return {
+            columnName: row.columnName,
+            displayName: row.columnName,
+            dataTypeID: postgresType,
+            fieldType: row.dataType,
+          };
+        }),
+      };
+    })
+  );
+  return allColumns;
+}