@quillsql/node 0.3.8 → 0.4.0

package/src/db/Postgres.ts

@@ -47,11 +47,18 @@ export async function getSchemasPostgres(pool: Pool): Promise<string[]> {
 
 export async function getTablesBySchemaPostgres(
   pool: Pool,
-  schemaName: string
-): Promise<string[]> {
-  const sql = `SELECT table_name FROM information_schema.tables WHERE table_schema = '${schemaName}'`;
-  const results = await runQueryPostgres(sql, pool);
-  return results.rows.map((row) => row.table_name);
+  schemaNames: string[]
+): Promise<{ tableName: string; schemaName: string }[]> {
+  const allColumns = await Promise.all(
+    schemaNames.map(async (schema) => {
+      const sql = `SELECT table_name, table_schema FROM information_schema.tables WHERE table_schema = '${schema}'`;
+      const results = await runQueryPostgres(sql, pool);
+      return results.rows.map((row) => {
+        return { tableName: row.table_name, schemaName: row.table_schema };
+      });
+    })
+  );
+  return allColumns.flat();
 }
 
 export async function getColumnsByTablePostgres(
@@ -81,7 +88,9 @@ export async function getForeignKeysPostgres(
   let foreignKeysString = results.rows.map((key) => {
     return key.column_name;
   });
-  foreignKeysString = foreignKeysString.filter((key) => key !== "id" && key !== "_id_");
+  foreignKeysString = foreignKeysString.filter(
+    (key) => key !== "id" && key !== "_id_"
+  );
   foreignKeysString = [...new Set(foreignKeysString)];
   if (foreignKeysString.length === 0) {
     sql = `SELECT column_name FROM information_schema.columns 
@@ -104,23 +113,23 @@ export async function getForeignKeysPostgres(
 export async function getSchemaColumnInfoPostgress(
   pool: Pool,
   schemaName: string,
-  tableNames: string[]
+  tableNames: { tableName: string; schemaName: string }[]
 ): Promise<
-  { tableName: string; columns: { columnName: string; dataTypeId: number }[] }[]
+  { tableName: string; columns: { columnName: string; dataTypeID: number }[] }[]
 > {
   const allColumns = await Promise.all(
     tableNames.map(async (tableName) => {
       const query = `
       SELECT column_name as "columnName", udt_name as "fieldType"
       FROM information_schema.columns
-      WHERE table_schema = '${schemaName}' 
-      AND table_name = '${tableName}'
+      WHERE table_schema = '${tableName.schemaName}' 
+      AND table_name = '${tableName.tableName}'
       ORDER BY ordinal_position;
     `;
       const results = await runQueryPostgres(query, pool);
       return {
-        tableName,
-        displayName: tableName,
+        tableName: `${tableName.schemaName}.${tableName.tableName}`,
+        displayName: `${tableName.schemaName}.${tableName.tableName}`,
         columns: results.rows.map((row) => {
           let pgType = PG_TYPES.find((pgType) => {
             return pgType.typname === row.fieldType;
@@ -131,7 +140,7 @@ export async function getSchemaColumnInfoPostgress(
           return {
             columnName: row.columnName,
             displayName: row.columnName,
-            dataTypeId: pgType,
+            dataTypeID: pgType,
             fieldType: row.fieldType,
           };
         }),

package/src/db/Snowflake.ts

@@ -66,11 +66,23 @@ export async function getSchemasSnowflake(
 
 export async function getTablesBySchemaSnowflake(
   connection: snowflake.Connection,
-  schema: string
-): Promise<string[]> {
-  const sql = `SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '${schema}'`;
-  const results = await runQuerySnowflake(sql, connection);
-  return results.rows.map((row) => row.TABLE_NAME);
+  schemaNames: string[]
+): Promise<{ tableName: string; schemaName: string }[]> {
+  const allColumns = await Promise.all(
+    schemaNames.map(async (schema) => {
+      const query = `SELECT 
+      TABLE_NAME as "tableName",
+      TABLE_SCHEMA as "schemaName"
+      FROM INFORMATION_SCHEMA.TABLES
+      WHERE  TABLE_SCHEMA = '${schema}';
+    `;
+      const results = await runQuerySnowflake(query, connection);
+      return results.rows.map((row) => {
+        return { tableName: row.tableName, schemaName: row.schemaName };
+      });
+    })
+  );
+  return allColumns.flat();
 }
 
 export async function getColumnsByTableSnowflake(
@@ -162,25 +174,25 @@ export async function getForeignKeysSnowflake(
 export async function getSchemaColumnInfoSnowflake(
   connection: snowflake.Connection,
   schemaName: string,
-  tableNames: string[]
+  tableNames: { tableName: string; schemaName: string }[]
 ) {
   const allColumns = await Promise.all(
     tableNames.map(async (tableName) => {
       const query = `SELECT 
       COLUMN_NAME as "columnName", DATA_TYPE as "dataType"
       FROM INFORMATION_SCHEMA.COLUMNS
-      WHERE TABLE_SCHEMA = '${schemaName}' AND TABLE_NAME = '${tableName}';
+      WHERE TABLE_SCHEMA = '${tableName.schemaName}' AND TABLE_NAME = '${tableName.tableName}';
     `;
       const results = await runQuerySnowflake(query, connection);
       return {
-        tableName,
-        displayName: tableName,
+        tableName: `${tableName.schemaName}.${tableName.tableName}`,
+        displayName: `${tableName.schemaName}.${tableName.tableName}`,
         columns: results.rows.map((row) => {
           const postgresType = POSTGRES_SNOWFLAKE_MAP[row.dataType];
           return {
             columnName: row.columnName,
             displayName: row.columnName,
-            dataTypeId: postgresType,
+            dataTypeID: postgresType,
             fieldType: row.dataType,
           };
         }),

package/src/index.ts

@@ -8,12 +8,7 @@ import {
 import { CachedConnection } from "./db/CachedConnection";
 import axios from "axios";
 import "dotenv/config";
-import {
-  getTableSchema,
-  mapQueries,
-  removeFields,
-} from "./utils/RunQueryProcesses";
-import { PG_TYPES } from "./assets/pgtypes";
+import { mapQueries, removeFields } from "./utils/RunQueryProcesses";
 import {
   DatabaseType,
   connectToDatabase,
@@ -25,6 +20,7 @@ import {
   getTablesBySchemaByDatabase,
   runQueryByDatabase,
 } from "./db/DatabaseHelper";
+import { convertTypeToPostgres } from "./utils/schemaConversion";
 
 const HOST =
   process.env.ENV === "development"
@@ -47,49 +43,49 @@ export default class QuillClass {
 
   constructor(
     privateKey: string,
-    databaseConnectionString: string,
-    cache: Partial<CacheCredentials> = {},
     databaseType: DatabaseType,
+    databaseConnectionString?: string,
+    databaseCredentials?: any,
+    cache: Partial<CacheCredentials> = {},
     metadataServerURL?: string
   ) {
     this.baseUrl = metadataServerURL ? metadataServerURL : HOST;
     this.config = { headers: { Authorization: `Bearer ${privateKey}` } };
+    let credentials = databaseCredentials;
+    if (databaseConnectionString) {
+      credentials = getDatabaseCredentials(
+        databaseType,
+        databaseConnectionString
+      );
+    }
     this.targetConnection = new CachedConnection(
       databaseType,
-      getDatabaseCredentials(databaseType, databaseConnectionString),
+      credentials,
       cache
     );
   }
 
   public async query({ orgId, metadata }: QuillQueryParams): Promise<any> {
     this.targetConnection.orgId = orgId;
-
     try {
-      // Initial Query Request
-      const limitedQueries = metadata.preQueries
-        ? metadata.preQueries?.map((query) => query + " limit 1")
-        : [];
       const preQueryResults = metadata.preQueries
-        ? await this.runQueries(limitedQueries)
-        : [];
-      const columns = metadata.preQueries
-        ? preQueryResults.queryResults[0].fields.map((field: any) => {
-            return {
-              fieldType: PG_TYPES.find((type) => field.dataTypeID === type.oid)
-                ? PG_TYPES.find((type) => field.dataTypeID === type.oid)
-                    ?.typname
-                : field.dataTypeID,
-              name: field.name,
-              displayName: field.name,
-              isVisible: true,
-              field: field.name,
-            };
-          })
-        : metadata.columns;
+        ? await this.runQueries(
+            metadata.preQueries,
+            this.targetConnection.databaseType,
+            metadata.databaseType,
+            metadata.runQueryConfig
+          )
+        : {};
+      if (metadata.runQueryConfig?.overridePost) {
+        return {
+          data: { queryResults: preQueryResults },
+          status: "success",
+        };
+      }
       const response = await this.postQuill(metadata.task, {
         ...metadata,
+        ...preQueryResults,
         orgId,
-        columns,
         viewQuery: metadata.preQueries ? metadata.preQueries[0] : undefined,
       });
       // if there is no metadata object in the response, create one
@@ -98,6 +94,8 @@ export default class QuillClass {
       }
       const results = await this.runQueries(
         response.queries,
+        this.targetConnection.databaseType,
+        metadata.databaseType,
         response.metadata.runQueryConfig
       );
       // QUICK JANKY FIX TO UPDATE METADATA AFTER GETTING MAPPED ARRAYS
@@ -134,37 +132,87 @@ export default class QuillClass {
 
   private async runQueries(
     queries: string[] | undefined,
+    pkDatabaseType: DatabaseType,
+    databaseType?: string,
     runQueryConfig?: AdditionalProcessing
   ) {
     let results: any;
     if (!queries) return { ...results, queryResults: [] };
-    if (!queries) return { ...results, queryResults: [] };
+    if (
+      databaseType &&
+      databaseType.toLowerCase() !== pkDatabaseType.toLowerCase()
+    ) {
+      return {
+        dbMismatched: true,
+        backendDatbaseType: pkDatabaseType,
+        queryResults: [],
+      };
+    }
     if (runQueryConfig?.arrayToMap) {
-      const mappedArray = await mapQueries(
-        queries,
-        runQueryConfig.arrayToMap,
-        this.targetConnection
-      );
+      const mappedArray = await mapQueries(queries, this.targetConnection);
       return { ...results, queryResults: [], mappedArray };
+    } else if (runQueryConfig?.getColumns) {
+      const queryResult = await this.targetConnection.query(
+        `${queries[0]} limit 1`
+      );
+      const columns = queryResult.fields.map((field: any) => {
+        return {
+          fieldType: convertTypeToPostgres(field.dataTypeID),
+          name: field.name,
+          displayName: field.name,
+          isVisible: true,
+          field: field.name,
+        };
+      });
+      return { columns };
+    } else if (runQueryConfig?.getTables) {
+      const queryResult = await getTablesBySchemaByDatabase(
+        this.targetConnection.databaseType,
+        this.targetConnection.pool,
+        runQueryConfig.schemaNames! || runQueryConfig.schema
+      );
+      const schemaInfo = await getColumnInfoBySchemaByDatabase(
+        this.targetConnection.databaseType,
+        this.targetConnection.pool,
+        runQueryConfig.schema!,
+        queryResult!
+      );
+      return schemaInfo;
     } else {
+      if (runQueryConfig?.limitThousand) {
+        queries = queries.map((query) => {
+          return query.replace(/;/, "") + " limit 1000;";
+        });
+      }
       const queryResults = await Promise.all(
         queries.map(async (query) => {
           return await this.targetConnection.query(query);
         })
       );
       results = { ...results, queryResults };
-      if (runQueryConfig?.getSchema) {
-        results = {
-          ...results,
-          columns: await getTableSchema(queryResults[0], this.targetConnection),
-        };
-      }
       if (runQueryConfig?.removeFields) {
         results = {
           ...results,
           queryResults: removeFields(queryResults, runQueryConfig.removeFields),
         };
       }
+      if (runQueryConfig?.convertDatatypes) {
+        results = queryResults.map((result) => {
+          return {
+            fields: result.fields.map((field: any) => {
+              return {
+                ...field,
+                fieldType: convertTypeToPostgres(field.dataTypeID),
+                isVisible: true,
+                field: field.name,
+                displayName: field.name,
+                name: field.name,
+              };
+            }),
+            rows: result.rows,
+          };
+        });
+      }
     }
     return results;
   }
@@ -189,6 +237,7 @@ export default class QuillClass {
 const Quill = ({
   privateKey,
   databaseConnectionString,
+  databaseConfig,
   cache,
   databaseType,
   metadataServerURL,
@@ -197,13 +246,15 @@ const Quill = ({
   databaseConnectionString: string;
   cache?: Partial<CacheCredentials>;
   databaseType: DatabaseType;
+  databaseConfig: any;
   metadataServerURL?: string;
 }) => {
   return new QuillClass(
     privateKey,
+    databaseType,
     databaseConnectionString,
+    databaseConfig,
     cache,
-    databaseType,
     metadataServerURL
   );
 };

package/src/index.uspec.ts

@@ -9,9 +9,9 @@ describe("Quill", () => {
   beforeEach(() => {
     quill = new Quill(
       "dummy_private_key",
+      DatabaseType.postgres,
       "dummy_db_url",
       {},
-      DatabaseType.postgres,
       undefined
     );
     quill.targetConnection.query = jest.fn().mockResolvedValue([]);

package/src/models/Quill.ts

@@ -7,6 +7,7 @@ export interface QuillRequestMetadata {
   // a query to be run
   queries?: string[];
   preQueries?: string[];
+  runQueryConfig?: AdditionalProcessing;
   query?: string;
   // a report to be fetched
   id?: string;
@@ -25,6 +26,7 @@ export interface QuillRequestMetadata {
   template?: boolean;
   clientId?: string;
   deleted?: boolean;
+  databaseType?: string
 }
 
 export interface QuillQueryParams {
@@ -51,8 +53,16 @@ export interface QuillConfig {
 
 export interface AdditionalProcessing {
   getSchema?: boolean;
+  getColumns?: boolean;
+  getTables?: boolean;
+  schema?: string,
+  schemaNames?: string[];
+  table?: string,
   removeFields?: string[];
   arrayToMap?: { arrayName: string; field: string };
+  overridePost?: boolean;
+  convertDatatypes?: boolean
+  limitThousand?: boolean;
 }
 
 export interface QuillClientResponse {

package/src/utils/RunQueryProcesses.ts

@@ -7,28 +7,6 @@ interface TableSchemaInfo {
   isVisible: boolean;
 }
 
-export async function getTableSchema(
-  queryResults: any,
-  targetConnection: CachedConnection
-) {
-  const typesQuery = await targetConnection.query(
-    "select typname, oid, typarray from pg_type order by oid;"
-  );
-  const schema: TableSchemaInfo[] = queryResults[0].fields.map(
-    (field: { dataTypeID: any; name: any }) => {
-      return {
-        fieldType: typesQuery.rows.filter(
-          (type: { oid: any }) => field.dataTypeID === type.oid
-        )[0].typname,
-        name: field.name,
-        displayName: field.name,
-        isVisible: true,
-      };
-    }
-  );
-  return schema;
-}
-
 export function removeFields(queryResults: any, fieldsToRemove: string[]): any {
   const fields = queryResults.fields.filter((field: { name: any }) =>
     fieldsToRemove.includes(field.name)
@@ -43,7 +21,6 @@ export function removeFields(queryResults: any, fieldsToRemove: string[]): any {
 
 export async function mapQueries(
   queries: string[],
-  arrayToMap: { arrayName: string; field: string },
   targetConnection: CachedConnection
 ): Promise<any[]> {
   const mappedArray = [];

package/src/utils/schemaConversion.ts

@@ -0,0 +1,11 @@
+import { PG_TYPES } from "../assets/pgtypes";
+
+export function convertTypeToPostgres(data_type_id: number): string {
+  const type = PG_TYPES.find((type) => data_type_id === type.oid)
+    ? PG_TYPES.find((type) => data_type_id === type.oid)?.typname
+    : undefined;
+  if (!type) {
+    return "varchar";
+  }
+  return type;
+}