@quillsql/node 0.3.7 → 0.4.0

package/src/index.ts

@@ -5,14 +5,22 @@ import {
   QuillClientResponse,
   QuillQueryParams,
 } from "./models/Quill";
-import { CachedPool } from "./db/CachedPools";
+import { CachedConnection } from "./db/CachedConnection";
 import axios from "axios";
 import "dotenv/config";
+import { mapQueries, removeFields } from "./utils/RunQueryProcesses";
 import {
-  getTableSchema,
-  mapQueries,
-  removeFields,
-} from "./utils/RunQueryProcesses";
+  DatabaseType,
+  connectToDatabase,
+  getColumnInfoBySchemaByDatabase,
+  getColumnsByTableByDatabase,
+  getDatabaseCredentials,
+  getForiegnKeysByDatabase,
+  getSchemasByDatabase,
+  getTablesBySchemaByDatabase,
+  runQueryByDatabase,
+} from "./db/DatabaseHelper";
+import { convertTypeToPostgres } from "./utils/schemaConversion";
 
 const HOST =
   process.env.ENV === "development"
@@ -25,9 +33,7 @@ const HOST =
 
 export default class QuillClass {
   // Configure cached connection pools with the given config.
-  private connectionString;
-  private ssl = { rejectUnauthorized: false };
-  public targetPool;
+  public targetConnection;
   private baseUrl: string;
   private config: {
     headers: {
@@ -37,29 +43,50 @@ export default class QuillClass {
 
   constructor(
     privateKey: string,
-    databaseConnectionString: string,
-    cache: Partial<CacheCredentials> = {}
+    databaseType: DatabaseType,
+    databaseConnectionString?: string,
+    databaseCredentials?: any,
+    cache: Partial<CacheCredentials> = {},
+    metadataServerURL?: string
   ) {
-    this.baseUrl = HOST;
+    this.baseUrl = metadataServerURL ? metadataServerURL : HOST;
     this.config = { headers: { Authorization: `Bearer ${privateKey}` } };
-    this.connectionString = databaseConnectionString;
-    this.ssl = { rejectUnauthorized: false };
-    this.targetPool = new CachedPool(
-      { connectionString: this.connectionString, ssl: this.ssl },
+    let credentials = databaseCredentials;
+    if (databaseConnectionString) {
+      credentials = getDatabaseCredentials(
+        databaseType,
+        databaseConnectionString
+      );
+    }
+    this.targetConnection = new CachedConnection(
+      databaseType,
+      credentials,
       cache
     );
   }
 
   public async query({ orgId, metadata }: QuillQueryParams): Promise<any> {
-    this.targetPool.orgId = orgId;
-
+    this.targetConnection.orgId = orgId;
     try {
-      // Initial Query Request
-      const preQueryResults = await this.runQueries(metadata.preQueries);
+      const preQueryResults = metadata.preQueries
+        ? await this.runQueries(
+            metadata.preQueries,
+            this.targetConnection.databaseType,
+            metadata.databaseType,
+            metadata.runQueryConfig
+          )
+        : {};
+      if (metadata.runQueryConfig?.overridePost) {
+        return {
+          data: { queryResults: preQueryResults },
+          status: "success",
+        };
+      }
       const response = await this.postQuill(metadata.task, {
         ...metadata,
+        ...preQueryResults,
         orgId,
-        preQueryResults,
+        viewQuery: metadata.preQueries ? metadata.preQueries[0] : undefined,
       });
       // if there is no metadata object in the response, create one
       if (!response.metadata) {
@@ -67,6 +94,8 @@ export default class QuillClass {
       }
       const results = await this.runQueries(
         response.queries,
+        this.targetConnection.databaseType,
+        metadata.databaseType,
         response.metadata.runQueryConfig
       );
       // QUICK JANKY FIX TO UPDATE METADATA AFTER GETTING MAPPED ARRAYS
@@ -103,36 +132,87 @@ export default class QuillClass {
 
   private async runQueries(
     queries: string[] | undefined,
+    pkDatabaseType: DatabaseType,
+    databaseType?: string,
     runQueryConfig?: AdditionalProcessing
   ) {
     let results: any;
     if (!queries) return { ...results, queryResults: [] };
+    if (
+      databaseType &&
+      databaseType.toLowerCase() !== pkDatabaseType.toLowerCase()
+    ) {
+      return {
+        dbMismatched: true,
+        backendDatbaseType: pkDatabaseType,
+        queryResults: [],
+      };
+    }
     if (runQueryConfig?.arrayToMap) {
-      const mappedArray = await mapQueries(
-        queries,
-        runQueryConfig.arrayToMap,
-        this.targetPool
-      );
+      const mappedArray = await mapQueries(queries, this.targetConnection);
       return { ...results, queryResults: [], mappedArray };
+    } else if (runQueryConfig?.getColumns) {
+      const queryResult = await this.targetConnection.query(
+        `${queries[0]} limit 1`
+      );
+      const columns = queryResult.fields.map((field: any) => {
+        return {
+          fieldType: convertTypeToPostgres(field.dataTypeID),
+          name: field.name,
+          displayName: field.name,
+          isVisible: true,
+          field: field.name,
+        };
+      });
+      return { columns };
+    } else if (runQueryConfig?.getTables) {
+      const queryResult = await getTablesBySchemaByDatabase(
+        this.targetConnection.databaseType,
+        this.targetConnection.pool,
+        runQueryConfig.schemaNames! || runQueryConfig.schema
+      );
+      const schemaInfo = await getColumnInfoBySchemaByDatabase(
+        this.targetConnection.databaseType,
+        this.targetConnection.pool,
+        runQueryConfig.schema!,
+        queryResult!
+      );
+      return schemaInfo;
     } else {
+      if (runQueryConfig?.limitThousand) {
+        queries = queries.map((query) => {
+          return query.replace(/;/, "") + " limit 1000;";
+        });
+      }
       const queryResults = await Promise.all(
         queries.map(async (query) => {
-          return await this.targetPool.query(query);
+          return await this.targetConnection.query(query);
         })
       );
       results = { ...results, queryResults };
-      if (runQueryConfig?.getSchema) {
-        results = {
-          ...results,
-          columns: await getTableSchema(queryResults[0], this.targetPool),
-        };
-      }
       if (runQueryConfig?.removeFields) {
         results = {
           ...results,
           queryResults: removeFields(queryResults, runQueryConfig.removeFields),
         };
       }
+      if (runQueryConfig?.convertDatatypes) {
+        results = queryResults.map((result) => {
+          return {
+            fields: result.fields.map((field: any) => {
+              return {
+                ...field,
+                fieldType: convertTypeToPostgres(field.dataTypeID),
+                isVisible: true,
+                field: field.name,
+                displayName: field.name,
+                name: field.name,
+              };
+            }),
+            rows: result.rows,
+          };
+        });
+      }
     }
     return results;
   }
@@ -150,22 +230,44 @@ export default class QuillClass {
   }
 
   public async close() {
-    await this.targetPool.close();
+    await this.targetConnection.close();
   }
 }
 
 const Quill = ({
   privateKey,
   databaseConnectionString,
+  databaseConfig,
   cache,
+  databaseType,
+  metadataServerURL,
 }: {
   privateKey: string;
   databaseConnectionString: string;
   cache?: Partial<CacheCredentials>;
+  databaseType: DatabaseType;
+  databaseConfig: any;
+  metadataServerURL?: string;
 }) => {
-  return new QuillClass(privateKey, databaseConnectionString, cache);
+  return new QuillClass(
+    privateKey,
+    databaseType,
+    databaseConnectionString,
+    databaseConfig,
+    cache,
+    metadataServerURL
+  );
 };
 
 module.exports = Quill;
 module.exports.Quill = Quill;
 module.exports.default = Quill;
+module.exports.getTablesBySchemaByDatabase = getTablesBySchemaByDatabase;
+module.exports.getDatabaseCredentials = getDatabaseCredentials;
+module.exports.getColumnsByTableByDatabase = getColumnsByTableByDatabase;
+module.exports.getForiegnKeysByDatabase = getForiegnKeysByDatabase;
+module.exports.getSchemasByDatabase = getSchemasByDatabase;
+module.exports.getColumnInfoBySchemaByDatabase =
+  getColumnInfoBySchemaByDatabase;
+module.exports.connectToDatabase = connectToDatabase;
+module.exports.runQueryByDatabase = runQueryByDatabase;

package/src/index.uspec.ts

@@ -1,4 +1,5 @@
 import Quill from ".";
+import { DatabaseType } from "./db/DatabaseHelper";
 
 jest.mock(".");
 
@@ -6,8 +7,14 @@ describe("Quill", () => {
   let quill: Quill;
 
   beforeEach(() => {
-    quill = new Quill("dummy_private_key", "dummy_db_url");
-    quill.targetPool.query = jest.fn().mockResolvedValue([]);
+    quill = new Quill(
+      "dummy_private_key",
+      DatabaseType.postgres,
+      "dummy_db_url",
+      {},
+      undefined
+    );
+    quill.targetConnection.query = jest.fn().mockResolvedValue([]);
   });
 
   describe("query", () => {

package/src/models/Client.ts

@@ -0,0 +1,29 @@
+export interface Client {
+  name: string;
+  databaseConnectionString: string;
+  etlDatabaseConnectionString: string;
+  stagingDatabaseConnectionString: string;
+  customerTableTitleFieldName: string;
+  databaseType: string;
+  // foreign key
+  customerFieldName: string;
+  // native key, ex: uuid
+  customerTableFieldName: string;
+  // table with all customers, ex: company_company
+  customerTableName: string;
+  customerView: string;
+  // sql type. ex: INT
+  customerFieldType: string;
+  // does db use ssl
+  useSsl: boolean;
+  serverCa: string;
+  clientCert: string;
+  clientKey: string;
+  defaultQuery: string;
+  ignoreDarkMode: boolean;
+  domainName: string;
+  hideSqlEditor: boolean;
+  adminCustomerId: string;
+  stagingAdminCustomerId: string;
+  cacheCloudConfig: { type: { cacheQueries: boolean }; default: null };
+}

package/src/models/Quill.ts

@@ -1,4 +1,3 @@
-import { CachedPool } from "../db/CachedPools";
 import { CacheCredentials } from "./Cache";
 import { DatabaseCredentials } from "./Database";
 import { FieldFormat, FormattedColumn } from "./Formats";
@@ -8,6 +7,7 @@ export interface QuillRequestMetadata {
   // a query to be run
   queries?: string[];
   preQueries?: string[];
+  runQueryConfig?: AdditionalProcessing;
   query?: string;
   // a report to be fetched
   id?: string;
@@ -26,6 +26,7 @@ export interface QuillRequestMetadata {
   template?: boolean;
   clientId?: string;
   deleted?: boolean;
+  databaseType?: string
 }
 
 export interface QuillQueryParams {
@@ -34,11 +35,6 @@ export interface QuillQueryParams {
   environment?: string;
 }
 
-export interface QuillTaskHandlerParams extends QuillQueryParams {
-  privateKey: string;
-  targetPool: CachedPool;
-}
-
 export interface QuillConfig {
   privateKey: string;
   db: Partial<DatabaseCredentials>;
@@ -57,8 +53,16 @@ export interface QuillConfig {
 
 export interface AdditionalProcessing {
   getSchema?: boolean;
+  getColumns?: boolean;
+  getTables?: boolean;
+  schema?: string,
+  schemaNames?: string[];
+  table?: string,
   removeFields?: string[];
   arrayToMap?: { arrayName: string; field: string };
+  overridePost?: boolean;
+  convertDatatypes?: boolean
+  limitThousand?: boolean;
 }
 
 export interface QuillClientResponse {

package/src/utils/RunQueryProcesses.ts

@@ -1,4 +1,4 @@
-import { CachedPool } from "../db/CachedPools";
+import { CachedConnection } from "../db/CachedConnection";
 
 interface TableSchemaInfo {
   fieldType: string;
@@ -7,28 +7,6 @@ interface TableSchemaInfo {
   isVisible: boolean;
 }
 
-export async function getTableSchema(
-  queryResults: any,
-  targetPool: CachedPool
-) {
-  const typesQuery = await targetPool.query(
-    "select typname, oid, typarray from pg_type order by oid;"
-  );
-  const schema: TableSchemaInfo[] = queryResults[0].fields.map(
-    (field: { dataTypeID: any; name: any }) => {
-      return {
-        fieldType: typesQuery.rows.filter(
-          (type: { oid: any }) => field.dataTypeID === type.oid
-        )[0].typname,
-        name: field.name,
-        displayName: field.name,
-        isVisible: true,
-      };
-    }
-  );
-  return schema;
-}
-
 export function removeFields(queryResults: any, fieldsToRemove: string[]): any {
   const fields = queryResults.fields.filter((field: { name: any }) =>
     fieldsToRemove.includes(field.name)
@@ -43,12 +21,11 @@ export function removeFields(queryResults: any, fieldsToRemove: string[]): any {
 
 export async function mapQueries(
   queries: string[],
-  arrayToMap: { arrayName: string; field: string },
-  targetPool: CachedPool
+  targetConnection: CachedConnection
 ): Promise<any[]> {
   const mappedArray = [];
   for (let i = 0; i < queries.length; i++) {
-    const queryResult = await targetPool.query(queries[i]);
+    const queryResult = await targetConnection.query(queries[i]);
     mappedArray.push(queryResult.rows);
   }
   return mappedArray;

package/src/utils/schemaConversion.ts

@@ -0,0 +1,11 @@
+import { PG_TYPES } from "../assets/pgtypes";
+
+export function convertTypeToPostgres(data_type_id: number): string {
+  const type = PG_TYPES.find((type) => data_type_id === type.oid)
+    ? PG_TYPES.find((type) => data_type_id === type.oid)?.typname
+    : undefined;
+  if (!type) {
+    return "varchar";
+  }
+  return type;
+}

package/src/utils/textProcessing.ts

@@ -0,0 +1,13 @@
+export function capitalize(text: string): string {
+  return text.charAt(0).toUpperCase() + text.slice(1);
+}
+
+export function depluralize(text: string): string {
+  if (text.endsWith("ies")) {
+    return text.slice(0, -3) + "y";
+  }
+  if (text.endsWith("s")) {
+    return text.slice(0, -1);
+  }
+  return text;
+}