@dbml/connector 3.7.4 → 3.8.0

package/dist/connectors/snowflakeConnector.js

@@ -0,0 +1,292 @@
+"use strict";
+/* eslint-disable camelcase */
+//
+// Description:
+// Snowflake info schema lacks reference direction for relationships.
+// Constraints and uniqueness cannot be auto-generated from Snowflake info schema.
+// Snowflake does not support ENUMs.
+// Snowflake does not support indexes for standard tables.
+//
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fetchSchemaJson = void 0;
+const snowflake_sdk_1 = __importDefault(require("snowflake-sdk"));
+const parseSchema_1 = require("../utils/parseSchema");
+const parseConnectionString = (connectionString) => {
+    const params = {};
+    const regex = /([^;=]+)=([^;]*)/g;
+    let match;
+    while ((match = regex.exec(connectionString)) !== null) {
+        params[match[1].trim()] = match[2].trim();
+    }
+    return params;
+};
+const connect = async (connection) => {
+    return new Promise((resolve, reject) => {
+        console.log('Attempting to connect...');
+        connection.connect((err) => {
+            if (err) {
+                reject(err);
+            }
+            else {
+                resolve();
+            }
+        });
+    });
+};
+const executeQuery = (connection, sqlText) => {
+    return new Promise((resolve, reject) => {
+        console.log('Executing query...');
+        connection.execute({
+            sqlText: sqlText,
+            complete: (err, stmt, rows) => {
+                if (err) {
+                    reject(err);
+                }
+                else {
+                    console.log('Query executed successfully.');
+                    if (rows) {
+                        resolve(rows);
+                    }
+                }
+            }
+        });
+    });
+};
+const isLogLevel = (value) => {
+    return ['ERROR', 'WARN', 'INFO', 'DEBUG', 'TRACE'].includes(value);
+};
+const connectToSnowflake = async (config) => {
+    const logLevel = isLogLevel(config.LOG_LEVEL) ? config.LOG_LEVEL : 'INFO';
+    const isDebugMode = config.IS_DEBUG_MODE === 'true';
+    snowflake_sdk_1.default.configure({
+        logLevel,
+        additionalLogToConsole: isDebugMode,
+    });
+    // "SERVER=myaccount.snowflakecomputing.com;UID=myusername;PWD=mypassword;DATABASE=mydatabase;WAREHOUSE=mywarehouse;ROLE=myrole";
+    const connection = snowflake_sdk_1.default.createConnection({
+        account: config.SERVER,
+        username: config.UID,
+        password: config.PWD,
+        database: config.DATABASE,
+        warehouse: config.WAREHOUSE,
+        sfRetryMaxLoginRetries: 3,
+        timeout: 10000,
+    });
+    try {
+        // Connect to Snowflake
+        await connect(connection);
+        // Execute the query
+        await executeQuery(connection, 'SELECT CURRENT_VERSION();');
+        return connection;
+    }
+    catch (err) {
+        throw err;
+    }
+};
+const convertQueryBoolean = (val) => val === 'YES';
+const getFieldType = (data_type, character_maximum_length, numeric_precision, numeric_scale) => {
+    if (character_maximum_length) {
+        return `${data_type}(${character_maximum_length})`;
+    }
+    if (numeric_precision && numeric_scale) {
+        return `${data_type}(${numeric_precision},${numeric_scale})`;
+    }
+    return data_type;
+};
+const getDbdefault = (column_default, default_type) => {
+    if (default_type === 'string') {
+        return {
+            type: default_type,
+            value: column_default.slice(1, -1),
+        };
+    }
+    return {
+        type: default_type,
+        value: column_default,
+    };
+};
+const generateField = (row) => {
+    const { COLUMN_NAME: column_name, DATA_TYPE: data_type, CHARACTER_MAXIMUM_LENGTH: character_maximum_length, NUMERIC_PRECISION: numeric_precision, NUMERIC_SCALE: numeric_scale, IDENTITY_INCREMENT: identity_increment, IS_NULLABLE: is_nullable, COLUMN_DEFAULT: column_default, DEFAULT_TYPE: default_type, COLUMN_COMMENT: column_comment, } = row;
+    const dbdefault = column_default ? getDbdefault(column_default, default_type) : null;
+    const fieldType = {
+        type_name: getFieldType(data_type, character_maximum_length, numeric_precision, numeric_scale),
+        schemaName: null,
+    };
+    return {
+        name: column_name,
+        type: fieldType,
+        dbdefault,
+        not_null: !convertQueryBoolean(is_nullable),
+        increment: !!identity_increment,
+        note: column_comment ? { value: column_comment } : { value: '' },
+    };
+};
+const generateTablesAndFields = async (conn, schemas) => {
+    const fields = {};
+    const schemaSql = schemas.length > 0 ? `AND c.table_schema IN (${schemas.map((schema) => `'${schema}'`).join(',')})` : '';
+    const tablesAndFieldsSql = `
+    SELECT
+        c.table_schema,
+        c.table_name,
+        c.column_name,
+        c.data_type,
+        c.character_maximum_length,
+        c.numeric_precision,
+        c.numeric_scale,
+        c.is_nullable,
+        c.column_default,
+        CASE
+            WHEN c.column_default IS NULL THEN NULL
+            WHEN c.column_default LIKE '''%' THEN 'string'
+            WHEN LOWER(c.column_default) = 'true' OR LOWER(c.column_default) = 'false' THEN 'boolean'
+            WHEN c.column_default REGEXP '^-?[0-9]+(\\.[0-9]+)?$' THEN 'number'
+            ELSE 'expression'
+        END AS default_type,
+        t.comment AS table_comment,
+        c.comment AS column_comment,
+        c.identity_increment
+    FROM
+        information_schema.columns c
+        JOIN information_schema.tables t
+            ON c.table_name = t.table_name
+            AND c.table_schema = t.table_schema
+    WHERE
+        t.table_type = 'BASE TABLE'
+        AND t.table_schema NOT IN ('INFORMATION_SCHEMA')
+        ${schemaSql}
+    ORDER BY
+        c.table_schema,
+        c.table_name,
+        c.ordinal_position;
+  `;
+    const tablesAndFieldsResult = await executeQuery(conn, tablesAndFieldsSql);
+    const tables = tablesAndFieldsResult.reduce((acc, row) => {
+        const { TABLE_SCHEMA: table_schema, TABLE_NAME: table_name, TABLE_COMMENT: table_comment } = row;
+        const key = `${table_schema}.${table_name}`;
+        if (!acc[key]) {
+            acc[key] = {
+                name: table_name,
+                schemaName: table_schema,
+                note: table_comment ? { value: table_comment } : { value: '' },
+            };
+        }
+        if (!fields[key])
+            fields[key] = [];
+        const field = generateField(row);
+        fields[key].push(field);
+        return acc;
+    }, {});
+    return {
+        tables: Object.values(tables),
+        fields,
+    };
+};
+const createConstraintKeysMap = (keys, schemas, constraintType) => {
+    return keys.reduce((acc, row) => {
+        const { schema_name, table_name, column_name, constraint_name } = row;
+        const selectedSchema = schemas.length > 0 ? schemas.includes(schema_name) : true;
+        if (!selectedSchema) {
+            return acc;
+        }
+        const key = `${schema_name}.${table_name}.${constraint_name}`;
+        if (acc[key]) {
+            const columnNames = acc[key].columnNames;
+            columnNames.push(column_name);
+            return acc;
+        }
+        acc[key] = {
+            schemaName: schema_name,
+            tableName: table_name,
+            constraintName: constraint_name,
+            columnNames: [column_name],
+            type: '',
+        };
+        acc[key][constraintType] = true;
+        return acc;
+    }, {});
+};
+const generateIndexes = async (conn, databaseName, schemas) => {
+    const getPrimaryKeysSql = `
+    SHOW PRIMARY KEYS IN DATABASE ${databaseName};
+  `;
+    const getUniqueKeysSql = `
+    SHOW UNIQUE KEYS IN DATABASE ${databaseName};
+  `;
+    const primaryKeys = await executeQuery(conn, getPrimaryKeysSql);
+    const uniqueKeys = await executeQuery(conn, getUniqueKeysSql);
+    const primaryKeysByConstraint = createConstraintKeysMap(primaryKeys, schemas, 'primary');
+    const uniqueKeysByConstraint = createConstraintKeysMap(uniqueKeys, schemas, 'unique');
+    const allConstraints = [Object.values(primaryKeysByConstraint), Object.values(uniqueKeysByConstraint)].flat();
+    const { indexes, tableConstraints } = allConstraints.reduce((acc, row) => {
+        const { schemaName, tableName, constraintName, columnNames, type, primary, unique } = row;
+        const key = `${schemaName}.${tableName}`;
+        if (columnNames.length < 2) {
+            acc.tableConstraints[key] = {
+                [columnNames[0]]: {
+                    pk: primary,
+                    unique,
+                },
+            };
+            return acc;
+        }
+        const index = {
+            name: constraintName,
+            type,
+            unique,
+            pk: primary,
+            columns: columnNames.map((columnName) => ({
+                type: 'column',
+                value: columnName,
+            })),
+        };
+        if (acc.indexes[key]) {
+            acc.indexes[key].push(index);
+        }
+        else {
+            acc.indexes[key] = [index];
+        }
+        return acc;
+    }, { indexes: {}, tableConstraints: {} });
+    return {
+        indexes,
+        tableConstraints,
+    };
+};
+const fetchSchemaJson = async (connection) => {
+    const config = parseConnectionString(connection);
+    const conn = await connectToSnowflake(config);
+    if (conn instanceof Error) {
+        throw conn;
+    }
+    // Schemas: schema1,schema2,schema3
+    const schemas = config.SCHEMAS ? (0, parseSchema_1.parseSchema)(config.SCHEMAS) : [];
+    const databaseName = config.DATABASE;
+    const tablesAndFieldsRes = generateTablesAndFields(conn, schemas);
+    const indexesRes = generateIndexes(conn, databaseName, schemas);
+    const res = await Promise.all([
+        tablesAndFieldsRes,
+        indexesRes,
+    ]);
+    conn.destroy((err) => {
+        if (err) {
+            throw err;
+        }
+        else {
+            console.log('Connection destroyed successfully.');
+        }
+    });
+    const { tables, fields } = res[0];
+    const { indexes, tableConstraints } = res[1];
+    return {
+        tables,
+        fields,
+        refs: [],
+        enums: [],
+        indexes,
+        tableConstraints,
+    };
+};
+exports.fetchSchemaJson = fetchSchemaJson;

package/dist/connectors/types.d.ts

@@ -16,6 +16,8 @@ interface Field {
     dbdefault: DefaultInfo | null;
     not_null: boolean;
     increment: boolean;
+    pk?: boolean;
+    unique?: boolean;
     note: NoteInfo;
 }
 interface Table {
@@ -77,4 +79,12 @@ interface DatabaseSchema {
     refs: Ref[];
     indexes: IndexesDictionary;
 }
-export { NoteInfo, TypeInfo, DefaultType, DefaultInfo, Field, Table, FieldsDictionary, TableConstraint, TableConstraintsDictionary, RefEndpoint, Ref, EnumValue, Enum, IndexColumn, Index, IndexesDictionary, DatabaseSchema, };
+interface BigQueryCredentials {
+    projectId: string;
+    credentials: {
+        clientEmail: string;
+        privateKey: string;
+    };
+    datasets: string[];
+}
+export { NoteInfo, TypeInfo, DefaultType, DefaultInfo, Field, Table, FieldsDictionary, TableConstraint, TableConstraintsDictionary, RefEndpoint, Ref, EnumValue, Enum, IndexColumn, Index, IndexesDictionary, DatabaseSchema, BigQueryCredentials, };

package/dist/utils/credential-loader.d.ts

@@ -0,0 +1,3 @@
+import { BigQueryCredentials } from '../connectors/types';
+export declare function loadCredentialFromFile(filename: string): Promise<string>;
+export declare function parseBigQueryCredential(credentialString: string): BigQueryCredentials;

package/dist/utils/credential-loader.js

@@ -0,0 +1,53 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadCredentialFromFile = loadCredentialFromFile;
+exports.parseBigQueryCredential = parseBigQueryCredential;
+const promises_1 = require("fs/promises");
+async function loadCredentialFromFile(filename) {
+    try {
+        const data = await (0, promises_1.readFile)(filename, { encoding: 'utf8' });
+        return data;
+    }
+    catch (error) {
+        if (error.name === 'TypeError' || error.name === 'Error') {
+            throw new Error(`Load credential error: ${error.message}`);
+        }
+        throw error;
+    }
+}
+function parseBigQueryCredential(credentialString) {
+    try {
+        const credentialJson = JSON.parse(credentialString);
+        const { project_id: projectId, client_email: clientEmail, private_key: privateKey, datasets, } = credentialJson;
+        if (typeof projectId !== 'string' || !projectId || !projectId.trim()) {
+            throw new Error('project_id must be a non-empty string');
+        }
+        if (typeof clientEmail !== 'string' || !clientEmail || !clientEmail.trim()) {
+            throw new Error('client_email must be a non-empty string');
+        }
+        if (typeof privateKey !== 'string' || !privateKey || !privateKey.trim()) {
+            throw new Error('private_key must be a non-empty string');
+        }
+        if (typeof clientEmail !== 'string' || !clientEmail || !clientEmail.trim()) {
+            throw new Error('client_email must be a non-empty string');
+        }
+        // valid datasets: ['dataset_1', 'dataset_2']
+        const parsedDatasets = !Array.isArray(datasets)
+            ? []
+            : datasets.filter((dataset) => typeof dataset === 'string');
+        return {
+            projectId,
+            credentials: {
+                clientEmail,
+                privateKey,
+            },
+            datasets: parsedDatasets,
+        };
+    }
+    catch (error) {
+        if (error instanceof SyntaxError) {
+            throw new Error('Credentials must be in JSON format');
+        }
+        throw error;
+    }
+}

package/dist/utils/helpers.d.ts

@@ -0,0 +1,7 @@
+import { FieldsDictionary, IndexesDictionary, Table, TableConstraintsDictionary } from '../connectors/types';
+export declare function getIntersection<T>(firstList: T[], secondList: T[]): T[];
+export declare function getTableSchemaKey(schemaName: string, tableName: string): string;
+export declare function mergeTables(firstTableList: Table[], secondTable: Table[]): Table[];
+export declare function mergeFieldDictionary(firstDict: FieldsDictionary, secondDict: FieldsDictionary): FieldsDictionary;
+export declare function mergeIndexDictionary(firstDict: IndexesDictionary, secondDict: IndexesDictionary): IndexesDictionary;
+export declare function mergeTableConstraintDictionary(firstDict: TableConstraintsDictionary, secondDict: TableConstraintsDictionary): TableConstraintsDictionary;

package/dist/utils/helpers.js

@@ -0,0 +1,50 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getIntersection = getIntersection;
+exports.getTableSchemaKey = getTableSchemaKey;
+exports.mergeTables = mergeTables;
+exports.mergeFieldDictionary = mergeFieldDictionary;
+exports.mergeIndexDictionary = mergeIndexDictionary;
+exports.mergeTableConstraintDictionary = mergeTableConstraintDictionary;
+function getIntersection(firstList, secondList) {
+    return firstList.filter((item) => secondList.includes(item));
+}
+function getTableSchemaKey(schemaName, tableName) {
+    if (!tableName || !tableName.trim()) {
+        throw new Error('Table name must be a non-empty string');
+    }
+    return schemaName ? `${schemaName}.${tableName}` : `${tableName}`;
+}
+function mergeTables(firstTableList, secondTable) {
+    return firstTableList.concat(secondTable);
+}
+function mergeFieldDictionary(firstDict, secondDict) {
+    const result = Object.assign({}, firstDict);
+    Object.keys(secondDict).forEach((key) => {
+        if (!result[key]) {
+            result[key] = [];
+        }
+        result[key] = result[key].concat(secondDict[key]);
+    });
+    return result;
+}
+function mergeIndexDictionary(firstDict, secondDict) {
+    const result = Object.assign({}, firstDict);
+    Object.keys(secondDict).forEach((key) => {
+        if (!result[key]) {
+            result[key] = [];
+        }
+        result[key] = result[key].concat(secondDict[key]);
+    });
+    return result;
+}
+function mergeTableConstraintDictionary(firstDict, secondDict) {
+    const result = Object.assign({}, firstDict);
+    Object.keys(secondDict).forEach((key) => {
+        if (!result[key]) {
+            result[key] = {};
+        }
+        result[key] = secondDict[key];
+    });
+    return result;
+}

package/dist/utils/parseSchema.d.ts

@@ -0,0 +1,10 @@
+export declare const parseJdbcSchema: (connectionString: string) => {
+    connectionString: string;
+    schemas: string[];
+};
+export declare const parseConnectionString: (connectionString: string, connectionStringType: "jdbc" | "odbc") => {
+    connectionString: string;
+    schemas: string[];
+};
+export declare const buildSchemaQuery: (columnName: string, schemas: string[], prefix?: string) => string;
+export declare const parseSchema: (schemas: string | undefined) => string[];

package/dist/utils/parseSchema.js

@@ -0,0 +1,59 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseSchema = exports.buildSchemaQuery = exports.parseConnectionString = exports.parseJdbcSchema = void 0;
+// You cannot use the "=" inside database's schema name. It will be treated as a delimiter.
+const SCHEMAS_DELIMITER = 'schemas=';
+const haveSchemas = (str) => {
+    return str.toLowerCase().includes(SCHEMAS_DELIMITER);
+};
+const getSchemasPart = (str) => {
+    return str ? str.toLowerCase().split(SCHEMAS_DELIMITER)[1] : '';
+};
+const noSchemaResult = (connectionString) => ({
+    connectionString,
+    schemas: [],
+});
+const parseOdbcSchema = (connectionString) => {
+    const connectionParts = connectionString.split(';');
+    const schemasPart = connectionParts.find((part) => haveSchemas(part));
+    return {
+        connectionString: connectionParts.filter((part) => !haveSchemas(part)).join(';'),
+        schemas: (0, exports.parseSchema)(getSchemasPart(schemasPart)),
+    };
+};
+const parseJdbcSchema = (connectionString) => {
+    const connectionParts = connectionString.split('?');
+    const connectionArgs = connectionParts[1].split('&');
+    const schemasPart = connectionArgs.find((part) => haveSchemas(part));
+    return {
+        connectionString: connectionParts[0],
+        schemas: (0, exports.parseSchema)(getSchemasPart(schemasPart)),
+    };
+};
+exports.parseJdbcSchema = parseJdbcSchema;
+const parseConnectionString = (connectionString, connectionStringType) => {
+    if (!haveSchemas(connectionString))
+        return noSchemaResult(connectionString);
+    return connectionStringType === 'jdbc'
+        ? (0, exports.parseJdbcSchema)(connectionString)
+        : parseOdbcSchema(connectionString);
+};
+exports.parseConnectionString = parseConnectionString;
+const buildSchemaQuery = (columnName, schemas, prefix = 'AND') => {
+    if (schemas.length === 0)
+        return '';
+    return `${prefix} ${columnName} IN (${schemas.map((schema) => `'${schema}'`).join(', ')})`;
+};
+exports.buildSchemaQuery = buildSchemaQuery;
+// Inputs: schemas string -> Output: schema array string[]
+// schema1, schema2 -> ['schema1', 'schema2']
+// schema3,schema4 -> ['schema3", 'schema4']
+// "schema5,gg", "schema6", schema7 -> ['schema5,gg', 'schema6', 'schema7']
+// schema8 -> ['schema8']
+const parseSchema = (schemas) => {
+    if (!schemas)
+        return [];
+    const schemasWithDoubleQuotes = schemas.split(/,(?=(?:[^"]*(?:"[^"]*")*)*$)/).map((s) => s.trim());
+    return schemasWithDoubleQuotes.map((s) => s.replace(/"/g, ''));
+};
+exports.parseSchema = parseSchema;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dbml/connector",
-  "version": "3.7.4",
+  "version": "3.8.0",
   "description": "This package was created to fetch the schema JSON from many kind of databases.",
   "author": "huy.phung.sw@gmail.com",
   "license": "MIT",
@@ -31,12 +31,14 @@
     "test": "jest"
   },
   "dependencies": {
+    "@google-cloud/bigquery": "^7.9.0",
     "mssql": "^11.0.1",
     "mysql2": "^3.11.0",
-    "pg": "^8.12.0"
+    "pg": "^8.12.0",
+    "snowflake-sdk": "^1.12.0"
   },
   "engines": {
     "node": ">=18"
   },
-  "gitHead": "316fd7ce4039ce9b622c30fbb2d03ab2198bd7eb"
+  "gitHead": "f106540bc455695c8a4c91ed63aa8fabc8ce645a"
 }