@dbml/connector 5.5.1 → 5.6.0-alpha.0

package/eslint.config.ts

@@ -19,7 +19,7 @@ export default defineConfig(
         'node_modules/*',
         'dist/*',
         'eslint.config.ts',
-        'jest.config.ts',
+        'vite.config.ts',
         '**/*.test.ts',
         '__tests__/**/*',
       ],
@@ -32,6 +32,7 @@ export default defineConfig(
           sourceType: 'module',
           ecmaVersion: 2020,
           project: './tsconfig.json',
+          tsconfigRootDir: import.meta.dirname,
         },
       },
       plugins: {

package/package.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://json.schemastore.org/package",
   "name": "@dbml/connector",
-  "version": "5.5.1",
+  "version": "5.6.0-alpha.0",
   "description": "This package was created to fetch the schema JSON from many kind of databases.",
   "author": "huy.phung.sw@gmail.com",
   "license": "MIT",
@@ -10,42 +10,25 @@
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "devDependencies": {
-    "@babel/core": "^7.21.4",
-    "@babel/node": "^7.20.7",
-    "@babel/plugin-transform-runtime": "^7.21.4",
-    "@babel/preset-env": "^7.21.4",
-    "@babel/preset-typescript": "^7.24.7",
-    "@babel/runtime": "^7.21.0",
-    "@stylistic/eslint-plugin": "^5.5.0",
-    "@types/jest": "^29.5.12",
     "@types/lodash": "^4.17.16",
     "@types/mssql": "^9.1.5",
     "@types/node": "^22.2.0",
     "@types/oracledb": "^6.10.0",
     "@types/pg": "^8.11.6",
-    "@typescript-eslint/eslint-plugin": "^8.46.3",
-    "@typescript-eslint/parser": "^8.46.3",
-    "babel-jest": "^29.5.0",
-    "eslint": "^9.39.1",
-    "jest": "^29.7.0",
-    "ts-jest": "^29.2.4",
-    "ts-node": "^10.9.2",
-    "typescript": "^5.9.3",
-    "typescript-eslint": "^8.46.3"
+    "ts-node": "^10.9.2"
   },
   "scripts": {
-    "build": "tsc",
+    "build": "rm -rf ./dist && vite build",
+    "dev": "vite build --watch",
     "start": "node dist/index.js",
-    "test": "jest --coverage=false",
-    "coverage": "jest --coverage",
+    "test:watch": "vitest tests",
+    "test": "vitest",
+    "coverage": "vitest --coverage",
     "lint": "eslint .",
     "lint:fix": "eslint --fix ."
   },
   "dependencies": {
     "@google-cloud/bigquery": "^7.9.0",
-    "@typescript-eslint/eslint-plugin": "^8.46.3",
-    "@typescript-eslint/parser": "^8.46.3",
-    "eslint": "^9.39.1",
     "lodash": "^4.17.21",
     "mssql": "^11.0.1",
     "mysql2": "^3.11.0",
@@ -56,5 +39,5 @@
   "engines": {
     "node": ">=18"
   },
-  "gitHead": "4519233511e253390842ffe363dd881e3413fd0a"
+  "gitHead": "e0cd0b00099344c05af0a83e30c2c9c2ccc325cc"
 }

package/vite.config.ts

@@ -0,0 +1,53 @@
+/// <reference types="vitest" />
+
+import path from 'path';
+import { defineConfig } from 'vite';
+import dts from 'vite-plugin-dts';
+
+export default defineConfig({
+  plugins: [
+    dts({ insertTypesEntry: true }),
+  ],
+  resolve: {
+    alias: {
+      "@": path.resolve(__dirname, "src/"),
+    },
+  },
+  build: {
+    target: 'node18',
+    lib: {
+      entry: path.resolve(__dirname, 'src/index.ts'),
+      fileName: 'index',
+      formats: ['cjs', 'es'],
+    },
+    rollupOptions: {
+      external: [
+        '@google-cloud/bigquery',
+        'lodash',
+        'mssql',
+        'mysql2',
+        'oracledb',
+        'pg',
+        'snowflake-sdk',
+        'fs',
+        'fs/promises',
+        'path',
+        'crypto',
+        'stream',
+        'util',
+        'timers',
+        'zlib',
+        'tls',
+        'url',
+        'net',
+      ],
+    },
+  },
+  test: {
+    globals: true,
+    coverage: {
+      provider: 'v8',
+      reporter: ['json', 'json-summary', 'html', 'text'],
+    },
+  },
+});

package/.babelrc

@@ -1,13 +0,0 @@
-{
-  "presets": [
-    "@babel/preset-env",
-    "@babel/preset-typescript"
-  ],
-  "env": {
-    "test": {
-      "plugins": [
-        "@babel/plugin-transform-runtime"
-      ]
-    }
-  }
-}

package/dist/connectors/bigquery/index.js

@@ -1,291 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.fetchSchemaJson = fetchSchemaJson;
-const bigquery_1 = require("@google-cloud/bigquery");
-const credential_loader_1 = require("../../utils/credential-loader");
-const helpers_1 = require("../../utils/helpers");
-// BigQuery client authentication strategy:
-// 1. Start with environment/default authentication:
-//    - GOOGLE_APPLICATION_CREDENTIALS env variable (service account key file)
-//    - Application Default Credentials (gcloud auth application-default login)
-//
-// 2. Override with explicitly provided properties:
-//    - If projectId is provided, override the default project
-//    - If credentials (clientEmail + privateKey) provided, override auth method
-//    - If datasets specified in config, fetch only those datasets
-async function connectBigQuery(credential) {
-    const config = {};
-    // Override project if explicitly provided
-    if (credential?.projectId) {
-        config.projectId = credential.projectId;
-    }
-    // Override credentials if both email and key are provided
-    if (credential?.credentials?.clientEmail && credential?.credentials?.privateKey) {
-        config.credentials = {
-            client_email: credential.credentials.clientEmail,
-            private_key: credential.credentials.privateKey,
-        };
-    }
-    // Create client with merged config
-    const client = new bigquery_1.BigQuery(config);
-    try {
-        const query = 'SELECT CURRENT_DATE() as today';
-        await client.query(query);
-        return client;
-    }
-    catch (error) {
-        if (error instanceof Error) {
-            throw new Error(`BigQuery connection error: ${error.message}`);
-        }
-        throw error;
-    }
-}
-function getDbDefault(columnDefault, defaultValueType) {
-    if (defaultValueType === null) {
-        return null;
-    }
-    if (defaultValueType === 'string') {
-        // bigquery store the double qoutes and quotes at the end and the beginning of the string: "string" or 'string'
-        return { value: columnDefault.slice(1, -1).replaceAll('\'', '\''), type: defaultValueType };
-    }
-    return { value: columnDefault, type: defaultValueType };
-}
-function generateField(row) {
-    const { columnName, isNullable, dataType, columnDefault, columnDefaultType, columnDescription, } = row;
-    const fieldDefaultValue = getDbDefault(columnDefault, columnDefaultType);
-    const field = {
-        name: columnName,
-        not_null: isNullable === 'NO',
-        type: {
-            type_name: dataType,
-            schemaName: null,
-        },
-        note: {
-            value: columnDescription || '',
-        },
-        dbdefault: fieldDefaultValue,
-        increment: false, // bigquery does not support native auto-increment
-    };
-    return field;
-}
-async function generateTablesAndFields(client, projectId, datasetId) {
-    const tableNameToFetch = `${projectId}.${datasetId}`;
-    const query = `
-    with
-      table_comments as (
-        select table_schema, table_name, option_value as description
-        from ${tableNameToFetch}.INFORMATION_SCHEMA.TABLE_OPTIONS
-        where option_name = 'description'
-      ),
-      column_comments as (
-        select column_name, table_schema, table_name, description
-        from ${tableNameToFetch}.INFORMATION_SCHEMA.COLUMN_FIELD_PATHS
-        where column_name = field_path
-      )
-    select
-      t.table_schema as tableSchema,
-      t.table_name as tableName,
-      (select description from table_comments tc where tc.table_name = t.table_name and tc.table_schema = t.table_schema) as tableDescription,
-      (
-        select description
-        from column_comments cc
-        where cc.table_name = c.table_name and cc.table_schema = c.table_schema and cc.column_name = c.column_name
-      ) as columnDescription,
-      c.column_name as columnName,
-      c.is_nullable as isNullable,
-      c.data_type as dataType,
-      c.column_default as columnDefault,
-      CASE
-        WHEN SAFE_CAST(c.column_default AS FLOAT64) IS NOT NULL THEN 'number'
-        WHEN SAFE_CAST(c.column_default AS BOOLEAN) IS NOT NULL THEN 'boolean'
-        -- string value is enclosed in single quotes, double quotes
-        WHEN REGEXP_CONTAINS(c.column_default, r"^['\\"].*['\\"]$") THEN 'string'
-        WHEN c.column_default = "NULL" THEN NULL
-        ELSE 'expression'
-      END AS columnDefaultType
-    from ${tableNameToFetch}.INFORMATION_SCHEMA.TABLES t
-    join ${tableNameToFetch}.INFORMATION_SCHEMA.COLUMNS c
-    on
-      t.table_schema = c.table_schema
-      and t.table_name = c.table_name
-    where t.table_type = 'BASE TABLE'
-    order by t.creation_time, c.ordinal_position;
-  `;
-    const [queryResult] = await client.query({ query });
-    const tableMap = {};
-    const fieldMap = {};
-    queryResult.forEach((row) => {
-        const { tableSchema, tableName, tableComment } = row;
-        const key = (0, helpers_1.getTableSchemaKey)(tableSchema, tableName);
-        if (!tableMap[key]) {
-            tableMap[key] = {
-                name: tableName,
-                note: { value: tableComment || '' },
-                schemaName: tableSchema,
-            };
-        }
-        if (!fieldMap[key]) {
-            fieldMap[key] = [];
-        }
-        const field = generateField(row);
-        fieldMap[key].push(field);
-    });
-    return {
-        tables: Object.values(tableMap),
-        fieldMap,
-    };
-}
-async function generateIndexes(client, projectId, datasetId) {
-    const tableNameToFetch = `${projectId}.${datasetId}`;
-    const query = `
-    select
-      s.index_schema as indexSchema,
-      s.table_name as tableName,
-      s.index_name as indexName,
-      string_agg(sc.index_column_name, ', ') as indexColumnNames
-    from ${tableNameToFetch}.INFORMATION_SCHEMA.SEARCH_INDEXES s
-    join ${tableNameToFetch}.INFORMATION_SCHEMA.SEARCH_INDEX_COLUMNS sc
-    on
-      s.index_schema = sc.index_schema
-      and s.table_name = sc.table_name
-      and s.index_name = sc.index_name
-    where sc.index_column_name = sc.index_field_path
-    group by s.index_schema, s.table_name, s.index_name
-    order by s.table_name, s.index_name;
-  `;
-    const [queryResult] = await client.query(query);
-    const indexMap = {};
-    queryResult.forEach((row) => {
-        const { indexSchema, tableName, indexName, indexColumnNames, } = row;
-        const indexColumns = indexColumnNames.split(', ').map((column) => {
-            return {
-                type: 'column',
-                value: column,
-            };
-        });
-        const index = {
-            name: indexName,
-            type: '',
-            columns: [...indexColumns],
-        };
-        const key = (0, helpers_1.getTableSchemaKey)(indexSchema, tableName);
-        if (indexMap[key]) {
-            indexMap[key].push(index);
-        }
-        else {
-            indexMap[key] = [index];
-        }
-    });
-    return indexMap;
-}
-async function generateTableConstraints(client, projectId, datasetId) {
-    const tableNameToFetch = `${projectId}.${datasetId}`;
-    const query = `
-    select
-      tc.table_schema as tableSchema,
-      tc.table_name as tableName,
-      tc.constraint_name as constraintName,
-      string_agg(kcu.column_name, ', ') as columnNames,
-      count(kcu.column_name) as columnCount,
-    from ${tableNameToFetch}.INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc
-    join ${tableNameToFetch}.INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu
-    on
-      tc.table_schema = kcu.table_schema
-      and tc.table_name = kcu.table_name
-      and tc.constraint_name = kcu.constraint_name
-    where tc.constraint_type = 'PRIMARY KEY'
-    group by tc.table_schema, tc.table_name, tc.constraint_name
-    order by tc.table_name, tc.constraint_name;
-  `;
-    const [queryResult] = await client.query(query);
-    const inlinePrimaryKeyList = queryResult.filter((row) => row.columnCount && row.columnCount === 1);
-    const compositePrimaryKeyList = queryResult.filter((row) => row.columnCount && row.columnCount > 1);
-    const inlinePrimaryKeyMap = {};
-    inlinePrimaryKeyList.forEach((inlineKey) => {
-        const { tableSchema, tableName, columnNames } = inlineKey;
-        const key = (0, helpers_1.getTableSchemaKey)(tableSchema, tableName);
-        if (!inlinePrimaryKeyMap[key]) {
-            inlinePrimaryKeyMap[key] = {};
-        }
-        if (!inlinePrimaryKeyMap[key][columnNames]) {
-            inlinePrimaryKeyMap[key][columnNames] = { checks: [] };
-        }
-        inlinePrimaryKeyMap[key][columnNames].pk = true;
-    });
-    const compositePrimaryKeyMap = {};
-    compositePrimaryKeyList.forEach((compositeKey) => {
-        const { tableSchema, tableName, constraintName, columnNames, } = compositeKey;
-        const indexColumns = columnNames.split(', ').map((column) => {
-            return {
-                type: 'column',
-                value: column,
-            };
-        });
-        const index = {
-            name: constraintName,
-            type: '',
-            columns: [...indexColumns],
-            pk: true,
-        };
-        const key = (0, helpers_1.getTableSchemaKey)(tableSchema, tableName);
-        if (compositePrimaryKeyMap[key]) {
-            compositePrimaryKeyMap[key].push(index);
-        }
-        else {
-            compositePrimaryKeyMap[key] = [index];
-        }
-    });
-    return {
-        inlinePrimaryKeyMap,
-        compositePrimaryKeyMap,
-    };
-}
-async function validateDatasetIdList(client, datasetIdList) {
-    const [datasets] = await client.getDatasets();
-    const rawDatasetIdList = datasets.map((dataset) => dataset.id).filter((id) => id);
-    // If user does not specify which datasets to be fetched, we will fetch all datasets
-    if (datasetIdList.length === 0) {
-        return rawDatasetIdList;
-    }
-    return (0, helpers_1.getIntersection)(datasetIdList, rawDatasetIdList);
-}
-async function fetchSchemaJsonByDataset(client, projectId, datasetId) {
-    const res = await Promise.all([
-        generateTablesAndFields(client, projectId, datasetId),
-        generateIndexes(client, projectId, datasetId),
-        generateTableConstraints(client, projectId, datasetId),
-    ]);
-    const { tables, fieldMap } = res[0];
-    const indexMap = res[1];
-    const { inlinePrimaryKeyMap, compositePrimaryKeyMap } = res[2];
-    return {
-        tables,
-        fields: fieldMap,
-        refs: [],
-        enums: [],
-        indexes: (0, helpers_1.mergeIndexDictionary)(indexMap, compositePrimaryKeyMap),
-        tableConstraints: inlinePrimaryKeyMap,
-        // Checks are not supported in BigQuery
-        checks: {},
-    };
-}
-async function fetchSchemaJson(keyFilename) {
-    const rawCredential = await (0, credential_loader_1.loadCredentialFromFile)(keyFilename);
-    const credentialJson = (0, credential_loader_1.parseBigQueryCredential)(rawCredential);
-    const client = await connectBigQuery(credentialJson);
-    const projectId = await client.getProjectId();
-    const datasetIdList = await validateDatasetIdList(client, credentialJson.datasets);
-    const querySchemaPromises = datasetIdList.map((dataset) => fetchSchemaJsonByDataset(client, projectId, dataset));
-    const databaseSchemaRes = await Promise.all(querySchemaPromises);
-    const res = databaseSchemaRes.reduce((acc, dbSchema) => {
-        const { tables, fields, indexes, tableConstraints, } = dbSchema;
-        acc.tables = (0, helpers_1.mergeTables)(acc.tables, tables);
-        acc.fields = (0, helpers_1.mergeFieldDictionary)(acc.fields, fields);
-        acc.indexes = (0, helpers_1.mergeIndexDictionary)(acc.indexes, indexes);
-        acc.tableConstraints = (0, helpers_1.mergeTableConstraintDictionary)(acc.tableConstraints, tableConstraints);
-        return acc;
-    }, {
-        tables: [], fields: {}, tableConstraints: {}, indexes: {}, refs: [], enums: [], checks: {},
-    });
-    return res;
-}

package/dist/connectors/connector.js

@@ -1,28 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.fetchSchemaJson = void 0;
-const postgres_1 = require("./postgres");
-const mssql_1 = require("./mssql");
-const mysql_1 = require("./mysql");
-const snowflake_1 = require("./snowflake");
-const bigquery_1 = require("./bigquery");
-const oracle_1 = require("./oracle");
-const fetchSchemaJson = async (connection, databaseType) => {
-    switch (databaseType) {
-        case 'postgres':
-            return (0, postgres_1.fetchSchemaJson)(connection);
-        case 'mssql':
-            return (0, mssql_1.fetchSchemaJson)(connection);
-        case 'mysql':
-            return (0, mysql_1.fetchSchemaJson)(connection);
-        case 'snowflake':
-            return (0, snowflake_1.fetchSchemaJson)(connection);
-        case 'bigquery':
-            return (0, bigquery_1.fetchSchemaJson)(connection);
-        case 'oracle':
-            return (0, oracle_1.fetchSchemaJson)(connection);
-        default:
-            throw new Error(`Unsupported database type: ${databaseType}`);
-    }
-};
-exports.fetchSchemaJson = fetchSchemaJson;