@dbml/connector 3.7.4 → 3.8.1

package/__tests__/connector.test.ts

@@ -1,10 +1,10 @@
-import path from 'path';
 import fs from 'fs';
+import path from 'path';
 import { scanDirNames } from '../jestHelpers.ts';
 import { connector } from '../src/index.ts';
 
 describe('@dbml/connector', () => {
-  const runTest = async (dirName) => {
+  const runTest = async (dirName: string) => {
     process.chdir(dirName);
 
     const connectionRaw = fs.readFileSync(path.join(dirName, './connection.json'), 'utf-8');

package/__tests__/connectors/postgres/expect-out-files/schema.json

@@ -1,5 +1,19 @@
 {
   "tables": [
+    {
+      "name": "table1",
+      "schemaName": "dbml_test",
+      "note": {
+        "value": ""
+      }
+    },
+    {
+      "name": "table3",
+      "schemaName": "dbml_test",
+      "note": {
+        "value": ""
+      }
+    },
     {
       "name": "all_default_values",
       "schemaName": "public",
@@ -49,6 +63,20 @@
         "value": ""
       }
     },
+    {
+      "name": "table1",
+      "schemaName": "public",
+      "note": {
+        "value": ""
+      }
+    },
+    {
+      "name": "table2",
+      "schemaName": "public",
+      "note": {
+        "value": ""
+      }
+    },
     {
       "name": "table_with_comments",
       "schemaName": "public",
@@ -72,6 +100,62 @@
     }
   ],
   "fields": {
+    "dbml_test.table1": [
+      {
+        "name": "id",
+        "type": {
+          "type_name": "int4",
+          "schemaName": null
+        },
+        "dbdefault": null,
+        "not_null": true,
+        "increment": true,
+        "note": {
+          "value": ""
+        }
+      },
+      {
+        "name": "status",
+        "type": {
+          "type_name": "enum_type1",
+          "schemaName": "dbml_test"
+        },
+        "dbdefault": null,
+        "not_null": false,
+        "increment": false,
+        "note": {
+          "value": ""
+        }
+      }
+    ],
+    "dbml_test.table3": [
+      {
+        "name": "id",
+        "type": {
+          "type_name": "int4",
+          "schemaName": null
+        },
+        "dbdefault": null,
+        "not_null": true,
+        "increment": true,
+        "note": {
+          "value": ""
+        }
+      },
+      {
+        "name": "status",
+        "type": {
+          "type_name": "enum_type3",
+          "schemaName": "dbml_test"
+        },
+        "dbdefault": null,
+        "not_null": false,
+        "increment": false,
+        "note": {
+          "value": ""
+        }
+      }
+    ],
     "public.all_default_values": [
       {
         "name": "id",
@@ -843,6 +927,62 @@
         }
       }
     ],
+    "public.table1": [
+      {
+        "name": "id",
+        "type": {
+          "type_name": "int4",
+          "schemaName": null
+        },
+        "dbdefault": null,
+        "not_null": true,
+        "increment": true,
+        "note": {
+          "value": ""
+        }
+      },
+      {
+        "name": "status",
+        "type": {
+          "type_name": "enum_type1",
+          "schemaName": "public"
+        },
+        "dbdefault": null,
+        "not_null": false,
+        "increment": false,
+        "note": {
+          "value": ""
+        }
+      }
+    ],
+    "public.table2": [
+      {
+        "name": "id",
+        "type": {
+          "type_name": "int4",
+          "schemaName": null
+        },
+        "dbdefault": null,
+        "not_null": true,
+        "increment": true,
+        "note": {
+          "value": ""
+        }
+      },
+      {
+        "name": "status",
+        "type": {
+          "type_name": "enum_type2",
+          "schemaName": "public"
+        },
+        "dbdefault": null,
+        "not_null": false,
+        "increment": false,
+        "note": {
+          "value": ""
+        }
+      }
+    ],
     "public.table_with_comments": [
       {
         "name": "id",
@@ -1229,6 +1369,54 @@
     }
   ],
   "enums": [
+    {
+      "name": "enum_type1",
+      "schemaName": "dbml_test",
+      "values": [
+        {
+          "name": "value1"
+        },
+        {
+          "name": "value2"
+        }
+      ]
+    },
+    {
+      "name": "enum_type3",
+      "schemaName": "dbml_test",
+      "values": [
+        {
+          "name": "value4"
+        },
+        {
+          "name": "value5"
+        }
+      ]
+    },
+    {
+      "name": "enum_type1",
+      "schemaName": "public",
+      "values": [
+        {
+          "name": "value1"
+        },
+        {
+          "name": "value2"
+        }
+      ]
+    },
+    {
+      "name": "enum_type2",
+      "schemaName": "public",
+      "values": [
+        {
+          "name": "value1"
+        },
+        {
+          "name": "value3"
+        }
+      ]
+    },
     {
       "name": "gender_type",
       "schemaName": "public",
@@ -1386,6 +1574,26 @@
         "pk": true
       }
     },
+    "public.table1": {
+      "id": {
+        "pk": true
+      }
+    },
+    "dbml_test.table1": {
+      "id": {
+        "pk": true
+      }
+    },
+    "public.table2": {
+      "id": {
+        "pk": true
+      }
+    },
+    "dbml_test.table3": {
+      "id": {
+        "pk": true
+      }
+    },
     "public.table_with_comments": {
       "id": {
         "pk": true

package/dist/connectors/bigqueryConnector.d.ts

@@ -0,0 +1,3 @@
+import { DatabaseSchema } from './types';
+declare function fetchSchemaJson(keyFilename: string): Promise<DatabaseSchema>;
+export { fetchSchemaJson };

package/dist/connectors/bigqueryConnector.js

@@ -0,0 +1,276 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fetchSchemaJson = fetchSchemaJson;
+const bigquery_1 = require("@google-cloud/bigquery");
+const credential_loader_1 = require("../utils/credential-loader");
+const helpers_1 = require("../utils/helpers");
+async function connectBigQuery(credential) {
+    const client = new bigquery_1.BigQuery({
+        projectId: credential.projectId,
+        credentials: {
+            private_key: credential.credentials.privateKey,
+            client_email: credential.credentials.clientEmail,
+        },
+    });
+    try {
+        const query = `SELECT CURRENT_DATE() as today`;
+        await client.query(query);
+        return client;
+    }
+    catch (error) {
+        if (error instanceof Error) {
+            throw new Error(`BigQuery connection error: ${error.message}`);
+        }
+        throw error;
+    }
+}
+function getDbDefault(columnDefault, defaultValueType) {
+    if (defaultValueType === null) {
+        return null;
+    }
+    if (defaultValueType === 'string') {
+        // bigquery store the double qoutes and quotes at the end and the beginning of the string: "string" or 'string'
+        return { value: columnDefault.slice(1, -1).replaceAll("'", "\'"), type: defaultValueType };
+    }
+    return { value: columnDefault, type: defaultValueType };
+}
+function generateField(row) {
+    const { columnName, isNullable, dataType, columnDefault, columnDefaultType, columnDescription, } = row;
+    const fieldDefaultValue = getDbDefault(columnDefault, columnDefaultType);
+    const field = {
+        name: columnName,
+        not_null: isNullable === 'NO',
+        type: {
+            type_name: dataType,
+            schemaName: null,
+        },
+        note: {
+            value: columnDescription || '',
+        },
+        dbdefault: fieldDefaultValue,
+        increment: false, // bigquery does not support native auto-increment
+    };
+    return field;
+}
+async function generateTablesAndFields(client, projectId, datasetId) {
+    const tableName = `${projectId}.${datasetId}`;
+    const query = `
+    with
+      table_comments as (
+        select table_schema, table_name, option_value as description
+        from ${tableName}.INFORMATION_SCHEMA.TABLE_OPTIONS
+        where option_name = 'description'
+      ),
+      column_comments as (
+        select column_name, table_schema, table_name, description
+        from ${tableName}.INFORMATION_SCHEMA.COLUMN_FIELD_PATHS
+        where column_name = field_path
+      )
+    select
+      t.table_schema as tableSchema,
+      t.table_name as tableName,
+      (select description from table_comments tc where tc.table_name = t.table_name and tc.table_schema = t.table_schema) as tableDescription,
+      (
+        select description
+        from column_comments cc
+        where cc.table_name = c.table_name and cc.table_schema = c.table_schema and cc.column_name = c.column_name
+      ) as columnDescription,
+      c.column_name as columnName,
+      c.is_nullable as isNullable,
+      c.data_type as dataType,
+      c.column_default as columnDefault,
+      CASE
+        WHEN SAFE_CAST(c.column_default AS FLOAT64) IS NOT NULL THEN 'number'
+        WHEN SAFE_CAST(c.column_default AS BOOLEAN) IS NOT NULL THEN 'boolean'
+        -- string value is enclosed in single quotes, double quotes
+        WHEN REGEXP_CONTAINS(c.column_default, r"^['\\"].*['\\"]$") THEN 'string'
+        WHEN c.column_default = "NULL" THEN NULL
+        ELSE 'expression'
+      END AS columnDefaultType
+    from ${tableName}.INFORMATION_SCHEMA.TABLES t
+    join ${tableName}.INFORMATION_SCHEMA.COLUMNS c
+    on
+      t.table_schema = c.table_schema
+      and t.table_name = c.table_name
+    where t.table_type = 'BASE TABLE'
+    order by t.table_name, c.ordinal_position;
+  `;
+    const [queryResult] = await client.query({
+        query,
+        params: {
+        // stringRegex: STRING_REGEX,
+        },
+    });
+    const tableMap = {};
+    const fieldMap = {};
+    queryResult.forEach((row) => {
+        const { tableSchema, tableName, tableComment } = row;
+        const key = (0, helpers_1.getTableSchemaKey)(tableSchema, tableName);
+        if (!tableMap[key]) {
+            tableMap[key] = {
+                name: tableName,
+                note: { value: tableComment || '' },
+                schemaName: tableSchema,
+            };
+        }
+        if (!fieldMap[key]) {
+            fieldMap[key] = [];
+        }
+        const field = generateField(row);
+        fieldMap[key].push(field);
+    });
+    return {
+        tables: Object.values(tableMap),
+        fieldMap,
+    };
+}
+async function generateIndexes(client, projectId, datasetId) {
+    const tableName = `${projectId}.${datasetId}`;
+    const query = `
+    select
+      s.index_schema as indexSchema,
+      s.table_name as tableName,
+      s.index_name as indexName,
+      string_agg(sc.index_column_name, ', ') as indexColumnNames
+    from ${tableName}.INFORMATION_SCHEMA.SEARCH_INDEXES s
+    join ${tableName}.INFORMATION_SCHEMA.SEARCH_INDEX_COLUMNS sc
+    on
+      s.index_schema = sc.index_schema
+      and s.table_name = sc.table_name
+      and s.index_name = sc.index_name
+    where sc.index_column_name = sc.index_field_path
+    group by s.index_schema, s.table_name, s.index_name
+    order by s.table_name, s.index_name;
+  `;
+    const [queryResult] = await client.query(query);
+    const indexMap = {};
+    queryResult.forEach((row) => {
+        const { indexSchema, tableName, indexName, indexColumnNames } = row;
+        const indexColumns = indexColumnNames.split(', ').map((column) => {
+            return {
+                type: 'column',
+                value: column,
+            };
+        });
+        const index = {
+            name: indexName,
+            type: '',
+            columns: [...indexColumns],
+        };
+        const key = (0, helpers_1.getTableSchemaKey)(indexSchema, tableName);
+        if (indexMap[key]) {
+            indexMap[key].push(index);
+        }
+        else {
+            indexMap[key] = [index];
+        }
+    });
+    return indexMap;
+}
+async function generateTableConstraints(client, projectId, datasetId) {
+    const tableName = `${projectId}.${datasetId}`;
+    const query = `
+    select
+      tc.table_schema as tableSchema,
+      tc.table_name as tableName,
+      tc.constraint_name as constraintName,
+      string_agg(kcu.column_name, ', ') as columnNames,
+      count(kcu.column_name) as columnCount,
+    from ${tableName}.INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc
+    join ${tableName}.INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu
+    on
+      tc.table_schema = kcu.table_schema
+      and tc.table_name = kcu.table_name
+      and tc.constraint_name = kcu.constraint_name
+    where tc.constraint_type = 'PRIMARY KEY'
+    group by tc.table_schema, tc.table_name, tc.constraint_name
+    order by tc.table_name, tc.constraint_name;
+  `;
+    const [queryResult] = await client.query(query);
+    const inlinePrimaryKeyList = queryResult.filter((row) => row.columnCount && row.columnCount === 1);
+    const compositePrimaryKeyList = queryResult.filter((row) => row.columnCount && row.columnCount > 1);
+    const inlinePrimaryKeyMap = {};
+    inlinePrimaryKeyList.forEach((inlineKey) => {
+        const { tableSchema, tableName, columnNames } = inlineKey;
+        const key = (0, helpers_1.getTableSchemaKey)(tableSchema, tableName);
+        if (!inlinePrimaryKeyMap[key]) {
+            inlinePrimaryKeyMap[key] = {};
+        }
+        if (!inlinePrimaryKeyMap[key][columnNames]) {
+            inlinePrimaryKeyMap[key][columnNames] = {};
+        }
+        inlinePrimaryKeyMap[key][columnNames].pk = true;
+    });
+    const compositePrimaryKeyMap = {};
+    compositePrimaryKeyList.forEach((compositeKey) => {
+        const { tableSchema, tableName, constraintName, columnNames } = compositeKey;
+        const indexColumns = columnNames.split(', ').map((column) => {
+            return {
+                type: 'column',
+                value: column,
+            };
+        });
+        const index = {
+            name: constraintName,
+            type: '',
+            columns: [...indexColumns],
+            pk: true,
+        };
+        const key = (0, helpers_1.getTableSchemaKey)(tableSchema, tableName);
+        if (compositePrimaryKeyMap[key]) {
+            compositePrimaryKeyMap[key].push(index);
+        }
+        else {
+            compositePrimaryKeyMap[key] = [index];
+        }
+    });
+    return {
+        inlinePrimaryKeyMap,
+        compositePrimaryKeyMap,
+    };
+}
+async function validateDatasetIdList(client, datasetIdList) {
+    const [datasets] = await client.getDatasets();
+    const rawDatasetIdList = datasets.map((dataset) => dataset.id).filter((id) => id);
+    // If user does not specify which datasets to be fetched, we will fetch all datasets
+    if (datasetIdList.length === 0) {
+        return rawDatasetIdList;
+    }
+    return (0, helpers_1.getIntersection)(datasetIdList, rawDatasetIdList);
+}
+async function fetchSchemaJsonByDataset(client, projectId, datasetId) {
+    const res = await Promise.all([
+        generateTablesAndFields(client, projectId, datasetId),
+        generateIndexes(client, projectId, datasetId),
+        generateTableConstraints(client, projectId, datasetId),
+    ]);
+    const { tables, fieldMap } = res[0];
+    const indexMap = res[1];
+    const { inlinePrimaryKeyMap, compositePrimaryKeyMap } = res[2];
+    return {
+        tables,
+        fields: fieldMap,
+        refs: [],
+        enums: [],
+        indexes: (0, helpers_1.mergeIndexDictionary)(indexMap, compositePrimaryKeyMap),
+        tableConstraints: inlinePrimaryKeyMap,
+    };
+}
+async function fetchSchemaJson(keyFilename) {
+    const rawCredential = await (0, credential_loader_1.loadCredentialFromFile)(keyFilename);
+    const credentialJson = (0, credential_loader_1.parseBigQueryCredential)(rawCredential);
+    const client = await connectBigQuery(credentialJson);
+    const projectId = await client.getProjectId();
+    const datasetIdList = await validateDatasetIdList(client, credentialJson.datasets);
+    const querySchemaPromises = datasetIdList.map((dataset) => fetchSchemaJsonByDataset(client, projectId, dataset));
+    const databaseSchemaRes = await Promise.all(querySchemaPromises);
+    const res = databaseSchemaRes.reduce((acc, dbSchema) => {
+        const { tables, fields, indexes, tableConstraints } = dbSchema;
+        acc.tables = (0, helpers_1.mergeTables)(acc.tables, tables);
+        acc.fields = (0, helpers_1.mergeFieldDictionary)(acc.fields, fields);
+        acc.indexes = (0, helpers_1.mergeIndexDictionary)(acc.indexes, indexes);
+        acc.tableConstraints = (0, helpers_1.mergeTableConstraintDictionary)(acc.tableConstraints, tableConstraints);
+        return acc;
+    }, { tables: [], fields: {}, tableConstraints: {}, indexes: {}, refs: [], enums: [] });
+    return res;
+}

package/dist/connectors/connector.d.ts

@@ -1,3 +1,3 @@
 import { DatabaseSchema } from './types';
-declare const fetchSchemaJson: (connection: string, format: string) => Promise<DatabaseSchema>;
+declare const fetchSchemaJson: (connection: string, databaseType: string) => Promise<DatabaseSchema>;
 export { fetchSchemaJson, };

package/dist/connectors/connector.js

@@ -4,16 +4,22 @@ exports.fetchSchemaJson = void 0;
 const postgresConnector_1 = require("./postgresConnector");
 const mssqlConnector_1 = require("./mssqlConnector");
 const mysqlConnector_1 = require("./mysqlConnector");
-const fetchSchemaJson = async (connection, format) => {
-    switch (format) {
+const snowflakeConnector_1 = require("./snowflakeConnector");
+const bigqueryConnector_1 = require("./bigqueryConnector");
+const fetchSchemaJson = async (connection, databaseType) => {
+    switch (databaseType) {
         case 'postgres':
             return (0, postgresConnector_1.fetchSchemaJson)(connection);
         case 'mssql':
             return (0, mssqlConnector_1.fetchSchemaJson)(connection);
         case 'mysql':
             return (0, mysqlConnector_1.fetchSchemaJson)(connection);
+        case 'snowflake':
+            return (0, snowflakeConnector_1.fetchSchemaJson)(connection);
+        case 'bigquery':
+            return (0, bigqueryConnector_1.fetchSchemaJson)(connection);
         default:
-            throw new Error(`Unsupported connection format: ${format}`);
+            throw new Error(`Unsupported database type: ${databaseType}`);
     }
 };
 exports.fetchSchemaJson = fetchSchemaJson;

package/dist/connectors/mssqlConnector.d.ts

@@ -1,3 +1,3 @@
 import { DatabaseSchema } from './types';
 declare const fetchSchemaJson: (connection: string) => Promise<DatabaseSchema>;
-export { fetchSchemaJson, };
+export { fetchSchemaJson };