@dbml/connector 3.7.4 → 3.8.1

package/src/connectors/postgresConnector.ts

@@ -1,5 +1,6 @@
 /* eslint-disable camelcase */
 import { Client } from 'pg';
+import { buildSchemaQuery, parseConnectionString } from '../utils/parseSchema';
 import {
   DatabaseSchema,
   DefaultInfo,
@@ -113,14 +114,14 @@ const generateField = (row: Record<string, any>): Field => {
   };
 };
 
-const generateTablesAndFields = async (client: Client): Promise<{
+const generateTablesAndFields = async (client: Client, schemas: string[]): Promise<{
   tables: Table[],
   fields: FieldsDictionary,
 }> => {
   const fields: FieldsDictionary = {};
   const tablesAndFieldsSql = `
     WITH comments AS (
-      SELECT
+      SELECT DISTINCT ON (pc.relname, pn.nspname, pa.attname)
         pc.relname AS table_name,
         pn.nspname AS table_schema,
         pa.attname AS column_name,
@@ -150,6 +151,7 @@ const generateTablesAndFields = async (client: Client): Promise<{
       c.identity_increment,
       c.is_nullable,
       c.column_default,
+      c.ordinal_position,
       CASE
         WHEN c.column_default IS NULL THEN NULL
         WHEN c.column_default LIKE 'nextval(%' THEN 'increment'
@@ -158,8 +160,8 @@ const generateTablesAndFields = async (client: Client): Promise<{
         WHEN c.column_default ~ '^-?[0-9]+(.[0-9]+)?$' THEN 'number'
         ELSE 'expression'
       END AS default_type,
-      (SELECT description FROM comments WHERE table_name = t.table_name AND table_schema = t.table_schema AND column_name IS NULL) AS table_comment,
-      (SELECT description FROM comments WHERE table_name = t.table_name AND table_schema = t.table_schema AND column_name = c.column_name) AS column_comment
+      (SELECT description FROM comments WHERE table_name = t.table_name AND table_schema = t.table_schema AND column_name IS NULL LIMIT 1) AS table_comment,
+      (SELECT description FROM comments WHERE table_name = t.table_name AND table_schema = t.table_schema AND column_name = c.column_name LIMIT 1) AS column_comment
     FROM
       information_schema.columns c
     JOIN
@@ -167,6 +169,7 @@ const generateTablesAndFields = async (client: Client): Promise<{
     WHERE
       t.table_type = 'BASE TABLE'
       AND t.table_schema NOT IN ('pg_catalog', 'information_schema')
+      ${buildSchemaQuery('t.table_schema', schemas)}
     ORDER BY
       t.table_schema,
       t.table_name,
@@ -200,7 +203,7 @@ const generateTablesAndFields = async (client: Client): Promise<{
   };
 };
 
-const generateRawRefs = async (client: Client): Promise<Ref[]> => {
+const generateRawRefs = async (client: Client, schemas: string[]): Promise<Ref[]> => {
   const refs: Ref[] = [];
 
   const refsListSql = `
@@ -226,6 +229,7 @@ const generateRawRefs = async (client: Client): Promise<Ref[]> => {
       AND tc.table_schema = rc.constraint_schema
     WHERE tc.constraint_type = 'FOREIGN KEY'
       AND tc.table_schema NOT IN ('pg_catalog', 'information_schema')
+      ${buildSchemaQuery('tc.table_schema', schemas)}
     GROUP BY
       tc.table_schema,
       tc.table_name,
@@ -279,8 +283,7 @@ const generateRawRefs = async (client: Client): Promise<Ref[]> => {
   return refs;
 };
 
-const generateIndexes = async (client: Client) => {
-  // const tableConstraints = {};
+const generateIndexes = async (client: Client, schemas: string[]) => {
   const indexListSql = `
     WITH user_tables AS (
       SELECT
@@ -293,6 +296,7 @@ const generateIndexes = async (client: Client) => {
     ),
     index_info AS (
       SELECT
+        t.relnamespace::regnamespace::text AS table_schema,
         t.relname AS table_name,
         i.relname AS index_name,
         ix.indisunique AS is_unique,
@@ -317,7 +321,7 @@ const generateIndexes = async (client: Client) => {
         AND t.relname NOT LIKE 'pg_%'
         AND t.relname NOT LIKE 'sql_%'
       GROUP BY
-        t.relname, i.relname, ix.indisunique, ix.indisprimary, am.amname, ix.indexprs, ix.indrelid
+        t.relnamespace, t.relname, i.relname, ix.indisunique, ix.indisprimary, am.amname, ix.indexprs, ix.indrelid
     )
     SELECT
       ut.tableschema AS table_schema,
@@ -328,12 +332,13 @@ const generateIndexes = async (client: Client) => {
       ii.index_type,
       ii.columns,
       ii.expressions,
-      ii.constraint_type  -- Added constraint type
+      ii.constraint_type  -- Retained constraint type here
     FROM
       user_tables ut
     LEFT JOIN
-      index_info ii ON ut.tablename = ii.table_name
+      index_info ii ON ut.tableschema = ii.table_schema AND ut.tablename = ii.table_name
     WHERE ii.columns IS NOT NULL
+    ${buildSchemaQuery('ut.tableschema', schemas)}
     ORDER BY
       ut.tablename,
       ii.constraint_type,
@@ -427,9 +432,9 @@ const generateIndexes = async (client: Client) => {
   };
 };
 
-const generateRawEnums = async (client: Client): Promise<Enum[]> => {
+const generateRawEnums = async (client: Client, schemas: string[]): Promise<Enum[]> => {
   const enumListSql = `
-    SELECT
+    SELECT DISTINCT
       n.nspname AS schema_name,
       t.typname AS enum_type,
       e.enumlabel AS enum_value,
@@ -440,6 +445,7 @@ const generateRawEnums = async (client: Client): Promise<Enum[]> => {
       pg_type t ON e.enumtypid = t.oid
     JOIN
       pg_namespace n ON t.typnamespace = n.oid
+    ${buildSchemaQuery('n.nspname', schemas, 'WHERE')}
     ORDER BY
       schema_name,
       enum_type,
@@ -449,15 +455,16 @@ const generateRawEnums = async (client: Client): Promise<Enum[]> => {
   const enumListResult = await client.query(enumListSql);
   const enums = enumListResult.rows.reduce((acc, row) => {
     const { schema_name, enum_type, enum_value } = row;
+    const key = `${schema_name}.${enum_type}`;
 
-    if (!acc[enum_type]) {
-      acc[enum_type] = {
+    if (!acc[key]) {
+      acc[key] = {
         name: enum_type,
         schemaName: schema_name,
         values: [],
       };
     }
-    acc[enum_type].values.push({
+    acc[key].values.push({
       name: enum_value,
     });
     return acc;
@@ -467,12 +474,13 @@ const generateRawEnums = async (client: Client): Promise<Enum[]> => {
 };
 
 const fetchSchemaJson = async (connection: string): Promise<DatabaseSchema> => {
-  const client = await getValidatedClient(connection);
+  const { connectionString, schemas } = parseConnectionString(connection, 'jdbc');
+  const client = await getValidatedClient(connectionString);
 
-  const tablesAndFieldsRes = generateTablesAndFields(client);
-  const indexesRes = generateIndexes(client);
-  const refsRes = generateRawRefs(client);
-  const enumsRes = generateRawEnums(client);
+  const tablesAndFieldsRes = generateTablesAndFields(client, schemas);
+  const indexesRes = generateIndexes(client, schemas);
+  const refsRes = generateRawRefs(client, schemas);
+  const enumsRes = generateRawEnums(client, schemas);
 
   const res = await Promise.all([
     tablesAndFieldsRes,
@@ -498,5 +506,5 @@ const fetchSchemaJson = async (connection: string): Promise<DatabaseSchema> => {
 };
 
 export {
-  fetchSchemaJson,
+  fetchSchemaJson
 };

package/src/connectors/snowflakeConnector.ts

@@ -0,0 +1,370 @@
+/* eslint-disable camelcase */
+//
+// Description:
+// Snowflake info schema lacks reference direction for relationships.
+// Constraints and uniqueness cannot be auto-generated from Snowflake info schema.
+// Snowflake does not support ENUMs.
+// Snowflake does not support indexes for standard tables.
+//
+
+import snowflake, { Connection, LogLevel } from 'snowflake-sdk';
+import { parseSchema } from '../utils/parseSchema';
+import {
+  DatabaseSchema,
+  DefaultInfo,
+  DefaultType,
+  Field,
+  FieldsDictionary,
+  Index,
+  IndexesDictionary,
+  Table,
+  TableConstraintsDictionary,
+} from './types';
+
+type ConstraintRow = {
+  schemaName: string;
+  tableName: string;
+  constraintName: string;
+  columnNames: string[];
+  type: string;
+  primary?: boolean;
+  unique?: boolean;
+};
+
+type GeneratedIndexes = {
+  indexes: IndexesDictionary;
+  tableConstraints: TableConstraintsDictionary;
+};
+
+const parseConnectionString = (connectionString: string): Record<string, string> => {
+  const params: Record<string, string> = {};
+  const regex = /([^;=]+)=([^;]*)/g;
+  let match;
+
+  while ((match = regex.exec(connectionString)) !== null) {
+      params[match[1].trim()] = match[2].trim();
+  }
+
+  return params;
+}
+
+
+const connect = async (connection: Connection): Promise<void> => {
+  return new Promise((resolve, reject) => {
+    console.log('Attempting to connect...');
+    connection.connect((err) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
+const executeQuery = (connection: Connection, sqlText: string): Promise<any[]> => {
+  return new Promise((resolve, reject) => {
+    console.log('Executing query...');
+    connection.execute({
+      sqlText: sqlText,
+      complete: (err, stmt, rows) => {
+        if (err) {
+          reject(err);
+        } else {
+          console.log('Query executed successfully.');
+          if (rows) {
+            resolve(rows);
+          }
+        }
+      }
+    });
+  });
+}
+
+const isLogLevel = (value: string): value is LogLevel => {
+  return ['ERROR', 'WARN', 'INFO', 'DEBUG', 'TRACE'].includes(value as LogLevel)
+}
+
+const connectToSnowflake = async (config: Record<string, string>): Promise<Connection> => {
+  const logLevel = isLogLevel(config.LOG_LEVEL) ? config.LOG_LEVEL : 'INFO';
+  const isDebugMode = config.IS_DEBUG_MODE === 'true';
+  snowflake.configure({
+    logLevel,
+    additionalLogToConsole: isDebugMode,
+  });
+
+  // "SERVER=myaccount.snowflakecomputing.com;UID=myusername;PWD=mypassword;DATABASE=mydatabase;WAREHOUSE=mywarehouse;ROLE=myrole";
+  const connection = snowflake.createConnection({
+    account: config.SERVER,
+    username: config.UID,
+    password: config.PWD,
+    database: config.DATABASE,
+    warehouse: config.WAREHOUSE,
+    sfRetryMaxLoginRetries: 3,
+    timeout: 10000,
+  });
+
+  try {
+    // Connect to Snowflake
+    await connect(connection);
+
+    // Execute the query
+    await executeQuery(connection, 'SELECT CURRENT_VERSION();');
+
+    return connection;
+  } catch (err) {
+    throw err;
+  }
+}
+
+
+const convertQueryBoolean = (val: string | null) => val === 'YES';
+
+const getFieldType = (data_type: string, character_maximum_length: number, numeric_precision: number, numeric_scale: number): string => {
+  if (character_maximum_length) {
+    return `${data_type}(${character_maximum_length})`;
+  }
+  if (numeric_precision && numeric_scale) {
+    return `${data_type}(${numeric_precision},${numeric_scale})`;
+  }
+  return data_type;
+};
+
+const getDbdefault = (column_default: string, default_type: DefaultType): DefaultInfo => {
+  if (default_type === 'string') {
+    return {
+      type: default_type,
+      value: column_default.slice(1, -1),
+    };
+  }
+  return {
+    type: default_type,
+    value: column_default,
+  };
+};
+
+const generateField = (row: Record<string, any>): Field => {
+  const {
+    COLUMN_NAME: column_name,
+    DATA_TYPE: data_type,
+    CHARACTER_MAXIMUM_LENGTH: character_maximum_length,
+    NUMERIC_PRECISION: numeric_precision,
+    NUMERIC_SCALE: numeric_scale,
+    IDENTITY_INCREMENT: identity_increment,
+    IS_NULLABLE: is_nullable,
+    COLUMN_DEFAULT: column_default,
+    DEFAULT_TYPE: default_type,
+    COLUMN_COMMENT: column_comment,
+  } = row;
+
+  const dbdefault = column_default ? getDbdefault(column_default, default_type) : null;
+
+  const fieldType = {
+    type_name: getFieldType(data_type, character_maximum_length, numeric_precision, numeric_scale),
+    schemaName: null,
+  };
+
+  return {
+    name: column_name,
+    type: fieldType,
+    dbdefault,
+    not_null: !convertQueryBoolean(is_nullable),
+    increment: !!identity_increment,
+    note: column_comment ? { value: column_comment } : { value: '' },
+  };
+};
+
+const generateTablesAndFields = async (conn: Connection, schemas: string[]): Promise<{
+  tables: Table[],
+  fields: FieldsDictionary,
+}> => {
+  const fields: FieldsDictionary = {};
+  const schemaSql = schemas.length > 0 ? `AND c.table_schema IN (${schemas.map((schema) => `'${schema}'`).join(',')})` : '';
+  const tablesAndFieldsSql = `
+    SELECT
+        c.table_schema,
+        c.table_name,
+        c.column_name,
+        c.data_type,
+        c.character_maximum_length,
+        c.numeric_precision,
+        c.numeric_scale,
+        c.is_nullable,
+        c.column_default,
+        CASE
+            WHEN c.column_default IS NULL THEN NULL
+            WHEN c.column_default LIKE '''%' THEN 'string'
+            WHEN LOWER(c.column_default) = 'true' OR LOWER(c.column_default) = 'false' THEN 'boolean'
+            WHEN c.column_default REGEXP '^-?[0-9]+(\\.[0-9]+)?$' THEN 'number'
+            ELSE 'expression'
+        END AS default_type,
+        t.comment AS table_comment,
+        c.comment AS column_comment,
+        c.identity_increment
+    FROM
+        information_schema.columns c
+        JOIN information_schema.tables t
+            ON c.table_name = t.table_name
+            AND c.table_schema = t.table_schema
+    WHERE
+        t.table_type = 'BASE TABLE'
+        AND t.table_schema NOT IN ('INFORMATION_SCHEMA')
+        ${schemaSql}
+    ORDER BY
+        c.table_schema,
+        c.table_name,
+        c.ordinal_position;
+  `;
+
+  const tablesAndFieldsResult = await executeQuery(conn, tablesAndFieldsSql);
+  const tables = tablesAndFieldsResult.reduce((acc: Record<string, Table>, row: Record<string, string>) => {
+    const { TABLE_SCHEMA: table_schema, TABLE_NAME: table_name, TABLE_COMMENT: table_comment } = row;
+    const key = `${table_schema}.${table_name}`;
+
+    if (!acc[key]) {
+      acc[key] = {
+        name: table_name,
+        schemaName: table_schema,
+        note: table_comment ? { value: table_comment } : { value: '' },
+      };
+    }
+
+    if (!fields[key]) fields[key] = [];
+    const field = generateField(row);
+    fields[key].push(field);
+
+    return acc;
+  }, {});
+
+  return {
+    tables: Object.values(tables),
+    fields,
+  };
+};
+
+const createConstraintKeysMap = (keys: Record<string, string>[], schemas: string[], constraintType: 'primary' | 'unique'): Record<string, ConstraintRow> => {
+  return keys.reduce((acc: Record<string, ConstraintRow>, row: Record<string, string>) => {
+    const { schema_name, table_name, column_name, constraint_name } = row;
+    const selectedSchema = schemas.length > 0 ? schemas.includes(schema_name) : true;
+    if (!selectedSchema) { return acc; }
+
+    const key = `${schema_name}.${table_name}.${constraint_name}`;
+
+    if (acc[key]) {
+      const columnNames = acc[key].columnNames;
+      columnNames.push(column_name);
+      return acc;
+    }
+
+    acc[key] = {
+      schemaName: schema_name,
+      tableName: table_name,
+      constraintName: constraint_name,
+      columnNames: [column_name],
+      type: '',
+    };
+    acc[key][constraintType] = true;
+
+    return acc;
+  }, {});
+};
+const generateIndexes = async (conn: Connection, databaseName: string, schemas: string[]) => {
+
+
+  const getPrimaryKeysSql = `
+    SHOW PRIMARY KEYS IN DATABASE ${databaseName};
+  `;
+
+  const getUniqueKeysSql = `
+    SHOW UNIQUE KEYS IN DATABASE ${databaseName};
+  `;
+
+  const primaryKeys = await executeQuery(conn, getPrimaryKeysSql);
+  const uniqueKeys = await executeQuery(conn, getUniqueKeysSql);
+
+  const primaryKeysByConstraint: Record<string, ConstraintRow> = createConstraintKeysMap(primaryKeys, schemas, 'primary');
+  const uniqueKeysByConstraint: Record<string, ConstraintRow> = createConstraintKeysMap(uniqueKeys, schemas, 'unique');
+
+  const allConstraints: ConstraintRow[] = [Object.values(primaryKeysByConstraint), Object.values(uniqueKeysByConstraint)].flat();
+  const { indexes, tableConstraints } = allConstraints.reduce((acc: GeneratedIndexes, row: ConstraintRow): GeneratedIndexes => {
+    const { schemaName, tableName, constraintName, columnNames, type, primary, unique } = row;
+    const key = `${schemaName}.${tableName}`;
+
+    if (columnNames.length < 2) {
+      acc.tableConstraints[key] = {
+        [columnNames[0]]: {
+          pk: primary,
+          unique,
+        },
+      };
+      return acc;
+    }
+
+    const index: Index = {
+      name: constraintName,
+      type,
+      unique,
+      pk: primary,
+      columns: columnNames.map((columnName) => ({
+        type: 'column',
+        value: columnName,
+      })),
+    };
+
+    if (acc.indexes[key]) {
+      acc.indexes[key].push(index);
+    } else {
+      acc.indexes[key] = [index];
+    }
+
+    return acc;
+  }, { indexes: {}, tableConstraints: {}});
+
+  return {
+    indexes,
+    tableConstraints,
+  };
+};
+
+const fetchSchemaJson = async (connection: string): Promise<DatabaseSchema> => {
+  const config = parseConnectionString(connection);
+  const conn = await connectToSnowflake(config);
+  if (conn instanceof Error) {
+    throw conn;
+  }
+  // Schemas: schema1,schema2,schema3
+  const schemas = config.SCHEMAS ? parseSchema(config.SCHEMAS) : [];
+  const databaseName = config.DATABASE;
+
+  const tablesAndFieldsRes = generateTablesAndFields(conn, schemas);
+  const indexesRes = generateIndexes(conn, databaseName, schemas);
+
+  const res = await Promise.all([
+    tablesAndFieldsRes,
+    indexesRes,
+  ]);
+
+  conn.destroy((err) => {
+    if (err) {
+      throw err;
+    } else {
+      console.log('Connection destroyed successfully.');
+    }
+  });
+
+  const { tables, fields } = res[0];
+  const { indexes, tableConstraints } = res[1];
+
+  return {
+    tables,
+    fields,
+    refs: [],
+    enums: [],
+    indexes,
+    tableConstraints,
+  };
+};
+
+export {
+  fetchSchemaJson,
+};

package/src/connectors/types.ts

@@ -21,6 +21,8 @@ interface Field {
   dbdefault: DefaultInfo | null;
   not_null: boolean;
   increment: boolean;
+  pk?: boolean;
+  unique?: boolean;
   note: NoteInfo;
 }
 
@@ -95,6 +97,15 @@ interface DatabaseSchema {
   indexes: IndexesDictionary; // Represents the indexes property
 }
 
+interface BigQueryCredentials {
+  projectId: string,
+  credentials: {
+    clientEmail: string,
+    privateKey: string,
+  },
+  datasets: string[],
+}
+
 export {
   NoteInfo,
   TypeInfo,
@@ -113,4 +124,5 @@ export {
   Index,
   IndexesDictionary,
   DatabaseSchema,
+  BigQueryCredentials,
 };

package/src/ddl_samples/bigquery.sql

@@ -0,0 +1,71 @@
+create schema if not exists dbml_test;
+
+create table if not exists dbml_test.test_data_type_default (
+    t_int INT64 default 1 OPTIONS(description="Int64 column"),
+    t_bool_1 BOOLEAN default true,
+    t_bool_2 BOOLEAN default True,
+    t_string_1 STRING default 'string column description 1',
+    t_string_2 STRING default "string column description 2",
+    t_string_3 STRING default "NULL",
+    t_float FLOAT64 default 123.67,
+    t_decimal DECIMAL default 10,
+    t_date DATE default CURRENT_DATE,
+    t_datetime DATETIME default CURRENT_DATETIME,
+    t_timestamp TIMESTAMP default CURRENT_TIMESTAMP,
+    t_array ARRAY<INT64> default ARRAY<INT64>[1, 2, 3],
+    t_struct STRUCT<
+        street STRING,
+        city STRING,
+        zip_code INT64
+    > default STRUCT('123 Example St', 'Sample City', 12345),
+    t_interval INTERVAL,
+    t_geo GEOGRAPHY
+) OPTIONS(description="Table description");
+
+create table if not exists dbml_test.test_constraint_1 (
+    id int64,
+    primary key (id) not enforced
+);
+
+create table if not exists dbml_test.test_constraint_2 (
+    id int64,
+    name string,
+
+    primary key (id, name) not enforced
+);
+
+create table if not exists dbml_test.test_constraint_3 (
+    id int64,
+    fk_id int64
+);
+alter table dbml_test.test_constraint_3
+add foreign key (fk_id) references dbml_test.test_constraint_1 (id) not enforced;
+
+create table if not exists dbml_test.test_constraint_4 (
+    fk_id int64,
+    fk_name string,
+    primary key (fk_id, fk_name) not enforced
+);
+alter table dbml_test.test_constraint_4
+add foreign key (fk_id, fk_name) references dbml_test.test_constraint_2 (id, name) not enforced;
+
+create table dbml_test.test_all_index (
+  a string,
+  b int64,
+  c struct <d int64, e array<string>, f struct<g string, h int64>>
+);
+
+create search index all_col_index
+on dbml_test.test_all_index(ALL COLUMNS)
+options (analyzer = 'NO_OP_ANALYZER');
+
+create table dbml_test.test_multi_index (a string, b int64, c json, d timestamp);
+
+create search index multi_index
+on dbml_test.test_multi_index (a, b, c)
+options ( data_types = ['string', 'int64', 'timestamp']);
+
+create table dbml_test.test_index (a string, b int64);
+
+create search index simple_index
+on dbml_test.test_index (a);