@dbml/cli 3.6.2 → 3.7.0

package/lib/connectors/PostgresConnector.js

@@ -0,0 +1,450 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.fetchSchemaJson = void 0;
+var _pg = require("pg");
+/* eslint-disable camelcase */
+
+const connectPg = async connection => {
+  const client = new _pg.Client(connection);
+  // bearer:disable javascript_lang_logger
+  client.on('error', err => console.log('PG connection error:', err));
+  await client.connect();
+  return client;
+};
+const convertQueryBoolean = val => val === 'YES';
+const getFieldType = (data_type, udt_name, character_maximum_length, numeric_precision, numeric_scale) => {
+  if (data_type === 'ARRAY') {
+    return `${udt_name.slice(1, udt_name.length)}[]`;
+  }
+  if (character_maximum_length) {
+    return `${udt_name}(${character_maximum_length})`;
+  }
+  if (numeric_precision && numeric_scale) {
+    return `${udt_name}(${numeric_precision},${numeric_scale})`;
+  }
+  return udt_name;
+};
+const getDbdefault = (data_type, column_default, default_type) => {
+  if (data_type === 'ARRAY') {
+    const values = column_default.slice(6, -1).split(',').map(value => {
+      return value.split('::')[0];
+    });
+    return {
+      type: default_type,
+      value: `ARRAY[${values.join(', ')}]`
+    };
+  }
+  if (default_type === 'string') {
+    const defaultValues = column_default.split('::')[0];
+    const isJson = data_type === 'json' || data_type === 'jsonb';
+    const type = isJson ? 'expression' : 'string';
+    return {
+      type,
+      value: defaultValues.slice(1, -1)
+    };
+  }
+  return {
+    type: default_type,
+    value: column_default
+  };
+};
+const generateField = row => {
+  const {
+    column_name,
+    data_type,
+    character_maximum_length,
+    numeric_precision,
+    numeric_scale,
+    udt_schema,
+    udt_name,
+    identity_increment,
+    is_nullable,
+    column_default,
+    default_type,
+    column_comment
+  } = row;
+  const dbdefault = column_default && default_type !== 'increment' ? getDbdefault(data_type, column_default, default_type) : null;
+  const fieldType = data_type === 'USER-DEFINED' ? {
+    type_name: udt_name,
+    schemaName: udt_schema
+  } : {
+    type_name: getFieldType(data_type, udt_name, character_maximum_length, numeric_precision, numeric_scale),
+    schemaname: null
+  };
+  return {
+    name: column_name,
+    type: fieldType,
+    dbdefault,
+    not_null: !convertQueryBoolean(is_nullable),
+    increment: !!identity_increment || default_type === 'increment',
+    note: column_comment ? {
+      value: column_comment
+    } : {
+      value: ''
+    }
+  };
+};
+const generateTablesAndFields = async client => {
+  const fields = {};
+  const tablesAndFieldsSql = `
+    WITH comments AS (
+      SELECT
+        pc.relname AS table_name,
+        pn.nspname AS table_schema,
+        pa.attname AS column_name,
+        pd.description
+      FROM
+        pg_description pd
+      JOIN
+        pg_class pc ON pd.objoid = pc.oid
+      JOIN
+        pg_namespace pn ON pc.relnamespace = pn.oid
+      LEFT JOIN
+        pg_attribute pa ON pd.objoid = pa.attrelid AND pd.objsubid = pa.attnum
+      WHERE
+        pc.relkind = 'r'
+        AND pn.nspname NOT IN ('pg_catalog', 'information_schema')
+    )
+    SELECT
+      t.table_schema,
+      t.table_name,
+      c.column_name,
+      c.data_type,
+      c.character_maximum_length,
+      c.numeric_precision,
+      c.numeric_scale,
+      c.udt_schema,
+      c.udt_name,
+      c.identity_increment,
+      c.is_nullable,
+      c.column_default,
+      CASE
+        WHEN c.column_default IS NULL THEN NULL
+        WHEN c.column_default LIKE 'nextval(%' THEN 'increment'
+        WHEN c.column_default LIKE '''%' THEN 'string'
+        WHEN c.column_default = 'true' OR c.column_default = 'false' THEN 'boolean'
+        WHEN c.column_default ~ '^-?[0-9]+(.[0-9]+)?$' THEN 'number'
+        ELSE 'expression'
+      END AS default_type,
+      (SELECT description FROM comments WHERE table_name = t.table_name AND table_schema = t.table_schema AND column_name IS NULL) AS table_comment,
+      (SELECT description FROM comments WHERE table_name = t.table_name AND table_schema = t.table_schema AND column_name = c.column_name) AS column_comment
+    FROM
+      information_schema.columns c
+    JOIN
+      information_schema.tables t ON c.table_name = t.table_name AND c.table_schema = t.table_schema
+    WHERE
+      t.table_type = 'BASE TABLE'
+      AND t.table_schema NOT IN ('pg_catalog', 'information_schema')
+    ORDER BY
+      t.table_schema,
+      t.table_name,
+      c.ordinal_position
+    ;
+  `;
+  const tablesAndFieldsResult = await client.query(tablesAndFieldsSql);
+  const tables = tablesAndFieldsResult.rows.reduce((acc, row) => {
+    const {
+      table_schema,
+      table_name,
+      table_comment
+    } = row;
+    if (!acc[table_name]) {
+      acc[table_name] = {
+        name: table_name,
+        schemaName: table_schema,
+        note: table_comment ? {
+          value: table_comment
+        } : {
+          value: ''
+        }
+      };
+    }
+    if (!fields[table_name]) fields[table_name] = [];
+    const field = generateField(row);
+    fields[table_name].push(field);
+    return acc;
+  }, {});
+  return {
+    tables: Object.values(tables),
+    fields
+  };
+};
+const generateRawRefs = async client => {
+  const refs = [];
+  const refsListSql = `
+    SELECT
+      tc.table_schema,
+      tc.table_name,
+      tc.constraint_name as fk_constraint_name,
+      STRING_AGG(DISTINCT kcu.column_name, ',') AS column_names,
+      ccu.table_schema AS foreign_table_schema,
+      ccu.table_name AS foreign_table_name,
+      STRING_AGG(DISTINCT ccu.column_name, ',') AS foreign_column_names,
+      tc.constraint_type,
+      rc.delete_rule AS on_delete,
+      rc.update_rule AS on_update
+    FROM information_schema.table_constraints AS tc
+    JOIN information_schema.key_column_usage AS kcu
+      ON tc.constraint_name = kcu.constraint_name
+      AND tc.table_schema = kcu.table_schema
+    JOIN information_schema.constraint_column_usage AS ccu
+      ON ccu.constraint_name = tc.constraint_name
+    JOIN information_schema.referential_constraints AS rc
+      ON tc.constraint_name = rc.constraint_name
+      AND tc.table_schema = rc.constraint_schema
+    WHERE tc.constraint_type = 'FOREIGN KEY'
+      AND tc.table_schema NOT IN ('pg_catalog', 'information_schema')
+    GROUP BY
+      tc.table_schema,
+      tc.table_name,
+      tc.constraint_name,
+      ccu.table_schema,
+      ccu.table_name,
+      tc.constraint_type,
+      rc.delete_rule,
+      rc.update_rule
+    ORDER BY
+      tc.table_schema,
+      tc.table_name;
+  `;
+  const refsQueryResult = await client.query(refsListSql);
+  refsQueryResult.rows.forEach(refRow => {
+    const {
+      table_schema,
+      fk_constraint_name,
+      table_name,
+      column_names,
+      foreign_table_schema,
+      foreign_table_name,
+      foreign_column_names,
+      on_delete,
+      on_update
+    } = refRow;
+    const ep1 = {
+      tableName: table_name,
+      schemaName: table_schema,
+      fieldNames: column_names.split(','),
+      relation: '*'
+    };
+    const ep2 = {
+      tableName: foreign_table_name,
+      schemaName: foreign_table_schema,
+      fieldNames: foreign_column_names.split(','),
+      relation: '1'
+    };
+    refs.push({
+      name: fk_constraint_name,
+      endpoints: [ep1, ep2],
+      onDelete: on_delete === 'NO ACTION' ? null : on_delete,
+      onUpdate: on_update === 'NO ACTION' ? null : on_update
+    });
+  });
+  return refs;
+};
+const generateIndexes = async client => {
+  // const tableConstraints = {};
+  const indexListSql = `
+    WITH user_tables AS (
+      SELECT tablename
+      FROM pg_tables
+      WHERE schemaname NOT IN ('pg_catalog', 'information_schema')  -- Exclude system schemas
+        AND tablename NOT LIKE 'pg_%'  -- Exclude PostgreSQL system tables
+        AND tablename NOT LIKE 'sql_%'  -- Exclude SQL standard tables
+    ),
+    index_info AS (
+      SELECT
+        t.relname AS table_name,
+        i.relname AS index_name,
+        ix.indisunique AS is_unique,
+        ix.indisprimary AS is_primary,
+        am.amname AS index_type,
+        array_to_string(array_agg(a.attname ORDER BY x.n), ', ') AS columns,
+        pg_get_expr(ix.indexprs, ix.indrelid) AS expressions,
+        CASE
+          WHEN ix.indisprimary THEN 'PRIMARY KEY'
+          WHEN ix.indisunique THEN 'UNIQUE'
+          ELSE NULL
+        END AS constraint_type
+      FROM
+        pg_class t
+        JOIN pg_index ix ON t.oid = ix.indrelid
+        JOIN pg_class i ON i.oid = ix.indexrelid
+        LEFT JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(ix.indkey)
+        JOIN pg_am am ON i.relam = am.oid
+        LEFT JOIN generate_subscripts(ix.indkey, 1) AS x(n) ON a.attnum = ix.indkey[x.n]
+      WHERE
+        t.relkind = 'r'
+        AND t.relname NOT LIKE 'pg_%'
+        AND t.relname NOT LIKE 'sql_%'
+      GROUP BY
+        t.relname, i.relname, ix.indisunique, ix.indisprimary, am.amname, ix.indexprs, ix.indrelid
+    )
+    SELECT
+      ut.tablename AS table_name,
+      ii.index_name,
+      ii.is_unique,
+      ii.is_primary,
+      ii.index_type,
+      ii.columns,
+      ii.expressions,
+      ii.constraint_type  -- Added constraint type
+    FROM
+      user_tables ut
+    LEFT JOIN
+      index_info ii ON ut.tablename = ii.table_name
+    WHERE ii.columns IS NOT NULL
+    ORDER BY
+      ut.tablename,
+      ii.constraint_type,
+      ii.index_name
+    ;
+  `;
+  const indexListResult = await client.query(indexListSql);
+  const {
+    outOfLineConstraints,
+    inlineConstraints
+  } = indexListResult.rows.reduce((acc, row) => {
+    const {
+      constraint_type,
+      columns
+    } = row;
+    if (columns === 'null' || columns.trim() === '') return acc;
+    const isSingleColumn = columns.split(',').length === 1;
+    const isInlineConstraint = isSingleColumn && (constraint_type === 'PRIMARY KEY' || constraint_type === 'UNIQUE');
+    if (isInlineConstraint) {
+      acc.inlineConstraints.push(row);
+    } else {
+      acc.outOfLineConstraints.push(row);
+    }
+    return acc;
+  }, {
+    outOfLineConstraints: [],
+    inlineConstraints: []
+  });
+  const indexes = outOfLineConstraints.reduce((acc, indexRow) => {
+    const {
+      table_name,
+      index_name,
+      index_type,
+      columns,
+      expressions
+    } = indexRow;
+    const indexColumns = columns.split(',').map(column => {
+      return {
+        type: 'column',
+        value: column.trim()
+      };
+    });
+    const indexExpressions = expressions ? expressions.split(',').map(expression => {
+      return {
+        type: 'expression',
+        value: expression
+      };
+    }) : [];
+    const index = {
+      name: index_name,
+      type: index_type,
+      columns: [...indexColumns, ...indexExpressions]
+    };
+    if (acc[table_name]) {
+      acc[table_name].push(index);
+    } else {
+      acc[table_name] = [index];
+    }
+    return acc;
+  }, {});
+  const tableConstraints = inlineConstraints.reduce((acc, row) => {
+    const {
+      table_name,
+      columns,
+      constraint_type
+    } = row;
+    if (!acc[table_name]) acc[table_name] = {};
+    const columnNames = columns.split(',').map(column => column.trim());
+    columnNames.forEach(columnName => {
+      if (!acc[table_name][columnName]) acc[table_name][columnName] = {};
+      if (constraint_type === 'PRIMARY KEY') {
+        acc[table_name][columnName].pk = true;
+      }
+      if (constraint_type === 'UNIQUE' && !acc[table_name][columnName].pk) {
+        acc[table_name][columnName].unique = true;
+      }
+    });
+    return acc;
+  }, {});
+  return {
+    indexes,
+    tableConstraints
+  };
+};
+const generateRawEnums = async client => {
+  const enumListSql = `
+    SELECT
+      n.nspname AS schema_name,
+      t.typname AS enum_type,
+      e.enumlabel AS enum_value,
+      e.enumsortorder AS sort_order
+    FROM
+      pg_enum e
+    JOIN
+      pg_type t ON e.enumtypid = t.oid
+    JOIN
+      pg_namespace n ON t.typnamespace = n.oid
+    ORDER BY
+      schema_name,
+      enum_type,
+      sort_order;
+    ;
+  `;
+  const enumListResult = await client.query(enumListSql);
+  const enums = enumListResult.rows.reduce((acc, row) => {
+    const {
+      schema_name,
+      enum_type,
+      enum_value
+    } = row;
+    if (!acc[enum_type]) {
+      acc[enum_type] = {
+        name: enum_type,
+        schemaName: schema_name,
+        values: []
+      };
+    }
+    acc[enum_type].values.push({
+      name: enum_value
+    });
+    return acc;
+  }, {});
+  return Object.values(enums);
+};
+const fetchSchemaJson = async connection => {
+  const client = await connectPg(connection);
+  const tablesAndFieldsRes = generateTablesAndFields(client);
+  const indexesRes = generateIndexes(client);
+  const refsRes = generateRawRefs(client);
+  const enumsRes = generateRawEnums(client);
+  const res = await Promise.all([tablesAndFieldsRes, indexesRes, refsRes, enumsRes]);
+  client.end();
+  const {
+    tables,
+    fields
+  } = res[0];
+  const {
+    indexes,
+    tableConstraints
+  } = res[1];
+  const refs = res[2];
+  const enums = res[3];
+  return {
+    tables,
+    fields,
+    refs,
+    enums,
+    indexes,
+    tableConstraints
+  };
+};
+exports.fetchSchemaJson = fetchSchemaJson;

package/lib/index.js

@@ -3,6 +3,12 @@
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
+Object.defineProperty(exports, "db2dbml", {
+  enumerable: true,
+  get: function () {
+    return _cli.db2dbml;
+  }
+});
 Object.defineProperty(exports, "dbml2sql", {
   enumerable: true,
   get: function () {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dbml/cli",
-  "version": "3.6.2",
+  "version": "3.7.0",
   "description": "",
   "main": "lib/index.js",
   "license": "Apache-2.0",
@@ -14,7 +14,8 @@
   },
   "bin": {
     "dbml2sql": "bin/dbml2sql.js",
-    "sql2dbml": "bin/sql2dbml.js"
+    "sql2dbml": "bin/sql2dbml.js",
+    "db2dbml": "bin/db2dbml.js"
   },
   "author": "Holistics <dev@holistics.io>",
   "homepage": "https://dbml.dbdiagram.io",
@@ -25,7 +26,7 @@
   ],
   "dependencies": {
     "@babel/cli": "^7.21.0",
-    "@dbml/core": "^3.6.2",
+    "@dbml/core": "^3.7.0",
     "bluebird": "^3.5.5",
     "chalk": "^2.4.2",
     "commander": "^2.20.0",
@@ -53,5 +54,5 @@
       "^.+\\.js$": "babel-jest"
     }
   },
-  "gitHead": "d3d82a3e8260f733098edc5fbfdbd3f8a381c6e3"
+  "gitHead": "3222baa4bf45b9ff74711a63d376505cf4bd9ac0"
 }

package/src/cli/connector.js

@@ -0,0 +1,31 @@
+import { importer } from '@dbml/core';
+import figures from 'figures';
+import chalk from 'chalk';
+import path from 'path';
+import {
+  resolvePaths,
+  getConnectionOpt,
+} from './utils';
+import OutputConsolePlugin from './outputPlugins/outputConsolePlugin';
+import OutputFilePlugin from './outputPlugins/outputFilePlugin';
+import logger from '../helpers/logger';
+
+export default async function connectionHandler (program) {
+  try {
+    const { connection, format } = getConnectionOpt(program.args);
+    const opts = program.opts();
+
+    if (!opts.outFile && !opts.outDir) {
+      const res = await importer.generateDbml(connection, format);
+      OutputConsolePlugin.write(res);
+    } else if (opts.outFile) {
+      const res = await importer.generateDbml(connection, format);
+      (new OutputFilePlugin(resolvePaths(opts.outFile))).write(res);
+
+      // bearer:disable javascript_lang_logger
+      console.log(`  ${chalk.green(figures.main.tick)} Generated DBML file from database's connection: ${path.basename(opts.outFile)}`);
+    }
+  } catch (error) {
+    logger.error(error);
+  }
+}

package/src/cli/index.js

@@ -1,8 +1,13 @@
 import program from 'commander';
 import importHandler from './import';
 import exportHandler from './export';
+import connectionHandler from './connector';
 import projectInfo from '../../package.json';
 
+function showHelp (args) {
+  if (args.length < 3) program.help();
+}
+
 function dbml2sql (args) {
   program.version(projectInfo.version);
 
@@ -15,6 +20,7 @@ function dbml2sql (args) {
     .option('-o, --out-file <pathspec>', 'compile all input files into a single files');
   // .option('-d, --out-dir <pathspec>', 'compile an input directory of dbml files into an output directory');
 
+  showHelp(args);
   program.parse(args);
 
   exportHandler(program);
@@ -34,12 +40,41 @@ function sql2dbml (args) {
     .option('-o, --out-file <pathspec>', 'compile all input files into a single files');
   // .option('-d, --out-dir <pathspec>', 'compile an input directory of sql files into an output directory');
 
+  showHelp(args);
   program.parse(args);
 
   importHandler(program);
 }
 
+function db2dbml (args) {
+  program.version(projectInfo.version);
+
+  // Q: How to write the arguments description for the below usage?
+  // A: The usage description is written in the following way:
+  //    - <format> your database format (postgres, mysql, mssql)
+  //    - <connection-string> your database connection string
+  //    - postgres: postgresql://user:password@localhost:5432/dbname
+  //    - mssql: 'Server=localhost,1433;Database=master;User Id=sa;Password=your_password;Encrypt=true;TrustServerCertificate
+  const description = `
+    <format> your database format (postgres, mysql, mssql)
+    <connection-string> your database connection string:
+      - postgres: postgresql://user:password@localhost:5432/dbname
+      - mysql: mysql://user:password@localhost:3306/dbname
+      - mssql: 'Server=localhost,1433;Database=master;User Id=sa;Password=your_password;Encrypt=true;TrustServerCertificate=true;'
+  `;
+  program
+    .usage('<format> <connection-string> [options]')
+    .description(description)
+    .option('-o, --out-file <pathspec>', 'compile all input files into a single files');
+
+  showHelp(args);
+  program.parse(args);
+
+  connectionHandler(program);
+}
+
 export {
   dbml2sql,
   sql2dbml,
+  db2dbml,
 };

package/src/cli/utils.js

@@ -1,6 +1,7 @@
 import path from 'path';
 import fs from 'fs';
 import { CompilerError } from '@dbml/core';
+import { reduce } from 'lodash';
 
 function resolvePaths (paths) {
   if (!Array.isArray(paths)) {
@@ -32,6 +33,27 @@ function getFormatOpt (opts) {
   return format;
 }
 
+function getConnectionOpt (args) {
+  const supportedDatabases = ['postgres', 'mysql', 'mssql'];
+  const defaultConnectionOpt = {
+    connection: args[0],
+    format: 'unknown',
+  };
+
+  return reduce(args, (connectionOpt, arg) => {
+    if (supportedDatabases.includes(arg)) connectionOpt.format = arg;
+    // Check if the arg is a connection string using regex
+    const connectionStringRegex = /^.*[:;]/;
+    if (connectionStringRegex.test(arg)) {
+      // Example: jdbc:mysql://localhost:3306/mydatabase
+      // Example: odbc:Driver={SQL Server};Server=myServerAddress;Database=myDataBase;Uid=myUsername;Pwd=myPassword;
+      connectionOpt.connection = arg;
+    }
+
+    return connectionOpt;
+  }, defaultConnectionOpt);
+}
+
 function generate (inputPaths, transform, outputPlugin) {
   inputPaths.forEach((_path) => {
     const source = fs.readFileSync(_path, 'utf-8');
@@ -49,5 +71,6 @@ export {
   resolvePaths,
   validateInputFilePaths,
   getFormatOpt,
+  getConnectionOpt,
   generate,
 };

package/src/index.js

@@ -1,6 +1,7 @@
-import { dbml2sql, sql2dbml } from './cli';
+import { dbml2sql, sql2dbml, db2dbml } from './cli';
 
 export {
   dbml2sql,
   sql2dbml,
+  db2dbml,
 };