@dbml/connector 5.2.0 → 5.3.0

package/src/connectors/oracle/indexes.ts

@@ -0,0 +1,71 @@
+import { Connection } from 'oracledb';
+import { Index, IndexesDictionary } from '../types';
+import { EXECUTE_OPTIONS } from './utils';
+
+export async function generateIndexes (client: Connection): Promise<IndexesDictionary> {
+  const query = `
+    SELECT
+      i.INDEX_NAME AS "index_name",
+      i.TABLE_NAME AS "table_name",
+      CASE
+        WHEN i.INDEX_TYPE LIKE '%NORMAL' THEN 'btree'
+        ELSE LOWER(REGEXP_REPLACE(i.INDEX_TYPE, '^FUNCTION-BASED ', ''))
+      END AS "index_type",
+      CASE WHEN i.UNIQUENESS = 'UNIQUE' THEN 1 ELSE 0 END AS "is_unique",
+      ic.COLUMN_NAME AS "column_name",
+      ic.COLUMN_POSITION AS "column_position",
+      ie.COLUMN_EXPRESSION AS "column_expression"
+    FROM USER_INDEXES i
+    LEFT JOIN USER_IND_COLUMNS ic
+      ON i.INDEX_NAME = ic.INDEX_NAME
+    LEFT JOIN USER_IND_EXPRESSIONS ie
+      ON i.INDEX_NAME = ie.INDEX_NAME
+      AND ic.COLUMN_POSITION = ie.COLUMN_POSITION
+    LEFT JOIN USER_OBJECTS obj
+      ON obj.OBJECT_NAME = i.TABLE_NAME
+      AND obj.OBJECT_TYPE = 'TABLE'
+    WHERE NOT EXISTS (
+      SELECT 1
+      FROM USER_CONSTRAINTS c
+      WHERE c.INDEX_NAME = i.INDEX_NAME
+    )
+    ORDER BY obj.CREATED, obj.OBJECT_ID, i.INDEX_NAME, ic.COLUMN_POSITION
+  `;
+
+  const res = await client.execute(query, [], EXECUTE_OPTIONS);
+
+  const indexes: IndexesDictionary = {};
+  const indexMap: Record<string, { table: string; type: string; unique: boolean; columns: Array<{ type: 'column' | 'expression'; value: string }> }> = {};
+
+  res.rows?.forEach((row) => {
+    const { index_name, table_name, index_type, is_unique, column_name, column_expression } = row as Record<string, unknown>;
+
+    const name = index_name as string;
+    const table = table_name as string;
+    const type = index_type as string;
+    const unique = !!(is_unique as number);
+
+    if (column_expression) {
+      if (!indexMap[name]) {
+        indexMap[name] = { table, type, unique, columns: [] };
+      }
+      indexMap[name].columns.push({ type: 'expression', value: column_expression as string });
+      return;
+    }
+    if (column_name) {
+      if (!indexMap[name]) {
+        indexMap[name] = { table, type, unique, columns: [] };
+      }
+      indexMap[name].columns.push({ type: 'column', value: column_name as string });
+      return;
+    }
+  });
+
+  Object.entries(indexMap).forEach(([name, { table, type, unique, columns }]) => {
+    const index: Index = { name, type, unique, columns };
+    if (!indexes[table]) indexes[table] = [];
+    indexes[table].push(index);
+  });
+
+  return indexes;
+}

package/src/connectors/oracle/refs.ts

@@ -0,0 +1,73 @@
+import { Connection } from 'oracledb';
+import { Ref, RefEndpoint } from '../types';
+import { EXECUTE_OPTIONS, LIST_SEPARATOR } from './utils';
+
+export async function generateRawRefs (client: Connection): Promise<Ref[]> {
+  const refs: Ref[] = [];
+
+  const refsListSql = `
+    SELECT
+      c.CONSTRAINT_NAME AS "fk_name",
+      c.TABLE_NAME AS "child_table",
+      LISTAGG(cc.COLUMN_NAME, '${LIST_SEPARATOR}') WITHIN GROUP (ORDER BY cc.POSITION) AS "child_columns",
+      p.TABLE_NAME AS "parent_table",
+      LISTAGG(pc.COLUMN_NAME, '${LIST_SEPARATOR}') WITHIN GROUP (ORDER BY pc.POSITION) AS "parent_columns",
+      c.DELETE_RULE AS "delete_rule"
+    FROM USER_CONSTRAINTS c
+    JOIN USER_CONSTRAINTS p
+      ON c.R_CONSTRAINT_NAME = p.CONSTRAINT_NAME
+    JOIN USER_CONS_COLUMNS cc
+      ON c.CONSTRAINT_NAME = cc.CONSTRAINT_NAME
+    JOIN USER_CONS_COLUMNS pc
+      ON p.CONSTRAINT_NAME = pc.CONSTRAINT_NAME
+      AND cc.POSITION = pc.POSITION
+    LEFT JOIN USER_OBJECTS obj_child
+      ON obj_child.OBJECT_NAME = c.TABLE_NAME
+      AND obj_child.OBJECT_TYPE = 'TABLE'
+    LEFT JOIN USER_OBJECTS obj_parent
+      ON obj_parent.OBJECT_NAME = p.TABLE_NAME
+      AND obj_parent.OBJECT_TYPE = 'TABLE'
+    WHERE c.CONSTRAINT_TYPE = 'R'
+    GROUP BY
+      c.CONSTRAINT_NAME,
+      c.TABLE_NAME,
+      p.TABLE_NAME,
+      c.DELETE_RULE,
+      obj_child.CREATED,
+      obj_child.OBJECT_ID,
+      obj_parent.CREATED,
+      obj_parent.OBJECT_ID
+    ORDER BY
+      obj_child.CREATED,
+      obj_child.OBJECT_ID,
+      c.CONSTRAINT_NAME
+  `;
+
+  const refsQueryResult = await client.execute(refsListSql, [], EXECUTE_OPTIONS);
+  refsQueryResult.rows?.forEach((refRow) => {
+    const { fk_name, child_table, child_columns, parent_table, parent_columns, delete_rule } = refRow as Record<string, string>;
+
+    const ep1: RefEndpoint = {
+      tableName: child_table,
+      schemaName: '',
+      fieldNames: child_columns.split(LIST_SEPARATOR),
+      relation: '*',
+    };
+
+    const ep2: RefEndpoint = {
+      tableName: parent_table,
+      schemaName: '',
+      fieldNames: parent_columns.split(LIST_SEPARATOR),
+      relation: '1',
+    };
+
+    refs.push({
+      name: fk_name,
+      endpoints: [ep1, ep2],
+      onDelete: delete_rule === 'NO ACTION' ? null : delete_rule,
+      onUpdate: null,
+    });
+  });
+
+  return refs;
+}

package/src/connectors/oracle/tables.ts

@@ -0,0 +1,305 @@
+import { Connection } from 'oracledb';
+import { DefaultInfo, DefaultType, Field, FieldsDictionary, Table } from '../types';
+import { EXECUTE_OPTIONS } from './utils';
+
+export async function generateTablesAndFields (client: Connection): Promise<{
+  tables: Table[];
+  fields: FieldsDictionary;
+}> {
+  const fields: FieldsDictionary = {};
+  // Note: DATA_DEFAULT is LONG type which doesn't support SQL string operations
+  // The oracledb driver fetches LONG as string, so we detect default type in JavaScript
+  const tablesAndFieldsSql = `
+    SELECT
+      cols.TABLE_NAME AS "table_name",
+      cols.COLUMN_NAME AS "column_name",
+      cols.COLUMN_ID AS "column_id",
+      cols.DATA_TYPE AS "data_type",
+      cols.DATA_LENGTH AS "data_length",
+      cols.DATA_PRECISION AS "data_precision",
+      cols.DATA_SCALE AS "data_scale",
+      CASE WHEN cols.IDENTITY_COLUMN = 'YES' THEN 1 ELSE 0 END AS "identity_increment",
+      CASE WHEN cols.NULLABLE = 'Y' THEN 1 ELSE 0 END AS "is_nullable",
+      cols.DATA_DEFAULT AS "data_default",
+      tcoms.COMMENTS AS "table_comment",
+      ccoms.COMMENTS AS "column_comment",
+      obj.CREATED AS "table_created"
+    FROM USER_TAB_COLUMNS cols
+    LEFT JOIN USER_COL_COMMENTS ccoms
+      ON ccoms.COLUMN_NAME = cols.COLUMN_NAME
+      AND ccoms.TABLE_NAME = cols.TABLE_NAME
+    LEFT JOIN USER_TAB_COMMENTS tcoms
+      ON tcoms.TABLE_NAME = cols.TABLE_NAME
+    LEFT JOIN USER_OBJECTS obj
+      ON obj.OBJECT_NAME = cols.TABLE_NAME
+      AND obj.OBJECT_TYPE = 'TABLE'
+    ORDER BY obj.CREATED, obj.OBJECT_ID, cols.COLUMN_ID
+  `;
+
+  const tablesAndFieldsResult = await client.execute(tablesAndFieldsSql, [], EXECUTE_OPTIONS);
+  const tableMap = tablesAndFieldsResult.rows?.reduce((acc: Record<string, Table>, row) => {
+    const { table_name, table_comment } = row as any;
+    if (!acc[table_name]) {
+      acc[table_name] = {
+        name: table_name,
+        schemaName: '',
+        note: table_comment ? { value: table_comment } : { value: '' },
+      };
+    }
+
+    if (!fields[table_name]) fields[table_name] = [];
+    const field = generateField(row as any);
+    fields[table_name].push(field);
+
+    return acc;
+  }, {} as Record<string, Table>) || {};
+
+  return {
+    tables: Object.values(tableMap),
+    fields,
+  };
+}
+
+// Field utils
+
+function getDefaultType (columnDefault: string | null | undefined): string {
+  if (columnDefault === null || columnDefault === undefined) {
+    return 'expression';
+  }
+  const trimmed = columnDefault.trim();
+  // String defaults are wrapped in single quotes: 'value'
+  // Example: `'string'`, `'string with quotes'''`
+  if (/^'.*'$/.test(trimmed)) {
+    return 'string';
+  }
+  // Number defaults are integers or non-scientific floats
+  // Example: `123`, `0.12`, `.12`, `12.`
+  if (/^[0-9]+$/.test(trimmed) || /^[0-9]*\.[0-9]*$/.test(trimmed)) {
+    return 'number';
+  }
+  // Example: Strings that are not enclosed in `'` or do not contain only numbers and `.`
+  //   * CURRENT_TIMESTAMP
+  //   * SYSDATE
+  //   * "number-column" + 2
+  return 'expression';
+}
+
+function generateField (row: Record<string, any>): Field {
+  const {
+    column_name,
+    data_type,
+    data_length,
+    data_precision,
+    data_scale,
+    identity_increment,
+    is_nullable,
+    data_default,
+    column_comment,
+  } = row;
+
+  const defaultType = getDefaultType(data_default);
+  const dbdefault = data_default && !identity_increment ? getDbdefault(data_default, defaultType) : null;
+
+  const fieldType = {
+    type_name: getFieldType(data_type, data_length, data_precision, data_scale),
+    schemaName: null,
+  };
+
+  return {
+    name: column_name,
+    type: fieldType,
+    dbdefault,
+    not_null: !is_nullable,
+    increment: !!identity_increment,
+    note: column_comment ? { value: column_comment } : { value: '' },
+  };
+}
+
+// Field default utils
+
+function getDbdefault (columnDefault: string | null, defaultType: string): DefaultInfo | null {
+  if (columnDefault === null) {
+    return null;
+  }
+  if (defaultType === 'string') {
+    const trimmed = columnDefault.trim();
+    return {
+      value: trimmed.slice(1, -1) // slice off single quotes at the start and end of the value
+        .replaceAll('\'\'', '\''), // escape sequences are preserved: in oracle, double single quotes = escape a single quote
+      type: defaultType,
+    };
+  }
+
+  return { value: columnDefault, type: defaultType as DefaultType };
+}
+
+// Field type utils
+
+// These datatypes are never qualified: data size, data scale and data precision are all ignore.
+// Ref: https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/Data-Types.html
+const UNQUALIFIED_DATA_TYPES = new Set([
+  // datetime
+  'date',
+  // number
+  'binary_float',
+  'binary_double',
+  // long and raw
+  'long',
+  'long raw',
+  // large object
+  'blob',
+  'clob',
+  'nclob',
+  'bfile',
+  // rowid
+  'rowid',
+  // ansi supported
+  'integer',
+  'int',
+  'smallint',
+  'double precision',
+  'real',
+  // any
+  'AnyData',
+  'AnyType',
+  'AnyDataSet',
+  // xml
+  'XMLType',
+  'URIType',
+  // spatial
+  'SDO_Geometry',
+  'SDO_Topo_Geometry',
+  'SDO_GeoRaster',
+]);
+
+// These datatypes can be and only be qualified with data size
+// For simplicity the data size in bytes is assumed
+// Ref: https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/Data-Types.html
+const SIZED_DATA_TYPES = new Set([
+  // character
+  'char',
+  'varchar2',
+  'nchar',
+  'nvarchar2',
+  // long and raw
+  'raw',
+  // rowid
+  'urowid',
+  // ansi supported
+  'character varying',
+  'character',
+  'char varying',
+  'nchar varying',
+  'varchar',
+  'national character varying',
+  'national character',
+  'national char varying',
+  'national char',
+]);
+
+// These datatypes can be and only be qualified with precision
+// For datetime, these are the types that only have 1 precision
+// Ref: https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/Data-Types.html
+const PRECISION_DATA_TYPES = new Set([
+  // number
+  'float',
+]);
+
+// This datatype can be qualified with a fractional second precision
+const TIMESTAMP_WITH_LOCAL_TIME_ZONE_DATA_TYPE = 'timestamp with local time zone';
+// This datatype can be qualified with a second precision
+const TIMESTAMP_WITH_TIME_ZONE_DATA_TYPE = 'timestamp with time zone';
+// This datatype can be qualified with a second precision
+const TIMESTAMP_DATA_TYPE = 'timestamp';
+// This datatype can be qualified with a year precision
+const INTERVAL_YEAR_DATA_TYPE = 'interval year to month';
+// This datatype can be qualified with a day precision and a fraction second precision
+const INTERVAL_DAY_DATA_TYPE = 'interval day to second';
+
+// These datatypes can be and only be qualified with precision and scale
+// For datetime, these are the types that can have 2 precisions
+// Ref: https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/Data-Types.html
+const PRECISION_SCALED_DATA_TYPES = new Set([
+  // number
+  'number',
+  // ansi supported
+  'numeric',
+  'decimal',
+  'dec',
+]);
+
+// Format the field type to a standard string for aesthetic when exporting to DBML (`ANYDATA`, `anydata` are all normalized to `AnyData`)
+// Some types contain multiple words with arbitrary spaces, for example, `TIMESTAMP (6)   WITH LOCAL TIME ZONE`, we should normalize this to allow processing to be uniform, for example when checking for equality
+function normalizeFieldType (dataType: string): string {
+  const normalizedDataType = dataType.split(' ').filter(Boolean).join(' ').toLowerCase();
+  switch (normalizedDataType) {
+    case 'anydata':
+      return 'AnyData';
+    case 'anytype':
+      return 'AnyType';
+    case 'anydataset':
+      return 'AnyDataSet';
+    case 'xmltype':
+      return 'XMLType';
+    case 'uritype':
+      return 'URIType';
+    case 'sdo_geometry':
+      return 'SDO_Geometry';
+    case 'sdo_topo_geometry':
+      return 'SDO_Topo_Geometry';
+    case 'sdo_georaster':
+      return 'SDO_GeoRaster';
+    default:
+      return normalizedDataType;
+  }
+}
+
+export function getFieldType (dataType: string, characterMaximumLength: number, numericPrecision: number | null, numericScale: number | null): string {
+  const normalizedDataType = normalizeFieldType(dataType);
+  if (normalizedDataType === TIMESTAMP_WITH_LOCAL_TIME_ZONE_DATA_TYPE) {
+    if (numericScale !== null) return `timestamp (${numericScale}) with local time zone`;
+    return normalizedDataType;
+  }
+  if (normalizedDataType === TIMESTAMP_WITH_TIME_ZONE_DATA_TYPE) {
+    if (numericScale !== null) return `timestamp (${numericScale}) with time zone`;
+    return normalizedDataType;
+  }
+  if (normalizedDataType === TIMESTAMP_DATA_TYPE) {
+    if (numericScale !== null) return `timestamp (${numericScale})`;
+    return normalizedDataType;
+  }
+  if (normalizedDataType === INTERVAL_YEAR_DATA_TYPE) {
+    if (numericPrecision !== null) return `interval year (${numericPrecision}) to month`;
+    return normalizedDataType;
+  }
+  if (normalizedDataType === INTERVAL_DAY_DATA_TYPE) {
+    if (numericPrecision !== null && numericScale !== null) return `interval day (${numericPrecision}) to second (${numericScale})`;
+    if (numericPrecision !== null) return `interval day (${numericPrecision}) to second`;
+    if (numericScale !== null) return `interval day to second (${numericScale})`;
+    return normalizedDataType;
+  }
+
+  if (SIZED_DATA_TYPES.has(normalizedDataType)) {
+    if (characterMaximumLength) return `${normalizedDataType}(${characterMaximumLength})`;
+    return normalizedDataType;
+  }
+
+  if (PRECISION_DATA_TYPES.has(normalizedDataType)) {
+    if (numericPrecision !== null) return `${normalizedDataType}(${numericPrecision})`;
+    return normalizedDataType;
+  }
+
+  if (PRECISION_SCALED_DATA_TYPES.has(normalizedDataType)) {
+    if (numericPrecision !== null && numericScale !== null) return `${normalizedDataType}(${numericPrecision}, ${numericScale})`;
+    if (numericPrecision !== null) return `${normalizedDataType}(${numericPrecision})`;
+    // There's no situation where numericScale is not null but numericPrecision is null
+    return normalizedDataType;
+  }
+
+  // This is not needed since we return `normalizedDataType` either if the condition holds true or not
+  // if (UNQUALIFIED_DATA_TYPES.has(normalizedDataType)) {
+  //   return normalizedDataType;
+  // }
+
+  return normalizedDataType;
+};

package/src/connectors/oracle/utils.ts

@@ -0,0 +1,23 @@
+import oracledb from 'oracledb';
+
+export const LIST_SEPARATOR = '@@@@@@@@'; // Used to join multiple fields into a string field to simulate array column types
+
+export const EXECUTE_OPTIONS = { outFormat: oracledb.OUT_FORMAT_OBJECT };
+
+// Expect an Easy Connect string format: username/password@[//]host[:port][/database]
+// Explanation of the format: https://www.orafaq.com/wiki/EZCONNECT
+export function processEasyConnectString (connection: string): { username: string; password: string; dbidentifier: string } {
+  // Capturing group `username`: a string without the characters `/` `@` `:`
+  // Capturing group `password`: a string without the characters `/` `@` `:`
+  // Capturing group `dbidentifier`: a string containing the `host[:port][/database]` part
+  const matches = connection.match(/^(?<username>[^/@:]+)\/(?<password>[^/@:]+)@(\/\/)?(?<dbidentifier>.+)$/);
+  const { username, password, dbidentifier } = matches?.groups || {};
+  if (!username || !password || !dbidentifier) {
+    throw new Error('Invalid Easy Connect string. Expect a string of format \'username/password@[//]host[:port][/database]\'');
+  }
+  return {
+    username,
+    password,
+    dbidentifier,
+  };
+}

package/src/connectors/{postgresConnector.ts → postgres/index.ts}

@@ -1,5 +1,5 @@
 import { Client } from 'pg';
-import { buildSchemaQuery, parseConnectionString } from '../utils/parseSchema';
+import { buildSchemaQuery, parseConnectionString } from '../../utils/parseSchema';
 import {
   CheckConstraintDictionary,
   DatabaseSchema,
@@ -13,7 +13,7 @@ import {
   RefEndpoint,
   Table,
   TableConstraintsDictionary,
-} from './types';
+} from '../types';
 
 const getValidatedClient = async (connection: string): Promise<Client> => {
   const client = new Client(connection);

package/src/connectors/{snowflakeConnector.ts → snowflake/index.ts}

@@ -8,7 +8,7 @@
 
 import snowflake from 'snowflake-sdk';
 import type { Connection, LogLevel, ConnectionOptions } from 'snowflake-sdk';
-import { parseSchema } from '../utils/parseSchema';
+import { parseSchema } from '../../utils/parseSchema';
 import {
   DatabaseSchema,
   DefaultInfo,
@@ -19,7 +19,7 @@ import {
   IndexesDictionary,
   Table,
   TableConstraintsDictionary,
-} from './types';
+} from '../types';
 
 type ConstraintRow = {
   schemaName: string;

package/src/connectors/types.ts

@@ -135,5 +135,6 @@ export {
   IndexesDictionary,
   DatabaseSchema,
   BigQueryCredentials,
+  CheckConstraint,
   CheckConstraintDictionary,
 };

package/test-output.log

@@ -1,18 +0,0 @@
-
-> @dbml/connector@5.1.0 test
-> jest
-
-PASS __tests__/connector.test.ts
-  @dbml/connector
-    ✓ connectors/mssql (264 ms)
-    ✓ connectors/mysql (31 ms)
-    ✓ connectors/postgres (55 ms)
-
-Test Suites: 1 passed, 1 total
-Tests:       3 passed, 3 total
-Snapshots:   0 total
-Time:        1.208 s, estimated 2 s
-Ran all test suites.
-Jest did not exit one second after the test run has completed.
-
-'This usually means that there are asynchronous operations that weren't stopped in your tests. Consider running Jest with `--detectOpenHandles` to troubleshoot this issue.