@dbml/connector 5.1.0 → 5.3.0

package/src/connectors/oracle/tables.ts

@@ -0,0 +1,305 @@
+import { Connection } from 'oracledb';
+import { DefaultInfo, DefaultType, Field, FieldsDictionary, Table } from '../types';
+import { EXECUTE_OPTIONS } from './utils';
+
+export async function generateTablesAndFields (client: Connection): Promise<{
+  tables: Table[];
+  fields: FieldsDictionary;
+}> {
+  const fields: FieldsDictionary = {};
+  // Note: DATA_DEFAULT is LONG type which doesn't support SQL string operations
+  // The oracledb driver fetches LONG as string, so we detect default type in JavaScript
+  const tablesAndFieldsSql = `
+    SELECT
+      cols.TABLE_NAME AS "table_name",
+      cols.COLUMN_NAME AS "column_name",
+      cols.COLUMN_ID AS "column_id",
+      cols.DATA_TYPE AS "data_type",
+      cols.DATA_LENGTH AS "data_length",
+      cols.DATA_PRECISION AS "data_precision",
+      cols.DATA_SCALE AS "data_scale",
+      CASE WHEN cols.IDENTITY_COLUMN = 'YES' THEN 1 ELSE 0 END AS "identity_increment",
+      CASE WHEN cols.NULLABLE = 'Y' THEN 1 ELSE 0 END AS "is_nullable",
+      cols.DATA_DEFAULT AS "data_default",
+      tcoms.COMMENTS AS "table_comment",
+      ccoms.COMMENTS AS "column_comment",
+      obj.CREATED AS "table_created"
+    FROM USER_TAB_COLUMNS cols
+    LEFT JOIN USER_COL_COMMENTS ccoms
+      ON ccoms.COLUMN_NAME = cols.COLUMN_NAME
+      AND ccoms.TABLE_NAME = cols.TABLE_NAME
+    LEFT JOIN USER_TAB_COMMENTS tcoms
+      ON tcoms.TABLE_NAME = cols.TABLE_NAME
+    LEFT JOIN USER_OBJECTS obj
+      ON obj.OBJECT_NAME = cols.TABLE_NAME
+      AND obj.OBJECT_TYPE = 'TABLE'
+    ORDER BY obj.CREATED, obj.OBJECT_ID, cols.COLUMN_ID
+  `;
+
+  const tablesAndFieldsResult = await client.execute(tablesAndFieldsSql, [], EXECUTE_OPTIONS);
+  const tableMap = tablesAndFieldsResult.rows?.reduce((acc: Record<string, Table>, row) => {
+    const { table_name, table_comment } = row as any;
+    if (!acc[table_name]) {
+      acc[table_name] = {
+        name: table_name,
+        schemaName: '',
+        note: table_comment ? { value: table_comment } : { value: '' },
+      };
+    }
+
+    if (!fields[table_name]) fields[table_name] = [];
+    const field = generateField(row as any);
+    fields[table_name].push(field);
+
+    return acc;
+  }, {} as Record<string, Table>) || {};
+
+  return {
+    tables: Object.values(tableMap),
+    fields,
+  };
+}
+
+// Field utils
+
+function getDefaultType (columnDefault: string | null | undefined): string {
+  if (columnDefault === null || columnDefault === undefined) {
+    return 'expression';
+  }
+  const trimmed = columnDefault.trim();
+  // String defaults are wrapped in single quotes: 'value'
+  // Example: `'string'`, `'string with quotes'''`
+  if (/^'.*'$/.test(trimmed)) {
+    return 'string';
+  }
+  // Number defaults are integers or non-scientific floats
+  // Example: `123`, `0.12`, `.12`, `12.`
+  if (/^[0-9]+$/.test(trimmed) || /^[0-9]*\.[0-9]*$/.test(trimmed)) {
+    return 'number';
+  }
+  // Example: Strings that are not enclosed in `'` or do not contain only numbers and `.`
+  //   * CURRENT_TIMESTAMP
+  //   * SYSDATE
+  //   * "number-column" + 2
+  return 'expression';
+}
+
+function generateField (row: Record<string, any>): Field {
+  const {
+    column_name,
+    data_type,
+    data_length,
+    data_precision,
+    data_scale,
+    identity_increment,
+    is_nullable,
+    data_default,
+    column_comment,
+  } = row;
+
+  const defaultType = getDefaultType(data_default);
+  const dbdefault = data_default && !identity_increment ? getDbdefault(data_default, defaultType) : null;
+
+  const fieldType = {
+    type_name: getFieldType(data_type, data_length, data_precision, data_scale),
+    schemaName: null,
+  };
+
+  return {
+    name: column_name,
+    type: fieldType,
+    dbdefault,
+    not_null: !is_nullable,
+    increment: !!identity_increment,
+    note: column_comment ? { value: column_comment } : { value: '' },
+  };
+}
+
+// Field default utils
+
+function getDbdefault (columnDefault: string | null, defaultType: string): DefaultInfo | null {
+  if (columnDefault === null) {
+    return null;
+  }
+  if (defaultType === 'string') {
+    const trimmed = columnDefault.trim();
+    return {
+      value: trimmed.slice(1, -1) // slice off single quotes at the start and end of the value
+        .replaceAll('\'\'', '\''), // escape sequences are preserved: in oracle, double single quotes = escape a single quote
+      type: defaultType,
+    };
+  }
+
+  return { value: columnDefault, type: defaultType as DefaultType };
+}
+
+// Field type utils
+
+// These datatypes are never qualified: data size, data scale and data precision are all ignore.
+// Ref: https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/Data-Types.html
+const UNQUALIFIED_DATA_TYPES = new Set([
+  // datetime
+  'date',
+  // number
+  'binary_float',
+  'binary_double',
+  // long and raw
+  'long',
+  'long raw',
+  // large object
+  'blob',
+  'clob',
+  'nclob',
+  'bfile',
+  // rowid
+  'rowid',
+  // ansi supported
+  'integer',
+  'int',
+  'smallint',
+  'double precision',
+  'real',
+  // any
+  'AnyData',
+  'AnyType',
+  'AnyDataSet',
+  // xml
+  'XMLType',
+  'URIType',
+  // spatial
+  'SDO_Geometry',
+  'SDO_Topo_Geometry',
+  'SDO_GeoRaster',
+]);
+
+// These datatypes can be and only be qualified with data size
+// For simplicity the data size in bytes is assumed
+// Ref: https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/Data-Types.html
+const SIZED_DATA_TYPES = new Set([
+  // character
+  'char',
+  'varchar2',
+  'nchar',
+  'nvarchar2',
+  // long and raw
+  'raw',
+  // rowid
+  'urowid',
+  // ansi supported
+  'character varying',
+  'character',
+  'char varying',
+  'nchar varying',
+  'varchar',
+  'national character varying',
+  'national character',
+  'national char varying',
+  'national char',
+]);
+
+// These datatypes can be and only be qualified with precision
+// For datetime, these are the types that only have 1 precision
+// Ref: https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/Data-Types.html
+const PRECISION_DATA_TYPES = new Set([
+  // number
+  'float',
+]);
+
+// This datatype can be qualified with a fractional second precision
+const TIMESTAMP_WITH_LOCAL_TIME_ZONE_DATA_TYPE = 'timestamp with local time zone';
+// This datatype can be qualified with a second precision
+const TIMESTAMP_WITH_TIME_ZONE_DATA_TYPE = 'timestamp with time zone';
+// This datatype can be qualified with a second precision
+const TIMESTAMP_DATA_TYPE = 'timestamp';
+// This datatype can be qualified with a year precision
+const INTERVAL_YEAR_DATA_TYPE = 'interval year to month';
+// This datatype can be qualified with a day precision and a fraction second precision
+const INTERVAL_DAY_DATA_TYPE = 'interval day to second';
+
+// These datatypes can be and only be qualified with precision and scale
+// For datetime, these are the types that can have 2 precisions
+// Ref: https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/Data-Types.html
+const PRECISION_SCALED_DATA_TYPES = new Set([
+  // number
+  'number',
+  // ansi supported
+  'numeric',
+  'decimal',
+  'dec',
+]);
+
+// Format the field type to a standard string for aesthetic when exporting to DBML (`ANYDATA`, `anydata` are all normalized to `AnyData`)
+// Some types contain multiple words with arbitrary spaces, for example, `TIMESTAMP (6)   WITH LOCAL TIME ZONE`, we should normalize this to allow processing to be uniform, for example when checking for equality
+function normalizeFieldType (dataType: string): string {
+  const normalizedDataType = dataType.split(' ').filter(Boolean).join(' ').toLowerCase();
+  switch (normalizedDataType) {
+    case 'anydata':
+      return 'AnyData';
+    case 'anytype':
+      return 'AnyType';
+    case 'anydataset':
+      return 'AnyDataSet';
+    case 'xmltype':
+      return 'XMLType';
+    case 'uritype':
+      return 'URIType';
+    case 'sdo_geometry':
+      return 'SDO_Geometry';
+    case 'sdo_topo_geometry':
+      return 'SDO_Topo_Geometry';
+    case 'sdo_georaster':
+      return 'SDO_GeoRaster';
+    default:
+      return normalizedDataType;
+  }
+}
+
+export function getFieldType (dataType: string, characterMaximumLength: number, numericPrecision: number | null, numericScale: number | null): string {
+  const normalizedDataType = normalizeFieldType(dataType);
+  if (normalizedDataType === TIMESTAMP_WITH_LOCAL_TIME_ZONE_DATA_TYPE) {
+    if (numericScale !== null) return `timestamp (${numericScale}) with local time zone`;
+    return normalizedDataType;
+  }
+  if (normalizedDataType === TIMESTAMP_WITH_TIME_ZONE_DATA_TYPE) {
+    if (numericScale !== null) return `timestamp (${numericScale}) with time zone`;
+    return normalizedDataType;
+  }
+  if (normalizedDataType === TIMESTAMP_DATA_TYPE) {
+    if (numericScale !== null) return `timestamp (${numericScale})`;
+    return normalizedDataType;
+  }
+  if (normalizedDataType === INTERVAL_YEAR_DATA_TYPE) {
+    if (numericPrecision !== null) return `interval year (${numericPrecision}) to month`;
+    return normalizedDataType;
+  }
+  if (normalizedDataType === INTERVAL_DAY_DATA_TYPE) {
+    if (numericPrecision !== null && numericScale !== null) return `interval day (${numericPrecision}) to second (${numericScale})`;
+    if (numericPrecision !== null) return `interval day (${numericPrecision}) to second`;
+    if (numericScale !== null) return `interval day to second (${numericScale})`;
+    return normalizedDataType;
+  }
+
+  if (SIZED_DATA_TYPES.has(normalizedDataType)) {
+    if (characterMaximumLength) return `${normalizedDataType}(${characterMaximumLength})`;
+    return normalizedDataType;
+  }
+
+  if (PRECISION_DATA_TYPES.has(normalizedDataType)) {
+    if (numericPrecision !== null) return `${normalizedDataType}(${numericPrecision})`;
+    return normalizedDataType;
+  }
+
+  if (PRECISION_SCALED_DATA_TYPES.has(normalizedDataType)) {
+    if (numericPrecision !== null && numericScale !== null) return `${normalizedDataType}(${numericPrecision}, ${numericScale})`;
+    if (numericPrecision !== null) return `${normalizedDataType}(${numericPrecision})`;
+    // There's no situation where numericScale is not null but numericPrecision is null
+    return normalizedDataType;
+  }
+
+  // This is not needed since we return `normalizedDataType` either if the condition holds true or not
+  // if (UNQUALIFIED_DATA_TYPES.has(normalizedDataType)) {
+  //   return normalizedDataType;
+  // }
+
+  return normalizedDataType;
+};

package/src/connectors/oracle/utils.ts

@@ -0,0 +1,23 @@
+import oracledb from 'oracledb';
+
+export const LIST_SEPARATOR = '@@@@@@@@'; // Used to join multiple fields into a string field to simulate array column types
+
+export const EXECUTE_OPTIONS = { outFormat: oracledb.OUT_FORMAT_OBJECT };
+
+// Expect an Easy Connect string format: username/password@[//]host[:port][/database]
+// Explanation of the format: https://www.orafaq.com/wiki/EZCONNECT
+export function processEasyConnectString (connection: string): { username: string; password: string; dbidentifier: string } {
+  // Capturing group `username`: a string without the characters `/` `@` `:`
+  // Capturing group `password`: a string without the characters `/` `@` `:`
+  // Capturing group `dbidentifier`: a string containing the `host[:port][/database]` part
+  const matches = connection.match(/^(?<username>[^/@:]+)\/(?<password>[^/@:]+)@(\/\/)?(?<dbidentifier>.+)$/);
+  const { username, password, dbidentifier } = matches?.groups || {};
+  if (!username || !password || !dbidentifier) {
+    throw new Error('Invalid Easy Connect string. Expect a string of format \'username/password@[//]host[:port][/database]\'');
+  }
+  return {
+    username,
+    password,
+    dbidentifier,
+  };
+}

package/src/connectors/{postgresConnector.ts → postgres/index.ts}

@@ -1,6 +1,5 @@
-/* eslint-disable camelcase */
 import { Client } from 'pg';
-import { buildSchemaQuery, parseConnectionString } from '../utils/parseSchema';
+import { buildSchemaQuery, parseConnectionString } from '../../utils/parseSchema';
 import {
   CheckConstraintDictionary,
   DatabaseSchema,
@@ -14,7 +13,7 @@ import {
   RefEndpoint,
   Table,
   TableConstraintsDictionary,
-} from './types';
+} from '../types';
 
 const getValidatedClient = async (connection: string): Promise<Client> => {
   const client = new Client(connection);
@@ -97,13 +96,15 @@ const generateField = (row: Record<string, any>): Field => {
 
   const dbdefault = column_default && default_type !== 'increment' ? getDbdefault(data_type, column_default, default_type) : null;
 
-  const fieldType = data_type === 'USER-DEFINED' ? {
-    type_name: udt_name,
-    schemaName: udt_schema,
-  } : {
-    type_name: getFieldType(data_type, udt_name, character_maximum_length, numeric_precision, numeric_scale),
-    schemaName: null,
-  };
+  const fieldType = data_type === 'USER-DEFINED'
+    ? {
+        type_name: udt_name,
+        schemaName: udt_schema,
+      }
+    : {
+        type_name: getFieldType(data_type, udt_name, character_maximum_length, numeric_precision, numeric_scale),
+        schemaName: null,
+      };
 
   return {
     name: column_name,
@@ -116,8 +117,8 @@ const generateField = (row: Record<string, any>): Field => {
 };
 
 const generateTablesAndFields = async (client: Client, schemas: string[]): Promise<{
-  tables: Table[],
-  fields: FieldsDictionary,
+  tables: Table[];
+  fields: FieldsDictionary;
 }> => {
   const fields: FieldsDictionary = {};
   const tablesAndFieldsSql = `
@@ -388,6 +389,8 @@ const generateIndexesAndConstraints = async (client: Client, schemas: string[])
       index_type,
       index_columns,
       index_expressions,
+      is_primary,
+      is_unique,
     } = indexRow;
     const indexColumns = index_columns.split(',').map((column: string) => {
       return {
@@ -396,12 +399,14 @@ const generateIndexesAndConstraints = async (client: Client, schemas: string[])
       };
     });
 
-    const indexExpressions = index_expressions ? index_expressions.split(',').map((expression: string) => {
-      return {
-        type: 'expression',
-        value: expression,
-      };
-    }) : [];
+    const indexExpressions = index_expressions
+      ? index_expressions.split(',').map((expression: string) => {
+          return {
+            type: 'expression',
+            value: expression,
+          };
+        })
+      : [];
 
     const index = {
       name: index_name,
@@ -410,6 +415,8 @@ const generateIndexesAndConstraints = async (client: Client, schemas: string[])
         ...indexColumns,
         ...indexExpressions,
       ],
+      pk: !!is_primary,
+      unique: !is_primary && !!is_unique,
     };
 
     const key = `${table_schema}.${table_name}`;

package/src/connectors/{snowflakeConnector.ts → snowflake/index.ts}

@@ -1,4 +1,3 @@
-/* eslint-disable camelcase */
 //
 // Description:
 // Snowflake info schema lacks reference direction for relationships.
@@ -9,7 +8,7 @@
 
 import snowflake from 'snowflake-sdk';
 import type { Connection, LogLevel, ConnectionOptions } from 'snowflake-sdk';
-import { parseSchema } from '../utils/parseSchema';
+import { parseSchema } from '../../utils/parseSchema';
 import {
   DatabaseSchema,
   DefaultInfo,
@@ -20,7 +19,7 @@ import {
   IndexesDictionary,
   Table,
   TableConstraintsDictionary,
-} from './types';
+} from '../types';
 
 type ConstraintRow = {
   schemaName: string;
@@ -196,8 +195,8 @@ const generateField = (row: Record<string, any>): Field => {
 };
 
 const generateTablesAndFields = async (conn: Connection, schemas: string[]): Promise<{
-  tables: Table[],
-  fields: FieldsDictionary,
+  tables: Table[];
+  fields: FieldsDictionary;
 }> => {
   const fields: FieldsDictionary = {};
   const schemaSql = schemas.length > 0 ? `AND c.table_schema IN (${schemas.map((schema) => `'${schema}'`).join(',')})` : '';
@@ -269,7 +268,9 @@ const createConstraintKeysMap = (keys: Record<string, string>[], schemas: string
       schema_name, table_name, column_name, constraint_name,
     } = row;
     const selectedSchema = schemas.length > 0 ? schemas.includes(schema_name) : true;
-    if (!selectedSchema) { return acc; }
+    if (!selectedSchema) {
+      return acc;
+    }
 
     const key = `${schema_name}.${table_name}.${constraint_name}`;
 

package/src/connectors/types.ts

@@ -10,7 +10,7 @@ interface TypeInfo {
 type DefaultType = 'string' | 'number' | 'boolean' | 'expression';
 
 interface DefaultInfo {
-  type: DefaultType,
+  type: DefaultType;
   value: string;
 }
 
@@ -45,12 +45,12 @@ interface CheckConstraintDictionary {
 }
 
 interface TableConstraint {
- [fieldName: string]: {
-   pk?: boolean;
-   unique?: boolean;
-   checks: CheckConstraint[];
+  [fieldName: string]: {
+    pk?: boolean;
+    unique?: boolean;
+    checks: CheckConstraint[];
   };
- }
+}
 
 interface TableConstraintsDictionary {
   [tableIdentifier: string]: TableConstraint; // Represents constraints within tables, indexed by schemaName.tableName
@@ -88,8 +88,8 @@ interface IndexColumn {
 interface Index {
   name: string;
   type: string;
-  unique?: boolean,
-  pk?: boolean,
+  unique?: boolean;
+  pk?: boolean;
   columns: IndexColumn[];
 }
 
@@ -108,12 +108,12 @@ interface DatabaseSchema {
 }
 
 interface BigQueryCredentials {
-  projectId?: string,
+  projectId?: string;
   credentials?: {
-    clientEmail: string,
-    privateKey: string,
-  },
-  datasets: string[],
+    clientEmail: string;
+    privateKey: string;
+  };
+  datasets: string[];
 }
 
 export {
@@ -135,5 +135,6 @@ export {
   IndexesDictionary,
   DatabaseSchema,
   BigQueryCredentials,
+  CheckConstraint,
   CheckConstraintDictionary,
 };

package/src/index.ts

@@ -1,4 +1,4 @@
-import * as connector from "./connectors/connector";
+import * as connector from './connectors/connector';
 
 export {
   connector,

package/src/utils/parseSchema.ts

@@ -14,6 +14,17 @@ const noSchemaResult = (connectionString: string) => ({
   schemas: [],
 });
 
+// Inputs: schemas string -> Output: schema array string[]
+// schema1, schema2 -> ['schema1', 'schema2']
+// schema3,schema4 -> ['schema3", 'schema4']
+// "schema5,gg", "schema6", schema7 -> ['schema5,gg', 'schema6', 'schema7']
+// schema8 -> ['schema8']
+export const parseSchema = (schemas: string | undefined): string[] => {
+  if (!schemas) return [];
+  const schemasWithDoubleQuotes = schemas.split(/,(?=(?:[^"]*(?:"[^"]*")*)*$)/).map((s) => s.trim());
+  return schemasWithDoubleQuotes.map((s) => s.replace(/"/g, ''));
+};
+
 const parseOdbcSchema = (connectionString: string) => {
   const connectionParts = connectionString.split(';');
   const schemasPart = connectionParts.find((part) => haveSchemas(part));
@@ -47,14 +58,3 @@ export const buildSchemaQuery = (columnName: string, schemas: string[], prefix =
   if (schemas.length === 0) return '';
   return `${prefix} ${columnName} IN (${schemas.map((schema) => `'${schema}'`).join(', ')})`;
 };
-
-// Inputs: schemas string -> Output: schema array string[]
-// schema1, schema2 -> ['schema1', 'schema2']
-// schema3,schema4 -> ['schema3", 'schema4']
-// "schema5,gg", "schema6", schema7 -> ['schema5,gg', 'schema6', 'schema7']
-// schema8 -> ['schema8']
-export const parseSchema = (schemas: string | undefined): string[] => {
-  if (!schemas) return [];
-  const schemasWithDoubleQuotes = schemas.split(/,(?=(?:[^"]*(?:"[^"]*")*)*$)/).map((s) => s.trim());
-  return schemasWithDoubleQuotes.map((s) => s.replace(/"/g, ''));
-};

package/tsconfig.json

@@ -9,7 +9,7 @@
     "esModuleInterop": true,
     "skipLibCheck": true,
     "forceConsistentCasingInFileNames": true,
-    "declaration": true, // Generates .d.ts files
+    "declaration": true // Generates .d.ts files
   },
   "include": ["./src"],
   "exclude": ["node_modules", "**/*.test.ts"]

package/.eslintrc.json

@@ -1,24 +0,0 @@
-{
-  "extends": [
-    "plugin:@typescript-eslint/recommended"
-  ],
-  "parser": "@typescript-eslint/parser",
-  "parserOptions": {
-    "sourceType": "module",
-    "ecmaVersion": 2020,
-    "project": "./tsconfig.json"
-  },
-  "settings": {
-    "import/resolver": {
-      "typescript": {
-        "alwaysTryTypes": true,
-        "project": "packages/*/{ts,js}config.json"
-      }
-    }
-  },
-  "plugins": ["@typescript-eslint"],
-  "ignorePatterns": ["**/*.test.ts"],
-  "rules": {
-    "@typescript-eslint/no-explicit-any": "off"
-  }
-}