@prairielearn/postgres-tools 1.0.0

package/src/describe.ts

@@ -0,0 +1,321 @@
+// @ts-check
+import pg from 'pg';
+import chalk from 'chalk';
+import { loadSqlEquiv, PostgresPool } from '@prairielearn/postgres';
+
+const sql = loadSqlEquiv(__filename);
+const pgArray = pg.types.arrayParser;
+
+interface ColumnDescription {
+  name: string;
+  type: string;
+  notnull: boolean;
+  default: any;
+}
+
+interface IndexDescription {
+  name: string;
+  isprimary: boolean;
+  isunique: boolean;
+  indexdef: string;
+  constraintdef: string;
+  contype: string;
+}
+
+interface ForeignKeyConstraintDescription {
+  name: string;
+  def: string;
+}
+
+interface ReferenceDescription {
+  name: string;
+  table: string;
+  condef: string;
+}
+
+interface CheckConstraintDescription {
+  name: string;
+  def: string;
+}
+
+interface TableDescription {
+  columns: ColumnDescription[];
+  indexes: IndexDescription[];
+  foreignKeyConstraints: ForeignKeyConstraintDescription[];
+  references: ReferenceDescription[];
+  checkConstraints: CheckConstraintDescription[];
+}
+
+export interface DatabaseDescription {
+  tables: Record<string, TableDescription>;
+  enums: Record<string, string[]>;
+}
+
+interface DescribeOptions {
+  ignoreTables?: string[];
+  ignoreColumns?: string[];
+  ignoreEnums?: string[];
+}
+
+function parsePostgresArray(arr: string): string[] {
+  // @ts-expect-error -- Incorrect type definitions in `pg-types`.
+  return pgArray.create(arr, String).parse();
+}
+
+async function describeWithPool(
+  pool: PostgresPool,
+  options: DescribeOptions
+): Promise<DatabaseDescription> {
+  const ignoreTables = options?.ignoreTables || [];
+  const ignoreEnums = options?.ignoreEnums || [];
+  let ignoreColumns: Record<string, string[]> = {};
+
+  const output: DatabaseDescription = {
+    tables: {},
+    enums: {},
+  };
+
+  // Get the names of the tables and filter out any ignored tables
+  const tablesRes = await pool.queryAsync(sql.get_tables, []);
+  const tables = tablesRes.rows.filter((table) => ignoreTables.indexOf(table.name) === -1);
+
+  // Transform ignored columns into a map from table names to arrays
+  // of column names
+  if (options.ignoreColumns && Array.isArray(options.ignoreColumns)) {
+    ignoreColumns = options.ignoreColumns
+      .filter((ignore) => {
+        return /^[^\s.]*\.[^\s.]*$/.test(ignore);
+      })
+      .reduce((result, value) => {
+        const res = /^(([^\s.]*)\.([^\s.]*))$/.exec(value);
+        if (!res) {
+          throw new Error(`Invalid ignore column: ${value}`);
+        }
+        const table = res[2];
+        const column = res[3];
+        (result[table] || (result[table] = [])).push(column);
+        return result;
+      }, {} as Record<string, string[]>);
+  }
+
+  // Get column info for each table
+  for (const table of tables) {
+    const columnResults = await pool.queryAsync(sql.get_columns_for_table, {
+      oid: table.oid,
+    });
+
+    const columns = columnResults.rows.filter((row) => {
+      return (ignoreColumns[table.name] || []).indexOf(row.name) === -1;
+    });
+
+    const indexResults = await pool.queryAsync(sql.get_indexes_for_table, {
+      oid: table.oid,
+    });
+
+    const foreignKeyConstraintResults = await pool.queryAsync(
+      sql.get_foreign_key_constraints_for_table,
+      {
+        oid: table.oid,
+      }
+    );
+
+    const referenceResults = await pool.queryAsync(sql.get_references_for_table, {
+      oid: table.oid,
+    });
+
+    // Filter out references from ignored tables
+    const references = referenceResults.rows.filter((row) => {
+      return ignoreTables.indexOf(row.table) === -1;
+    });
+
+    const checkConstraintResults = await pool.queryAsync(sql.get_check_constraints_for_table, {
+      oid: table.oid,
+    });
+
+    output.tables[table.name] = {
+      columns: columns,
+      indexes: indexResults.rows,
+      foreignKeyConstraints: foreignKeyConstraintResults.rows,
+      references: references,
+      checkConstraints: checkConstraintResults.rows,
+    };
+  }
+
+  // Get all enums
+  const enumsRes = await pool.queryAsync(sql.get_enums, []);
+
+  // Filter ignored enums
+  const rows = enumsRes.rows.filter((row) => {
+    return ignoreEnums.indexOf(row.name) === -1;
+  });
+
+  rows.forEach((row) => {
+    output.enums[row.name] = parsePostgresArray(row.values);
+  });
+
+  return output;
+}
+
+/**
+ * Will produce a description of a given database's schema. This will include
+ * information about tables, enums, constraints, indices, etc.
+ */
+export async function describeDatabase(
+  databaseName: string,
+  options: DescribeOptions = {}
+): Promise<DatabaseDescription> {
+  // Connect to the database.
+  const pool = new PostgresPool();
+  const pgConfig = {
+    user: 'postgres',
+    database: databaseName,
+    host: 'localhost',
+    max: 10,
+    idleTimeoutMillis: 30000,
+  };
+  function idleErrorHandler(err: Error) {
+    throw err;
+  }
+  await pool.initAsync(pgConfig, idleErrorHandler);
+
+  try {
+    return await describeWithPool(pool, options);
+  } finally {
+    await pool.closeAsync();
+  }
+}
+
+export function formatDatabaseDescription(
+  description: DatabaseDescription,
+  options = { coloredOutput: true }
+): { tables: Record<string, string>; enums: Record<string, string> } {
+  const output = {
+    tables: {} as Record<string, string>,
+    enums: {} as Record<string, string>,
+  };
+
+  Object.keys(description.tables).forEach((tableName) => (output.tables[tableName] = ''));
+
+  /**
+   * Optionally applies the given formatter to the text if colored output is
+   * enabled.
+   */
+  function formatText(text: string, formatter: (s: string) => string): string {
+    if (options.coloredOutput) {
+      return formatter(text);
+    }
+    return text;
+  }
+
+  Object.entries(description.tables).forEach(([tableName, table]) => {
+    if (table.columns.length > 0) {
+      output.tables[tableName] += formatText('columns\n', chalk.underline);
+      output.tables[tableName] += table.columns
+        .map((row) => {
+          let rowText = formatText(`    ${row.name}`, chalk.bold);
+          rowText += ':' + formatText(` ${row.type}`, chalk.green);
+          if (row.notnull) {
+            rowText += formatText(' not null', chalk.gray);
+          }
+          if (row.default) {
+            rowText += formatText(` default ${row.default}`, chalk.gray);
+          }
+          return rowText;
+        })
+        .join('\n');
+    }
+
+    if (table.indexes.length > 0) {
+      if (output.tables[tableName].length !== 0) {
+        output.tables[tableName] += '\n\n';
+      }
+      output.tables[tableName] += formatText('indexes\n', chalk.underline);
+      output.tables[tableName] += table.indexes
+        .map((row) => {
+          const using = row.indexdef.substring(row.indexdef.indexOf('USING '));
+          let rowText = formatText(`    ${row.name}`, chalk.bold) + ':';
+          // Primary indexes are implicitly unique, so we don't need to
+          // capture that explicitly.
+          if (row.isunique && !row.isprimary) {
+            if (!row.constraintdef || row.constraintdef.indexOf('UNIQUE') === -1) {
+              // Some unique indexes don't include the UNIQUE constraint
+              // as part of the constraint definition, so we need to capture
+              // that manually.
+              rowText += formatText(` UNIQUE`, chalk.green);
+            }
+          }
+          rowText += row.constraintdef ? formatText(` ${row.constraintdef}`, chalk.green) : '';
+          rowText += using ? formatText(` ${using}`, chalk.green) : '';
+          return rowText;
+        })
+        .join('\n');
+    }
+
+    if (table.checkConstraints.length > 0) {
+      if (output.tables[tableName].length !== 0) {
+        output.tables[tableName] += '\n\n';
+      }
+      output.tables[tableName] += formatText('check constraints\n', chalk.underline);
+      output.tables[tableName] += table.checkConstraints
+        .map((row) => {
+          // Particularly long constraints are formatted as multiple lines.
+          // We'll collapse them into a single line for better appearance in
+          // the resulting file.
+          //
+          // The first replace handles lines that end with a parenthesis: we
+          // want to avoid spaces between the parenthesis and the next token.
+          //
+          // The second replace handles all other lines: we want to collapse
+          // all leading whitespace into a single space.
+          const def = row.def.replace(/\(\n/g, '(').replace(/\n\s*/g, ' ');
+
+          let rowText = formatText(`    ${row.name}:`, chalk.bold);
+          rowText += formatText(` ${def}`, chalk.green);
+          return rowText;
+        })
+        .join('\n');
+    }
+
+    if (table.foreignKeyConstraints.length > 0) {
+      if (output.tables[tableName].length !== 0) {
+        output.tables[tableName] += '\n\n';
+      }
+      output.tables[tableName] += formatText('foreign-key constraints\n', chalk.underline);
+      output.tables[tableName] += table.foreignKeyConstraints
+        .map((row) => {
+          let rowText = formatText(`    ${row.name}:`, chalk.bold);
+          rowText += formatText(` ${row.def}`, chalk.green);
+          return rowText;
+        })
+        .join('\n');
+    }
+
+    if (table.references.length > 0) {
+      if (output.tables[tableName].length !== 0) {
+        output.tables[tableName] += '\n\n';
+      }
+      output.tables[tableName] += formatText('referenced by\n', chalk.underline);
+      output.tables[tableName] += table.references
+        ?.map((row) => {
+          let rowText = formatText(`    ${row.table}:`, chalk.bold);
+          rowText += formatText(` ${row.condef}`, chalk.green);
+          return rowText;
+        })
+        .join('\n');
+    }
+  });
+
+  Object.entries(description.enums).forEach(([enumName, enumValues]) => {
+    output.enums[enumName] = formatText(enumValues.join(', '), chalk.gray);
+  });
+
+  // We need to tack on a newline to everything.
+  Object.entries(output.tables).forEach(([tableName, table]) => {
+    output.tables[tableName] = table + '\n';
+  });
+  Object.entries(output.enums).forEach(([enumName, enumValues]) => {
+    output.enums[enumName] = enumValues + '\n';
+  });
+
+  return output;
+}

package/src/diff.ts

@@ -0,0 +1,232 @@
+// @ts-check
+import fs from 'fs-extra';
+import path from 'node:path';
+import chalk from 'chalk';
+import _ from 'lodash';
+import { diffLines } from 'diff';
+
+import { describeDatabase, formatDatabaseDescription } from './describe';
+
+type DatabaseInfo = { type: 'database'; name: string };
+type DirectoryInfo = { type: 'directory'; path: string };
+type DiffTarget = DatabaseInfo | DirectoryInfo;
+type DiffOptions = { coloredOutput?: boolean };
+type Description = {
+  tables: Record<string, string>;
+  enums: Record<string, string>;
+};
+
+async function diff(db1: DiffTarget, db2: DiffTarget, options: DiffOptions): Promise<string> {
+  function formatText(text: string, formatter?: ((s: string) => string) | null): string {
+    if (options.coloredOutput && formatter) {
+      return formatter(text);
+    }
+    return text;
+  }
+
+  const db2Name = db2.type === 'database' ? db2.name : db2.path;
+  const db2NameBold = formatText(db2Name, chalk.bold);
+
+  let result = '';
+
+  const description1 = await loadDescription(db1);
+  const description2 = await loadDescription(db2);
+
+  // Determine if both databases have the same tables
+  const tablesMissingFrom1 = _.difference(_.keys(description2.tables), _.keys(description1.tables));
+  const tablesMissingFrom2 = _.difference(_.keys(description1.tables), _.keys(description2.tables));
+
+  if (tablesMissingFrom1.length > 0) {
+    result += formatText(`Tables added to ${db2NameBold} (${db2.type})\n`, chalk.underline);
+    result += formatText(
+      tablesMissingFrom1.map((table) => `+ ${table}`).join('\n') + '\n\n',
+      chalk.green
+    );
+  }
+
+  if (tablesMissingFrom2.length > 0) {
+    result += formatText(`Tables missing from ${db2NameBold} (${db2.type})\n`, chalk.underline);
+    result += formatText(
+      tablesMissingFrom2.map((table) => `- ${table}`).join('\n') + '\n\n',
+      chalk.red
+    );
+  }
+
+  // Determine if both databases have the same enums
+  const enumsMissingFrom1 = _.difference(_.keys(description2.enums), _.keys(description1.enums));
+  const enumsMissingFrom2 = _.difference(_.keys(description1.enums), _.keys(description2.enums));
+
+  if (enumsMissingFrom1.length > 0) {
+    result += formatText(`Enums added to ${db2NameBold} (${db1.type})\n`, chalk.underline);
+    result += formatText(
+      enumsMissingFrom1.map((enumName) => `+ ${enumName}`).join('\n') + '\n\n',
+      chalk.green
+    );
+  }
+
+  if (enumsMissingFrom2.length > 0) {
+    result += formatText(`Enums missing from ${db2NameBold} (${db2.type})\n`, chalk.underline);
+    result += formatText(
+      enumsMissingFrom2.map((enumName) => `- ${enumName}`).join('\n') + '\n\n',
+      chalk.red
+    );
+  }
+
+  // Determine if the columns of any table differ
+  const intersection = _.intersection(_.keys(description1.tables), _.keys(description2.tables));
+  _.forEach(intersection, (table) => {
+    // We normalize each blob to end with a newline to make diffs print cleaner
+    const diff = diffLines(
+      description1.tables[table].trim() + '\n',
+      description2.tables[table].trim() + '\n'
+    );
+    if (diff.length === 1) return;
+
+    const boldTable = formatText(table, chalk.bold);
+    result += formatText(`Differences in ${boldTable} table\n`, chalk.underline);
+
+    // Shift around the newlines so that we can cleanly show +/- symbols
+    for (let i = 1; i < diff.length; i++) {
+      const prev = diff[i - 1].value;
+      if (prev[prev.length - 1] === '\n') {
+        diff[i - 1].value = prev.slice(0, -1);
+        diff[i].value = '\n' + diff[i].value;
+      }
+    }
+
+    _.forEach(diff, (part, index) => {
+      if (index === 0) {
+        part.value = '\n' + part.value;
+      }
+      const mark = part.added ? '+ ' : part.removed ? '- ' : '  ';
+      let change = part.value.split('\n').join(`\n${mark}`);
+      if (index === 0) {
+        change = change.slice(1, change.length);
+      }
+      if (part.added || part.removed) {
+        result += formatText(change, part.added ? chalk.green : part.removed ? chalk.red : null);
+      }
+    });
+    result += '\n\n';
+  });
+
+  // Determine if the values of any enums differ
+  const enumsIntersection = _.intersection(_.keys(description1.enums), _.keys(description2.enums));
+  _.forEach(enumsIntersection, (enumName) => {
+    // We don't need to do a particularly fancy diff here, since
+    // enums are just represented here as strings
+    if (description1.enums[enumName].trim() !== description2.enums[enumName].trim()) {
+      const boldEnum = formatText(enumName, chalk.bold);
+      result += formatText(`Differences in ${boldEnum} enum\n`);
+      result += formatText(`- ${description1.enums[enumName].trim()}\n`, chalk.red);
+      result += formatText(`+ ${description2.enums[enumName].trim()}\n`, chalk.green);
+      result += '\n\n';
+    }
+  });
+
+  return result;
+}
+
+async function loadDescriptionFromDisk(dirPath: string): Promise<Description> {
+  const description: Description = {
+    tables: {},
+    enums: {},
+  };
+
+  const tables = await fs.readdir(path.join(dirPath, 'tables'));
+  for (const table of tables) {
+    const data = await fs.readFile(path.join(dirPath, 'tables', table), 'utf8');
+    description.tables[table.replace('.pg', '')] = data;
+  }
+
+  const enums = await fs.readdir(path.join(dirPath, 'enums'));
+  for (const enumName of enums) {
+    const data = await fs.readFile(path.join(dirPath, 'enums', enumName), 'utf8');
+    description.enums[enumName.replace('.pg', '')] = data;
+  }
+
+  return description;
+}
+
+async function loadDescriptionFromDatabase(name: string) {
+  const description = await describeDatabase(name);
+  return formatDatabaseDescription(description, { coloredOutput: false });
+}
+
+async function loadDescription(db: DiffTarget): Promise<Description> {
+  if (db.type === 'database') {
+    return loadDescriptionFromDatabase(db.name);
+  } else if (db.type === 'directory') {
+    return loadDescriptionFromDisk(db.path);
+  } else {
+    throw new Error('Invalid database type');
+  }
+}
+
+export async function diffDatabases(database1: string, database2: string, options: DiffOptions) {
+  return diff(
+    {
+      type: 'database',
+      name: database1,
+    },
+    {
+      type: 'database',
+      name: database2,
+    },
+    options
+  );
+}
+
+export async function diffDatabaseAndDirectory(
+  database: string,
+  directory: string,
+  options: DiffOptions
+) {
+  return diff(
+    {
+      type: 'database',
+      name: database,
+    },
+    {
+      type: 'directory',
+      path: directory,
+    },
+    options
+  );
+}
+
+export async function diffDirectoryAndDatabase(
+  directory: string,
+  database: string,
+  options: DiffOptions
+) {
+  return diff(
+    {
+      type: 'directory',
+      path: directory,
+    },
+    {
+      type: 'database',
+      name: database,
+    },
+    options
+  );
+}
+
+export async function diffDirectories(
+  directory1: string,
+  directory2: string,
+  options: DiffOptions
+) {
+  return diff(
+    {
+      type: 'directory',
+      path: directory1,
+    },
+    {
+      type: 'directory',
+      path: directory2,
+    },
+    options
+  );
+}

package/src/index.ts

@@ -0,0 +1,7 @@
+export { describeDatabase, formatDatabaseDescription } from './describe';
+export {
+  diffDatabases,
+  diffDirectories,
+  diffDatabaseAndDirectory,
+  diffDirectoryAndDatabase,
+} from './diff';

package/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "@prairielearn/tsconfig",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "types": ["mocha", "node"],
+  }
+}