@axinom/mosaic-cli 0.14.2-rc.8 → 0.15.0-rc.0

package/src/commands/pg-dump/generate.ts

@@ -0,0 +1,146 @@
+import {
+  assertError,
+  getBasicDbConfigDefinitions,
+  getValidatedConfig,
+  isNullOrWhitespace,
+  pick,
+} from '@axinom/mosaic-service-common';
+import * as chalk from 'chalk';
+import { exec, ExecException } from 'child_process';
+import { existsSync, mkdirSync } from 'fs';
+import { dirname, join, normalize, relative } from 'path';
+import { PgDumpOptions } from './pg-dump-options';
+
+/**
+ * Calculates a path to `docker-compose.yaml` file based on initial path.
+ * For example, if `scripts/infra` is provided as initial path, it will look for `scripts/infra/docker-compose.yml` in current folder, then in folder one hierarchy level up, traversing directories towards root.
+ * Returns a value like `../../../scripts/infra` or throws an error.
+ * Eliminates the need to calculate how many `..` you must have in the path to docker compose file which is passed as a cli parameter.
+ */
+const findComposePath = (initialPath: string): string => {
+  let startPath = process.cwd();
+  let lastPath = '';
+
+  while (lastPath !== startPath) {
+    const lookupPath = join(startPath, initialPath, 'docker-compose.yml');
+    if (existsSync(lookupPath)) {
+      return dirname(relative(process.cwd(), lookupPath));
+    } else {
+      // Preserve currently checked path for while condition comparison
+      lastPath = startPath;
+      // Remove last segment from path, move to directory closer to root
+      startPath = normalize(join(startPath, '..'));
+    }
+  }
+  throw Error(
+    `'${initialPath}/docker-compose.yml' file not found. Please make sure that correct composeFolderPath parameter is provided.`,
+  );
+};
+
+/**
+ * Loads and validates all required environment variables that are used to create a shadow database connection string.
+ * For this to work, CLI script must be called with pre-loaded variables. e.g.
+ * - using dotenv-cli script would look like this: `dotenv -- mosaic pg-dump -c scripts/infra`
+ * - using env-cmd script could look like this: `env-cmd -f .env env-cmd -f ../../../.env mosaic pg-dump -c scripts/infra`
+ */
+const getDumpDbConnectionString = (connectionString?: string): string => {
+  if (!isNullOrWhitespace(connectionString)) {
+    return connectionString;
+  }
+
+  try {
+    const configDefinitions = pick(
+      getBasicDbConfigDefinitions(),
+      'pgHost',
+      'pgPort',
+      'dbName',
+      'dbOwner',
+      'dbOwnerPassword',
+      'dbOwnerConnectionString',
+      'dbShadowConnectionString',
+      'pgUserSuffix',
+    );
+    const config = getValidatedConfig(configDefinitions);
+    return config.dbShadowConnectionString;
+  } catch (error) {
+    assertError(error);
+    throw new Error(
+      `${error.message}\n\nSpecified environment variables must be pre-loaded, or a database connection string can be specified explicitly using 'connectionString' cli parameter.`,
+    );
+  }
+};
+
+/**
+ * Creates a dump directory (if not existing already), where resulting sql dump file will be created/updated.
+ * Returns absolute path to dump file, including filename.
+ */
+const ensureDumpDirectoryExist = (targetPath: string): string => {
+  const dbSchemaExportPath = join(process.cwd(), targetPath);
+  const dirPath = dirname(dbSchemaExportPath);
+  if (!existsSync(dirPath)) {
+    mkdirSync(dirPath, { recursive: true });
+  }
+  return dbSchemaExportPath;
+};
+
+/**
+ * Returns a readable timestamp in current locale time, e.g. `[2021-05-13 13:02:12.685]`
+ */
+const getTimestamp = (): string => {
+  const offset = new Date().getTimezoneOffset() * 60000; //offset in milliseconds
+  const localISOTime = new Date(Date.now() - offset)
+    .toISOString()
+    .slice(0, -1)
+    .replace('T', ' ');
+  return `[${localISOTime}]`;
+};
+
+export const generate = (options: PgDumpOptions): void => {
+  try {
+    console.log(getTimestamp(), `Starting pg-dump command...`);
+    const composePath = findComposePath(options.composeFolderPath);
+    const connectionString = getDumpDbConnectionString(
+      options.connectionString,
+    );
+    const dumpPath = ensureDumpDirectoryExist(options.dumpPath);
+    const excludeSchemas = isNullOrWhitespace(options.excludeSchemas)
+      ? []
+      : options.excludeSchemas
+          .split(',')
+          .map((schemaName) => `--exclude-schema=${schemaName}`);
+    const dumpOptions = [
+      '--no-sync',
+      '--schema-only',
+      '--no-owner',
+      ...excludeSchemas,
+      connectionString,
+    ].join(' ');
+
+    exec(
+      `cd ${composePath} && docker-compose exec -T postgres pg_dump ${dumpOptions} > ${dumpPath}`,
+      (_error: ExecException | null, stdout: string, stderr: string) => {
+        if (stdout) {
+          console.log(getTimestamp(), 'pg_dump info:', stdout);
+        }
+        if (stderr) {
+          console.log(
+            getTimestamp(),
+            chalk.red('Command failed:'),
+            stderr.trim(),
+          );
+        } else {
+          console.log(
+            getTimestamp(),
+            chalk.green('Success:'),
+            `Database schema dump created at '${chalk.blueBright(
+              options.dumpPath,
+            )}'`,
+          );
+        }
+      },
+    );
+  } catch (err) {
+    assertError(err);
+    console.log(getTimestamp(), chalk.red('Command failed:'), err.message);
+  }
+};

package/src/commands/pg-dump/index.ts

@@ -0,0 +1,39 @@
+import { CommandModule } from 'yargs';
+import { generate } from './generate';
+import { PgDumpOptions } from './pg-dump-options';
+
+export const pgDump: CommandModule<unknown, PgDumpOptions> = {
+  command: 'pg-dump',
+  describe:
+    'Development command that uses a default PostgreSQL pg_dump utility to create a schema dump file of a specific database based on an existing docker-compose.yml file.',
+  builder: (yargs) =>
+    yargs
+      .option('composeFolderPath', {
+        alias: 'c',
+        default: 'infra',
+        describe:
+          'Relative path to docker-compose.yml file, e.g. `infra` or `scripts/infra`. Script will look for such directory starting from current project, traversing directories closer to root, so no need to specify a path like `../../../infra`',
+        string: true,
+      })
+      .option('dumpPath', {
+        alias: 'd',
+        default: 'src/generated/db/schema.sql',
+        describe:
+          'Relative path from working directory to a schema SQL file which will be created by pg_dump utility, e.g. `src/generated/db/schema.sql`',
+        string: true,
+      })
+      .option('excludeSchemas', {
+        alias: 'e',
+        default: 'graphile_migrate,graphile_worker',
+        describe:
+          'A comma-separated string of database schema names that should be excluded from pg_dump, e.g. `schema_one,schema_two`',
+        string: true,
+      })
+      .option('connectionString', {
+        alias: 's',
+        describe:
+          'The connection string to the database to be used to take a schema dump. Can be omitted if database-related environment variables are preloaded',
+        string: true,
+      }),
+  handler: generate,
+};

package/src/commands/pg-dump/pg-dump-options.ts

@@ -0,0 +1,6 @@
+export interface PgDumpOptions {
+  composeFolderPath: string;
+  dumpPath: string;
+  excludeSchemas: string;
+  connectionString?: string;
+}

package/src/commands/publish-schema-to-db/README.md

@@ -0,0 +1,130 @@
+# Overview
+
+This a CLI tool for bootstrapping the development process of a catalog-like
+service. It generates a Postgres database schema together with corresponding
+PostGraphile Smart Tags from an array of content type publish format schemas.
+
+This tool will output two files:
+
+```bash
+${YOUR_PROJECT}
+├── migrations
+│   └── current.sql
+└── src
+    └── plugins
+        └── smart-tags-plugin.ts
+```
+
+**NOTE:** These files will be overwritten each time you run this tool.
+
+# Usage
+
+Example:
+
+```bash
+yarn mosaic publish-schema-to-db -i '/path/to/schemas/**/*-published-event.json' -o '/path/to/your/service/root'
+```
+
+# Assumptions and terminology
+
+**Content metadata schema** - JSON schema describing the publish format for
+content metadata (movie, collection, TV show etc.).
+
+When mapping the publish message JSON schema to a Postgres schema the following
+conventions are used (this applies for each defined content type).
+
+**Content metadata schema** is decomposed into:
+
+- **Content entity** - root entity of content metadata.
+- An array of **primitive properties**.
+- An array of **object properties**.
+- A _relations_ table linking the content to other **content entities** (e.g.
+  related items, genres).
+
+## Customizing output
+
+In addition to the general assumptions described above it is also possible to
+customize schema generation per content type via postprocessors, see
+`src/postprocessors`.
+
+## Limitations
+
+This tool assumes at most one level of nested objects in the input JSON schema.
+
+SUPPORTED
+
+```json
+{
+  "$schema": "http://json-schema.org/draft-04/schema",
+  "properties": {
+    "object_prop": {
+      "type": "object",
+      "properties": {
+        "a": {
+          "type": "string"
+        },
+        "b": {
+          "type": "number"
+        }
+      }
+    }
+  }
+}
+```
+
+NOT SUPPORTED
+
+```json
+{
+  "$schema": "http://json-schema.org/draft-04/schema",
+  "properties": {
+    "object_prop": {
+      "type": "object",
+      "properties": {
+        "a": {
+          "type": "string"
+        },
+        "nested_object_prop": {
+          "type": "object",
+          "properties": {
+            "b": {
+              "type": "number"
+            }
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+Mapping of supported property types to DB constructs:
+
+- primitive (string, int etc.) - primitive type field
+- primitive array - array of primitive type
+- object - 1-to-1
+- object array - 1-to-many
+
+# TODO
+
+- Named indexes.
+- Generate migrations.
+  - https://github.com/michaelsogos/pg-diff
+  - https://www.npmjs.com/package/pg-compare
+  - https://databaseci.com/docs/migra
+- Test table generators.
+- Better logging.
+- Support nullable (required/not required)
+- Improve property exclusion logic
+
+  Frank's comment:
+
+  > Checked a bit on the implementation. The "ignored properties" seems to be
+  > just a flat string list. So it is not possible to exclude propertyA from
+  > movie but keep propertyB on TV show - right? In the used env variable I see
+  > $schema,related_items. The schema part I understand (and I would even hard-code that potentially - or even exclude anything starting with $).
+  > But for "related_items" I am very unsure about. The name can be custom, it
+  > can change over time etc. I would rather say to hard code the \$ part. And
+  > then have some callback function that you can implement in code that would
+  > get the content type (and for objects also that type). And then this could
+  > be customized to exclude if it is needed.

package/src/commands/publish-schema-to-db/abstractions/base-smart-tags.ts

@@ -0,0 +1,6 @@
+import { PgSmartTagTags } from 'graphile-utils/node8plus/makePgSmartTagsPlugin';
+
+export interface BaseSmartTags {
+  tags?: PgSmartTagTags;
+  description?: string;
+}

package/src/commands/publish-schema-to-db/abstractions/index.ts

@@ -0,0 +1,5 @@
+export * from './base-smart-tags';
+export * from './pg-column';
+export * from './pg-fk-column';
+export * from './pg-table';
+export * from './pg-type';

package/src/commands/publish-schema-to-db/abstractions/pg-column.ts

@@ -0,0 +1,31 @@
+import { BaseSmartTags } from './base-smart-tags';
+import { PgTable } from './pg-table';
+import { PgType } from './pg-type';
+
+/**
+ * Abstraction over a Postgres table column.
+ */
+export interface PgColumn {
+  /** Name of the column. */
+  name: string;
+
+  /** Optional display name to override the `name` in the generated GQL API. */
+  displayName?: string;
+
+  /** Description of the property. */
+  readonly description?: string;
+
+  /** Reference to the parent table that holds this column. */
+  readonly table: PgTable;
+
+  /** Postgres data type associated with this column. */
+  readonly type: PgType;
+
+  /** Expression defining the column. It will be used inside the CREATE TABLE statement. */
+  buildExpression(): string;
+
+  buildSmartTags(): BaseSmartTags;
+
+  /** Additional statements associated with the column e.g. indexes etc. */
+  buildAdditionalStatements(): string[];
+}

package/src/commands/publish-schema-to-db/abstractions/pg-fk-column.ts

@@ -0,0 +1,6 @@
+import { PgColumn } from './pg-column';
+
+export interface PgFkColumn extends PgColumn {
+  readonly targetPk: PgColumn;
+  readonly fkName?: string;
+}

package/src/commands/publish-schema-to-db/abstractions/pg-table.ts

@@ -0,0 +1,55 @@
+import { PgSmartTagTags } from 'graphile-utils';
+import { BaseSmartTags } from './base-smart-tags';
+import { PgColumn } from './pg-column';
+import { PgFkColumn } from './pg-fk-column';
+
+// TODO: Consider adding a separate property for FKs for consistency.
+/**
+ * Abstraction over a Postgres table.
+ */
+export interface PgTable {
+  /** Name of the table. */
+  name: string;
+
+  /** Optional display name to override the `name` in the generated GQL API. */
+  displayName?: string;
+
+  /** Description of the table. */
+  readonly description?: string;
+
+  /** Table's PK */
+  readonly pk: PgColumn;
+
+  /** FKs to other tables. */
+  readonly fks: PgFkColumn[];
+
+  /** Virtual FKs to other tables (PostGraphile-specific). */
+  readonly virtualFks: PgFkColumn[];
+
+  /** Array of regular 'data' columns in the table. */
+  readonly columns: PgColumn[];
+
+  /** Builds a fully qualified table name - [schema].[table_name]. */
+  buildFullName(): string;
+
+  /**
+   * Builds an (ordered) array  of SQL statements required for creating a table.
+   */
+  buildStatements(): string[];
+
+  /**
+   * Builds additional smart tags for PostGraphile.
+   */
+  buildSmartTags(): TableSmartTags;
+}
+
+export interface TableSmartTags {
+  tags?: PgSmartTagTags;
+  description?: string;
+  attribute?: {
+    [attributeName: string]: BaseSmartTags;
+  };
+  constraint?: {
+    [constraintName: string]: BaseSmartTags;
+  };
+}

package/src/commands/publish-schema-to-db/abstractions/pg-type.ts

@@ -0,0 +1,8 @@
+/** Supported Postgres data types. */
+export type PgType =
+  | 'TEXT'
+  | 'TEXT[]'
+  | 'INTEGER'
+  | 'INTEGER[]'
+  | 'BOOLEAN'
+  | 'TIMESTAMPTZ';

package/src/commands/publish-schema-to-db/content-entity-model.ts

@@ -0,0 +1,93 @@
+/* eslint-disable no-console */
+import { JSONSchema4 } from 'json-schema';
+import { PgTable, TableSmartTags } from './abstractions';
+import { separateProperties } from './pg-models/json-schema-parse-utils';
+import { isReservedPgWord } from './pg-models/pg-sql-gen-utils';
+import { ContentEntityTable, ObjectPropertyTable } from './pg-models/tables';
+import { PublishSchemaToDbOptions } from './publish-schema-to-db-options';
+
+export class ContentEntityModel {
+  public name: string;
+  public contentEntity: PgTable;
+  public relatedObjects: PgTable[] = [];
+
+  constructor(options: PublishSchemaToDbOptions, schema: JSONSchema4) {
+    if (!schema.title) {
+      throw Error('Content type schema title is undefined.');
+    }
+
+    // TODO: Too restrictive.
+    this.name = schema.title.substring(
+      0,
+      schema.title.indexOf('_published_event'),
+    );
+
+    // 1. Split properties into primitive and object properties.
+    const [primitiveProperties, objectProperties] = separateProperties(
+      schema,
+      options.ignoredProperties,
+    );
+
+    // 2. Create the content entity table.
+    this.contentEntity = new ContentEntityTable(
+      options,
+      this.name,
+      primitiveProperties,
+      schema.description,
+    );
+
+    // 3. Create related object property tables.
+    for (const key in objectProperties) {
+      this.relatedObjects.push(
+        new ObjectPropertyTable(
+          options,
+          key,
+          objectProperties[key],
+          this.contentEntity.pk,
+        ),
+      );
+    }
+  }
+
+  public validate(): void {
+    console.log(`Validating content entity model for ${this.name}`);
+    const tables = [this.contentEntity, ...this.relatedObjects];
+    for (const table of tables) {
+      if (isReservedPgWord(table.name)) {
+        throw new Error(
+          `Table name ${table.name} is a reserved word. Please specify a name override in the postprocessor.`,
+        );
+      }
+      for (const column of table.columns) {
+        if (isReservedPgWord(column.name)) {
+          throw new Error(
+            `Column name ${table.name}.${column.name} is a reserved word. Please specify a name override in the postprocessor.`,
+          );
+        }
+      }
+    }
+  }
+
+  public buildStatements(): string[] {
+    return ([] as string[]).concat(
+      `--`,
+      `-- #${this.name}`,
+      `--`,
+      ...[this.contentEntity, ...this.relatedObjects].map((t) =>
+        t.buildStatements(),
+      ),
+      `--`,
+      `--`,
+      `--`,
+    );
+  }
+
+  public buildSmartTags(): { [name: string]: TableSmartTags } {
+    return [this.contentEntity, ...this.relatedObjects].reduce((acc, t) => {
+      if (t.buildSmartTags()) {
+        acc[t.name] = t.buildSmartTags();
+      }
+      return acc;
+    }, {} as { [name: string]: TableSmartTags });
+  }
+}

package/src/commands/publish-schema-to-db/generate.ts

@@ -0,0 +1,82 @@
+/* eslint-disable no-console */
+import { JSONPgSmartTags } from 'graphile-utils';
+import * as path from 'path';
+import { ContentEntityModel } from './content-entity-model';
+import { getContentMetadataSchemas } from './pg-models/json-schema-parse-utils';
+import { generateMigration } from './pg-models/pg-sql-gen-utils';
+import { generateSmartTagsPlugin } from './pg-models/pgl-utils';
+import {
+  collectionPostprocessor,
+  ContentEntityModelPostprocessor,
+  episodePostprocessor,
+  moviePostprocessor,
+  seasonPostprocessor,
+  tvshowPostprocessor,
+} from './postprocessors';
+import { PublishSchemaToDbOptions } from './publish-schema-to-db-options';
+
+/**
+ * Generates a database schema from an asset publish format JSON schema.
+ */
+export async function generate(
+  options: PublishSchemaToDbOptions,
+): Promise<void> {
+  console.log(`Searching for schemas matching pattern ${options.inputGlob}`);
+
+  // 1. Read all publish events.
+  const contentTypeSchemas = await getContentMetadataSchemas(options.inputGlob);
+  const modelPostProcessors: {
+    [name: string]: ContentEntityModelPostprocessor;
+  } = {
+    movie: moviePostprocessor,
+    tvshow: tvshowPostprocessor,
+    season: seasonPostprocessor,
+    episode: episodePostprocessor,
+    collection: collectionPostprocessor,
+  };
+
+  const contentEntityModels: ContentEntityModel[] = [];
+  const statements: string[] = [];
+  const smartTags: JSONPgSmartTags = {
+    version: 1,
+    config: {
+      class: {},
+    },
+  };
+
+  for (const contentTypeSchema of contentTypeSchemas) {
+    contentEntityModels.push(
+      new ContentEntityModel(options, contentTypeSchema),
+    );
+  }
+
+  for (const model of contentEntityModels) {
+    const otherModels = contentEntityModels.filter(
+      (m) => m.contentEntity.name !== model.contentEntity.name,
+    );
+    const postprocess = modelPostProcessors[model.contentEntity.name];
+    if (postprocess !== undefined) {
+      console.log(`Found model postprocessor for ${model.contentEntity.name}.`);
+      postprocess(model, otherModels, options);
+    }
+
+    model.validate();
+
+    smartTags.config.class = {
+      ...smartTags.config.class,
+      ...model.buildSmartTags(),
+    };
+    statements.push(...model.buildStatements());
+  }
+
+  // 3. Generate SQL.
+  await generateMigration(
+    statements,
+    path.join(options.outputRoot, 'migrations', 'current.sql'),
+  );
+
+  generateSmartTagsPlugin(
+    smartTags,
+    path.join(options.outputRoot, 'src', 'plugins', 'smart-tags-plugin.ts'),
+  );
+}

package/src/commands/publish-schema-to-db/index.ts

@@ -0,0 +1,49 @@
+import { CommandModule } from 'yargs';
+import { generate } from './generate';
+import { PublishSchemaToDbOptions } from './publish-schema-to-db-options';
+
+export const publishSchemaToDb: CommandModule<
+  unknown,
+  PublishSchemaToDbOptions
+> = {
+  command: 'publish-schema-to-db',
+  describe:
+    'Tool to generate the initial database schema for a catalog-like service from publish format definitions.',
+  builder: (yargs) =>
+    yargs
+      .option('inputGlob', {
+        // This should be a positional but making it required does not work.
+        alias: 'i',
+        describe:
+          'Glob pattern matching the publish format schemas to be used for code generation.',
+        type: 'string',
+        demandOption: true,
+      })
+      .option('idKey', {
+        alias: 'k',
+        describe: 'Key containing the published ID of a content entity.',
+        type: 'string',
+        default: 'content_id',
+      })
+      .option('ignoredProperties', {
+        alias: 'n',
+        describe: 'List of schema keys to ignore.',
+        type: 'array',
+        default: ['$schema', 'related_items', 'genres'],
+      })
+      .option('dbSchema', {
+        alias: 's',
+        describe: 'Name of the DB schema for SQL generation.',
+        type: 'string',
+        default: 'app_public',
+      })
+      .option('outputRoot', {
+        alias: 'o',
+        describe: 'Path to the catalog service directory.',
+        type: 'string',
+        demandOption: true,
+      }),
+  handler: async (argv) => {
+    await generate(argv);
+  },
+};

package/src/commands/publish-schema-to-db/jest.config.js

@@ -0,0 +1,9 @@
+/* eslint-disable @typescript-eslint/no-var-requires */
+const base = require('../../../../../jest.config.base.js');
+
+module.exports = {
+  ...base,
+  displayName: 'publish-schema-to-db',
+  testEnvironment: 'node',
+  transformIgnorePatterns: ['^.+\\.js$'],
+};