@axinom/mosaic-cli 0.14.2-rc.9 → 0.15.0-rc.0

package/src/commands/msg-diff/msg-diff.ts

@@ -0,0 +1,364 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+/* eslint-disable no-console */
+import { diff, DiffOutputItem } from '@asyncapi/diff';
+import { AsyncAPIDocument, Channel, parse } from '@asyncapi/parser';
+import { green, red, white } from 'chalk';
+import * as jsonDiff from 'diff';
+import * as fs from 'fs';
+import * as jsonSchemaDiff from 'json-schema-diff';
+import * as path from 'path';
+import { override } from './asyncapi-override';
+import { GitCheckoutTmp } from './git-checkout-tmp';
+import { MessageDiffOptions as MessageDiffOptions } from './message-diff-options';
+
+/**
+ * Diff results. Each file (asyncapi document or payload) is classified to one of 'skipped', 'same', 'changed' or 'breaking'.
+ */
+interface Results {
+  skipped: number;
+  same: number;
+  changed: number;
+  breaking: number;
+}
+
+/**
+ * Finds AsyncAPI documents found in git, and compares them with the current file system.
+ * The following changes are considered breaking:
+ *  - Removed / renamed asyncapi document
+ *  - Breaking changes in any asyncapi document
+ *  - Breaking changes in any message payload
+ * Message payloads are referenced through asyncapi channel definitions. It is safe to rename payload files if
+ * file references in the asyncapi document are updated accordingly.
+ *
+ * @param options (object):
+ *  schemaRoot      Path to directory to scan. All files required to parse the schema must be inside this dir
+ *                  & subdirectories. This includes JSON Schema references.
+ *  filePattern     Regular expression that matches suitable input files.
+ *  excludePattern  Regular expression that matches suitable input files.
+ *  branch          Git branch to compare to.
+ *  verbose         Verbose output includes full diff for all files, not only where breaking changes were found.
+ */
+export class MessageDiff {
+  private readonly schemaRoot: string;
+  private readonly filePattern: RegExp;
+  private readonly excludePattern: RegExp | undefined;
+  private readonly branch: string;
+  private readonly verbose: boolean;
+
+  public readonly results = {
+    skipped: 0,
+    same: 0,
+    changed: 0,
+    breaking: 0,
+  };
+
+  constructor(options: MessageDiffOptions) {
+    this.schemaRoot = path.resolve(options.inputDir);
+    this.branch = options.branch;
+    this.verbose = options.verbose;
+
+    try {
+      this.filePattern = new RegExp(options.filePattern);
+    } catch (error) {
+      console.error(
+        `${options.filePattern} is not a valid regular expression.`,
+      );
+      process.exit(2);
+    }
+    try {
+      this.excludePattern =
+        options.excludePattern === undefined
+          ? undefined
+          : new RegExp(options.excludePattern);
+    } catch (error) {
+      console.error(
+        `${options.excludePattern} is not a valid regular expression.`,
+      );
+      process.exit(2);
+    }
+  }
+
+  public async run(): Promise<void> {
+    // checkout branch to temp dir
+    const gitTmp = new GitCheckoutTmp();
+    gitTmp.checkout(this.schemaRoot, this.branch);
+    const rootDir1 = gitTmp.tmpDir;
+    const rootDir2 = gitTmp.gitRoot;
+    if (rootDir1 === undefined || rootDir2 === undefined) {
+      throw `Git checkout failed`;
+    }
+
+    // scan the checked out files
+    try {
+      const relPath = path.relative(rootDir2, this.schemaRoot);
+      await this.walk(
+        path.join(rootDir1, relPath),
+        path.join(rootDir2, relPath),
+      );
+    } finally {
+      // remove tmp files
+      gitTmp.cleanUp();
+    }
+
+    // check results
+    const totalFiles =
+      this.results.skipped +
+      this.results.same +
+      this.results.changed +
+      this.results.breaking;
+
+    if (totalFiles === 0) {
+      throw `No files found matching the given arguments`;
+    }
+
+    const color =
+      this.results.breaking > 0
+        ? red
+        : this.results.skipped > 0
+        ? white
+        : green;
+    console.log(
+      color(
+        `${totalFiles} file${totalFiles === 1 ? ' was' : 's were'} checked. ${
+          this.results.skipped
+        } ${this.results.skipped === 1 ? 'was' : 'were'} skipped. ${
+          this.results.same
+        } ${this.results.same === 1 ? 'is' : 'are'} the same. ${
+          this.results.changed
+        } ${
+          this.results.changed === 1 ? 'has' : 'have'
+        } non-breaking changes. ${this.results.breaking} ${
+          this.results.breaking === 1 ? 'has' : 'have'
+        } breaking changes.`,
+      ),
+    );
+
+    if (this.results.breaking > 0) {
+      process.exit(1);
+    }
+  }
+
+  /**
+   * Recursively walks a directory comparing matching files to another directory.
+   * @param dir1 - Root dir to scan for matching files.
+   * @param dir2 - Root dir for comparision.
+   */
+  public async walk(dir1: string, dir2: string): Promise<void> {
+    const items = await fs.promises.readdir(dir1);
+
+    const fullPathItems = items.map((i) => path.join(dir1, i));
+    const dirs = fullPathItems.filter((i) => fs.statSync(i).isDirectory());
+    const files = fullPathItems.filter(
+      (i) =>
+        fs.statSync(i).isFile() &&
+        this.filePattern.test(i) &&
+        (!this.excludePattern || !this.excludePattern.test(i)),
+    );
+
+    if ([...files, ...dirs].length === 0) {
+      return;
+    }
+
+    for (const file1 of files) {
+      const file2 = path.join(dir2, path.relative(dir1, file1));
+      await this.asyncApiDiff(file1, file2);
+    }
+
+    for (const subDir1 of dirs) {
+      const subDir2 = path.join(dir2, path.relative(dir1, subDir1));
+      await this.walk(subDir1, subDir2);
+    }
+  }
+
+  /**
+   * Full comparison of asyncapi documents. Each channel payload is also compared.
+   */
+  private async asyncApiDiff(file1: string, file2: string): Promise<void> {
+    console.log(
+      `Comparing asyncapi document '${file2}'. Changes from branch '${this.branch}':`,
+    );
+    let document1: AsyncAPIDocument;
+    let document2: AsyncAPIDocument;
+
+    // load asyncapi document from git
+    try {
+      const text = fs.readFileSync(file1, 'utf8');
+      document1 = await parse(text, { path: file1 });
+    } catch (e: any) {
+      throw `Error reading ${file1}: ${e}`;
+    }
+
+    // load local asyncapi document
+    try {
+      if (!fs.existsSync(file2)) {
+        console.log(
+          red(
+            `❌ File not found. This document may have been removed. Assuming this is a breaking change.`,
+          ),
+        );
+        this.results.breaking++;
+        return;
+      }
+      const text = fs.readFileSync(file2, 'utf8');
+      document2 = await parse(text, { path: file2 });
+    } catch (e: any) {
+      throw `Error reading ${file2}: ${e}`;
+    }
+
+    // diff document
+    const resultKey = await this.documentDiff(document1, document2);
+    this.results[resultKey]++;
+
+    const channels1 = document1.channels();
+    const channels2 = document2.channels();
+
+    // diff payloads for channels which exist in both documents
+    // removed / renamed channels are already marked as breaking
+    for (const [k, channel1] of Object.entries(channels1)) {
+      if (!(k in channels2)) {
+        continue;
+      }
+      console.log(
+        `Comparing payload for channel ${k} in document ${file2}. Changes from branch ${this.branch}`,
+      );
+      const channel2 = channels2[k];
+      const resultKey = await this.payloadDiff(channel1, channel2);
+      this.results[resultKey]++;
+    }
+  }
+
+  /**
+   * Compare asyncapi documents using AsyncAPI Diff: https://github.com/asyncapi/diff
+   * Note: non standard rules are defined in file: asyncapi-override.ts
+   */
+  public async documentDiff(
+    document1: AsyncAPIDocument,
+    document2: AsyncAPIDocument,
+  ): Promise<keyof Results> {
+    // generate diff
+    // overrides are applied through options here:
+    const output = diff(document1.json(), document2.json(), { override });
+    const breaking = output.breaking();
+    const nonBreaking = output.nonBreaking();
+    const unclassified = output.unclassified();
+
+    // log full results if breaking changes were found or verbose is set
+    const itemMsg = (change: string | DiffOutputItem): string =>
+      typeof change === 'string' ? change : `${change.action} ${change.path}`;
+
+    if (this.verbose) {
+      for (const item of unclassified) {
+        console.log(`  ？ ${itemMsg(item)}`);
+      }
+    }
+    if (breaking.length > 0 || this.verbose) {
+      for (const item of nonBreaking) {
+        console.log(`  ✔ ${itemMsg(item)}`);
+      }
+    }
+    for (const item of breaking) {
+      console.log(`  ✖ ${itemMsg(item)}`);
+    }
+
+    // log summary
+    const color = breaking.length > 0 ? red : green;
+    console.log(
+      color(
+        `${breaking.length > 0 ? '❌ ' : ''}${
+          nonBreaking.length
+        } non-breaking change${nonBreaking.length === 1 ? '' : 's'}, ${
+          breaking.length
+        } breaking change${
+          breaking.length === 1 ? '' : 's'
+        } in asyncapi document`,
+      ),
+    );
+
+    return breaking.length > 0
+      ? 'breaking'
+      : nonBreaking.length > 0
+      ? 'changed'
+      : 'same';
+  }
+
+  /**
+   * Compare payload schemas using JsonSchema Diff: https://www.npmjs.com/package/json-schema-diff
+   * NOTE: It might be possible to define rules for asyncapi-diff to compare properties in message schemas but
+   *       it would not have the ability to properly evaluate breaking changes as json-schema-diff does.
+   *       e.g. adding an existing property to the list of required properties is a breaking change.
+   *
+   * Only schema version http://json-schema.org/draft-07/schema# is supported.
+   * If any other version is used (in local file or git) then the comparison result will be 'skipped'.
+   */
+  public async payloadDiff(
+    channel1: Channel,
+    channel2: Channel,
+  ): Promise<keyof Results> {
+    const getChannelPayload = (channel: Channel): any => {
+      const json = channel.json();
+      return (json.subscribe ?? json.publish)?.message?.payload;
+    };
+    const schema1 = getChannelPayload(channel1);
+    const schema2 = getChannelPayload(channel2);
+    if (schema1 === undefined) {
+      throw `Could not find payload schema in ${this.branch}`;
+    }
+    if (schema2 === undefined) {
+      throw `Could not find payload schema`;
+    }
+
+    let result: jsonSchemaDiff.DiffResult;
+    try {
+      result = await jsonSchemaDiff.diffSchemas({
+        sourceSchema: schema1,
+        destinationSchema: schema2,
+      });
+    } catch (e: any) {
+      // Skip if unsupported schema version is used by either payload
+      if (
+        e instanceof Error &&
+        e.message.startsWith('no schema with key or ref')
+      ) {
+        console.log(
+          `WARNING: Unsupported JsonSchema version. This payload will be skipped. Both versions of the file must use $schema: "http://json-schema.org/draft-07/schema#".`,
+        );
+        return 'skipped';
+      }
+      throw e;
+    }
+
+    if (result.removalsFound || this.verbose) {
+      // json-schema-diff does not give a useful report of what are the breaking changes. See:
+      // https://bitbucket.org/atlassian/json-schema-diff/issues/6/is-it-supposed-to-report-only-what-has
+      // So we display a line-by-line diff of the schemas..its not targeted to what is breaking, but its more useful
+      // than just logging both versions in full which is the observed behavior of json-schema-diff.
+      const diffResult = jsonDiff.diffJson(schema1, schema2);
+      diffResult.forEach((r) => {
+        for (const line of r.value.split(/\r?\n/)) {
+          // strip blank lines and annotations added by asyncapi parser
+          if (!line || line.match(/^\s+"x-parser-schema/)) {
+            continue;
+          }
+          console.log(`${r.removed ? '  -' : r.added ? '  +' : '   '} ${line}`);
+        }
+      });
+    }
+
+    const color = result.removalsFound ? red : green;
+    console.log(
+      color(
+        result.removalsFound
+          ? '❌ Breaking changes in payload'
+          : result.additionsFound
+          ? 'No breaking changes in payload'
+          : 'No changes in payload',
+      ),
+    );
+
+    return result.removalsFound
+      ? 'breaking'
+      : result.additionsFound
+      ? 'changed'
+      : 'same';
+  }
+}

package/src/commands/msg-diff/test-resources/0/1-asyncapi.yml

@@ -0,0 +1,38 @@
+asyncapi: 2.0.0
+info:
+  title: Service 1
+  version: '1.0.0'
+  description: |
+    Testing 1
+# AsyncAPI extension, used to define service id. If not defined, as service id will be used info.title in lower-kebab-case
+x-service-id: ax-service-template
+
+channels:
+ # commands
+  'cmd-channel':
+    bindings:
+      amqp:
+        queue:
+          name: example_command
+    publish:
+      message:
+        $ref: '#/components/messages/command-ref'
+  # events
+  'evt-channel':
+    bindings:
+      amqp:
+        queue:
+          name: example_event
+    publish:
+      message:
+        $ref: '#/components/messages/event-ref'
+components:
+  messages:
+    command-ref:
+      contentType: application/json
+      payload:
+        $ref: 'command.json'
+    event-ref:
+      contentType: application/json
+      payload:
+        $ref: 'event.json'

package/src/commands/msg-diff/test-resources/0/2-asyncapi.yml

@@ -0,0 +1,36 @@
+asyncapi: 2.0.0
+info:
+  title: Service 2
+  version: '1.0.0'
+  description: |
+    Testing 2
+
+channels:
+ # commands
+  'cmd-channel':
+    bindings:
+      amqp:
+        queue:
+          name: example_command
+    publish:
+      message:
+        $ref: '#/components/messages/command-ref'
+  # events
+  'evt-channel':
+    bindings:
+      amqp:
+        queue:
+          name: example_event
+    publish:
+      message:
+        $ref: '#/components/messages/event-ref'
+components:
+  messages:
+    command-ref:
+      contentType: application/json
+      payload:
+        $ref: 'command.json'
+    event-ref:
+      contentType: application/json
+      payload:
+        $ref: 'event.json'

package/src/commands/msg-diff/test-resources/0/command.json

@@ -0,0 +1,74 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type":"object",
+  "title": "synchronize_claim_definitions_start_command",
+  "description": "Synchronize claim definitions start command schema.",
+  "additionalProperties": false,
+  "required": ["claim_definition_groups"],
+  "properties": {
+    "claim_definition_groups": {
+      "type": "array",
+      "description": "An array of claim definition groups to be registered.",
+      "items": {
+        "$ref": "#/definitions/claim_definition_group",
+        "description": "Defines a group of related claim definitions"
+      },
+      "minItems": 1
+    }
+  },
+  "definitions": {
+    "claim_definition_group": {
+      "type": "object",
+      "title": "claim_definition_group",
+      "additionalProperties": false,
+      "required": [
+        "title",
+        "selection_mode",
+        "claim_definitions"
+      ],
+      "properties": {
+        "title": {
+          "$ref": "../common.json#/definitions/non_empty_string",
+          "description": "Unique claim definition group title."
+        },
+        "selection_mode": {
+          "type": "string",
+          "title": "claim_selection_mode",
+          "description": "Selection options for a claim definition group.",
+          "enum": [
+            "SINGLE",
+            "MULTIPLE"
+          ]
+        },
+        "claim_definitions": {
+          "type": "array",
+          "description": "An array of claim definitions.",
+          "items": {
+            "$ref": "#/definitions/claim_definition",
+            "description": "Definition of a claim to be included in a claim set."
+          },
+          "minItems": 1
+        }
+      }
+    },
+    "claim_definition": {
+      "type": "object",
+      "title": "claim_definition",
+      "additionalProperties": false,
+      "required": [
+        "claim",
+        "title"
+      ],
+      "properties": {
+        "claim": {
+          "$ref": "../common.json#/definitions/non_empty_string",
+          "description": "The claim."
+        },
+        "title": {
+          "$ref": "../common.json#/definitions/non_empty_string",
+          "description": "Display name for the claim."
+        }
+      }
+    }
+  }
+}

package/src/commands/msg-diff/test-resources/0/event.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type":"object",
+  "title": "example_post_created_event",
+  "description": "New post created event.",
+  "properties": {
+    "title": {
+      "description": "Title of the post.",
+      "type": "string"
+    },
+    "content": {
+      "description": "Post body, actual content.",
+      "type": "string"
+    },
+    "is_public": {
+      "description": "Indicates whether the post is public.",
+      "type": "boolean"
+    }
+  },
+  "additionalProperties": false,
+  "required": [
+    "title",
+    "content"
+  ]
+}

package/src/commands/msg-diff/test-resources/1/1-asyncapi.yml

@@ -0,0 +1,25 @@
+asyncapi: 2.0.0
+info:
+  title: Template Service
+  version: '1.0.0'
+  description: |
+    Template Management
+# AsyncAPI extension, used to define service id. If not defined, as service id will be used info.title in lower-kebab-case
+x-service-id: ax-service-template
+
+channels:
+  # events
+  'evt-channel':
+    bindings:
+      amqp:
+        queue:
+          name: example_event
+    publish:
+      message:
+        $ref: '#/components/messages/event-ref'
+components:
+  messages:
+    event-ref:
+      contentType: application/json
+      payload:
+        $ref: 'moved-event.json'

package/src/commands/msg-diff/test-resources/1/moved-event.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type":"object",
+  "title": "example_post_created_event",
+  "description": "New post created event.",
+  "properties": {
+    "title": {
+      "description": "Title of the post.",
+      "type": "string"
+    },
+    "content": {
+      "description": "Post body, actual content.",
+      "type": "string"
+    },
+    "is_public": {
+      "description": "Indicates whether the post is public.",
+      "type": "boolean"
+    }
+  },
+  "additionalProperties": false,
+  "required": [
+    "title",
+    "content"
+  ]
+}

package/src/commands/msg-diff/test-resources/common.json

@@ -0,0 +1,20 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "common",
+  "description": "Common definitions.",
+  "definitions": {
+    "non_empty_string": {
+      "type": "string",
+      "minLength": 1,
+      "pattern": "^$|.*\\S.*",
+      "description": "The string has a minimum length of one character and it cannot consist of only whitespace characters."
+    },
+    "non_empty_uuid": {
+      "type": "string",
+      "minLength": 32,
+      "maxLength": 36,
+      "format": "uuid",
+      "description": "A UUID."
+    }
+  }
+}

package/src/commands/pg-dump/README.md

@@ -0,0 +1,21 @@
+# Overview
+
+Development command to create a database schema dump (using current service
+config) and put it into `src/generated/db/schema.sql` file of the current
+project.
+
+Environment variables must be pre-loaded so that a database connection string
+could be constructed.
+
+Shadow database is used as a target to provide a clean schema dump. Shadow
+database is re-created on each migration commit, migration uncommit, and
+database reset in development as a testing ground for migrations, so it always
+has up-to-date schema without unexpected changes.
+
+Example call with preload can look like this:
+`dotenv -- mosaic pg-dump -c scripts/infra`, where `dotenv` is a cli command to
+load environment variables using `dotenv-cli` npm package. Alternatively,
+`env-cmd` or some other tool can be used instead of `dotenv`
+
+Example call without preload can look like this:
+`mosaic pg-dump -c scripts/infra -s postgres://username:password@localhost:5432/database_name`