node-server-engine 1.0.5

package/src/entities/PubSub/PubSub.test.ts

@@ -0,0 +1,297 @@
+import {
+  PubSub as PubsubClient,
+  IAM,
+  Topic,
+  Subscription
+} from '@google-cloud/pubsub';
+import { expect } from 'chai';
+import { faker } from '@faker-js/faker';
+import { stub, fake, match, SinonSpy, SinonStub } from 'sinon';
+import { PubSub } from './PubSub';
+import { EngineError } from 'entities/EngineError';
+
+describe('Entities - Pub/Sub', function () {
+  let topicExistsFake: SinonSpy;
+  let topicIamFake: SinonSpy;
+  let topicPublishFake: SinonSpy;
+  let topicFlushFake: SinonSpy;
+  let topicStub: SinonStub;
+  let subscriptionExistsFake: SinonSpy;
+  let subscriptionIamFake: SinonSpy;
+  let subscriptionOnFake: SinonSpy;
+  let subscriptionCloseFake: SinonSpy;
+  let subscriptionStub: SinonStub;
+
+  beforeEach(() => {
+    topicExistsFake = fake.resolves([true]);
+
+    topicIamFake = fake(async (permissions: Array<string>) =>
+      permissions.reduce(
+        (result, permission) => {
+          result[0][permission] = true;
+          return result;
+        },
+        [{} as Record<string, boolean>]
+      )
+    );
+    topicPublishFake = fake.resolves(undefined);
+    topicFlushFake = fake.resolves(undefined);
+    topicStub = stub(PubsubClient.prototype, 'topic').callsFake(
+      () =>
+        ({
+          exists: topicExistsFake,
+          iam: { testPermissions: topicIamFake } as unknown as IAM,
+          publishMessage: topicPublishFake,
+          flush: topicFlushFake
+        }) as unknown as Topic
+    );
+
+    subscriptionExistsFake = fake.resolves([true]);
+
+    subscriptionIamFake = fake(async (permissions: Array<string>) =>
+      permissions.reduce(
+        (result, permission) => {
+          result[0][permission] = true;
+          return result;
+        },
+        [{} as Record<string, boolean>]
+      )
+    );
+    subscriptionOnFake = fake();
+    subscriptionCloseFake = fake.resolves(undefined);
+    subscriptionStub = stub(PubsubClient.prototype, 'subscription').callsFake(
+      () =>
+        ({
+          exists: subscriptionExistsFake,
+          iam: { testPermissions: subscriptionIamFake } as unknown as IAM,
+          on: subscriptionOnFake,
+          close: subscriptionCloseFake
+        }) as unknown as Subscription
+    );
+  });
+
+  afterEach(async () => {
+    await PubSub.shutdown();
+  });
+
+  describe('addPublisher', () => {
+    it('should register a publisher only once', () => {
+      const topic = faker.lorem.slug();
+
+      PubSub.addPublisher(topic);
+      PubSub.addPublisher(topic);
+      PubSub.addPublisher(topic);
+
+      expect(topicStub).to.have.been.calledOnceWithExactly(topic, match.object);
+    });
+
+    it('should register multiple publishers', () => {
+      const [topic1, topic2, topic3] = [
+        faker.lorem.slug(),
+        faker.lorem.slug(),
+        faker.lorem.slug()
+      ];
+
+      PubSub.addPublisher(topic1);
+      PubSub.addPublisher([topic2, topic3]);
+
+      expect(topicStub).to.have.been.callCount(3);
+      expect(topicStub).to.have.been.calledWithExactly(topic1, match.object);
+      expect(topicStub).to.have.been.calledWithExactly(topic2, match.object);
+      expect(topicStub).to.have.been.calledWithExactly(topic3, match.object);
+    });
+  });
+
+  describe('addSubscriber', () => {
+    it('should register a subscription only once', () => {
+      const subscription = faker.lorem.slug();
+      const handler = fake();
+
+      PubSub.addSubscriber(subscription, handler);
+      PubSub.addSubscriber(subscription, handler);
+      PubSub.addSubscriber(subscription, handler);
+
+      expect(subscriptionStub).to.have.been.calledOnceWithExactly(
+        subscription,
+        match.object
+      );
+    });
+
+    it('should register a subscription and making them run first when a new message arrives', () => {
+      const subscription = faker.lorem.slug();
+      const handler = fake();
+
+      PubSub.addSubscriber(subscription, handler, { first: true });
+      expect(subscriptionStub).to.have.been.calledOnceWithExactly(
+        subscription,
+        match.object
+      );
+    });
+
+    it('should register a subscription with debezium events', () => {
+      const subscription = faker.lorem.slug();
+      const handler = fake();
+
+      PubSub.addSubscriber(subscription, handler, { isDebezium: true });
+      expect(subscriptionStub).to.have.been.calledOnceWithExactly(
+        subscription,
+        match.object
+      );
+    });
+
+    it('should register multiple subscriptions', () => {
+      const handler = fake();
+      const [subscription1, subscription2, subscription3] = [
+        faker.lorem.slug(),
+        faker.lorem.slug(),
+        faker.lorem.slug()
+      ];
+
+      PubSub.addSubscriber(subscription1, handler);
+      PubSub.addSubscriber(subscription2, handler);
+      PubSub.addSubscriber(subscription3, handler);
+
+      expect(subscriptionStub).to.have.been.callCount(3);
+      expect(subscriptionStub).to.have.been.calledWithExactly(
+        subscription1,
+        match.object
+      );
+      expect(subscriptionStub).to.have.been.calledWithExactly(
+        subscription2,
+        match.object
+      );
+      expect(subscriptionStub).to.have.been.calledWithExactly(
+        subscription3,
+        match.object
+      );
+    });
+  });
+
+  describe('init', () => {
+    it('should throw an error if a topic does not exist', async () => {
+      topicExistsFake = fake.resolves([false]);
+
+      PubSub.addPublisher(faker.lorem.slug());
+
+      await expect(PubSub.init()).to.be.rejectedWith(EngineError);
+    });
+
+    it('should throw an error if an instance has no permission on the topic', async () => {
+      topicIamFake = fake(async (permissions: Array<string>) =>
+        permissions.reduce(
+          (result, permission) => {
+            result[0][permission] = false;
+            return result;
+          },
+          [{} as Record<string, boolean>]
+        )
+      );
+
+      PubSub.addPublisher(faker.lorem.slug());
+
+      await expect(PubSub.init()).to.be.rejectedWith(EngineError);
+    });
+
+    it('should throw an error if a subscription does not exist', async () => {
+      const handler = fake();
+      subscriptionExistsFake = fake.resolves([false]);
+
+      PubSub.addSubscriber(faker.lorem.slug(), handler);
+
+      await expect(PubSub.init()).to.be.rejectedWith(EngineError);
+    });
+
+    it('should throw an error if an instance has no permission on the subscription', async () => {
+      const handler = fake();
+
+      subscriptionIamFake = fake(async (permissions: Array<string>) =>
+        permissions.reduce(
+          (result, permission) => {
+            result[0][permission] = false;
+            return result;
+          },
+          [{} as Record<string, boolean>]
+        )
+      );
+
+      PubSub.addSubscriber(faker.lorem.slug(), handler);
+
+      await expect(PubSub.init()).to.be.rejectedWith(EngineError);
+    });
+
+    it('should successfully init', async () => {
+      const topic = faker.lorem.slug();
+      const subscription = faker.lorem.slug();
+      const subscriptionHandler = fake();
+
+      PubSub.addPublisher(topic);
+      PubSub.addSubscriber(subscription, subscriptionHandler);
+
+      await PubSub.init();
+
+      expect(topicExistsFake).to.have.been.calledOnceWithExactly();
+      expect(topicIamFake).to.have.been.calledOnceWithExactly([
+        'pubsub.topics.publish'
+      ]);
+      expect(subscriptionExistsFake).to.have.been.calledOnceWithExactly();
+      expect(subscriptionIamFake).to.have.been.calledOnceWithExactly([
+        'pubsub.subscriptions.consume'
+      ]);
+      expect(subscriptionOnFake).to.have.been.calledOnceWithExactly(
+        'message',
+        match.func
+      );
+    });
+  });
+
+  describe('shutdown', () => {
+    it('should flush all publishers', async () => {
+      PubSub.addPublisher([
+        faker.lorem.slug(),
+        faker.lorem.slug(),
+        faker.lorem.slug()
+      ]);
+
+      await PubSub.shutdown();
+
+      expect(topicFlushFake).to.have.callCount(3);
+    });
+
+    it('should close all subscriptions', async () => {
+      const handler = fake();
+      PubSub.addSubscriber(faker.lorem.slug(), handler);
+      PubSub.addSubscriber(faker.lorem.slug(), handler);
+      PubSub.addSubscriber(faker.lorem.slug(), handler);
+
+      await PubSub.shutdown();
+
+      expect(subscriptionCloseFake).to.have.callCount(3);
+    });
+  });
+
+  describe('publish', () => {
+    it('should throw an error if the topi was not previously declared', async () => {
+      const message = { [faker.lorem.word()]: faker.lorem.word() };
+      const attributes = { [faker.lorem.word()]: faker.lorem.word() };
+
+      await expect(
+        PubSub.publish(faker.lorem.slug(), message, attributes)
+      ).to.be.rejectedWith(EngineError);
+    });
+
+    it('should publish a message', async () => {
+      const topic = faker.lorem.slug();
+      const message = { [faker.lorem.word()]: faker.lorem.word() };
+      const attributes = { [faker.lorem.word()]: faker.lorem.word() };
+
+      PubSub.addPublisher(topic);
+      await PubSub.publish(topic, message, attributes);
+
+      expect(topicPublishFake).to.have.been.calledOnceWithExactly({
+        json: message,
+        attributes,
+        orderingKey: undefined
+      });
+    });
+  });
+});

package/src/entities/PubSub/PubSub.ts

@@ -0,0 +1,202 @@
+import {
+  Attributes,
+  PubSub as PubsubClient,
+  Message
+} from '@google-cloud/pubsub';
+import { handleMessage } from './handleMessage';
+import {
+  PubSubTopicsMap,
+  PubSubSubscriptionsMap,
+  PubSubAddSubscriberOptions,
+  PubSubMessageHandler
+} from './PubSub.types';
+import { EngineError } from 'entities/EngineError';
+import { LifecycleController } from 'entities/LifecycleController';
+import { reportDebug, reportError } from 'utils';
+
+const namespace = 'engine:PubSub';
+
+/**
+ * Associative containing all the subscriptions sorted by topic registered for publishing
+ */
+const subscriptions: PubSubSubscriptionsMap = {};
+
+/**
+ * Associative containing all the topics registered for publishing
+ */
+const topics: PubSubTopicsMap = {};
+
+const pubSubClient = new PubsubClient();
+
+/**
+ * This is a singleton service who's role is to manage pub/sub connections
+ * It is the preferred way to interact with Pub/Sub as it manages the underlying subscription and handler objects
+ * It avoids potential subscription splitting on an instance
+ */
+export const PubSub = {
+  /**
+   * Initialize the registered publishers and subscribers
+   */
+  async init(): Promise<void> {
+    reportDebug({ namespace, message: `Initializing Pub/Sub` });
+    // Check that all topics/permissions exist, and that we have permission on them
+    await Promise.all([
+      // Test that all topics actually exist and that we have permissions
+      ...Object.values(topics).map(async (topic) => {
+        const [exists] = await topic.exists();
+        if (!exists)
+          throw new EngineError(`Topic ${topic.name} does not exist`);
+        // Test permission ton publish to topic
+        const [permissions] = await topic.iam.testPermissions([
+          'pubsub.topics.publish'
+        ]);
+        const hasPermissions = Object.values(permissions).every(
+          (success) => success
+        );
+        if (!hasPermissions)
+          throw new EngineError(
+            `Instance does not have appropriate permissions to publish to topic ${topic.name}`
+          );
+      }),
+      // Test that all subscriptions actually exist and that we have permissions
+      ...Object.values(subscriptions).map(async ({ subscription }) => {
+        const [exists] = await subscription.exists();
+        if (!exists)
+          throw new EngineError(
+            `Subscription ${subscription.name} does not exist`
+          );
+        // Test permission ton consume subscription
+        const [permissions] = await subscription.iam.testPermissions([
+          'pubsub.subscriptions.consume'
+        ]);
+        const hasPermissions = Object.values(permissions).every(
+          (success) => success
+        );
+        if (!hasPermissions)
+          throw new EngineError(
+            `Instance does not have appropriate permissions to subscribe to subscription ${subscription.name}`
+          );
+      })
+    ]);
+
+    // Register handlers on the subscriptions to activate them
+    for (const { subscription, handlers, isDebezium } of Object.values(
+      subscriptions
+    )) {
+      subscription.on('message', (message) => {
+        handleMessage(
+          message as Message,
+          subscription,
+          handlers,
+          isDebezium
+        ).catch((error) => {
+          reportError(error);
+        });
+      });
+    }
+
+    LifecycleController.register(PubSub);
+  },
+
+  /**
+   * Shutdown the registered publishers and subscribers
+   */
+  async shutdown(): Promise<void> {
+    reportDebug({ namespace, message: `Shutting down Pub/Sub` });
+    // Flush all pending messages and shut down connections
+    await Promise.all([
+      ...Object.values(topics).map(async (topic) => topic.flush()),
+      ...Object.values(subscriptions).map(async ({ subscription }) =>
+        subscription.close()
+      )
+    ]);
+    // Delete the stored topics/subscriptions
+    Object.keys(topics).forEach((topic) => delete topics[topic]);
+    Object.keys(subscriptions).forEach(
+      (subscription) => delete subscriptions[subscription]
+    );
+    // Close the client
+    await pubSubClient.close();
+  },
+
+  /**
+   * Creates a publisher if it does not exist yet
+   */
+  addPublisher(topic: string | Array<string>): void {
+    const topicArray = topic instanceof Array ? topic : [topic];
+    reportDebug({
+      namespace,
+      message: `Registering publisher for topics '${topicArray.join("', '")}'`
+    });
+    // Register a new topic if it does not exist yet
+    for (const topic of topicArray) {
+      if (!Object.keys(topics).includes(topic)) {
+        topics[topic] = pubSubClient.topic(topic, {
+          enableOpenTelemetryTracing: true
+        });
+      }
+    }
+  },
+
+  /**
+   * Add message handlers to a subscription
+   * @param subscription - Name of the Pub/Sub subscription
+   * @param handler - Function or list of functions to set as message handlers
+   * @param options - Indicates that the additional options need to handle on Subscriber
+   */
+  addSubscriber<T>(
+    subscription: string,
+    handler: PubSubMessageHandler<T> | Array<PubSubMessageHandler<T>>,
+    options: PubSubAddSubscriberOptions = {}
+  ): void {
+    const handlers = handler instanceof Array ? handler : [handler];
+    const isDebezium = options?.isDebezium ?? false;
+    const first = options?.first ?? false;
+
+    reportDebug({
+      namespace,
+      message: `Registering subscriber for subscription '${subscription}'`,
+      data: { first, handlersCount: handlers.length, isDebezium }
+    });
+    if (!Object.keys(subscriptions).includes(subscription)) {
+      subscriptions[subscription] = {
+        subscription: pubSubClient.subscription(subscription, {
+          enableOpenTelemetryTracing: true
+        }),
+        handlers: []
+      };
+    }
+
+    if (isDebezium) subscriptions[subscription].isDebezium = true;
+
+    if (first) subscriptions[subscription].handlers.push(...handlers);
+    else subscriptions[subscription].handlers.unshift(...handlers);
+  },
+
+  /**
+   * Send a message to a Pub/Sub topic
+   * @param topic - Name of the topic
+   * @param message - Content of the message, must be an object that can be JSON stringified
+   * @param attributes - Attributes for this message, can be used to filter messages on the subscription
+   * @param orderingKey - Enforce order guarantee of messages sent with the same key
+   */
+  async publish(
+    topic: string,
+    message: unknown,
+    attributes?: Attributes,
+    orderingKey?: string
+  ): Promise<void> {
+    if (!topics[topic])
+      throw new EngineError(`No publisher was registered for topic '${topic}'`);
+    reportDebug({
+      namespace,
+      message: `Sending message to topic '${topic}'`,
+      data: { message, attributes, orderingKey }
+    });
+    await topics[topic].publishMessage({
+      json: message,
+      attributes,
+      orderingKey
+    });
+  }
+};

package/src/entities/PubSub/PubSub.types.ts

@@ -0,0 +1,76 @@
+import { Topic, Subscription, Attributes } from '@google-cloud/pubsub';
+import { DebeziumEvent } from 'const/DebeziumEvent';
+
+/** List of all active topics */
+export interface PubSubTopicsMap {
+  [topic: string]: Topic;
+}
+
+/** List of subscriptions handled for each topic */
+export interface PubSubSubscriptionsMap {
+  [topic: string]: {
+    /** The Pub/Sub subscription object */
+    subscription: Subscription;
+    /** The message handling functions */
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    handlers: Array<PubSubMessageHandler<any>>;
+    /** Indicates that the subscription from Debezium events */
+    isDebezium?: boolean;
+  };
+}
+
+/** Options to setup a subscription  */
+export interface PubSubAddSubscriberOptions {
+  /** Should the handlers declared here be added to the top of the queue, and called first when an new message arrives */
+  first?: boolean;
+  /** Indicates that the topic is a Debezium CDC topic, additional processing is done on the message */
+  isDebezium?: boolean;
+}
+
+export type PubSubMessageHandler<T> = (
+  message: T,
+  attributes: Attributes,
+  publishedAt: Date
+) => void | Promise<void>;
+
+/** Debezium create event payload */
+interface PubSubDebeziumCreatePayload<T> {
+  /** The instance properties before the operation */
+  before: null;
+  /** The instance properties after the operation */
+  after: T;
+  /** The operation that was executed */
+  op: DebeziumEvent.Create;
+  /** List of fields that have changed */
+  diff: Array<string>;
+}
+
+/** Debezium update event payload */
+interface PubSubDebeziumUpdatePayload<T> {
+  /** The instance properties before the operation */
+  before: T;
+  /** The instance properties after the operation */
+  after: T;
+  /** The operation that was executed */
+  op: DebeziumEvent.Update;
+  /** List of fields that have changed */
+  diff: Array<string>;
+}
+
+/** Debezium delete event payload */
+interface PubSubDebeziumDeletePayload<T> {
+  /** The instance properties before the operation */
+  before: T;
+  /** The instance properties after the operation */
+  after: null;
+  /** The operation that was executed */
+  op: DebeziumEvent.Delete;
+  /** List of fields that have changed */
+  diff: Array<string>;
+}
+
+/** Payload of Debezium messages */
+export type PubSubDebeziumPayload<T> =
+  | PubSubDebeziumCreatePayload<T>
+  | PubSubDebeziumUpdatePayload<T>
+  | PubSubDebeziumDeletePayload<T>;

package/src/entities/PubSub/handleMessage/handleMessage.test.ts

@@ -0,0 +1,102 @@
+import { PreciseDate } from '@google-cloud/precise-date';
+import {
+  Subscription,
+  PubSub,
+  Attributes,
+  Message
+} from '@google-cloud/pubsub';
+import { expect } from 'chai';
+import { faker } from '@faker-js/faker';
+import { fake, SinonSpy } from 'sinon';
+import { handleMessage } from './handleMessage';
+
+describe('Entities - Pub/Sub - Handle Message', () => {
+  let subscription: Subscription;
+  let payload: Record<string, unknown>;
+  let attributes: Attributes;
+  let ack: SinonSpy;
+  let message: Message;
+  let publishTime: PreciseDate;
+
+  beforeEach(() => {
+    subscription = new Subscription(new PubSub(), faker.lorem.slug());
+
+    payload = { [faker.lorem.word()]: faker.lorem.word() };
+
+    attributes = { [faker.lorem.word()]: faker.lorem.word() };
+    const data = Buffer.from(JSON.stringify(payload));
+
+    publishTime = new PreciseDate(faker.date.recent());
+
+    ack = fake();
+    message = {
+      id: faker.string.uuid(),
+      ack,
+      ackId: faker.string.uuid(),
+      data,
+      attributes,
+      publishTime,
+      deliveryAttempt: 0,
+      received: 0
+    } as unknown as Message;
+  });
+
+  it('should parse the message and forward it to all the handlers', async () => {
+    const handlers = [fake(), fake(), fake()];
+
+    await handleMessage(message, subscription, handlers);
+
+    for (const handler of handlers) {
+      expect(handler).to.have.been.calledOnceWithExactly(
+        payload,
+        attributes,
+        publishTime
+      );
+    }
+  });
+
+  it('should parse a debezium message [create]', async () => {
+    const handler = fake();
+    payload = { before: null, after: { a: 'a', b: 'b', c: 'c' } };
+    message.data = Buffer.from(JSON.stringify(payload), 'utf-8');
+
+    await handleMessage(message, subscription, [handler], true);
+
+    expect(handler).to.have.been.calledOnceWithExactly(
+      { ...payload, diff: ['a', 'b', 'c'] },
+      attributes,
+      publishTime
+    );
+  });
+
+  it('should parse a debezium message [destroy]', async () => {
+    const handler = fake();
+    payload = { before: { a: 'a', b: 'b', c: 'c' }, after: null };
+    message.data = Buffer.from(JSON.stringify(payload), 'utf-8');
+
+    await handleMessage(message, subscription, [handler], true);
+
+    expect(handler).to.have.been.calledOnceWithExactly(
+      { ...payload, diff: ['a', 'b', 'c'] },
+      attributes,
+      publishTime
+    );
+  });
+
+  it('should parse a debezium message [update]', async () => {
+    const handler = fake();
+    payload = {
+      before: { a: 'a', b: 'b', c: 'c' },
+      after: { a: 'a', b: 'y', c: 'c' }
+    };
+    message.data = Buffer.from(JSON.stringify(payload), 'utf-8');
+
+    await handleMessage(message, subscription, [handler], true);
+
+    expect(handler).to.have.been.calledOnceWithExactly(
+      { ...payload, diff: ['b'] },
+      attributes,
+      publishTime
+    );
+  });
+});