@furvester/upstream-sync 1.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Furvester
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,5 @@
+# Upstream Sync
+
+[![CI/CD](https://github.com/Furvester/upstream-sync/actions/workflows/cicd.yml/badge.svg)](https://github.com/Furvester/upstream-sync/actions/workflows/cicd.yml)
+
+Sync from upstream via RabbitMQ.

package/dist/index.d.ts

@@ -0,0 +1,67 @@
+/// <reference types="node" resolution-mode="require"/>
+import type { EntityClass, EntityManager, MikroORM } from "@mikro-orm/postgresql";
+import { type CollectionPageParams, type Selector } from "jsonapi-zod-query";
+import type { Logger } from "logforth";
+import { type ConnectionOptions } from "rabbitmq-client";
+import type { z } from "zod";
+export type SyncManagerConfig = {
+    rabbitmq: ConnectionOptions;
+    orm: MikroORM;
+    logger: Logger;
+    namespace: string;
+    apiGatewayUrl: string;
+    apiKey: string;
+};
+export type SyncableEntityClass = EntityClass<{
+    id: string;
+    upstreamVersion: number;
+}>;
+export type PreSyncResource = {
+    id: string;
+    version: number;
+};
+export type PreSyncDocument<T extends PreSyncResource> = {
+    data: T[];
+    pageParams: CollectionPageParams;
+};
+export type MessageEventHandler<T = unknown> = (em: EntityManager, data: T) => Promise<void> | void;
+export type MessageEvent<T extends z.ZodTypeAny> = {
+    key: string;
+    schema: T;
+    handler: MessageEventHandler<ReturnType<T["parse"]>>;
+};
+export type PreSyncHandler<T> = (em: EntityManager, resource: T) => Promise<void> | void;
+export type PreSync<TSelector extends Selector<PreSyncDocument<PreSyncResource>>> = {
+    basePath: string;
+    searchParams?: URLSearchParams;
+    selector: TSelector;
+    entityClass: SyncableEntityClass;
+    create: PreSyncHandler<ReturnType<TSelector>["data"][number]>;
+    update: PreSyncHandler<ReturnType<TSelector>["data"][number]>;
+};
+export type UpstreamEntity = {
+    routingKeyPrefix: string;
+    events: MessageEvent<z.ZodTypeAny>[];
+    preSync: PreSync<Selector<PreSyncDocument<PreSyncResource>>>;
+};
+export type UpstreamService = {
+    serviceName: string;
+    entities: UpstreamEntity[];
+};
+export declare const createMessageEvent: <T extends z.ZodTypeAny>(event: MessageEvent<T>) => MessageEvent<T>;
+export declare const createPreSync: <T extends Selector<PreSyncDocument<PreSyncResource>>>(preSync: PreSync<T>) => PreSync<T>;
+export declare const createUpstreamEntity: (entity: UpstreamEntity) => UpstreamEntity;
+export declare const createUpstreamService: (service: UpstreamService) => UpstreamService;
+export declare class SyncManager {
+    private readonly rabbitmq;
+    private readonly config;
+    private readonly upstreamServices;
+    private readonly initialSyncMutex;
+    constructor(config: SyncManagerConfig);
+    addUpstreamService(service: UpstreamService): this;
+    run(): Promise<void>;
+    private createConsumer;
+    private runPreSync;
+    private preSyncEntity;
+    private waitReady;
+}

package/dist/index.js

@@ -0,0 +1,155 @@
+import { Mutex } from "async-mutex";
+import { handleJsonApiError, injectPageParams, } from "jsonapi-zod-query";
+import { Connection, } from "rabbitmq-client";
+export const createMessageEvent = (event) => event;
+export const createPreSync = (preSync) => preSync;
+export const createUpstreamEntity = (entity) => entity;
+export const createUpstreamService = (service) => service;
+export class SyncManager {
+    rabbitmq;
+    config;
+    upstreamServices = [];
+    initialSyncMutex = new Mutex();
+    constructor(config) {
+        this.config = config;
+        this.rabbitmq = new Connection(config.rabbitmq);
+        this.rabbitmq.on("error", (error) => {
+            config.logger.info("RabbitMQ connection error", { error });
+        });
+        this.rabbitmq.on("connection", () => {
+            config.logger.info("RabbitMQ connection successfully (re)established");
+        });
+    }
+    addUpstreamService(service) {
+        this.upstreamServices.push(service);
+        return this;
+    }
+    async run() {
+        await this.initialSyncMutex.acquire();
+        const consumer = this.createConsumer();
+        const handleShutdown = async () => {
+            await consumer.close();
+            await this.rabbitmq.close();
+            await this.config.orm.close(true);
+        };
+        process.on("SIGINT", handleShutdown);
+        process.on("SIGTERM", handleShutdown);
+        await this.runPreSync();
+        this.initialSyncMutex.release();
+    }
+    createConsumer() {
+        const queue = `${this.config.namespace}.upstream-sync`;
+        const exchanges = [];
+        const queueBindings = [];
+        const eventHandlers = new Map();
+        for (const service of this.upstreamServices) {
+            exchanges.push({
+                exchange: service.serviceName,
+                type: "topic",
+            });
+            for (const entity of service.entities) {
+                queueBindings.push({
+                    exchange: service.serviceName,
+                    queue,
+                    routingKey: `${entity.routingKeyPrefix}.*`,
+                });
+                for (const event of entity.events) {
+                    eventHandlers.set(`${entity.routingKeyPrefix}.${event.key}`, event);
+                }
+            }
+        }
+        return this.rabbitmq.createConsumer({
+            queue,
+            concurrency: 1,
+            qos: { prefetchCount: 1 },
+            exchanges,
+            queueBindings,
+        }, async (message) => {
+            await this.initialSyncMutex.waitForUnlock();
+            const eventHandler = eventHandlers.get(message.routingKey);
+            if (!eventHandler) {
+                this.config.logger.debug(`No event handler found for ${message.routingKey}`);
+                return;
+            }
+            const result = eventHandler.schema.safeParse(message.body);
+            if (!result.success) {
+                this.config.logger.error("Failed to parse message", {
+                    issues: result.error.issues,
+                });
+                return;
+            }
+            await this.config.orm.em.fork().transactional(async (em) => {
+                await eventHandler.handler(em, result.data);
+            });
+        });
+    }
+    async runPreSync() {
+        for (const service of this.upstreamServices) {
+            await this.waitReady(`/${service.serviceName}/health`, service.serviceName);
+            for (const entity of service.entities) {
+                await this.preSyncEntity(service, entity);
+            }
+        }
+    }
+    async preSyncEntity(service, entity) {
+        const em = this.config.orm.em.fork();
+        const versions = new Map((await em
+            .createQueryBuilder(entity.preSync.entityClass)
+            .select(["id", "upstreamVersion"])
+            .execute("all", false)).map(({ id, upstream_version }) => [id, upstream_version]));
+        const baseUrl = new URL(`/${service.serviceName}${entity.preSync.basePath}`, this.config.apiGatewayUrl);
+        if (entity.preSync.searchParams) {
+            baseUrl.search = entity.preSync.searchParams.toString();
+        }
+        let url = baseUrl;
+        do {
+            const response = await fetch(url, {
+                headers: {
+                    accept: "application/vnd.api+json",
+                    authorization: `Bearer ${this.config.apiKey}`,
+                },
+            });
+            await handleJsonApiError(response);
+            const document = entity.preSync.selector(await response.json());
+            for (const resource of document.data) {
+                const version = versions.get(resource.id);
+                if (version === undefined) {
+                    await entity.preSync.create(em, resource);
+                    continue;
+                }
+                versions.delete(resource.id);
+                if (version === resource.version) {
+                    continue;
+                }
+                await entity.preSync.update(em, resource);
+            }
+            if (document.pageParams.next) {
+                url = new URL(baseUrl);
+                injectPageParams(url, document.pageParams.next);
+                continue;
+            }
+            url = null;
+        } while (url !== null);
+        if (versions.size > 0) {
+            await em.nativeDelete(entity.preSync.entityClass, {
+                id: { $in: [...versions.keys()] },
+            });
+        }
+    }
+    async waitReady(path, serviceName) {
+        const healthUrl = new URL(path, this.config.apiGatewayUrl);
+        for (let i = 0; i < 12; ++i) {
+            try {
+                const response = await fetch(healthUrl);
+                if (response.ok) {
+                    return;
+                }
+            }
+            catch {
+                this.config.logger.info("Unable to verify health, backing off");
+                await new Promise((resolve) => setTimeout(resolve, 5000));
+            }
+        }
+        throw new Error(`Service "${serviceName}" didn't turn ready within 60 seconds`);
+    }
+}

package/package.json

@@ -0,0 +1,59 @@
+{
+  "name": "@furvester/upstream-sync",
+  "version": "1.0.0",
+  "description": "Sync from upstream via RabbitMQ",
+  "type": "module",
+  "author": "Ben Scholzen 'DASPRiD'",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/furvester/upstream-sync.git"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": [
+    "dist/**/*"
+  ],
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      }
+    }
+  },
+  "types": "./dist/index.d.ts",
+  "module": "./dist/index.js",
+  "scripts": {
+    "build": "tsc",
+    "coverage": "vitest run --coverage",
+    "format": "biome format . --write",
+    "check": "biome check . --apply"
+  },
+  "devDependencies": {
+    "@biomejs/biome": "1.7.1",
+    "@commitlint/cli": "^19.3.0",
+    "@commitlint/config-conventional": "^19.2.2",
+    "@mikro-orm/postgresql": "^6.3.13",
+    "@tsconfig/node20": "^20.1.4",
+    "@types/node": "^20.12.7",
+    "jsonapi-zod-query": "^2.1.1",
+    "lefthook": "^1.6.10",
+    "logforth": "^1.2.2",
+    "rabbitmq-client": "^5.0.0",
+    "typescript": "^5.4.5",
+    "zod": "^3.23.4"
+  },
+  "peerDependencies": {
+    "@mikro-orm/postgresql": "^6.3.13",
+    "jsonapi-zod-query": "^2.1.1",
+    "logforth": "^1.2.2",
+    "rabbitmq-client": "^5.0.0",
+    "zod": "^3.22.4"
+  },
+  "packageManager": "pnpm@9.9.0+sha512.60c18acd138bff695d339be6ad13f7e936eea6745660d4cc4a776d5247c540d0edee1a563695c183a66eb917ef88f2b4feb1fc25f32a7adcadc7aaf3438e99c1",
+  "dependencies": {
+    "async-mutex": "^0.5.0"
+  }
+}