@webiny/api-sync-pg-to-opensearch 0.0.0-unstable.0d717d18dd

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) Webiny
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,11 @@
+# @webiny/api-sync-pg-to-opensearch
+
+> [!NOTE]
+> This package is part of the [Webiny](https://www.webiny.com) monorepo.
+> It’s **included in every Webiny project by default** and is not meant to be used as a standalone package.
+
+📘 **Documentation:** [https://www.webiny.com/docs](https://www.webiny.com/docs)
+
+---
+
+_This README file is automatically generated during the publish process._

package/createPgToOpenSearchHandler.d.ts

@@ -0,0 +1,4 @@
+import type { Client } from "@webiny/api-opensearch/client.js";
+import type { PgWalChangeRecord } from "./types.js";
+export type PgToOpenSearchHandler = (records: PgWalChangeRecord[]) => Promise<void>;
+export declare const createPgToOpenSearchHandler: (client: Client) => PgToOpenSearchHandler;

package/createPgToOpenSearchHandler.js

@@ -0,0 +1,33 @@
+import { Container } from "@webiny/feature/api";
+import { TimerFeature } from "@webiny/utils/features/Timer/feature.js";
+import { ProcessEnvFeature } from "@webiny/stdlib/node";
+import { OperationsBuilder } from "@webiny/api-sync-to-opensearch/features/OperationsBuilder/abstraction.js";
+import { ExecuteSyncWithRetry } from "@webiny/api-sync-to-opensearch/features/ExecuteSyncWithRetry/abstraction.js";
+import { PgToOpenSearchFeature } from "./features/PgToOpenSearchFeature.js";
+const MAX_RUNNING_TIME = 900;
+const createPgToOpenSearchHandler = (client)=>{
+    const container = new Container();
+    ProcessEnvFeature.register(container);
+    TimerFeature.register(container, {
+        getRemainingSeconds: ()=>MAX_RUNNING_TIME,
+        getRemainingMilliseconds: ()=>1000 * MAX_RUNNING_TIME
+    });
+    PgToOpenSearchFeature.register(container, {
+        client
+    });
+    const builder = container.resolve(OperationsBuilder);
+    const executeSyncWithRetry = container.resolve(ExecuteSyncWithRetry);
+    return async (records)=>{
+        const operations = await builder.build({
+            records
+        });
+        if (0 === operations.total) return;
+        await executeSyncWithRetry.execute({
+            maxRunningTime: MAX_RUNNING_TIME,
+            operations
+        });
+    };
+};
+export { createPgToOpenSearchHandler };
+
+//# sourceMappingURL=createPgToOpenSearchHandler.js.map

package/createPgToOpenSearchHandler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"createPgToOpenSearchHandler.js","sources":["../src/createPgToOpenSearchHandler.ts"],"sourcesContent":["import { Container } from \"@webiny/feature/api\";\nimport { TimerFeature } from \"@webiny/utils/features/Timer/feature.js\";\nimport { ProcessEnvFeature } from \"@webiny/stdlib/node\";\nimport { OperationsBuilder } from \"@webiny/api-sync-to-opensearch/features/OperationsBuilder/abstraction.js\";\nimport { ExecuteSyncWithRetry } from \"@webiny/api-sync-to-opensearch/features/ExecuteSyncWithRetry/abstraction.js\";\nimport { PgToOpenSearchFeature } from \"./features/PgToOpenSearchFeature.js\";\nimport type { Client } from \"@webiny/api-opensearch/client.js\";\nimport type { PgWalChangeRecord } from \"./types.js\";\n\nconst MAX_RUNNING_TIME = 900;\n\nexport type PgToOpenSearchHandler = (records: PgWalChangeRecord[]) => Promise<void>;\n\nexport const createPgToOpenSearchHandler = (client: Client): PgToOpenSearchHandler => {\n    const container = new Container();\n\n    ProcessEnvFeature.register(container);\n    TimerFeature.register(container, {\n        getRemainingSeconds: () => MAX_RUNNING_TIME,\n        getRemainingMilliseconds: () => MAX_RUNNING_TIME * 1000\n    });\n\n    PgToOpenSearchFeature.register(container, { client });\n\n    const builder = container.resolve(OperationsBuilder);\n    const executeSyncWithRetry = container.resolve(ExecuteSyncWithRetry);\n\n    return async (records: PgWalChangeRecord[]): Promise<void> => {\n        const operations = await builder.build({ records });\n\n        if (operations.total === 0) {\n            return;\n        }\n\n        await executeSyncWithRetry.execute({\n            maxRunningTime: MAX_RUNNING_TIME,\n            operations\n        });\n    };\n};\n"],"names":["MAX_RUNNING_TIME","createPgToOpenSearchHandler","client","container","Container","ProcessEnvFeature","TimerFeature","PgToOpenSearchFeature","builder","OperationsBuilder","executeSyncWithRetry","ExecuteSyncWithRetry","records","operations"],"mappings":";;;;;;AASA,MAAMA,mBAAmB;AAIlB,MAAMC,8BAA8B,CAACC;IACxC,MAAMC,YAAY,IAAIC;IAEtBC,kBAAkB,QAAQ,CAACF;IAC3BG,aAAa,QAAQ,CAACH,WAAW;QAC7B,qBAAqB,IAAMH;QAC3B,0BAA0B,IAAMA,AAAmB,OAAnBA;IACpC;IAEAO,sBAAsB,QAAQ,CAACJ,WAAW;QAAED;IAAO;IAEnD,MAAMM,UAAUL,UAAU,OAAO,CAACM;IAClC,MAAMC,uBAAuBP,UAAU,OAAO,CAACQ;IAE/C,OAAO,OAAOC;QACV,MAAMC,aAAa,MAAML,QAAQ,KAAK,CAAC;YAAEI;QAAQ;QAEjD,IAAIC,AAAqB,MAArBA,WAAW,KAAK,EAChB;QAGJ,MAAMH,qBAAqB,OAAO,CAAC;YAC/B,gBAAgBV;YAChBa;QACJ;IACJ;AACJ"}

package/features/PgOperationsBuilder/PgOperationsBuilder.d.ts

@@ -0,0 +1,17 @@
+import { OperationsBuilder } from "@webiny/api-sync-to-opensearch/features/OperationsBuilder/abstraction.js";
+import type { Operations } from "@webiny/api-sync-to-opensearch/features/Operations/abstractions/Operations.js";
+import { OperationsFactory } from "@webiny/api-sync-to-opensearch/features/Operations/abstractions/OperationsFactory.js";
+import { CompressionHandler } from "@webiny/utils/exports/api.js";
+import type { PgWalChangeRecord } from "../../types.js";
+declare class PgOperationsBuilderImpl implements OperationsBuilder.Interface<PgWalChangeRecord> {
+    private readonly compressor;
+    private readonly operationsFactory;
+    constructor(compressor: CompressionHandler.Interface, operationsFactory: OperationsFactory.Interface);
+    build(params: {
+        records: PgWalChangeRecord[];
+    }): Promise<Operations.Interface>;
+}
+export declare const PgOperationsBuilder: typeof PgOperationsBuilderImpl & {
+    __abstraction: import("@webiny/di").Abstraction<import("@webiny/api-sync-to-opensearch/features/OperationsBuilder/abstraction.js").IOperationsBuilder<unknown>>;
+};
+export {};

package/features/PgOperationsBuilder/PgOperationsBuilder.js

@@ -0,0 +1,49 @@
+import { OperationsBuilder } from "@webiny/api-sync-to-opensearch/features/OperationsBuilder/abstraction.js";
+import { OperationType } from "@webiny/api-sync-to-opensearch/features/Operations/Operations.js";
+import { OperationsFactory } from "@webiny/api-sync-to-opensearch/features/Operations/abstractions/OperationsFactory.js";
+import { CompressionHandler } from "@webiny/utils/exports/api.js";
+class PgOperationsBuilderImpl {
+    constructor(compressor, operationsFactory){
+        this.compressor = compressor;
+        this.operationsFactory = operationsFactory;
+    }
+    async build(params) {
+        const operations = this.operationsFactory.create();
+        for (const record of params.records){
+            if (!record.id || !record.index) {
+                console.error("Missing id or index in sync record, skipping.");
+                continue;
+            }
+            if (record.operation === OperationType.INSERT || record.operation === OperationType.MODIFY) {
+                if (!record.data?.value) {
+                    console.error(`Missing data for ${record.operation} operation, ID ${record.id}. Skipping.`);
+                    continue;
+                }
+                const data = await this.compressor.decompress(record.data);
+                if (null == data) {
+                    console.error(`Could not decompress data for operation "${record.operation}", ID ${record.id}. Skipping.`);
+                    continue;
+                }
+                operations.insert({
+                    id: record.id,
+                    index: record.index,
+                    data
+                });
+            } else if (record.operation === OperationType.REMOVE) operations.delete({
+                id: record.id,
+                index: record.index
+            });
+        }
+        return operations;
+    }
+}
+const PgOperationsBuilder = OperationsBuilder.createImplementation({
+    implementation: PgOperationsBuilderImpl,
+    dependencies: [
+        CompressionHandler,
+        OperationsFactory
+    ]
+});
+export { PgOperationsBuilder };
+
+//# sourceMappingURL=PgOperationsBuilder.js.map

package/features/PgOperationsBuilder/PgOperationsBuilder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"features/PgOperationsBuilder/PgOperationsBuilder.js","sources":["../../../src/features/PgOperationsBuilder/PgOperationsBuilder.ts"],"sourcesContent":["import { OperationsBuilder } from \"@webiny/api-sync-to-opensearch/features/OperationsBuilder/abstraction.js\";\nimport { OperationType } from \"@webiny/api-sync-to-opensearch/features/Operations/Operations.js\";\nimport type { Operations } from \"@webiny/api-sync-to-opensearch/features/Operations/abstractions/Operations.js\";\nimport { OperationsFactory } from \"@webiny/api-sync-to-opensearch/features/Operations/abstractions/OperationsFactory.js\";\nimport { CompressionHandler } from \"@webiny/utils/exports/api.js\";\nimport type { PgWalChangeRecord } from \"~/types.js\";\n\nclass PgOperationsBuilderImpl implements OperationsBuilder.Interface<PgWalChangeRecord> {\n    public constructor(\n        private readonly compressor: CompressionHandler.Interface,\n        private readonly operationsFactory: OperationsFactory.Interface\n    ) {}\n\n    public async build(params: { records: PgWalChangeRecord[] }): Promise<Operations.Interface> {\n        const operations = this.operationsFactory.create();\n        for (const record of params.records) {\n            if (!record.id || !record.index) {\n                console.error(`Missing id or index in sync record, skipping.`);\n                continue;\n            }\n\n            if (\n                record.operation === OperationType.INSERT ||\n                record.operation === OperationType.MODIFY\n            ) {\n                if (!record.data?.value) {\n                    console.error(\n                        `Missing data for ${record.operation} operation, ID ${record.id}. Skipping.`\n                    );\n                    continue;\n                }\n                const data = await this.compressor.decompress(record.data);\n                if (data === undefined || data === null) {\n                    console.error(\n                        `Could not decompress data for operation \"${record.operation}\", ID ${record.id}. Skipping.`\n                    );\n                    continue;\n                }\n\n                operations.insert({\n                    id: record.id,\n                    index: record.index,\n                    data\n                });\n            } else if (record.operation === OperationType.REMOVE) {\n                operations.delete({\n                    id: record.id,\n                    index: record.index\n                });\n            }\n        }\n        return operations;\n    }\n}\n\nexport const PgOperationsBuilder = OperationsBuilder.createImplementation({\n    implementation: PgOperationsBuilderImpl,\n    dependencies: [CompressionHandler, OperationsFactory]\n});\n"],"names":["PgOperationsBuilderImpl","compressor","operationsFactory","params","operations","record","console","OperationType","data","PgOperationsBuilder","OperationsBuilder","CompressionHandler","OperationsFactory"],"mappings":";;;;AAOA,MAAMA;IACF,YACqBC,UAAwC,EACxCC,iBAA8C,CACjE;aAFmBD,UAAU,GAAVA;aACAC,iBAAiB,GAAjBA;IAClB;IAEH,MAAa,MAAMC,MAAwC,EAAiC;QACxF,MAAMC,aAAa,IAAI,CAAC,iBAAiB,CAAC,MAAM;QAChD,KAAK,MAAMC,UAAUF,OAAO,OAAO,CAAE;YACjC,IAAI,CAACE,OAAO,EAAE,IAAI,CAACA,OAAO,KAAK,EAAE;gBAC7BC,QAAQ,KAAK,CAAC;gBACd;YACJ;YAEA,IACID,OAAO,SAAS,KAAKE,cAAc,MAAM,IACzCF,OAAO,SAAS,KAAKE,cAAc,MAAM,EAC3C;gBACE,IAAI,CAACF,OAAO,IAAI,EAAE,OAAO;oBACrBC,QAAQ,KAAK,CACT,CAAC,iBAAiB,EAAED,OAAO,SAAS,CAAC,eAAe,EAAEA,OAAO,EAAE,CAAC,WAAW,CAAC;oBAEhF;gBACJ;gBACA,MAAMG,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAACH,OAAO,IAAI;gBACzD,IAAIG,QAAAA,MAAqC;oBACrCF,QAAQ,KAAK,CACT,CAAC,yCAAyC,EAAED,OAAO,SAAS,CAAC,MAAM,EAAEA,OAAO,EAAE,CAAC,WAAW,CAAC;oBAE/F;gBACJ;gBAEAD,WAAW,MAAM,CAAC;oBACd,IAAIC,OAAO,EAAE;oBACb,OAAOA,OAAO,KAAK;oBACnBG;gBACJ;YACJ,OAAO,IAAIH,OAAO,SAAS,KAAKE,cAAc,MAAM,EAChDH,WAAW,MAAM,CAAC;gBACd,IAAIC,OAAO,EAAE;gBACb,OAAOA,OAAO,KAAK;YACvB;QAER;QACA,OAAOD;IACX;AACJ;AAEO,MAAMK,sBAAsBC,kBAAkB,oBAAoB,CAAC;IACtE,gBAAgBV;IAChB,cAAc;QAACW;QAAoBC;KAAkB;AACzD"}

package/features/PgOperationsBuilder/feature.d.ts

@@ -0,0 +1,4 @@
+export declare const PgOperationsBuilderFeature: {
+    name: string;
+    register(container: import("@webiny/di").Container): void;
+};

package/features/PgOperationsBuilder/feature.js

@@ -0,0 +1,11 @@
+import { createFeature } from "@webiny/feature/api";
+import { PgOperationsBuilder } from "./PgOperationsBuilder.js";
+const PgOperationsBuilderFeature = createFeature({
+    name: "sync.pg.operationsBuilder",
+    register (container) {
+        container.register(PgOperationsBuilder);
+    }
+});
+export { PgOperationsBuilderFeature };
+
+//# sourceMappingURL=feature.js.map

package/features/PgOperationsBuilder/feature.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"features/PgOperationsBuilder/feature.js","sources":["../../../src/features/PgOperationsBuilder/feature.ts"],"sourcesContent":["import { createFeature } from \"@webiny/feature/api\";\nimport { PgOperationsBuilder } from \"./PgOperationsBuilder.js\";\n\nexport const PgOperationsBuilderFeature = createFeature({\n    name: \"sync.pg.operationsBuilder\",\n    register(container) {\n        container.register(PgOperationsBuilder);\n    }\n});\n"],"names":["PgOperationsBuilderFeature","createFeature","container","PgOperationsBuilder"],"mappings":";;AAGO,MAAMA,6BAA6BC,cAAc;IACpD,MAAM;IACN,UAASC,SAAS;QACdA,UAAU,QAAQ,CAACC;IACvB;AACJ"}

package/features/PgToOpenSearchFeature.d.ts

@@ -0,0 +1,8 @@
+import type { Client } from "@webiny/api-opensearch/client.js";
+export interface PgToOpenSearchFeatureConfig {
+    client: Client;
+}
+export declare const PgToOpenSearchFeature: {
+    name: string;
+    register(container: import("@webiny/di").Container, context: PgToOpenSearchFeatureConfig): void;
+};

package/features/PgToOpenSearchFeature.js

@@ -0,0 +1,23 @@
+import { createFeature } from "@webiny/feature/api";
+import { OperationsFactoryFeature } from "@webiny/api-sync-to-opensearch/features/Operations/feature.js";
+import { ExecuteSyncFeature } from "@webiny/api-sync-to-opensearch/features/ExecuteSync/feature.js";
+import { ExecuteSyncWithRetryFeature } from "@webiny/api-sync-to-opensearch/features/ExecuteSyncWithRetry/feature.js";
+import { SynchronizationBuilderFeature } from "@webiny/api-sync-to-opensearch/features/SynchronizationBuilder/feature.js";
+import { PgOperationsBuilderFeature } from "./PgOperationsBuilder/feature.js";
+import { OpenSearchClientFeature } from "@webiny/api-opensearch/features/OpenSearchClient/feature.js";
+import { CompressionFeature } from "@webiny/utils/features/compression/feature.js";
+const PgToOpenSearchFeature = createFeature({
+    name: "sync.pg-to-opensearch",
+    register (container, config) {
+        OpenSearchClientFeature.register(container, config.client);
+        CompressionFeature.register(container);
+        OperationsFactoryFeature.register(container);
+        ExecuteSyncFeature.register(container);
+        ExecuteSyncWithRetryFeature.register(container);
+        SynchronizationBuilderFeature.register(container);
+        PgOperationsBuilderFeature.register(container);
+    }
+});
+export { PgToOpenSearchFeature };
+
+//# sourceMappingURL=PgToOpenSearchFeature.js.map

package/features/PgToOpenSearchFeature.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"features/PgToOpenSearchFeature.js","sources":["../../src/features/PgToOpenSearchFeature.ts"],"sourcesContent":["import { createFeature } from \"@webiny/feature/api\";\nimport type { Client } from \"@webiny/api-opensearch/client.js\";\nimport { OperationsFactoryFeature } from \"@webiny/api-sync-to-opensearch/features/Operations/feature.js\";\nimport { ExecuteSyncFeature } from \"@webiny/api-sync-to-opensearch/features/ExecuteSync/feature.js\";\nimport { ExecuteSyncWithRetryFeature } from \"@webiny/api-sync-to-opensearch/features/ExecuteSyncWithRetry/feature.js\";\nimport { SynchronizationBuilderFeature } from \"@webiny/api-sync-to-opensearch/features/SynchronizationBuilder/feature.js\";\nimport { PgOperationsBuilderFeature } from \"./PgOperationsBuilder/feature.js\";\nimport { OpenSearchClientFeature } from \"@webiny/api-opensearch/features/OpenSearchClient/feature.js\";\nimport { CompressionFeature } from \"@webiny/utils/features/compression/feature.js\";\n\nexport interface PgToOpenSearchFeatureConfig {\n    client: Client;\n}\n\nexport const PgToOpenSearchFeature = createFeature<PgToOpenSearchFeatureConfig>({\n    name: \"sync.pg-to-opensearch\",\n    register(container, config) {\n        OpenSearchClientFeature.register(container, config.client);\n        CompressionFeature.register(container);\n\n        OperationsFactoryFeature.register(container);\n        ExecuteSyncFeature.register(container);\n        ExecuteSyncWithRetryFeature.register(container);\n        SynchronizationBuilderFeature.register(container);\n\n        PgOperationsBuilderFeature.register(container);\n    }\n});\n"],"names":["PgToOpenSearchFeature","createFeature","container","config","OpenSearchClientFeature","CompressionFeature","OperationsFactoryFeature","ExecuteSyncFeature","ExecuteSyncWithRetryFeature","SynchronizationBuilderFeature","PgOperationsBuilderFeature"],"mappings":";;;;;;;;AAcO,MAAMA,wBAAwBC,cAA2C;IAC5E,MAAM;IACN,UAASC,SAAS,EAAEC,MAAM;QACtBC,wBAAwB,QAAQ,CAACF,WAAWC,OAAO,MAAM;QACzDE,mBAAmB,QAAQ,CAACH;QAE5BI,yBAAyB,QAAQ,CAACJ;QAClCK,mBAAmB,QAAQ,CAACL;QAC5BM,4BAA4B,QAAQ,CAACN;QACrCO,8BAA8B,QAAQ,CAACP;QAEvCQ,2BAA2B,QAAQ,CAACR;IACxC;AACJ"}

package/index.d.ts

@@ -0,0 +1,3 @@
+export type { PgWalChangeRecord, PgWalChangeRecordData } from "./types.js";
+export { createPgToOpenSearchHandler, type PgToOpenSearchHandler } from "./createPgToOpenSearchHandler.js";
+export { PgToOpenSearchFeature, type PgToOpenSearchFeatureConfig } from "./features/PgToOpenSearchFeature.js";

package/index.js

@@ -0,0 +1,2 @@
+export { createPgToOpenSearchHandler } from "./createPgToOpenSearchHandler.js";
+export { PgToOpenSearchFeature } from "./features/PgToOpenSearchFeature.js";

package/package.json

@@ -0,0 +1,39 @@
+{
+	"name": "@webiny/api-sync-pg-to-opensearch",
+	"version": "0.0.0-unstable.0d717d18dd",
+	"type": "module",
+	"exports": {
+		".": "./index.js",
+		"./*": "./*"
+	},
+	"repository": {
+		"type": "git",
+		"url": "https://github.com/webiny/webiny-js.git",
+		"directory": "packages/api-sync-pg-to-opensearch"
+	},
+	"description": "PostgreSQL to OpenSearch synchronization adapter via WAL.",
+	"license": "MIT",
+	"author": "Webiny Ltd.",
+	"dependencies": {
+		"@webiny/api-opensearch": "0.0.0-unstable.0d717d18dd",
+		"@webiny/api-sync-to-opensearch": "0.0.0-unstable.0d717d18dd",
+		"@webiny/feature": "0.0.0-unstable.0d717d18dd",
+		"@webiny/stdlib": "0.0.15",
+		"@webiny/utils": "0.0.0-unstable.0d717d18dd"
+	},
+	"devDependencies": {
+		"@webiny/build-tools": "0.0.0-unstable.0d717d18dd",
+		"typescript": "7.0.2"
+	},
+	"publishConfig": {
+		"access": "public"
+	},
+	"adio": {
+		"ignoreDirs": [
+			"__tests__"
+		]
+	},
+	"webiny": {
+		"publishFrom": "dist"
+	}
+}

package/types.d.ts

@@ -0,0 +1,12 @@
+export interface PgWalChangeRecordData {
+    compression: string;
+    value: string;
+}
+export interface PgWalChangeRecord {
+    id: string;
+    entryId: string;
+    index: string;
+    operation: string;
+    data: PgWalChangeRecordData;
+    tenant: string;
+}