@webiny/api-search-index-tasks-ddb-os 6.6.0-alpha.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) Webiny
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,11 @@
+# @webiny/api-search-index-tasks-ddb-os
+
+> [!NOTE]
+> This package is part of the [Webiny](https://www.webiny.com) monorepo.
+> It’s **included in every Webiny project by default** and is not meant to be used as a standalone package.
+
+📘 **Documentation:** [https://www.webiny.com/docs](https://www.webiny.com/docs)
+
+---
+
+_This README file is automatically generated during the publish process._

package/index.d.ts

@@ -0,0 +1,4 @@
+export declare const SearchIndexTasksDdbOsFeature: {
+    name: string;
+    register(container: import("@webiny/di").Container): void;
+};

package/index.js

@@ -0,0 +1,15 @@
+import { createFeature } from "@webiny/feature/api";
+import { SearchIndexTasksOsFeature } from "@webiny/api-search-index-tasks-os";
+import { DdbStorageScanner } from "./storage/StorageScanner.js";
+import { DdbStorageWriter } from "./storage/StorageWriter.js";
+const SearchIndexTasksDdbOsFeature = createFeature({
+    name: "SearchIndexTasksDdbOs",
+    register (container) {
+        SearchIndexTasksOsFeature.register(container);
+        container.register(DdbStorageScanner);
+        container.register(DdbStorageWriter);
+    }
+});
+export { SearchIndexTasksDdbOsFeature };
+
+//# sourceMappingURL=index.js.map

package/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sources":["../src/index.ts"],"sourcesContent":["import { createFeature } from \"@webiny/feature/api\";\nimport { SearchIndexTasksOsFeature } from \"@webiny/api-search-index-tasks-os\";\nimport { DdbStorageScanner } from \"~/storage/StorageScanner.js\";\nimport { DdbStorageWriter } from \"~/storage/StorageWriter.js\";\n\nexport const SearchIndexTasksDdbOsFeature = createFeature({\n    name: \"SearchIndexTasksDdbOs\",\n    register(container) {\n        SearchIndexTasksOsFeature.register(container);\n        container.register(DdbStorageScanner);\n        container.register(DdbStorageWriter);\n    }\n});\n"],"names":["SearchIndexTasksDdbOsFeature","createFeature","container","SearchIndexTasksOsFeature","DdbStorageScanner","DdbStorageWriter"],"mappings":";;;;AAKO,MAAMA,+BAA+BC,cAAc;IACtD,MAAM;IACN,UAASC,SAAS;QACdC,0BAA0B,QAAQ,CAACD;QACnCA,UAAU,QAAQ,CAACE;QACnBF,UAAU,QAAQ,CAACG;IACvB;AACJ"}

package/package.json

@@ -0,0 +1,40 @@
+{
+	"name": "@webiny/api-search-index-tasks-ddb-os",
+	"version": "6.6.0-alpha.0",
+	"type": "module",
+	"exports": {
+		".": "./index.js",
+		"./*": "./*"
+	},
+	"repository": {
+		"type": "git",
+		"url": "https://github.com/webiny/webiny-js.git",
+		"directory": "packages/api-search-index-tasks-ddb-os"
+	},
+	"description": "DynamoDB + OpenSearch bridge for @webiny/api-search-index-tasks.",
+	"license": "MIT",
+	"author": "Webiny Ltd.",
+	"dependencies": {
+		"@webiny/api-opensearch": "6.6.0-alpha.0",
+		"@webiny/api-search-index-tasks": "6.6.0-alpha.0",
+		"@webiny/api-search-index-tasks-os": "6.6.0-alpha.0",
+		"@webiny/db-dynamodb": "6.6.0-alpha.0",
+		"@webiny/feature": "6.6.0-alpha.0"
+	},
+	"devDependencies": {
+		"@webiny/build-tools": "6.6.0-alpha.0",
+		"typescript": "7.0.2",
+		"vitest": "4.1.10"
+	},
+	"publishConfig": {
+		"access": "public"
+	},
+	"adio": {
+		"ignoreDirs": [
+			"__tests__"
+		]
+	},
+	"webiny": {
+		"publishFrom": "dist"
+	}
+}

package/storage/StorageScanner.d.ts

@@ -0,0 +1,11 @@
+import { StorageScanner as Abstraction } from "@webiny/api-search-index-tasks/abstractions/StorageScanner.js";
+import { DynamoDBClient } from "@webiny/db-dynamodb/exports/api/db.js";
+declare class DdbStorageScannerImpl implements Abstraction.Interface {
+    private readonly table;
+    constructor(dynamoDBClient: DynamoDBClient.Interface);
+    scan(cursor: string | undefined, limit: number): Promise<Abstraction.Result>;
+}
+export declare const DdbStorageScanner: typeof DdbStorageScannerImpl & {
+    __abstraction: import("@webiny/di").Abstraction<import("@webiny/api-search-index-tasks/abstractions/StorageScanner.js").IStorageScanner>;
+};
+export {};

package/storage/StorageScanner.js

@@ -0,0 +1,45 @@
+import { StorageScanner } from "@webiny/api-search-index-tasks/abstractions/StorageScanner.js";
+import { scan } from "@webiny/db-dynamodb";
+import { createOpenSearchTable } from "@webiny/api-opensearch";
+import { DynamoDBClient } from "@webiny/db-dynamodb/exports/api/db.js";
+class DdbStorageScannerImpl {
+    constructor(dynamoDBClient){
+        this.table = createOpenSearchTable({
+            documentClient: dynamoDBClient.client
+        });
+    }
+    async scan(cursor, limit) {
+        const startKey = cursor ? JSON.parse(cursor) : void 0;
+        const results = await scan({
+            table: this.table.table,
+            options: {
+                startKey,
+                limit
+            }
+        });
+        const items = results.items.map((item)=>({
+                index: item.index,
+                entity: item._et || item.entity,
+                data: item,
+                modified: item.modified
+            }));
+        let nextCursor;
+        if (results.lastEvaluatedKey?.PK && results.lastEvaluatedKey?.SK) nextCursor = JSON.stringify({
+            PK: results.lastEvaluatedKey.PK,
+            SK: results.lastEvaluatedKey.SK
+        });
+        return {
+            items,
+            cursor: nextCursor
+        };
+    }
+}
+const DdbStorageScanner = StorageScanner.createImplementation({
+    implementation: DdbStorageScannerImpl,
+    dependencies: [
+        DynamoDBClient
+    ]
+});
+export { DdbStorageScanner };
+
+//# sourceMappingURL=StorageScanner.js.map

package/storage/StorageScanner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage/StorageScanner.js","sources":["../../src/storage/StorageScanner.ts"],"sourcesContent":["import { StorageScanner as Abstraction } from \"@webiny/api-search-index-tasks/abstractions/StorageScanner.js\";\nimport { scan } from \"@webiny/db-dynamodb\";\nimport { createOpenSearchTable } from \"@webiny/api-opensearch\";\nimport { DynamoDBClient } from \"@webiny/db-dynamodb/exports/api/db.js\";\n\ninterface IDynamoDbElasticsearchRecord {\n    PK: string;\n    SK: string;\n    index: string;\n    _et?: string;\n    entity: string;\n    data: Record<string, any>;\n    modified: string;\n    [key: string]: any;\n}\n\ninterface IDdbCursor {\n    PK: string;\n    SK: string;\n}\n\nclass DdbStorageScannerImpl implements Abstraction.Interface {\n    private readonly table;\n\n    constructor(dynamoDBClient: DynamoDBClient.Interface) {\n        this.table = createOpenSearchTable({\n            documentClient: dynamoDBClient.client\n        });\n    }\n\n    async scan(cursor: string | undefined, limit: number): Promise<Abstraction.Result> {\n        const startKey = cursor ? (JSON.parse(cursor) as IDdbCursor) : undefined;\n\n        const results = await scan<IDynamoDbElasticsearchRecord>({\n            table: this.table.table,\n            options: {\n                startKey,\n                limit\n            }\n        });\n\n        const items = results.items.map(item => ({\n            index: item.index,\n            entity: item._et || item.entity,\n            data: item,\n            modified: item.modified\n        }));\n\n        let nextCursor: string | undefined;\n        if (results.lastEvaluatedKey?.PK && results.lastEvaluatedKey?.SK) {\n            nextCursor = JSON.stringify({\n                PK: results.lastEvaluatedKey.PK,\n                SK: results.lastEvaluatedKey.SK\n            });\n        }\n\n        return {\n            items,\n            cursor: nextCursor\n        };\n    }\n}\n\nexport const DdbStorageScanner = Abstraction.createImplementation({\n    implementation: DdbStorageScannerImpl,\n    dependencies: [DynamoDBClient]\n});\n"],"names":["DdbStorageScannerImpl","dynamoDBClient","createOpenSearchTable","cursor","limit","startKey","JSON","undefined","results","scan","items","item","nextCursor","DdbStorageScanner","Abstraction","DynamoDBClient"],"mappings":";;;;AAqBA,MAAMA;IAGF,YAAYC,cAAwC,CAAE;QAClD,IAAI,CAAC,KAAK,GAAGC,sBAAsB;YAC/B,gBAAgBD,eAAe,MAAM;QACzC;IACJ;IAEA,MAAM,KAAKE,MAA0B,EAAEC,KAAa,EAA+B;QAC/E,MAAMC,WAAWF,SAAUG,KAAK,KAAK,CAACH,UAAyBI;QAE/D,MAAMC,UAAU,MAAMC,KAAmC;YACrD,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK;YACvB,SAAS;gBACLJ;gBACAD;YACJ;QACJ;QAEA,MAAMM,QAAQF,QAAQ,KAAK,CAAC,GAAG,CAACG,CAAAA,OAAS;gBACrC,OAAOA,KAAK,KAAK;gBACjB,QAAQA,KAAK,GAAG,IAAIA,KAAK,MAAM;gBAC/B,MAAMA;gBACN,UAAUA,KAAK,QAAQ;YAC3B;QAEA,IAAIC;QACJ,IAAIJ,QAAQ,gBAAgB,EAAE,MAAMA,QAAQ,gBAAgB,EAAE,IAC1DI,aAAaN,KAAK,SAAS,CAAC;YACxB,IAAIE,QAAQ,gBAAgB,CAAC,EAAE;YAC/B,IAAIA,QAAQ,gBAAgB,CAAC,EAAE;QACnC;QAGJ,OAAO;YACHE;YACA,QAAQE;QACZ;IACJ;AACJ;AAEO,MAAMC,oBAAoBC,eAAAA,oBAAgC,CAAC;IAC9D,gBAAgBd;IAChB,cAAc;QAACe;KAAe;AAClC"}

package/storage/StorageWriter.d.ts

@@ -0,0 +1,15 @@
+import { StorageWriter as Abstraction } from "@webiny/api-search-index-tasks/abstractions/StorageWriter.js";
+import { DynamoDBClient } from "@webiny/db-dynamodb/exports/api/db.js";
+declare class DdbStorageWriterImpl implements Abstraction.Interface {
+    private readonly table;
+    private readonly entities;
+    private buffer;
+    constructor(dynamoDBClient: DynamoDBClient.Interface);
+    put(record: Abstraction.Record): void;
+    execute(): Promise<void>;
+    private getEntity;
+}
+export declare const DdbStorageWriter: typeof DdbStorageWriterImpl & {
+    __abstraction: import("@webiny/di").Abstraction<import("@webiny/api-search-index-tasks/abstractions/StorageWriter.js").IStorageWriter>;
+};
+export {};

package/storage/StorageWriter.js

@@ -0,0 +1,45 @@
+import { StorageWriter } from "@webiny/api-search-index-tasks/abstractions/StorageWriter.js";
+import { createTableWriteBatch } from "@webiny/db-dynamodb";
+import { createOpenSearchEntity, createOpenSearchTable } from "@webiny/api-opensearch";
+import { DynamoDBClient } from "@webiny/db-dynamodb/exports/api/db.js";
+class DdbStorageWriterImpl {
+    constructor(dynamoDBClient){
+        this.entities = {};
+        this.buffer = [];
+        this.table = createOpenSearchTable({
+            documentClient: dynamoDBClient.client
+        });
+    }
+    put(record) {
+        const entity = this.getEntity(record.entity);
+        this.buffer.push({
+            entity,
+            data: record.data
+        });
+    }
+    async execute() {
+        if (0 === this.buffer.length) return;
+        const batch = createTableWriteBatch({
+            table: this.table.table
+        });
+        for (const { entity, data } of this.buffer)batch.put(entity.entity, data);
+        await batch.execute();
+        this.buffer = [];
+    }
+    getEntity(name) {
+        if (this.entities[name]) return this.entities[name];
+        return this.entities[name] = createOpenSearchEntity({
+            table: this.table,
+            entityName: name
+        });
+    }
+}
+const DdbStorageWriter = StorageWriter.createImplementation({
+    implementation: DdbStorageWriterImpl,
+    dependencies: [
+        DynamoDBClient
+    ]
+});
+export { DdbStorageWriter };
+
+//# sourceMappingURL=StorageWriter.js.map

package/storage/StorageWriter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage/StorageWriter.js","sources":["../../src/storage/StorageWriter.ts"],"sourcesContent":["import { StorageWriter as Abstraction } from \"@webiny/api-search-index-tasks/abstractions/StorageWriter.js\";\nimport { createTableWriteBatch, type IEntity, type IPutBatchItem } from \"@webiny/db-dynamodb\";\nimport { createOpenSearchTable, createOpenSearchEntity } from \"@webiny/api-opensearch\";\nimport { DynamoDBClient } from \"@webiny/db-dynamodb/exports/api/db.js\";\n\ninterface IBufferedRecord {\n    entity: IEntity;\n    data: Record<string, any>;\n}\n\nclass DdbStorageWriterImpl implements Abstraction.Interface {\n    private readonly table;\n    private readonly entities: Record<string, IEntity> = {};\n    private buffer: IBufferedRecord[] = [];\n\n    constructor(dynamoDBClient: DynamoDBClient.Interface) {\n        this.table = createOpenSearchTable({\n            documentClient: dynamoDBClient.client\n        });\n    }\n\n    put(record: Abstraction.Record): void {\n        const entity = this.getEntity(record.entity);\n        this.buffer.push({\n            entity,\n            data: record.data\n        });\n    }\n\n    async execute(): Promise<void> {\n        if (this.buffer.length === 0) {\n            return;\n        }\n\n        const batch = createTableWriteBatch({\n            table: this.table.table\n        });\n\n        for (const { entity, data } of this.buffer) {\n            batch.put(entity.entity, data as IPutBatchItem);\n        }\n\n        await batch.execute();\n        this.buffer = [];\n    }\n\n    private getEntity(name: string): IEntity {\n        if (this.entities[name]) {\n            return this.entities[name];\n        }\n\n        return (this.entities[name] = createOpenSearchEntity({\n            table: this.table,\n            entityName: name\n        }));\n    }\n}\n\nexport const DdbStorageWriter = Abstraction.createImplementation({\n    implementation: DdbStorageWriterImpl,\n    dependencies: [DynamoDBClient]\n});\n"],"names":["DdbStorageWriterImpl","dynamoDBClient","createOpenSearchTable","record","entity","batch","createTableWriteBatch","data","name","createOpenSearchEntity","DdbStorageWriter","Abstraction","DynamoDBClient"],"mappings":";;;;AAUA,MAAMA;IAKF,YAAYC,cAAwC,CAAE;aAHrC,QAAQ,GAA4B,CAAC;aAC9C,MAAM,GAAsB,EAAE;QAGlC,IAAI,CAAC,KAAK,GAAGC,sBAAsB;YAC/B,gBAAgBD,eAAe,MAAM;QACzC;IACJ;IAEA,IAAIE,MAA0B,EAAQ;QAClC,MAAMC,SAAS,IAAI,CAAC,SAAS,CAACD,OAAO,MAAM;QAC3C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;YACbC;YACA,MAAMD,OAAO,IAAI;QACrB;IACJ;IAEA,MAAM,UAAyB;QAC3B,IAAI,AAAuB,MAAvB,IAAI,CAAC,MAAM,CAAC,MAAM,EAClB;QAGJ,MAAME,QAAQC,sBAAsB;YAChC,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK;QAC3B;QAEA,KAAK,MAAM,EAAEF,MAAM,EAAEG,IAAI,EAAE,IAAI,IAAI,CAAC,MAAM,CACtCF,MAAM,GAAG,CAACD,OAAO,MAAM,EAAEG;QAG7B,MAAMF,MAAM,OAAO;QACnB,IAAI,CAAC,MAAM,GAAG,EAAE;IACpB;IAEQ,UAAUG,IAAY,EAAW;QACrC,IAAI,IAAI,CAAC,QAAQ,CAACA,KAAK,EACnB,OAAO,IAAI,CAAC,QAAQ,CAACA,KAAK;QAG9B,OAAQ,IAAI,CAAC,QAAQ,CAACA,KAAK,GAAGC,uBAAuB;YACjD,OAAO,IAAI,CAAC,KAAK;YACjB,YAAYD;QAChB;IACJ;AACJ;AAEO,MAAME,mBAAmBC,cAAAA,oBAAgC,CAAC;IAC7D,gBAAgBX;IAChB,cAAc;QAACY;KAAe;AAClC"}