@golemio/core 1.20.1-dev.1696905644 → 1.20.1-dev.1699014653
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/helpers/data-access/table-storage/providers/AzureTableStorageService.d.ts +3 -1
- package/dist/helpers/data-access/table-storage/providers/AzureTableStorageService.js +34 -8
- package/dist/helpers/data-access/table-storage/providers/AzureTableStorageService.js.map +1 -1
- package/dist/helpers/data-access/table-storage/providers/interfaces/IAzureTableStorageConfig.d.ts +1 -0
- package/package.json +1 -1
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
import { ILogger } from "
|
|
1
|
+
import { ILogger } from "@golemio/validator/dist/Logger";
|
|
2
2
|
import { IAzureTableStorageConfig } from "./interfaces/IAzureTableStorageConfig";
|
|
3
3
|
import { ITableStorageService } from "./interfaces/ITableStorageService";
|
|
4
4
|
export declare class AzureTableStorageService implements ITableStorageService {
|
|
5
5
|
private config;
|
|
6
6
|
private log;
|
|
7
|
+
private readonly maxPageSize;
|
|
8
|
+
private readonly pageQueueLimit;
|
|
7
9
|
private tableClientDict;
|
|
8
10
|
constructor(config: IAzureTableStorageConfig, log: ILogger);
|
|
9
11
|
createEntities<T extends object>(tableName: string, entities: T[]): Promise<void>;
|
|
@@ -6,12 +6,17 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
6
6
|
exports.AzureTableStorageService = void 0;
|
|
7
7
|
const _helpers_1 = require("../../..");
|
|
8
8
|
const data_tables_1 = require("@azure/data-tables");
|
|
9
|
-
const crypto_1 = __importDefault(require("crypto"));
|
|
10
9
|
const identity_1 = require("@azure/identity");
|
|
10
|
+
const node_crypto_1 = __importDefault(require("node:crypto"));
|
|
11
|
+
const MAX_ALLOWED_PAGE_SIZE = 1000;
|
|
12
|
+
/** Azure tables may start returning 503 if we go over this number */
|
|
13
|
+
const MAX_ADMISSIBLE_PARALLEL_REQUESTS = 30;
|
|
11
14
|
class AzureTableStorageService {
|
|
12
15
|
constructor(config, log) {
|
|
13
16
|
this.config = config;
|
|
14
17
|
this.log = log;
|
|
18
|
+
this.maxPageSize = config.maxPageSize ? Math.min(config.maxPageSize, MAX_ALLOWED_PAGE_SIZE) : MAX_ALLOWED_PAGE_SIZE;
|
|
19
|
+
this.pageQueueLimit = Math.ceil(2 * ((MAX_ADMISSIBLE_PARALLEL_REQUESTS * config.entityBatchSize) / this.maxPageSize));
|
|
15
20
|
this.tableClientDict = new Map();
|
|
16
21
|
}
|
|
17
22
|
async createEntities(tableName, entities) {
|
|
@@ -20,7 +25,7 @@ class AzureTableStorageService {
|
|
|
20
25
|
}
|
|
21
26
|
const nowDateTime = new _helpers_1.DateTime(new Date(), { timeZone: "Europe/Prague" });
|
|
22
27
|
const fallbackPartitionKey = nowDateTime.format("yyyy-MM-dd");
|
|
23
|
-
const rowKeyPrefix = nowDateTime.format("HH_mm_ss.SSS") + "_" +
|
|
28
|
+
const rowKeyPrefix = nowDateTime.format("HH_mm_ss.SSS") + "_" + node_crypto_1.default.randomBytes(3).toString("hex");
|
|
24
29
|
const tableEntities = entities.map((entity, i) => {
|
|
25
30
|
if (!this.isEntityValid(entity)) {
|
|
26
31
|
entity = {
|
|
@@ -47,16 +52,37 @@ class AzureTableStorageService {
|
|
|
47
52
|
select: ["PartitionKey", "RowKey"],
|
|
48
53
|
},
|
|
49
54
|
})
|
|
50
|
-
.byPage({ maxPageSize: this.
|
|
55
|
+
.byPage({ maxPageSize: this.maxPageSize });
|
|
56
|
+
let pageQueue = [];
|
|
51
57
|
for await (const page of resultsIterator) {
|
|
52
|
-
if (
|
|
58
|
+
if (page.length === 0 && pageQueue.length === 0) {
|
|
53
59
|
break;
|
|
54
|
-
const partitionKeyMap = page.reduce((groups, el) => groups.set(el.partitionKey, [...(groups.get(el.partitionKey) || []), el]), new Map());
|
|
55
|
-
for (const [_, value] of partitionKeyMap) {
|
|
56
|
-
const actions = value.map((entity) => ["delete", entity]);
|
|
57
|
-
await tableClient.submitTransaction(actions);
|
|
58
60
|
}
|
|
61
|
+
pageQueue.push(page);
|
|
62
|
+
if (pageQueue.length < this.pageQueueLimit && page.length === this.maxPageSize) {
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
65
|
+
const partitionKeyMap = pageQueue
|
|
66
|
+
.flat()
|
|
67
|
+
.reduce((groups, el) => groups.set(el.partitionKey, [...(groups.get(el.partitionKey) || []), el]), new Map());
|
|
68
|
+
let promiseQueue = [];
|
|
69
|
+
for (const [_, entities] of partitionKeyMap) {
|
|
70
|
+
for (let i = 0; i < entities.length; i += this.config.entityBatchSize) {
|
|
71
|
+
const entityChunk = entities.slice(i, i + this.config.entityBatchSize);
|
|
72
|
+
const actions = entityChunk.map((entity) => ["delete", entity]);
|
|
73
|
+
promiseQueue.push(tableClient.submitTransaction(actions));
|
|
74
|
+
if (promiseQueue.length >= MAX_ADMISSIBLE_PARALLEL_REQUESTS) {
|
|
75
|
+
await Promise.all(promiseQueue).catch((err) => resultsIterator.throw?.(err));
|
|
76
|
+
promiseQueue = [];
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
if (promiseQueue.length > 0) {
|
|
81
|
+
await Promise.all(promiseQueue).catch((err) => resultsIterator.throw?.(err));
|
|
82
|
+
}
|
|
83
|
+
pageQueue = [];
|
|
59
84
|
}
|
|
85
|
+
await resultsIterator.return?.([]);
|
|
60
86
|
}
|
|
61
87
|
getTableClient(tableName) {
|
|
62
88
|
if (this.tableClientDict.has(tableName)) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AzureTableStorageService.js","sourceRoot":"","sources":["../../../../../src/helpers/data-access/table-storage/providers/AzureTableStorageService.ts"],"names":[],"mappings":";;;;;;AAAA,uCAAoC;
|
|
1
|
+
{"version":3,"file":"AzureTableStorageService.js","sourceRoot":"","sources":["../../../../../src/helpers/data-access/table-storage/providers/AzureTableStorageService.ts"],"names":[],"mappings":";;;;;;AAAA,uCAAoC;AACpC,oDAO4B;AAC5B,8CAAyD;AAEzD,8DAAiC;AAIjC,MAAM,qBAAqB,GAAG,IAAI,CAAC;AACnC,qEAAqE;AACrE,MAAM,gCAAgC,GAAG,EAAE,CAAC;AAE5C,MAAa,wBAAwB;IAKjC,YAAoB,MAAgC,EAAU,GAAY;QAAtD,WAAM,GAAN,MAAM,CAA0B;QAAU,QAAG,GAAH,GAAG,CAAS;QACtE,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,EAAE,qBAAqB,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC;QACpH,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,gCAAgC,GAAG,MAAM,CAAC,eAAe,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;QACtH,IAAI,CAAC,eAAe,GAAG,IAAI,GAAG,EAAE,CAAC;IACrC,CAAC;IAEM,KAAK,CAAC,cAAc,CAAmB,SAAiB,EAAE,QAAa;QAC1E,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;YAClB,OAAO;SACV;QAED,MAAM,WAAW,GAAG,IAAI,mBAAQ,CAAC,IAAI,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,eAAe,EAAE,CAAC,CAAC;QAC5E,MAAM,oBAAoB,GAAG,WAAW,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QAC9D,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,CAAC,cAAc,CAAC,GAAG,GAAG,GAAG,qBAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACtG,MAAM,aAAa,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAC7C,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE;gBAC7B,MAAM,GAAG;oBACL,YAAY,EAAE,oBAAoB;oBAClC,MAAM,EAAE,YAAY,GAAG,GAAG,GAAG,CAAC;oBAC9B,GAAG,MAAM;iBACZ,CAAC;aACL;YAED,OAAO,MAAwB,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;QAEnD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE;YACxE,MAAM,WAAW,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;YAC5E,MAAM,OAAO,GAA+B,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;YAE5F,MAAM,WAAW,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;SAChD;IACL,CAAC;IAEM,KAAK,CAAC,uBAAuB,CAAmB,SAAiB,EAAE,SAAiB;QACvF,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;QACnD,MAAM,eAAe,GAAG,WAAW;aAC9B,YAAY,CAAI;YACb,YAAY,EAAE;gBACV,MAAM,EAAE,yBAAyB,SAAS,GAAG;gBAC7C,MAAM,EAAE,CAAC,cAAc,EAAE,QAAQ,CAAC;aACrC;SACJ,CAAC;aACD,MAAM,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QAE/C,IAAI,SAAS,GAAoC,EAAE,CAAC;QAEpD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,eAAe,EAAE;YACtC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE;gBAC7C,MAAM;aACT;YAED,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAErB,IAAI,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,WAAW,EAAE;gBAC5E,SAAS;aACZ;YAED,MAAM,eAAe,GAA6C,SAAS;iBACtE,IAAI,EAAE;iBACN,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;YAElH,IAAI,YAAY,GAA6C,EAAE,CAAC;YAEhE,KAAK,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,eAAe,EAAE;gBACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE;oBACnE,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;oBACvE,MAAM,OAAO,GAA+B,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,QAAQ,EAAE,MAAwB,CAAC,CAAC,CAAC;oBAC9G,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;oBAE1D,IAAI,YAAY,CAAC,MAAM,IAAI,gCAAgC,EAAE;wBACzD,MAAM,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;wBAC7E,YAAY,GAAG,EAAE,CAAC;qBACrB;iBACJ;aACJ;YAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE;gBACzB,MAAM,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;aAChF;YAED,SAAS,GAAG,EAAE,CAAC;SAClB;QAED,MAAM,eAAe,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IAEO,cAAc,CAAC,SAAiB;QACpC,IAAI,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;YACrC,OAAO,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC;SAC/C;QACD,MAAM,WAAW,GAAG,IAAI,iCAAsB,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QACrH,MAAM,WAAW,GAAG,IAAI,yBAAW,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,OAAO,yBAAyB,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;QACrH,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QACjD,OAAO,WAAW,CAAC;IACvB,CAAC;IAEO,aAAa,CAAmB,MAAS;QAC7C,OAAO,MAAM,CAAC,cAAc,CAAC,cAAc,CAAC,IAAI,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IACpF,CAAC;CACJ;AA3GD,4DA2GC"}
|