@jupiterone/integration-sdk-runtime 9.10.0 → 9.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/execution/uploader.d.ts +2 -1
- package/dist/src/execution/uploader.js +7 -2
- package/dist/src/execution/uploader.js.map +1 -1
- package/dist/src/synchronization/batchBySize.d.ts +12 -0
- package/dist/src/synchronization/batchBySize.js +80 -0
- package/dist/src/synchronization/batchBySize.js.map +1 -0
- package/dist/src/synchronization/index.d.ts +2 -2
- package/dist/src/synchronization/index.js +18 -5
- package/dist/src/synchronization/index.js.map +1 -1
- package/dist/src/synchronization/shrinkBatchRawData.d.ts +1 -0
- package/dist/src/synchronization/shrinkBatchRawData.js +3 -3
- package/dist/src/synchronization/shrinkBatchRawData.js.map +1 -1
- package/dist/tsconfig.dist.tsbuildinfo +1 -1
- package/package.json +4 -4
|
@@ -19,5 +19,6 @@ export interface CreatePersisterApiStepGraphObjectDataUploaderParams {
|
|
|
19
19
|
uploadConcurrency: number;
|
|
20
20
|
uploadBatchSize?: number;
|
|
21
21
|
uploadRelationshipsBatchSize?: number;
|
|
22
|
+
uploadBatchSizeInBytes?: number;
|
|
22
23
|
}
|
|
23
|
-
export declare function createPersisterApiStepGraphObjectDataUploader({ stepId, synchronizationJobContext, uploadConcurrency, uploadBatchSize, uploadRelationshipsBatchSize, }: CreatePersisterApiStepGraphObjectDataUploaderParams): StepGraphObjectDataUploader;
|
|
24
|
+
export declare function createPersisterApiStepGraphObjectDataUploader({ stepId, synchronizationJobContext, uploadConcurrency, uploadBatchSize, uploadRelationshipsBatchSize, uploadBatchSizeInBytes, }: CreatePersisterApiStepGraphObjectDataUploaderParams): StepGraphObjectDataUploader;
|
|
@@ -8,6 +8,7 @@ const integration_sdk_core_1 = require("@jupiterone/integration-sdk-core");
|
|
|
8
8
|
const dist_1 = __importDefault(require("p-queue/dist"));
|
|
9
9
|
const synchronization_1 = require("../synchronization");
|
|
10
10
|
const crypto_1 = require("crypto");
|
|
11
|
+
const shrinkBatchRawData_1 = require("../synchronization/shrinkBatchRawData");
|
|
11
12
|
function createQueuedStepGraphObjectDataUploader({ stepId, uploadConcurrency: maximumQueueSize, upload, onThrottleEnqueue, }) {
|
|
12
13
|
const queue = new dist_1.default({
|
|
13
14
|
concurrency: maximumQueueSize,
|
|
@@ -90,7 +91,11 @@ function jobContextWithUploaderMetadataLogger({ synchronizationJobContext, uploa
|
|
|
90
91
|
}),
|
|
91
92
|
};
|
|
92
93
|
}
|
|
93
|
-
function createPersisterApiStepGraphObjectDataUploader({ stepId, synchronizationJobContext, uploadConcurrency, uploadBatchSize, uploadRelationshipsBatchSize, }) {
|
|
94
|
+
function createPersisterApiStepGraphObjectDataUploader({ stepId, synchronizationJobContext, uploadConcurrency, uploadBatchSize, uploadRelationshipsBatchSize, uploadBatchSizeInBytes, }) {
|
|
95
|
+
if (uploadBatchSizeInBytes &&
|
|
96
|
+
uploadBatchSizeInBytes > shrinkBatchRawData_1.MAX_BATCH_SIZE_IN_BYTES) {
|
|
97
|
+
uploadBatchSizeInBytes = shrinkBatchRawData_1.MAX_BATCH_SIZE_IN_BYTES;
|
|
98
|
+
}
|
|
94
99
|
return createQueuedStepGraphObjectDataUploader({
|
|
95
100
|
stepId,
|
|
96
101
|
uploadConcurrency,
|
|
@@ -101,7 +106,7 @@ function createPersisterApiStepGraphObjectDataUploader({ stepId, synchronization
|
|
|
101
106
|
stepId: stepId,
|
|
102
107
|
});
|
|
103
108
|
try {
|
|
104
|
-
await (0, synchronization_1.uploadGraphObjectData)(context, graphObjectData, uploadBatchSize, uploadRelationshipsBatchSize);
|
|
109
|
+
await (0, synchronization_1.uploadGraphObjectData)(context, graphObjectData, uploadBatchSize, uploadRelationshipsBatchSize, uploadBatchSizeInBytes);
|
|
105
110
|
}
|
|
106
111
|
catch (err) {
|
|
107
112
|
context.logger.error({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.js","sourceRoot":"","sources":["../../../src/execution/uploader.ts"],"names":[],"mappings":";;;;;;AAAA,2EAAoE;AACpE,wDAAkC;AAElC,wDAG4B;AAC5B,mCAA4C;
|
|
1
|
+
{"version":3,"file":"uploader.js","sourceRoot":"","sources":["../../../src/execution/uploader.ts"],"names":[],"mappings":";;;;;;AAAA,2EAAoE;AACpE,wDAAkC;AAElC,wDAG4B;AAC5B,mCAA4C;AAC5C,8EAAgF;AAmBhF,SAAgB,uCAAuC,CAAC,EACtD,MAAM,EACN,iBAAiB,EAAE,gBAAgB,EACnC,MAAM,EACN,iBAAiB,GAC6B;IAC9C,MAAM,KAAK,GAAG,IAAI,cAAM,CAAC;QACvB,WAAW,EAAE,gBAAgB;KAC9B,CAAC,CAAC;IAEH,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,MAAM,YAAY,GAAY,EAAE,CAAC;IAEjC,OAAO;QACL,MAAM;QACN,KAAK,CAAC,OAAO,CAAC,eAAe;YAC3B,IAAI,SAAS,EAAE;gBACb,qEAAqE;gBACrE,iDAAiD;gBACjD,OAAO;aACR;YAED,gEAAgE;YAChE,qEAAqE;YACrE,yEAAyE;YACzE,uCAAuC;YACvC,EAAE;YACF,0EAA0E;YAC1E,yEAAyE;YACzE,sEAAsE;YACtE,wEAAwE;YACxE,uCAAuC;YACvC,IAAI,KAAK,CAAC,IAAI,IAAI,gBAAgB,EAAE;gBAClC,IAAI,iBAAiB,EAAE;oBACrB,iEAAiE;oBACjE,iBAAiB,EAAE,CAAC;iBACrB;gBAED,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;aACvB;YAED,KAAK;iBACF,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;iBAClC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACb,qEAAqE;gBACrE,oEAAoE;gBACpE,qEAAqE;gBACrE,sEAAsE;gBACtE,qEAAqE;gBACrE,6DAA6D;gBAC7D,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;QACP,CAAC;QAED,KAAK,CAAC,wBAAwB;YAC5B,IAAI;gBACF,wEAAwE;gBACxE,oEAAoE;gBACpE,yEAAyE;gBACzE,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC;aACtB;oBAAS;gBACR,uEAAuE;gBACvE,sEAAsE;gBACtE,uBAAuB;gBACvB,SAAS,GAAG,IAAI,CAAC;aAClB;YAED,IAAI,YAAY,CAAC,MAAM,EAAE;gBACvB,MAAM,IAAI,uCAAgB,CAAC;oBACzB,IAAI,EAAE,cAAc;oBACpB,OAAO,EAAE,gDAAgD,MAAM,mBAAmB,YAAY,CAAC,IAAI,CACjG,GAAG,CACJ,GAAG;oBACJ,kEAAkE;oBAClE,yDAAyD;oBACzD,KAAK,EAAE,YAAY,CAAC,CAAC,CAAC;iBACvB,CAAC,CAAC;aACJ;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAhFD,0FAgFC;AAED;;;GAGG;AACH,SAAS,oCAAoC,CAAC,EAC5C,yBAAyB,EACzB,QAAQ,EACR,MAAM,GAKP;IACC,OAAO;QACL,GAAG,yBAAyB;QAC5B,MAAM,EAAE,yBAAyB,CAAC,MAAM,CAAC,KAAK,CAAC;YAC7C,QAAQ;YACR,MAAM;SACP,CAAC;KACH,CAAC;AACJ,CAAC;AAWD,SAAgB,6CAA6C,CAAC,EAC5D,MAAM,EACN,yBAAyB,EACzB,iBAAiB,EACjB,eAAe,EACf,4BAA4B,EAC5B,sBAAsB,GAC8B;IACpD,IACE,sBAAsB;QACtB,sBAAsB,GAAG,4CAAuB,EAChD;QACA,sBAAsB,GAAG,4CAAuB,CAAC;KAClD;IACD,OAAO,uCAAuC,CAAC;QAC7C,MAAM;QACN,iBAAiB;QACjB,KAAK,CAAC,MAAM,CAAC,eAAe;YAC1B,MAAM,OAAO,GAAG,oCAAoC,CAAC;gBACnD,yBAAyB;gBACzB,QAAQ,EAAE,IAAA,mBAAI,GAAE;gBAChB,MAAM,EAAE,MAAM;aACf,CAAC,CAAC;YAEH,IAAI;gBACF,MAAM,IAAA,uCAAqB,EACzB,OAAO,EACP,eAAe,EACf,eAAe,EACf,4BAA4B,EAC5B,sBAAsB,CACvB,CAAC;aACH;YAAC,OAAO,GAAG,EAAE;gBACZ,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB;oBACE,GAAG;oBACH,iBAAiB;oBACjB,eAAe;oBACf,4BAA4B;iBAC7B,EACD,mCAAmC,CACpC,CAAC;gBACF,MAAM,GAAG,CAAC;aACX;QACH,CAAC;KACF,CAAC,CAAC;AACL,CAAC;AA9CD,sGA8CC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { IntegrationLogger } from '@jupiterone/integration-sdk-core';
|
|
2
|
+
import { UploadDataLookup } from '.';
|
|
3
|
+
/**
|
|
4
|
+
* Converts an array of graphObjects into an array of groups of this objects.
|
|
5
|
+
* @param graphObjects Entities or relationships to group.
|
|
6
|
+
*
|
|
7
|
+
* @param uploadBatchSizeInBytes The maximum size of the group of objects.
|
|
8
|
+
*
|
|
9
|
+
* @param logger
|
|
10
|
+
*/
|
|
11
|
+
export declare function batchGraphObjectsBySizeInBytes<T extends UploadDataLookup, K extends keyof T>(graphObjects: T[K][], uploadBatchSizeInBytes: number, logger: IntegrationLogger): T[K][][];
|
|
12
|
+
export declare function getSizeOfObject(object: any): number;
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.getSizeOfObject = exports.batchGraphObjectsBySizeInBytes = void 0;
|
|
4
|
+
const integration_sdk_core_1 = require("@jupiterone/integration-sdk-core");
|
|
5
|
+
/**
|
|
6
|
+
* Converts an array of graphObjects into an array of groups of this objects.
|
|
7
|
+
* @param graphObjects Entities or relationships to group.
|
|
8
|
+
*
|
|
9
|
+
* @param uploadBatchSizeInBytes The maximum size of the group of objects.
|
|
10
|
+
*
|
|
11
|
+
* @param logger
|
|
12
|
+
*/
|
|
13
|
+
function batchGraphObjectsBySizeInBytes(graphObjects, uploadBatchSizeInBytes, logger) {
|
|
14
|
+
const batches = [];
|
|
15
|
+
let currentBatch = [];
|
|
16
|
+
let currentBatchSize = 0;
|
|
17
|
+
for (const graphObject of graphObjects) {
|
|
18
|
+
const typedGraphObject = graphObject;
|
|
19
|
+
let graphObjectSizeInBytes = 0;
|
|
20
|
+
try {
|
|
21
|
+
graphObjectSizeInBytes = getSizeOfObject(graphObject);
|
|
22
|
+
}
|
|
23
|
+
catch (err) {
|
|
24
|
+
logger.warn({ err, _key: typedGraphObject._key }, 'Failed to calculate size of object in bytes');
|
|
25
|
+
logger.publishWarnEvent({
|
|
26
|
+
name: integration_sdk_core_1.IntegrationWarnEventName.IngestionLimitEncountered,
|
|
27
|
+
description: 'Graph object is larger than what can me measured. Will skip graphObject',
|
|
28
|
+
});
|
|
29
|
+
continue;
|
|
30
|
+
}
|
|
31
|
+
if (graphObjectSizeInBytes > uploadBatchSizeInBytes) {
|
|
32
|
+
const rawDataSizeInBytes = typedGraphObject._rawData
|
|
33
|
+
? getSizeOfObject(typedGraphObject._rawData)
|
|
34
|
+
: 0;
|
|
35
|
+
if (graphObjectSizeInBytes - rawDataSizeInBytes >
|
|
36
|
+
uploadBatchSizeInBytes) {
|
|
37
|
+
logger.warn({
|
|
38
|
+
_key: typedGraphObject._key,
|
|
39
|
+
_type: typedGraphObject.type,
|
|
40
|
+
uploadBatchSizeInBytes,
|
|
41
|
+
graphObjectSizeInBytes,
|
|
42
|
+
rawDataSizeInBytes,
|
|
43
|
+
}, 'Graph object is larger than batch size in bytes; cannot upload graph object.');
|
|
44
|
+
logger.publishWarnEvent({
|
|
45
|
+
name: integration_sdk_core_1.IntegrationWarnEventName.IngestionLimitEncountered,
|
|
46
|
+
description: 'Graph object is larger than batch size in bytes; cannot upload graph object. Will skip graph object.',
|
|
47
|
+
});
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
else {
|
|
51
|
+
logger.info({
|
|
52
|
+
_key: typedGraphObject._key,
|
|
53
|
+
_type: typedGraphObject.type,
|
|
54
|
+
uploadBatchSizeInBytes,
|
|
55
|
+
graphObjectSizeInBytes,
|
|
56
|
+
rawDataSizeInBytes,
|
|
57
|
+
}, 'Graph object is larger than batch size in bytes; removing raw data.');
|
|
58
|
+
graphObjectSizeInBytes -= rawDataSizeInBytes;
|
|
59
|
+
typedGraphObject._rawData = undefined;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
if (graphObjectSizeInBytes + currentBatchSize >= uploadBatchSizeInBytes) {
|
|
63
|
+
batches.push(currentBatch);
|
|
64
|
+
currentBatch = [];
|
|
65
|
+
currentBatchSize = 0;
|
|
66
|
+
}
|
|
67
|
+
currentBatch.push(graphObject);
|
|
68
|
+
currentBatchSize += graphObjectSizeInBytes;
|
|
69
|
+
}
|
|
70
|
+
if (currentBatch.length > 0) {
|
|
71
|
+
batches.push(currentBatch);
|
|
72
|
+
}
|
|
73
|
+
return batches;
|
|
74
|
+
}
|
|
75
|
+
exports.batchGraphObjectsBySizeInBytes = batchGraphObjectsBySizeInBytes;
|
|
76
|
+
function getSizeOfObject(object) {
|
|
77
|
+
return Buffer.byteLength(JSON.stringify(object), 'utf8');
|
|
78
|
+
}
|
|
79
|
+
exports.getSizeOfObject = getSizeOfObject;
|
|
80
|
+
//# sourceMappingURL=batchBySize.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batchBySize.js","sourceRoot":"","sources":["../../../src/synchronization/batchBySize.ts"],"names":[],"mappings":";;;AAAA,2EAK0C;AAG1C;;;;;;;GAOG;AACH,SAAgB,8BAA8B,CAI5C,YAAoB,EACpB,sBAA8B,EAC9B,MAAyB;IAEzB,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,IAAI,YAAY,GAAW,EAAE,CAAC;IAC9B,IAAI,gBAAgB,GAAW,CAAC,CAAC;IACjC,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE;QACtC,MAAM,gBAAgB,GAAG,WAAoC,CAAC;QAE9D,IAAI,sBAAsB,GAAG,CAAC,CAAC;QAC/B,IAAI;YACF,sBAAsB,GAAG,eAAe,CAAC,WAAW,CAAC,CAAC;SACvD;QAAC,OAAO,GAAG,EAAE;YACZ,MAAM,CAAC,IAAI,CACT,EAAE,GAAG,EAAE,IAAI,EAAE,gBAAgB,CAAC,IAAI,EAAE,EACpC,6CAA6C,CAC9C,CAAC;YACF,MAAM,CAAC,gBAAgB,CAAC;gBACtB,IAAI,EAAE,+CAAwB,CAAC,yBAAyB;gBACxD,WAAW,EACT,yEAAyE;aAC5E,CAAC,CAAC;YACH,SAAS;SACV;QAED,IAAI,sBAAsB,GAAG,sBAAsB,EAAE;YACnD,MAAM,kBAAkB,GAAG,gBAAgB,CAAC,QAAQ;gBAClD,CAAC,CAAC,eAAe,CAAC,gBAAgB,CAAC,QAAQ,CAAC;gBAC5C,CAAC,CAAC,CAAC,CAAC;YAEN,IACE,sBAAsB,GAAG,kBAAkB;gBAC3C,sBAAsB,EACtB;gBACA,MAAM,CAAC,IAAI,CACT;oBACE,IAAI,EAAE,gBAAgB,CAAC,IAAI;oBAC3B,KAAK,EAAE,gBAAgB,CAAC,IAAI;oBAC5B,sBAAsB;oBACtB,sBAAsB;oBACtB,kBAAkB;iBACnB,EACD,8EAA8E,CAC/E,CAAC;gBACF,MAAM,CAAC,gBAAgB,CAAC;oBACtB,IAAI,EAAE,+CAAwB,CAAC,yBAAyB;oBACxD,WAAW,EACT,sGAAsG;iBACzG,CAAC,CAAC;gBACH,SAAS;aACV;iBAAM;gBACL,MAAM,CAAC,IAAI,CACT;oBACE,IAAI,EAAE,gBAAgB,CAAC,IAAI;oBAC3B,KAAK,EAAE,gBAAgB,CAAC,IAAI;oBAC5B,sBAAsB;oBACtB,sBAAsB;oBACtB,kBAAkB;iBACnB,EACD,qEAAqE,CACtE,CAAC;gBACF,sBAAsB,IAAI,kBAAkB,CAAC;gBAC7C,gBAAgB,CAAC,QAAQ,GAAG,SAAS,CAAC;aACvC;SACF;QAED,IAAI,sBAAsB,GAAG,gBAAgB,IAAI,sBAAsB,EAAE;YACvE,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC3B,YAAY,GAAG,EAAE,CAAC;YAClB,gBAAgB,GAAG,CAAC,CAAC;SACtB;QACD,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC/B,gBAAgB,IAAI,sBAAsB,CAAC;KAC5C;IAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE;QAC3B,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;KAC5B;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AArFD,wEAqFC;AAED,SAAgB,eAAe,CAAC,MAAW;IACzC,OAAO,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;AAC3D,CAAC;AAFD,0CAEC"}
|
|
@@ -68,7 +68,7 @@ interface FinalizeSynchronizationInput extends SynchronizationJobContext {
|
|
|
68
68
|
* the synchronization of all uploaded entities and relationships.
|
|
69
69
|
*/
|
|
70
70
|
export declare function finalizeSynchronization({ apiClient, job, logger, partialDatasets, }: FinalizeSynchronizationInput): Promise<SynchronizationJob>;
|
|
71
|
-
export declare function uploadGraphObjectData(synchronizationJobContext: SynchronizationJobContext, graphObjectData: FlushedGraphObjectData, uploadBatchSize?: number, uploadRelationshipsBatchSize?: number): Promise<void>;
|
|
71
|
+
export declare function uploadGraphObjectData(synchronizationJobContext: SynchronizationJobContext, graphObjectData: FlushedGraphObjectData, uploadBatchSize?: number, uploadRelationshipsBatchSize?: number, uploadBatchSizeInBytes?: number): Promise<void>;
|
|
72
72
|
/**
|
|
73
73
|
* Uploads data collected by the integration.
|
|
74
74
|
*/
|
|
@@ -108,7 +108,7 @@ type SystemErrorResponseData = {
|
|
|
108
108
|
*/
|
|
109
109
|
export declare function getSystemErrorResponseData(err: any): SystemErrorResponseData | undefined;
|
|
110
110
|
export declare function uploadDataChunk<T extends UploadDataLookup, K extends keyof T>({ logger, apiClient, jobId, type, batch }: UploadDataChunkParams<T, K>): Promise<void>;
|
|
111
|
-
export declare function uploadData<T extends UploadDataLookup, K extends keyof T>({ job, apiClient, logger }: SynchronizationJobContext, type: K, data: T[K][], uploadBatchSize?: number): Promise<void>;
|
|
111
|
+
export declare function uploadData<T extends UploadDataLookup, K extends keyof T>({ job, apiClient, logger }: SynchronizationJobContext, type: K, data: T[K][], uploadBatchSize?: number, uploadBatchSizeInBytes?: number): Promise<void>;
|
|
112
112
|
interface AbortSynchronizationInput extends SynchronizationJobContext {
|
|
113
113
|
reason?: string;
|
|
114
114
|
}
|
|
@@ -17,6 +17,7 @@ const crypto_1 = require("crypto");
|
|
|
17
17
|
const events_1 = require("./events");
|
|
18
18
|
const __1 = require("..");
|
|
19
19
|
const shrinkBatchRawData_1 = require("./shrinkBatchRawData");
|
|
20
|
+
const batchBySize_1 = require("./batchBySize");
|
|
20
21
|
var events_2 = require("./events");
|
|
21
22
|
Object.defineProperty(exports, "createEventPublishingQueue", { enumerable: true, get: function () { return events_2.createEventPublishingQueue; } });
|
|
22
23
|
exports.DEFAULT_UPLOAD_BATCH_SIZE = 250;
|
|
@@ -148,7 +149,7 @@ async function getPartialDatasets() {
|
|
|
148
149
|
const summary = await (0, fileSystem_1.readJsonFromPath)(path_1.default.resolve((0, fileSystem_1.getRootStorageDirectory)(), 'summary.json'));
|
|
149
150
|
return summary.metadata.partialDatasets;
|
|
150
151
|
}
|
|
151
|
-
async function uploadGraphObjectData(synchronizationJobContext, graphObjectData, uploadBatchSize, uploadRelationshipsBatchSize) {
|
|
152
|
+
async function uploadGraphObjectData(synchronizationJobContext, graphObjectData, uploadBatchSize, uploadRelationshipsBatchSize, uploadBatchSizeInBytes) {
|
|
152
153
|
const entityBatchSize = uploadBatchSize;
|
|
153
154
|
const relationshipsBatchSize = uploadRelationshipsBatchSize || uploadBatchSize;
|
|
154
155
|
try {
|
|
@@ -157,7 +158,7 @@ async function uploadGraphObjectData(synchronizationJobContext, graphObjectData,
|
|
|
157
158
|
synchronizationJobContext.logger.debug({
|
|
158
159
|
entities: graphObjectData.entities.length,
|
|
159
160
|
}, 'Preparing batches of entities for upload');
|
|
160
|
-
await uploadData(synchronizationJobContext, 'entities', graphObjectData.entities, entityBatchSize);
|
|
161
|
+
await uploadData(synchronizationJobContext, 'entities', graphObjectData.entities, entityBatchSize, uploadBatchSizeInBytes);
|
|
161
162
|
synchronizationJobContext.logger.debug({
|
|
162
163
|
entities: graphObjectData.entities.length,
|
|
163
164
|
}, 'Successfully uploaded entities');
|
|
@@ -167,7 +168,7 @@ async function uploadGraphObjectData(synchronizationJobContext, graphObjectData,
|
|
|
167
168
|
synchronizationJobContext.logger.debug({
|
|
168
169
|
relationships: graphObjectData.relationships.length,
|
|
169
170
|
}, 'Preparing batches of relationships for upload');
|
|
170
|
-
await uploadData(synchronizationJobContext, 'relationships', graphObjectData.relationships, relationshipsBatchSize);
|
|
171
|
+
await uploadData(synchronizationJobContext, 'relationships', graphObjectData.relationships, relationshipsBatchSize, uploadBatchSizeInBytes);
|
|
171
172
|
synchronizationJobContext.logger.debug({
|
|
172
173
|
relationships: graphObjectData.relationships.length,
|
|
173
174
|
}, 'Successfully uploaded relationships');
|
|
@@ -296,8 +297,20 @@ async function uploadDataChunk({ logger, apiClient, jobId, type, batch }) {
|
|
|
296
297
|
});
|
|
297
298
|
}
|
|
298
299
|
exports.uploadDataChunk = uploadDataChunk;
|
|
299
|
-
async function uploadData({ job, apiClient, logger }, type, data, uploadBatchSize) {
|
|
300
|
-
|
|
300
|
+
async function uploadData({ job, apiClient, logger }, type, data, uploadBatchSize, uploadBatchSizeInBytes) {
|
|
301
|
+
let batches;
|
|
302
|
+
try {
|
|
303
|
+
if (uploadBatchSizeInBytes) {
|
|
304
|
+
batches = (0, batchBySize_1.batchGraphObjectsBySizeInBytes)(data, uploadBatchSizeInBytes, logger);
|
|
305
|
+
}
|
|
306
|
+
else {
|
|
307
|
+
batches = (0, chunk_1.default)(data, uploadBatchSize || exports.DEFAULT_UPLOAD_BATCH_SIZE);
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
catch (error) {
|
|
311
|
+
logger.warn({ error }, 'Batching by size failed');
|
|
312
|
+
batches = (0, chunk_1.default)(data, uploadBatchSize || exports.DEFAULT_UPLOAD_BATCH_SIZE);
|
|
313
|
+
}
|
|
301
314
|
await (0, p_map_1.default)(batches, async (batch) => {
|
|
302
315
|
if (batch.length) {
|
|
303
316
|
await uploadDataChunk({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/synchronization/index.ts"],"names":[],"mappings":";;;;;;AAAA,gDAAwB;AACxB,yDAAiC;AACjC,kDAAyB;AAEzB,2EAM0C;AAM1C,8CAA0E;AAC1E,mCAAkD;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/synchronization/index.ts"],"names":[],"mappings":";;;;;;AAAA,gDAAwB;AACxB,yDAAiC;AACjC,kDAAyB;AAEzB,2EAM0C;AAM1C,8CAA0E;AAC1E,mCAAkD;AAYzC,wGAZA,+BAAuB,OAYA;AAVhC,wCAA2C;AAE3C,+CAA0D;AAC1D,mCAA4C;AAC5C,qCAAsD;AAEtD,0BAA6C;AAC7C,6DAA0D;AAC1D,+CAA+D;AAG/D,mCAAsD;AAA7C,oHAAA,0BAA0B,OAAA;AAEtB,QAAA,yBAAyB,GAAG,GAAG,CAAC;AAE7C,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAE7B,IAAY,cAEX;AAFD,WAAY,cAAc;IACxB,6DAA2C,CAAA;AAC7C,CAAC,EAFW,cAAc,GAAd,sBAAc,KAAd,sBAAc,QAEzB;AA0CD;;GAEG;AACI,KAAK,UAAU,wBAAwB,CAC5C,KAAuB;IAEvB,MAAM,UAAU,GAAG,MAAM,uBAAuB,CAAC,KAAK,CAAC,CAAC;IACxD,MAAM,oBAAoB,GAAG,IAAA,mCAA0B,EAAC,UAAU,CAAC,CAAC;IACpE,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,oBAAoB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IAE9E,IAAI;QACF,MAAM,mBAAmB,CAAC,UAAU,CAAC,CAAC;QACtC,IAAI,KAAK,CAAC,YAAY,EAAE;YACtB,OAAO,MAAM,qBAAqB,CAAC,UAAU,CAAC,CAAC;SAChD;aAAM;YACL,OAAO,MAAM,uBAAuB,CAAC;gBACnC,GAAG,UAAU;gBACb,eAAe,EAAE,MAAM,kBAAkB,EAAE;aAC5C,CAAC,CAAC;SACJ;KACF;IAAC,OAAO,GAAG,EAAE;QACZ,UAAU,CAAC,MAAM,CAAC,KAAK,CACrB,GAAG,EACH,mDAAmD,CACpD,CAAC;QAEF,IAAI;YACF,MAAM,oBAAoB,CAAC,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;SACpE;QAAC,OAAO,UAAU,EAAE;YACnB,UAAU,CAAC,MAAM,CAAC,KAAK,CACrB,UAAU,EACV,oDAAoD,CACrD,CAAC;YACF,MAAM,UAAU,CAAC;SAClB;QAED,MAAM,GAAG,CAAC;KACX;YAAS;QACR,MAAM,oBAAoB,CAAC,MAAM,EAAE,CAAC;KACrC;AACH,CAAC;AArCD,4DAqCC;AAED;;;;;;;;;;GAUG;AACH,SAAS,qBAAqB,CAAC,EAC7B,MAAM,EACN,KAAK,EACL,qBAAqB,EACrB,gBAAgB,GACC;IACjB,OAAO,MAAM,KAAK,KAAK;QACrB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE;QACnB,CAAC,CAAC,EAAE,MAAM,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,CAAC;AAC1D,CAAC;AAUD;;GAEG;AACI,KAAK,UAAU,uBAAuB,CAC3C,KAAuB;IAEvB,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,2BAA2B,EAAE,GACvE,KAAK,CAAC;IAER,MAAM,gBAAgB,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;IAEtD,MAAM,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;IAEjD,IAAI,GAAuB,CAAC;IAC5B,IAAI;QACF,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,IAAI,CACnC,iCAAiC,EACjC,gBAAgB,CACjB,CAAC;QACF,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC;KACzB;IAAC,OAAO,GAAG,EAAE;QACZ,MAAM,CAAC,IAAI,CACT;YACE,GAAG;YACH,eAAe,EAAG,GAAW,EAAE,SAAS;SACzC,EACD,qDAAqD,CACtD,CAAC;QACF,MAAM,IAAA,+BAAuB,EAC3B,GAAG,EACH,qDAAqD,CACtD,CAAC;KACH;IAED,OAAO;QACL,SAAS;QACT,GAAG;QACH,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC;YACnB,GAAG,gBAAgB;YACnB,gBAAgB,EACd,gBAAgB,CAAC,gBAAgB,IAAI,GAAG,CAAC,gBAAgB;YAC3D,oBAAoB,EAAE,GAAG,CAAC,EAAE;SAC7B,CAAC;QACF,eAAe;QACf,2BAA2B;KAC5B,CAAC;AACJ,CAAC;AA3CD,0DA2CC;AAIM,KAAK,UAAU,qBAAqB,CAAC,EAC1C,SAAS,EACT,GAAG,GACwB;IAC3B,IAAI,MAA0B,CAAC;IAE/B,IAAI;QACF,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,GAAG,CAClC,mCAAmC,GAAG,CAAC,EAAE,EAAE,CAC5C,CAAC;QACF,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC;KAC5B;IAAC,OAAO,GAAG,EAAE;QACZ,MAAM,IAAA,+BAAuB,EAC3B,GAAG,EACH,qDAAqD,CACtD,CAAC;KACH;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAnBD,sDAmBC;AAMD;;;GAGG;AACI,KAAK,UAAU,uBAAuB,CAAC,EAC5C,SAAS,EACT,GAAG,EACH,MAAM,EACN,eAAe,GACc;IAC7B,MAAM,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;IAE7C,IAAI,YAAgC,CAAC;IAErC,IAAI;QACF,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,IAAI,CACnC,mCAAmC,GAAG,CAAC,EAAE,WAAW,EACpD;YACE,eAAe;SAChB,CACF,CAAC;QACF,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC;KAClC;IAAC,OAAO,GAAG,EAAE;QACZ,MAAM,CAAC,IAAI,CACT;YACE,GAAG;YACH,eAAe,EAAG,GAAW,EAAE,SAAS;SACzC,EACD,qDAAqD,CACtD,CAAC;QACF,MAAM,IAAA,+BAAuB,EAC3B,GAAG,EACH,sDAAsD,CACvD,CAAC;KACH;IAED,MAAM,CAAC,IAAI,CACT,EAAE,kBAAkB,EAAE,YAAY,EAAE,EACpC,sCAAsC,CACvC,CAAC;IAEF,OAAO,YAAY,CAAC;AACtB,CAAC;AAtCD,0DAsCC;AAED,KAAK,UAAU,kBAAkB;IAC/B,MAAM,OAAO,GAAG,MAAM,IAAA,6BAAgB,EACpC,cAAI,CAAC,OAAO,CAAC,IAAA,oCAAuB,GAAE,EAAE,cAAc,CAAC,CACxD,CAAC;IAEF,OAAO,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC;AAC1C,CAAC;AAEM,KAAK,UAAU,qBAAqB,CACzC,yBAAoD,EACpD,eAAuC,EACvC,eAAwB,EACxB,4BAAqC,EACrC,sBAA+B;IAE/B,MAAM,eAAe,GAAG,eAAe,CAAC;IACxC,MAAM,sBAAsB,GAC1B,4BAA4B,IAAI,eAAe,CAAC;IAElD,IAAI;QACF,IACE,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,QAAQ,CAAC;YACvC,eAAe,CAAC,QAAQ,CAAC,MAAM,IAAI,CAAC,EACpC;YACA,yBAAyB,CAAC,MAAM,CAAC,KAAK,CACpC;gBACE,QAAQ,EAAE,eAAe,CAAC,QAAQ,CAAC,MAAM;aAC1C,EACD,0CAA0C,CAC3C,CAAC;YAEF,MAAM,UAAU,CACd,yBAAyB,EACzB,UAAU,EACV,eAAe,CAAC,QAAQ,EACxB,eAAe,EACf,sBAAsB,CACvB,CAAC;YAEF,yBAAyB,CAAC,MAAM,CAAC,KAAK,CACpC;gBACE,QAAQ,EAAE,eAAe,CAAC,QAAQ,CAAC,MAAM;aAC1C,EACD,gCAAgC,CACjC,CAAC;SACH;QAED,IACE,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,aAAa,CAAC;YAC5C,eAAe,CAAC,aAAa,CAAC,MAAM,IAAI,CAAC,EACzC;YACA,yBAAyB,CAAC,MAAM,CAAC,KAAK,CACpC;gBACE,aAAa,EAAE,eAAe,CAAC,aAAa,CAAC,MAAM;aACpD,EACD,+CAA+C,CAChD,CAAC;YAEF,MAAM,UAAU,CACd,yBAAyB,EACzB,eAAe,EACf,eAAe,CAAC,aAAa,EAC7B,sBAAsB,EACtB,sBAAsB,CACvB,CAAC;YAEF,yBAAyB,CAAC,MAAM,CAAC,KAAK,CACpC;gBACE,aAAa,EAAE,eAAe,CAAC,aAAa,CAAC,MAAM;aACpD,EACD,qCAAqC,CACtC,CAAC;SACH;KACF;IAAC,OAAO,GAAG,EAAE;QACZ,MAAM,IAAA,+BAAuB,EAAC,GAAG,EAAE,gCAAgC,CAAC,CAAC;KACtE;AACH,CAAC;AApED,sDAoEC;AAED;;GAEG;AACI,KAAK,UAAU,mBAAmB,CAAC,OAAkC;IAC1E,OAAO,CAAC,MAAM,CAAC,0BAA0B,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAEvD,KAAK,UAAU,qBAAqB,CAAC,UAAkC;QACrE,MAAM,qBAAqB,CACzB,OAAO,EACP,UAAU,EACV,OAAO,CAAC,eAAe,EACvB,OAAO,CAAC,2BAA2B,CACpC,CAAC;IACJ,CAAC;IAED,MAAM,IAAA,uBAAa,EAAC;QAClB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,UAAU,EAAE,sBAAsB;QAClC,SAAS,EAAE,GAAG,EAAE,CAAC,IAAA,2BAAuB,EAAC,qBAAqB,CAAC;KAChE,CAAC,CAAC;IAEH,OAAO,CAAC,MAAM,CAAC,wBAAwB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;AACvD,CAAC;AAnBD,kDAmBC;AAeD,SAAS,4BAA4B,CAAC,GAAG;IACvC,OAAO,CACL,GAAG,CAAC,IAAI,KAAK,gCAAgC;QAC7C,GAAG,CAAC,QAAQ,EAAE,MAAM,KAAK,GAAG,CAC7B,CAAC;AACJ,CAAC;AAcD;;;;;;;;;;GAUG;AACH,SAAgB,0BAA0B,CACxC,GAAQ;IAER,OAAO,GAAG,CAAC,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC;AACnC,CAAC;AAJD,gEAIC;AAUD,SAAS,0BAA0B,CAAC,EAClC,GAAG,EACH,cAAc,EACd,MAAM,EACN,KAAK,EACL,mBAAmB,GACc;IACjC;;;;;;;;;;OAUG;IACH,MAAM,uBAAuB,GAAG,0BAA0B,CAAC,GAAG,CAAC,CAAC;IAEhE,MAAM,CAAC,IAAI,CACT;QACE,GAAG;QACH,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,UAAU,EAAE,cAAc,CAAC,UAAU;QACrC,uBAAuB;QACvB,iBAAiB,EAAE,cAAc,CAAC,iBAAiB;QACnD,mBAAmB;QACnB,eAAe,EAAG,GAAW,EAAE,SAAS;KACzC,EACD,0BAA0B,CAC3B,CAAC;IAEF,IAAI,4BAA4B,CAAC,GAAG,CAAC,EAAE;QACrC,kEAAkE;QAClE,IAAA,uCAAkB,EAAC,KAAK,EAAE,MAAM,CAAC,CAAC;KACnC;SAAM,IAAI,uBAAuB,EAAE,IAAI,KAAK,0BAA0B,EAAE;QACvE,MAAM,IAAI,uCAAgB,CAAC;YACzB,IAAI,EAAE,oCAAoC;YAC1C,KAAK,EAAE,GAAG;YACV,KAAK,EAAE,IAAI;YACX,OAAO,EACL,iEAAiE;SACpE,CAAC,CAAC;KACJ;AACH,CAAC;AAEM,KAAK,UAAU,eAAe,CAGnC,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAA+B;IACtE,MAAM,mBAAmB,GAAG,IAAA,mBAAI,GAAE,CAAC;IAEnC,MAAM,IAAA,eAAK,EACT,KAAK,EAAE,GAAG,EAAE,EAAE;QACZ,iEAAiE;QACjE,0CAA0C;QAC1C,MAAM,CAAC,KAAK,CACV;YACE,mBAAmB;YACnB,UAAU,EAAE,IAAI;YAChB,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,SAAS,EAAE,KAAK,CAAC,MAAM;SACxB,EACD,mBAAmB,CACpB,CAAC;QAEF,MAAM,SAAS,CAAC,IAAI,CAClB,mCAAmC,KAAK,IAAI,IAAc,EAAE,EAC5D;YACE,CAAC,IAAI,CAAC,EAAE,KAAK;SACd,EACD;YACE,OAAO,EAAE;gBACP,qEAAqE;gBACrE,uBAAuB;gBACvB,CAAC,cAAc,CAAC,aAAa,CAAC,EAAE,mBAAmB;aACpD;SACF,CACF,CAAC;QAEF,MAAM,CAAC,KAAK,CACV;YACE,mBAAmB;YACnB,UAAU,EAAE,IAAI;YAChB,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,SAAS,EAAE,KAAK,CAAC,MAAM;SACxB,EACD,0BAA0B,CAC3B,CAAC;IACJ,CAAC,EACD;QACE,WAAW,EAAE,CAAC;QACd,KAAK,EAAE,GAAG;QACV,MAAM,EAAE,IAAI;QACZ,WAAW,CAAC,GAAG,EAAE,cAAc;YAC7B,IAAI;gBACF,0BAA0B,CAAC;oBACzB,GAAG;oBACH,cAAc;oBACd,MAAM;oBACN,KAAK;oBACL,mBAAmB;iBACpB,CAAC,CAAC;aACJ;YAAC,OAAO,KAAK,EAAE;gBACd,MAAM,CAAC,IAAI,CACT,EAAE,KAAK,EAAE,EACT,2CAA2C,EAC3C,KAAK,CACN,CAAC;gBACF,MAAM,KAAK,CAAC;aACb;QACH,CAAC;KACF,CACF,CAAC;AACJ,CAAC;AApED,0CAoEC;AAEM,KAAK,UAAU,UAAU,CAC9B,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,EAA6B,EACrD,IAAO,EACP,IAAY,EACZ,eAAwB,EACxB,sBAA+B;IAE/B,IAAI,OAAiB,CAAC;IACtB,IAAI;QACF,IAAI,sBAAsB,EAAE;YAC1B,OAAO,GAAG,IAAA,4CAA8B,EACtC,IAAI,EACJ,sBAAsB,EACtB,MAAM,CACP,CAAC;SACH;aAAM;YACL,OAAO,GAAG,IAAA,eAAK,EAAC,IAAI,EAAE,eAAe,IAAI,iCAAyB,CAAC,CAAC;SACrE;KACF;IAAC,OAAO,KAAK,EAAE;QACd,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,yBAAyB,CAAC,CAAC;QAClD,OAAO,GAAG,IAAA,eAAK,EAAC,IAAI,EAAE,eAAe,IAAI,iCAAyB,CAAC,CAAC;KACrE;IACD,MAAM,IAAA,eAAI,EACR,OAAO,EACP,KAAK,EAAE,KAAa,EAAE,EAAE;QACtB,IAAI,KAAK,CAAC,MAAM,EAAE;YAChB,MAAM,eAAe,CAAC;gBACpB,SAAS;gBACT,MAAM;gBACN,KAAK,EAAE,GAAG,CAAC,EAAE;gBACb,IAAI;gBACJ,KAAK;aACN,CAAC,CAAC;SACJ;IACH,CAAC,EACD,EAAE,WAAW,EAAE,kBAAkB,EAAE,CACpC,CAAC;AACJ,CAAC;AArCD,gCAqCC;AAKD;;GAEG;AACI,KAAK,UAAU,oBAAoB,CAAC,EACzC,MAAM,EACN,SAAS,EACT,GAAG,EACH,MAAM,GACoB;IAC1B,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;IAE/C,IAAI,UAA8B,CAAC;IAEnC,IAAI;QACF,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,IAAI,CACnC,mCAAmC,GAAG,CAAC,EAAE,QAAQ,EACjD,EAAE,MAAM,EAAE,CACX,CAAC;QACF,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC;KAChC;IAAC,OAAO,GAAG,EAAE;QACZ,MAAM,IAAA,+BAAuB,EAC3B,GAAG,EACH,mDAAmD,CACpD,CAAC;KACH;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAxBD,oDAwBC"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { UploadDataLookup } from '.';
|
|
2
2
|
import { IntegrationLogger } from '../logger';
|
|
3
|
+
export declare const MAX_BATCH_SIZE_IN_BYTES = 5500000;
|
|
3
4
|
/**
|
|
4
5
|
* Removes data from the rawData of the largest entity until the overall size
|
|
5
6
|
* of the data object is less than maxSize (defaulted to MAX_BATCH_SIZE).
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.shrinkBatchRawData = void 0;
|
|
3
|
+
exports.shrinkBatchRawData = exports.MAX_BATCH_SIZE_IN_BYTES = void 0;
|
|
4
4
|
const integration_sdk_core_1 = require("@jupiterone/integration-sdk-core");
|
|
5
5
|
// Uploads above 6 MiB will fail. This is technically
|
|
6
6
|
// 6144000 bytes, but we need header space. Most web
|
|
@@ -9,7 +9,7 @@ const integration_sdk_core_1 = require("@jupiterone/integration-sdk-core");
|
|
|
9
9
|
// however, we do have to consider the size of whatever aws sdk wraps
|
|
10
10
|
// our payload in. In practice we have seen batches as small as 5800000
|
|
11
11
|
// fail. To be completely safe, we are using 5500000 bytes as default
|
|
12
|
-
|
|
12
|
+
exports.MAX_BATCH_SIZE_IN_BYTES = 5500000;
|
|
13
13
|
// TODO [INT-3707]: uncomment and use when implementing method
|
|
14
14
|
// to shrink single entity's rawData until that entity is < 1MB
|
|
15
15
|
// const MAX_RAW_DATA_SIZE = 1194304;
|
|
@@ -19,7 +19,7 @@ const MAX_BATCH_SIZE = 5500000;
|
|
|
19
19
|
*
|
|
20
20
|
* @param batchData
|
|
21
21
|
*/
|
|
22
|
-
function shrinkBatchRawData(batchData, logger, maxBatchSize =
|
|
22
|
+
function shrinkBatchRawData(batchData, logger, maxBatchSize = exports.MAX_BATCH_SIZE_IN_BYTES) {
|
|
23
23
|
logger.info(`Attempting to shrink rawData`);
|
|
24
24
|
const startTimeInMilliseconds = Date.now();
|
|
25
25
|
let totalBatchSize = Buffer.byteLength(JSON.stringify(batchData));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shrinkBatchRawData.js","sourceRoot":"","sources":["../../../src/synchronization/shrinkBatchRawData.ts"],"names":[],"mappings":";;;AAAA,2EAK0C;AAI1C,sDAAsD;AACtD,qDAAqD;AACrD,sDAAsD;AACtD,2CAA2C;AAC3C,qEAAqE;AACrE,uEAAuE;AACvE,qEAAqE;
|
|
1
|
+
{"version":3,"file":"shrinkBatchRawData.js","sourceRoot":"","sources":["../../../src/synchronization/shrinkBatchRawData.ts"],"names":[],"mappings":";;;AAAA,2EAK0C;AAI1C,sDAAsD;AACtD,qDAAqD;AACrD,sDAAsD;AACtD,2CAA2C;AAC3C,qEAAqE;AACrE,uEAAuE;AACvE,qEAAqE;AACxD,QAAA,uBAAuB,GAAG,OAAS,CAAC;AAEjD,8DAA8D;AAC9D,+DAA+D;AAE/D,qCAAqC;AAErC;;;;;GAKG;AACH,SAAgB,kBAAkB,CAChC,SAAqD,EACrD,MAAyB,EACzB,YAAY,GAAG,+BAAuB;IAEtC,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAC5C,MAAM,uBAAuB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3C,IAAI,cAAc,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;IAClE,MAAM,gBAAgB,GAAG,cAAc,CAAC;IACxC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,MAAM,eAAe,GAAG,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;IAEzD,GAAG;QACD,mCAAmC;QACnC,MAAM,wBAAwB,GAC5B,oCAAoC,CAAC,SAAS,CAAC,CAAC;QAElD,mFAAmF;QACnF,mDAAmD;QACnD,IAAI,wBAAwB,EAAE,QAAQ,EAAE,MAAM,EAAE;YAC9C,6FAA6F;YAC7F,MAAM,mBAAmB,GAAG,sBAAsB,CAChD,wBAAwB,CAAC,QAAQ,CAClC,CAAC;YACF,wCAAwC;YACxC,MAAM,iBAAiB,GAAG,4BAA4B,CACpD,mBAAmB,CAAC,OAAO,CAC5B,CAAC;YAEF,kDAAkD;YAClD,IAAI,iBAAiB,CAAC,IAAI,KAAK,eAAe,EAAE;gBAC9C,MAAM,gBAAgB,GAAG,EAAE,CAAC;gBAC5B,KAAK,MAAM,MAAM,IAAI,SAAS,EAAE;oBAC9B,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,UAAU,CAC/C,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CACvB,CAAC;iBACH;gBACD,MAAM,CAAC,KAAK,CACV;oBACE,wBAAwB,EAAE,wBAAwB,CAChD,uBAAuB,CAAC,SAAS,CAAC,CACnC;oBACD,cAAc,EAAE,cAAc;oBAC9B,gBAAgB;iBACjB,EACD,8HAA8H,CAC/H,CAAC;gBACF,MAAM,CAAC,iBAAiB,CAAC;oBACvB,IAAI,EAAE,gDAAyB,CAAC,0BAA0B;oBAC1D,WAAW,EAAE,qFAAqF,SAAS,CAAC,MAAM,sBAAsB,cAAc,oDAAoD;iBAC3M,CAAC,CAAC;gBACH,MAAM,IAAI,uCAAgB,CAAC;oBACzB,IAAI,EAAE,2BAA2B;oBACjC,KAAK,EAAE,KAAK;oBACZ,OAAO,EACL,yHAAyH;iBAC5H,CAAC,CAAC;aACJ;YAED,uGAAuG;YACvG,mBAAmB,CAAC,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC;YACjE,YAAY,IAAI,CAAC,CAAC;YAClB,cAAc;gBACZ,cAAc,GAAG,iBAAiB,CAAC,IAAI,GAAG,eAAe,CAAC;SAC7D;aAAM;YACL,+CAA+C;YAC/C,MAAM,IAAI,uCAAgB,CAAC;gBACzB,IAAI,EAAE,2BAA2B;gBACjC,KAAK,EAAE,KAAK;gBACZ,OAAO,EACL,kFAAkF;aACrF,CAAC,CAAC;SACJ;KACF,QAAQ,cAAc,GAAG,YAAY,EAAE;IAExC,MAAM,qBAAqB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzC,MAAM,CAAC,IAAI,CACT;QACE,WAAW,EAAE,gBAAgB;QAC7B,SAAS,EAAE,cAAc;QACzB,YAAY;QACZ,SAAS,EAAE,qBAAqB,GAAG,uBAAuB;KAC3D,EACD,wBAAwB,CACzB,CAAC;AACJ,CAAC;AArFD,gDAqFC;AAQD;;;;;;;GAOG;AACH,SAAS,4BAA4B,CAAC,IAAS;IAC7C,MAAM,WAAW,GAAe,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;IACrD,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC;YACvB,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YAC/C,CAAC,CAAC,CAAC,CAAC;QACN,IAAI,MAAM,GAAG,WAAW,CAAC,IAAI,EAAE;YAC7B,WAAW,CAAC,GAAG,GAAG,IAAI,CAAC;YACvB,WAAW,CAAC,IAAI,GAAG,MAAM,CAAC;SAC3B;KACF;IAED,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,oCAAoC,CAC3C,IAAgD;IAEhD,IAAI,sBAAsB,CAAC;IAC3B,IAAI,kBAAkB,GAAG,MAAM,CAAC,gBAAgB,CAAC;IAEjD,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE;QACvB,MAAM,MAAM,GAAG,IAAI,EAAE,QAAQ;YAC3B,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClD,CAAC,CAAC,CAAC,CAAC;QACN,IAAI,MAAM,GAAG,kBAAkB,EAAE;YAC/B,sBAAsB,GAAG,IAAI,CAAC;YAC9B,kBAAkB,GAAG,MAAM,CAAC;SAC7B;KACF;IACD,OAAO,sBAAsB,CAAC;AAChC,CAAC;AAED,SAAS,uBAAuB,CAC9B,IAAgD;IAEhD,IAAI,aAAa,CAAC;IAClB,IAAI,iBAAiB,GAAG,MAAM,CAAC,gBAAgB,CAAC;IAEhD,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClE,IAAI,MAAM,GAAG,iBAAiB,EAAE;YAC9B,aAAa,GAAG,IAAI,CAAC;YACrB,iBAAiB,GAAG,MAAM,CAAC;SAC5B;KACF;IACD,OAAO,aAAa,CAAC;AACvB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,sBAAsB,CAAC,IAAqB;IACnD,IAAI,WAAW,CAAC;IAChB,IAAI,eAAe,GAAG,MAAM,CAAC,gBAAgB,CAAC;IAE9C,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClE,IAAI,MAAM,GAAG,eAAe,EAAE;YAC5B,WAAW,GAAG,IAAI,CAAC;YACnB,eAAe,GAAG,MAAM,CAAC;SAC1B;KACF;IAED,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,wBAAwB,CAAC,IAAY;IAC5C,MAAM,WAAW,GAAG,EAAE,CAAC;IAEvB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;QAC/C,IAAI,KAAK,KAAK,SAAS,EAAE;YACvB,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;SACtB;aAAM;YACL,WAAW,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;SAC7D;KACF;IAED,OAAO,WAAW,CAAC;AACrB,CAAC"}
|