@jupiterone/integration-sdk-runtime 10.7.1 → 11.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/execution/uploader.d.ts +1 -3
- package/dist/src/execution/uploader.js +3 -9
- package/dist/src/execution/uploader.js.map +1 -1
- package/dist/src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.d.ts +1 -11
- package/dist/src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.js +5 -5
- package/dist/src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.js.map +1 -1
- package/dist/src/synchronization/index.d.ts +2 -7
- package/dist/src/synchronization/index.js +8 -14
- package/dist/src/synchronization/index.js.map +1 -1
- package/dist/src/synchronization/shrinkBatchRawData.d.ts +1 -2
- package/dist/src/synchronization/shrinkBatchRawData.js +2 -10
- package/dist/src/synchronization/shrinkBatchRawData.js.map +1 -1
- package/dist/tsconfig.dist.tsbuildinfo +1 -1
- package/package.json +4 -4
|
@@ -17,8 +17,6 @@ export interface CreatePersisterApiStepGraphObjectDataUploaderParams {
|
|
|
17
17
|
stepId: string;
|
|
18
18
|
synchronizationJobContext: SynchronizationJobContext;
|
|
19
19
|
uploadConcurrency: number;
|
|
20
|
-
uploadBatchSize?: number;
|
|
21
|
-
uploadRelationshipsBatchSize?: number;
|
|
22
20
|
uploadBatchSizeInBytes?: number;
|
|
23
21
|
}
|
|
24
|
-
export declare function createPersisterApiStepGraphObjectDataUploader({ stepId, synchronizationJobContext, uploadConcurrency,
|
|
22
|
+
export declare function createPersisterApiStepGraphObjectDataUploader({ stepId, synchronizationJobContext, uploadConcurrency, uploadBatchSizeInBytes, }: CreatePersisterApiStepGraphObjectDataUploaderParams): StepGraphObjectDataUploader;
|
|
@@ -8,7 +8,6 @@ const integration_sdk_core_1 = require("@jupiterone/integration-sdk-core");
|
|
|
8
8
|
const dist_1 = __importDefault(require("p-queue/dist"));
|
|
9
9
|
const synchronization_1 = require("../synchronization");
|
|
10
10
|
const crypto_1 = require("crypto");
|
|
11
|
-
const shrinkBatchRawData_1 = require("../synchronization/shrinkBatchRawData");
|
|
12
11
|
function createQueuedStepGraphObjectDataUploader({ stepId, uploadConcurrency: maximumQueueSize, upload, onThrottleEnqueue, }) {
|
|
13
12
|
const queue = new dist_1.default({
|
|
14
13
|
concurrency: maximumQueueSize,
|
|
@@ -91,11 +90,7 @@ function jobContextWithUploaderMetadataLogger({ synchronizationJobContext, uploa
|
|
|
91
90
|
}),
|
|
92
91
|
};
|
|
93
92
|
}
|
|
94
|
-
function createPersisterApiStepGraphObjectDataUploader({ stepId, synchronizationJobContext, uploadConcurrency,
|
|
95
|
-
if (uploadBatchSizeInBytes &&
|
|
96
|
-
uploadBatchSizeInBytes > shrinkBatchRawData_1.MAX_BATCH_SIZE_IN_BYTES) {
|
|
97
|
-
uploadBatchSizeInBytes = shrinkBatchRawData_1.MAX_BATCH_SIZE_IN_BYTES;
|
|
98
|
-
}
|
|
93
|
+
function createPersisterApiStepGraphObjectDataUploader({ stepId, synchronizationJobContext, uploadConcurrency, uploadBatchSizeInBytes = synchronization_1.DEFAULT_UPLOAD_BATCH_SIZE_IN_BYTES, }) {
|
|
99
94
|
return createQueuedStepGraphObjectDataUploader({
|
|
100
95
|
stepId,
|
|
101
96
|
uploadConcurrency,
|
|
@@ -106,14 +101,13 @@ function createPersisterApiStepGraphObjectDataUploader({ stepId, synchronization
|
|
|
106
101
|
stepId: stepId,
|
|
107
102
|
});
|
|
108
103
|
try {
|
|
109
|
-
await (0, synchronization_1.uploadGraphObjectData)(context, graphObjectData,
|
|
104
|
+
await (0, synchronization_1.uploadGraphObjectData)(context, graphObjectData, uploadBatchSizeInBytes);
|
|
110
105
|
}
|
|
111
106
|
catch (err) {
|
|
112
107
|
context.logger.error({
|
|
113
108
|
err,
|
|
114
109
|
uploadConcurrency,
|
|
115
|
-
|
|
116
|
-
uploadRelationshipsBatchSize,
|
|
110
|
+
uploadBatchSizeInBytes,
|
|
117
111
|
}, 'Error uploading graph object data');
|
|
118
112
|
throw err;
|
|
119
113
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.js","sourceRoot":"","sources":["../../../src/execution/uploader.ts"],"names":[],"mappings":";;;;;;AAAA,2EAAoE;AACpE,wDAAkC;AAElC,
|
|
1
|
+
{"version":3,"file":"uploader.js","sourceRoot":"","sources":["../../../src/execution/uploader.ts"],"names":[],"mappings":";;;;;;AAAA,2EAAoE;AACpE,wDAAkC;AAElC,wDAI4B;AAC5B,mCAA4C;AAmB5C,SAAgB,uCAAuC,CAAC,EACtD,MAAM,EACN,iBAAiB,EAAE,gBAAgB,EACnC,MAAM,EACN,iBAAiB,GAC6B;IAC9C,MAAM,KAAK,GAAG,IAAI,cAAM,CAAC;QACvB,WAAW,EAAE,gBAAgB;KAC9B,CAAC,CAAC;IAEH,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,MAAM,YAAY,GAAY,EAAE,CAAC;IAEjC,OAAO;QACL,MAAM;QACN,KAAK,CAAC,OAAO,CAAC,eAAe;YAC3B,IAAI,SAAS,EAAE;gBACb,qEAAqE;gBACrE,iDAAiD;gBACjD,OAAO;aACR;YAED,gEAAgE;YAChE,qEAAqE;YACrE,yEAAyE;YACzE,uCAAuC;YACvC,EAAE;YACF,0EAA0E;YAC1E,yEAAyE;YACzE,sEAAsE;YACtE,wEAAwE;YACxE,uCAAuC;YACvC,IAAI,KAAK,CAAC,IAAI,IAAI,gBAAgB,EAAE;gBAClC,IAAI,iBAAiB,EAAE;oBACrB,iEAAiE;oBACjE,iBAAiB,EAAE,CAAC;iBACrB;gBAED,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;aACvB;YAED,KAAK;iBACF,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;iBAClC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACb,qEAAqE;gBACrE,oEAAoE;gBACpE,qEAAqE;gBACrE,sEAAsE;gBACtE,qEAAqE;gBACrE,6DAA6D;gBAC7D,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;QACP,CAAC;QAED,KAAK,CAAC,wBAAwB;YAC5B,IAAI;gBACF,wEAAwE;gBACxE,oEAAoE;gBACpE,yEAAyE;gBACzE,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC;aACtB;oBAAS;gBACR,uEAAuE;gBACvE,sEAAsE;gBACtE,uBAAuB;gBACvB,SAAS,GAAG,IAAI,CAAC;aAClB;YAED,IAAI,YAAY,CAAC,MAAM,EAAE;gBACvB,MAAM,IAAI,uCAAgB,CAAC;oBACzB,IAAI,EAAE,cAAc;oBACpB,OAAO,EAAE,gDAAgD,MAAM,mBAAmB,YAAY,CAAC,IAAI,CACjG,GAAG,CACJ,GAAG;oBACJ,kEAAkE;oBAClE,yDAAyD;oBACzD,KAAK,EAAE,YAAY,CAAC,CAAC,CAAC;iBACvB,CAAC,CAAC;aACJ;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAhFD,0FAgFC;AAED;;;GAGG;AACH,SAAS,oCAAoC,CAAC,EAC5C,yBAAyB,EACzB,QAAQ,EACR,MAAM,GAKP;IACC,OAAO;QACL,GAAG,yBAAyB;QAC5B,MAAM,EAAE,yBAAyB,CAAC,MAAM,CAAC,KAAK,CAAC;YAC7C,QAAQ;YACR,MAAM;SACP,CAAC;KACH,CAAC;AACJ,CAAC;AASD,SAAgB,6CAA6C,CAAC,EAC5D,MAAM,EACN,yBAAyB,EACzB,iBAAiB,EACjB,sBAAsB,GAAG,oDAAkC,GACP;IACpD,OAAO,uCAAuC,CAAC;QAC7C,MAAM;QACN,iBAAiB;QACjB,KAAK,CAAC,MAAM,CAAC,eAAe;YAC1B,MAAM,OAAO,GAAG,oCAAoC,CAAC;gBACnD,yBAAyB;gBACzB,QAAQ,EAAE,IAAA,mBAAI,GAAE;gBAChB,MAAM,EAAE,MAAM;aACf,CAAC,CAAC;YAEH,IAAI;gBACF,MAAM,IAAA,uCAAqB,EACzB,OAAO,EACP,eAAe,EACf,sBAAsB,CACvB,CAAC;aACH;YAAC,OAAO,GAAG,EAAE;gBACZ,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB;oBACE,GAAG;oBACH,iBAAiB;oBACjB,sBAAsB;iBACvB,EACD,mCAAmC,CACpC,CAAC;gBACF,MAAM,GAAG,CAAC;aACX;QACH,CAAC;KACF,CAAC,CAAC;AACL,CAAC;AAnCD,sGAmCC"}
|
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import { Entity, GraphObjectFilter, GraphObjectIteratee, Relationship, GraphObjectStore, GraphObjectIndexMetadata, GetIndexMetadataForGraphObjectTypeParams, IntegrationStep } from '@jupiterone/integration-sdk-core';
|
|
2
2
|
export declare const DEFAULT_GRAPH_OBJECT_BUFFER_THRESHOLD = 500;
|
|
3
3
|
export declare const DEFAULT_GRAPH_OBJECT_FILE_SIZE = 500;
|
|
4
|
-
export declare const
|
|
5
|
-
export declare const MAX_GRAPH_OBJECT_BUFFER_THRESHOLD_IN_BYTES = 1000000000;
|
|
4
|
+
export declare const MAX_GRAPH_OBJECT_BUFFER_THRESHOLD_IN_BYTES: 1073741824;
|
|
6
5
|
export interface FileSystemGraphObjectStoreParams {
|
|
7
6
|
integrationSteps?: IntegrationStep[];
|
|
8
7
|
/**
|
|
@@ -10,15 +9,6 @@ export interface FileSystemGraphObjectStoreParams {
|
|
|
10
9
|
* default: 5_000_000
|
|
11
10
|
*/
|
|
12
11
|
graphObjectBufferThresholdInBytes?: number;
|
|
13
|
-
/**
|
|
14
|
-
* The maximum number of graph objects that this store can buffer in memory
|
|
15
|
-
* before writing to disk. Machines with more memory should consider bumping
|
|
16
|
-
* this value up.
|
|
17
|
-
*
|
|
18
|
-
* Default: 500
|
|
19
|
-
* @deprecated this argument is no longer used. Please use `graphObjectBufferThresholdInBytes` instead.
|
|
20
|
-
*/
|
|
21
|
-
graphObjectBufferThreshold?: number;
|
|
22
12
|
/**
|
|
23
13
|
* The maximum number of entities/relationships stored in each file.
|
|
24
14
|
*/
|
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.FileSystemGraphObjectStore = exports.MAX_GRAPH_OBJECT_BUFFER_THRESHOLD_IN_BYTES = exports.
|
|
6
|
+
exports.FileSystemGraphObjectStore = exports.MAX_GRAPH_OBJECT_BUFFER_THRESHOLD_IN_BYTES = exports.DEFAULT_GRAPH_OBJECT_FILE_SIZE = exports.DEFAULT_GRAPH_OBJECT_BUFFER_THRESHOLD = void 0;
|
|
7
7
|
const async_sema_1 = require("async-sema");
|
|
8
8
|
const p_map_1 = __importDefault(require("p-map"));
|
|
9
9
|
const flushDataToDisk_1 = require("./flushDataToDisk");
|
|
@@ -12,11 +12,11 @@ const memory_1 = require("../memory");
|
|
|
12
12
|
const fileSystem_1 = require("../../fileSystem");
|
|
13
13
|
const bigMap_1 = require("../../execution/utils/bigMap");
|
|
14
14
|
const lodash_1 = require("lodash");
|
|
15
|
+
const synchronization_1 = require("../../synchronization");
|
|
15
16
|
exports.DEFAULT_GRAPH_OBJECT_BUFFER_THRESHOLD = 500;
|
|
16
17
|
exports.DEFAULT_GRAPH_OBJECT_FILE_SIZE = 500;
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
exports.MAX_GRAPH_OBJECT_BUFFER_THRESHOLD_IN_BYTES = 1000000000;
|
|
18
|
+
// no more than 2^30 bytes (1GB)
|
|
19
|
+
exports.MAX_GRAPH_OBJECT_BUFFER_THRESHOLD_IN_BYTES = 1073741824;
|
|
20
20
|
// it is important that this value is set to 1
|
|
21
21
|
// to ensure that only one operation can be performed at a time.
|
|
22
22
|
const BINARY_SEMAPHORE_CONCURRENCY = 1;
|
|
@@ -60,7 +60,7 @@ class FileSystemGraphObjectStore {
|
|
|
60
60
|
this.prettifyFiles = params?.prettifyFiles || false;
|
|
61
61
|
this.graphObjectBufferThresholdInBytes = (0, lodash_1.min)([
|
|
62
62
|
params?.graphObjectBufferThresholdInBytes ||
|
|
63
|
-
|
|
63
|
+
synchronization_1.DEFAULT_UPLOAD_BATCH_SIZE_IN_BYTES,
|
|
64
64
|
exports.MAX_GRAPH_OBJECT_BUFFER_THRESHOLD_IN_BYTES,
|
|
65
65
|
]);
|
|
66
66
|
if (params?.integrationSteps) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FileSystemGraphObjectStore.js","sourceRoot":"","sources":["../../../../src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.ts"],"names":[],"mappings":";;;;;;AAAA,2CAAkC;AAClC,kDAAyB;AAazB,uDAAoD;AACpD,uCAImB;AACnB,sCAAqD;AAErD,iDAA8D;AAC9D,yDAAsD;AACtD,mCAAoC;
|
|
1
|
+
{"version":3,"file":"FileSystemGraphObjectStore.js","sourceRoot":"","sources":["../../../../src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.ts"],"names":[],"mappings":";;;;;;AAAA,2CAAkC;AAClC,kDAAyB;AAazB,uDAAoD;AACpD,uCAImB;AACnB,sCAAqD;AAErD,iDAA8D;AAC9D,yDAAsD;AACtD,mCAAoC;AACpC,2DAA2E;AAE9D,QAAA,qCAAqC,GAAG,GAAG,CAAC;AAC5C,QAAA,8BAA8B,GAAG,GAAG,CAAC;AAElD,gCAAgC;AACnB,QAAA,0CAA0C,GACrD,UAAsB,CAAC;AAEzB,8CAA8C;AAC9C,gEAAgE;AAChE,MAAM,4BAA4B,GAAG,CAAC,CAAC;AAgCvC;;GAEG;AACH,SAAS,6CAA6C,CACpD,gBAAmC;IAEnC,MAAM,mCAAmC,GAAG,IAAI,GAAG,EAGhD,CAAC;IAEJ,KAAK,MAAM,IAAI,IAAI,gBAAgB,EAAE;QACnC,MAAM,WAAW,GAAgC;YAC/C,QAAQ,EAAE,IAAI,GAAG,EAAE;YACnB,aAAa,EAAE,IAAI,GAAG,EAAE;SACzB,CAAC;QAEF,KAAK,MAAM,cAAc,IAAI,IAAI,CAAC,QAAQ,EAAE;YAC1C,IAAI,cAAc,CAAC,aAAa,EAAE;gBAChC,WAAW,CAAC,QAAQ,CAAC,GAAG,CACtB,cAAc,CAAC,KAAK,EACpB,cAAc,CAAC,aAAa,CAC7B,CAAC;aACH;SACF;QAED,KAAK,MAAM,oBAAoB,IAAI,IAAI,CAAC,aAAa,EAAE;YACrD,IAAI,oBAAoB,CAAC,aAAa,EAAE;gBACtC,WAAW,CAAC,aAAa,CAAC,GAAG,CAC3B,oBAAoB,CAAC,KAAK,EAC1B,oBAAoB,CAAC,aAAa,CACnC,CAAC;aACH;SACF;QAED,mCAAmC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;KAC/D;IAED,OAAO,mCAAmC,CAAC;AAC7C,CAAC;AAeD,MAAM,6CAA6C,GAAG,OAAO,CAAC;AAE9D,MAAa,0BAA0B;IACpB,SAAS,CAAO;IAChB,qBAAqB,GAAG,IAAI,iCAAwB,EAAE,CAAC;IACvD,mBAAmB,CAAS;IAC5B,iCAAiC,CAAS;IAC1C,aAAa,CAAU;IACvB,mCAAmC,CAGlD;IACe,uBAAuB,GAAG,IAAI,eAAM,CAGnD,6CAA6C,CAAC,CAAC;IAEjD,YAAY,MAAyC;QACnD,IAAI,CAAC,SAAS,GAAG,IAAI,iBAAI,CAAC,4BAA4B,CAAC,CAAC;QACxD,IAAI,CAAC,mBAAmB;YACtB,MAAM,EAAE,mBAAmB,IAAI,sCAA8B,CAAC;QAEhE,IAAI,CAAC,aAAa,GAAG,MAAM,EAAE,aAAa,IAAI,KAAK,CAAC;QACpD,IAAI,CAAC,iCAAiC,GAAG,IAAA,YAAG,EAAC;YAC3C,MAAM,EAAE,iCAAiC;gBACvC,oDAAkC;YACpC,kDAA0C;SAC3C,CAAE,CAAC;QACJ,IAAI,MAAM,EAAE,gBAAgB,EAAE;YAC5B,IAAI,CAAC,mCAAmC;gBACtC,6CAA6C,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;SAC1E;IACH,CAAC;IAED,KAAK,CAAC,WAAW,CACf,MAAc,EACd,WAAqB,EACrB,iBAAyD;QAEzD,MAAM,IAAI,CAAC,qBAAqB,CAAC,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAElE,IACE,IAAI,CAAC,qBAAqB,CAAC,yBAAyB,EAAE;YACtD,IAAI,CAAC,iCAAiC,EACtC;YACA,MAAM,IAAI,CAAC,mBAAmB,CAAC,iBAAiB,CAAC,CAAC;SACnD;IACH,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,MAAc,EACd,gBAAgC,EAChC,sBAAyE;QAEzE,MAAM,IAAI,CAAC,qBAAqB,CAAC,gBAAgB,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;QAE5E,IACE,IAAI,CAAC,qBAAqB,CAAC,+BAA+B,EAAE;YAC5D,IAAI,CAAC,iCAAiC,EACtC;YACA,MAAM,IAAI,CAAC,wBAAwB,CAAC,sBAAsB,CAAC,CAAC;SAC7D;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,UAAU,CAAC,IAAwB;QACvC,IAAI,CAAC,IAAI;YAAE,OAAO;QAClB,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACzE,IAAI,cAAc,EAAE;YAClB,OAAO,cAAc,CAAC;SACvB;QAED,MAAM,oBAAoB,GAAG,IAAI,CAAC,uBAAuB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpE,IAAI,CAAC,oBAAoB;YAAE,OAAO;QAElC,MAAM,QAAQ,GAAG,IAAA,uCAA0B,EACzC,oBAAoB,CAAC,aAAa,CACnC,CAAC;QACF,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAA,6BAAmB,EAAoB;YAChE,QAAQ;SACT,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,eAAe,CACnB,MAAyB,EACzB,QAAgC;QAEhC,MAAM,IAAI,CAAC,qBAAqB,CAAC,eAAe,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAEnE,MAAM,IAAA,gCAAsB,EAAC;YAC3B,IAAI,EAAE,MAAM,CAAC,KAAK;YAClB,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,oBAAoB,CACxB,MAAyB,EACzB,QAAgC;QAEhC,MAAM,IAAI,CAAC,qBAAqB,CAAC,oBAAoB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAExE,MAAM,IAAA,sCAA4B,EAAC;YACjC,IAAI,EAAE,MAAM,CAAC,KAAK;YAClB,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CACT,iBAAyD,EACzD,sBAAyE;QAEzE,MAAM,OAAO,CAAC,GAAG,CAAC;YAChB,IAAI,CAAC,mBAAmB,CAAC,iBAAiB,CAAC;YAC3C,IAAI,CAAC,wBAAwB,CAAC,sBAAsB,CAAC;SACtD,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,mBAAmB,CACvB,iBAAyD;QAEzD,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,CAC5B,IAAA,eAAI,EACF,IAAI,CAAC,qBAAqB,CAAC,qBAAqB,EAAE,EAClD,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,EAAE;YAC3B,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;gBACtC,MAAM,aAAa,GAAG,IAAI,CAAC,kCAAkC,CAAC;oBAC5D,MAAM;oBACN,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,yBAAyB,EAAE,UAAU;iBACtC,CAAC,CAAC;gBAEH,IAAI,OAAO,aAAa,KAAK,WAAW,EAAE;oBACxC,OAAO,IAAI,CAAC;iBACb;gBAED,OAAO,aAAa,CAAC,OAAO,KAAK,IAAI,CAAC;YACxC,CAAC,CAAC,CAAC;YAEH,IAAI,SAAS,CAAC,MAAM,EAAE;gBACpB,MAAM,OAAO,CAAC,GAAG,CACf,IAAA,cAAK,EAAC,SAAS,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;oBAC5D,MAAM,uBAAuB,GAAG,MAAM,IAAA,iCAAe,EAAC;wBACpD,oBAAoB,EAAE,MAAM;wBAC5B,cAAc,EAAE,UAAU;wBAC1B,IAAI;wBACJ,MAAM,EAAE,IAAI,CAAC,aAAa;qBAC3B,CAAC,CAAC;oBAEH,KAAK,MAAM,EACT,aAAa,EACb,UAAU,GACX,IAAI,uBAAuB,EAAE;wBAC5B,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE;4BAC7C,IAAI,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE;gCACvC,aAAa;gCACb,KAAK;6BACN,CAAC,CAAC;yBACJ;qBACF;gBACH,CAAC,CAAC,CACH,CAAC;aACH;YAED,IAAI,CAAC,qBAAqB,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAE3D,IAAI,iBAAiB,EAAE;gBACrB,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAC;aACnC;QACH,CAAC,CACF,CACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,wBAAwB,CAC5B,sBAAyE;QAEzE,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,CAC5B,IAAA,eAAI,EACF,IAAI,CAAC,qBAAqB,CAAC,0BAA0B,EAAE,EACvD,KAAK,EAAE,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,EAAE;YAChC,MAAM,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;gBAC3C,MAAM,aAAa,GAAG,IAAI,CAAC,kCAAkC,CAAC;oBAC5D,MAAM;oBACN,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,yBAAyB,EAAE,eAAe;iBAC3C,CAAC,CAAC;gBAEH,IAAI,OAAO,aAAa,KAAK,WAAW,EAAE;oBACxC,OAAO,IAAI,CAAC;iBACb;gBAED,OAAO,aAAa,CAAC,OAAO,KAAK,IAAI,CAAC;YACxC,CAAC,CAAC,CAAC;YAEH,IAAI,SAAS,CAAC,MAAM,EAAE;gBACpB,MAAM,OAAO,CAAC,GAAG,CACf,IAAA,cAAK,EAAC,SAAS,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;oBAC5D,MAAM,IAAA,iCAAe,EAAC;wBACpB,oBAAoB,EAAE,MAAM;wBAC5B,cAAc,EAAE,eAAe;wBAC/B,IAAI;wBACJ,MAAM,EAAE,IAAI,CAAC,aAAa;qBAC3B,CAAC,CAAC;gBACL,CAAC,CAAC,CACH,CAAC;aACH;YAED,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;YAErE,IAAI,sBAAsB,EAAE;gBAC1B,MAAM,sBAAsB,CAAC,aAAa,CAAC,CAAC;aAC7C;QACH,CAAC,CACF,CACF,CAAC;IACJ,CAAC;IAED,kCAAkC,CAAC,EACjC,MAAM,EACN,KAAK,EACL,yBAAyB,GACgB;QAGzC,IAAI,CAAC,IAAI,CAAC,mCAAmC,EAAE;YAC7C,OAAO,SAAS,CAAC;SAClB;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,mCAAmC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACjE,OAAO,GAAG,IAAI,GAAG,CAAC,yBAAyB,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC1D,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACK,KAAK,CAAC,aAAa,CAAI,SAA2B;QACxD,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QAC/B,IAAI;YACF,MAAM,SAAS,EAAE,CAAC;SACnB;gBAAS;YACR,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;SAC1B;IACH,CAAC;CACF;AAxQD,gEAwQC"}
|
|
@@ -6,7 +6,6 @@ import { FlushedGraphObjectData } from '../storage/types';
|
|
|
6
6
|
import { AxiosInstance } from 'axios';
|
|
7
7
|
export { synchronizationApiError };
|
|
8
8
|
export { createEventPublishingQueue } from './events';
|
|
9
|
-
export declare const DEFAULT_UPLOAD_BATCH_SIZE = 250;
|
|
10
9
|
export declare const DEFAULT_UPLOAD_BATCH_SIZE_IN_BYTES = 5000000;
|
|
11
10
|
export declare enum RequestHeaders {
|
|
12
11
|
CorrelationId = "JupiterOne-Correlation-Id"
|
|
@@ -40,8 +39,6 @@ export interface SynchronizeInput {
|
|
|
40
39
|
* When no value is provided, a new job will be created.
|
|
41
40
|
*/
|
|
42
41
|
integrationJobId?: string;
|
|
43
|
-
uploadBatchSize?: number | undefined;
|
|
44
|
-
uploadRelationshipBatchSize?: number | undefined;
|
|
45
42
|
skipFinalize?: boolean;
|
|
46
43
|
}
|
|
47
44
|
/**
|
|
@@ -52,8 +49,6 @@ export interface SynchronizationJobContext {
|
|
|
52
49
|
apiClient: ApiClient;
|
|
53
50
|
job: SynchronizationJob;
|
|
54
51
|
logger: IntegrationLogger;
|
|
55
|
-
uploadBatchSize?: number | undefined;
|
|
56
|
-
uploadRelationshipBatchSize?: number | undefined;
|
|
57
52
|
}
|
|
58
53
|
/**
|
|
59
54
|
* Initializes a synchronization job
|
|
@@ -69,7 +64,7 @@ interface FinalizeSynchronizationInput extends SynchronizationJobContext {
|
|
|
69
64
|
* the synchronization of all uploaded entities and relationships.
|
|
70
65
|
*/
|
|
71
66
|
export declare function finalizeSynchronization({ apiClient, job, logger, partialDatasets, }: FinalizeSynchronizationInput): Promise<SynchronizationJob>;
|
|
72
|
-
export declare function uploadGraphObjectData(synchronizationJobContext: SynchronizationJobContext, graphObjectData: FlushedGraphObjectData,
|
|
67
|
+
export declare function uploadGraphObjectData(synchronizationJobContext: SynchronizationJobContext, graphObjectData: FlushedGraphObjectData, uploadBatchSizeInBytes?: number): Promise<void>;
|
|
73
68
|
/**
|
|
74
69
|
* Uploads data collected by the integration.
|
|
75
70
|
*/
|
|
@@ -109,7 +104,7 @@ type SystemErrorResponseData = {
|
|
|
109
104
|
*/
|
|
110
105
|
export declare function getSystemErrorResponseData(err: any): SystemErrorResponseData | undefined;
|
|
111
106
|
export declare function uploadDataChunk<T extends UploadDataLookup, K extends keyof T>({ logger, apiClient, jobId, type, batch }: UploadDataChunkParams<T, K>): Promise<void>;
|
|
112
|
-
export declare function uploadData<T extends UploadDataLookup, K extends keyof T>({ job, apiClient, logger }: SynchronizationJobContext, type: K, data: T[K][],
|
|
107
|
+
export declare function uploadData<T extends UploadDataLookup, K extends keyof T>({ job, apiClient, logger }: SynchronizationJobContext, type: K, data: T[K][], uploadBatchSizeInBytes?: number): Promise<void>;
|
|
113
108
|
interface AbortSynchronizationInput extends SynchronizationJobContext {
|
|
114
109
|
reason?: string;
|
|
115
110
|
}
|
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.abortSynchronization = exports.uploadData = exports.uploadDataChunk = exports.getSystemErrorResponseData = exports.uploadCollectedData = exports.uploadGraphObjectData = exports.finalizeSynchronization = exports.synchronizationStatus = exports.initiateSynchronization = exports.synchronizeCollectedData = exports.RequestHeaders = exports.DEFAULT_UPLOAD_BATCH_SIZE_IN_BYTES = exports.
|
|
6
|
+
exports.abortSynchronization = exports.uploadData = exports.uploadDataChunk = exports.getSystemErrorResponseData = exports.uploadCollectedData = exports.uploadGraphObjectData = exports.finalizeSynchronization = exports.synchronizationStatus = exports.initiateSynchronization = exports.synchronizeCollectedData = exports.RequestHeaders = exports.DEFAULT_UPLOAD_BATCH_SIZE_IN_BYTES = exports.createEventPublishingQueue = exports.synchronizationApiError = void 0;
|
|
7
7
|
const path_1 = __importDefault(require("path"));
|
|
8
8
|
const p_map_1 = __importDefault(require("p-map"));
|
|
9
9
|
const integration_sdk_core_1 = require("@jupiterone/integration-sdk-core");
|
|
@@ -19,7 +19,6 @@ const shrinkBatchRawData_1 = require("./shrinkBatchRawData");
|
|
|
19
19
|
const batchBySize_1 = require("./batchBySize");
|
|
20
20
|
var events_2 = require("./events");
|
|
21
21
|
Object.defineProperty(exports, "createEventPublishingQueue", { enumerable: true, get: function () { return events_2.createEventPublishingQueue; } });
|
|
22
|
-
exports.DEFAULT_UPLOAD_BATCH_SIZE = 250;
|
|
23
22
|
exports.DEFAULT_UPLOAD_BATCH_SIZE_IN_BYTES = 5000000;
|
|
24
23
|
const UPLOAD_CONCURRENCY = 6;
|
|
25
24
|
var RequestHeaders;
|
|
@@ -81,7 +80,7 @@ function buildJobConfiguration({ source, scope, integrationInstanceId, integrati
|
|
|
81
80
|
* Initializes a synchronization job
|
|
82
81
|
*/
|
|
83
82
|
async function initiateSynchronization(input) {
|
|
84
|
-
const { logger, apiClient
|
|
83
|
+
const { logger, apiClient } = input;
|
|
85
84
|
const jobConfiguration = buildJobConfiguration(input);
|
|
86
85
|
logger.info('Initiating synchronization job...');
|
|
87
86
|
let job;
|
|
@@ -104,8 +103,6 @@ async function initiateSynchronization(input) {
|
|
|
104
103
|
integrationJobId: jobConfiguration.integrationJobId ?? job.integrationJobId,
|
|
105
104
|
synchronizationJobId: job.id,
|
|
106
105
|
}),
|
|
107
|
-
uploadBatchSize,
|
|
108
|
-
uploadRelationshipBatchSize,
|
|
109
106
|
};
|
|
110
107
|
}
|
|
111
108
|
exports.initiateSynchronization = initiateSynchronization;
|
|
@@ -148,16 +145,14 @@ async function getPartialDatasets() {
|
|
|
148
145
|
const summary = await (0, fileSystem_1.readJsonFromPath)(path_1.default.resolve((0, fileSystem_1.getRootStorageDirectory)(), 'summary.json'));
|
|
149
146
|
return summary.metadata.partialDatasets;
|
|
150
147
|
}
|
|
151
|
-
async function uploadGraphObjectData(synchronizationJobContext, graphObjectData,
|
|
152
|
-
const entityBatchSize = uploadBatchSize;
|
|
153
|
-
const relationshipsBatchSize = uploadRelationshipsBatchSize || uploadBatchSize;
|
|
148
|
+
async function uploadGraphObjectData(synchronizationJobContext, graphObjectData, uploadBatchSizeInBytes) {
|
|
154
149
|
try {
|
|
155
150
|
if (Array.isArray(graphObjectData.entities) &&
|
|
156
151
|
graphObjectData.entities.length != 0) {
|
|
157
152
|
synchronizationJobContext.logger.debug({
|
|
158
153
|
entities: graphObjectData.entities.length,
|
|
159
154
|
}, 'Preparing batches of entities for upload');
|
|
160
|
-
await uploadData(synchronizationJobContext, 'entities', graphObjectData.entities,
|
|
155
|
+
await uploadData(synchronizationJobContext, 'entities', graphObjectData.entities, uploadBatchSizeInBytes);
|
|
161
156
|
synchronizationJobContext.logger.debug({
|
|
162
157
|
entities: graphObjectData.entities.length,
|
|
163
158
|
}, 'Successfully uploaded entities');
|
|
@@ -167,7 +162,7 @@ async function uploadGraphObjectData(synchronizationJobContext, graphObjectData,
|
|
|
167
162
|
synchronizationJobContext.logger.debug({
|
|
168
163
|
relationships: graphObjectData.relationships.length,
|
|
169
164
|
}, 'Preparing batches of relationships for upload');
|
|
170
|
-
await uploadData(synchronizationJobContext, 'relationships', graphObjectData.relationships,
|
|
165
|
+
await uploadData(synchronizationJobContext, 'relationships', graphObjectData.relationships, uploadBatchSizeInBytes);
|
|
171
166
|
synchronizationJobContext.logger.debug({
|
|
172
167
|
relationships: graphObjectData.relationships.length,
|
|
173
168
|
}, 'Successfully uploaded relationships');
|
|
@@ -184,7 +179,7 @@ exports.uploadGraphObjectData = uploadGraphObjectData;
|
|
|
184
179
|
async function uploadCollectedData(context) {
|
|
185
180
|
context.logger.synchronizationUploadStart(context.job);
|
|
186
181
|
async function uploadGraphObjectFile(parsedData) {
|
|
187
|
-
await uploadGraphObjectData(context, parsedData
|
|
182
|
+
await uploadGraphObjectData(context, parsedData);
|
|
188
183
|
}
|
|
189
184
|
await (0, metrics_1.timeOperation)({
|
|
190
185
|
logger: context.logger,
|
|
@@ -236,8 +231,7 @@ function handleUploadDataChunkError({ err, attemptContext, logger, batch, upload
|
|
|
236
231
|
'err.$response': err?.$response,
|
|
237
232
|
}, 'Handling upload error...');
|
|
238
233
|
if (isRequestUploadTooLargeError(err)) {
|
|
239
|
-
|
|
240
|
-
(0, shrinkBatchRawData_1.shrinkBatchRawData)(batch, logger);
|
|
234
|
+
(0, shrinkBatchRawData_1.shrinkBatchRawData)(batch, logger, exports.DEFAULT_UPLOAD_BATCH_SIZE_IN_BYTES);
|
|
241
235
|
}
|
|
242
236
|
else if (systemErrorResponseData?.code === 'JOB_NOT_AWAITING_UPLOADS') {
|
|
243
237
|
throw new integration_sdk_core_1.IntegrationError({
|
|
@@ -308,7 +302,7 @@ async function uploadDataChunk({ logger, apiClient, jobId, type, batch }) {
|
|
|
308
302
|
});
|
|
309
303
|
}
|
|
310
304
|
exports.uploadDataChunk = uploadDataChunk;
|
|
311
|
-
async function uploadData({ job, apiClient, logger }, type, data,
|
|
305
|
+
async function uploadData({ job, apiClient, logger }, type, data, uploadBatchSizeInBytes) {
|
|
312
306
|
const batches = (0, batchBySize_1.batchGraphObjectsBySizeInBytes)(data, uploadBatchSizeInBytes || exports.DEFAULT_UPLOAD_BATCH_SIZE_IN_BYTES, logger);
|
|
313
307
|
await (0, p_map_1.default)(batches, async (batch) => {
|
|
314
308
|
if (batch.length) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/synchronization/index.ts"],"names":[],"mappings":";;;;;;AAAA,gDAAwB;AACxB,kDAAyB;AAEzB,2EAM0C;AAM1C,8CAA0E;AAC1E,mCAAkD;AAYzC,wGAZA,+BAAuB,OAYA;AAVhC,wCAA2C;AAE3C,+CAA0D;AAC1D,mCAA4C;AAC5C,qCAAsD;AAEtD,0BAA6C;AAC7C,6DAA0D;AAC1D,+CAAgF;AAGhF,mCAAsD;AAA7C,oHAAA,0BAA0B,OAAA;AAEtB,QAAA,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/synchronization/index.ts"],"names":[],"mappings":";;;;;;AAAA,gDAAwB;AACxB,kDAAyB;AAEzB,2EAM0C;AAM1C,8CAA0E;AAC1E,mCAAkD;AAYzC,wGAZA,+BAAuB,OAYA;AAVhC,wCAA2C;AAE3C,+CAA0D;AAC1D,mCAA4C;AAC5C,qCAAsD;AAEtD,0BAA6C;AAC7C,6DAA0D;AAC1D,+CAAgF;AAGhF,mCAAsD;AAA7C,oHAAA,0BAA0B,OAAA;AAEtB,QAAA,kCAAkC,GAAG,OAAS,CAAC;AAE5D,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAE7B,IAAY,cAEX;AAFD,WAAY,cAAc;IACxB,6DAA2C,CAAA;AAC7C,CAAC,EAFW,cAAc,8BAAd,cAAc,QAEzB;AAuCD;;GAEG;AACI,KAAK,UAAU,wBAAwB,CAC5C,KAAuB;IAEvB,MAAM,UAAU,GAAG,MAAM,uBAAuB,CAAC,KAAK,CAAC,CAAC;IACxD,MAAM,oBAAoB,GAAG,IAAA,mCAA0B,EAAC,UAAU,CAAC,CAAC;IACpE,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,oBAAoB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IAE9E,IAAI;QACF,MAAM,mBAAmB,CAAC,UAAU,CAAC,CAAC;QACtC,IAAI,KAAK,CAAC,YAAY,EAAE;YACtB,OAAO,MAAM,qBAAqB,CAAC,UAAU,CAAC,CAAC;SAChD;aAAM;YACL,OAAO,MAAM,uBAAuB,CAAC;gBACnC,GAAG,UAAU;gBACb,eAAe,EAAE,MAAM,kBAAkB,EAAE;aAC5C,CAAC,CAAC;SACJ;KACF;IAAC,OAAO,GAAG,EAAE;QACZ,UAAU,CAAC,MAAM,CAAC,KAAK,CACrB,GAAG,EACH,mDAAmD,CACpD,CAAC;QAEF,IAAI;YACF,MAAM,oBAAoB,CAAC,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;SACpE;QAAC,OAAO,UAAU,EAAE;YACnB,UAAU,CAAC,MAAM,CAAC,KAAK,CACrB,UAAU,EACV,oDAAoD,CACrD,CAAC;YACF,MAAM,UAAU,CAAC;SAClB;QAED,MAAM,GAAG,CAAC;KACX;YAAS;QACR,MAAM,oBAAoB,CAAC,MAAM,EAAE,CAAC;KACrC;AACH,CAAC;AArCD,4DAqCC;AAED;;;;;;;;;;GAUG;AACH,SAAS,qBAAqB,CAAC,EAC7B,MAAM,EACN,KAAK,EACL,qBAAqB,EACrB,gBAAgB,GACC;IACjB,OAAO,MAAM,KAAK,KAAK;QACrB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE;QACnB,CAAC,CAAC,EAAE,MAAM,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,CAAC;AAC1D,CAAC;AAQD;;GAEG;AACI,KAAK,UAAU,uBAAuB,CAC3C,KAAuB;IAEvB,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,KAAK,CAAC;IAEpC,MAAM,gBAAgB,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;IAEtD,MAAM,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;IAEjD,IAAI,GAAuB,CAAC;IAC5B,IAAI;QACF,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,IAAI,CACnC,iCAAiC,EACjC,gBAAgB,CACjB,CAAC;QACF,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC;KACzB;IAAC,OAAO,GAAG,EAAE;QACZ,MAAM,CAAC,IAAI,CACT;YACE,GAAG;YACH,eAAe,EAAG,GAAW,EAAE,SAAS;SACzC,EACD,qDAAqD,CACtD,CAAC;QACF,MAAM,IAAA,+BAAuB,EAC3B,GAAG,EACH,qDAAqD,CACtD,CAAC;KACH;IAED,OAAO;QACL,SAAS;QACT,GAAG;QACH,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC;YACnB,GAAG,gBAAgB;YACnB,gBAAgB,EACd,gBAAgB,CAAC,gBAAgB,IAAI,GAAG,CAAC,gBAAgB;YAC3D,oBAAoB,EAAE,GAAG,CAAC,EAAE;SAC7B,CAAC;KACH,CAAC;AACJ,CAAC;AAxCD,0DAwCC;AAIM,KAAK,UAAU,qBAAqB,CAAC,EAC1C,SAAS,EACT,GAAG,GACwB;IAC3B,IAAI,MAA0B,CAAC;IAE/B,IAAI;QACF,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,GAAG,CAClC,mCAAmC,GAAG,CAAC,EAAE,EAAE,CAC5C,CAAC;QACF,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC;KAC5B;IAAC,OAAO,GAAG,EAAE;QACZ,MAAM,IAAA,+BAAuB,EAC3B,GAAG,EACH,qDAAqD,CACtD,CAAC;KACH;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAnBD,sDAmBC;AAMD;;;GAGG;AACI,KAAK,UAAU,uBAAuB,CAAC,EAC5C,SAAS,EACT,GAAG,EACH,MAAM,EACN,eAAe,GACc;IAC7B,MAAM,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;IAE7C,IAAI,YAAgC,CAAC;IAErC,IAAI;QACF,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,IAAI,CACnC,mCAAmC,GAAG,CAAC,EAAE,WAAW,EACpD;YACE,eAAe;SAChB,CACF,CAAC;QACF,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC;KAClC;IAAC,OAAO,GAAG,EAAE;QACZ,MAAM,CAAC,IAAI,CACT;YACE,GAAG;YACH,eAAe,EAAG,GAAW,EAAE,SAAS;SACzC,EACD,qDAAqD,CACtD,CAAC;QACF,MAAM,IAAA,+BAAuB,EAC3B,GAAG,EACH,sDAAsD,CACvD,CAAC;KACH;IAED,OAAO,YAAY,CAAC;AACtB,CAAC;AAjCD,0DAiCC;AAED,KAAK,UAAU,kBAAkB;IAC/B,MAAM,OAAO,GAAG,MAAM,IAAA,6BAAgB,EACpC,cAAI,CAAC,OAAO,CAAC,IAAA,oCAAuB,GAAE,EAAE,cAAc,CAAC,CACxD,CAAC;IAEF,OAAO,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC;AAC1C,CAAC;AAEM,KAAK,UAAU,qBAAqB,CACzC,yBAAoD,EACpD,eAAuC,EACvC,sBAA+B;IAE/B,IAAI;QACF,IACE,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,QAAQ,CAAC;YACvC,eAAe,CAAC,QAAQ,CAAC,MAAM,IAAI,CAAC,EACpC;YACA,yBAAyB,CAAC,MAAM,CAAC,KAAK,CACpC;gBACE,QAAQ,EAAE,eAAe,CAAC,QAAQ,CAAC,MAAM;aAC1C,EACD,0CAA0C,CAC3C,CAAC;YAEF,MAAM,UAAU,CACd,yBAAyB,EACzB,UAAU,EACV,eAAe,CAAC,QAAQ,EACxB,sBAAsB,CACvB,CAAC;YAEF,yBAAyB,CAAC,MAAM,CAAC,KAAK,CACpC;gBACE,QAAQ,EAAE,eAAe,CAAC,QAAQ,CAAC,MAAM;aAC1C,EACD,gCAAgC,CACjC,CAAC;SACH;QAED,IACE,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,aAAa,CAAC;YAC5C,eAAe,CAAC,aAAa,CAAC,MAAM,IAAI,CAAC,EACzC;YACA,yBAAyB,CAAC,MAAM,CAAC,KAAK,CACpC;gBACE,aAAa,EAAE,eAAe,CAAC,aAAa,CAAC,MAAM;aACpD,EACD,+CAA+C,CAChD,CAAC;YAEF,MAAM,UAAU,CACd,yBAAyB,EACzB,eAAe,EACf,eAAe,CAAC,aAAa,EAC7B,sBAAsB,CACvB,CAAC;YAEF,yBAAyB,CAAC,MAAM,CAAC,KAAK,CACpC;gBACE,aAAa,EAAE,eAAe,CAAC,aAAa,CAAC,MAAM;aACpD,EACD,qCAAqC,CACtC,CAAC;SACH;KACF;IAAC,OAAO,GAAG,EAAE;QACZ,MAAM,IAAA,+BAAuB,EAAC,GAAG,EAAE,gCAAgC,CAAC,CAAC;KACtE;AACH,CAAC;AA5DD,sDA4DC;AAED;;GAEG;AACI,KAAK,UAAU,mBAAmB,CAAC,OAAkC;IAC1E,OAAO,CAAC,MAAM,CAAC,0BAA0B,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAEvD,KAAK,UAAU,qBAAqB,CAAC,UAAkC;QACrE,MAAM,qBAAqB,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;IACnD,CAAC;IAED,MAAM,IAAA,uBAAa,EAAC;QAClB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,UAAU,EAAE,sBAAsB;QAClC,SAAS,EAAE,GAAG,EAAE,CAAC,IAAA,2BAAuB,EAAC,qBAAqB,CAAC;KAChE,CAAC,CAAC;IAEH,OAAO,CAAC,MAAM,CAAC,wBAAwB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;AACvD,CAAC;AAdD,kDAcC;AAeD,SAAS,4BAA4B,CAAC,GAAG;IACvC,OAAO,CACL,GAAG,CAAC,IAAI,KAAK,gCAAgC;QAC7C,GAAG,CAAC,QAAQ,EAAE,MAAM,KAAK,GAAG,CAC7B,CAAC;AACJ,CAAC;AAcD;;;;;;;;;;GAUG;AACH,SAAgB,0BAA0B,CACxC,GAAQ;IAER,OAAO,GAAG,CAAC,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC;AACnC,CAAC;AAJD,gEAIC;AAUD,SAAS,0BAA0B,CAAC,EAClC,GAAG,EACH,cAAc,EACd,MAAM,EACN,KAAK,EACL,mBAAmB,GACc;IACjC;;;;;;;;;;OAUG;IACH,MAAM,uBAAuB,GAAG,0BAA0B,CAAC,GAAG,CAAC,CAAC;IAEhE,MAAM,CAAC,IAAI,CACT;QACE,GAAG;QACH,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,UAAU,EAAE,cAAc,CAAC,UAAU;QACrC,uBAAuB;QACvB,iBAAiB,EAAE,cAAc,CAAC,iBAAiB;QACnD,mBAAmB;QACnB,eAAe,EAAG,GAAW,EAAE,SAAS;KACzC,EACD,0BAA0B,CAC3B,CAAC;IAEF,IAAI,4BAA4B,CAAC,GAAG,CAAC,EAAE;QACrC,IAAA,uCAAkB,EAAC,KAAK,EAAE,MAAM,EAAE,0CAAkC,CAAC,CAAC;KACvE;SAAM,IAAI,uBAAuB,EAAE,IAAI,KAAK,0BAA0B,EAAE;QACvE,MAAM,IAAI,uCAAgB,CAAC;YACzB,IAAI,EAAE,oCAAoC;YAC1C,KAAK,EAAE,GAAG;YACV,KAAK,EAAE,IAAI;YACX,OAAO,EACL,iEAAiE;SACpE,CAAC,CAAC;KACJ;AACH,CAAC;AAEM,KAAK,UAAU,eAAe,CAGnC,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAA+B;IACtE,MAAM,mBAAmB,GAAG,IAAA,mBAAI,GAAE,CAAC;IAEnC,MAAM,IAAA,eAAK,EACT,KAAK,EAAE,GAAG,EAAE,EAAE;QACZ,iEAAiE;QACjE,0CAA0C;QAC1C,MAAM,CAAC,KAAK,CACV;YACE,mBAAmB;YACnB,UAAU,EAAE,IAAI;YAChB,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,SAAS,EAAE,KAAK,CAAC,MAAM;SACxB,EACD,mBAAmB,CACpB,CAAC;QACF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,SAAS,CAAC,IAAI,CAClB,mCAAmC,KAAK,IAAI,IAAc,EAAE,EAC5D;YACE,CAAC,IAAI,CAAC,EAAE,KAAK;SACd,EACD;YACE,OAAO,EAAE;gBACP,qEAAqE;gBACrE,uBAAuB;gBACvB,CAAC,cAAc,CAAC,aAAa,CAAC,EAAE,mBAAmB;aACpD;SACF,CACF,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QACxC,IAAI,QAAQ,IAAI,KAAM,EAAE;YACtB,MAAM,CAAC,IAAI,CACT;gBACE,mBAAmB;gBACnB,UAAU,EAAE,IAAI;gBAChB,UAAU,EAAE,GAAG,CAAC,UAAU;gBAC1B,SAAS,EAAE,KAAK,CAAC,MAAM;gBACvB,gBAAgB,EAAE,IAAA,6BAAe,EAAC,KAAK,CAAC;gBACxC,cAAc,EAAE,QAAQ;aACzB,EACD,8BAA8B,CAC/B,CAAC;SACH;QACD,MAAM,CAAC,KAAK,CACV;YACE,mBAAmB;YACnB,UAAU,EAAE,IAAI;YAChB,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,SAAS,EAAE,KAAK,CAAC,MAAM;SACxB,EACD,0BAA0B,CAC3B,CAAC;IACJ,CAAC,EACD;QACE,WAAW,EAAE,CAAC;QACd,KAAK,EAAE,GAAG;QACV,MAAM,EAAE,IAAI;QACZ,WAAW,CAAC,GAAG,EAAE,cAAc;YAC7B,IAAI;gBACF,0BAA0B,CAAC;oBACzB,GAAG;oBACH,cAAc;oBACd,MAAM;oBACN,KAAK;oBACL,mBAAmB;iBACpB,CAAC,CAAC;aACJ;YAAC,OAAO,KAAK,EAAE;gBACd,MAAM,CAAC,IAAI,CACT,EAAE,KAAK,EAAE,EACT,2CAA2C,EAC3C,KAAK,CACN,CAAC;gBACF,MAAM,KAAK,CAAC;aACb;QACH,CAAC;KACF,CACF,CAAC;AACJ,CAAC;AAjFD,0CAiFC;AAEM,KAAK,UAAU,UAAU,CAC9B,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,EAA6B,EACrD,IAAO,EACP,IAAY,EACZ,sBAA+B;IAE/B,MAAM,OAAO,GAAa,IAAA,4CAA8B,EACtD,IAAI,EACJ,sBAAsB,IAAI,0CAAkC,EAC5D,MAAM,CACP,CAAC;IAEF,MAAM,IAAA,eAAI,EACR,OAAO,EACP,KAAK,EAAE,KAAa,EAAE,EAAE;QACtB,IAAI,KAAK,CAAC,MAAM,EAAE;YAChB,MAAM,eAAe,CAAC;gBACpB,SAAS;gBACT,MAAM;gBACN,KAAK,EAAE,GAAG,CAAC,EAAE;gBACb,IAAI;gBACJ,KAAK;aACN,CAAC,CAAC;SACJ;IACH,CAAC,EACD,EAAE,WAAW,EAAE,kBAAkB,EAAE,CACpC,CAAC;AACJ,CAAC;AA3BD,gCA2BC;AAKD;;GAEG;AACI,KAAK,UAAU,oBAAoB,CAAC,EACzC,MAAM,EACN,SAAS,EACT,GAAG,EACH,MAAM,GACoB;IAC1B,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;IAE/C,IAAI,UAA8B,CAAC;IAEnC,IAAI;QACF,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,IAAI,CACnC,mCAAmC,GAAG,CAAC,EAAE,QAAQ,EACjD,EAAE,MAAM,EAAE,CACX,CAAC;QACF,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC;KAChC;IAAC,OAAO,GAAG,EAAE;QACZ,MAAM,IAAA,+BAAuB,EAC3B,GAAG,EACH,mDAAmD,CACpD,CAAC;KACH;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAxBD,oDAwBC"}
|
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
import { UploadDataLookup } from '.';
|
|
2
2
|
import { IntegrationLogger } from '../logger';
|
|
3
|
-
export declare const MAX_BATCH_SIZE_IN_BYTES = 5500000;
|
|
4
3
|
/**
|
|
5
4
|
* Removes data from the rawData of the largest entity until the overall size
|
|
6
5
|
* of the data object is less than maxSize (defaulted to MAX_BATCH_SIZE).
|
|
7
6
|
*
|
|
8
7
|
* @param batchData
|
|
9
8
|
*/
|
|
10
|
-
export declare function shrinkBatchRawData(batchData: UploadDataLookup[keyof UploadDataLookup][], logger: IntegrationLogger, maxBatchSize
|
|
9
|
+
export declare function shrinkBatchRawData(batchData: UploadDataLookup[keyof UploadDataLookup][], logger: IntegrationLogger, maxBatchSize: number): void;
|
|
@@ -1,15 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.shrinkBatchRawData =
|
|
3
|
+
exports.shrinkBatchRawData = void 0;
|
|
4
4
|
const integration_sdk_core_1 = require("@jupiterone/integration-sdk-core");
|
|
5
|
-
// Uploads above 6 MiB will fail. This is technically
|
|
6
|
-
// 6144000 bytes, but we need header space. Most web
|
|
7
|
-
// servers will only allow 8KB or 16KB as a max header
|
|
8
|
-
// size, so 6144000 - 16384 = 6127616 bytes
|
|
9
|
-
// however, we do have to consider the size of whatever aws sdk wraps
|
|
10
|
-
// our payload in. In practice we have seen batches as small as 5800000
|
|
11
|
-
// fail. To be completely safe, we are using 5500000 bytes as default
|
|
12
|
-
exports.MAX_BATCH_SIZE_IN_BYTES = 5500000;
|
|
13
5
|
// TODO [INT-3707]: uncomment and use when implementing method
|
|
14
6
|
// to shrink single entity's rawData until that entity is < 1MB
|
|
15
7
|
// const MAX_RAW_DATA_SIZE = 1194304;
|
|
@@ -19,7 +11,7 @@ exports.MAX_BATCH_SIZE_IN_BYTES = 5500000;
|
|
|
19
11
|
*
|
|
20
12
|
* @param batchData
|
|
21
13
|
*/
|
|
22
|
-
function shrinkBatchRawData(batchData, logger, maxBatchSize
|
|
14
|
+
function shrinkBatchRawData(batchData, logger, maxBatchSize) {
|
|
23
15
|
logger.info(`Attempting to shrink rawData`);
|
|
24
16
|
const startTimeInMilliseconds = Date.now();
|
|
25
17
|
let totalBatchSize = Buffer.byteLength(JSON.stringify(batchData));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shrinkBatchRawData.js","sourceRoot":"","sources":["../../../src/synchronization/shrinkBatchRawData.ts"],"names":[],"mappings":";;;AAAA,2EAK0C;AAI1C,
|
|
1
|
+
{"version":3,"file":"shrinkBatchRawData.js","sourceRoot":"","sources":["../../../src/synchronization/shrinkBatchRawData.ts"],"names":[],"mappings":";;;AAAA,2EAK0C;AAI1C,8DAA8D;AAC9D,+DAA+D;AAE/D,qCAAqC;AAErC;;;;;GAKG;AACH,SAAgB,kBAAkB,CAChC,SAAqD,EACrD,MAAyB,EACzB,YAAoB;IAEpB,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAC5C,MAAM,uBAAuB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3C,IAAI,cAAc,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;IAClE,MAAM,gBAAgB,GAAG,cAAc,CAAC;IACxC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,MAAM,eAAe,GAAG,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;IAEzD,GAAG;QACD,mCAAmC;QACnC,MAAM,wBAAwB,GAC5B,oCAAoC,CAAC,SAAS,CAAC,CAAC;QAElD,mFAAmF;QACnF,mDAAmD;QACnD,IAAI,wBAAwB,EAAE,QAAQ,EAAE,MAAM,EAAE;YAC9C,6FAA6F;YAC7F,MAAM,mBAAmB,GAAG,sBAAsB,CAChD,wBAAwB,CAAC,QAAQ,CAClC,CAAC;YACF,wCAAwC;YACxC,MAAM,iBAAiB,GAAG,4BAA4B,CACpD,mBAAmB,CAAC,OAAO,CAC5B,CAAC;YAEF,kDAAkD;YAClD,IAAI,iBAAiB,CAAC,IAAI,KAAK,eAAe,EAAE;gBAC9C,MAAM,gBAAgB,GAAG,EAAE,CAAC;gBAC5B,KAAK,MAAM,MAAM,IAAI,SAAS,EAAE;oBAC9B,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,UAAU,CAC/C,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CACvB,CAAC;iBACH;gBACD,MAAM,CAAC,KAAK,CACV;oBACE,wBAAwB,EAAE,wBAAwB,CAChD,uBAAuB,CAAC,SAAS,CAAC,CACnC;oBACD,cAAc,EAAE,cAAc;oBAC9B,gBAAgB;iBACjB,EACD,8HAA8H,CAC/H,CAAC;gBACF,MAAM,CAAC,iBAAiB,CAAC;oBACvB,IAAI,EAAE,gDAAyB,CAAC,0BAA0B;oBAC1D,WAAW,EAAE,qFAAqF,SAAS,CAAC,MAAM,sBAAsB,cAAc,oDAAoD;iBAC3M,CAAC,CAAC;gBACH,MAAM,IAAI,uCAAgB,CAAC;oBACzB,IAAI,EAAE,2BAA2B;oBACjC,KAAK,EAAE,KAAK;oBACZ,OAAO,EACL,yHAAyH;iBAC5H,CAAC,CAAC;aACJ;YAED,uGAAuG;YACvG,mBAAmB,CAAC,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC;YACjE,YAAY,IAAI,CAAC,CAAC;YAClB,cAAc;gBACZ,cAAc,GAAG,iBAAiB,CAAC,IAAI,GAAG,eAAe,CAAC;SAC7D;aAAM;YACL,+CAA+C;YAC/C,MAAM,IAAI,uCAAgB,CAAC;gBACzB,IAAI,EAAE,2BAA2B;gBACjC,KAAK,EAAE,KAAK;gBACZ,OAAO,EACL,kFAAkF;aACrF,CAAC,CAAC;SACJ;KACF,QAAQ,cAAc,GAAG,YAAY,EAAE;IAExC,MAAM,qBAAqB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzC,MAAM,CAAC,IAAI,CACT;QACE,WAAW,EAAE,gBAAgB;QAC7B,SAAS,EAAE,cAAc;QACzB,YAAY;QACZ,SAAS,EAAE,qBAAqB,GAAG,uBAAuB;KAC3D,EACD,wBAAwB,CACzB,CAAC;AACJ,CAAC;AArFD,gDAqFC;AAQD;;;;;;;GAOG;AACH,SAAS,4BAA4B,CAAC,IAAS;IAC7C,MAAM,WAAW,GAAe,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;IACrD,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC;YACvB,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YAC/C,CAAC,CAAC,CAAC,CAAC;QACN,IAAI,MAAM,GAAG,WAAW,CAAC,IAAI,EAAE;YAC7B,WAAW,CAAC,GAAG,GAAG,IAAI,CAAC;YACvB,WAAW,CAAC,IAAI,GAAG,MAAM,CAAC;SAC3B;KACF;IAED,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,oCAAoC,CAC3C,IAAgD;IAEhD,IAAI,sBAAsB,CAAC;IAC3B,IAAI,kBAAkB,GAAG,MAAM,CAAC,gBAAgB,CAAC;IAEjD,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE;QACvB,MAAM,MAAM,GAAG,IAAI,EAAE,QAAQ;YAC3B,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClD,CAAC,CAAC,CAAC,CAAC;QACN,IAAI,MAAM,GAAG,kBAAkB,EAAE;YAC/B,sBAAsB,GAAG,IAAI,CAAC;YAC9B,kBAAkB,GAAG,MAAM,CAAC;SAC7B;KACF;IACD,OAAO,sBAAsB,CAAC;AAChC,CAAC;AAED,SAAS,uBAAuB,CAC9B,IAAgD;IAEhD,IAAI,aAAa,CAAC;IAClB,IAAI,iBAAiB,GAAG,MAAM,CAAC,gBAAgB,CAAC;IAEhD,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClE,IAAI,MAAM,GAAG,iBAAiB,EAAE;YAC9B,aAAa,GAAG,IAAI,CAAC;YACrB,iBAAiB,GAAG,MAAM,CAAC;SAC5B;KACF;IACD,OAAO,aAAa,CAAC;AACvB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,sBAAsB,CAAC,IAAqB;IACnD,IAAI,WAAW,CAAC;IAChB,IAAI,eAAe,GAAG,MAAM,CAAC,gBAAgB,CAAC;IAE9C,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClE,IAAI,MAAM,GAAG,eAAe,EAAE;YAC5B,WAAW,GAAG,IAAI,CAAC;YACnB,eAAe,GAAG,MAAM,CAAC;SAC1B;KACF;IAED,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,wBAAwB,CAAC,IAAY;IAC5C,MAAM,WAAW,GAAG,EAAE,CAAC;IAEvB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;QAC/C,IAAI,KAAK,KAAK,SAAS,EAAE;YACvB,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;SACtB;aAAM;YACL,WAAW,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;SAC7D;KACF;IAED,OAAO,WAAW,CAAC;AACrB,CAAC"}
|