@jupiterone/integration-sdk-runtime 10.5.1 → 10.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/execution/duplicateKeyTracker.js +2 -0
- package/dist/src/execution/duplicateKeyTracker.js.map +1 -1
- package/dist/src/execution/jobState.js +2 -6
- package/dist/src/execution/jobState.js.map +1 -1
- package/dist/src/execution/utils/bigMap.js +2 -1
- package/dist/src/execution/utils/bigMap.js.map +1 -1
- package/dist/src/logger/index.js +15 -1
- package/dist/src/logger/index.js.map +1 -1
- package/dist/src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.d.ts +5 -2
- package/dist/src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.js +22 -12
- package/dist/src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.js.map +1 -1
- package/dist/src/storage/memory.d.ts +14 -2
- package/dist/src/storage/memory.js +55 -32
- package/dist/src/storage/memory.js.map +1 -1
- package/dist/src/synchronization/events.js +2 -1
- package/dist/src/synchronization/events.js.map +1 -1
- package/dist/src/synchronization/index.js +3 -2
- package/dist/src/synchronization/index.js.map +1 -1
- package/dist/tsconfig.dist.tsbuildinfo +1 -1
- package/package.json +4 -4
|
@@ -13,6 +13,8 @@ const DUPLICATE_KEY_TRACKER_DEFAULT_MAP_KEY_SPACE = 2000000;
|
|
|
13
13
|
* table.
|
|
14
14
|
*/
|
|
15
15
|
class DuplicateKeyTracker {
|
|
16
|
+
graphObjectKeyMap;
|
|
17
|
+
normalizationFunction;
|
|
16
18
|
constructor(normalizationFunction) {
|
|
17
19
|
this.normalizationFunction = normalizationFunction || ((_key) => _key);
|
|
18
20
|
this.graphObjectKeyMap = new bigMap_1.BigMap(DUPLICATE_KEY_TRACKER_DEFAULT_MAP_KEY_SPACE);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"duplicateKeyTracker.js","sourceRoot":"","sources":["../../../src/execution/duplicateKeyTracker.ts"],"names":[],"mappings":";;;AAAA,2EAK0C;AAC1C,mCAAyC;AACzC,2CAAwC;AAExC,MAAM,2CAA2C,GAAG,OAAO,CAAC;AAE5D;;;;;;GAMG;AACH,MAAa,mBAAmB;
|
|
1
|
+
{"version":3,"file":"duplicateKeyTracker.js","sourceRoot":"","sources":["../../../src/execution/duplicateKeyTracker.ts"],"names":[],"mappings":";;;AAAA,2EAK0C;AAC1C,mCAAyC;AACzC,2CAAwC;AAExC,MAAM,2CAA2C,GAAG,OAAO,CAAC;AAE5D;;;;;;GAMG;AACH,MAAa,mBAAmB;IACb,iBAAiB,CAAyB;IAC1C,qBAAqB,CAA2B;IAEjE,YAAY,qBAAgD;QAC1D,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;QAEvE,IAAI,CAAC,iBAAiB,GAAG,IAAI,eAAM,CACjC,2CAA2C,CAC5C,CAAC;IACJ,CAAC;IAED,WAAW,CAAC,IAAY;QACtB,MAAM,aAAa,GAAG,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QACvD,IAAI,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE;YAC7C,MAAM,IAAI,mDAA4B,CACpC,iCAAiC,IAAI,GAAG,CACzC,CAAC;SACH;QAED,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;IAClD,CAAC;IAED,sBAAsB,CAAC,IAAY;QACjC,OAAO,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAC;IACtE,CAAC;IAED,MAAM,CAAC,IAAY;QACjB,OAAO,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAC;IACtE,CAAC;CACF;AA9BD,kDA8BC;AASM,KAAK,UAAU,2BAA2B,CAAC,EAChD,eAAe,EACf,OAAO,EACP,mBAAmB,EACnB,gBAAgB,GACS;IACzB,MAAM,yBAAyB,GAAG,4BAA4B,CAC5D,OAAO,EACP,eAAe,CAAC,IAAI,EACpB,mBAAmB,CACpB,CAAC;IAEF,IAAI,yBAAyB,EAAE;QAC7B,OAAO,eAAe,CAAC,yBAAyB,EAAE,eAAe,CAAC,CAAC;KACpE;SAAM;QACL,MAAM,kCAAkC,GACtC,MAAM,gBAAgB,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QAC1D,OAAO,eAAe,CACpB,kCAAmC,EACnC,eAAe,CAChB,CAAC;KACH;AACH,CAAC;AAtBD,kEAsBC;AAED;;;;;;;;GAQG;AACH,SAAS,4BAA4B,CACnC,OAAiB,EACjB,IAAY,EACZ,mBAA2B;IAE3B,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC3B,IAAI,CAAC,IAAI,mBAAmB,EAAE;YAC5B,OAAO,SAAS,CAAC;SAClB;aAAM,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI,EAAE;YAC1B,OAAO,CAAC,CAAC;SACV;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,SAAS,iBAAiB,CAAC,CAAM,EAAE,CAAM;IACvC,IAAI;QACF,IAAA,wBAAe,EAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACtB,OAAO,IAAI,CAAC;KACb;IAAC,MAAM;QACN,OAAO,KAAK,CAAC;KACd;AACH,CAAC;AAQD;;;;;;GAMG;AACH,SAAS,eAAe,CAAC,CAAS,EAAE,CAAS;IAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7C,MAAM,CAAC,QAAQ,GAAG,SAAS,CAAC;IAC5B,MAAM,CAAC,QAAQ,GAAG,SAAS,CAAC;IAE5B,OAAO;QACL,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,YAAY,EAAE,iBAAiB,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC;QACvD,eAAe,EAAE,iBAAiB,CAAC,MAAM,EAAE,MAAM,CAAC;KACnD,CAAC;AACJ,CAAC"}
|
|
@@ -3,9 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.createStepJobState = exports.MemoryDataStore = exports.TypeTracker = void 0;
|
|
4
4
|
const duplicateKeyTracker_1 = require("./duplicateKeyTracker");
|
|
5
5
|
class TypeTracker {
|
|
6
|
-
|
|
7
|
-
this.graphObjectTypeSummaryByStep = new Map();
|
|
8
|
-
}
|
|
6
|
+
graphObjectTypeSummaryByStep = new Map();
|
|
9
7
|
addStepGraphObjectType(input) {
|
|
10
8
|
const existingStepSummary = this.graphObjectTypeSummaryByStep.get(input.stepId);
|
|
11
9
|
if (existingStepSummary) {
|
|
@@ -49,9 +47,7 @@ class TypeTracker {
|
|
|
49
47
|
}
|
|
50
48
|
exports.TypeTracker = TypeTracker;
|
|
51
49
|
class MemoryDataStore {
|
|
52
|
-
|
|
53
|
-
this.data = new Map();
|
|
54
|
-
}
|
|
50
|
+
data = new Map();
|
|
55
51
|
set(key, data) {
|
|
56
52
|
this.data[key] = data;
|
|
57
53
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jobState.js","sourceRoot":"","sources":["../../../src/execution/jobState.ts"],"names":[],"mappings":";;;AAOA,+DAI+B;AAY/B,MAAa,WAAW;
|
|
1
|
+
{"version":3,"file":"jobState.js","sourceRoot":"","sources":["../../../src/execution/jobState.ts"],"names":[],"mappings":";;;AAOA,+DAI+B;AAY/B,MAAa,WAAW;IACL,4BAA4B,GAAG,IAAI,GAAG,EAGpD,CAAC;IAEJ,sBAAsB,CAAC,KAItB;QACC,MAAM,mBAAmB,GAAG,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAC/D,KAAK,CAAC,MAAM,CACb,CAAC;QAEF,IAAI,mBAAmB,EAAE;YACvB,MAAM,eAAe,GAAG,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAE7D,IAAI,eAAe,EAAE;gBACnB,eAAe,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC;aACtC;iBAAM;gBACL,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;aAC9D;SACF;aAAM;YACL,IAAI,CAAC,4BAA4B,CAAC,GAAG,CACnC,KAAK,CAAC,MAAM,EACZ,IAAI,GAAG,CAAqC;gBAC1C,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;aACtC,CAAC,CACH,CAAC;SACH;IACH,CAAC;IAED,0BAA0B,CAAC,MAAc;QACvC,MAAM,mBAAmB,GAAG,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC1E,OAAO,mBAAmB,CAAC,CAAC,CAAC,CAAC,GAAG,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACpE,CAAC;IAED,sBAAsB;QACpB,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAU,CAAC;QAE3C,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,4BAA4B,CAAC,MAAM,EAAE,EAAE;YACpE,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,IAAI,EAAE,EAAE;gBACtC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;aAC7B;SACF;QAED,OAAO,CAAC,GAAG,gBAAgB,CAAC,CAAC;IAC/B,CAAC;IAED,aAAa,CAAC,MAAc;QAC1B,MAAM,WAAW,GAA2B,EAAE,CAAC;QAE/C,MAAM,mBAAmB,GAAG,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAE1E,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,mBAAmB,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;YACnE,WAAW,CAAC,KAAK,CAAC,GAAG;gBACnB,GAAG,OAAO;aACX,CAAC;SACH;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;CACF;AA/DD,kCA+DC;AAED,MAAa,eAAe;IACT,IAAI,GAAG,IAAI,GAAG,EAAmB,CAAC;IAEnD,GAAG,CAAC,GAAW,EAAE,IAAa;QAC5B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,GAAG,CAAC,GAAW;QACb,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC;IAED,MAAM,CAAC,GAAW;QAChB,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC;CACF;AAdD,0CAcC;AAoCD,SAAgB,kBAAkB,CAAC,EACjC,MAAM,EACN,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,SAAS,EACT,eAAe,EACf,qBAAqB,EACrB,cAAc,EACd,oBAAoB,EACpB,QAAQ,EACR,oBAAoB,GACK;IACzB,MAAM,WAAW,GAAG,KAAK,EAAE,QAAkB,EAAqB,EAAE;QAClE,IAAI,eAAe,EAAE;YACnB,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;SAC1C;QAED,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE;YAChD,IAAI;gBACF,mBAAmB,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;aAC9C;YAAC,OAAO,GAAG,EAAE;gBACZ,MAAM,qBAAqB,GAAG,MAAM,IAAA,iDAA2B,EAAC;oBAC9D,eAAe,EAAE,MAAM;oBACvB,OAAO,EAAE,QAAQ;oBACjB,mBAAmB,EAAE,KAAK;oBAC1B,gBAAgB;iBACjB,CAAC,CAAC;gBACH,oBAAoB,CAAC,qBAAqB,CAAC,CAAC;gBAC5C,MAAM,GAAG,CAAC;aACX;YAED,WAAW,CAAC,sBAAsB,CAAC;gBACjC,MAAM;gBACN,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;SACJ;QAED,MAAM,gBAAgB,CAAC,WAAW,CAChC,MAAM,EACN,QAAQ,EACR,KAAK,EAAE,QAAQ,EAAE,EAAE,CACjB,QAAQ,EAAE,OAAO,CAAC;YAChB,QAAQ;YACR,aAAa,EAAE,EAAE;SAClB,CAAC,CACL,CAAC;QAEF,IAAI,cAAc,EAAE;YAClB,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;SAC5C;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;IAEF,SAAS,8BAA8B,CAAC,CAAe;QACrD,mBAAmB,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAExC,WAAW,CAAC,sBAAsB,CAAC;YACjC,MAAM;YACN,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;IACL,CAAC;IAED,MAAM,gBAAgB,GAAG,KAAK,EAAE,kBAAkC,EAAE,EAAE;QACpE,IAAI,aAA6B,CAAC;QAElC,IAAI,qBAAqB,EAAE;YACzB,aAAa,GAAG,EAAE,CAAC;YAEnB,KAAK,MAAM,YAAY,IAAI,kBAAkB,EAAE;gBAC7C,MAAM,eAAe,GAAG,MAAM,qBAAqB,CAAC,YAAY,CAAC,CAAC;gBAClE,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;gBAEpC,iEAAiE;gBACjE,mBAAmB;gBACnB,8BAA8B,CAAC,eAAe,CAAC,CAAC;aACjD;SACF;aAAM;YACL,aAAa,GAAG,kBAAkB,CAAC;YACnC,aAAa,CAAC,OAAO,CAAC,8BAA8B,CAAC,CAAC;SACvD;QAED,MAAM,gBAAgB,CAAC,gBAAgB,CACrC,MAAM,EACN,aAAa,EACb,KAAK,EAAE,aAAa,EAAE,EAAE,CACtB,QAAQ,EAAE,OAAO,CAAC;YAChB,QAAQ,EAAE,EAAE;YACZ,aAAa;SACd,CAAC,CACL,CAAC;QAEF,IAAI,oBAAoB,EAAE;YACxB,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC;SACvD;IACH,CAAC,CAAC;IAEF,OAAO;QACL,OAAO,EAAE,KAAK,EAAK,GAAW,EAAE,IAAO,EAAiB,EAAE;YACxD,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YACzB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,CAAC;QAED,OAAO,EAAE,KAAK,EAAK,GAAW,EAA0B,EAAE;YACxD,OAAO,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC,CAAC;QAClD,CAAC;QAED,UAAU,EAAE,KAAK,EAAK,GAAW,EAAiB,EAAE;YAClD,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACtB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,CAAC;QAED,SAAS,EAAE,KAAK,EAAE,MAAc,EAAmB,EAAE;YACnD,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;YAC9C,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC;QACD,WAAW;QAEX,eAAe,EAAE,CAAC,YAA0B,EAAE,EAAE;YAC9C,OAAO,gBAAgB,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC;QAC1C,CAAC;QACD,gBAAgB;QAEhB,UAAU,EAAE,KAAK,EAAE,IAAwB,EAAE,EAAE;YAC7C,IAAI,CAAC,IAAI;gBAAE,OAAO,IAAI,CAAC;YACvB,MAAM,eAAe,GAAG,mBAAmB,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;YAEzE,IAAI,CAAC,eAAe,EAAE;gBACpB,OAAO,IAAI,CAAC;aACb;YAED,OAAO,CAAC,MAAM,gBAAgB,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC,IAAI,IAAI,CAAC;QACtE,CAAC;QAED,MAAM,EAAE,CAAC,IAAwB,EAAE,EAAE;YACnC,IAAI,CAAC,IAAI;gBAAE,OAAO,KAAK,CAAC;YACxB,OAAO,mBAAmB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC1C,CAAC;QAED,eAAe,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE,CACpC,gBAAgB,CAAC,eAAe,CAAC,MAAM,EAAE,QAAQ,CAAC;QAEpD,oBAAoB,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE,CACzC,gBAAgB,CAAC,oBAAoB,CAAC,MAAM,EAAE,QAAQ,CAAC;QAEzD,KAAK,EAAE,GAAG,EAAE,CACV,gBAAgB,CAAC,KAAK,CACpB,KAAK,EAAE,QAAQ,EAAE,EAAE,CACjB,QAAQ,EAAE,OAAO,CAAC;YAChB,QAAQ;YACR,aAAa,EAAE,EAAE;SAClB,CAAC,EACJ,KAAK,EAAE,aAAa,EAAE,EAAE,CACtB,QAAQ,EAAE,OAAO,CAAC;YAChB,QAAQ,EAAE,EAAE;YACZ,aAAa;SACd,CAAC,CACL;QAEH,KAAK,CAAC,wBAAwB;YAC5B,MAAM,QAAQ,EAAE,wBAAwB,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC;AACJ,CAAC;AAtKD,gDAsKC"}
|
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.BigMap = void 0;
|
|
4
4
|
class BigMap {
|
|
5
|
+
maps = [new Map()];
|
|
6
|
+
maximumMapKeySpace;
|
|
5
7
|
constructor(maximumMapKeySpace) {
|
|
6
|
-
this.maps = [new Map()];
|
|
7
8
|
this.maximumMapKeySpace = maximumMapKeySpace;
|
|
8
9
|
}
|
|
9
10
|
set(key, value) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bigMap.js","sourceRoot":"","sources":["../../../../src/execution/utils/bigMap.ts"],"names":[],"mappings":";;;AAAA,MAAa,MAAM;
|
|
1
|
+
{"version":3,"file":"bigMap.js","sourceRoot":"","sources":["../../../../src/execution/utils/bigMap.ts"],"names":[],"mappings":";;;AAAA,MAAa,MAAM;IACT,IAAI,GAAgB,CAAC,IAAI,GAAG,EAAQ,CAAC,CAAC;IAErC,kBAAkB,CAAS;IAEpC,YAAY,kBAA0B;QACpC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;IAC/C,CAAC;IAED,GAAG,CAAC,GAAM,EAAE,KAAQ;QAClB,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAE5C,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,kBAAkB,EAAE;YACxC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAQ,CAAC;YAC/B,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACtC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,OAAO,MAAM,CAAC;SACf;aAAM;YACL,OAAO,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;SAC5B;IACH,CAAC;IAED,GAAG,CAAC,GAAM;QACR,OAAO,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,KAAK,SAAS,CAAC;IACjD,CAAC;IAED,GAAG,CAAC,GAAM;QACR,OAAO,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACrC,CAAC;IAED,aAAa;QACX,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;IAC1B,CAAC;CACF;AAjCD,wBAiCC;AAED,SAAS,SAAS,CAAO,IAAiB,EAAE,GAAM;IAChD,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE;QACrD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QAExB,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YAChB,OAAO,GAAG,CAAC;SACZ;KACF;AACH,CAAC;AAED,SAAS,WAAW,CAAO,IAAiB,EAAE,GAAM;IAClD,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE;QACrD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QACxB,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAE3B,IAAI,KAAK,KAAK,SAAS,EAAE;YACvB,OAAO,KAAK,CAAC;SACd;KACF;AACH,CAAC"}
|
package/dist/src/logger/index.js
CHANGED
|
@@ -19,6 +19,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
19
19
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
20
20
|
exports.createErrorEventDescription = exports.IntegrationLogger = exports.createIntegrationLogger = exports.createLogger = exports.PROVIDER_AUTH_ERROR_HELP = void 0;
|
|
21
21
|
const bunyan_1 = __importDefault(require("bunyan"));
|
|
22
|
+
const util_1 = require("util");
|
|
22
23
|
const events_1 = require("events");
|
|
23
24
|
const crypto_1 = require("crypto");
|
|
24
25
|
const integration_sdk_core_1 = require("@jupiterone/integration-sdk-core");
|
|
@@ -33,7 +34,17 @@ function createLogger({ name, pretty, serializers, onFailure, }) {
|
|
|
33
34
|
name,
|
|
34
35
|
level: (process.env.LOG_LEVEL || 'info'),
|
|
35
36
|
serializers: {
|
|
36
|
-
err:
|
|
37
|
+
err: function (err) {
|
|
38
|
+
if (!err || !err.stack)
|
|
39
|
+
return err;
|
|
40
|
+
return {
|
|
41
|
+
message: err.message,
|
|
42
|
+
name: err.name,
|
|
43
|
+
stack: (0, util_1.inspect)(err, false, 10),
|
|
44
|
+
code: err.code,
|
|
45
|
+
signal: err.signal,
|
|
46
|
+
};
|
|
47
|
+
},
|
|
37
48
|
},
|
|
38
49
|
};
|
|
39
50
|
if (pretty) {
|
|
@@ -96,6 +107,9 @@ function createInstanceConfigSerializer(fields) {
|
|
|
96
107
|
};
|
|
97
108
|
}
|
|
98
109
|
class IntegrationLogger extends events_1.EventEmitter {
|
|
110
|
+
_logger;
|
|
111
|
+
_errorSet;
|
|
112
|
+
onFailure;
|
|
99
113
|
constructor(input) {
|
|
100
114
|
super();
|
|
101
115
|
this._logger = input.logger;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/logger/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,oDAA4B;AAC5B,mCAAsC;AACtC,mCAA4C;AAE5C,2EA4B0C;AAE1C,0DAAwC;AAE3B,QAAA,wBAAwB,GACnC,sCAAsC;IACtC,4GAA4G;IAC5G,gFAAgF,CAAC;AAEnF,2BAA2B;AAC3B,MAAM,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;AAuB9C,SAAgB,YAAY,CAG1B,EACA,IAAI,EACJ,MAAM,EACN,WAAW,EACX,SAAS,GAIV;IACC,MAAM,YAAY,GAAyB;QACzC,IAAI;QACJ,KAAK,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM,CAAoB;QAC3D,WAAW,EAAE;YACX,GAAG,EAAE,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/logger/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,oDAA4B;AAC5B,+BAA+B;AAC/B,mCAAsC;AACtC,mCAA4C;AAE5C,2EA4B0C;AAE1C,0DAAwC;AAE3B,QAAA,wBAAwB,GACnC,sCAAsC;IACtC,4GAA4G;IAC5G,gFAAgF,CAAC;AAEnF,2BAA2B;AAC3B,MAAM,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;AAuB9C,SAAgB,YAAY,CAG1B,EACA,IAAI,EACJ,MAAM,EACN,WAAW,EACX,SAAS,GAIV;IACC,MAAM,YAAY,GAAyB;QACzC,IAAI;QACJ,KAAK,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM,CAAoB;QAC3D,WAAW,EAAE;YACX,GAAG,EAAE,UAAU,GAAG;gBAChB,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK;oBAAE,OAAO,GAAG,CAAC;gBACnC,OAAO;oBACL,OAAO,EAAE,GAAG,CAAC,OAAO;oBACpB,IAAI,EAAE,GAAG,CAAC,IAAI;oBACd,KAAK,EAAE,IAAA,cAAO,EAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC;oBAC9B,IAAI,EAAE,GAAG,CAAC,IAAI;oBACd,MAAM,EAAE,GAAG,CAAC,MAAM;iBACnB,CAAC;YACJ,CAAC;SACF;KACF,CAAC;IAEF,IAAI,MAAM,EAAE;QACV,YAAY,CAAC,OAAO,GAAG,CAAC,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;KAC5E;IAED,MAAM,MAAM,GAAG,gBAAM,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;IAEjD,IAAI,WAAW,EAAE;QACf,MAAM,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;KACpC;IAED,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAS,CAAC;IAElC,OAAO,IAAI,iBAAiB,CAAC;QAC3B,MAAM;QACN,QAAQ;QACR,SAAS;KACV,CAAC,CAAC;AACL,CAAC;AA9CD,oCA8CC;AAED;;;GAGG;AACH,SAAgB,uBAAuB,CAGrC,EACA,IAAI,EACJ,gBAAgB,EAChB,MAAM,EACN,WAAW,EACX,SAAS,GACwC;IACjD,MAAM,uBAAuB,GAAG,8BAA8B,CAC5D,gBAAgB,EAAE,oBAAoB,CACvC,CAAC;IAEF,OAAO,YAAY,CAAC;QAClB,IAAI;QACJ,MAAM;QACN,WAAW,EAAE;YACX,yBAAyB,EAAE,uBAAuB;YAClD,oCAAoC;YACpC,QAAQ,EAAE,CAAC,QAA6B,EAAE,EAAE,CAAC,CAAC;gBAC5C,GAAG,QAAQ;gBACX,MAAM,EAAE,QAAQ,CAAC,MAAM;oBACrB,CAAC,CAAC,uBAAuB,CAAC,QAAQ,CAAC,MAAM,CAAC;oBAC1C,CAAC,CAAC,SAAS;aACd,CAAC;YACF,GAAG,WAAW;SACf;QACD,SAAS;KACV,CAAC,CAAC;AACL,CAAC;AA9BD,0DA8BC;AAED,SAAS,8BAA8B,CAErC,MAAmD;IACnD,OAAO,CAAC,MAAW,EAAE,EAAE;QACrB,IAAI,CAAC,MAAM,EAAE;YACX,OAAO,MAAM,CAAC;SACf;aAAM;YACL,MAAM,UAAU,GAAQ,EAAE,CAAC;YAC3B,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE;gBACnC,MAAM,KAAK,GAAG,MAAM,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;gBAClC,IAAI,KAAK,EAAE;oBACT,UAAU,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;iBAChD;qBAAM;oBACL,UAAU,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;iBACvB;aACF;YACD,OAAO,UAAU,CAAC;SACnB;IACH,CAAC,CAAC;AACJ,CAAC;AAmBD,MAAa,iBACX,SAAQ,qBAAY;IAGZ,OAAO,CAAS;IAChB,SAAS,CAAa;IACrB,SAAS,CAAoB;IAEtC,YAAY,KAA6B;QACvC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,QAAQ,CAAC;QAEhC,gEAAgE;QAChE,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACjD,CAAC;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,cAAc,CAAC,GAAU;QACvB,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,GAAG,MAAa;QACpB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC;IACvC,CAAC;IACD,IAAI,CAAC,GAAG,MAAa;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;IACtC,CAAC;IAED,IAAI,CAAC,GAAG,MAAa;QACnB,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAElC,IAAI,CAAC,aAAa,CAChB;YACE,IAAI,EAAE,aAAa;YACnB,KAAK,EAAE,CAAC;SACT,EACD;YACE,SAAS,EAAE,KAAK;SACjB,CACF,CAAC;QAEF,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,GAAG,MAAa;QACpB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,GAAG,MAAa;QACpB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE;YACvB,OAAO;SACR;QAED,IAAI,gBAAgB,GAAwB,EAAE,CAAC;QAC/C,IAAI,aAAa,GAAU,MAAM,CAAC;QAElC,IAAI,MAAM,CAAC,CAAC,CAAC,YAAY,KAAK,EAAE;YAC9B,gBAAgB,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YACtC,aAAa,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SACjC;aAAM,IAAI,OAAO,MAAM,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE;YACxC,gBAAgB,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YAC7B,aAAa,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SACjC;QAED,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CACvB,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,gBAAgB,EAAE,EACtC,GAAG,aAAa,CACjB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,GAAG,MAAa;QACpB,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAElC,IAAI,CAAC,aAAa,CAChB;YACE,IAAI,EAAE,cAAc;YACpB,KAAK,EAAE,CAAC;SACT,EACD;YACE,SAAS,EAAE,KAAK;SACjB,CACF,CAAC;QAEF,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,UAAkB,EAAE,EAAE,MAAgB;QAC1C,MAAM,WAAW,GAAG,IAAI,iBAAiB,CAAC;YACxC,QAAQ,EAAE,IAAI,CAAC,SAAS;YACxB,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC;YAC3C,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAC;QAEH,wBAAwB;QACxB,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QAC5D,WAAW,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;QAE9D,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,IAAI,CACF,IAAO,EACP,IAAU;QAEV,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAChC,CAAC;IAED,SAAS,CAAC,IAAkB;QAC1B,MAAM,IAAI,GAAG,YAAY,CAAC;QAC1B,MAAM,WAAW,GAAG,kBAAkB,IAAI,CAAC,IAAI,MAAM,CAAC;QACtD,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACvB,IAAI,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,WAAW,CAAC,IAAkB;QAC5B,MAAM,IAAI,GAAG,UAAU,CAAC;QACxB,MAAM,WAAW,GAAG,mBAAmB,IAAI,CAAC,IAAI,IAAI,CAAC;QACrD,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACvB,IAAI,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,QAAQ,CACN,IAAkB,EAClB,MAA0B,EAC1B,iBAA4C;QAE5C,IAAI,CAAC,MAAM,IAAI,MAAM,KAAK,yCAAkB,CAAC,IAAI,EAAE;YACjD,OAAO;SACR;QAED,MAAM,IAAI,GAAG,WAAW,CAAC;QACzB,IAAI,WAAW,GAAG,iBAAiB,IAAI,CAAC,IAAI,KAAK,CAAC;QAElD,QAAQ,MAAM,EAAE;YACd,KAAK,yCAAkB,CAAC,WAAW,CAAC,CAAC;gBACnC,WAAW,IAAI,yKAAyK,CAAC;gBACzL,MAAM;aACP;YACD,KAAK,yCAAkB,CAAC,IAAI,CAAC,CAAC;gBAC5B,WAAW,IAAI,iDAAiD,CAAC;gBACjE,MAAM;aACP;YACD,KAAK,yCAAkB,CAAC,UAAU,CAAC,CAAC;gBAClC,WAAW,IAAI,gEAAgE,CAAC;gBAChF,MAAM;aACP;YACD,KAAK,yCAAkB,CAAC,MAAM,CAAC,CAAC;gBAC9B,WAAW,IAAI,wEAAwE,CAAC;gBACxF,MAAM;aACP;YACD,KAAK,yCAAkB,CAAC,WAAW,CAAC,CAAC;gBACnC,WAAW,IAAI,wEAAwE,CAAC;gBACxF,MAAM;aACP;YACD,KAAK,yCAAkB,CAAC,eAAe,CAAC,CAAC;gBACvC,MAAM,cAAc,GAAG,iBAAiB,EAAE,UAAU,EAAE,IAAI,CAAC;gBAC3D,IAAI,CAAC,cAAc,EAAE;oBACnB,IAAI,CAAC,IAAI,CAAC,2CAA2C,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;iBACnE;gBACD,WAAW,IAAI,cAAc;oBAC3B,CAAC,CAAC,0CAA0C,cAAc,6GAA6G;oBACvK,CAAC,CAAC,iJAAiJ,CAAC;gBACtJ,MAAM;aACP;SACF;QAED,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACvB,IAAI,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,WAAW,CAAC,IAAkB,EAAE,GAAU;QACxC,MAAM,SAAS,GAAG,cAAc,CAAC;QACjC,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,2BAA2B,CAC1D,GAAG,EACH,SAAS,IAAI,CAAC,IAAI,oCAAoC,CACvD,CAAC;QACF,IAAI,CAAC,aAAa,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,CAAC,CAAC;IAC/D,CAAC;IAED,0BAA0B,CAAC,GAAuB;QAChD,MAAM,IAAI,GAAG,mBAAmB,CAAC;QACjC,MAAM,WAAW,GAAG,iDAAiD,CAAC;QACtE,IAAI,CAAC,IAAI,CACP;YACE,oBAAoB,EAAE,GAAG,CAAC,EAAE;SAC7B,EACD,WAAW,CACZ,CAAC;QACF,IAAI,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,wBAAwB,CAAC,GAAuB;QAC9C,MAAM,IAAI,GAAG,iBAAiB,CAAC;QAC/B,MAAM,WAAW,GAAG,kBAAkB,CAAC;QACvC,IAAI,CAAC,IAAI,CACP;YACE,oBAAoB,EAAE,GAAG,CAAC,EAAE;SAC7B,EACD,WAAW,CACZ,CAAC;QACF,IAAI,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,iBAAiB,CAAC,GAAU;QAC1B,MAAM,SAAS,GAAG,oBAAoB,CAAC;QACvC,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,2BAA2B,CAC1D,GAAG,EACH,4DAA4D,CAC7D,CAAC;QACF,IAAI,CAAC,aAAa,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,CAAC,CAAC;IAC/D,CAAC;IAEO,aAAa,CAAC,OAKrB;QACC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;QAEzD,uEAAuE;QACvE,yEAAyE;QACzE,wEAAwE;QACxE,MAAM,IAAI,GAAI,GAAW,CAAC,IAAI,CAAC;QAE/B,IAAI,IAAA,kDAA2B,EAAC,GAAG,CAAC,EAAE;YACpC,IAAI,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,WAAW,CAAC,CAAC;YAChD,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;SACzB;aAAM;YACL,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,WAAW,CAAC,CAAC;SAChD;QAED,IAAI,CAAC,YAAY,CAAC;YAChB,IAAI,EAAE,SAAS;YACf,WAAW;YACX,KAAK,EAAE,wCAAiB,CAAC,KAAK;SAC/B,CAAC,CAAC;IACL,CAAC;IAED,aAAa,CACX,MAAiC,EACjC,EAAE,SAAS,GAAG,IAAI,KAA2B,EAAE;QAE/C,MAAM,mBAAmB,GAAG;YAC1B,GAAG,MAAM;YACT,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC;QAEF,IAAI,SAAS,EAAE;YACb,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,mBAAmB,EAAE,EAAE,mBAAmB,CAAC,CAAC;SACjE;QAED,2DAA2D;QAC3D,2BAA2B;QAC3B,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,mBAAmB,CAAC,CAAC;IAClD,CAAC;IAED,YAAY,CAAC,KAAwB;QACnC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACxB,GAAG,KAAK;YACR,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,wCAAiB,CAAC,IAAI;SAC7C,CAAC,CAAC;IACL,CAAC;IAED,gBAAgB,CAAC,KAA4B;QAC3C,OAAO,IAAI,CAAC,YAAY,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,wCAAiB,CAAC,IAAI,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,gBAAgB,CAAC,KAA4B;QAC3C,OAAO,IAAI,CAAC,YAAY,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,wCAAiB,CAAC,IAAI,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,iBAAiB,CAAC,KAA6B;QAC7C,OAAO,IAAI,CAAC,YAAY,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,wCAAiB,CAAC,KAAK,EAAE,CAAC,CAAC;IACzE,CAAC;IAEO,iBAAiB,CAAC,MAAW;QACnC,IAAI,MAAM,YAAY,KAAK,EAAE;YAC3B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;SAC5B;aAAM,IAAI,MAAM,EAAE,GAAG,YAAY,KAAK,EAAE;YACvC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;SAChC;IACH,CAAC;CACF;AA7SD,8CA6SC;AAID,SAAgB,2BAA2B,CACzC,GAA6B,EAC7B,OAAe;AAEf;;;GAGG;AACH,SAAkB;IAElB,MAAM,OAAO,GAAG,IAAA,mBAAI,GAAE,CAAC;IAEvB,IAAI,SAAiB,CAAC;IACtB,IAAI,WAAmB,CAAC;IAExB,IAAI,GAAG,YAAY,uCAAgB,EAAE;QACnC,SAAS,GAAG,GAAG,CAAC,IAAI,CAAC;QACrB,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC;KAC3B;SAAM;QACL,SAAS,GAAG,4CAAqB,CAAC;QAClC,WAAW,GAAG,8CAAuB,CAAC;KACvC;IAED,IAAI,IAAA,0CAAmB,EAAC,GAAG,CAAC,EAAE;QAC5B,uDAAuD;QACvD,wCAAwC;QACxC,OAAO,IAAI,gCAAwB,CAAC;KACrC;IAED,MAAM,cAAc,GAAoB;QACtC,CAAC,WAAW,EAAE,SAAS,CAAC;QACxB,CAAC,QAAQ,EAAE,WAAW,CAAC;KACxB,CAAC;IAEF,IAAI,SAAS,EAAE;QACb,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE;YACxC,cAAc,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;SAC5C;KACF;IAED,MAAM,YAAY,GAAG,cAAc;SAChC,GAAG,CAAC,CAAC,aAAa,EAAE,EAAE;QACrB,OAAO,GAAG,aAAa,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACnE,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,OAAO;QACL,OAAO;QACP,WAAW,EAAE,GAAG,OAAO,KAAK,YAAY,GAAG;KAC5C,CAAC;AACJ,CAAC;AAlDD,kEAkDC"}
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { Entity, GraphObjectFilter, GraphObjectIteratee, Relationship, GraphObjectStore, GraphObjectIndexMetadata, GetIndexMetadataForGraphObjectTypeParams, IntegrationStep } from '@jupiterone/integration-sdk-core';
|
|
2
2
|
export declare const DEFAULT_GRAPH_OBJECT_BUFFER_THRESHOLD = 500;
|
|
3
3
|
export declare const DEFAULT_GRAPH_OBJECT_FILE_SIZE = 500;
|
|
4
|
+
export declare const DEFAULT_GRAPH_OBJECT_BUFFER_THRESHOLD_IN_BYTES = 5000000;
|
|
5
|
+
export declare const MAX_GRAPH_OBJECT_BUFFER_THRESHOLD_IN_BYTES = 1000000000;
|
|
4
6
|
export interface FileSystemGraphObjectStoreParams {
|
|
5
7
|
integrationSteps?: IntegrationStep[];
|
|
6
8
|
/**
|
|
7
9
|
* The maximum size in bytes of entities/relationships stored in memory at one time.
|
|
8
|
-
* default:
|
|
10
|
+
* default: 5_000_000
|
|
9
11
|
*/
|
|
10
12
|
graphObjectBufferThresholdInBytes?: number;
|
|
11
13
|
/**
|
|
@@ -14,6 +16,7 @@ export interface FileSystemGraphObjectStoreParams {
|
|
|
14
16
|
* this value up.
|
|
15
17
|
*
|
|
16
18
|
* Default: 500
|
|
19
|
+
* @deprecated this argument is no longer used. Please use `graphObjectBufferThresholdInBytes` instead.
|
|
17
20
|
*/
|
|
18
21
|
graphObjectBufferThreshold?: number;
|
|
19
22
|
/**
|
|
@@ -28,8 +31,8 @@ export interface FileSystemGraphObjectStoreParams {
|
|
|
28
31
|
export declare class FileSystemGraphObjectStore implements GraphObjectStore {
|
|
29
32
|
private readonly semaphore;
|
|
30
33
|
private readonly localGraphObjectStore;
|
|
31
|
-
private readonly graphObjectBufferThreshold;
|
|
32
34
|
private readonly graphObjectFileSize;
|
|
35
|
+
private readonly graphObjectBufferThresholdInBytes;
|
|
33
36
|
private readonly prettifyFiles;
|
|
34
37
|
private readonly stepIdToGraphObjectIndexMetadataMap;
|
|
35
38
|
private readonly entityOnDiskLocationMap;
|
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.FileSystemGraphObjectStore = exports.DEFAULT_GRAPH_OBJECT_FILE_SIZE = exports.DEFAULT_GRAPH_OBJECT_BUFFER_THRESHOLD = void 0;
|
|
6
|
+
exports.FileSystemGraphObjectStore = exports.MAX_GRAPH_OBJECT_BUFFER_THRESHOLD_IN_BYTES = exports.DEFAULT_GRAPH_OBJECT_BUFFER_THRESHOLD_IN_BYTES = exports.DEFAULT_GRAPH_OBJECT_FILE_SIZE = exports.DEFAULT_GRAPH_OBJECT_BUFFER_THRESHOLD = void 0;
|
|
7
7
|
const async_sema_1 = require("async-sema");
|
|
8
8
|
const p_map_1 = __importDefault(require("p-map"));
|
|
9
9
|
const flushDataToDisk_1 = require("./flushDataToDisk");
|
|
@@ -14,6 +14,9 @@ const bigMap_1 = require("../../execution/utils/bigMap");
|
|
|
14
14
|
const lodash_1 = require("lodash");
|
|
15
15
|
exports.DEFAULT_GRAPH_OBJECT_BUFFER_THRESHOLD = 500;
|
|
16
16
|
exports.DEFAULT_GRAPH_OBJECT_FILE_SIZE = 500;
|
|
17
|
+
exports.DEFAULT_GRAPH_OBJECT_BUFFER_THRESHOLD_IN_BYTES = 5000000;
|
|
18
|
+
// no more than 10^9 bytes
|
|
19
|
+
exports.MAX_GRAPH_OBJECT_BUFFER_THRESHOLD_IN_BYTES = 1000000000;
|
|
17
20
|
// it is important that this value is set to 1
|
|
18
21
|
// to ensure that only one operation can be performed at a time.
|
|
19
22
|
const BINARY_SEMAPHORE_CONCURRENCY = 1;
|
|
@@ -43,16 +46,23 @@ function integrationStepsToGraphObjectIndexMetadataMap(integrationSteps) {
|
|
|
43
46
|
}
|
|
44
47
|
const ENTITY_LOCATION_ON_DISK_DEFAULT_MAP_KEY_SPACE = 2000000;
|
|
45
48
|
class FileSystemGraphObjectStore {
|
|
49
|
+
semaphore;
|
|
50
|
+
localGraphObjectStore = new memory_1.InMemoryGraphObjectStore();
|
|
51
|
+
graphObjectFileSize;
|
|
52
|
+
graphObjectBufferThresholdInBytes;
|
|
53
|
+
prettifyFiles;
|
|
54
|
+
stepIdToGraphObjectIndexMetadataMap;
|
|
55
|
+
entityOnDiskLocationMap = new bigMap_1.BigMap(ENTITY_LOCATION_ON_DISK_DEFAULT_MAP_KEY_SPACE);
|
|
46
56
|
constructor(params) {
|
|
47
|
-
this.localGraphObjectStore = new memory_1.InMemoryGraphObjectStore();
|
|
48
|
-
this.entityOnDiskLocationMap = new bigMap_1.BigMap(ENTITY_LOCATION_ON_DISK_DEFAULT_MAP_KEY_SPACE);
|
|
49
57
|
this.semaphore = new async_sema_1.Sema(BINARY_SEMAPHORE_CONCURRENCY);
|
|
50
|
-
this.graphObjectBufferThreshold =
|
|
51
|
-
params?.graphObjectBufferThreshold ||
|
|
52
|
-
exports.DEFAULT_GRAPH_OBJECT_BUFFER_THRESHOLD;
|
|
53
58
|
this.graphObjectFileSize =
|
|
54
59
|
params?.graphObjectFileSize || exports.DEFAULT_GRAPH_OBJECT_FILE_SIZE;
|
|
55
60
|
this.prettifyFiles = params?.prettifyFiles || false;
|
|
61
|
+
this.graphObjectBufferThresholdInBytes = (0, lodash_1.min)([
|
|
62
|
+
params?.graphObjectBufferThresholdInBytes ||
|
|
63
|
+
exports.DEFAULT_GRAPH_OBJECT_BUFFER_THRESHOLD_IN_BYTES,
|
|
64
|
+
exports.MAX_GRAPH_OBJECT_BUFFER_THRESHOLD_IN_BYTES,
|
|
65
|
+
]);
|
|
56
66
|
if (params?.integrationSteps) {
|
|
57
67
|
this.stepIdToGraphObjectIndexMetadataMap =
|
|
58
68
|
integrationStepsToGraphObjectIndexMetadataMap(params.integrationSteps);
|
|
@@ -60,15 +70,15 @@ class FileSystemGraphObjectStore {
|
|
|
60
70
|
}
|
|
61
71
|
async addEntities(stepId, newEntities, onEntitiesFlushed) {
|
|
62
72
|
await this.localGraphObjectStore.addEntities(stepId, newEntities);
|
|
63
|
-
if (this.localGraphObjectStore.
|
|
64
|
-
this.
|
|
73
|
+
if (this.localGraphObjectStore.getTotalEntitySizeInBytes() >=
|
|
74
|
+
this.graphObjectBufferThresholdInBytes) {
|
|
65
75
|
await this.flushEntitiesToDisk(onEntitiesFlushed);
|
|
66
76
|
}
|
|
67
77
|
}
|
|
68
78
|
async addRelationships(stepId, newRelationships, onRelationshipsFlushed) {
|
|
69
79
|
await this.localGraphObjectStore.addRelationships(stepId, newRelationships);
|
|
70
|
-
if (this.localGraphObjectStore.
|
|
71
|
-
this.
|
|
80
|
+
if (this.localGraphObjectStore.getTotalRelationshipSizeInBytes() >=
|
|
81
|
+
this.graphObjectBufferThresholdInBytes) {
|
|
72
82
|
await this.flushRelationshipsToDisk(onRelationshipsFlushed);
|
|
73
83
|
}
|
|
74
84
|
}
|
|
@@ -144,7 +154,7 @@ class FileSystemGraphObjectStore {
|
|
|
144
154
|
}
|
|
145
155
|
}));
|
|
146
156
|
}
|
|
147
|
-
this.localGraphObjectStore.flushEntities(entities);
|
|
157
|
+
this.localGraphObjectStore.flushEntities(entities, stepId);
|
|
148
158
|
if (onEntitiesFlushed) {
|
|
149
159
|
await onEntitiesFlushed(entities);
|
|
150
160
|
}
|
|
@@ -173,7 +183,7 @@ class FileSystemGraphObjectStore {
|
|
|
173
183
|
});
|
|
174
184
|
}));
|
|
175
185
|
}
|
|
176
|
-
this.localGraphObjectStore.flushRelationships(relationships);
|
|
186
|
+
this.localGraphObjectStore.flushRelationships(relationships, stepId);
|
|
177
187
|
if (onRelationshipsFlushed) {
|
|
178
188
|
await onRelationshipsFlushed(relationships);
|
|
179
189
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FileSystemGraphObjectStore.js","sourceRoot":"","sources":["../../../../src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.ts"],"names":[],"mappings":";;;;;;AAAA,2CAAkC;AAClC,kDAAyB;AAazB,uDAAoD;AACpD,uCAImB;AACnB,sCAAqD;AAErD,iDAA8D;AAC9D,yDAAsD;AACtD,
|
|
1
|
+
{"version":3,"file":"FileSystemGraphObjectStore.js","sourceRoot":"","sources":["../../../../src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.ts"],"names":[],"mappings":";;;;;;AAAA,2CAAkC;AAClC,kDAAyB;AAazB,uDAAoD;AACpD,uCAImB;AACnB,sCAAqD;AAErD,iDAA8D;AAC9D,yDAAsD;AACtD,mCAAoC;AAEvB,QAAA,qCAAqC,GAAG,GAAG,CAAC;AAC5C,QAAA,8BAA8B,GAAG,GAAG,CAAC;AAErC,QAAA,8CAA8C,GAAG,OAAS,CAAC;AACxE,0BAA0B;AACb,QAAA,0CAA0C,GAAG,UAAa,CAAC;AAExE,8CAA8C;AAC9C,gEAAgE;AAChE,MAAM,4BAA4B,GAAG,CAAC,CAAC;AAyCvC;;GAEG;AACH,SAAS,6CAA6C,CACpD,gBAAmC;IAEnC,MAAM,mCAAmC,GAAG,IAAI,GAAG,EAGhD,CAAC;IAEJ,KAAK,MAAM,IAAI,IAAI,gBAAgB,EAAE;QACnC,MAAM,WAAW,GAAgC;YAC/C,QAAQ,EAAE,IAAI,GAAG,EAAE;YACnB,aAAa,EAAE,IAAI,GAAG,EAAE;SACzB,CAAC;QAEF,KAAK,MAAM,cAAc,IAAI,IAAI,CAAC,QAAQ,EAAE;YAC1C,IAAI,cAAc,CAAC,aAAa,EAAE;gBAChC,WAAW,CAAC,QAAQ,CAAC,GAAG,CACtB,cAAc,CAAC,KAAK,EACpB,cAAc,CAAC,aAAa,CAC7B,CAAC;aACH;SACF;QAED,KAAK,MAAM,oBAAoB,IAAI,IAAI,CAAC,aAAa,EAAE;YACrD,IAAI,oBAAoB,CAAC,aAAa,EAAE;gBACtC,WAAW,CAAC,aAAa,CAAC,GAAG,CAC3B,oBAAoB,CAAC,KAAK,EAC1B,oBAAoB,CAAC,aAAa,CACnC,CAAC;aACH;SACF;QAED,mCAAmC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;KAC/D;IAED,OAAO,mCAAmC,CAAC;AAC7C,CAAC;AAeD,MAAM,6CAA6C,GAAG,OAAO,CAAC;AAE9D,MAAa,0BAA0B;IACpB,SAAS,CAAO;IAChB,qBAAqB,GAAG,IAAI,iCAAwB,EAAE,CAAC;IACvD,mBAAmB,CAAS;IAC5B,iCAAiC,CAAS;IAC1C,aAAa,CAAU;IACvB,mCAAmC,CAGlD;IACe,uBAAuB,GAAG,IAAI,eAAM,CAGnD,6CAA6C,CAAC,CAAC;IAEjD,YAAY,MAAyC;QACnD,IAAI,CAAC,SAAS,GAAG,IAAI,iBAAI,CAAC,4BAA4B,CAAC,CAAC;QACxD,IAAI,CAAC,mBAAmB;YACtB,MAAM,EAAE,mBAAmB,IAAI,sCAA8B,CAAC;QAEhE,IAAI,CAAC,aAAa,GAAG,MAAM,EAAE,aAAa,IAAI,KAAK,CAAC;QACpD,IAAI,CAAC,iCAAiC,GAAG,IAAA,YAAG,EAAC;YAC3C,MAAM,EAAE,iCAAiC;gBACvC,sDAA8C;YAChD,kDAA0C;SAC3C,CAAE,CAAC;QACJ,IAAI,MAAM,EAAE,gBAAgB,EAAE;YAC5B,IAAI,CAAC,mCAAmC;gBACtC,6CAA6C,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;SAC1E;IACH,CAAC;IAED,KAAK,CAAC,WAAW,CACf,MAAc,EACd,WAAqB,EACrB,iBAAyD;QAEzD,MAAM,IAAI,CAAC,qBAAqB,CAAC,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAElE,IACE,IAAI,CAAC,qBAAqB,CAAC,yBAAyB,EAAE;YACtD,IAAI,CAAC,iCAAiC,EACtC;YACA,MAAM,IAAI,CAAC,mBAAmB,CAAC,iBAAiB,CAAC,CAAC;SACnD;IACH,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,MAAc,EACd,gBAAgC,EAChC,sBAAyE;QAEzE,MAAM,IAAI,CAAC,qBAAqB,CAAC,gBAAgB,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;QAE5E,IACE,IAAI,CAAC,qBAAqB,CAAC,+BAA+B,EAAE;YAC5D,IAAI,CAAC,iCAAiC,EACtC;YACA,MAAM,IAAI,CAAC,wBAAwB,CAAC,sBAAsB,CAAC,CAAC;SAC7D;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,UAAU,CAAC,IAAwB;QACvC,IAAI,CAAC,IAAI;YAAE,OAAO;QAClB,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACzE,IAAI,cAAc,EAAE;YAClB,OAAO,cAAc,CAAC;SACvB;QAED,MAAM,oBAAoB,GAAG,IAAI,CAAC,uBAAuB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpE,IAAI,CAAC,oBAAoB;YAAE,OAAO;QAElC,MAAM,QAAQ,GAAG,IAAA,uCAA0B,EACzC,oBAAoB,CAAC,aAAa,CACnC,CAAC;QACF,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAA,6BAAmB,EAAoB;YAChE,QAAQ;SACT,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,eAAe,CACnB,MAAyB,EACzB,QAAgC;QAEhC,MAAM,IAAI,CAAC,qBAAqB,CAAC,eAAe,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAEnE,MAAM,IAAA,gCAAsB,EAAC;YAC3B,IAAI,EAAE,MAAM,CAAC,KAAK;YAClB,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,oBAAoB,CACxB,MAAyB,EACzB,QAAgC;QAEhC,MAAM,IAAI,CAAC,qBAAqB,CAAC,oBAAoB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAExE,MAAM,IAAA,sCAA4B,EAAC;YACjC,IAAI,EAAE,MAAM,CAAC,KAAK;YAClB,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CACT,iBAAyD,EACzD,sBAAyE;QAEzE,MAAM,OAAO,CAAC,GAAG,CAAC;YAChB,IAAI,CAAC,mBAAmB,CAAC,iBAAiB,CAAC;YAC3C,IAAI,CAAC,wBAAwB,CAAC,sBAAsB,CAAC;SACtD,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,mBAAmB,CACvB,iBAAyD;QAEzD,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,CAC5B,IAAA,eAAI,EACF,IAAI,CAAC,qBAAqB,CAAC,qBAAqB,EAAE,EAClD,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,EAAE;YAC3B,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;gBACtC,MAAM,aAAa,GAAG,IAAI,CAAC,kCAAkC,CAAC;oBAC5D,MAAM;oBACN,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,yBAAyB,EAAE,UAAU;iBACtC,CAAC,CAAC;gBAEH,IAAI,OAAO,aAAa,KAAK,WAAW,EAAE;oBACxC,OAAO,IAAI,CAAC;iBACb;gBAED,OAAO,aAAa,CAAC,OAAO,KAAK,IAAI,CAAC;YACxC,CAAC,CAAC,CAAC;YAEH,IAAI,SAAS,CAAC,MAAM,EAAE;gBACpB,MAAM,OAAO,CAAC,GAAG,CACf,IAAA,cAAK,EAAC,SAAS,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;oBAC5D,MAAM,uBAAuB,GAAG,MAAM,IAAA,iCAAe,EAAC;wBACpD,oBAAoB,EAAE,MAAM;wBAC5B,cAAc,EAAE,UAAU;wBAC1B,IAAI;wBACJ,MAAM,EAAE,IAAI,CAAC,aAAa;qBAC3B,CAAC,CAAC;oBAEH,KAAK,MAAM,EACT,aAAa,EACb,UAAU,GACX,IAAI,uBAAuB,EAAE;wBAC5B,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE;4BAC7C,IAAI,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE;gCACvC,aAAa;gCACb,KAAK;6BACN,CAAC,CAAC;yBACJ;qBACF;gBACH,CAAC,CAAC,CACH,CAAC;aACH;YAED,IAAI,CAAC,qBAAqB,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAE3D,IAAI,iBAAiB,EAAE;gBACrB,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAC;aACnC;QACH,CAAC,CACF,CACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,wBAAwB,CAC5B,sBAAyE;QAEzE,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,CAC5B,IAAA,eAAI,EACF,IAAI,CAAC,qBAAqB,CAAC,0BAA0B,EAAE,EACvD,KAAK,EAAE,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,EAAE;YAChC,MAAM,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;gBAC3C,MAAM,aAAa,GAAG,IAAI,CAAC,kCAAkC,CAAC;oBAC5D,MAAM;oBACN,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,yBAAyB,EAAE,eAAe;iBAC3C,CAAC,CAAC;gBAEH,IAAI,OAAO,aAAa,KAAK,WAAW,EAAE;oBACxC,OAAO,IAAI,CAAC;iBACb;gBAED,OAAO,aAAa,CAAC,OAAO,KAAK,IAAI,CAAC;YACxC,CAAC,CAAC,CAAC;YAEH,IAAI,SAAS,CAAC,MAAM,EAAE;gBACpB,MAAM,OAAO,CAAC,GAAG,CACf,IAAA,cAAK,EAAC,SAAS,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;oBAC5D,MAAM,IAAA,iCAAe,EAAC;wBACpB,oBAAoB,EAAE,MAAM;wBAC5B,cAAc,EAAE,eAAe;wBAC/B,IAAI;wBACJ,MAAM,EAAE,IAAI,CAAC,aAAa;qBAC3B,CAAC,CAAC;gBACL,CAAC,CAAC,CACH,CAAC;aACH;YAED,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;YAErE,IAAI,sBAAsB,EAAE;gBAC1B,MAAM,sBAAsB,CAAC,aAAa,CAAC,CAAC;aAC7C;QACH,CAAC,CACF,CACF,CAAC;IACJ,CAAC;IAED,kCAAkC,CAAC,EACjC,MAAM,EACN,KAAK,EACL,yBAAyB,GACgB;QAGzC,IAAI,CAAC,IAAI,CAAC,mCAAmC,EAAE;YAC7C,OAAO,SAAS,CAAC;SAClB;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,mCAAmC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACjE,OAAO,GAAG,IAAI,GAAG,CAAC,yBAAyB,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC1D,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACK,KAAK,CAAC,aAAa,CAAI,SAA2B;QACxD,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QAC/B,IAAI;YACF,MAAM,SAAS,EAAE,CAAC;SACnB;gBAAS;YACR,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;SAC1B;IACH,CAAC;CACF;AAxQD,gEAwQC"}
|
|
@@ -17,6 +17,12 @@ export declare class InMemoryGraphObjectStore implements GraphObjectStore {
|
|
|
17
17
|
*/
|
|
18
18
|
private readonly entityKeyToEntityMap;
|
|
19
19
|
private readonly relationshipKeyToRelationshipMap;
|
|
20
|
+
/**
|
|
21
|
+
* Store the size in bytes for entities and relationships generated by each step.
|
|
22
|
+
*
|
|
23
|
+
*/
|
|
24
|
+
private entitySizeInBytes;
|
|
25
|
+
private relationshipSizeInBytes;
|
|
20
26
|
/**
|
|
21
27
|
* Maps to lookup all entity/relationship _key's by _type. The value of this
|
|
22
28
|
* data structure is Map<_key{string}, boolean> to make deleting individual
|
|
@@ -50,20 +56,26 @@ export declare class InMemoryGraphObjectStore implements GraphObjectStore {
|
|
|
50
56
|
* steps have no interdependence. If we cleared it out, it's possible that
|
|
51
57
|
* two steps could be running in parallel and one of the steps may clear out
|
|
52
58
|
* the maps while the other is still relying on it.
|
|
59
|
+
* @remarks This should only be used to remove *ALL* the entities from a single step from the store
|
|
60
|
+
* @see {@link https://github.com/JupiterOne/sdk/pull/944#discussion_r1303284408}
|
|
53
61
|
*/
|
|
54
|
-
flushEntities(entities: Entity[]): void;
|
|
62
|
+
flushEntities(entities: Entity[], stepId: string): void;
|
|
55
63
|
/**
|
|
56
64
|
* Instead of clearing the entire map, we want to delete old data individually.
|
|
57
65
|
* This will allow us to eventually run steps in parallel without locking when
|
|
58
66
|
* steps have no interdependence. If we cleared it out, it's possible that
|
|
59
67
|
* two steps could be running in parallel and one of the steps may clear out
|
|
60
68
|
* the maps while the other is still relying on it.
|
|
69
|
+
* @remarks This should only be used to remove *ALL* the relationships from a single step from the store
|
|
70
|
+
* @see {@link https://github.com/JupiterOne/sdk/pull/944#discussion_r1303284408}
|
|
61
71
|
*/
|
|
62
|
-
flushRelationships(relationships: Relationship[]): void;
|
|
72
|
+
flushRelationships(relationships: Relationship[], stepId: string): void;
|
|
63
73
|
collectEntities(): Entity[];
|
|
64
74
|
collectRelationships(): Relationship[];
|
|
65
75
|
collectEntitiesByStep(): Map<string, Entity[]>;
|
|
66
76
|
collectRelationshipsByStep(): Map<string, Relationship[]>;
|
|
67
77
|
getTotalEntityItemCount(): number;
|
|
68
78
|
getTotalRelationshipItemCount(): number;
|
|
79
|
+
getTotalEntitySizeInBytes(): number;
|
|
80
|
+
getTotalRelationshipSizeInBytes(): number;
|
|
69
81
|
}
|
|
@@ -2,41 +2,46 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.InMemoryGraphObjectStore = void 0;
|
|
4
4
|
const integration_sdk_core_1 = require("@jupiterone/integration-sdk-core");
|
|
5
|
+
const batchBySize_1 = require("../synchronization/batchBySize");
|
|
5
6
|
/**
|
|
6
7
|
* Stores entities and relationships in memory and may be used to buffer data in
|
|
7
8
|
* memory before flushing to another datastore (e.g. disk).
|
|
8
9
|
*/
|
|
9
10
|
class InMemoryGraphObjectStore {
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
11
|
+
/**
|
|
12
|
+
* Maps to lookup entity/relationship with metadata by _key
|
|
13
|
+
*
|
|
14
|
+
* {
|
|
15
|
+
* "my_user_key1": { stepId: 'fetch-users', entity: { ...ENTITY1 } },
|
|
16
|
+
* "my_user_key2": { stepId: 'fetch-users', entity: { ...ENTITY2 } }
|
|
17
|
+
* }
|
|
18
|
+
*/
|
|
19
|
+
entityKeyToEntityMap = new Map();
|
|
20
|
+
relationshipKeyToRelationshipMap = new Map();
|
|
21
|
+
/**
|
|
22
|
+
* Store the size in bytes for entities and relationships generated by each step.
|
|
23
|
+
*
|
|
24
|
+
*/
|
|
25
|
+
entitySizeInBytes = {};
|
|
26
|
+
relationshipSizeInBytes = {};
|
|
27
|
+
/**
|
|
28
|
+
* Maps to lookup all entity/relationship _key's by _type. The value of this
|
|
29
|
+
* data structure is Map<_key{string}, boolean> to make deleting individual
|
|
30
|
+
* keys super fast. See "flushEntities" for more information.
|
|
31
|
+
*
|
|
32
|
+
* {
|
|
33
|
+
* 'my_token_type': {
|
|
34
|
+
* 'my_token_key1': true,
|
|
35
|
+
* 'my_token_key2': true
|
|
36
|
+
* },
|
|
37
|
+
* 'my_user_type': {
|
|
38
|
+
* 'my_user_key1': true,
|
|
39
|
+
* 'my_user_key2': true
|
|
40
|
+
* }
|
|
41
|
+
* }
|
|
42
|
+
*/
|
|
43
|
+
entityTypeToKeysMap = new Map();
|
|
44
|
+
relationshipTypeToKeysMap = new Map();
|
|
40
45
|
addEntities(stepId, newEntities) {
|
|
41
46
|
for (const entity of newEntities) {
|
|
42
47
|
this.entityKeyToEntityMap.set(entity._key, {
|
|
@@ -51,6 +56,9 @@ class InMemoryGraphObjectStore {
|
|
|
51
56
|
this.entityTypeToKeysMap.set(entity._type, new Map([[entity._key, true]]));
|
|
52
57
|
}
|
|
53
58
|
}
|
|
59
|
+
this.entitySizeInBytes[stepId] = this.entitySizeInBytes[stepId]
|
|
60
|
+
? this.entitySizeInBytes[stepId] + (0, batchBySize_1.getSizeOfObject)(newEntities)
|
|
61
|
+
: (0, batchBySize_1.getSizeOfObject)(newEntities);
|
|
54
62
|
return Promise.resolve();
|
|
55
63
|
}
|
|
56
64
|
addRelationships(stepId, newRelationships) {
|
|
@@ -67,6 +75,9 @@ class InMemoryGraphObjectStore {
|
|
|
67
75
|
this.relationshipTypeToKeysMap.set(relationship._type, new Map([[relationship._key, true]]));
|
|
68
76
|
}
|
|
69
77
|
}
|
|
78
|
+
this.relationshipSizeInBytes[stepId] = this.relationshipSizeInBytes[stepId]
|
|
79
|
+
? this.relationshipSizeInBytes[stepId] + (0, batchBySize_1.getSizeOfObject)(newRelationships)
|
|
80
|
+
: (0, batchBySize_1.getSizeOfObject)(newRelationships);
|
|
70
81
|
return Promise.resolve();
|
|
71
82
|
}
|
|
72
83
|
findEntity(_key) {
|
|
@@ -120,8 +131,10 @@ class InMemoryGraphObjectStore {
|
|
|
120
131
|
* steps have no interdependence. If we cleared it out, it's possible that
|
|
121
132
|
* two steps could be running in parallel and one of the steps may clear out
|
|
122
133
|
* the maps while the other is still relying on it.
|
|
134
|
+
* @remarks This should only be used to remove *ALL* the entities from a single step from the store
|
|
135
|
+
* @see {@link https://github.com/JupiterOne/sdk/pull/944#discussion_r1303284408}
|
|
123
136
|
*/
|
|
124
|
-
flushEntities(entities) {
|
|
137
|
+
flushEntities(entities, stepId) {
|
|
125
138
|
for (const entity of entities) {
|
|
126
139
|
this.entityKeyToEntityMap.delete(entity._key);
|
|
127
140
|
const entityTypeKeysMap = this.entityTypeToKeysMap.get(entity._type);
|
|
@@ -132,6 +145,7 @@ class InMemoryGraphObjectStore {
|
|
|
132
145
|
}
|
|
133
146
|
entityTypeKeysMap.delete(entity._key);
|
|
134
147
|
}
|
|
148
|
+
delete this.entitySizeInBytes[stepId];
|
|
135
149
|
}
|
|
136
150
|
/**
|
|
137
151
|
* Instead of clearing the entire map, we want to delete old data individually.
|
|
@@ -139,8 +153,10 @@ class InMemoryGraphObjectStore {
|
|
|
139
153
|
* steps have no interdependence. If we cleared it out, it's possible that
|
|
140
154
|
* two steps could be running in parallel and one of the steps may clear out
|
|
141
155
|
* the maps while the other is still relying on it.
|
|
156
|
+
* @remarks This should only be used to remove *ALL* the relationships from a single step from the store
|
|
157
|
+
* @see {@link https://github.com/JupiterOne/sdk/pull/944#discussion_r1303284408}
|
|
142
158
|
*/
|
|
143
|
-
flushRelationships(relationships) {
|
|
159
|
+
flushRelationships(relationships, stepId) {
|
|
144
160
|
for (const relationship of relationships) {
|
|
145
161
|
this.relationshipKeyToRelationshipMap.delete(relationship._key);
|
|
146
162
|
const relationshipTypeKeysMap = this.relationshipTypeToKeysMap.get(relationship._type);
|
|
@@ -151,6 +167,7 @@ class InMemoryGraphObjectStore {
|
|
|
151
167
|
}
|
|
152
168
|
relationshipTypeKeysMap.delete(relationship._key);
|
|
153
169
|
}
|
|
170
|
+
delete this.relationshipSizeInBytes[stepId];
|
|
154
171
|
}
|
|
155
172
|
collectEntities() {
|
|
156
173
|
const entities = [];
|
|
@@ -202,6 +219,12 @@ class InMemoryGraphObjectStore {
|
|
|
202
219
|
getTotalRelationshipItemCount() {
|
|
203
220
|
return this.relationshipKeyToRelationshipMap.size;
|
|
204
221
|
}
|
|
222
|
+
getTotalEntitySizeInBytes() {
|
|
223
|
+
return Object.values(this.entitySizeInBytes).reduce((partialSum, stepBytes) => partialSum + stepBytes, 0);
|
|
224
|
+
}
|
|
225
|
+
getTotalRelationshipSizeInBytes() {
|
|
226
|
+
return Object.values(this.relationshipSizeInBytes).reduce((partialSum, stepBytes) => partialSum + stepBytes, 0);
|
|
227
|
+
}
|
|
205
228
|
}
|
|
206
229
|
exports.InMemoryGraphObjectStore = InMemoryGraphObjectStore;
|
|
207
230
|
//# sourceMappingURL=memory.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory.js","sourceRoot":"","sources":["../../../src/storage/memory.ts"],"names":[],"mappings":";;;AAAA,2EAQ0C;
|
|
1
|
+
{"version":3,"file":"memory.js","sourceRoot":"","sources":["../../../src/storage/memory.ts"],"names":[],"mappings":";;;AAAA,2EAQ0C;AAC1C,gEAAiE;AAcjE;;;GAGG;AACH,MAAa,wBAAwB;IACnC;;;;;;;OAOG;IACc,oBAAoB,GAAG,IAAI,GAAG,EAG5C,CAAC;IACa,gCAAgC,GAAG,IAAI,GAAG,EAGxD,CAAC;IAEJ;;;OAGG;IACK,iBAAiB,GAA+B,EAAE,CAAC;IACnD,uBAAuB,GAA+B,EAAE,CAAC;IACjE;;;;;;;;;;;;;;;OAeG;IACc,mBAAmB,GAClC,IAAI,GAAG,EAAE,CAAC;IACK,yBAAyB,GAGtC,IAAI,GAAG,EAAE,CAAC;IAEd,WAAW,CAAC,MAAc,EAAE,WAAqB;QAC/C,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE;YAChC,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE;gBACzC,MAAM;gBACN,MAAM;aACP,CAAC,CAAC;YAEH,MAAM,iBAAiB,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACrE,IAAI,iBAAiB,EAAE;gBACrB,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;aAC1C;iBAAM;gBACL,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAC1B,MAAM,CAAC,KAAK,EACZ,IAAI,GAAG,CAAkB,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAChD,CAAC;aACH;SACF;QACD,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC;YAC7D,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,GAAG,IAAA,6BAAe,EAAC,WAAW,CAAC;YAC/D,CAAC,CAAC,IAAA,6BAAe,EAAC,WAAW,CAAC,CAAC;QACjC,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAED,gBAAgB,CACd,MAAc,EACd,gBAAgC;QAEhC,KAAK,MAAM,YAAY,IAAI,gBAAgB,EAAE;YAC3C,IAAI,CAAC,gCAAgC,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,EAAE;gBAC3D,MAAM;gBACN,YAAY;aACb,CAAC,CAAC;YAEH,MAAM,uBAAuB,GAAG,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAChE,YAAY,CAAC,KAAK,CACnB,CAAC;YAEF,IAAI,uBAAuB,EAAE;gBAC3B,uBAAuB,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;aACtD;iBAAM;gBACL,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAChC,YAAY,CAAC,KAAK,EAClB,IAAI,GAAG,CAAkB,CAAC,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CACtD,CAAC;aACH;SACF;QACD,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC;YACzE,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,GAAG,IAAA,6BAAe,EAAC,gBAAgB,CAAC;YAC1E,CAAC,CAAC,IAAA,6BAAe,EAAC,gBAAgB,CAAC,CAAC;QACtC,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAED,UAAU,CAAC,IAAwB;QACjC,MAAM,MAAM,GAAG,IAAI;YACjB,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM;YAC7C,CAAC,CAAC,SAAS,CAAC;QACd,OAAO,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,KAAK;QACH,MAAM,IAAI,uCAAgB,CAAC;YACzB,OAAO,EACL,mIAAmI;YACrI,IAAI,EAAE,uBAAuB;SAC9B,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,eAAe,CACnB,MAAyB,EACzB,QAAgC;QAEhC,MAAM,iBAAiB,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAErE,IAAI,CAAC,iBAAiB,EAAE;YACtB,OAAO;SACR;QAED,KAAK,MAAM,CAAC,IAAI,CAAC,IAAI,iBAAiB,EAAE;YACtC,MAAM,eAAe,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAE5D,IAAI,CAAC,eAAe,EAAE;gBACpB,qEAAqE;gBACrE,QAAQ;gBACR,MAAM,IAAI,iDAA0B,CAClC,gCAAgC,MAAM,CAAC,KAAK,UAAU,IAAI,GAAG,CAC9D,CAAC;aACH;YAED,MAAM,QAAQ,CAAC,eAAe,CAAC,MAAW,CAAC,CAAC;SAC7C;IACH,CAAC;IAED,KAAK,CAAC,oBAAoB,CACxB,MAAyB,EACzB,QAAgC;QAEhC,MAAM,uBAAuB,GAAG,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAChE,MAAM,CAAC,KAAK,CACb,CAAC;QAEF,IAAI,CAAC,uBAAuB,EAAE;YAC5B,OAAO;SACR;QAED,KAAK,MAAM,CAAC,IAAI,CAAC,IAAI,uBAAuB,EAAE;YAC5C,MAAM,eAAe,GAAG,IAAI,CAAC,gCAAgC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAExE,IAAI,CAAC,eAAe,EAAE;gBACpB,qEAAqE;gBACrE,QAAQ;gBACR,MAAM,IAAI,iDAA0B,CAClC,sCAAsC,MAAM,CAAC,KAAK,UAAU,IAAI,GAAG,CACpE,CAAC;aACH;YAED,MAAM,QAAQ,CAAC,eAAe,CAAC,YAAiB,CAAC,CAAC;SACnD;IACH,CAAC;IAED;;;;;;;;OAQG;IACH,aAAa,CAAC,QAAkB,EAAE,MAAc;QAC9C,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE;YAC7B,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,iBAAiB,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAErE,IAAI,CAAC,iBAAiB,EAAE;gBACtB,oEAAoE;gBACpE,sCAAsC;gBACtC,MAAM,IAAI,KAAK,CACb,oDAAoD,MAAM,CAAC,IAAI,WAAW,MAAM,CAAC,KAAK,GAAG,CAC1F,CAAC;aACH;YAED,iBAAiB,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;SACvC;QACD,OAAO,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC;IAED;;;;;;;;OAQG;IACH,kBAAkB,CAAC,aAA6B,EAAE,MAAc;QAC9D,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE;YACxC,IAAI,CAAC,gCAAgC,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAChE,MAAM,uBAAuB,GAAG,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAChE,YAAY,CAAC,KAAK,CACnB,CAAC;YAEF,IAAI,CAAC,uBAAuB,EAAE;gBAC5B,oEAAoE;gBACpE,sCAAsC;gBACtC,MAAM,IAAI,KAAK,CACb,0DAA0D,YAAY,CAAC,IAAI,WAAW,YAAY,CAAC,KAAK,GAAG,CAC5G,CAAC;aACH;YAED,uBAAuB,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;SACnD;QACD,OAAO,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC;IAC9C,CAAC;IAED,eAAe;QACb,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,IAAI,IAAI,CAAC,oBAAoB,EAAE;YAC/D,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;SACvC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,oBAAoB;QAClB,MAAM,aAAa,GAAmB,EAAE,CAAC;QACzC,KAAK,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,IAAI,IAAI;aACvC,gCAAgC,EAAE;YACnC,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;SAClD;QACD,OAAO,aAAa,CAAC;IACvB,CAAC;IAED,qBAAqB;QACnB,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAoB,CAAC;QAEtD,KAAK,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,IAAI,IAAI,CAAC,oBAAoB,EAAE;YAC/D,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,eAAe,CAAC;YAE3C,MAAM,mBAAmB,GAAG,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAC1D,IAAI,mBAAmB,EAAE;gBACvB,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aAClC;iBAAM;gBACL,iBAAiB,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;aACzC;SACF;QAED,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAED,0BAA0B;QACxB,MAAM,sBAAsB,GAAG,IAAI,GAAG,EAA0B,CAAC;QAEjE,KAAK,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,IAAI,IAAI;aACvC,gCAAgC,EAAE;YACnC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,GAAG,eAAe,CAAC;YAEjD,MAAM,wBAAwB,GAAG,sBAAsB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACpE,IAAI,wBAAwB,EAAE;gBAC5B,wBAAwB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;aAC7C;iBAAM;gBACL,sBAAsB,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC;aACpD;SACF;QAED,OAAO,sBAAsB,CAAC;IAChC,CAAC;IAED,uBAAuB;QACrB,OAAO,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC;IACxC,CAAC;IAED,6BAA6B;QAC3B,OAAO,IAAI,CAAC,gCAAgC,CAAC,IAAI,CAAC;IACpD,CAAC;IAED,yBAAyB;QACvB,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,MAAM,CACjD,CAAC,UAAU,EAAE,SAAS,EAAE,EAAE,CAAC,UAAU,GAAG,SAAS,EACjD,CAAC,CACF,CAAC;IACJ,CAAC;IAED,+BAA+B;QAC7B,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC,MAAM,CACvD,CAAC,UAAU,EAAE,SAAS,EAAE,EAAE,CAAC,UAAU,GAAG,SAAS,EACjD,CAAC,CACF,CAAC;IACJ,CAAC;CACF;AA1SD,4DA0SC"}
|
|
@@ -7,8 +7,9 @@ exports.createEventPublishingQueue = void 0;
|
|
|
7
7
|
const p_queue_1 = __importDefault(require("p-queue"));
|
|
8
8
|
const synchronization_1 = require("../synchronization");
|
|
9
9
|
const createEventPublishingQueue = ({ apiClient, logger, job }, config) => {
|
|
10
|
-
if (!job.integrationJobId)
|
|
10
|
+
if (!job.integrationJobId) {
|
|
11
11
|
return createNoopEventPublishingQueue();
|
|
12
|
+
}
|
|
12
13
|
const queue = new p_queue_1.default({ concurrency: 1 });
|
|
13
14
|
return {
|
|
14
15
|
enqueue(event) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"events.js","sourceRoot":"","sources":["../../../src/synchronization/events.ts"],"names":[],"mappings":";;;;;;AAEA,sDAAmC;AAEnC,wDAG4B;AAOrB,MAAM,0BAA0B,GAAG,CACxC,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,EAA6B,EACrD,MAA2B,EACL,EAAE;IACxB,IAAI,CAAC,GAAG,CAAC,gBAAgB;
|
|
1
|
+
{"version":3,"file":"events.js","sourceRoot":"","sources":["../../../src/synchronization/events.ts"],"names":[],"mappings":";;;;;;AAEA,sDAAmC;AAEnC,wDAG4B;AAOrB,MAAM,0BAA0B,GAAG,CACxC,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,EAA6B,EACrD,MAA2B,EACL,EAAE;IACxB,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE;QACzB,OAAO,8BAA8B,EAAE,CAAC;KACzC;IAED,MAAM,KAAK,GAAG,IAAI,iBAAY,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC;IAEnD,OAAO;QACL,OAAO,CAAC,KAAuB;YAC7B,KAAK,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;gBACxB,IAAI;oBACF,MAAM,SAAS,CAAC,IAAI,CAClB,mCAAmC,GAAG,CAAC,EAAE,SAAS,EAClD;wBACE,MAAM,EAAE;4BACN;gCACE,IAAI,EAAE,KAAK,CAAC,IAAI;gCAChB,WAAW,EAAE,KAAK,CAAC,WAAW;gCAC9B,iFAAiF;6BAClF;yBACF;qBACF,EACD,MAAM,CACP,CAAC;iBACH;gBAAC,OAAO,GAAG,EAAE;oBACZ,MAAM,uBAAuB,GAAG,IAAA,4CAA0B,EAAC,GAAG,CAAC,CAAC;oBAChE,MAAM,CAAC,KAAK,CACV,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,uBAAuB,EAAE,EAChD,qCAAqC,CACtC,CAAC;iBACH;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QACD,MAAM,EAAE,KAAK,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE;KACnC,CAAC;AACJ,CAAC,CAAC;AAtCW,QAAA,0BAA0B,8BAsCrC;AAEF;;GAEG;AACH,SAAS,8BAA8B;IACrC,OAAO;QACL,OAAO,CAAC,MAAM;YACZ,OAAO;QACT,CAAC;QACD,KAAK,CAAC,MAAM;YACV,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -141,7 +141,6 @@ async function finalizeSynchronization({ apiClient, job, logger, partialDatasets
|
|
|
141
141
|
}, 'Error occurred while finalizing synchronization job');
|
|
142
142
|
throw (0, error_1.synchronizationApiError)(err, 'Error occurred while finalizing synchronization job.');
|
|
143
143
|
}
|
|
144
|
-
logger.info({ synchronizationJob: finalizedJob }, 'Synchronization finalization result.');
|
|
145
144
|
return finalizedJob;
|
|
146
145
|
}
|
|
147
146
|
exports.finalizeSynchronization = finalizeSynchronization;
|
|
@@ -270,13 +269,15 @@ async function uploadDataChunk({ logger, apiClient, jobId, type, batch }) {
|
|
|
270
269
|
[RequestHeaders.CorrelationId]: uploadCorrelationId,
|
|
271
270
|
},
|
|
272
271
|
});
|
|
273
|
-
|
|
272
|
+
const duration = Date.now() - startTime;
|
|
273
|
+
if (duration >= 10000) {
|
|
274
274
|
logger.info({
|
|
275
275
|
uploadCorrelationId,
|
|
276
276
|
uploadType: type,
|
|
277
277
|
attemptNum: ctx.attemptNum,
|
|
278
278
|
batchSize: batch.length,
|
|
279
279
|
batchSizeInBytes: (0, batchBySize_1.getSizeOfObject)(batch),
|
|
280
|
+
uploadDuration: duration,
|
|
280
281
|
}, 'Finished uploading big batch');
|
|
281
282
|
}
|
|
282
283
|
logger.debug({
|