@jupiterone/integration-sdk-runtime 10.3.0 → 10.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.d.ts +2 -4
- package/dist/src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.js +12 -16
- package/dist/src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.js.map +1 -1
- package/dist/src/storage/memory.d.ts +2 -14
- package/dist/src/storage/memory.js +2 -27
- package/dist/src/storage/memory.js.map +1 -1
- package/dist/tsconfig.dist.tsbuildinfo +1 -1
- package/package.json +4 -4
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { Entity, GraphObjectFilter, GraphObjectIteratee, Relationship, GraphObjectStore, GraphObjectIndexMetadata, GetIndexMetadataForGraphObjectTypeParams, IntegrationStep } from '@jupiterone/integration-sdk-core';
|
|
2
|
+
export declare const DEFAULT_GRAPH_OBJECT_BUFFER_THRESHOLD = 500;
|
|
2
3
|
export declare const DEFAULT_GRAPH_OBJECT_FILE_SIZE = 500;
|
|
3
|
-
export declare const DEFAULT_GRAPH_OBJECT_BUFFER_THRESHOLD_IN_BYTES = 5000000;
|
|
4
|
-
export declare const MAX_GRAPH_OBJECT_BUFFER_THRESHOLD_IN_BYTES = 1000000000;
|
|
5
4
|
export interface FileSystemGraphObjectStoreParams {
|
|
6
5
|
integrationSteps?: IntegrationStep[];
|
|
7
6
|
/**
|
|
@@ -15,7 +14,6 @@ export interface FileSystemGraphObjectStoreParams {
|
|
|
15
14
|
* this value up.
|
|
16
15
|
*
|
|
17
16
|
* Default: 500
|
|
18
|
-
* @deprecated this argument is no longer used. Please use `graphObjectBufferThresholdInBytes` instead.
|
|
19
17
|
*/
|
|
20
18
|
graphObjectBufferThreshold?: number;
|
|
21
19
|
/**
|
|
@@ -30,8 +28,8 @@ export interface FileSystemGraphObjectStoreParams {
|
|
|
30
28
|
export declare class FileSystemGraphObjectStore implements GraphObjectStore {
|
|
31
29
|
private readonly semaphore;
|
|
32
30
|
private readonly localGraphObjectStore;
|
|
31
|
+
private readonly graphObjectBufferThreshold;
|
|
33
32
|
private readonly graphObjectFileSize;
|
|
34
|
-
private readonly graphObjectBufferThresholdInBytes;
|
|
35
33
|
private readonly prettifyFiles;
|
|
36
34
|
private readonly stepIdToGraphObjectIndexMetadataMap;
|
|
37
35
|
private readonly entityOnDiskLocationMap;
|
|
@@ -3,19 +3,17 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.FileSystemGraphObjectStore = exports.
|
|
6
|
+
exports.FileSystemGraphObjectStore = exports.DEFAULT_GRAPH_OBJECT_FILE_SIZE = exports.DEFAULT_GRAPH_OBJECT_BUFFER_THRESHOLD = void 0;
|
|
7
7
|
const async_sema_1 = require("async-sema");
|
|
8
8
|
const p_map_1 = __importDefault(require("p-map"));
|
|
9
9
|
const flushDataToDisk_1 = require("./flushDataToDisk");
|
|
10
10
|
const indices_1 = require("./indices");
|
|
11
|
+
const memory_1 = require("../memory");
|
|
11
12
|
const fileSystem_1 = require("../../fileSystem");
|
|
12
13
|
const bigMap_1 = require("../../execution/utils/bigMap");
|
|
13
14
|
const lodash_1 = require("lodash");
|
|
14
|
-
|
|
15
|
+
exports.DEFAULT_GRAPH_OBJECT_BUFFER_THRESHOLD = 500;
|
|
15
16
|
exports.DEFAULT_GRAPH_OBJECT_FILE_SIZE = 500;
|
|
16
|
-
exports.DEFAULT_GRAPH_OBJECT_BUFFER_THRESHOLD_IN_BYTES = 5000000;
|
|
17
|
-
// no more than 10^9 bytes
|
|
18
|
-
exports.MAX_GRAPH_OBJECT_BUFFER_THRESHOLD_IN_BYTES = 1000000000;
|
|
19
17
|
// it is important that this value is set to 1
|
|
20
18
|
// to ensure that only one operation can be performed at a time.
|
|
21
19
|
const BINARY_SEMAPHORE_CONCURRENCY = 1;
|
|
@@ -49,13 +47,11 @@ class FileSystemGraphObjectStore {
|
|
|
49
47
|
this.localGraphObjectStore = new memory_1.InMemoryGraphObjectStore();
|
|
50
48
|
this.entityOnDiskLocationMap = new bigMap_1.BigMap(ENTITY_LOCATION_ON_DISK_DEFAULT_MAP_KEY_SPACE);
|
|
51
49
|
this.semaphore = new async_sema_1.Sema(BINARY_SEMAPHORE_CONCURRENCY);
|
|
50
|
+
this.graphObjectBufferThreshold =
|
|
51
|
+
params?.graphObjectBufferThreshold ||
|
|
52
|
+
exports.DEFAULT_GRAPH_OBJECT_BUFFER_THRESHOLD;
|
|
52
53
|
this.graphObjectFileSize =
|
|
53
54
|
params?.graphObjectFileSize || exports.DEFAULT_GRAPH_OBJECT_FILE_SIZE;
|
|
54
|
-
this.graphObjectBufferThresholdInBytes = (0, lodash_1.min)([
|
|
55
|
-
params?.graphObjectBufferThresholdInBytes ||
|
|
56
|
-
exports.DEFAULT_GRAPH_OBJECT_BUFFER_THRESHOLD_IN_BYTES,
|
|
57
|
-
exports.MAX_GRAPH_OBJECT_BUFFER_THRESHOLD_IN_BYTES,
|
|
58
|
-
]);
|
|
59
55
|
this.prettifyFiles = params?.prettifyFiles || false;
|
|
60
56
|
if (params?.integrationSteps) {
|
|
61
57
|
this.stepIdToGraphObjectIndexMetadataMap =
|
|
@@ -64,15 +60,15 @@ class FileSystemGraphObjectStore {
|
|
|
64
60
|
}
|
|
65
61
|
async addEntities(stepId, newEntities, onEntitiesFlushed) {
|
|
66
62
|
await this.localGraphObjectStore.addEntities(stepId, newEntities);
|
|
67
|
-
if (this.localGraphObjectStore.
|
|
68
|
-
this.
|
|
63
|
+
if (this.localGraphObjectStore.getTotalEntityItemCount() >=
|
|
64
|
+
this.graphObjectBufferThreshold) {
|
|
69
65
|
await this.flushEntitiesToDisk(onEntitiesFlushed);
|
|
70
66
|
}
|
|
71
67
|
}
|
|
72
68
|
async addRelationships(stepId, newRelationships, onRelationshipsFlushed) {
|
|
73
69
|
await this.localGraphObjectStore.addRelationships(stepId, newRelationships);
|
|
74
|
-
if (this.localGraphObjectStore.
|
|
75
|
-
this.
|
|
70
|
+
if (this.localGraphObjectStore.getTotalRelationshipItemCount() >=
|
|
71
|
+
this.graphObjectBufferThreshold) {
|
|
76
72
|
await this.flushRelationshipsToDisk(onRelationshipsFlushed);
|
|
77
73
|
}
|
|
78
74
|
}
|
|
@@ -148,7 +144,7 @@ class FileSystemGraphObjectStore {
|
|
|
148
144
|
}
|
|
149
145
|
}));
|
|
150
146
|
}
|
|
151
|
-
this.localGraphObjectStore.flushEntities(entities
|
|
147
|
+
this.localGraphObjectStore.flushEntities(entities);
|
|
152
148
|
if (onEntitiesFlushed) {
|
|
153
149
|
await onEntitiesFlushed(entities);
|
|
154
150
|
}
|
|
@@ -177,7 +173,7 @@ class FileSystemGraphObjectStore {
|
|
|
177
173
|
});
|
|
178
174
|
}));
|
|
179
175
|
}
|
|
180
|
-
this.localGraphObjectStore.flushRelationships(relationships
|
|
176
|
+
this.localGraphObjectStore.flushRelationships(relationships);
|
|
181
177
|
if (onRelationshipsFlushed) {
|
|
182
178
|
await onRelationshipsFlushed(relationships);
|
|
183
179
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FileSystemGraphObjectStore.js","sourceRoot":"","sources":["../../../../src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.ts"],"names":[],"mappings":";;;;;;AAAA,2CAAkC;AAClC,kDAAyB;AAazB,uDAAoD;AACpD,uCAImB;
|
|
1
|
+
{"version":3,"file":"FileSystemGraphObjectStore.js","sourceRoot":"","sources":["../../../../src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.ts"],"names":[],"mappings":";;;;;;AAAA,2CAAkC;AAClC,kDAAyB;AAazB,uDAAoD;AACpD,uCAImB;AACnB,sCAAqD;AAErD,iDAA8D;AAC9D,yDAAsD;AACtD,mCAA+B;AAElB,QAAA,qCAAqC,GAAG,GAAG,CAAC;AAC5C,QAAA,8BAA8B,GAAG,GAAG,CAAC;AAElD,8CAA8C;AAC9C,gEAAgE;AAChE,MAAM,4BAA4B,GAAG,CAAC,CAAC;AAwCvC;;GAEG;AACH,SAAS,6CAA6C,CACpD,gBAAmC;IAEnC,MAAM,mCAAmC,GAAG,IAAI,GAAG,EAGhD,CAAC;IAEJ,KAAK,MAAM,IAAI,IAAI,gBAAgB,EAAE;QACnC,MAAM,WAAW,GAAgC;YAC/C,QAAQ,EAAE,IAAI,GAAG,EAAE;YACnB,aAAa,EAAE,IAAI,GAAG,EAAE;SACzB,CAAC;QAEF,KAAK,MAAM,cAAc,IAAI,IAAI,CAAC,QAAQ,EAAE;YAC1C,IAAI,cAAc,CAAC,aAAa,EAAE;gBAChC,WAAW,CAAC,QAAQ,CAAC,GAAG,CACtB,cAAc,CAAC,KAAK,EACpB,cAAc,CAAC,aAAa,CAC7B,CAAC;aACH;SACF;QAED,KAAK,MAAM,oBAAoB,IAAI,IAAI,CAAC,aAAa,EAAE;YACrD,IAAI,oBAAoB,CAAC,aAAa,EAAE;gBACtC,WAAW,CAAC,aAAa,CAAC,GAAG,CAC3B,oBAAoB,CAAC,KAAK,EAC1B,oBAAoB,CAAC,aAAa,CACnC,CAAC;aACH;SACF;QAED,mCAAmC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;KAC/D;IAED,OAAO,mCAAmC,CAAC;AAC7C,CAAC;AAeD,MAAM,6CAA6C,GAAG,OAAO,CAAC;AAE9D,MAAa,0BAA0B;IAerC,YAAY,MAAyC;QAbpC,0BAAqB,GAAG,IAAI,iCAAwB,EAAE,CAAC;QAQvD,4BAAuB,GAAG,IAAI,eAAM,CAGnD,6CAA6C,CAAC,CAAC;QAG/C,IAAI,CAAC,SAAS,GAAG,IAAI,iBAAI,CAAC,4BAA4B,CAAC,CAAC;QACxD,IAAI,CAAC,0BAA0B;YAC7B,MAAM,EAAE,0BAA0B;gBAClC,6CAAqC,CAAC;QACxC,IAAI,CAAC,mBAAmB;YACtB,MAAM,EAAE,mBAAmB,IAAI,sCAA8B,CAAC;QAEhE,IAAI,CAAC,aAAa,GAAG,MAAM,EAAE,aAAa,IAAI,KAAK,CAAC;QAEpD,IAAI,MAAM,EAAE,gBAAgB,EAAE;YAC5B,IAAI,CAAC,mCAAmC;gBACtC,6CAA6C,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;SAC1E;IACH,CAAC;IAED,KAAK,CAAC,WAAW,CACf,MAAc,EACd,WAAqB,EACrB,iBAAyD;QAEzD,MAAM,IAAI,CAAC,qBAAqB,CAAC,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAElE,IACE,IAAI,CAAC,qBAAqB,CAAC,uBAAuB,EAAE;YACpD,IAAI,CAAC,0BAA0B,EAC/B;YACA,MAAM,IAAI,CAAC,mBAAmB,CAAC,iBAAiB,CAAC,CAAC;SACnD;IACH,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,MAAc,EACd,gBAAgC,EAChC,sBAAyE;QAEzE,MAAM,IAAI,CAAC,qBAAqB,CAAC,gBAAgB,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;QAE5E,IACE,IAAI,CAAC,qBAAqB,CAAC,6BAA6B,EAAE;YAC1D,IAAI,CAAC,0BAA0B,EAC/B;YACA,MAAM,IAAI,CAAC,wBAAwB,CAAC,sBAAsB,CAAC,CAAC;SAC7D;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,UAAU,CAAC,IAAwB;QACvC,IAAI,CAAC,IAAI;YAAE,OAAO;QAClB,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACzE,IAAI,cAAc,EAAE;YAClB,OAAO,cAAc,CAAC;SACvB;QAED,MAAM,oBAAoB,GAAG,IAAI,CAAC,uBAAuB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpE,IAAI,CAAC,oBAAoB;YAAE,OAAO;QAElC,MAAM,QAAQ,GAAG,IAAA,uCAA0B,EACzC,oBAAoB,CAAC,aAAa,CACnC,CAAC;QACF,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAA,6BAAmB,EAAoB;YAChE,QAAQ;SACT,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,eAAe,CACnB,MAAyB,EACzB,QAAgC;QAEhC,MAAM,IAAI,CAAC,qBAAqB,CAAC,eAAe,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAEnE,MAAM,IAAA,gCAAsB,EAAC;YAC3B,IAAI,EAAE,MAAM,CAAC,KAAK;YAClB,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,oBAAoB,CACxB,MAAyB,EACzB,QAAgC;QAEhC,MAAM,IAAI,CAAC,qBAAqB,CAAC,oBAAoB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAExE,MAAM,IAAA,sCAA4B,EAAC;YACjC,IAAI,EAAE,MAAM,CAAC,KAAK;YAClB,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CACT,iBAAyD,EACzD,sBAAyE;QAEzE,MAAM,OAAO,CAAC,GAAG,CAAC;YAChB,IAAI,CAAC,mBAAmB,CAAC,iBAAiB,CAAC;YAC3C,IAAI,CAAC,wBAAwB,CAAC,sBAAsB,CAAC;SACtD,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,mBAAmB,CACvB,iBAAyD;QAEzD,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,CAC5B,IAAA,eAAI,EACF,IAAI,CAAC,qBAAqB,CAAC,qBAAqB,EAAE,EAClD,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,EAAE;YAC3B,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;gBACtC,MAAM,aAAa,GAAG,IAAI,CAAC,kCAAkC,CAAC;oBAC5D,MAAM;oBACN,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,yBAAyB,EAAE,UAAU;iBACtC,CAAC,CAAC;gBAEH,IAAI,OAAO,aAAa,KAAK,WAAW,EAAE;oBACxC,OAAO,IAAI,CAAC;iBACb;gBAED,OAAO,aAAa,CAAC,OAAO,KAAK,IAAI,CAAC;YACxC,CAAC,CAAC,CAAC;YAEH,IAAI,SAAS,CAAC,MAAM,EAAE;gBACpB,MAAM,OAAO,CAAC,GAAG,CACf,IAAA,cAAK,EAAC,SAAS,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;oBAC5D,MAAM,uBAAuB,GAAG,MAAM,IAAA,iCAAe,EAAC;wBACpD,oBAAoB,EAAE,MAAM;wBAC5B,cAAc,EAAE,UAAU;wBAC1B,IAAI;wBACJ,MAAM,EAAE,IAAI,CAAC,aAAa;qBAC3B,CAAC,CAAC;oBAEH,KAAK,MAAM,EACT,aAAa,EACb,UAAU,GACX,IAAI,uBAAuB,EAAE;wBAC5B,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE;4BAC7C,IAAI,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE;gCACvC,aAAa;gCACb,KAAK;6BACN,CAAC,CAAC;yBACJ;qBACF;gBACH,CAAC,CAAC,CACH,CAAC;aACH;YAED,IAAI,CAAC,qBAAqB,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;YAEnD,IAAI,iBAAiB,EAAE;gBACrB,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAC;aACnC;QACH,CAAC,CACF,CACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,wBAAwB,CAC5B,sBAAyE;QAEzE,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,CAC5B,IAAA,eAAI,EACF,IAAI,CAAC,qBAAqB,CAAC,0BAA0B,EAAE,EACvD,KAAK,EAAE,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,EAAE;YAChC,MAAM,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;gBAC3C,MAAM,aAAa,GAAG,IAAI,CAAC,kCAAkC,CAAC;oBAC5D,MAAM;oBACN,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,yBAAyB,EAAE,eAAe;iBAC3C,CAAC,CAAC;gBAEH,IAAI,OAAO,aAAa,KAAK,WAAW,EAAE;oBACxC,OAAO,IAAI,CAAC;iBACb;gBAED,OAAO,aAAa,CAAC,OAAO,KAAK,IAAI,CAAC;YACxC,CAAC,CAAC,CAAC;YAEH,IAAI,SAAS,CAAC,MAAM,EAAE;gBACpB,MAAM,OAAO,CAAC,GAAG,CACf,IAAA,cAAK,EAAC,SAAS,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;oBAC5D,MAAM,IAAA,iCAAe,EAAC;wBACpB,oBAAoB,EAAE,MAAM;wBAC5B,cAAc,EAAE,eAAe;wBAC/B,IAAI;wBACJ,MAAM,EAAE,IAAI,CAAC,aAAa;qBAC3B,CAAC,CAAC;gBACL,CAAC,CAAC,CACH,CAAC;aACH;YAED,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,CAAC,aAAa,CAAC,CAAC;YAE7D,IAAI,sBAAsB,EAAE;gBAC1B,MAAM,sBAAsB,CAAC,aAAa,CAAC,CAAC;aAC7C;QACH,CAAC,CACF,CACF,CAAC;IACJ,CAAC;IAED,kCAAkC,CAAC,EACjC,MAAM,EACN,KAAK,EACL,yBAAyB,GACgB;QAGzC,IAAI,CAAC,IAAI,CAAC,mCAAmC,EAAE;YAC7C,OAAO,SAAS,CAAC;SAClB;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,mCAAmC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACjE,OAAO,GAAG,IAAI,GAAG,CAAC,yBAAyB,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC1D,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACK,KAAK,CAAC,aAAa,CAAI,SAA2B;QACxD,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QAC/B,IAAI;YACF,MAAM,SAAS,EAAE,CAAC;SACnB;gBAAS;YACR,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;SAC1B;IACH,CAAC;CACF;AAvQD,gEAuQC"}
|
|
@@ -7,12 +7,6 @@ export interface GraphObjectMetadata {
|
|
|
7
7
|
* memory before flushing to another datastore (e.g. disk).
|
|
8
8
|
*/
|
|
9
9
|
export declare class InMemoryGraphObjectStore implements GraphObjectStore {
|
|
10
|
-
/**
|
|
11
|
-
* Store the size in bytes for entities and relationships generated by each step.
|
|
12
|
-
*
|
|
13
|
-
*/
|
|
14
|
-
private entitySizeInBytes;
|
|
15
|
-
private relationshipSizeInBytes;
|
|
16
10
|
/**
|
|
17
11
|
* Maps to lookup entity/relationship with metadata by _key
|
|
18
12
|
*
|
|
@@ -56,26 +50,20 @@ export declare class InMemoryGraphObjectStore implements GraphObjectStore {
|
|
|
56
50
|
* steps have no interdependence. If we cleared it out, it's possible that
|
|
57
51
|
* two steps could be running in parallel and one of the steps may clear out
|
|
58
52
|
* the maps while the other is still relying on it.
|
|
59
|
-
* @remarks This should only be used to remove *ALL* the entities from a single step from the store
|
|
60
|
-
* @see {@link https://github.com/JupiterOne/sdk/pull/944#discussion_r1303284408}
|
|
61
53
|
*/
|
|
62
|
-
flushEntities(entities: Entity[]
|
|
54
|
+
flushEntities(entities: Entity[]): void;
|
|
63
55
|
/**
|
|
64
56
|
* Instead of clearing the entire map, we want to delete old data individually.
|
|
65
57
|
* This will allow us to eventually run steps in parallel without locking when
|
|
66
58
|
* steps have no interdependence. If we cleared it out, it's possible that
|
|
67
59
|
* two steps could be running in parallel and one of the steps may clear out
|
|
68
60
|
* the maps while the other is still relying on it.
|
|
69
|
-
* @remarks This should only be used to remove *ALL* the relationships from a single step from the store
|
|
70
|
-
* @see {@link https://github.com/JupiterOne/sdk/pull/944#discussion_r1303284408}
|
|
71
61
|
*/
|
|
72
|
-
flushRelationships(relationships: Relationship[]
|
|
62
|
+
flushRelationships(relationships: Relationship[]): void;
|
|
73
63
|
collectEntities(): Entity[];
|
|
74
64
|
collectRelationships(): Relationship[];
|
|
75
65
|
collectEntitiesByStep(): Map<string, Entity[]>;
|
|
76
66
|
collectRelationshipsByStep(): Map<string, Relationship[]>;
|
|
77
67
|
getTotalEntityItemCount(): number;
|
|
78
68
|
getTotalRelationshipItemCount(): number;
|
|
79
|
-
getTotalEntitySizeInBytes(): number;
|
|
80
|
-
getTotalRelationshipSizeInBytes(): number;
|
|
81
69
|
}
|
|
@@ -2,19 +2,12 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.InMemoryGraphObjectStore = void 0;
|
|
4
4
|
const integration_sdk_core_1 = require("@jupiterone/integration-sdk-core");
|
|
5
|
-
const batchBySize_1 = require("../synchronization/batchBySize");
|
|
6
5
|
/**
|
|
7
6
|
* Stores entities and relationships in memory and may be used to buffer data in
|
|
8
7
|
* memory before flushing to another datastore (e.g. disk).
|
|
9
8
|
*/
|
|
10
9
|
class InMemoryGraphObjectStore {
|
|
11
10
|
constructor() {
|
|
12
|
-
/**
|
|
13
|
-
* Store the size in bytes for entities and relationships generated by each step.
|
|
14
|
-
*
|
|
15
|
-
*/
|
|
16
|
-
this.entitySizeInBytes = {};
|
|
17
|
-
this.relationshipSizeInBytes = {};
|
|
18
11
|
/**
|
|
19
12
|
* Maps to lookup entity/relationship with metadata by _key
|
|
20
13
|
*
|
|
@@ -58,9 +51,6 @@ class InMemoryGraphObjectStore {
|
|
|
58
51
|
this.entityTypeToKeysMap.set(entity._type, new Map([[entity._key, true]]));
|
|
59
52
|
}
|
|
60
53
|
}
|
|
61
|
-
this.entitySizeInBytes[stepId] = this.entitySizeInBytes[stepId]
|
|
62
|
-
? this.entitySizeInBytes[stepId] + (0, batchBySize_1.getSizeOfObject)(newEntities)
|
|
63
|
-
: (0, batchBySize_1.getSizeOfObject)(newEntities);
|
|
64
54
|
return Promise.resolve();
|
|
65
55
|
}
|
|
66
56
|
addRelationships(stepId, newRelationships) {
|
|
@@ -77,9 +67,6 @@ class InMemoryGraphObjectStore {
|
|
|
77
67
|
this.relationshipTypeToKeysMap.set(relationship._type, new Map([[relationship._key, true]]));
|
|
78
68
|
}
|
|
79
69
|
}
|
|
80
|
-
this.relationshipSizeInBytes[stepId] = this.relationshipSizeInBytes[stepId]
|
|
81
|
-
? this.relationshipSizeInBytes[stepId] + (0, batchBySize_1.getSizeOfObject)(newRelationships)
|
|
82
|
-
: (0, batchBySize_1.getSizeOfObject)(newRelationships);
|
|
83
70
|
return Promise.resolve();
|
|
84
71
|
}
|
|
85
72
|
findEntity(_key) {
|
|
@@ -133,10 +120,8 @@ class InMemoryGraphObjectStore {
|
|
|
133
120
|
* steps have no interdependence. If we cleared it out, it's possible that
|
|
134
121
|
* two steps could be running in parallel and one of the steps may clear out
|
|
135
122
|
* the maps while the other is still relying on it.
|
|
136
|
-
* @remarks This should only be used to remove *ALL* the entities from a single step from the store
|
|
137
|
-
* @see {@link https://github.com/JupiterOne/sdk/pull/944#discussion_r1303284408}
|
|
138
123
|
*/
|
|
139
|
-
flushEntities(entities
|
|
124
|
+
flushEntities(entities) {
|
|
140
125
|
for (const entity of entities) {
|
|
141
126
|
this.entityKeyToEntityMap.delete(entity._key);
|
|
142
127
|
const entityTypeKeysMap = this.entityTypeToKeysMap.get(entity._type);
|
|
@@ -147,7 +132,6 @@ class InMemoryGraphObjectStore {
|
|
|
147
132
|
}
|
|
148
133
|
entityTypeKeysMap.delete(entity._key);
|
|
149
134
|
}
|
|
150
|
-
delete this.entitySizeInBytes[stepId];
|
|
151
135
|
}
|
|
152
136
|
/**
|
|
153
137
|
* Instead of clearing the entire map, we want to delete old data individually.
|
|
@@ -155,10 +139,8 @@ class InMemoryGraphObjectStore {
|
|
|
155
139
|
* steps have no interdependence. If we cleared it out, it's possible that
|
|
156
140
|
* two steps could be running in parallel and one of the steps may clear out
|
|
157
141
|
* the maps while the other is still relying on it.
|
|
158
|
-
* @remarks This should only be used to remove *ALL* the relationships from a single step from the store
|
|
159
|
-
* @see {@link https://github.com/JupiterOne/sdk/pull/944#discussion_r1303284408}
|
|
160
142
|
*/
|
|
161
|
-
flushRelationships(relationships
|
|
143
|
+
flushRelationships(relationships) {
|
|
162
144
|
for (const relationship of relationships) {
|
|
163
145
|
this.relationshipKeyToRelationshipMap.delete(relationship._key);
|
|
164
146
|
const relationshipTypeKeysMap = this.relationshipTypeToKeysMap.get(relationship._type);
|
|
@@ -169,7 +151,6 @@ class InMemoryGraphObjectStore {
|
|
|
169
151
|
}
|
|
170
152
|
relationshipTypeKeysMap.delete(relationship._key);
|
|
171
153
|
}
|
|
172
|
-
delete this.relationshipSizeInBytes[stepId];
|
|
173
154
|
}
|
|
174
155
|
collectEntities() {
|
|
175
156
|
const entities = [];
|
|
@@ -221,12 +202,6 @@ class InMemoryGraphObjectStore {
|
|
|
221
202
|
getTotalRelationshipItemCount() {
|
|
222
203
|
return this.relationshipKeyToRelationshipMap.size;
|
|
223
204
|
}
|
|
224
|
-
getTotalEntitySizeInBytes() {
|
|
225
|
-
return Object.values(this.entitySizeInBytes).reduce((partialSum, stepBytes) => partialSum + stepBytes, 0);
|
|
226
|
-
}
|
|
227
|
-
getTotalRelationshipSizeInBytes() {
|
|
228
|
-
return Object.values(this.relationshipSizeInBytes).reduce((partialSum, stepBytes) => partialSum + stepBytes, 0);
|
|
229
|
-
}
|
|
230
205
|
}
|
|
231
206
|
exports.InMemoryGraphObjectStore = InMemoryGraphObjectStore;
|
|
232
207
|
//# sourceMappingURL=memory.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory.js","sourceRoot":"","sources":["../../../src/storage/memory.ts"],"names":[],"mappings":";;;AAAA,2EAQ0C;
|
|
1
|
+
{"version":3,"file":"memory.js","sourceRoot":"","sources":["../../../src/storage/memory.ts"],"names":[],"mappings":";;;AAAA,2EAQ0C;AAc1C;;;GAGG;AACH,MAAa,wBAAwB;IAArC;QACE;;;;;;;WAOG;QACc,yBAAoB,GAAG,IAAI,GAAG,EAG5C,CAAC;QACa,qCAAgC,GAAG,IAAI,GAAG,EAGxD,CAAC;QAEJ;;;;;;;;;;;;;;;WAeG;QACc,wBAAmB,GAClC,IAAI,GAAG,EAAE,CAAC;QACK,8BAAyB,GAGtC,IAAI,GAAG,EAAE,CAAC;IAmOhB,CAAC;IAjOC,WAAW,CAAC,MAAc,EAAE,WAAqB;QAC/C,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE;YAChC,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE;gBACzC,MAAM;gBACN,MAAM;aACP,CAAC,CAAC;YAEH,MAAM,iBAAiB,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACrE,IAAI,iBAAiB,EAAE;gBACrB,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;aAC1C;iBAAM;gBACL,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAC1B,MAAM,CAAC,KAAK,EACZ,IAAI,GAAG,CAAkB,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAChD,CAAC;aACH;SACF;QACD,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAED,gBAAgB,CACd,MAAc,EACd,gBAAgC;QAEhC,KAAK,MAAM,YAAY,IAAI,gBAAgB,EAAE;YAC3C,IAAI,CAAC,gCAAgC,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,EAAE;gBAC3D,MAAM;gBACN,YAAY;aACb,CAAC,CAAC;YAEH,MAAM,uBAAuB,GAAG,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAChE,YAAY,CAAC,KAAK,CACnB,CAAC;YAEF,IAAI,uBAAuB,EAAE;gBAC3B,uBAAuB,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;aACtD;iBAAM;gBACL,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAChC,YAAY,CAAC,KAAK,EAClB,IAAI,GAAG,CAAkB,CAAC,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CACtD,CAAC;aACH;SACF;QACD,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAED,UAAU,CAAC,IAAwB;QACjC,MAAM,MAAM,GAAG,IAAI;YACjB,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM;YAC7C,CAAC,CAAC,SAAS,CAAC;QACd,OAAO,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,KAAK;QACH,MAAM,IAAI,uCAAgB,CAAC;YACzB,OAAO,EACL,mIAAmI;YACrI,IAAI,EAAE,uBAAuB;SAC9B,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,eAAe,CACnB,MAAyB,EACzB,QAAgC;QAEhC,MAAM,iBAAiB,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAErE,IAAI,CAAC,iBAAiB,EAAE;YACtB,OAAO;SACR;QAED,KAAK,MAAM,CAAC,IAAI,CAAC,IAAI,iBAAiB,EAAE;YACtC,MAAM,eAAe,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAE5D,IAAI,CAAC,eAAe,EAAE;gBACpB,qEAAqE;gBACrE,QAAQ;gBACR,MAAM,IAAI,iDAA0B,CAClC,gCAAgC,MAAM,CAAC,KAAK,UAAU,IAAI,GAAG,CAC9D,CAAC;aACH;YAED,MAAM,QAAQ,CAAC,eAAe,CAAC,MAAW,CAAC,CAAC;SAC7C;IACH,CAAC;IAED,KAAK,CAAC,oBAAoB,CACxB,MAAyB,EACzB,QAAgC;QAEhC,MAAM,uBAAuB,GAAG,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAChE,MAAM,CAAC,KAAK,CACb,CAAC;QAEF,IAAI,CAAC,uBAAuB,EAAE;YAC5B,OAAO;SACR;QAED,KAAK,MAAM,CAAC,IAAI,CAAC,IAAI,uBAAuB,EAAE;YAC5C,MAAM,eAAe,GAAG,IAAI,CAAC,gCAAgC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAExE,IAAI,CAAC,eAAe,EAAE;gBACpB,qEAAqE;gBACrE,QAAQ;gBACR,MAAM,IAAI,iDAA0B,CAClC,sCAAsC,MAAM,CAAC,KAAK,UAAU,IAAI,GAAG,CACpE,CAAC;aACH;YAED,MAAM,QAAQ,CAAC,eAAe,CAAC,YAAiB,CAAC,CAAC;SACnD;IACH,CAAC;IAED;;;;;;OAMG;IACH,aAAa,CAAC,QAAkB;QAC9B,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE;YAC7B,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,iBAAiB,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAErE,IAAI,CAAC,iBAAiB,EAAE;gBACtB,oEAAoE;gBACpE,sCAAsC;gBACtC,MAAM,IAAI,KAAK,CACb,oDAAoD,MAAM,CAAC,IAAI,WAAW,MAAM,CAAC,KAAK,GAAG,CAC1F,CAAC;aACH;YAED,iBAAiB,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;SACvC;IACH,CAAC;IAED;;;;;;OAMG;IACH,kBAAkB,CAAC,aAA6B;QAC9C,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE;YACxC,IAAI,CAAC,gCAAgC,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAChE,MAAM,uBAAuB,GAAG,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAChE,YAAY,CAAC,KAAK,CACnB,CAAC;YAEF,IAAI,CAAC,uBAAuB,EAAE;gBAC5B,oEAAoE;gBACpE,sCAAsC;gBACtC,MAAM,IAAI,KAAK,CACb,0DAA0D,YAAY,CAAC,IAAI,WAAW,YAAY,CAAC,KAAK,GAAG,CAC5G,CAAC;aACH;YAED,uBAAuB,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;SACnD;IACH,CAAC;IAED,eAAe;QACb,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,IAAI,IAAI,CAAC,oBAAoB,EAAE;YAC/D,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;SACvC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,oBAAoB;QAClB,MAAM,aAAa,GAAmB,EAAE,CAAC;QACzC,KAAK,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,IAAI,IAAI;aACvC,gCAAgC,EAAE;YACnC,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;SAClD;QACD,OAAO,aAAa,CAAC;IACvB,CAAC;IAED,qBAAqB;QACnB,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAoB,CAAC;QAEtD,KAAK,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,IAAI,IAAI,CAAC,oBAAoB,EAAE;YAC/D,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,eAAe,CAAC;YAE3C,MAAM,mBAAmB,GAAG,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAC1D,IAAI,mBAAmB,EAAE;gBACvB,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aAClC;iBAAM;gBACL,iBAAiB,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;aACzC;SACF;QAED,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAED,0BAA0B;QACxB,MAAM,sBAAsB,GAAG,IAAI,GAAG,EAA0B,CAAC;QAEjE,KAAK,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,IAAI,IAAI;aACvC,gCAAgC,EAAE;YACnC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,GAAG,eAAe,CAAC;YAEjD,MAAM,wBAAwB,GAAG,sBAAsB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACpE,IAAI,wBAAwB,EAAE;gBAC5B,wBAAwB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;aAC7C;iBAAM;gBACL,sBAAsB,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC;aACpD;SACF;QAED,OAAO,sBAAsB,CAAC;IAChC,CAAC;IAED,uBAAuB;QACrB,OAAO,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC;IACxC,CAAC;IAED,6BAA6B;QAC3B,OAAO,IAAI,CAAC,gCAAgC,CAAC,IAAI,CAAC;IACpD,CAAC;CACF;AA1QD,4DA0QC"}
|