@jupiterone/integration-sdk-runtime 17.0.2 → 17.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/api/index.d.ts +2 -1
- package/dist/src/api/index.js +24 -2
- package/dist/src/api/index.js.map +1 -1
- package/dist/src/execution/executeIntegration.js +3 -1
- package/dist/src/execution/executeIntegration.js.map +1 -1
- package/dist/src/execution/uploader.d.ts +3 -1
- package/dist/src/execution/uploader.js +8 -1
- package/dist/src/execution/uploader.js.map +1 -1
- package/dist/src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.d.ts +6 -1
- package/dist/src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.js +97 -25
- package/dist/src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.js.map +1 -1
- package/dist/src/storage/FileSystemGraphObjectStore/flushDataToDisk.d.ts +2 -1
- package/dist/src/storage/FileSystemGraphObjectStore/flushDataToDisk.js +54 -17
- package/dist/src/storage/FileSystemGraphObjectStore/flushDataToDisk.js.map +1 -1
- package/dist/src/synchronization/index.js +2 -25
- package/dist/src/synchronization/index.js.map +1 -1
- package/dist/tsconfig.dist.tsbuildinfo +1 -1
- package/package.json +4 -4
package/dist/src/api/index.d.ts
CHANGED
|
@@ -7,6 +7,7 @@ interface CreateApiClientInput {
|
|
|
7
7
|
retryOptions?: RetryOptions;
|
|
8
8
|
compressUploads?: boolean;
|
|
9
9
|
alphaOptions?: AlphaOptions;
|
|
10
|
+
proxyUrl?: string;
|
|
10
11
|
}
|
|
11
12
|
interface RetryOptions {
|
|
12
13
|
attempts?: number;
|
|
@@ -23,7 +24,7 @@ interface RetryOptions {
|
|
|
23
24
|
* In managed environments, this can be configured to
|
|
24
25
|
* hit public and private JupiterOne APIs.
|
|
25
26
|
*/
|
|
26
|
-
export declare function createApiClient({ apiBaseUrl, account, accessToken, retryOptions, compressUploads, alphaOptions, }: CreateApiClientInput): ApiClient;
|
|
27
|
+
export declare function createApiClient({ apiBaseUrl, account, accessToken, retryOptions, compressUploads, alphaOptions, proxyUrl, }: CreateApiClientInput): ApiClient;
|
|
27
28
|
export declare const compressRequest: AlphaInterceptor;
|
|
28
29
|
interface GetApiBaseUrlInput {
|
|
29
30
|
dev: boolean;
|
package/dist/src/api/index.js
CHANGED
|
@@ -20,18 +20,21 @@ const util_1 = require("../synchronization/util");
|
|
|
20
20
|
* In managed environments, this can be configured to
|
|
21
21
|
* hit public and private JupiterOne APIs.
|
|
22
22
|
*/
|
|
23
|
-
function createApiClient({ apiBaseUrl, account, accessToken, retryOptions, compressUploads, alphaOptions, }) {
|
|
23
|
+
function createApiClient({ apiBaseUrl, account, accessToken, retryOptions, compressUploads, alphaOptions, proxyUrl, }) {
|
|
24
24
|
const headers = {
|
|
25
|
-
'
|
|
25
|
+
'JupiterOne-Account': account,
|
|
26
26
|
'Content-Type': 'application/json',
|
|
27
27
|
};
|
|
28
28
|
if (accessToken) {
|
|
29
29
|
headers.Authorization = `Bearer ${accessToken}`;
|
|
30
30
|
}
|
|
31
|
+
const proxyUrlString = proxyUrl || getProxyFromEnvironment();
|
|
32
|
+
const proxy = proxyUrlString ? parseProxyUrl(proxyUrlString) : undefined;
|
|
31
33
|
const opts = {
|
|
32
34
|
baseURL: apiBaseUrl,
|
|
33
35
|
headers,
|
|
34
36
|
retry: retryOptions ?? {},
|
|
37
|
+
...(proxy && { proxy }),
|
|
35
38
|
...alphaOptions,
|
|
36
39
|
};
|
|
37
40
|
const client = new alpha_1.Alpha(opts);
|
|
@@ -103,4 +106,23 @@ const getApiKeyFromEnvironment = () => getFromEnv('JUPITERONE_API_KEY', error_1.
|
|
|
103
106
|
exports.getApiKeyFromEnvironment = getApiKeyFromEnvironment;
|
|
104
107
|
const getAccountFromEnvironment = () => getFromEnv('JUPITERONE_ACCOUNT', error_1.IntegrationAccountRequiredError);
|
|
105
108
|
exports.getAccountFromEnvironment = getAccountFromEnvironment;
|
|
109
|
+
function parseProxyUrl(proxyUrl) {
|
|
110
|
+
const url = new URL(proxyUrl);
|
|
111
|
+
const proxy = {
|
|
112
|
+
host: url.hostname,
|
|
113
|
+
port: parseInt(url.port) || (url.protocol === 'https:' ? 443 : 80),
|
|
114
|
+
protocol: url.protocol.replace(':', ''),
|
|
115
|
+
};
|
|
116
|
+
if (url.username && url.password) {
|
|
117
|
+
proxy.auth = {
|
|
118
|
+
username: decodeURIComponent(url.username),
|
|
119
|
+
password: decodeURIComponent(url.password),
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
return proxy;
|
|
123
|
+
}
|
|
124
|
+
function getProxyFromEnvironment() {
|
|
125
|
+
(0, dotenv_expand_1.default)(dotenv_1.default.config());
|
|
126
|
+
return process.env.HTTPS_PROXY || process.env.https_proxy;
|
|
127
|
+
}
|
|
106
128
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/api/index.ts"],"names":[],"mappings":";;;;;;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/api/index.ts"],"names":[],"mappings":";;;;;;AAwCA,0CAwEC;AA+BD,sCAMC;AArJD,2CAAwE;AAGxE,oDAA4B;AAC5B,kEAAyC;AAEzC,mCAGiB;AACjB,kDAAmD;AAqBnD;;;;;;;;GAQG;AACH,SAAgB,eAAe,CAAC,EAC9B,UAAU,EACV,OAAO,EACP,WAAW,EACX,YAAY,EACZ,eAAe,EACf,YAAY,EACZ,QAAQ,GACa;IACrB,MAAM,OAAO,GAA2B;QACtC,oBAAoB,EAAE,OAAO;QAC7B,cAAc,EAAE,kBAAkB;KACnC,CAAC;IAEF,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO,CAAC,aAAa,GAAG,UAAU,WAAW,EAAE,CAAC;IAClD,CAAC;IAED,MAAM,cAAc,GAAG,QAAQ,IAAI,uBAAuB,EAAE,CAAC;IAC7D,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEzE,MAAM,IAAI,GAAiB;QACzB,OAAO,EAAE,UAAU;QACnB,OAAO;QACP,KAAK,EAAE,YAAY,IAAI,EAAE;QACzB,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;QACvB,GAAG,YAAY;KAChB,CAAC;IAEF,MAAM,MAAM,GAAG,IAAI,aAAK,CAAC,IAAI,CAAc,CAAC;IAE5C,kDAAkD;IAClD,MAAM,CAAC,YAAY,EAAE,QAAQ,EAAE,GAAG,CAChC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,EACtB,CAAC,KAAU,EAAE,EAAE;QACb,IAAI,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;YAC3B,KAAK,CAAC,MAAM,CAAC,OAAO,GAAG,YAAY,CAAC;QACtC,CAAC;QAED,IAAI,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;YACrC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,GAAG,YAAY,CAAC;QAC/C,CAAC;QAED,IAAI,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChD,KAAK,CAAC,OAAO,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CACnD,kCAAkC,EAClC,2BAA2B,CAC5B,CAAC;QACJ,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,CAAC,qBAAqB,CAChD,KAAK,CAAC,OAAO,IAAI,EAAE,CACpB,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC;QACrD,IAAI,aAAa,EAAE,CAAC;YAClB,MAAM,UAAU,GAAI,KAAK,CAAC,OAAe,CAAC,aAAa,CAAC,CAAC;YACzD,IAAI,UAAU,EAAE,aAAa,EAAE,CAAC;gBAC9B,UAAU,CAAC,aAAa,GAAG,YAAY,CAAC;YAC1C,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC,CACF,CAAC;IAEF,IAAI,eAAe,EAAE,CAAC;QACpB,wEAAwE;QACxE,2EAA2E;QAC3E,kEAAkE;QAClE,0EAA0E;QAC1E,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAe,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAEM,MAAM,eAAe,GAAqB,KAAK,WACpD,MAAoB;IAEpB,IACE,MAAM,CAAC,MAAM,KAAK,MAAM;QACxB,MAAM,CAAC,GAAG;QACV,6EAA6E,CAAC,IAAI,CAChF,MAAM,CAAC,GAAG,CACX,EACD,CAAC;QACD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,GAAG,MAAM,CAAC;QAC9C,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,OAAO,GAAG;gBACf,kBAAkB,EAAE,MAAM;aAC3B,CAAC;QACJ,CAAC;QACD,MAAM,CAAC,IAAI,GAAG,MAAM,IAAA,eAAQ,EAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AApBW,QAAA,eAAe,mBAoB1B;AAMW,QAAA,4BAA4B,GAAG,8BAA8B,CAAC;AAC9D,QAAA,2BAA2B,GAAG,+BAA+B,CAAC;AAE3E,SAAgB,aAAa,CAAC,EAAE,GAAG,KAAyB,EAAE,GAAG,EAAE,KAAK,EAAE;IACxE,IAAI,GAAG,EAAE,CAAC;QACR,OAAO,mCAA2B,CAAC;IACrC,CAAC;SAAM,CAAC;QACN,OAAO,oCAA4B,CAAC;IACtC,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CACjB,YAAoB,EACpB,YAAwC;IAExC,IAAA,uBAAY,EAAC,gBAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAE9B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IAExC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,YAAY,EAAE,CAAC;IAC3B,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAEM,MAAM,wBAAwB,GAAG,GAAG,EAAE,CAC3C,UAAU,CAAC,oBAAoB,EAAE,sCAA8B,CAAC,CAAC;AADtD,QAAA,wBAAwB,4BAC8B;AAE5D,MAAM,yBAAyB,GAAG,GAAG,EAAE,CAC5C,UAAU,CAAC,oBAAoB,EAAE,uCAA+B,CAAC,CAAC;AADvD,QAAA,yBAAyB,6BAC8B;AAEpE,SAAS,aAAa,CAAC,QAAgB;IACrC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC9B,MAAM,KAAK,GAAqB;QAC9B,IAAI,EAAE,GAAG,CAAC,QAAQ;QAClB,IAAI,EAAE,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC;KACxC,CAAC;IAEF,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,GAAG;YACX,QAAQ,EAAE,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC1C,QAAQ,EAAE,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC3C,CAAC;IACJ,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,uBAAuB;IAC9B,IAAA,uBAAY,EAAC,gBAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9B,OAAO,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC;AAC5D,CAAC"}
|
|
@@ -132,7 +132,9 @@ async function executeWithContext(context, config, options = {}) {
|
|
|
132
132
|
if (shouldPublishDiskUsageMetric) {
|
|
133
133
|
diskUsagePublishInterval = createDiskUsagePublishInterval();
|
|
134
134
|
}
|
|
135
|
-
const { graphObjectStore = new storage_1.FileSystemGraphObjectStore(
|
|
135
|
+
const { graphObjectStore = new storage_1.FileSystemGraphObjectStore({
|
|
136
|
+
logger,
|
|
137
|
+
}), createStepGraphObjectDataUploader, resultsCallback, } = options;
|
|
136
138
|
let duplicateKeyTracker = new duplicateKeyTracker_1.InMemoryDuplicateKeyTracker(config.normalizeGraphObjectKey);
|
|
137
139
|
if (process.env.USE_ON_DISK_DKT) {
|
|
138
140
|
// conditionally require so the dependency can remain optional
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executeIntegration.js","sourceRoot":"","sources":["../../../src/execution/executeIntegration.ts"],"names":[],"mappings":";;;;;AA4EA,8DAyBC;AAKD,gEAkCC;AA2BD,
|
|
1
|
+
{"version":3,"file":"executeIntegration.js","sourceRoot":"","sources":["../../../src/execution/executeIntegration.ts"],"names":[],"mappings":";;;;;AA4EA,8DAyBC;AAKD,gEAkCC;AA2BD,gDAwMC;AA/WD,2EAa0C;AAE1C,8CAOuB;AACvB,sCAImB;AACnB,wCAA2C;AAC3C,wCAA0E;AAC1E,gDAAwB;AACxB,yCAAwE;AACxE,yCAA6C;AAC7C,iCAIgB;AAEhB,6DAA0D;AAC1D,+DAA4D;AAC5D,6CAAuD;AACvD,+EAA4E;AAC5E,+DAG+B;AAC/B,iGAA8F;AAuB9F,MAAM,kCAAkC,GAAG,KAAK,GAAG,CAAC,CAAC;AAErD;;;GAGG;AACI,KAAK,UAAU,yBAAyB,CAC7C,MAAmC,EACnC,gBAAkC,EAClC,OAAmC;IAEnC,MAAM,MAAM,GAAG,IAAA,gCAAuB,EAAC;QACrC,IAAI,EAAE,OAAO;QACb,gBAAgB,EAAE,MAAM;QACxB,MAAM,EAAE,OAAO,EAAE,MAAM;KACxB,CAAC,CAAC;IACH,MAAM,wBAAwB,GAAG,IAAA,+CAAsC,EACrE,GAAG,EAAE,CAAC,MAAM,CACb,CAAC;IACF,MAAM,MAAM,GAAG,MAAM,0BAA0B,CAC7C,MAAM,EACN,IAAA,qDAA0C,EAAC,MAAM,CAAC,EAClD,MAAM,EACN,gBAAgB,EAChB;QACE,GAAG,OAAO;QACV,sBAAsB,EAAE,OAAO,EAAE,sBAAsB,IAAI,IAAI;KAChE,CACF,CAAC;IACF,IAAA,iDAAwC,EAAC,wBAAwB,CAAC,CAAC;IACnE,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACI,KAAK,UAAU,0BAA0B,CAI9C,MAAyB,EACzB,QAAiD,EACjD,MAAuD,EACvD,gBAAkC,EAClC,UAAqC,EAAE;IAEvC,IAAI,OAAO,CAAC,sBAAsB,KAAK,IAAI,EAAE,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,qCAAqC,GAAG,MAAM,CAAC;IAC7D,CAAC;IACD,MAAM,yBAAyB,GAAG,IAAA,mCAAgB,EAChD,QAAQ,EACR,IAAA,iCAAe,EAAC,MAAM,CAAC,CACxB,CAAC;IAEF,OAAO,IAAA,uBAAa,EAAC;QACnB,MAAM;QACN,UAAU,EAAE,kBAAkB;QAC9B,SAAS,EAAE,GAAG,EAAE,CACd,kBAAkB,CAChB;YACE,QAAQ,EAAE,yBAAyB;YACnC,MAAM;YACN,gBAAgB;YAChB,eAAe,EACb,MAAM,CAAC,mBAAmB,EAAE,CAAC,yBAAyB,CAAC,IAAI,EAAE;SAChE,EACD,MAAM,EACN,OAAO,CACR;KACJ,CAAC,CAAC;AACL,CAAC;AAED,SAAS,sBAAsB,CAC7B,OAA0B,EAC1B,IAAY;IAEZ,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC;QAC3B,IAAI,EAAE,YAAY;QAClB,KAAK,EAAE,IAAI;QACX,IAAI,EAAE,OAAO;KACd,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,yBAAyB,CAEtC,OAA0B;IAC1B,IAAI,CAAC,CAAC,MAAM,IAAA,0CAA6B,GAAE,CAAC,EAAE,CAAC;QAC7C,OAAO;IACT,CAAC;IAED,sBAAsB,CAAC,OAAO,EAAE,MAAM,IAAA,wCAA2B,GAAE,CAAC,CAAC;AACvE,CAAC;AAED;;;GAGG;AACI,KAAK,UAAU,kBAAkB,CAItC,OAA0B,EAC1B,MAAkE,EAClE,UAAqC,EAAE;IAEvC,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAE3B,wEAAwE;IACxE,oEAAoE;IACpE,MAAM,4BAA4B,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC;IAE5E,MAAM,CAAC,IAAI,CACT;QACE,kBAAkB,EAAE,IAAA,iCAAoB,GAAE;KAC3C,EACD,mCAAmC,CACpC,CAAC;IAEF,IAAI,wBAAoD,CAAC;IACzD,IAAI,iBAAiB,GAAG,KAAK,CAAC;IAE9B,SAAS,8BAA8B;QACrC,OAAO,WAAW,CAAC,GAAG,EAAE;YACtB,IAAA,0CAA6B,GAAE;iBAC5B,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;gBAChB,IAAI,CAAC,OAAO;oBAAE,OAAO;gBAErB,OAAO,IAAA,wCAA2B,GAAE,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;oBACjD,IAAI,iBAAiB,EAAE,CAAC;wBACtB,OAAO;oBACT,CAAC;oBAED,sBAAsB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;gBACxC,CAAC,CAAC,CAAC;YACL,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACb,IAAI,iBAAiB,EAAE,CAAC;oBACtB,OAAO;gBACT,CAAC;gBAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,oCAAoC,CAAC,CAAC;YACtE,CAAC,CAAC,CAAC;QACP,CAAC,EAAE,kCAAkC,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,CAAC;QACH,IAAI,4BAA4B,EAAE,CAAC;YACjC,MAAM,yBAAyB,CAAC,OAAO,CAAC,CAAC;QAC3C,CAAC;QAED,IAAI,CAAC;YACH,MAAM,IAAA,mCAAsB,GAAE,CAAC;YAE/B,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,kBAAkB,EAAE,CAAC,OAAO,CAAC,CAAC;YAC7C,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC;gBAC9B,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,MAAM,uBAAuB,GAC3B,MAAM,MAAM,CAAC,kBAAkB,EAAE,CAAC,OAAO,CAAC,CAAC;YAE7C,MAAM,qBAAqB,GACzB,uBAAuB;gBACvB,IAAA,gCAAyB,EAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;YAErD,IAAA,oCAAuB,EAAC,MAAM,CAAC,gBAAgB,EAAE,qBAAqB,CAAC,CAAC;YAExE,MAAM,eAAe,GAAG,IAAA,qEAAiC,EAAC;gBACxD,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,qBAAqB;gBACrB,eAAe,EAAE,OAAO,CAAC,QAAQ,EAAE,eAAe,EAAE,GAAG,CACrD,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,iBAAiB,CACrC;aACF,CAAC,CAAC;YAEH,IAAI,4BAA4B,EAAE,CAAC;gBACjC,wBAAwB,GAAG,8BAA8B,EAAE,CAAC;YAC9D,CAAC;YAED,MAAM,EACJ,gBAAgB,GAAG,IAAI,oCAA0B,CAAC;gBAChD,MAAM;aACP,CAAC,EACF,iCAAiC,EACjC,eAAe,GAChB,GAAG,OAAO,CAAC;YAEZ,IAAI,mBAAmB,GACrB,IAAI,iDAA2B,CAAC,MAAM,CAAC,uBAAuB,CAAC,CAAC;YAElE,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC;gBAChC,8DAA8D;gBAC9D,IAAI,CAAC;oBACH,MAAM,EACJ,yBAAyB,GAC1B,GAAG,OAAO,CAAC,6BAA6B,CAAC,CAAC;oBAC3C,mBAAmB,GAAG,IAAI,yBAAyB,CAAC;wBAClD,QAAQ,EAAE,cAAI,CAAC,IAAI,CACjB,OAAO,CAAC,GAAG,EAAE,EACb,2CAA8B,EAC9B,gBAAgB,CACjB;qBACF,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,CAAC,IAAI,CACT,EAAE,GAAG,EAAE,EACP,qGAAqG,CACtG,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,MAAM,sBAAsB,GAAG,MAAM,IAAA,mBAAY,EAAC;gBAChD,gBAAgB,EAAE,OAAO;gBACzB,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,eAAe;gBACf,eAAe,EAAE,MAAM,CAAC,eAAe;gBACvC,mBAAmB;gBACnB,gBAAgB;gBAChB,SAAS,EAAE,IAAI,0BAAe,EAAE;gBAChC,iCAAiC;gBACjC,eAAe,EAAE,MAAM,CAAC,eAAe;gBACvC,qBAAqB,EAAE,MAAM,CAAC,qBAAqB;gBACnD,cAAc,EAAE,SAAS;gBACzB,oBAAoB,EAAE,SAAS;gBAC/B,oBAAoB,EAAE,MAAM,CAAC,oBAAoB;gBACjD,uBAAuB,EAAE,MAAM,CAAC,uBAAuB;aACxD,CAAC,CAAC;YAEH,MAAM,eAAe,GAAG,IAAA,uDAAgD,EACtE,sBAAsB,CACvB,CAAC;YAEF,MAAM,OAAO,GAA6B;gBACxC,sBAAsB;gBACtB,eAAe,EAAE,MAAM,CAAC,sBAAsB;oBAC5C,CAAC,CAAC,mBAAmB,CAAC,kBAAkB,EAAE;oBAC1C,CAAC,CAAC,SAAS;gBACb,QAAQ,EAAE;oBACR,eAAe;iBAChB;aACF,CAAC;YAEF,MAAM,IAAA,4BAAe,EAAC;gBACpB,IAAI,EAAE,cAAc;gBACpB,IAAI,EAAE,OAAO;aACd,CAAC,CAAC;YAEH,IAAI,eAAe,IAAI,IAAI,EAAE,CAAC;gBAC5B,IAAI,CAAC;oBACH,MAAM,eAAe,CAAC,OAAO,CAAC,CAAC;gBACjC,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,OAAO,CAAC,MAAM,CAAC,IAAI,CACjB,GAAG,EACH,oDAAoD,CACrD,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,IAAA,mDAAwB,EAAC,OAAO,EAAE,CAAC,IAAI,EAAE,eAAe,EAAE,EAAE;gBAC1D,IACE,IAAI,CAAC,MAAM,KAAK,uCAAgB,CAAC,OAAO;oBACxC,eAAe,CAAC,MAAM,EACtB,CAAC;oBACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,EAAE,eAAe,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,EACpC,iGAAiG;wBAC/F,wDAAwD,CAC3D,CAAC;gBACJ,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,OAAO,OAAO,CAAC;QACjB,CAAC;gBAAS,CAAC;YACT,iBAAiB,GAAG,IAAI,CAAC;YAEzB,IAAI,wBAAwB,EAAE,CAAC;gBAC7B,aAAa,CAAC,wBAAwB,CAAC,CAAC;YAC1C,CAAC;YAED,IAAI,4BAA4B,EAAE,CAAC;gBACjC,MAAM,yBAAyB,CAAC,OAAO,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;IACH,CAAC;YAAS,CAAC;QACT,IAAI,MAAM,EAAE,cAAc,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YACvC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,0DAA0D;gBAC1D,uEAAuE;gBACvE,4BAA4B;gBAC5B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,wCAAwC,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { IntegrationLogger } from '@jupiterone/integration-sdk-core';
|
|
1
2
|
import { FlushedGraphObjectData } from '../storage/types';
|
|
2
3
|
import { SynchronizationJobContext } from '../synchronization';
|
|
3
4
|
export interface StepGraphObjectDataUploader {
|
|
@@ -7,12 +8,13 @@ export interface StepGraphObjectDataUploader {
|
|
|
7
8
|
}
|
|
8
9
|
export type CreateStepGraphObjectDataUploaderFunction = (stepId: string) => StepGraphObjectDataUploader;
|
|
9
10
|
export interface CreateQueuedStepGraphObjectDataUploaderParams {
|
|
11
|
+
logger: IntegrationLogger;
|
|
10
12
|
stepId: string;
|
|
11
13
|
uploadConcurrency: number;
|
|
12
14
|
upload: (graphObjectData: FlushedGraphObjectData) => Promise<void>;
|
|
13
15
|
onThrottleEnqueue?: () => void;
|
|
14
16
|
}
|
|
15
|
-
export declare function createQueuedStepGraphObjectDataUploader({ stepId, uploadConcurrency: maximumQueueSize, upload, onThrottleEnqueue, }: CreateQueuedStepGraphObjectDataUploaderParams): StepGraphObjectDataUploader;
|
|
17
|
+
export declare function createQueuedStepGraphObjectDataUploader({ logger, stepId, uploadConcurrency: maximumQueueSize, upload, onThrottleEnqueue, }: CreateQueuedStepGraphObjectDataUploaderParams): StepGraphObjectDataUploader;
|
|
16
18
|
export interface CreatePersisterApiStepGraphObjectDataUploaderParams {
|
|
17
19
|
stepId: string;
|
|
18
20
|
synchronizationJobContext: SynchronizationJobContext;
|
|
@@ -9,7 +9,7 @@ const integration_sdk_core_1 = require("@jupiterone/integration-sdk-core");
|
|
|
9
9
|
const dist_1 = __importDefault(require("p-queue/dist"));
|
|
10
10
|
const synchronization_1 = require("../synchronization");
|
|
11
11
|
const crypto_1 = require("crypto");
|
|
12
|
-
function createQueuedStepGraphObjectDataUploader({ stepId, uploadConcurrency: maximumQueueSize, upload, onThrottleEnqueue, }) {
|
|
12
|
+
function createQueuedStepGraphObjectDataUploader({ logger, stepId, uploadConcurrency: maximumQueueSize, upload, onThrottleEnqueue, }) {
|
|
13
13
|
const queue = new dist_1.default({
|
|
14
14
|
concurrency: maximumQueueSize,
|
|
15
15
|
});
|
|
@@ -44,6 +44,7 @@ function createQueuedStepGraphObjectDataUploader({ stepId, uploadConcurrency: ma
|
|
|
44
44
|
queue
|
|
45
45
|
.add(() => upload(graphObjectData))
|
|
46
46
|
.catch((err) => {
|
|
47
|
+
logger.warn({ err, stepId, graphObjectData }, 'Error uploading graph object data batch');
|
|
47
48
|
// Do not pause the queue entirely. We will try to prevent additional
|
|
48
49
|
// tasks from being added to the queue, but even if an error occurs,
|
|
49
50
|
// we should try uploading the remaining data that we have queued up.
|
|
@@ -69,6 +70,11 @@ function createQueuedStepGraphObjectDataUploader({ stepId, uploadConcurrency: ma
|
|
|
69
70
|
// this time, we could be receiving additional tasks in our queue that
|
|
70
71
|
// will grow the queue.
|
|
71
72
|
completed = true;
|
|
73
|
+
logger.debug({
|
|
74
|
+
stepId,
|
|
75
|
+
uploadErrorCount: uploadErrors.length,
|
|
76
|
+
typesInvolvedInFailures: Array.from(typesInvolvedInFailures),
|
|
77
|
+
}, 'Upload queue processing complete');
|
|
72
78
|
}
|
|
73
79
|
if (uploadErrors.length) {
|
|
74
80
|
throw new integration_sdk_core_1.UploadError(`Error(s) uploading graph object data (stepId=${stepId}, errorMessages=${uploadErrors.join(',')})`, Array.from(typesInvolvedInFailures.values()));
|
|
@@ -91,6 +97,7 @@ function jobContextWithUploaderMetadataLogger({ synchronizationJobContext, uploa
|
|
|
91
97
|
}
|
|
92
98
|
function createPersisterApiStepGraphObjectDataUploader({ stepId, synchronizationJobContext, uploadConcurrency, uploadBatchSizeInBytes = synchronization_1.DEFAULT_UPLOAD_BATCH_SIZE_IN_BYTES, }) {
|
|
93
99
|
return createQueuedStepGraphObjectDataUploader({
|
|
100
|
+
logger: synchronizationJobContext.logger,
|
|
94
101
|
stepId,
|
|
95
102
|
uploadConcurrency,
|
|
96
103
|
async upload(graphObjectData) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.js","sourceRoot":"","sources":["../../../src/execution/uploader.ts"],"names":[],"mappings":";;;;;
|
|
1
|
+
{"version":3,"file":"uploader.js","sourceRoot":"","sources":["../../../src/execution/uploader.ts"],"names":[],"mappings":";;;;;AA+BA,0FAqGC;AA+BD,sGAoCC;AAvMD,2EAG0C;AAC1C,wDAAkC;AAElC,wDAI4B;AAC5B,mCAA4C;AAoB5C,SAAgB,uCAAuC,CAAC,EACtD,MAAM,EACN,MAAM,EACN,iBAAiB,EAAE,gBAAgB,EACnC,MAAM,EACN,iBAAiB,GAC6B;IAC9C,MAAM,KAAK,GAAG,IAAI,cAAM,CAAC;QACvB,WAAW,EAAE,gBAAgB;KAC9B,CAAC,CAAC;IAEH,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,MAAM,YAAY,GAAY,EAAE,CAAC;IACjC,MAAM,uBAAuB,GAAG,IAAI,GAAG,EAAU,CAAC;IAClD,OAAO;QACL,MAAM;QACN,KAAK,CAAC,OAAO,CAAC,eAAe;YAC3B,IAAI,SAAS,EAAE,CAAC;gBACd,qEAAqE;gBACrE,iDAAiD;gBACjD,OAAO;YACT,CAAC;YAED,gEAAgE;YAChE,qEAAqE;YACrE,yEAAyE;YACzE,uCAAuC;YACvC,EAAE;YACF,0EAA0E;YAC1E,yEAAyE;YACzE,sEAAsE;YACtE,wEAAwE;YACxE,uCAAuC;YACvC,IAAI,KAAK,CAAC,IAAI,IAAI,gBAAgB,EAAE,CAAC;gBACnC,IAAI,iBAAiB,EAAE,CAAC;oBACtB,iEAAiE;oBACjE,iBAAiB,EAAE,CAAC;gBACtB,CAAC;gBAED,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;YACxB,CAAC;YAED,KAAK;iBACF,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;iBAClC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACb,MAAM,CAAC,IAAI,CACT,EAAE,GAAG,EAAE,MAAM,EAAE,eAAe,EAAE,EAChC,yCAAyC,CAC1C,CAAC;gBACF,qEAAqE;gBACrE,oEAAoE;gBACpE,qEAAqE;gBACrE,sEAAsE;gBACtE,qEAAqE;gBACrE,6DAA6D;gBAC7D,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACvB,IAAI,eAAe,EAAE,CAAC;oBACpB,eAAe,CAAC,QAAQ,CAAC,OAAO,CAC9B,CAAC,MAAM,EAAE,EAAE,CAAC,uBAAuB,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,EACrD,uBAAuB,CACxB,CAAC;oBACF,eAAe,CAAC,aAAa,CAAC,OAAO,CACnC,CAAC,YAAY,EAAE,EAAE,CAAC,uBAAuB,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,EACjE,uBAAuB,CACxB,CAAC;gBACJ,CAAC;YACH,CAAC,CAAC,CAAC;QACP,CAAC;QAED,KAAK,CAAC,wBAAwB;YAC5B,IAAI,CAAC;gBACH,wEAAwE;gBACxE,oEAAoE;gBACpE,yEAAyE;gBACzE,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC;YACvB,CAAC;oBAAS,CAAC;gBACT,uEAAuE;gBACvE,sEAAsE;gBACtE,uBAAuB;gBACvB,SAAS,GAAG,IAAI,CAAC;gBAEjB,MAAM,CAAC,KAAK,CACV;oBACE,MAAM;oBACN,gBAAgB,EAAE,YAAY,CAAC,MAAM;oBACrC,uBAAuB,EAAE,KAAK,CAAC,IAAI,CAAC,uBAAuB,CAAC;iBAC7D,EACD,kCAAkC,CACnC,CAAC;YACJ,CAAC;YAED,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;gBACxB,MAAM,IAAI,kCAAW,CACnB,gDAAgD,MAAM,mBAAmB,YAAY,CAAC,IAAI,CACxF,GAAG,CACJ,GAAG,EACJ,KAAK,CAAC,IAAI,CAAC,uBAAuB,CAAC,MAAM,EAAE,CAAC,CAC7C,CAAC;YACJ,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,oCAAoC,CAAC,EAC5C,yBAAyB,EACzB,QAAQ,EACR,MAAM,GAKP;IACC,OAAO;QACL,GAAG,yBAAyB;QAC5B,MAAM,EAAE,yBAAyB,CAAC,MAAM,CAAC,KAAK,CAAC;YAC7C,QAAQ;YACR,MAAM;SACP,CAAC;KACH,CAAC;AACJ,CAAC;AASD,SAAgB,6CAA6C,CAAC,EAC5D,MAAM,EACN,yBAAyB,EACzB,iBAAiB,EACjB,sBAAsB,GAAG,oDAAkC,GACP;IACpD,OAAO,uCAAuC,CAAC;QAC7C,MAAM,EAAE,yBAAyB,CAAC,MAAM;QACxC,MAAM;QACN,iBAAiB;QACjB,KAAK,CAAC,MAAM,CAAC,eAAe;YAC1B,MAAM,OAAO,GAAG,oCAAoC,CAAC;gBACnD,yBAAyB;gBACzB,QAAQ,EAAE,IAAA,mBAAI,GAAE;gBAChB,MAAM,EAAE,MAAM;aACf,CAAC,CAAC;YAEH,IAAI,CAAC;gBACH,MAAM,IAAA,uCAAqB,EACzB,OAAO,EACP,eAAe,EACf,sBAAsB,CACvB,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB;oBACE,GAAG;oBACH,iBAAiB;oBACjB,sBAAsB;iBACvB,EACD,mCAAmC,CACpC,CAAC;gBACF,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Entity, GraphObjectFilter, GraphObjectIteratee, Relationship, GraphObjectStore, GraphObjectIndexMetadata, GetIndexMetadataForGraphObjectTypeParams, IntegrationStep, GraphObjectIterateeOptions } from '@jupiterone/integration-sdk-core';
|
|
1
|
+
import { Entity, GraphObjectFilter, GraphObjectIteratee, Relationship, GraphObjectStore, GraphObjectIndexMetadata, GetIndexMetadataForGraphObjectTypeParams, IntegrationStep, GraphObjectIterateeOptions, IntegrationLogger } from '@jupiterone/integration-sdk-core';
|
|
2
2
|
export declare const DEFAULT_GRAPH_OBJECT_FILE_SIZE = 500;
|
|
3
3
|
export declare const MAX_GRAPH_OBJECT_BUFFER_THRESHOLD_IN_BYTES: 1073741824;
|
|
4
4
|
export interface FileSystemGraphObjectStoreParams {
|
|
@@ -16,6 +16,10 @@ export interface FileSystemGraphObjectStoreParams {
|
|
|
16
16
|
* Whether the files that are written to disk should be minified or not
|
|
17
17
|
*/
|
|
18
18
|
prettifyFiles?: boolean;
|
|
19
|
+
/**
|
|
20
|
+
* Optional logger for debugging and tracking data flow
|
|
21
|
+
*/
|
|
22
|
+
logger?: IntegrationLogger;
|
|
19
23
|
}
|
|
20
24
|
export declare class FileSystemGraphObjectStore implements GraphObjectStore {
|
|
21
25
|
private readonly semaphore;
|
|
@@ -25,6 +29,7 @@ export declare class FileSystemGraphObjectStore implements GraphObjectStore {
|
|
|
25
29
|
private readonly prettifyFiles;
|
|
26
30
|
private readonly stepIdToGraphObjectIndexMetadataMap;
|
|
27
31
|
private readonly entityOnDiskLocationMap;
|
|
32
|
+
private readonly logger?;
|
|
28
33
|
constructor(params?: FileSystemGraphObjectStoreParams);
|
|
29
34
|
addEntities(stepId: string, newEntities: Entity[], onEntitiesFlushed?: (entities: Entity[]) => Promise<void>): Promise<void>;
|
|
30
35
|
addRelationships(stepId: string, newRelationships: Relationship[], onRelationshipsFlushed?: (relationships: Relationship[]) => Promise<void>): Promise<void>;
|
|
@@ -48,11 +48,13 @@ class FileSystemGraphObjectStore {
|
|
|
48
48
|
prettifyFiles;
|
|
49
49
|
stepIdToGraphObjectIndexMetadataMap;
|
|
50
50
|
entityOnDiskLocationMap = new bigMap_1.BigMap(ENTITY_LOCATION_ON_DISK_DEFAULT_MAP_KEY_SPACE);
|
|
51
|
+
logger;
|
|
51
52
|
constructor(params) {
|
|
52
53
|
this.semaphore = new async_sema_1.Sema(BINARY_SEMAPHORE_CONCURRENCY);
|
|
53
54
|
this.graphObjectFileSize =
|
|
54
55
|
params?.graphObjectFileSize || exports.DEFAULT_GRAPH_OBJECT_FILE_SIZE;
|
|
55
56
|
this.prettifyFiles = params?.prettifyFiles || false;
|
|
57
|
+
this.logger = params?.logger;
|
|
56
58
|
this.graphObjectBufferThresholdInBytes = (0, lodash_1.min)([
|
|
57
59
|
params?.graphObjectBufferThresholdInBytes ||
|
|
58
60
|
synchronization_1.DEFAULT_UPLOAD_BATCH_SIZE_IN_BYTES,
|
|
@@ -185,28 +187,63 @@ class FileSystemGraphObjectStore {
|
|
|
185
187
|
return indexMetadata.enabled === true;
|
|
186
188
|
});
|
|
187
189
|
if (indexable.length) {
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
190
|
+
const chunks = (0, lodash_1.chunk)(indexable, this.graphObjectFileSize);
|
|
191
|
+
this.logger?.debug({
|
|
192
|
+
stepId,
|
|
193
|
+
entityCount: indexable.length,
|
|
194
|
+
chunkCount: chunks.length,
|
|
195
|
+
chunkSize: this.graphObjectFileSize,
|
|
196
|
+
}, 'Flushing entity chunks to disk');
|
|
197
|
+
try {
|
|
198
|
+
await Promise.all(chunks.map(async (data, chunkIndex) => {
|
|
199
|
+
const graphObjectsToFilePaths = await (0, flushDataToDisk_1.flushDataToDisk)({
|
|
200
|
+
storageDirectoryPath: stepId,
|
|
201
|
+
collectionType: 'entities',
|
|
202
|
+
data,
|
|
203
|
+
pretty: this.prettifyFiles,
|
|
204
|
+
logger: this.logger,
|
|
205
|
+
});
|
|
206
|
+
for (const { graphDataPath, collection, } of graphObjectsToFilePaths) {
|
|
207
|
+
for (const [index, e] of collection.entries()) {
|
|
208
|
+
this.entityOnDiskLocationMap.set(e._key, {
|
|
209
|
+
graphDataPath,
|
|
210
|
+
index,
|
|
211
|
+
});
|
|
212
|
+
}
|
|
201
213
|
}
|
|
202
|
-
|
|
203
|
-
|
|
214
|
+
this.logger?.debug({
|
|
215
|
+
stepId,
|
|
216
|
+
chunkIndex,
|
|
217
|
+
entitiesInChunk: data.length,
|
|
218
|
+
filesCreated: graphObjectsToFilePaths.length,
|
|
219
|
+
}, 'Entity chunk flushed successfully');
|
|
220
|
+
}));
|
|
221
|
+
}
|
|
222
|
+
catch (error) {
|
|
223
|
+
this.logger?.error({
|
|
224
|
+
stepId,
|
|
225
|
+
entityCount: indexable.length,
|
|
226
|
+
chunkCount: chunks.length,
|
|
227
|
+
error: error.message,
|
|
228
|
+
errorStack: error.stack,
|
|
229
|
+
}, 'Failed to flush entity chunks to disk');
|
|
230
|
+
throw error;
|
|
231
|
+
}
|
|
204
232
|
}
|
|
205
233
|
this.localGraphObjectStore.flushEntities(entities, stepId);
|
|
206
234
|
entitiesToUpload = entitiesToUpload.concat(entities);
|
|
207
235
|
}
|
|
208
236
|
if (onEntitiesFlushed) {
|
|
209
|
-
|
|
237
|
+
try {
|
|
238
|
+
await onEntitiesFlushed(entitiesToUpload);
|
|
239
|
+
}
|
|
240
|
+
catch (err) {
|
|
241
|
+
this.logger?.error({
|
|
242
|
+
entityCount: entitiesToUpload.length,
|
|
243
|
+
err,
|
|
244
|
+
}, 'onEntitiesFlushed callback failed');
|
|
245
|
+
throw err;
|
|
246
|
+
}
|
|
210
247
|
}
|
|
211
248
|
});
|
|
212
249
|
}
|
|
@@ -249,20 +286,55 @@ class FileSystemGraphObjectStore {
|
|
|
249
286
|
return indexMetadata.enabled === true;
|
|
250
287
|
});
|
|
251
288
|
if (indexable.length) {
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
289
|
+
const chunks = (0, lodash_1.chunk)(indexable, this.graphObjectFileSize);
|
|
290
|
+
this.logger?.debug({
|
|
291
|
+
stepId,
|
|
292
|
+
relationshipCount: indexable.length,
|
|
293
|
+
chunkCount: chunks.length,
|
|
294
|
+
chunkSize: this.graphObjectFileSize,
|
|
295
|
+
}, 'Flushing relationship chunks to disk');
|
|
296
|
+
try {
|
|
297
|
+
await Promise.all(chunks.map(async (data, chunkIndex) => {
|
|
298
|
+
await (0, flushDataToDisk_1.flushDataToDisk)({
|
|
299
|
+
storageDirectoryPath: stepId,
|
|
300
|
+
collectionType: 'relationships',
|
|
301
|
+
data,
|
|
302
|
+
pretty: this.prettifyFiles,
|
|
303
|
+
logger: this.logger,
|
|
304
|
+
});
|
|
305
|
+
this.logger?.debug({
|
|
306
|
+
stepId,
|
|
307
|
+
chunkIndex,
|
|
308
|
+
relationshipsInChunk: data.length,
|
|
309
|
+
}, 'Relationship chunk flushed successfully');
|
|
310
|
+
}));
|
|
311
|
+
}
|
|
312
|
+
catch (error) {
|
|
313
|
+
this.logger?.error({
|
|
314
|
+
stepId,
|
|
315
|
+
relationshipCount: indexable.length,
|
|
316
|
+
chunkCount: chunks.length,
|
|
317
|
+
error: error.message,
|
|
318
|
+
errorStack: error.stack,
|
|
319
|
+
}, 'Failed to flush relationship chunks to disk');
|
|
320
|
+
throw error;
|
|
321
|
+
}
|
|
260
322
|
}
|
|
261
323
|
this.localGraphObjectStore.flushRelationships(relationships, stepId);
|
|
262
324
|
relationshipsToUpload = relationshipsToUpload.concat(relationships);
|
|
263
325
|
}
|
|
264
326
|
if (onRelationshipsFlushed) {
|
|
265
|
-
|
|
327
|
+
try {
|
|
328
|
+
await onRelationshipsFlushed(relationshipsToUpload);
|
|
329
|
+
}
|
|
330
|
+
catch (error) {
|
|
331
|
+
this.logger?.error({
|
|
332
|
+
relationshipCount: relationshipsToUpload.length,
|
|
333
|
+
error: error.message,
|
|
334
|
+
errorStack: error.stack,
|
|
335
|
+
}, 'onRelationshipsFlushed callback failed');
|
|
336
|
+
throw error;
|
|
337
|
+
}
|
|
266
338
|
}
|
|
267
339
|
});
|
|
268
340
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FileSystemGraphObjectStore.js","sourceRoot":"","sources":["../../../../src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.ts"],"names":[],"mappings":";;;AAAA,2CAAkC;
|
|
1
|
+
{"version":3,"file":"FileSystemGraphObjectStore.js","sourceRoot":"","sources":["../../../../src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.ts"],"names":[],"mappings":";;;AAAA,2CAAkC;AAelC,uDAAoD;AACpD,uCAImB;AACnB,sCAAqD;AAErD,iDAA8D;AAC9D,yDAAsD;AACtD,mCAAoC;AACpC,2DAA2E;AAE9D,QAAA,8BAA8B,GAAG,GAAG,CAAC;AAElD,gCAAgC;AACnB,QAAA,0CAA0C,GACrD,aAAsB,CAAC;AAEzB,8CAA8C;AAC9C,gEAAgE;AAChE,MAAM,4BAA4B,GAAG,CAAC,CAAC;AAqCvC;;GAEG;AACH,SAAS,6CAA6C,CACpD,gBAAmC;IAEnC,MAAM,mCAAmC,GAAG,IAAI,GAAG,EAGhD,CAAC;IAEJ,KAAK,MAAM,IAAI,IAAI,gBAAgB,EAAE,CAAC;QACpC,MAAM,WAAW,GAAgC;YAC/C,QAAQ,EAAE,IAAI,GAAG,EAAE;YACnB,aAAa,EAAE,IAAI,GAAG,EAAE;SACzB,CAAC;QAEF,KAAK,MAAM,cAAc,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC3C,IAAI,cAAc,CAAC,aAAa,EAAE,CAAC;gBACjC,WAAW,CAAC,QAAQ,CAAC,GAAG,CACtB,cAAc,CAAC,KAAK,EACpB,cAAc,CAAC,aAAa,CAC7B,CAAC;YACJ,CAAC;QACH,CAAC;QAED,KAAK,MAAM,oBAAoB,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACtD,IAAI,oBAAoB,CAAC,aAAa,EAAE,CAAC;gBACvC,WAAW,CAAC,aAAa,CAAC,GAAG,CAC3B,oBAAoB,CAAC,KAAK,EAC1B,oBAAoB,CAAC,aAAa,CACnC,CAAC;YACJ,CAAC;QACH,CAAC;QAED,mCAAmC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;IAChE,CAAC;IAED,OAAO,mCAAmC,CAAC;AAC7C,CAAC;AAeD,MAAM,6CAA6C,GAAG,OAAO,CAAC;AAE9D,MAAa,0BAA0B;IACpB,SAAS,CAAO;IAChB,qBAAqB,GAAG,IAAI,iCAAwB,EAAE,CAAC;IACvD,mBAAmB,CAAS;IAC5B,iCAAiC,CAAS;IAC1C,aAAa,CAAU;IACvB,mCAAmC,CAGlD;IACe,uBAAuB,GAAG,IAAI,eAAM,CAGnD,6CAA6C,CAAC,CAAC;IAChC,MAAM,CAAqB;IAE5C,YAAY,MAAyC;QACnD,IAAI,CAAC,SAAS,GAAG,IAAI,iBAAI,CAAC,4BAA4B,CAAC,CAAC;QACxD,IAAI,CAAC,mBAAmB;YACtB,MAAM,EAAE,mBAAmB,IAAI,sCAA8B,CAAC;QAEhE,IAAI,CAAC,aAAa,GAAG,MAAM,EAAE,aAAa,IAAI,KAAK,CAAC;QACpD,IAAI,CAAC,MAAM,GAAG,MAAM,EAAE,MAAM,CAAC;QAC7B,IAAI,CAAC,iCAAiC,GAAG,IAAA,YAAG,EAAC;YAC3C,MAAM,EAAE,iCAAiC;gBACvC,oDAAkC;YACpC,kDAA0C;SAC3C,CAAE,CAAC;QACJ,IAAI,MAAM,EAAE,gBAAgB,EAAE,CAAC;YAC7B,IAAI,CAAC,mCAAmC;gBACtC,6CAA6C,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW,CACf,MAAc,EACd,WAAqB,EACrB,iBAAyD;QAEzD,MAAM,IAAI,CAAC,qBAAqB,CAAC,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAElE,IACE,IAAI,CAAC,qBAAqB,CAAC,yBAAyB,EAAE;YACtD,IAAI,CAAC,iCAAiC,EACtC,CAAC;YACD,MAAM,IAAI,CAAC,mBAAmB,CAAC,iBAAiB,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,MAAc,EACd,gBAAgC,EAChC,sBAAyE;QAEzE,MAAM,IAAI,CAAC,qBAAqB,CAAC,gBAAgB,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;QAE5E,IACE,IAAI,CAAC,qBAAqB,CAAC,+BAA+B,EAAE;YAC5D,IAAI,CAAC,iCAAiC,EACtC,CAAC;YACD,MAAM,IAAI,CAAC,wBAAwB,CAAC,sBAAsB,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,UAAU,CAAC,IAAwB;QACvC,IAAI,CAAC,IAAI;YAAE,OAAO;QAClB,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACzE,IAAI,cAAc,EAAE,CAAC;YACnB,OAAO,cAAc,CAAC;QACxB,CAAC;QAED,MAAM,oBAAoB,GAAG,IAAI,CAAC,uBAAuB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpE,IAAI,CAAC,oBAAoB;YAAE,OAAO;QAElC,MAAM,QAAQ,GAAG,IAAA,uCAA0B,EACzC,oBAAoB,CAAC,aAAa,CACnC,CAAC;QACF,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAA,6BAAmB,EAAoB;YAChE,QAAQ;SACT,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,eAAe,CACnB,MAAyB,EACzB,QAAgC,EAChC,OAAoC;QAEpC,kFAAkF;QAClF,gCAAgC;QAChC,mFAAmF;QACnF,gGAAgG;QAChG,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAmB,CAAC;QACpD,MAAM,IAAI,CAAC,qBAAqB,CAAC,eAAe,CAC9C,MAAM,EACN,CAAC,GAAgB,EAAE,EAAE;YACnB,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACrC,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC,EACD,OAAO,CACR,CAAC;QAEF,MAAM,IAAA,gCAAsB,EAAC;YAC3B,IAAI,EAAE,MAAM,CAAC,KAAK;YAClB,OAAO;YACP,QAAQ,EAAE,CAAC,GAAgB,EAAE,EAAE;gBAC7B,IAAI,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;oBACnC,OAAO;gBACT,CAAC;gBACD,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC;YACvB,CAAC;SACF,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,oBAAoB,CACxB,MAAyB,EACzB,QAAgC,EAChC,OAAoC;QAEpC,uFAAuF;QACvF,gCAAgC;QAChC,mFAAmF;QACnF,gGAAgG;QAChG,MAAM,qBAAqB,GAAG,IAAI,GAAG,EAAmB,CAAC;QACzD,MAAM,IAAI,CAAC,qBAAqB,CAAC,oBAAoB,CACnD,MAAM,EACN,CAAC,GAAgB,EAAE,EAAE;YACnB,qBAAqB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC1C,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC,CACF,CAAC;QAEF,MAAM,IAAA,sCAA4B,EAAC;YACjC,IAAI,EAAE,MAAM,CAAC,KAAK;YAClB,OAAO;YACP,QAAQ,EAAE,CAAC,GAAgB,EAAE,EAAE;gBAC7B,IAAI,qBAAqB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;oBACxC,OAAO;gBACT,CAAC;gBACD,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC;YACvB,CAAC;SACF,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CACT,iBAAyD,EACzD,sBAAyE;QAEzE,MAAM,OAAO,CAAC,GAAG,CAAC;YAChB,IAAI,CAAC,mBAAmB,CAAC,iBAAiB,EAAE,IAAI,CAAC;YACjD,IAAI,CAAC,wBAAwB,CAAC,sBAAsB,EAAE,IAAI,CAAC;SAC5D,CAAC,CAAC;IACL,CAAC;IACD;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,mBAAmB,CACvB,iBAAyD,EACzD,QAAiB,KAAK;QAEtB,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,IAAI,EAAE;YAClC,mGAAmG;YACnG,mIAAmI;YACnI,mGAAmG;YACnG,IACE,CAAC,KAAK;gBACN,IAAI,CAAC,qBAAqB,CAAC,yBAAyB,EAAE;oBACpD,IAAI,CAAC,iCAAiC,EACxC,CAAC;gBACD,OAAO;YACT,CAAC;YAED,MAAM,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC,qBAAqB,EAAE,CAAC;YAC1E,IAAI,gBAAgB,GAAa,EAAE,CAAC;YACpC,KAAK,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,cAAc,EAAE,CAAC;gBAChD,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;oBACtC,MAAM,aAAa,GAAG,IAAI,CAAC,kCAAkC,CAAC;wBAC5D,MAAM;wBACN,KAAK,EAAE,CAAC,CAAC,KAAK;wBACd,yBAAyB,EAAE,UAAU;qBACtC,CAAC,CAAC;oBAEH,IAAI,OAAO,aAAa,KAAK,WAAW,EAAE,CAAC;wBACzC,OAAO,IAAI,CAAC;oBACd,CAAC;oBAED,OAAO,aAAa,CAAC,OAAO,KAAK,IAAI,CAAC;gBACxC,CAAC,CAAC,CAAC;gBAEH,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;oBACrB,MAAM,MAAM,GAAG,IAAA,cAAK,EAAC,SAAS,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;oBAC1D,IAAI,CAAC,MAAM,EAAE,KAAK,CAChB;wBACE,MAAM;wBACN,WAAW,EAAE,SAAS,CAAC,MAAM;wBAC7B,UAAU,EAAE,MAAM,CAAC,MAAM;wBACzB,SAAS,EAAE,IAAI,CAAC,mBAAmB;qBACpC,EACD,gCAAgC,CACjC,CAAC;oBAEF,IAAI,CAAC;wBACH,MAAM,OAAO,CAAC,GAAG,CACf,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE;4BACpC,MAAM,uBAAuB,GAAG,MAAM,IAAA,iCAAe,EAAC;gCACpD,oBAAoB,EAAE,MAAM;gCAC5B,cAAc,EAAE,UAAU;gCAC1B,IAAI;gCACJ,MAAM,EAAE,IAAI,CAAC,aAAa;gCAC1B,MAAM,EAAE,IAAI,CAAC,MAAM;6BACpB,CAAC,CAAC;4BAEH,KAAK,MAAM,EACT,aAAa,EACb,UAAU,GACX,IAAI,uBAAuB,EAAE,CAAC;gCAC7B,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;oCAC9C,IAAI,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE;wCACvC,aAAa;wCACb,KAAK;qCACN,CAAC,CAAC;gCACL,CAAC;4BACH,CAAC;4BAED,IAAI,CAAC,MAAM,EAAE,KAAK,CAChB;gCACE,MAAM;gCACN,UAAU;gCACV,eAAe,EAAE,IAAI,CAAC,MAAM;gCAC5B,YAAY,EAAE,uBAAuB,CAAC,MAAM;6BAC7C,EACD,mCAAmC,CACpC,CAAC;wBACJ,CAAC,CAAC,CACH,CAAC;oBACJ,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,IAAI,CAAC,MAAM,EAAE,KAAK,CAChB;4BACE,MAAM;4BACN,WAAW,EAAE,SAAS,CAAC,MAAM;4BAC7B,UAAU,EAAE,MAAM,CAAC,MAAM;4BACzB,KAAK,EAAE,KAAK,CAAC,OAAO;4BACpB,UAAU,EAAE,KAAK,CAAC,KAAK;yBACxB,EACD,uCAAuC,CACxC,CAAC;wBACF,MAAM,KAAK,CAAC;oBACd,CAAC;gBACH,CAAC;gBAED,IAAI,CAAC,qBAAqB,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBAC3D,gBAAgB,GAAG,gBAAgB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACvD,CAAC;YAED,IAAI,iBAAiB,EAAE,CAAC;gBACtB,IAAI,CAAC;oBACH,MAAM,iBAAiB,CAAC,gBAAgB,CAAC,CAAC;gBAC5C,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,IAAI,CAAC,MAAM,EAAE,KAAK,CAChB;wBACE,WAAW,EAAE,gBAAgB,CAAC,MAAM;wBACpC,GAAG;qBACJ,EACD,mCAAmC,CACpC,CAAC;oBACF,MAAM,GAAG,CAAC;gBACZ,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IACD;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,wBAAwB,CAC5B,sBAAyE,EACzE,QAAiB,KAAK;QAEtB,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,IAAI,EAAE;YAClC,mGAAmG;YACnG,mIAAmI;YACnI,mGAAmG;YACnG,IACE,CAAC,KAAK;gBACN,IAAI,CAAC,qBAAqB,CAAC,+BAA+B,EAAE;oBAC1D,IAAI,CAAC,iCAAiC,EACxC,CAAC;gBACD,OAAO;YACT,CAAC;YACD,MAAM,mBAAmB,GACvB,IAAI,CAAC,qBAAqB,CAAC,0BAA0B,EAAE,CAAC;YAC1D,IAAI,qBAAqB,GAAmB,EAAE,CAAC;YAC/C,KAAK,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,IAAI,mBAAmB,EAAE,CAAC;gBAC1D,MAAM,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;oBAC3C,MAAM,aAAa,GAAG,IAAI,CAAC,kCAAkC,CAAC;wBAC5D,MAAM;wBACN,KAAK,EAAE,CAAC,CAAC,KAAK;wBACd,yBAAyB,EAAE,eAAe;qBAC3C,CAAC,CAAC;oBAEH,IAAI,OAAO,aAAa,KAAK,WAAW,EAAE,CAAC;wBACzC,OAAO,IAAI,CAAC;oBACd,CAAC;oBAED,OAAO,aAAa,CAAC,OAAO,KAAK,IAAI,CAAC;gBACxC,CAAC,CAAC,CAAC;gBAEH,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;oBACrB,MAAM,MAAM,GAAG,IAAA,cAAK,EAAC,SAAS,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;oBAC1D,IAAI,CAAC,MAAM,EAAE,KAAK,CAChB;wBACE,MAAM;wBACN,iBAAiB,EAAE,SAAS,CAAC,MAAM;wBACnC,UAAU,EAAE,MAAM,CAAC,MAAM;wBACzB,SAAS,EAAE,IAAI,CAAC,mBAAmB;qBACpC,EACD,sCAAsC,CACvC,CAAC;oBAEF,IAAI,CAAC;wBACH,MAAM,OAAO,CAAC,GAAG,CACf,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE;4BACpC,MAAM,IAAA,iCAAe,EAAC;gCACpB,oBAAoB,EAAE,MAAM;gCAC5B,cAAc,EAAE,eAAe;gCAC/B,IAAI;gCACJ,MAAM,EAAE,IAAI,CAAC,aAAa;gCAC1B,MAAM,EAAE,IAAI,CAAC,MAAM;6BACpB,CAAC,CAAC;4BAEH,IAAI,CAAC,MAAM,EAAE,KAAK,CAChB;gCACE,MAAM;gCACN,UAAU;gCACV,oBAAoB,EAAE,IAAI,CAAC,MAAM;6BAClC,EACD,yCAAyC,CAC1C,CAAC;wBACJ,CAAC,CAAC,CACH,CAAC;oBACJ,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,IAAI,CAAC,MAAM,EAAE,KAAK,CAChB;4BACE,MAAM;4BACN,iBAAiB,EAAE,SAAS,CAAC,MAAM;4BACnC,UAAU,EAAE,MAAM,CAAC,MAAM;4BACzB,KAAK,EAAE,KAAK,CAAC,OAAO;4BACpB,UAAU,EAAE,KAAK,CAAC,KAAK;yBACxB,EACD,6CAA6C,CAC9C,CAAC;wBACF,MAAM,KAAK,CAAC;oBACd,CAAC;gBACH,CAAC;gBAED,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;gBACrE,qBAAqB,GAAG,qBAAqB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACtE,CAAC;YAED,IAAI,sBAAsB,EAAE,CAAC;gBAC3B,IAAI,CAAC;oBACH,MAAM,sBAAsB,CAAC,qBAAqB,CAAC,CAAC;gBACtD,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAI,CAAC,MAAM,EAAE,KAAK,CAChB;wBACE,iBAAiB,EAAE,qBAAqB,CAAC,MAAM;wBAC/C,KAAK,EAAE,KAAK,CAAC,OAAO;wBACpB,UAAU,EAAE,KAAK,CAAC,KAAK;qBACxB,EACD,wCAAwC,CACzC,CAAC;oBACF,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,kCAAkC,CAAC,EACjC,MAAM,EACN,KAAK,EACL,yBAAyB,GACgB;QAGzC,IAAI,CAAC,IAAI,CAAC,mCAAmC,EAAE,CAAC;YAC9C,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,mCAAmC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACjE,OAAO,GAAG,IAAI,GAAG,CAAC,yBAAyB,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC1D,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACK,KAAK,CAAC,aAAa,CAAI,SAA2B;QACxD,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QAC/B,IAAI,CAAC;YACH,MAAM,SAAS,EAAE,CAAC;QACpB,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC;CACF;AA/bD,gEA+bC"}
|
|
@@ -5,6 +5,7 @@ interface FlushDataToDiskInput<TGraphObject = Entity | Relationship> {
|
|
|
5
5
|
collectionType: CollectionType;
|
|
6
6
|
data: TGraphObject[];
|
|
7
7
|
pretty?: boolean;
|
|
8
|
+
logger?: any;
|
|
8
9
|
}
|
|
9
10
|
interface GraphObjectToFilePath<TGraphObject = Entity | Relationship> {
|
|
10
11
|
graphDataPath: string;
|
|
@@ -15,5 +16,5 @@ interface GraphObjectToFilePath<TGraphObject = Entity | Relationship> {
|
|
|
15
16
|
* creating a symlink in the 'index' directory
|
|
16
17
|
* based on the entity or relationship '_type'.
|
|
17
18
|
*/
|
|
18
|
-
export declare function flushDataToDisk<TGraphObject = Entity | Relationship>({ storageDirectoryPath, collectionType, data, pretty, }: FlushDataToDiskInput<TGraphObject>): Promise<GraphObjectToFilePath<TGraphObject>[]>;
|
|
19
|
+
export declare function flushDataToDisk<TGraphObject = Entity | Relationship>({ storageDirectoryPath, collectionType, data, pretty, logger, }: FlushDataToDiskInput<TGraphObject>): Promise<GraphObjectToFilePath<TGraphObject>[]>;
|
|
19
20
|
export {};
|
|
@@ -14,12 +14,24 @@ const path_1 = require("./path");
|
|
|
14
14
|
* creating a symlink in the 'index' directory
|
|
15
15
|
* based on the entity or relationship '_type'.
|
|
16
16
|
*/
|
|
17
|
-
async function flushDataToDisk({ storageDirectoryPath, collectionType, data, pretty, }) {
|
|
17
|
+
async function flushDataToDisk({ storageDirectoryPath, collectionType, data, pretty, logger, }) {
|
|
18
18
|
// split the data by type first
|
|
19
19
|
const groupedCollections = (0, groupBy_1.default)(data, '_type');
|
|
20
|
+
const totalObjects = data.length;
|
|
21
|
+
const typeCount = Object.keys(groupedCollections).length;
|
|
22
|
+
const chunkInfo = Object.entries(groupedCollections)
|
|
23
|
+
.map(([type, items]) => `${type}:${items.length}`)
|
|
24
|
+
.join(', ');
|
|
25
|
+
logger?.debug({
|
|
26
|
+
stepId: storageDirectoryPath,
|
|
27
|
+
collectionType,
|
|
28
|
+
totalObjects,
|
|
29
|
+
typeCount,
|
|
30
|
+
chunkInfo,
|
|
31
|
+
}, 'Flushing data to disk');
|
|
20
32
|
// for each collection, write the data to disk,
|
|
21
33
|
// then symlink to index directory
|
|
22
|
-
|
|
34
|
+
const results = await (0, p_map_1.default)(Object.entries(groupedCollections), async ([type, collection]) => {
|
|
23
35
|
const filename = generateJsonFilename();
|
|
24
36
|
const graphDataPath = (0, path_1.buildObjectCollectionFilePath)({
|
|
25
37
|
storageDirectoryPath,
|
|
@@ -27,22 +39,47 @@ async function flushDataToDisk({ storageDirectoryPath, collectionType, data, pre
|
|
|
27
39
|
filename,
|
|
28
40
|
});
|
|
29
41
|
const indexPath = (0, path_1.buildIndexFilePath)({ type, collectionType, filename });
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
42
|
+
try {
|
|
43
|
+
await (0, fileSystem_1.writeJsonToPath)({
|
|
44
|
+
path: graphDataPath,
|
|
45
|
+
data: {
|
|
46
|
+
[collectionType]: collection,
|
|
47
|
+
},
|
|
48
|
+
pretty,
|
|
49
|
+
});
|
|
50
|
+
await (0, fileSystem_1.symlink)({
|
|
51
|
+
sourcePath: graphDataPath,
|
|
52
|
+
destinationPath: indexPath,
|
|
53
|
+
});
|
|
54
|
+
logger?.debug({
|
|
55
|
+
type,
|
|
56
|
+
count: collection.length,
|
|
57
|
+
filename,
|
|
58
|
+
graphDataPath,
|
|
59
|
+
}, 'Chunk written to disk successfully');
|
|
60
|
+
return {
|
|
61
|
+
graphDataPath,
|
|
62
|
+
collection,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
catch (error) {
|
|
66
|
+
logger?.error({
|
|
67
|
+
error,
|
|
68
|
+
type,
|
|
69
|
+
count: collection.length,
|
|
70
|
+
filename,
|
|
71
|
+
graphDataPath,
|
|
72
|
+
}, 'Failed to write chunk to disk');
|
|
73
|
+
throw error;
|
|
74
|
+
}
|
|
45
75
|
}, { concurrency: 3 });
|
|
76
|
+
logger?.info({
|
|
77
|
+
stepId: storageDirectoryPath,
|
|
78
|
+
collectionType,
|
|
79
|
+
chunksWritten: results.length,
|
|
80
|
+
totalObjectsWritten: results.reduce((sum, r) => sum + r.collection.length, 0),
|
|
81
|
+
}, 'Successfully flushed all chunks to disk');
|
|
82
|
+
return results;
|
|
46
83
|
}
|
|
47
84
|
function generateJsonFilename() {
|
|
48
85
|
return `${(0, crypto_1.randomUUID)()}.json`;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"flushDataToDisk.js","sourceRoot":"","sources":["../../../../src/storage/FileSystemGraphObjectStore/flushDataToDisk.ts"],"names":[],"mappings":";;;;;
|
|
1
|
+
{"version":3,"file":"flushDataToDisk.js","sourceRoot":"","sources":["../../../../src/storage/FileSystemGraphObjectStore/flushDataToDisk.ts"],"names":[],"mappings":";;;;;AAgCA,0CAqGC;AArID,kDAAyB;AACzB,mCAA4C;AAC5C,6DAAqC;AAIrC,iDAA4D;AAE5D,iCAIgB;AAehB;;;;GAIG;AACI,KAAK,UAAU,eAAe,CAAuC,EAC1E,oBAAoB,EACpB,cAAc,EACd,IAAI,EACJ,MAAM,EACN,MAAM,GAC6B;IAGnC,+BAA+B;IAC/B,MAAM,kBAAkB,GAAG,IAAA,iBAAO,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAElD,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC;IACjC,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC;IACzD,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC;SACjD,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;SACjD,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,MAAM,EAAE,KAAK,CACX;QACE,MAAM,EAAE,oBAAoB;QAC5B,cAAc;QACd,YAAY;QACZ,SAAS;QACT,SAAS;KACV,EACD,uBAAuB,CACxB,CAAC;IAEF,+CAA+C;IAC/C,kCAAkC;IAClC,MAAM,OAAO,GAAG,MAAM,IAAA,eAAI,EACxB,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,EAClC,KAAK,EAAE,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,EAAE;QAC3B,MAAM,QAAQ,GAAG,oBAAoB,EAAE,CAAC;QACxC,MAAM,aAAa,GAAG,IAAA,oCAA6B,EAAC;YAClD,oBAAoB;YACpB,cAAc;YACd,QAAQ;SACT,CAAC,CAAC;QACH,MAAM,SAAS,GAAG,IAAA,yBAAkB,EAAC,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAC,CAAC;QAEzE,IAAI,CAAC;YACH,MAAM,IAAA,4BAAe,EAAC;gBACpB,IAAI,EAAE,aAAa;gBACnB,IAAI,EAAE;oBACJ,CAAC,cAAc,CAAC,EAAE,UAAU;iBAC7B;gBACD,MAAM;aACP,CAAC,CAAC;YAEH,MAAM,IAAA,oBAAO,EAAC;gBACZ,UAAU,EAAE,aAAa;gBACzB,eAAe,EAAE,SAAS;aAC3B,CAAC,CAAC;YAEH,MAAM,EAAE,KAAK,CACX;gBACE,IAAI;gBACJ,KAAK,EAAE,UAAU,CAAC,MAAM;gBACxB,QAAQ;gBACR,aAAa;aACd,EACD,oCAAoC,CACrC,CAAC;YAEF,OAAO;gBACL,aAAa;gBACb,UAAU;aACX,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,EAAE,KAAK,CACX;gBACE,KAAK;gBACL,IAAI;gBACJ,KAAK,EAAE,UAAU,CAAC,MAAM;gBACxB,QAAQ;gBACR,aAAa;aACd,EACD,+BAA+B,CAChC,CAAC;YACF,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC,EACD,EAAE,WAAW,EAAE,CAAC,EAAE,CACnB,CAAC;IAEF,MAAM,EAAE,IAAI,CACV;QACE,MAAM,EAAE,oBAAoB;QAC5B,cAAc;QACd,aAAa,EAAE,OAAO,CAAC,MAAM;QAC7B,mBAAmB,EAAE,OAAO,CAAC,MAAM,CACjC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EACrC,CAAC,CACF;KACF,EACD,yCAAyC,CAC1C,CAAC;IAEF,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,oBAAoB;IAC3B,OAAO,GAAG,IAAA,mBAAI,GAAE,OAAO,CAAC;AAC1B,CAAC"}
|
|
@@ -335,31 +335,8 @@ async function uploadData({ job, apiClient, logger }, type, data, uploadBatchSiz
|
|
|
335
335
|
*/
|
|
336
336
|
async function abortSynchronization({ logger, apiClient, job, reason, terminalStatus, }) {
|
|
337
337
|
logger.info({ integrationJobId: job.id, reason, terminalStatus }, 'Aborting synchronization job...');
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
return response.data.job;
|
|
341
|
-
}, {
|
|
342
|
-
maxAttempts: 5,
|
|
343
|
-
delay: 200,
|
|
344
|
-
factor: 1.05,
|
|
345
|
-
handleError(err, context) {
|
|
346
|
-
if (context.attemptsRemaining > 0) {
|
|
347
|
-
logger.warn({
|
|
348
|
-
err,
|
|
349
|
-
'err.$response': err?.$response,
|
|
350
|
-
context,
|
|
351
|
-
}, 'Error occurred while aborting synchronization job. Retrying request.');
|
|
352
|
-
}
|
|
353
|
-
if (context.attemptsRemaining === 0) {
|
|
354
|
-
logger.error({
|
|
355
|
-
err,
|
|
356
|
-
'err.$response': err?.$response,
|
|
357
|
-
context,
|
|
358
|
-
}, 'Error occurred while aborting synchronization job');
|
|
359
|
-
throw (0, error_1.synchronizationApiError)(err, 'Error occurred while aborting synchronization job.');
|
|
360
|
-
}
|
|
361
|
-
},
|
|
362
|
-
});
|
|
338
|
+
const response = await apiClient.post(`/persister/synchronization/jobs/${job.id}/abort`, { reason, terminalStatus });
|
|
339
|
+
return response.data.job;
|
|
363
340
|
}
|
|
364
341
|
function cleanAxiosError(err) {
|
|
365
342
|
if (err.config?.headers?.Authorization) {
|