@jupiterone/integration-sdk-runtime 17.1.0 → 17.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/execution/executeIntegration.js +3 -1
- package/dist/src/execution/executeIntegration.js.map +1 -1
- package/dist/src/execution/uploader.d.ts +3 -1
- package/dist/src/execution/uploader.js +8 -1
- package/dist/src/execution/uploader.js.map +1 -1
- package/dist/src/logger/index.js +2 -0
- package/dist/src/logger/index.js.map +1 -1
- package/dist/src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.d.ts +6 -1
- package/dist/src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.js +97 -25
- package/dist/src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.js.map +1 -1
- package/dist/src/storage/FileSystemGraphObjectStore/flushDataToDisk.d.ts +2 -1
- package/dist/src/storage/FileSystemGraphObjectStore/flushDataToDisk.js +54 -17
- package/dist/src/storage/FileSystemGraphObjectStore/flushDataToDisk.js.map +1 -1
- package/dist/tsconfig.dist.tsbuildinfo +1 -1
- package/package.json +4 -4
|
@@ -132,7 +132,9 @@ async function executeWithContext(context, config, options = {}) {
|
|
|
132
132
|
if (shouldPublishDiskUsageMetric) {
|
|
133
133
|
diskUsagePublishInterval = createDiskUsagePublishInterval();
|
|
134
134
|
}
|
|
135
|
-
const { graphObjectStore = new storage_1.FileSystemGraphObjectStore(
|
|
135
|
+
const { graphObjectStore = new storage_1.FileSystemGraphObjectStore({
|
|
136
|
+
logger,
|
|
137
|
+
}), createStepGraphObjectDataUploader, resultsCallback, } = options;
|
|
136
138
|
let duplicateKeyTracker = new duplicateKeyTracker_1.InMemoryDuplicateKeyTracker(config.normalizeGraphObjectKey);
|
|
137
139
|
if (process.env.USE_ON_DISK_DKT) {
|
|
138
140
|
// conditionally require so the dependency can remain optional
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executeIntegration.js","sourceRoot":"","sources":["../../../src/execution/executeIntegration.ts"],"names":[],"mappings":";;;;;AA4EA,8DAyBC;AAKD,gEAkCC;AA2BD,
|
|
1
|
+
{"version":3,"file":"executeIntegration.js","sourceRoot":"","sources":["../../../src/execution/executeIntegration.ts"],"names":[],"mappings":";;;;;AA4EA,8DAyBC;AAKD,gEAkCC;AA2BD,gDAwMC;AA/WD,2EAa0C;AAE1C,8CAOuB;AACvB,sCAImB;AACnB,wCAA2C;AAC3C,wCAA0E;AAC1E,gDAAwB;AACxB,yCAAwE;AACxE,yCAA6C;AAC7C,iCAIgB;AAEhB,6DAA0D;AAC1D,+DAA4D;AAC5D,6CAAuD;AACvD,+EAA4E;AAC5E,+DAG+B;AAC/B,iGAA8F;AAuB9F,MAAM,kCAAkC,GAAG,KAAK,GAAG,CAAC,CAAC;AAErD;;;GAGG;AACI,KAAK,UAAU,yBAAyB,CAC7C,MAAmC,EACnC,gBAAkC,EAClC,OAAmC;IAEnC,MAAM,MAAM,GAAG,IAAA,gCAAuB,EAAC;QACrC,IAAI,EAAE,OAAO;QACb,gBAAgB,EAAE,MAAM;QACxB,MAAM,EAAE,OAAO,EAAE,MAAM;KACxB,CAAC,CAAC;IACH,MAAM,wBAAwB,GAAG,IAAA,+CAAsC,EACrE,GAAG,EAAE,CAAC,MAAM,CACb,CAAC;IACF,MAAM,MAAM,GAAG,MAAM,0BAA0B,CAC7C,MAAM,EACN,IAAA,qDAA0C,EAAC,MAAM,CAAC,EAClD,MAAM,EACN,gBAAgB,EAChB;QACE,GAAG,OAAO;QACV,sBAAsB,EAAE,OAAO,EAAE,sBAAsB,IAAI,IAAI;KAChE,CACF,CAAC;IACF,IAAA,iDAAwC,EAAC,wBAAwB,CAAC,CAAC;IACnE,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACI,KAAK,UAAU,0BAA0B,CAI9C,MAAyB,EACzB,QAAiD,EACjD,MAAuD,EACvD,gBAAkC,EAClC,UAAqC,EAAE;IAEvC,IAAI,OAAO,CAAC,sBAAsB,KAAK,IAAI,EAAE,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,qCAAqC,GAAG,MAAM,CAAC;IAC7D,CAAC;IACD,MAAM,yBAAyB,GAAG,IAAA,mCAAgB,EAChD,QAAQ,EACR,IAAA,iCAAe,EAAC,MAAM,CAAC,CACxB,CAAC;IAEF,OAAO,IAAA,uBAAa,EAAC;QACnB,MAAM;QACN,UAAU,EAAE,kBAAkB;QAC9B,SAAS,EAAE,GAAG,EAAE,CACd,kBAAkB,CAChB;YACE,QAAQ,EAAE,yBAAyB;YACnC,MAAM;YACN,gBAAgB;YAChB,eAAe,EACb,MAAM,CAAC,mBAAmB,EAAE,CAAC,yBAAyB,CAAC,IAAI,EAAE;SAChE,EACD,MAAM,EACN,OAAO,CACR;KACJ,CAAC,CAAC;AACL,CAAC;AAED,SAAS,sBAAsB,CAC7B,OAA0B,EAC1B,IAAY;IAEZ,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC;QAC3B,IAAI,EAAE,YAAY;QAClB,KAAK,EAAE,IAAI;QACX,IAAI,EAAE,OAAO;KACd,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,yBAAyB,CAEtC,OAA0B;IAC1B,IAAI,CAAC,CAAC,MAAM,IAAA,0CAA6B,GAAE,CAAC,EAAE,CAAC;QAC7C,OAAO;IACT,CAAC;IAED,sBAAsB,CAAC,OAAO,EAAE,MAAM,IAAA,wCAA2B,GAAE,CAAC,CAAC;AACvE,CAAC;AAED;;;GAGG;AACI,KAAK,UAAU,kBAAkB,CAItC,OAA0B,EAC1B,MAAkE,EAClE,UAAqC,EAAE;IAEvC,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAE3B,wEAAwE;IACxE,oEAAoE;IACpE,MAAM,4BAA4B,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC;IAE5E,MAAM,CAAC,IAAI,CACT;QACE,kBAAkB,EAAE,IAAA,iCAAoB,GAAE;KAC3C,EACD,mCAAmC,CACpC,CAAC;IAEF,IAAI,wBAAoD,CAAC;IACzD,IAAI,iBAAiB,GAAG,KAAK,CAAC;IAE9B,SAAS,8BAA8B;QACrC,OAAO,WAAW,CAAC,GAAG,EAAE;YACtB,IAAA,0CAA6B,GAAE;iBAC5B,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;gBAChB,IAAI,CAAC,OAAO;oBAAE,OAAO;gBAErB,OAAO,IAAA,wCAA2B,GAAE,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;oBACjD,IAAI,iBAAiB,EAAE,CAAC;wBACtB,OAAO;oBACT,CAAC;oBAED,sBAAsB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;gBACxC,CAAC,CAAC,CAAC;YACL,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACb,IAAI,iBAAiB,EAAE,CAAC;oBACtB,OAAO;gBACT,CAAC;gBAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,oCAAoC,CAAC,CAAC;YACtE,CAAC,CAAC,CAAC;QACP,CAAC,EAAE,kCAAkC,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,CAAC;QACH,IAAI,4BAA4B,EAAE,CAAC;YACjC,MAAM,yBAAyB,CAAC,OAAO,CAAC,CAAC;QAC3C,CAAC;QAED,IAAI,CAAC;YACH,MAAM,IAAA,mCAAsB,GAAE,CAAC;YAE/B,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,kBAAkB,EAAE,CAAC,OAAO,CAAC,CAAC;YAC7C,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC;gBAC9B,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,MAAM,uBAAuB,GAC3B,MAAM,MAAM,CAAC,kBAAkB,EAAE,CAAC,OAAO,CAAC,CAAC;YAE7C,MAAM,qBAAqB,GACzB,uBAAuB;gBACvB,IAAA,gCAAyB,EAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;YAErD,IAAA,oCAAuB,EAAC,MAAM,CAAC,gBAAgB,EAAE,qBAAqB,CAAC,CAAC;YAExE,MAAM,eAAe,GAAG,IAAA,qEAAiC,EAAC;gBACxD,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,qBAAqB;gBACrB,eAAe,EAAE,OAAO,CAAC,QAAQ,EAAE,eAAe,EAAE,GAAG,CACrD,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,iBAAiB,CACrC;aACF,CAAC,CAAC;YAEH,IAAI,4BAA4B,EAAE,CAAC;gBACjC,wBAAwB,GAAG,8BAA8B,EAAE,CAAC;YAC9D,CAAC;YAED,MAAM,EACJ,gBAAgB,GAAG,IAAI,oCAA0B,CAAC;gBAChD,MAAM;aACP,CAAC,EACF,iCAAiC,EACjC,eAAe,GAChB,GAAG,OAAO,CAAC;YAEZ,IAAI,mBAAmB,GACrB,IAAI,iDAA2B,CAAC,MAAM,CAAC,uBAAuB,CAAC,CAAC;YAElE,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC;gBAChC,8DAA8D;gBAC9D,IAAI,CAAC;oBACH,MAAM,EACJ,yBAAyB,GAC1B,GAAG,OAAO,CAAC,6BAA6B,CAAC,CAAC;oBAC3C,mBAAmB,GAAG,IAAI,yBAAyB,CAAC;wBAClD,QAAQ,EAAE,cAAI,CAAC,IAAI,CACjB,OAAO,CAAC,GAAG,EAAE,EACb,2CAA8B,EAC9B,gBAAgB,CACjB;qBACF,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,CAAC,IAAI,CACT,EAAE,GAAG,EAAE,EACP,qGAAqG,CACtG,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,MAAM,sBAAsB,GAAG,MAAM,IAAA,mBAAY,EAAC;gBAChD,gBAAgB,EAAE,OAAO;gBACzB,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,eAAe;gBACf,eAAe,EAAE,MAAM,CAAC,eAAe;gBACvC,mBAAmB;gBACnB,gBAAgB;gBAChB,SAAS,EAAE,IAAI,0BAAe,EAAE;gBAChC,iCAAiC;gBACjC,eAAe,EAAE,MAAM,CAAC,eAAe;gBACvC,qBAAqB,EAAE,MAAM,CAAC,qBAAqB;gBACnD,cAAc,EAAE,SAAS;gBACzB,oBAAoB,EAAE,SAAS;gBAC/B,oBAAoB,EAAE,MAAM,CAAC,oBAAoB;gBACjD,uBAAuB,EAAE,MAAM,CAAC,uBAAuB;aACxD,CAAC,CAAC;YAEH,MAAM,eAAe,GAAG,IAAA,uDAAgD,EACtE,sBAAsB,CACvB,CAAC;YAEF,MAAM,OAAO,GAA6B;gBACxC,sBAAsB;gBACtB,eAAe,EAAE,MAAM,CAAC,sBAAsB;oBAC5C,CAAC,CAAC,mBAAmB,CAAC,kBAAkB,EAAE;oBAC1C,CAAC,CAAC,SAAS;gBACb,QAAQ,EAAE;oBACR,eAAe;iBAChB;aACF,CAAC;YAEF,MAAM,IAAA,4BAAe,EAAC;gBACpB,IAAI,EAAE,cAAc;gBACpB,IAAI,EAAE,OAAO;aACd,CAAC,CAAC;YAEH,IAAI,eAAe,IAAI,IAAI,EAAE,CAAC;gBAC5B,IAAI,CAAC;oBACH,MAAM,eAAe,CAAC,OAAO,CAAC,CAAC;gBACjC,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,OAAO,CAAC,MAAM,CAAC,IAAI,CACjB,GAAG,EACH,oDAAoD,CACrD,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,IAAA,mDAAwB,EAAC,OAAO,EAAE,CAAC,IAAI,EAAE,eAAe,EAAE,EAAE;gBAC1D,IACE,IAAI,CAAC,MAAM,KAAK,uCAAgB,CAAC,OAAO;oBACxC,eAAe,CAAC,MAAM,EACtB,CAAC;oBACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,EAAE,eAAe,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,EACpC,iGAAiG;wBAC/F,wDAAwD,CAC3D,CAAC;gBACJ,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,OAAO,OAAO,CAAC;QACjB,CAAC;gBAAS,CAAC;YACT,iBAAiB,GAAG,IAAI,CAAC;YAEzB,IAAI,wBAAwB,EAAE,CAAC;gBAC7B,aAAa,CAAC,wBAAwB,CAAC,CAAC;YAC1C,CAAC;YAED,IAAI,4BAA4B,EAAE,CAAC;gBACjC,MAAM,yBAAyB,CAAC,OAAO,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;IACH,CAAC;YAAS,CAAC;QACT,IAAI,MAAM,EAAE,cAAc,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YACvC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,0DAA0D;gBAC1D,uEAAuE;gBACvE,4BAA4B;gBAC5B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,wCAAwC,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { IntegrationLogger } from '@jupiterone/integration-sdk-core';
|
|
1
2
|
import { FlushedGraphObjectData } from '../storage/types';
|
|
2
3
|
import { SynchronizationJobContext } from '../synchronization';
|
|
3
4
|
export interface StepGraphObjectDataUploader {
|
|
@@ -7,12 +8,13 @@ export interface StepGraphObjectDataUploader {
|
|
|
7
8
|
}
|
|
8
9
|
export type CreateStepGraphObjectDataUploaderFunction = (stepId: string) => StepGraphObjectDataUploader;
|
|
9
10
|
export interface CreateQueuedStepGraphObjectDataUploaderParams {
|
|
11
|
+
logger: IntegrationLogger;
|
|
10
12
|
stepId: string;
|
|
11
13
|
uploadConcurrency: number;
|
|
12
14
|
upload: (graphObjectData: FlushedGraphObjectData) => Promise<void>;
|
|
13
15
|
onThrottleEnqueue?: () => void;
|
|
14
16
|
}
|
|
15
|
-
export declare function createQueuedStepGraphObjectDataUploader({ stepId, uploadConcurrency: maximumQueueSize, upload, onThrottleEnqueue, }: CreateQueuedStepGraphObjectDataUploaderParams): StepGraphObjectDataUploader;
|
|
17
|
+
export declare function createQueuedStepGraphObjectDataUploader({ logger, stepId, uploadConcurrency: maximumQueueSize, upload, onThrottleEnqueue, }: CreateQueuedStepGraphObjectDataUploaderParams): StepGraphObjectDataUploader;
|
|
16
18
|
export interface CreatePersisterApiStepGraphObjectDataUploaderParams {
|
|
17
19
|
stepId: string;
|
|
18
20
|
synchronizationJobContext: SynchronizationJobContext;
|
|
@@ -9,7 +9,7 @@ const integration_sdk_core_1 = require("@jupiterone/integration-sdk-core");
|
|
|
9
9
|
const dist_1 = __importDefault(require("p-queue/dist"));
|
|
10
10
|
const synchronization_1 = require("../synchronization");
|
|
11
11
|
const crypto_1 = require("crypto");
|
|
12
|
-
function createQueuedStepGraphObjectDataUploader({ stepId, uploadConcurrency: maximumQueueSize, upload, onThrottleEnqueue, }) {
|
|
12
|
+
function createQueuedStepGraphObjectDataUploader({ logger, stepId, uploadConcurrency: maximumQueueSize, upload, onThrottleEnqueue, }) {
|
|
13
13
|
const queue = new dist_1.default({
|
|
14
14
|
concurrency: maximumQueueSize,
|
|
15
15
|
});
|
|
@@ -44,6 +44,7 @@ function createQueuedStepGraphObjectDataUploader({ stepId, uploadConcurrency: ma
|
|
|
44
44
|
queue
|
|
45
45
|
.add(() => upload(graphObjectData))
|
|
46
46
|
.catch((err) => {
|
|
47
|
+
logger.warn({ err, stepId, graphObjectData }, 'Error uploading graph object data batch');
|
|
47
48
|
// Do not pause the queue entirely. We will try to prevent additional
|
|
48
49
|
// tasks from being added to the queue, but even if an error occurs,
|
|
49
50
|
// we should try uploading the remaining data that we have queued up.
|
|
@@ -69,6 +70,11 @@ function createQueuedStepGraphObjectDataUploader({ stepId, uploadConcurrency: ma
|
|
|
69
70
|
// this time, we could be receiving additional tasks in our queue that
|
|
70
71
|
// will grow the queue.
|
|
71
72
|
completed = true;
|
|
73
|
+
logger.debug({
|
|
74
|
+
stepId,
|
|
75
|
+
uploadErrorCount: uploadErrors.length,
|
|
76
|
+
typesInvolvedInFailures: Array.from(typesInvolvedInFailures),
|
|
77
|
+
}, 'Upload queue processing complete');
|
|
72
78
|
}
|
|
73
79
|
if (uploadErrors.length) {
|
|
74
80
|
throw new integration_sdk_core_1.UploadError(`Error(s) uploading graph object data (stepId=${stepId}, errorMessages=${uploadErrors.join(',')})`, Array.from(typesInvolvedInFailures.values()));
|
|
@@ -91,6 +97,7 @@ function jobContextWithUploaderMetadataLogger({ synchronizationJobContext, uploa
|
|
|
91
97
|
}
|
|
92
98
|
function createPersisterApiStepGraphObjectDataUploader({ stepId, synchronizationJobContext, uploadConcurrency, uploadBatchSizeInBytes = synchronization_1.DEFAULT_UPLOAD_BATCH_SIZE_IN_BYTES, }) {
|
|
93
99
|
return createQueuedStepGraphObjectDataUploader({
|
|
100
|
+
logger: synchronizationJobContext.logger,
|
|
94
101
|
stepId,
|
|
95
102
|
uploadConcurrency,
|
|
96
103
|
async upload(graphObjectData) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.js","sourceRoot":"","sources":["../../../src/execution/uploader.ts"],"names":[],"mappings":";;;;;
|
|
1
|
+
{"version":3,"file":"uploader.js","sourceRoot":"","sources":["../../../src/execution/uploader.ts"],"names":[],"mappings":";;;;;AA+BA,0FAqGC;AA+BD,sGAoCC;AAvMD,2EAG0C;AAC1C,wDAAkC;AAElC,wDAI4B;AAC5B,mCAA4C;AAoB5C,SAAgB,uCAAuC,CAAC,EACtD,MAAM,EACN,MAAM,EACN,iBAAiB,EAAE,gBAAgB,EACnC,MAAM,EACN,iBAAiB,GAC6B;IAC9C,MAAM,KAAK,GAAG,IAAI,cAAM,CAAC;QACvB,WAAW,EAAE,gBAAgB;KAC9B,CAAC,CAAC;IAEH,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,MAAM,YAAY,GAAY,EAAE,CAAC;IACjC,MAAM,uBAAuB,GAAG,IAAI,GAAG,EAAU,CAAC;IAClD,OAAO;QACL,MAAM;QACN,KAAK,CAAC,OAAO,CAAC,eAAe;YAC3B,IAAI,SAAS,EAAE,CAAC;gBACd,qEAAqE;gBACrE,iDAAiD;gBACjD,OAAO;YACT,CAAC;YAED,gEAAgE;YAChE,qEAAqE;YACrE,yEAAyE;YACzE,uCAAuC;YACvC,EAAE;YACF,0EAA0E;YAC1E,yEAAyE;YACzE,sEAAsE;YACtE,wEAAwE;YACxE,uCAAuC;YACvC,IAAI,KAAK,CAAC,IAAI,IAAI,gBAAgB,EAAE,CAAC;gBACnC,IAAI,iBAAiB,EAAE,CAAC;oBACtB,iEAAiE;oBACjE,iBAAiB,EAAE,CAAC;gBACtB,CAAC;gBAED,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;YACxB,CAAC;YAED,KAAK;iBACF,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;iBAClC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACb,MAAM,CAAC,IAAI,CACT,EAAE,GAAG,EAAE,MAAM,EAAE,eAAe,EAAE,EAChC,yCAAyC,CAC1C,CAAC;gBACF,qEAAqE;gBACrE,oEAAoE;gBACpE,qEAAqE;gBACrE,sEAAsE;gBACtE,qEAAqE;gBACrE,6DAA6D;gBAC7D,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACvB,IAAI,eAAe,EAAE,CAAC;oBACpB,eAAe,CAAC,QAAQ,CAAC,OAAO,CAC9B,CAAC,MAAM,EAAE,EAAE,CAAC,uBAAuB,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,EACrD,uBAAuB,CACxB,CAAC;oBACF,eAAe,CAAC,aAAa,CAAC,OAAO,CACnC,CAAC,YAAY,EAAE,EAAE,CAAC,uBAAuB,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,EACjE,uBAAuB,CACxB,CAAC;gBACJ,CAAC;YACH,CAAC,CAAC,CAAC;QACP,CAAC;QAED,KAAK,CAAC,wBAAwB;YAC5B,IAAI,CAAC;gBACH,wEAAwE;gBACxE,oEAAoE;gBACpE,yEAAyE;gBACzE,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC;YACvB,CAAC;oBAAS,CAAC;gBACT,uEAAuE;gBACvE,sEAAsE;gBACtE,uBAAuB;gBACvB,SAAS,GAAG,IAAI,CAAC;gBAEjB,MAAM,CAAC,KAAK,CACV;oBACE,MAAM;oBACN,gBAAgB,EAAE,YAAY,CAAC,MAAM;oBACrC,uBAAuB,EAAE,KAAK,CAAC,IAAI,CAAC,uBAAuB,CAAC;iBAC7D,EACD,kCAAkC,CACnC,CAAC;YACJ,CAAC;YAED,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;gBACxB,MAAM,IAAI,kCAAW,CACnB,gDAAgD,MAAM,mBAAmB,YAAY,CAAC,IAAI,CACxF,GAAG,CACJ,GAAG,EACJ,KAAK,CAAC,IAAI,CAAC,uBAAuB,CAAC,MAAM,EAAE,CAAC,CAC7C,CAAC;YACJ,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,oCAAoC,CAAC,EAC5C,yBAAyB,EACzB,QAAQ,EACR,MAAM,GAKP;IACC,OAAO;QACL,GAAG,yBAAyB;QAC5B,MAAM,EAAE,yBAAyB,CAAC,MAAM,CAAC,KAAK,CAAC;YAC7C,QAAQ;YACR,MAAM;SACP,CAAC;KACH,CAAC;AACJ,CAAC;AASD,SAAgB,6CAA6C,CAAC,EAC5D,MAAM,EACN,yBAAyB,EACzB,iBAAiB,EACjB,sBAAsB,GAAG,oDAAkC,GACP;IACpD,OAAO,uCAAuC,CAAC;QAC7C,MAAM,EAAE,yBAAyB,CAAC,MAAM;QACxC,MAAM;QACN,iBAAiB;QACjB,KAAK,CAAC,MAAM,CAAC,eAAe;YAC1B,MAAM,OAAO,GAAG,oCAAoC,CAAC;gBACnD,yBAAyB;gBACzB,QAAQ,EAAE,IAAA,mBAAI,GAAE;gBAChB,MAAM,EAAE,MAAM;aACf,CAAC,CAAC;YAEH,IAAI,CAAC;gBACH,MAAM,IAAA,uCAAqB,EACzB,OAAO,EACP,eAAe,EACf,sBAAsB,CACvB,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB;oBACE,GAAG;oBACH,iBAAiB;oBACjB,sBAAsB;iBACvB,EACD,mCAAmC,CACpC,CAAC;gBACF,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}
|
package/dist/src/logger/index.js
CHANGED
|
@@ -253,6 +253,7 @@ class IntegrationLogger extends events_1.EventEmitter {
|
|
|
253
253
|
}
|
|
254
254
|
stepFailure(step, err) {
|
|
255
255
|
const eventName = 'step_failure';
|
|
256
|
+
this._logger.info({ errconstructorname: err.constructor?.name }, 'Error Constructorname');
|
|
256
257
|
const { errorId, description } = createErrorEventDescription(err, `Step "${step.name}" failed to complete due to error.`);
|
|
257
258
|
this.handleFailure({ eventName, errorId, err, description });
|
|
258
259
|
}
|
|
@@ -274,6 +275,7 @@ class IntegrationLogger extends events_1.EventEmitter {
|
|
|
274
275
|
}
|
|
275
276
|
validationFailure(err) {
|
|
276
277
|
const eventName = 'validation_failure';
|
|
278
|
+
this._logger.info({ errconstructorname: err.constructor?.name }, 'Error Constructorname');
|
|
277
279
|
const { errorId, description } = createErrorEventDescription(err, `Error occurred while validating integration configuration.`);
|
|
278
280
|
this.handleFailure({ eventName, errorId, err, description });
|
|
279
281
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/logger/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAkEA,oCA8CC;AAMD,0DA8BC;AA4WD,kEAkDC;AAljBD,oDAA4B;AAC5B,+BAA+B;AAC/B,mCAAsC;AACtC,mCAA4C;AAE5C,2EA4B0C;AAE1C,0DAAwC;AAE3B,QAAA,wBAAwB,GACnC,sCAAsC;IACtC,4GAA4G;IAC5G,gFAAgF,CAAC;AAEnF,2BAA2B;AAC3B,MAAM,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;AAuB9C,SAAgB,YAAY,CAG1B,EACA,IAAI,EACJ,MAAM,EACN,WAAW,EACX,SAAS,GAIV;IACC,MAAM,YAAY,GAAyB;QACzC,IAAI;QACJ,KAAK,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM,CAAoB;QAC3D,WAAW,EAAE;YACX,GAAG,EAAE,UAAU,GAAG;gBAChB,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK;oBAAE,OAAO,GAAG,CAAC;gBACnC,OAAO;oBACL,OAAO,EAAE,GAAG,CAAC,OAAO;oBACpB,IAAI,EAAE,GAAG,CAAC,IAAI;oBACd,KAAK,EAAE,IAAA,cAAO,EAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC;oBAC9B,IAAI,EAAE,GAAG,CAAC,IAAI;oBACd,MAAM,EAAE,GAAG,CAAC,MAAM;iBACnB,CAAC;YACJ,CAAC;SACF;KACF,CAAC;IAEF,IAAI,MAAM,EAAE,CAAC;QACX,YAAY,CAAC,OAAO,GAAG,CAAC,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;IAC7E,CAAC;IAED,MAAM,MAAM,GAAG,gBAAM,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;IAEjD,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;IACrC,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAS,CAAC;IAElC,OAAO,IAAI,iBAAiB,CAAC;QAC3B,MAAM;QACN,QAAQ;QACR,SAAS;KACV,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,SAAgB,uBAAuB,CAGrC,EACA,IAAI,EACJ,gBAAgB,EAChB,MAAM,EACN,WAAW,EACX,SAAS,GACwC;IACjD,MAAM,uBAAuB,GAAG,8BAA8B,CAC5D,gBAAgB,EAAE,oBAAoB,CACvC,CAAC;IAEF,OAAO,YAAY,CAAC;QAClB,IAAI;QACJ,MAAM;QACN,WAAW,EAAE;YACX,yBAAyB,EAAE,uBAAuB;YAClD,oCAAoC;YACpC,QAAQ,EAAE,CAAC,QAA6B,EAAE,EAAE,CAAC,CAAC;gBAC5C,GAAG,QAAQ;gBACX,MAAM,EAAE,QAAQ,CAAC,MAAM;oBACrB,CAAC,CAAC,uBAAuB,CAAC,QAAQ,CAAC,MAAM,CAAC;oBAC1C,CAAC,CAAC,SAAS;aACd,CAAC;YACF,GAAG,WAAW;SACf;QACD,SAAS;KACV,CAAC,CAAC;AACL,CAAC;AAED,SAAS,8BAA8B,CAErC,MAAmD;IACnD,OAAO,CAAC,MAAW,EAAE,EAAE;QACrB,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,MAAM,CAAC;QAChB,CAAC;aAAM,CAAC;YACN,MAAM,UAAU,GAAQ,EAAE,CAAC;YAC3B,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACpC,MAAM,KAAK,GAAG,MAAM,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;gBAClC,IAAI,KAAK,EAAE,CAAC;oBACV,UAAU,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBACjD,CAAC;qBAAM,CAAC;oBACN,UAAU,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;gBACxB,CAAC;YACH,CAAC;YACD,OAAO,UAAU,CAAC;QACpB,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAmBD,MAAa,iBACX,SAAQ,qBAAY;IAGZ,OAAO,CAAS;IAChB,SAAS,CAAa;IACrB,SAAS,CAAoB;IAEtC,YAAY,KAA6B;QACvC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,QAAQ,CAAC;QAEhC,gEAAgE;QAChE,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACjD,CAAC;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,cAAc,CAAC,GAAU;QACvB,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,GAAG,MAAa;QACpB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC;IACvC,CAAC;IACD,IAAI,CAAC,GAAG,MAAa;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;IACtC,CAAC;IAED,IAAI,CAAC,GAAG,MAAa;QACnB,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAElC,IAAI,CAAC,aAAa,CAChB;YACE,IAAI,EAAE,aAAa;YACnB,KAAK,EAAE,CAAC;SACT,EACD;YACE,SAAS,EAAE,KAAK;SACjB,CACF,CAAC;QAEF,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,GAAG,MAAa;QACpB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,GAAG,MAAa;QACpB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QAED,IAAI,gBAAgB,GAAwB,EAAE,CAAC;QAC/C,IAAI,aAAa,GAAU,MAAM,CAAC;QAElC,IAAI,MAAM,CAAC,CAAC,CAAC,YAAY,KAAK,EAAE,CAAC;YAC/B,gBAAgB,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YACtC,aAAa,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;aAAM,IAAI,OAAO,MAAM,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;YACzC,gBAAgB,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YAC7B,aAAa,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;QAED,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CACvB,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,gBAAgB,EAAE,EACtC,GAAG,aAAa,CACjB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,GAAG,MAAa;QACpB,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAElC,IAAI,CAAC,aAAa,CAChB;YACE,IAAI,EAAE,cAAc;YACpB,KAAK,EAAE,CAAC;SACT,EACD;YACE,SAAS,EAAE,KAAK;SACjB,CACF,CAAC;QAEF,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,UAAkB,EAAE,EAAE,MAAgB;QAC1C,MAAM,WAAW,GAAG,IAAI,iBAAiB,CAAC;YACxC,QAAQ,EAAE,IAAI,CAAC,SAAS;YACxB,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC;YAC3C,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAC;QAEH,wBAAwB;QACxB,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QAC5D,WAAW,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;QAE9D,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,IAAI,CACF,IAAO,EACP,IAAU;QAEV,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAChC,CAAC;IAED,SAAS,CAAC,IAAkB;QAC1B,MAAM,IAAI,GAAG,YAAY,CAAC;QAC1B,MAAM,WAAW,GAAG,kBAAkB,IAAI,CAAC,IAAI,MAAM,CAAC;QACtD,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACxB,IAAI,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,WAAW,CAAC,IAAkB;QAC5B,MAAM,IAAI,GAAG,UAAU,CAAC;QACxB,MAAM,WAAW,GAAG,mBAAmB,IAAI,CAAC,IAAI,IAAI,CAAC;QACrD,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACxB,IAAI,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,QAAQ,CACN,IAAkB,EAClB,MAA0B,EAC1B,iBAA4C;QAE5C,IAAI,CAAC,MAAM,IAAI,MAAM,KAAK,yCAAkB,CAAC,IAAI,EAAE,CAAC;YAClD,OAAO;QACT,CAAC;QAED,MAAM,IAAI,GAAG,WAAW,CAAC;QACzB,IAAI,WAAW,GAAG,iBAAiB,IAAI,CAAC,IAAI,KAAK,CAAC;QAElD,QAAQ,MAAM,EAAE,CAAC;YACf,KAAK,yCAAkB,CAAC,WAAW,CAAC,CAAC,CAAC;gBACpC,WAAW,IAAI,yKAAyK,CAAC;gBACzL,MAAM;YACR,CAAC;YACD,KAAK,yCAAkB,CAAC,IAAI,CAAC,CAAC,CAAC;gBAC7B,WAAW,IAAI,iDAAiD,CAAC;gBACjE,MAAM;YACR,CAAC;YACD,KAAK,yCAAkB,CAAC,UAAU,CAAC,CAAC,CAAC;gBACnC,WAAW,IAAI,gEAAgE,CAAC;gBAChF,MAAM;YACR,CAAC;YACD,KAAK,yCAAkB,CAAC,MAAM,CAAC,CAAC,CAAC;gBAC/B,WAAW,IAAI,wEAAwE,CAAC;gBACxF,MAAM;YACR,CAAC;YACD,KAAK,yCAAkB,CAAC,WAAW,CAAC,CAAC,CAAC;gBACpC,WAAW,IAAI,wEAAwE,CAAC;gBACxF,MAAM;YACR,CAAC;YACD,KAAK,yCAAkB,CAAC,eAAe,CAAC,CAAC,CAAC;gBACxC,MAAM,cAAc,GAAG,iBAAiB,EAAE,UAAU,EAAE,IAAI,CAAC;gBAC3D,IAAI,CAAC,cAAc,EAAE,CAAC;oBACpB,IAAI,CAAC,IAAI,CAAC,2CAA2C,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;gBACpE,CAAC;gBACD,WAAW,IAAI,cAAc;oBAC3B,CAAC,CAAC,0CAA0C,cAAc,6GAA6G;oBACvK,CAAC,CAAC,iJAAiJ,CAAC;gBACtJ,MAAM;YACR,CAAC;QACH,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACvB,IAAI,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,WAAW,CAAC,IAAkB,EAAE,GAAU;QACxC,MAAM,SAAS,GAAG,cAAc,CAAC;QACjC,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,2BAA2B,CAC1D,GAAG,EACH,SAAS,IAAI,CAAC,IAAI,oCAAoC,CACvD,CAAC;QACF,IAAI,CAAC,aAAa,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,CAAC,CAAC;IAC/D,CAAC;IAED,0BAA0B,CAAC,GAAuB;QAChD,MAAM,IAAI,GAAG,mBAAmB,CAAC;QACjC,MAAM,WAAW,GAAG,iDAAiD,CAAC;QACtE,IAAI,CAAC,IAAI,CACP;YACE,oBAAoB,EAAE,GAAG,CAAC,EAAE;SAC7B,EACD,WAAW,CACZ,CAAC;QACF,IAAI,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,wBAAwB,CAAC,GAAuB;QAC9C,MAAM,IAAI,GAAG,iBAAiB,CAAC;QAC/B,MAAM,WAAW,GAAG,kBAAkB,CAAC;QACvC,IAAI,CAAC,IAAI,CACP;YACE,oBAAoB,EAAE,GAAG,CAAC,EAAE;SAC7B,EACD,WAAW,CACZ,CAAC;QACF,IAAI,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,iBAAiB,CAAC,GAAU;QAC1B,MAAM,SAAS,GAAG,oBAAoB,CAAC;QACvC,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,2BAA2B,CAC1D,GAAG,EACH,4DAA4D,CAC7D,CAAC;QACF,IAAI,CAAC,aAAa,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,CAAC,CAAC;IAC/D,CAAC;IAEO,aAAa,CAAC,OAKrB;QACC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;QAEzD,uEAAuE;QACvE,yEAAyE;QACzE,wEAAwE;QACxE,MAAM,IAAI,GAAI,GAAW,CAAC,IAAI,CAAC;QAE/B,IAAI,IAAA,kDAA2B,EAAC,GAAG,CAAC,EAAE,CAAC;YACrC,IAAI,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,WAAW,CAAC,CAAC;YAChD,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,WAAW,CAAC,CAAC;QACjD,CAAC;QAED,IAAI,CAAC,YAAY,CAAC;YAChB,IAAI,EAAE,SAAS;YACf,WAAW;YACX,KAAK,EAAE,wCAAiB,CAAC,KAAK;SAC/B,CAAC,CAAC;IACL,CAAC;IAED,aAAa,CACX,MAAiC,EACjC,EAAE,SAAS,GAAG,IAAI,KAA2B,EAAE;QAE/C,MAAM,mBAAmB,GAAG;YAC1B,GAAG,MAAM;YACT,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC;QAEF,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,mBAAmB,EAAE,EAAE,mBAAmB,CAAC,CAAC;QACnE,CAAC;QAED,2DAA2D;QAC3D,2BAA2B;QAC3B,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,mBAAmB,CAAC,CAAC;IAClD,CAAC;IAED,YAAY,CAAC,KAAwB;QACnC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACxB,GAAG,KAAK;YACR,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,wCAAiB,CAAC,IAAI;SAC7C,CAAC,CAAC;IACL,CAAC;IAED,gBAAgB,CAAC,KAA4B;QAC3C,OAAO,IAAI,CAAC,YAAY,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,wCAAiB,CAAC,IAAI,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,gBAAgB,CAAC,KAA4B;QAC3C,OAAO,IAAI,CAAC,YAAY,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,wCAAiB,CAAC,IAAI,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,iBAAiB,CAAC,KAA6B;QAC7C,OAAO,IAAI,CAAC,YAAY,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,wCAAiB,CAAC,KAAK,EAAE,CAAC,CAAC;IACzE,CAAC;IAEO,iBAAiB,CAAC,MAAW;QACnC,IAAI,MAAM,YAAY,KAAK,EAAE,CAAC;YAC5B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC7B,CAAC;aAAM,IAAI,MAAM,EAAE,GAAG,YAAY,KAAK,EAAE,CAAC;YACxC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;CACF;AA7SD,8CA6SC;AAID;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,GAAQ;IAClC,OAAO,CACL,GAAG;QACH,OAAO,GAAG,KAAK,QAAQ;QACvB,MAAM,IAAI,GAAG;QACb,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ;QAC5B,OAAO,IAAI,GAAG;QACd,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,EAAE,UAAU,CAAC,aAAa,CAAC;YAC/C,GAAG,CAAC,WAAW,EAAE,IAAI,KAAK,iCAAiC;YAC3D,GAAG,CAAC,WAAW,EAAE,IAAI,KAAK,mCAAmC;YAC7D,GAAG,CAAC,WAAW,EAAE,IAAI,KAAK,aAAa,CAAC,CAC3C,CAAC;AACJ,CAAC;AAED,SAAgB,2BAA2B,CACzC,GAA6B,EAC7B,OAAe;AAEf;;;GAGG;AACH,SAAkB;IAElB,MAAM,OAAO,GAAG,IAAA,mBAAI,GAAE,CAAC;IAEvB,IAAI,SAAiB,CAAC;IACtB,IAAI,WAAmB,CAAC;IAExB,IAAI,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5B,SAAS,GAAI,GAAwB,CAAC,IAAI,CAAC;QAC3C,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC;IAC5B,CAAC;SAAM,CAAC;QACN,SAAS,GAAG,4CAAqB,CAAC;QAClC,WAAW,GAAG,8CAAuB,CAAC;IACxC,CAAC;IAED,IAAI,IAAA,0CAAmB,EAAC,GAAG,CAAC,EAAE,CAAC;QAC7B,uDAAuD;QACvD,wCAAwC;QACxC,OAAO,IAAI,gCAAwB,CAAC;IACtC,CAAC;IAED,MAAM,cAAc,GAAoB;QACtC,CAAC,WAAW,EAAE,SAAS,CAAC;QACxB,CAAC,QAAQ,EAAE,WAAW,CAAC;KACxB,CAAC;IAEF,IAAI,SAAS,EAAE,CAAC;QACd,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACzC,cAAc,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAG,cAAc;SAChC,GAAG,CAAC,CAAC,aAAa,EAAE,EAAE;QACrB,OAAO,GAAG,aAAa,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACnE,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,OAAO;QACL,OAAO;QACP,WAAW,EAAE,GAAG,OAAO,KAAK,YAAY,GAAG;KAC5C,CAAC;AACJ,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/logger/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAkEA,oCA8CC;AAMD,0DA8BC;AAoXD,kEAkDC;AA1jBD,oDAA4B;AAC5B,+BAA+B;AAC/B,mCAAsC;AACtC,mCAA4C;AAE5C,2EA4B0C;AAE1C,0DAAwC;AAE3B,QAAA,wBAAwB,GACnC,sCAAsC;IACtC,4GAA4G;IAC5G,gFAAgF,CAAC;AAEnF,2BAA2B;AAC3B,MAAM,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;AAuB9C,SAAgB,YAAY,CAG1B,EACA,IAAI,EACJ,MAAM,EACN,WAAW,EACX,SAAS,GAIV;IACC,MAAM,YAAY,GAAyB;QACzC,IAAI;QACJ,KAAK,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM,CAAoB;QAC3D,WAAW,EAAE;YACX,GAAG,EAAE,UAAU,GAAG;gBAChB,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK;oBAAE,OAAO,GAAG,CAAC;gBACnC,OAAO;oBACL,OAAO,EAAE,GAAG,CAAC,OAAO;oBACpB,IAAI,EAAE,GAAG,CAAC,IAAI;oBACd,KAAK,EAAE,IAAA,cAAO,EAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC;oBAC9B,IAAI,EAAE,GAAG,CAAC,IAAI;oBACd,MAAM,EAAE,GAAG,CAAC,MAAM;iBACnB,CAAC;YACJ,CAAC;SACF;KACF,CAAC;IAEF,IAAI,MAAM,EAAE,CAAC;QACX,YAAY,CAAC,OAAO,GAAG,CAAC,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;IAC7E,CAAC;IAED,MAAM,MAAM,GAAG,gBAAM,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;IAEjD,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;IACrC,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAS,CAAC;IAElC,OAAO,IAAI,iBAAiB,CAAC;QAC3B,MAAM;QACN,QAAQ;QACR,SAAS;KACV,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,SAAgB,uBAAuB,CAGrC,EACA,IAAI,EACJ,gBAAgB,EAChB,MAAM,EACN,WAAW,EACX,SAAS,GACwC;IACjD,MAAM,uBAAuB,GAAG,8BAA8B,CAC5D,gBAAgB,EAAE,oBAAoB,CACvC,CAAC;IAEF,OAAO,YAAY,CAAC;QAClB,IAAI;QACJ,MAAM;QACN,WAAW,EAAE;YACX,yBAAyB,EAAE,uBAAuB;YAClD,oCAAoC;YACpC,QAAQ,EAAE,CAAC,QAA6B,EAAE,EAAE,CAAC,CAAC;gBAC5C,GAAG,QAAQ;gBACX,MAAM,EAAE,QAAQ,CAAC,MAAM;oBACrB,CAAC,CAAC,uBAAuB,CAAC,QAAQ,CAAC,MAAM,CAAC;oBAC1C,CAAC,CAAC,SAAS;aACd,CAAC;YACF,GAAG,WAAW;SACf;QACD,SAAS;KACV,CAAC,CAAC;AACL,CAAC;AAED,SAAS,8BAA8B,CAErC,MAAmD;IACnD,OAAO,CAAC,MAAW,EAAE,EAAE;QACrB,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,MAAM,CAAC;QAChB,CAAC;aAAM,CAAC;YACN,MAAM,UAAU,GAAQ,EAAE,CAAC;YAC3B,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACpC,MAAM,KAAK,GAAG,MAAM,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;gBAClC,IAAI,KAAK,EAAE,CAAC;oBACV,UAAU,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBACjD,CAAC;qBAAM,CAAC;oBACN,UAAU,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;gBACxB,CAAC;YACH,CAAC;YACD,OAAO,UAAU,CAAC;QACpB,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAmBD,MAAa,iBACX,SAAQ,qBAAY;IAGZ,OAAO,CAAS;IAChB,SAAS,CAAa;IACrB,SAAS,CAAoB;IAEtC,YAAY,KAA6B;QACvC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,QAAQ,CAAC;QAEhC,gEAAgE;QAChE,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACjD,CAAC;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,cAAc,CAAC,GAAU;QACvB,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,GAAG,MAAa;QACpB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC;IACvC,CAAC;IACD,IAAI,CAAC,GAAG,MAAa;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;IACtC,CAAC;IAED,IAAI,CAAC,GAAG,MAAa;QACnB,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAElC,IAAI,CAAC,aAAa,CAChB;YACE,IAAI,EAAE,aAAa;YACnB,KAAK,EAAE,CAAC;SACT,EACD;YACE,SAAS,EAAE,KAAK;SACjB,CACF,CAAC;QAEF,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,GAAG,MAAa;QACpB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,GAAG,MAAa;QACpB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QAED,IAAI,gBAAgB,GAAwB,EAAE,CAAC;QAC/C,IAAI,aAAa,GAAU,MAAM,CAAC;QAElC,IAAI,MAAM,CAAC,CAAC,CAAC,YAAY,KAAK,EAAE,CAAC;YAC/B,gBAAgB,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YACtC,aAAa,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;aAAM,IAAI,OAAO,MAAM,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;YACzC,gBAAgB,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YAC7B,aAAa,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;QAED,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CACvB,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,gBAAgB,EAAE,EACtC,GAAG,aAAa,CACjB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,GAAG,MAAa;QACpB,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAElC,IAAI,CAAC,aAAa,CAChB;YACE,IAAI,EAAE,cAAc;YACpB,KAAK,EAAE,CAAC;SACT,EACD;YACE,SAAS,EAAE,KAAK;SACjB,CACF,CAAC;QAEF,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,UAAkB,EAAE,EAAE,MAAgB;QAC1C,MAAM,WAAW,GAAG,IAAI,iBAAiB,CAAC;YACxC,QAAQ,EAAE,IAAI,CAAC,SAAS;YACxB,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC;YAC3C,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAC;QAEH,wBAAwB;QACxB,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QAC5D,WAAW,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;QAE9D,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,IAAI,CACF,IAAO,EACP,IAAU;QAEV,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAChC,CAAC;IAED,SAAS,CAAC,IAAkB;QAC1B,MAAM,IAAI,GAAG,YAAY,CAAC;QAC1B,MAAM,WAAW,GAAG,kBAAkB,IAAI,CAAC,IAAI,MAAM,CAAC;QACtD,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACxB,IAAI,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,WAAW,CAAC,IAAkB;QAC5B,MAAM,IAAI,GAAG,UAAU,CAAC;QACxB,MAAM,WAAW,GAAG,mBAAmB,IAAI,CAAC,IAAI,IAAI,CAAC;QACrD,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACxB,IAAI,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,QAAQ,CACN,IAAkB,EAClB,MAA0B,EAC1B,iBAA4C;QAE5C,IAAI,CAAC,MAAM,IAAI,MAAM,KAAK,yCAAkB,CAAC,IAAI,EAAE,CAAC;YAClD,OAAO;QACT,CAAC;QAED,MAAM,IAAI,GAAG,WAAW,CAAC;QACzB,IAAI,WAAW,GAAG,iBAAiB,IAAI,CAAC,IAAI,KAAK,CAAC;QAElD,QAAQ,MAAM,EAAE,CAAC;YACf,KAAK,yCAAkB,CAAC,WAAW,CAAC,CAAC,CAAC;gBACpC,WAAW,IAAI,yKAAyK,CAAC;gBACzL,MAAM;YACR,CAAC;YACD,KAAK,yCAAkB,CAAC,IAAI,CAAC,CAAC,CAAC;gBAC7B,WAAW,IAAI,iDAAiD,CAAC;gBACjE,MAAM;YACR,CAAC;YACD,KAAK,yCAAkB,CAAC,UAAU,CAAC,CAAC,CAAC;gBACnC,WAAW,IAAI,gEAAgE,CAAC;gBAChF,MAAM;YACR,CAAC;YACD,KAAK,yCAAkB,CAAC,MAAM,CAAC,CAAC,CAAC;gBAC/B,WAAW,IAAI,wEAAwE,CAAC;gBACxF,MAAM;YACR,CAAC;YACD,KAAK,yCAAkB,CAAC,WAAW,CAAC,CAAC,CAAC;gBACpC,WAAW,IAAI,wEAAwE,CAAC;gBACxF,MAAM;YACR,CAAC;YACD,KAAK,yCAAkB,CAAC,eAAe,CAAC,CAAC,CAAC;gBACxC,MAAM,cAAc,GAAG,iBAAiB,EAAE,UAAU,EAAE,IAAI,CAAC;gBAC3D,IAAI,CAAC,cAAc,EAAE,CAAC;oBACpB,IAAI,CAAC,IAAI,CAAC,2CAA2C,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;gBACpE,CAAC;gBACD,WAAW,IAAI,cAAc;oBAC3B,CAAC,CAAC,0CAA0C,cAAc,6GAA6G;oBACvK,CAAC,CAAC,iJAAiJ,CAAC;gBACtJ,MAAM;YACR,CAAC;QACH,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACvB,IAAI,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,WAAW,CAAC,IAAkB,EAAE,GAAU;QACxC,MAAM,SAAS,GAAG,cAAc,CAAC;QACjC,IAAI,CAAC,OAAO,CAAC,IAAI,CACf,EAAE,kBAAkB,EAAE,GAAG,CAAC,WAAW,EAAE,IAAI,EAAE,EAC7C,uBAAuB,CACxB,CAAC;QACF,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,2BAA2B,CAC1D,GAAG,EACH,SAAS,IAAI,CAAC,IAAI,oCAAoC,CACvD,CAAC;QACF,IAAI,CAAC,aAAa,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,CAAC,CAAC;IAC/D,CAAC;IAED,0BAA0B,CAAC,GAAuB;QAChD,MAAM,IAAI,GAAG,mBAAmB,CAAC;QACjC,MAAM,WAAW,GAAG,iDAAiD,CAAC;QACtE,IAAI,CAAC,IAAI,CACP;YACE,oBAAoB,EAAE,GAAG,CAAC,EAAE;SAC7B,EACD,WAAW,CACZ,CAAC;QACF,IAAI,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,wBAAwB,CAAC,GAAuB;QAC9C,MAAM,IAAI,GAAG,iBAAiB,CAAC;QAC/B,MAAM,WAAW,GAAG,kBAAkB,CAAC;QACvC,IAAI,CAAC,IAAI,CACP;YACE,oBAAoB,EAAE,GAAG,CAAC,EAAE;SAC7B,EACD,WAAW,CACZ,CAAC;QACF,IAAI,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,iBAAiB,CAAC,GAAU;QAC1B,MAAM,SAAS,GAAG,oBAAoB,CAAC;QACvC,IAAI,CAAC,OAAO,CAAC,IAAI,CACf,EAAE,kBAAkB,EAAE,GAAG,CAAC,WAAW,EAAE,IAAI,EAAE,EAC7C,uBAAuB,CACxB,CAAC;QACF,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,2BAA2B,CAC1D,GAAG,EACH,4DAA4D,CAC7D,CAAC;QACF,IAAI,CAAC,aAAa,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,CAAC,CAAC;IAC/D,CAAC;IAEO,aAAa,CAAC,OAKrB;QACC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;QAEzD,uEAAuE;QACvE,yEAAyE;QACzE,wEAAwE;QACxE,MAAM,IAAI,GAAI,GAAW,CAAC,IAAI,CAAC;QAE/B,IAAI,IAAA,kDAA2B,EAAC,GAAG,CAAC,EAAE,CAAC;YACrC,IAAI,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,WAAW,CAAC,CAAC;YAChD,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,WAAW,CAAC,CAAC;QACjD,CAAC;QAED,IAAI,CAAC,YAAY,CAAC;YAChB,IAAI,EAAE,SAAS;YACf,WAAW;YACX,KAAK,EAAE,wCAAiB,CAAC,KAAK;SAC/B,CAAC,CAAC;IACL,CAAC;IAED,aAAa,CACX,MAAiC,EACjC,EAAE,SAAS,GAAG,IAAI,KAA2B,EAAE;QAE/C,MAAM,mBAAmB,GAAG;YAC1B,GAAG,MAAM;YACT,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC;QAEF,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,mBAAmB,EAAE,EAAE,mBAAmB,CAAC,CAAC;QACnE,CAAC;QAED,2DAA2D;QAC3D,2BAA2B;QAC3B,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,mBAAmB,CAAC,CAAC;IAClD,CAAC;IAED,YAAY,CAAC,KAAwB;QACnC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACxB,GAAG,KAAK;YACR,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,wCAAiB,CAAC,IAAI;SAC7C,CAAC,CAAC;IACL,CAAC;IAED,gBAAgB,CAAC,KAA4B;QAC3C,OAAO,IAAI,CAAC,YAAY,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,wCAAiB,CAAC,IAAI,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,gBAAgB,CAAC,KAA4B;QAC3C,OAAO,IAAI,CAAC,YAAY,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,wCAAiB,CAAC,IAAI,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,iBAAiB,CAAC,KAA6B;QAC7C,OAAO,IAAI,CAAC,YAAY,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,wCAAiB,CAAC,KAAK,EAAE,CAAC,CAAC;IACzE,CAAC;IAEO,iBAAiB,CAAC,MAAW;QACnC,IAAI,MAAM,YAAY,KAAK,EAAE,CAAC;YAC5B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC7B,CAAC;aAAM,IAAI,MAAM,EAAE,GAAG,YAAY,KAAK,EAAE,CAAC;YACxC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;CACF;AArTD,8CAqTC;AAID;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,GAAQ;IAClC,OAAO,CACL,GAAG;QACH,OAAO,GAAG,KAAK,QAAQ;QACvB,MAAM,IAAI,GAAG;QACb,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ;QAC5B,OAAO,IAAI,GAAG;QACd,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,EAAE,UAAU,CAAC,aAAa,CAAC;YAC/C,GAAG,CAAC,WAAW,EAAE,IAAI,KAAK,iCAAiC;YAC3D,GAAG,CAAC,WAAW,EAAE,IAAI,KAAK,mCAAmC;YAC7D,GAAG,CAAC,WAAW,EAAE,IAAI,KAAK,aAAa,CAAC,CAC3C,CAAC;AACJ,CAAC;AAED,SAAgB,2BAA2B,CACzC,GAA6B,EAC7B,OAAe;AAEf;;;GAGG;AACH,SAAkB;IAElB,MAAM,OAAO,GAAG,IAAA,mBAAI,GAAE,CAAC;IAEvB,IAAI,SAAiB,CAAC;IACtB,IAAI,WAAmB,CAAC;IAExB,IAAI,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5B,SAAS,GAAI,GAAwB,CAAC,IAAI,CAAC;QAC3C,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC;IAC5B,CAAC;SAAM,CAAC;QACN,SAAS,GAAG,4CAAqB,CAAC;QAClC,WAAW,GAAG,8CAAuB,CAAC;IACxC,CAAC;IAED,IAAI,IAAA,0CAAmB,EAAC,GAAG,CAAC,EAAE,CAAC;QAC7B,uDAAuD;QACvD,wCAAwC;QACxC,OAAO,IAAI,gCAAwB,CAAC;IACtC,CAAC;IAED,MAAM,cAAc,GAAoB;QACtC,CAAC,WAAW,EAAE,SAAS,CAAC;QACxB,CAAC,QAAQ,EAAE,WAAW,CAAC;KACxB,CAAC;IAEF,IAAI,SAAS,EAAE,CAAC;QACd,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACzC,cAAc,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAG,cAAc;SAChC,GAAG,CAAC,CAAC,aAAa,EAAE,EAAE;QACrB,OAAO,GAAG,aAAa,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACnE,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,OAAO;QACL,OAAO;QACP,WAAW,EAAE,GAAG,OAAO,KAAK,YAAY,GAAG;KAC5C,CAAC;AACJ,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Entity, GraphObjectFilter, GraphObjectIteratee, Relationship, GraphObjectStore, GraphObjectIndexMetadata, GetIndexMetadataForGraphObjectTypeParams, IntegrationStep, GraphObjectIterateeOptions } from '@jupiterone/integration-sdk-core';
|
|
1
|
+
import { Entity, GraphObjectFilter, GraphObjectIteratee, Relationship, GraphObjectStore, GraphObjectIndexMetadata, GetIndexMetadataForGraphObjectTypeParams, IntegrationStep, GraphObjectIterateeOptions, IntegrationLogger } from '@jupiterone/integration-sdk-core';
|
|
2
2
|
export declare const DEFAULT_GRAPH_OBJECT_FILE_SIZE = 500;
|
|
3
3
|
export declare const MAX_GRAPH_OBJECT_BUFFER_THRESHOLD_IN_BYTES: 1073741824;
|
|
4
4
|
export interface FileSystemGraphObjectStoreParams {
|
|
@@ -16,6 +16,10 @@ export interface FileSystemGraphObjectStoreParams {
|
|
|
16
16
|
* Whether the files that are written to disk should be minified or not
|
|
17
17
|
*/
|
|
18
18
|
prettifyFiles?: boolean;
|
|
19
|
+
/**
|
|
20
|
+
* Optional logger for debugging and tracking data flow
|
|
21
|
+
*/
|
|
22
|
+
logger?: IntegrationLogger;
|
|
19
23
|
}
|
|
20
24
|
export declare class FileSystemGraphObjectStore implements GraphObjectStore {
|
|
21
25
|
private readonly semaphore;
|
|
@@ -25,6 +29,7 @@ export declare class FileSystemGraphObjectStore implements GraphObjectStore {
|
|
|
25
29
|
private readonly prettifyFiles;
|
|
26
30
|
private readonly stepIdToGraphObjectIndexMetadataMap;
|
|
27
31
|
private readonly entityOnDiskLocationMap;
|
|
32
|
+
private readonly logger?;
|
|
28
33
|
constructor(params?: FileSystemGraphObjectStoreParams);
|
|
29
34
|
addEntities(stepId: string, newEntities: Entity[], onEntitiesFlushed?: (entities: Entity[]) => Promise<void>): Promise<void>;
|
|
30
35
|
addRelationships(stepId: string, newRelationships: Relationship[], onRelationshipsFlushed?: (relationships: Relationship[]) => Promise<void>): Promise<void>;
|
|
@@ -48,11 +48,13 @@ class FileSystemGraphObjectStore {
|
|
|
48
48
|
prettifyFiles;
|
|
49
49
|
stepIdToGraphObjectIndexMetadataMap;
|
|
50
50
|
entityOnDiskLocationMap = new bigMap_1.BigMap(ENTITY_LOCATION_ON_DISK_DEFAULT_MAP_KEY_SPACE);
|
|
51
|
+
logger;
|
|
51
52
|
constructor(params) {
|
|
52
53
|
this.semaphore = new async_sema_1.Sema(BINARY_SEMAPHORE_CONCURRENCY);
|
|
53
54
|
this.graphObjectFileSize =
|
|
54
55
|
params?.graphObjectFileSize || exports.DEFAULT_GRAPH_OBJECT_FILE_SIZE;
|
|
55
56
|
this.prettifyFiles = params?.prettifyFiles || false;
|
|
57
|
+
this.logger = params?.logger;
|
|
56
58
|
this.graphObjectBufferThresholdInBytes = (0, lodash_1.min)([
|
|
57
59
|
params?.graphObjectBufferThresholdInBytes ||
|
|
58
60
|
synchronization_1.DEFAULT_UPLOAD_BATCH_SIZE_IN_BYTES,
|
|
@@ -185,28 +187,63 @@ class FileSystemGraphObjectStore {
|
|
|
185
187
|
return indexMetadata.enabled === true;
|
|
186
188
|
});
|
|
187
189
|
if (indexable.length) {
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
190
|
+
const chunks = (0, lodash_1.chunk)(indexable, this.graphObjectFileSize);
|
|
191
|
+
this.logger?.debug({
|
|
192
|
+
stepId,
|
|
193
|
+
entityCount: indexable.length,
|
|
194
|
+
chunkCount: chunks.length,
|
|
195
|
+
chunkSize: this.graphObjectFileSize,
|
|
196
|
+
}, 'Flushing entity chunks to disk');
|
|
197
|
+
try {
|
|
198
|
+
await Promise.all(chunks.map(async (data, chunkIndex) => {
|
|
199
|
+
const graphObjectsToFilePaths = await (0, flushDataToDisk_1.flushDataToDisk)({
|
|
200
|
+
storageDirectoryPath: stepId,
|
|
201
|
+
collectionType: 'entities',
|
|
202
|
+
data,
|
|
203
|
+
pretty: this.prettifyFiles,
|
|
204
|
+
logger: this.logger,
|
|
205
|
+
});
|
|
206
|
+
for (const { graphDataPath, collection, } of graphObjectsToFilePaths) {
|
|
207
|
+
for (const [index, e] of collection.entries()) {
|
|
208
|
+
this.entityOnDiskLocationMap.set(e._key, {
|
|
209
|
+
graphDataPath,
|
|
210
|
+
index,
|
|
211
|
+
});
|
|
212
|
+
}
|
|
201
213
|
}
|
|
202
|
-
|
|
203
|
-
|
|
214
|
+
this.logger?.debug({
|
|
215
|
+
stepId,
|
|
216
|
+
chunkIndex,
|
|
217
|
+
entitiesInChunk: data.length,
|
|
218
|
+
filesCreated: graphObjectsToFilePaths.length,
|
|
219
|
+
}, 'Entity chunk flushed successfully');
|
|
220
|
+
}));
|
|
221
|
+
}
|
|
222
|
+
catch (error) {
|
|
223
|
+
this.logger?.error({
|
|
224
|
+
stepId,
|
|
225
|
+
entityCount: indexable.length,
|
|
226
|
+
chunkCount: chunks.length,
|
|
227
|
+
error: error.message,
|
|
228
|
+
errorStack: error.stack,
|
|
229
|
+
}, 'Failed to flush entity chunks to disk');
|
|
230
|
+
throw error;
|
|
231
|
+
}
|
|
204
232
|
}
|
|
205
233
|
this.localGraphObjectStore.flushEntities(entities, stepId);
|
|
206
234
|
entitiesToUpload = entitiesToUpload.concat(entities);
|
|
207
235
|
}
|
|
208
236
|
if (onEntitiesFlushed) {
|
|
209
|
-
|
|
237
|
+
try {
|
|
238
|
+
await onEntitiesFlushed(entitiesToUpload);
|
|
239
|
+
}
|
|
240
|
+
catch (err) {
|
|
241
|
+
this.logger?.error({
|
|
242
|
+
entityCount: entitiesToUpload.length,
|
|
243
|
+
err,
|
|
244
|
+
}, 'onEntitiesFlushed callback failed');
|
|
245
|
+
throw err;
|
|
246
|
+
}
|
|
210
247
|
}
|
|
211
248
|
});
|
|
212
249
|
}
|
|
@@ -249,20 +286,55 @@ class FileSystemGraphObjectStore {
|
|
|
249
286
|
return indexMetadata.enabled === true;
|
|
250
287
|
});
|
|
251
288
|
if (indexable.length) {
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
289
|
+
const chunks = (0, lodash_1.chunk)(indexable, this.graphObjectFileSize);
|
|
290
|
+
this.logger?.debug({
|
|
291
|
+
stepId,
|
|
292
|
+
relationshipCount: indexable.length,
|
|
293
|
+
chunkCount: chunks.length,
|
|
294
|
+
chunkSize: this.graphObjectFileSize,
|
|
295
|
+
}, 'Flushing relationship chunks to disk');
|
|
296
|
+
try {
|
|
297
|
+
await Promise.all(chunks.map(async (data, chunkIndex) => {
|
|
298
|
+
await (0, flushDataToDisk_1.flushDataToDisk)({
|
|
299
|
+
storageDirectoryPath: stepId,
|
|
300
|
+
collectionType: 'relationships',
|
|
301
|
+
data,
|
|
302
|
+
pretty: this.prettifyFiles,
|
|
303
|
+
logger: this.logger,
|
|
304
|
+
});
|
|
305
|
+
this.logger?.debug({
|
|
306
|
+
stepId,
|
|
307
|
+
chunkIndex,
|
|
308
|
+
relationshipsInChunk: data.length,
|
|
309
|
+
}, 'Relationship chunk flushed successfully');
|
|
310
|
+
}));
|
|
311
|
+
}
|
|
312
|
+
catch (error) {
|
|
313
|
+
this.logger?.error({
|
|
314
|
+
stepId,
|
|
315
|
+
relationshipCount: indexable.length,
|
|
316
|
+
chunkCount: chunks.length,
|
|
317
|
+
error: error.message,
|
|
318
|
+
errorStack: error.stack,
|
|
319
|
+
}, 'Failed to flush relationship chunks to disk');
|
|
320
|
+
throw error;
|
|
321
|
+
}
|
|
260
322
|
}
|
|
261
323
|
this.localGraphObjectStore.flushRelationships(relationships, stepId);
|
|
262
324
|
relationshipsToUpload = relationshipsToUpload.concat(relationships);
|
|
263
325
|
}
|
|
264
326
|
if (onRelationshipsFlushed) {
|
|
265
|
-
|
|
327
|
+
try {
|
|
328
|
+
await onRelationshipsFlushed(relationshipsToUpload);
|
|
329
|
+
}
|
|
330
|
+
catch (error) {
|
|
331
|
+
this.logger?.error({
|
|
332
|
+
relationshipCount: relationshipsToUpload.length,
|
|
333
|
+
error: error.message,
|
|
334
|
+
errorStack: error.stack,
|
|
335
|
+
}, 'onRelationshipsFlushed callback failed');
|
|
336
|
+
throw error;
|
|
337
|
+
}
|
|
266
338
|
}
|
|
267
339
|
});
|
|
268
340
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FileSystemGraphObjectStore.js","sourceRoot":"","sources":["../../../../src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.ts"],"names":[],"mappings":";;;AAAA,2CAAkC;
|
|
1
|
+
{"version":3,"file":"FileSystemGraphObjectStore.js","sourceRoot":"","sources":["../../../../src/storage/FileSystemGraphObjectStore/FileSystemGraphObjectStore.ts"],"names":[],"mappings":";;;AAAA,2CAAkC;AAelC,uDAAoD;AACpD,uCAImB;AACnB,sCAAqD;AAErD,iDAA8D;AAC9D,yDAAsD;AACtD,mCAAoC;AACpC,2DAA2E;AAE9D,QAAA,8BAA8B,GAAG,GAAG,CAAC;AAElD,gCAAgC;AACnB,QAAA,0CAA0C,GACrD,aAAsB,CAAC;AAEzB,8CAA8C;AAC9C,gEAAgE;AAChE,MAAM,4BAA4B,GAAG,CAAC,CAAC;AAqCvC;;GAEG;AACH,SAAS,6CAA6C,CACpD,gBAAmC;IAEnC,MAAM,mCAAmC,GAAG,IAAI,GAAG,EAGhD,CAAC;IAEJ,KAAK,MAAM,IAAI,IAAI,gBAAgB,EAAE,CAAC;QACpC,MAAM,WAAW,GAAgC;YAC/C,QAAQ,EAAE,IAAI,GAAG,EAAE;YACnB,aAAa,EAAE,IAAI,GAAG,EAAE;SACzB,CAAC;QAEF,KAAK,MAAM,cAAc,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC3C,IAAI,cAAc,CAAC,aAAa,EAAE,CAAC;gBACjC,WAAW,CAAC,QAAQ,CAAC,GAAG,CACtB,cAAc,CAAC,KAAK,EACpB,cAAc,CAAC,aAAa,CAC7B,CAAC;YACJ,CAAC;QACH,CAAC;QAED,KAAK,MAAM,oBAAoB,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACtD,IAAI,oBAAoB,CAAC,aAAa,EAAE,CAAC;gBACvC,WAAW,CAAC,aAAa,CAAC,GAAG,CAC3B,oBAAoB,CAAC,KAAK,EAC1B,oBAAoB,CAAC,aAAa,CACnC,CAAC;YACJ,CAAC;QACH,CAAC;QAED,mCAAmC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;IAChE,CAAC;IAED,OAAO,mCAAmC,CAAC;AAC7C,CAAC;AAeD,MAAM,6CAA6C,GAAG,OAAO,CAAC;AAE9D,MAAa,0BAA0B;IACpB,SAAS,CAAO;IAChB,qBAAqB,GAAG,IAAI,iCAAwB,EAAE,CAAC;IACvD,mBAAmB,CAAS;IAC5B,iCAAiC,CAAS;IAC1C,aAAa,CAAU;IACvB,mCAAmC,CAGlD;IACe,uBAAuB,GAAG,IAAI,eAAM,CAGnD,6CAA6C,CAAC,CAAC;IAChC,MAAM,CAAqB;IAE5C,YAAY,MAAyC;QACnD,IAAI,CAAC,SAAS,GAAG,IAAI,iBAAI,CAAC,4BAA4B,CAAC,CAAC;QACxD,IAAI,CAAC,mBAAmB;YACtB,MAAM,EAAE,mBAAmB,IAAI,sCAA8B,CAAC;QAEhE,IAAI,CAAC,aAAa,GAAG,MAAM,EAAE,aAAa,IAAI,KAAK,CAAC;QACpD,IAAI,CAAC,MAAM,GAAG,MAAM,EAAE,MAAM,CAAC;QAC7B,IAAI,CAAC,iCAAiC,GAAG,IAAA,YAAG,EAAC;YAC3C,MAAM,EAAE,iCAAiC;gBACvC,oDAAkC;YACpC,kDAA0C;SAC3C,CAAE,CAAC;QACJ,IAAI,MAAM,EAAE,gBAAgB,EAAE,CAAC;YAC7B,IAAI,CAAC,mCAAmC;gBACtC,6CAA6C,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW,CACf,MAAc,EACd,WAAqB,EACrB,iBAAyD;QAEzD,MAAM,IAAI,CAAC,qBAAqB,CAAC,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAElE,IACE,IAAI,CAAC,qBAAqB,CAAC,yBAAyB,EAAE;YACtD,IAAI,CAAC,iCAAiC,EACtC,CAAC;YACD,MAAM,IAAI,CAAC,mBAAmB,CAAC,iBAAiB,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,MAAc,EACd,gBAAgC,EAChC,sBAAyE;QAEzE,MAAM,IAAI,CAAC,qBAAqB,CAAC,gBAAgB,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;QAE5E,IACE,IAAI,CAAC,qBAAqB,CAAC,+BAA+B,EAAE;YAC5D,IAAI,CAAC,iCAAiC,EACtC,CAAC;YACD,MAAM,IAAI,CAAC,wBAAwB,CAAC,sBAAsB,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,UAAU,CAAC,IAAwB;QACvC,IAAI,CAAC,IAAI;YAAE,OAAO;QAClB,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACzE,IAAI,cAAc,EAAE,CAAC;YACnB,OAAO,cAAc,CAAC;QACxB,CAAC;QAED,MAAM,oBAAoB,GAAG,IAAI,CAAC,uBAAuB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpE,IAAI,CAAC,oBAAoB;YAAE,OAAO;QAElC,MAAM,QAAQ,GAAG,IAAA,uCAA0B,EACzC,oBAAoB,CAAC,aAAa,CACnC,CAAC;QACF,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAA,6BAAmB,EAAoB;YAChE,QAAQ;SACT,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,eAAe,CACnB,MAAyB,EACzB,QAAgC,EAChC,OAAoC;QAEpC,kFAAkF;QAClF,gCAAgC;QAChC,mFAAmF;QACnF,gGAAgG;QAChG,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAmB,CAAC;QACpD,MAAM,IAAI,CAAC,qBAAqB,CAAC,eAAe,CAC9C,MAAM,EACN,CAAC,GAAgB,EAAE,EAAE;YACnB,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACrC,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC,EACD,OAAO,CACR,CAAC;QAEF,MAAM,IAAA,gCAAsB,EAAC;YAC3B,IAAI,EAAE,MAAM,CAAC,KAAK;YAClB,OAAO;YACP,QAAQ,EAAE,CAAC,GAAgB,EAAE,EAAE;gBAC7B,IAAI,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;oBACnC,OAAO;gBACT,CAAC;gBACD,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC;YACvB,CAAC;SACF,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,oBAAoB,CACxB,MAAyB,EACzB,QAAgC,EAChC,OAAoC;QAEpC,uFAAuF;QACvF,gCAAgC;QAChC,mFAAmF;QACnF,gGAAgG;QAChG,MAAM,qBAAqB,GAAG,IAAI,GAAG,EAAmB,CAAC;QACzD,MAAM,IAAI,CAAC,qBAAqB,CAAC,oBAAoB,CACnD,MAAM,EACN,CAAC,GAAgB,EAAE,EAAE;YACnB,qBAAqB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC1C,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC,CACF,CAAC;QAEF,MAAM,IAAA,sCAA4B,EAAC;YACjC,IAAI,EAAE,MAAM,CAAC,KAAK;YAClB,OAAO;YACP,QAAQ,EAAE,CAAC,GAAgB,EAAE,EAAE;gBAC7B,IAAI,qBAAqB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;oBACxC,OAAO;gBACT,CAAC;gBACD,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC;YACvB,CAAC;SACF,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CACT,iBAAyD,EACzD,sBAAyE;QAEzE,MAAM,OAAO,CAAC,GAAG,CAAC;YAChB,IAAI,CAAC,mBAAmB,CAAC,iBAAiB,EAAE,IAAI,CAAC;YACjD,IAAI,CAAC,wBAAwB,CAAC,sBAAsB,EAAE,IAAI,CAAC;SAC5D,CAAC,CAAC;IACL,CAAC;IACD;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,mBAAmB,CACvB,iBAAyD,EACzD,QAAiB,KAAK;QAEtB,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,IAAI,EAAE;YAClC,mGAAmG;YACnG,mIAAmI;YACnI,mGAAmG;YACnG,IACE,CAAC,KAAK;gBACN,IAAI,CAAC,qBAAqB,CAAC,yBAAyB,EAAE;oBACpD,IAAI,CAAC,iCAAiC,EACxC,CAAC;gBACD,OAAO;YACT,CAAC;YAED,MAAM,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC,qBAAqB,EAAE,CAAC;YAC1E,IAAI,gBAAgB,GAAa,EAAE,CAAC;YACpC,KAAK,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,cAAc,EAAE,CAAC;gBAChD,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;oBACtC,MAAM,aAAa,GAAG,IAAI,CAAC,kCAAkC,CAAC;wBAC5D,MAAM;wBACN,KAAK,EAAE,CAAC,CAAC,KAAK;wBACd,yBAAyB,EAAE,UAAU;qBACtC,CAAC,CAAC;oBAEH,IAAI,OAAO,aAAa,KAAK,WAAW,EAAE,CAAC;wBACzC,OAAO,IAAI,CAAC;oBACd,CAAC;oBAED,OAAO,aAAa,CAAC,OAAO,KAAK,IAAI,CAAC;gBACxC,CAAC,CAAC,CAAC;gBAEH,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;oBACrB,MAAM,MAAM,GAAG,IAAA,cAAK,EAAC,SAAS,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;oBAC1D,IAAI,CAAC,MAAM,EAAE,KAAK,CAChB;wBACE,MAAM;wBACN,WAAW,EAAE,SAAS,CAAC,MAAM;wBAC7B,UAAU,EAAE,MAAM,CAAC,MAAM;wBACzB,SAAS,EAAE,IAAI,CAAC,mBAAmB;qBACpC,EACD,gCAAgC,CACjC,CAAC;oBAEF,IAAI,CAAC;wBACH,MAAM,OAAO,CAAC,GAAG,CACf,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE;4BACpC,MAAM,uBAAuB,GAAG,MAAM,IAAA,iCAAe,EAAC;gCACpD,oBAAoB,EAAE,MAAM;gCAC5B,cAAc,EAAE,UAAU;gCAC1B,IAAI;gCACJ,MAAM,EAAE,IAAI,CAAC,aAAa;gCAC1B,MAAM,EAAE,IAAI,CAAC,MAAM;6BACpB,CAAC,CAAC;4BAEH,KAAK,MAAM,EACT,aAAa,EACb,UAAU,GACX,IAAI,uBAAuB,EAAE,CAAC;gCAC7B,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;oCAC9C,IAAI,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE;wCACvC,aAAa;wCACb,KAAK;qCACN,CAAC,CAAC;gCACL,CAAC;4BACH,CAAC;4BAED,IAAI,CAAC,MAAM,EAAE,KAAK,CAChB;gCACE,MAAM;gCACN,UAAU;gCACV,eAAe,EAAE,IAAI,CAAC,MAAM;gCAC5B,YAAY,EAAE,uBAAuB,CAAC,MAAM;6BAC7C,EACD,mCAAmC,CACpC,CAAC;wBACJ,CAAC,CAAC,CACH,CAAC;oBACJ,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,IAAI,CAAC,MAAM,EAAE,KAAK,CAChB;4BACE,MAAM;4BACN,WAAW,EAAE,SAAS,CAAC,MAAM;4BAC7B,UAAU,EAAE,MAAM,CAAC,MAAM;4BACzB,KAAK,EAAE,KAAK,CAAC,OAAO;4BACpB,UAAU,EAAE,KAAK,CAAC,KAAK;yBACxB,EACD,uCAAuC,CACxC,CAAC;wBACF,MAAM,KAAK,CAAC;oBACd,CAAC;gBACH,CAAC;gBAED,IAAI,CAAC,qBAAqB,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBAC3D,gBAAgB,GAAG,gBAAgB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACvD,CAAC;YAED,IAAI,iBAAiB,EAAE,CAAC;gBACtB,IAAI,CAAC;oBACH,MAAM,iBAAiB,CAAC,gBAAgB,CAAC,CAAC;gBAC5C,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,IAAI,CAAC,MAAM,EAAE,KAAK,CAChB;wBACE,WAAW,EAAE,gBAAgB,CAAC,MAAM;wBACpC,GAAG;qBACJ,EACD,mCAAmC,CACpC,CAAC;oBACF,MAAM,GAAG,CAAC;gBACZ,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IACD;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,wBAAwB,CAC5B,sBAAyE,EACzE,QAAiB,KAAK;QAEtB,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,IAAI,EAAE;YAClC,mGAAmG;YACnG,mIAAmI;YACnI,mGAAmG;YACnG,IACE,CAAC,KAAK;gBACN,IAAI,CAAC,qBAAqB,CAAC,+BAA+B,EAAE;oBAC1D,IAAI,CAAC,iCAAiC,EACxC,CAAC;gBACD,OAAO;YACT,CAAC;YACD,MAAM,mBAAmB,GACvB,IAAI,CAAC,qBAAqB,CAAC,0BAA0B,EAAE,CAAC;YAC1D,IAAI,qBAAqB,GAAmB,EAAE,CAAC;YAC/C,KAAK,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,IAAI,mBAAmB,EAAE,CAAC;gBAC1D,MAAM,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;oBAC3C,MAAM,aAAa,GAAG,IAAI,CAAC,kCAAkC,CAAC;wBAC5D,MAAM;wBACN,KAAK,EAAE,CAAC,CAAC,KAAK;wBACd,yBAAyB,EAAE,eAAe;qBAC3C,CAAC,CAAC;oBAEH,IAAI,OAAO,aAAa,KAAK,WAAW,EAAE,CAAC;wBACzC,OAAO,IAAI,CAAC;oBACd,CAAC;oBAED,OAAO,aAAa,CAAC,OAAO,KAAK,IAAI,CAAC;gBACxC,CAAC,CAAC,CAAC;gBAEH,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;oBACrB,MAAM,MAAM,GAAG,IAAA,cAAK,EAAC,SAAS,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;oBAC1D,IAAI,CAAC,MAAM,EAAE,KAAK,CAChB;wBACE,MAAM;wBACN,iBAAiB,EAAE,SAAS,CAAC,MAAM;wBACnC,UAAU,EAAE,MAAM,CAAC,MAAM;wBACzB,SAAS,EAAE,IAAI,CAAC,mBAAmB;qBACpC,EACD,sCAAsC,CACvC,CAAC;oBAEF,IAAI,CAAC;wBACH,MAAM,OAAO,CAAC,GAAG,CACf,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE;4BACpC,MAAM,IAAA,iCAAe,EAAC;gCACpB,oBAAoB,EAAE,MAAM;gCAC5B,cAAc,EAAE,eAAe;gCAC/B,IAAI;gCACJ,MAAM,EAAE,IAAI,CAAC,aAAa;gCAC1B,MAAM,EAAE,IAAI,CAAC,MAAM;6BACpB,CAAC,CAAC;4BAEH,IAAI,CAAC,MAAM,EAAE,KAAK,CAChB;gCACE,MAAM;gCACN,UAAU;gCACV,oBAAoB,EAAE,IAAI,CAAC,MAAM;6BAClC,EACD,yCAAyC,CAC1C,CAAC;wBACJ,CAAC,CAAC,CACH,CAAC;oBACJ,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,IAAI,CAAC,MAAM,EAAE,KAAK,CAChB;4BACE,MAAM;4BACN,iBAAiB,EAAE,SAAS,CAAC,MAAM;4BACnC,UAAU,EAAE,MAAM,CAAC,MAAM;4BACzB,KAAK,EAAE,KAAK,CAAC,OAAO;4BACpB,UAAU,EAAE,KAAK,CAAC,KAAK;yBACxB,EACD,6CAA6C,CAC9C,CAAC;wBACF,MAAM,KAAK,CAAC;oBACd,CAAC;gBACH,CAAC;gBAED,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;gBACrE,qBAAqB,GAAG,qBAAqB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACtE,CAAC;YAED,IAAI,sBAAsB,EAAE,CAAC;gBAC3B,IAAI,CAAC;oBACH,MAAM,sBAAsB,CAAC,qBAAqB,CAAC,CAAC;gBACtD,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAI,CAAC,MAAM,EAAE,KAAK,CAChB;wBACE,iBAAiB,EAAE,qBAAqB,CAAC,MAAM;wBAC/C,KAAK,EAAE,KAAK,CAAC,OAAO;wBACpB,UAAU,EAAE,KAAK,CAAC,KAAK;qBACxB,EACD,wCAAwC,CACzC,CAAC;oBACF,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,kCAAkC,CAAC,EACjC,MAAM,EACN,KAAK,EACL,yBAAyB,GACgB;QAGzC,IAAI,CAAC,IAAI,CAAC,mCAAmC,EAAE,CAAC;YAC9C,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,mCAAmC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACjE,OAAO,GAAG,IAAI,GAAG,CAAC,yBAAyB,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC1D,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACK,KAAK,CAAC,aAAa,CAAI,SAA2B;QACxD,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QAC/B,IAAI,CAAC;YACH,MAAM,SAAS,EAAE,CAAC;QACpB,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC;CACF;AA/bD,gEA+bC"}
|
|
@@ -5,6 +5,7 @@ interface FlushDataToDiskInput<TGraphObject = Entity | Relationship> {
|
|
|
5
5
|
collectionType: CollectionType;
|
|
6
6
|
data: TGraphObject[];
|
|
7
7
|
pretty?: boolean;
|
|
8
|
+
logger?: any;
|
|
8
9
|
}
|
|
9
10
|
interface GraphObjectToFilePath<TGraphObject = Entity | Relationship> {
|
|
10
11
|
graphDataPath: string;
|
|
@@ -15,5 +16,5 @@ interface GraphObjectToFilePath<TGraphObject = Entity | Relationship> {
|
|
|
15
16
|
* creating a symlink in the 'index' directory
|
|
16
17
|
* based on the entity or relationship '_type'.
|
|
17
18
|
*/
|
|
18
|
-
export declare function flushDataToDisk<TGraphObject = Entity | Relationship>({ storageDirectoryPath, collectionType, data, pretty, }: FlushDataToDiskInput<TGraphObject>): Promise<GraphObjectToFilePath<TGraphObject>[]>;
|
|
19
|
+
export declare function flushDataToDisk<TGraphObject = Entity | Relationship>({ storageDirectoryPath, collectionType, data, pretty, logger, }: FlushDataToDiskInput<TGraphObject>): Promise<GraphObjectToFilePath<TGraphObject>[]>;
|
|
19
20
|
export {};
|
|
@@ -14,12 +14,24 @@ const path_1 = require("./path");
|
|
|
14
14
|
* creating a symlink in the 'index' directory
|
|
15
15
|
* based on the entity or relationship '_type'.
|
|
16
16
|
*/
|
|
17
|
-
async function flushDataToDisk({ storageDirectoryPath, collectionType, data, pretty, }) {
|
|
17
|
+
async function flushDataToDisk({ storageDirectoryPath, collectionType, data, pretty, logger, }) {
|
|
18
18
|
// split the data by type first
|
|
19
19
|
const groupedCollections = (0, groupBy_1.default)(data, '_type');
|
|
20
|
+
const totalObjects = data.length;
|
|
21
|
+
const typeCount = Object.keys(groupedCollections).length;
|
|
22
|
+
const chunkInfo = Object.entries(groupedCollections)
|
|
23
|
+
.map(([type, items]) => `${type}:${items.length}`)
|
|
24
|
+
.join(', ');
|
|
25
|
+
logger?.debug({
|
|
26
|
+
stepId: storageDirectoryPath,
|
|
27
|
+
collectionType,
|
|
28
|
+
totalObjects,
|
|
29
|
+
typeCount,
|
|
30
|
+
chunkInfo,
|
|
31
|
+
}, 'Flushing data to disk');
|
|
20
32
|
// for each collection, write the data to disk,
|
|
21
33
|
// then symlink to index directory
|
|
22
|
-
|
|
34
|
+
const results = await (0, p_map_1.default)(Object.entries(groupedCollections), async ([type, collection]) => {
|
|
23
35
|
const filename = generateJsonFilename();
|
|
24
36
|
const graphDataPath = (0, path_1.buildObjectCollectionFilePath)({
|
|
25
37
|
storageDirectoryPath,
|
|
@@ -27,22 +39,47 @@ async function flushDataToDisk({ storageDirectoryPath, collectionType, data, pre
|
|
|
27
39
|
filename,
|
|
28
40
|
});
|
|
29
41
|
const indexPath = (0, path_1.buildIndexFilePath)({ type, collectionType, filename });
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
42
|
+
try {
|
|
43
|
+
await (0, fileSystem_1.writeJsonToPath)({
|
|
44
|
+
path: graphDataPath,
|
|
45
|
+
data: {
|
|
46
|
+
[collectionType]: collection,
|
|
47
|
+
},
|
|
48
|
+
pretty,
|
|
49
|
+
});
|
|
50
|
+
await (0, fileSystem_1.symlink)({
|
|
51
|
+
sourcePath: graphDataPath,
|
|
52
|
+
destinationPath: indexPath,
|
|
53
|
+
});
|
|
54
|
+
logger?.debug({
|
|
55
|
+
type,
|
|
56
|
+
count: collection.length,
|
|
57
|
+
filename,
|
|
58
|
+
graphDataPath,
|
|
59
|
+
}, 'Chunk written to disk successfully');
|
|
60
|
+
return {
|
|
61
|
+
graphDataPath,
|
|
62
|
+
collection,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
catch (error) {
|
|
66
|
+
logger?.error({
|
|
67
|
+
error,
|
|
68
|
+
type,
|
|
69
|
+
count: collection.length,
|
|
70
|
+
filename,
|
|
71
|
+
graphDataPath,
|
|
72
|
+
}, 'Failed to write chunk to disk');
|
|
73
|
+
throw error;
|
|
74
|
+
}
|
|
45
75
|
}, { concurrency: 3 });
|
|
76
|
+
logger?.info({
|
|
77
|
+
stepId: storageDirectoryPath,
|
|
78
|
+
collectionType,
|
|
79
|
+
chunksWritten: results.length,
|
|
80
|
+
totalObjectsWritten: results.reduce((sum, r) => sum + r.collection.length, 0),
|
|
81
|
+
}, 'Successfully flushed all chunks to disk');
|
|
82
|
+
return results;
|
|
46
83
|
}
|
|
47
84
|
function generateJsonFilename() {
|
|
48
85
|
return `${(0, crypto_1.randomUUID)()}.json`;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"flushDataToDisk.js","sourceRoot":"","sources":["../../../../src/storage/FileSystemGraphObjectStore/flushDataToDisk.ts"],"names":[],"mappings":";;;;;
|
|
1
|
+
{"version":3,"file":"flushDataToDisk.js","sourceRoot":"","sources":["../../../../src/storage/FileSystemGraphObjectStore/flushDataToDisk.ts"],"names":[],"mappings":";;;;;AAgCA,0CAqGC;AArID,kDAAyB;AACzB,mCAA4C;AAC5C,6DAAqC;AAIrC,iDAA4D;AAE5D,iCAIgB;AAehB;;;;GAIG;AACI,KAAK,UAAU,eAAe,CAAuC,EAC1E,oBAAoB,EACpB,cAAc,EACd,IAAI,EACJ,MAAM,EACN,MAAM,GAC6B;IAGnC,+BAA+B;IAC/B,MAAM,kBAAkB,GAAG,IAAA,iBAAO,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAElD,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC;IACjC,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC;IACzD,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC;SACjD,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;SACjD,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,MAAM,EAAE,KAAK,CACX;QACE,MAAM,EAAE,oBAAoB;QAC5B,cAAc;QACd,YAAY;QACZ,SAAS;QACT,SAAS;KACV,EACD,uBAAuB,CACxB,CAAC;IAEF,+CAA+C;IAC/C,kCAAkC;IAClC,MAAM,OAAO,GAAG,MAAM,IAAA,eAAI,EACxB,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,EAClC,KAAK,EAAE,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,EAAE;QAC3B,MAAM,QAAQ,GAAG,oBAAoB,EAAE,CAAC;QACxC,MAAM,aAAa,GAAG,IAAA,oCAA6B,EAAC;YAClD,oBAAoB;YACpB,cAAc;YACd,QAAQ;SACT,CAAC,CAAC;QACH,MAAM,SAAS,GAAG,IAAA,yBAAkB,EAAC,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAC,CAAC;QAEzE,IAAI,CAAC;YACH,MAAM,IAAA,4BAAe,EAAC;gBACpB,IAAI,EAAE,aAAa;gBACnB,IAAI,EAAE;oBACJ,CAAC,cAAc,CAAC,EAAE,UAAU;iBAC7B;gBACD,MAAM;aACP,CAAC,CAAC;YAEH,MAAM,IAAA,oBAAO,EAAC;gBACZ,UAAU,EAAE,aAAa;gBACzB,eAAe,EAAE,SAAS;aAC3B,CAAC,CAAC;YAEH,MAAM,EAAE,KAAK,CACX;gBACE,IAAI;gBACJ,KAAK,EAAE,UAAU,CAAC,MAAM;gBACxB,QAAQ;gBACR,aAAa;aACd,EACD,oCAAoC,CACrC,CAAC;YAEF,OAAO;gBACL,aAAa;gBACb,UAAU;aACX,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,EAAE,KAAK,CACX;gBACE,KAAK;gBACL,IAAI;gBACJ,KAAK,EAAE,UAAU,CAAC,MAAM;gBACxB,QAAQ;gBACR,aAAa;aACd,EACD,+BAA+B,CAChC,CAAC;YACF,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC,EACD,EAAE,WAAW,EAAE,CAAC,EAAE,CACnB,CAAC;IAEF,MAAM,EAAE,IAAI,CACV;QACE,MAAM,EAAE,oBAAoB;QAC5B,cAAc;QACd,aAAa,EAAE,OAAO,CAAC,MAAM;QAC7B,mBAAmB,EAAE,OAAO,CAAC,MAAM,CACjC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EACrC,CAAC,CACF;KACF,EACD,yCAAyC,CAC1C,CAAC;IAEF,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,oBAAoB;IAC3B,OAAO,GAAG,IAAA,mBAAI,GAAE,OAAO,CAAC;AAC1B,CAAC"}
|