@devrev/ts-adaas 1.13.3 → 1.13.4-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -1
- package/dist/common/helpers.d.ts +28 -21
- package/dist/common/helpers.d.ts.map +1 -1
- package/dist/common/helpers.js +18 -156
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -3
- package/dist/multithreading/create-worker.d.ts.map +1 -0
- package/dist/multithreading/create-worker.test.d.ts.map +1 -0
- package/dist/multithreading/process-task.d.ts.map +1 -0
- package/dist/{workers → multithreading}/process-task.js +1 -1
- package/dist/{workers → multithreading/spawn}/spawn.d.ts +1 -1
- package/dist/multithreading/spawn/spawn.d.ts.map +1 -0
- package/dist/multithreading/spawn/spawn.helpers.d.ts +21 -0
- package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -0
- package/dist/multithreading/spawn/spawn.helpers.js +114 -0
- package/dist/multithreading/spawn/spawn.helpers.test.d.ts +2 -0
- package/dist/multithreading/spawn/spawn.helpers.test.d.ts.map +1 -0
- package/dist/multithreading/spawn/spawn.helpers.test.js +293 -0
- package/dist/{workers → multithreading/spawn}/spawn.js +11 -10
- package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts.map +1 -0
- package/dist/{workers → multithreading/worker-adapter}/worker-adapter.artifacts.test.js +4 -4
- package/dist/{workers → multithreading/worker-adapter}/worker-adapter.d.ts +8 -8
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts +22 -0
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.js +64 -0
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.js +514 -0
- package/dist/{workers → multithreading/worker-adapter}/worker-adapter.js +16 -16
- package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts.map +1 -0
- package/dist/{workers → multithreading/worker-adapter}/worker-adapter.test.js +10 -10
- package/dist/{workers → multithreading}/worker.d.ts.map +1 -1
- package/dist/state/state.d.ts.map +1 -1
- package/dist/state/state.js +1 -2
- package/dist/types/extraction.d.ts +1 -1
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/types/workers.d.ts +1 -1
- package/dist/types/workers.d.ts.map +1 -1
- package/dist/uploader/uploader.d.ts +1 -26
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.d.ts +32 -0
- package/dist/uploader/uploader.helpers.d.ts.map +1 -0
- package/dist/uploader/uploader.helpers.js +140 -0
- package/dist/uploader/uploader.helpers.test.d.ts +2 -0
- package/dist/uploader/uploader.helpers.test.d.ts.map +1 -0
- package/dist/uploader/uploader.helpers.test.js +265 -0
- package/dist/uploader/uploader.js +7 -117
- package/dist/uploader/uploader.test.js +561 -72
- package/package.json +1 -1
- package/dist/common/helpers.test.d.ts +0 -2
- package/dist/common/helpers.test.d.ts.map +0 -1
- package/dist/common/helpers.test.js +0 -158
- package/dist/workers/create-worker.d.ts.map +0 -1
- package/dist/workers/create-worker.test.d.ts.map +0 -1
- package/dist/workers/process-task.d.ts.map +0 -1
- package/dist/workers/spawn.d.ts.map +0 -1
- package/dist/workers/worker-adapter.artifacts.test.d.ts.map +0 -1
- package/dist/workers/worker-adapter.d.ts.map +0 -1
- package/dist/workers/worker-adapter.test.d.ts.map +0 -1
- /package/dist/{workers → multithreading}/create-worker.d.ts +0 -0
- /package/dist/{workers → multithreading}/create-worker.js +0 -0
- /package/dist/{workers → multithreading}/create-worker.test.d.ts +0 -0
- /package/dist/{workers → multithreading}/create-worker.test.js +0 -0
- /package/dist/{workers → multithreading}/process-task.d.ts +0 -0
- /package/dist/{workers → multithreading/worker-adapter}/worker-adapter.artifacts.test.d.ts +0 -0
- /package/dist/{workers → multithreading/worker-adapter}/worker-adapter.test.d.ts +0 -0
- /package/dist/{workers → multithreading}/worker.d.ts +0 -0
- /package/dist/{workers → multithreading}/worker.js +0 -0
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { ExternalSystemAttachmentStreamingFunction, NormalizedAttachment } from '../types';
|
|
2
|
-
import { WorkerAdapter } from '../
|
|
2
|
+
import { WorkerAdapter } from '../multithreading/worker-adapter/worker-adapter';
|
|
3
3
|
export interface AttachmentsStreamingPoolParams<ConnectorState> {
|
|
4
4
|
adapter: WorkerAdapter<ConnectorState>;
|
|
5
5
|
attachments: NormalizedAttachment[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"attachments-streaming-pool.interfaces.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,yCAAyC,EACzC,oBAAoB,EACrB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,aAAa,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"attachments-streaming-pool.interfaces.d.ts","sourceRoot":"","sources":["../../src/attachments-streaming/attachments-streaming-pool.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,yCAAyC,EACzC,oBAAoB,EACrB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,aAAa,EAAE,MAAM,iDAAiD,CAAC;AAEhF,MAAM,WAAW,8BAA8B,CAAC,cAAc;IAC5D,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IACvC,WAAW,EAAE,oBAAoB,EAAE,CAAC;IACpC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,yCAAyC,CAAC;CACnD"}
|
package/dist/common/helpers.d.ts
CHANGED
|
@@ -1,27 +1,30 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
};
|
|
6
|
-
export declare function getNoScriptEventType(eventType: EventType): {
|
|
7
|
-
eventType: ExtractorEventType;
|
|
8
|
-
} | {
|
|
9
|
-
eventType: LoaderEventType;
|
|
10
|
-
};
|
|
11
|
-
export declare function getSyncDirection({ event }: {
|
|
12
|
-
event: AirdropEvent;
|
|
13
|
-
}): string;
|
|
14
|
-
export declare function getFilesToLoad({ supportedItemTypes, statsFile, }: {
|
|
15
|
-
supportedItemTypes: string[];
|
|
16
|
-
statsFile: StatsFileObject[];
|
|
17
|
-
}): FileToLoad[];
|
|
18
|
-
export declare function addReportToLoaderReport({ loaderReports, report, }: {
|
|
19
|
-
loaderReports: LoaderReport[];
|
|
20
|
-
report: LoaderReport;
|
|
21
|
-
}): LoaderReport[];
|
|
1
|
+
/**
|
|
2
|
+
* Gets the library version from the package.json file.
|
|
3
|
+
* @returns {string} The library version
|
|
4
|
+
*/
|
|
22
5
|
export declare function getLibraryVersion(): any;
|
|
6
|
+
/**
|
|
7
|
+
* Sleeps for a given number of milliseconds.
|
|
8
|
+
* @param {number} ms - The number of milliseconds to sleep
|
|
9
|
+
* @returns {Promise<void>} A promise that resolves after the given number of milliseconds
|
|
10
|
+
*/
|
|
23
11
|
export declare function sleep(ms: number): Promise<unknown>;
|
|
12
|
+
/**
|
|
13
|
+
* Truncates a filename if it exceeds the maximum allowed length.
|
|
14
|
+
* @param {string} filename - The filename to truncate
|
|
15
|
+
* @returns {string} The truncated filename
|
|
16
|
+
*/
|
|
24
17
|
export declare function truncateFilename(filename: string): string;
|
|
18
|
+
/**
|
|
19
|
+
* MemoryInfo is an interface that represents the memory usage information.
|
|
20
|
+
* @interface MemoryInfo
|
|
21
|
+
* @property {string} rssUsedMB - The RSS used in MB
|
|
22
|
+
* @property {string} rssUsedPercent - The RSS used percentage
|
|
23
|
+
* @property {string} heapUsedPercent - The heap used percentage
|
|
24
|
+
* @property {string} externalMB - The external memory used in MB
|
|
25
|
+
* @property {string} arrayBuffersMB - The array buffers memory used in MB
|
|
26
|
+
* @property {string} formattedMessage - The formatted message
|
|
27
|
+
*/
|
|
25
28
|
export interface MemoryInfo {
|
|
26
29
|
rssUsedMB: string;
|
|
27
30
|
rssUsedPercent: string;
|
|
@@ -30,5 +33,9 @@ export interface MemoryInfo {
|
|
|
30
33
|
arrayBuffersMB: string;
|
|
31
34
|
formattedMessage: string;
|
|
32
35
|
}
|
|
36
|
+
/**
|
|
37
|
+
* Gets the memory usage information.
|
|
38
|
+
* @returns {MemoryInfo} The memory usage information
|
|
39
|
+
*/
|
|
33
40
|
export declare function getMemoryUsage(): MemoryInfo;
|
|
34
41
|
//# sourceMappingURL=helpers.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../src/common/helpers.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../src/common/helpers.ts"],"names":[],"mappings":"AASA;;;GAGG;AACH,wBAAgB,iBAAiB,QAiBhC;AAED;;;;GAIG;AACH,wBAAsB,KAAK,CAAC,EAAE,EAAE,MAAM,oBAGrC;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAmBzD;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED;;;GAGG;AACH,wBAAgB,cAAc,IAAI,UAAU,CAmD3C"}
|
package/dist/common/helpers.js
CHANGED
|
@@ -33,11 +33,6 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.getTimeoutErrorEventType = getTimeoutErrorEventType;
|
|
37
|
-
exports.getNoScriptEventType = getNoScriptEventType;
|
|
38
|
-
exports.getSyncDirection = getSyncDirection;
|
|
39
|
-
exports.getFilesToLoad = getFilesToLoad;
|
|
40
|
-
exports.addReportToLoaderReport = addReportToLoaderReport;
|
|
41
36
|
exports.getLibraryVersion = getLibraryVersion;
|
|
42
37
|
exports.sleep = sleep;
|
|
43
38
|
exports.truncateFilename = truncateFilename;
|
|
@@ -45,158 +40,11 @@ exports.getMemoryUsage = getMemoryUsage;
|
|
|
45
40
|
const fs_1 = require("fs");
|
|
46
41
|
const path = __importStar(require("path"));
|
|
47
42
|
const v8 = __importStar(require("v8"));
|
|
48
|
-
const extraction_1 = require("../types/extraction");
|
|
49
|
-
const loading_1 = require("../types/loading");
|
|
50
43
|
const constants_1 = require("./constants");
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
case extraction_1.EventType.ExtractionMetadataStart:
|
|
56
|
-
return {
|
|
57
|
-
eventType: extraction_1.ExtractorEventType.MetadataExtractionError,
|
|
58
|
-
};
|
|
59
|
-
// Data extraction (handles both old and new enum members)
|
|
60
|
-
case extraction_1.EventType.StartExtractingData:
|
|
61
|
-
case extraction_1.EventType.ContinueExtractingData:
|
|
62
|
-
case extraction_1.EventType.ExtractionDataStart:
|
|
63
|
-
case extraction_1.EventType.ExtractionDataContinue:
|
|
64
|
-
return {
|
|
65
|
-
eventType: extraction_1.ExtractorEventType.DataExtractionError,
|
|
66
|
-
};
|
|
67
|
-
// Data deletion (handles both old and new enum members)
|
|
68
|
-
case extraction_1.EventType.StartDeletingExtractorState:
|
|
69
|
-
case extraction_1.EventType.ExtractionDataDelete:
|
|
70
|
-
return {
|
|
71
|
-
eventType: extraction_1.ExtractorEventType.ExtractorStateDeletionError,
|
|
72
|
-
};
|
|
73
|
-
// Attachments extraction (handles both old and new enum members)
|
|
74
|
-
case extraction_1.EventType.StartExtractingAttachments:
|
|
75
|
-
case extraction_1.EventType.ContinueExtractingAttachments:
|
|
76
|
-
case extraction_1.EventType.ExtractionAttachmentsStart:
|
|
77
|
-
case extraction_1.EventType.ExtractionAttachmentsContinue:
|
|
78
|
-
return {
|
|
79
|
-
eventType: extraction_1.ExtractorEventType.AttachmentExtractionError,
|
|
80
|
-
};
|
|
81
|
-
// Attachments deletion (handles both old and new enum members)
|
|
82
|
-
case extraction_1.EventType.StartDeletingExtractorAttachmentsState:
|
|
83
|
-
case extraction_1.EventType.ExtractionAttachmentsDelete:
|
|
84
|
-
return {
|
|
85
|
-
eventType: extraction_1.ExtractorEventType.ExtractorAttachmentsStateDeletionError,
|
|
86
|
-
};
|
|
87
|
-
// External sync units (handles both old and new enum members)
|
|
88
|
-
case extraction_1.EventType.StartExtractingExternalSyncUnits:
|
|
89
|
-
case extraction_1.EventType.ExtractionExternalSyncUnitsStart:
|
|
90
|
-
return {
|
|
91
|
-
eventType: extraction_1.ExtractorEventType.ExternalSyncUnitExtractionError,
|
|
92
|
-
};
|
|
93
|
-
// Loading data
|
|
94
|
-
case extraction_1.EventType.StartLoadingData:
|
|
95
|
-
case extraction_1.EventType.ContinueLoadingData:
|
|
96
|
-
return {
|
|
97
|
-
eventType: loading_1.LoaderEventType.DataLoadingError,
|
|
98
|
-
};
|
|
99
|
-
// Deleting loader state
|
|
100
|
-
case extraction_1.EventType.StartDeletingLoaderState:
|
|
101
|
-
return {
|
|
102
|
-
eventType: loading_1.LoaderEventType.LoaderStateDeletionError,
|
|
103
|
-
};
|
|
104
|
-
// Loading attachments
|
|
105
|
-
case extraction_1.EventType.StartLoadingAttachments:
|
|
106
|
-
case extraction_1.EventType.ContinueLoadingAttachments:
|
|
107
|
-
return {
|
|
108
|
-
eventType: loading_1.LoaderEventType.AttachmentLoadingError,
|
|
109
|
-
};
|
|
110
|
-
// Deleting loader attachment state
|
|
111
|
-
case extraction_1.EventType.StartDeletingLoaderAttachmentState:
|
|
112
|
-
return {
|
|
113
|
-
eventType: loading_1.LoaderEventType.LoaderAttachmentStateDeletionError,
|
|
114
|
-
};
|
|
115
|
-
default:
|
|
116
|
-
console.error('Event type not recognized in getTimeoutErrorEventType function: ' +
|
|
117
|
-
eventType);
|
|
118
|
-
return {
|
|
119
|
-
eventType: loading_1.LoaderEventType.UnknownEventType,
|
|
120
|
-
};
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
function getNoScriptEventType(eventType) {
|
|
124
|
-
switch (eventType) {
|
|
125
|
-
case extraction_1.EventType.StartDeletingExtractorState:
|
|
126
|
-
return {
|
|
127
|
-
eventType: extraction_1.ExtractorEventType.ExtractorStateDeletionDone,
|
|
128
|
-
};
|
|
129
|
-
case extraction_1.EventType.StartDeletingExtractorAttachmentsState:
|
|
130
|
-
return {
|
|
131
|
-
eventType: extraction_1.ExtractorEventType.ExtractorAttachmentsStateDeletionDone,
|
|
132
|
-
};
|
|
133
|
-
case extraction_1.EventType.StartDeletingLoaderState:
|
|
134
|
-
return {
|
|
135
|
-
eventType: loading_1.LoaderEventType.LoaderStateDeletionDone,
|
|
136
|
-
};
|
|
137
|
-
case extraction_1.EventType.StartDeletingLoaderAttachmentState:
|
|
138
|
-
return {
|
|
139
|
-
eventType: loading_1.LoaderEventType.LoaderAttachmentStateDeletionDone,
|
|
140
|
-
};
|
|
141
|
-
default:
|
|
142
|
-
console.error('Event type not recognized in getNoScriptEventType function: ' +
|
|
143
|
-
eventType);
|
|
144
|
-
return {
|
|
145
|
-
eventType: loading_1.LoaderEventType.UnknownEventType,
|
|
146
|
-
};
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
function getSyncDirection({ event }) {
|
|
150
|
-
return event.payload.event_context.mode;
|
|
151
|
-
}
|
|
152
|
-
function getFilesToLoad({ supportedItemTypes, statsFile, }) {
|
|
153
|
-
const filesToLoad = [];
|
|
154
|
-
if (supportedItemTypes.length === 0 || statsFile.length === 0) {
|
|
155
|
-
return [];
|
|
156
|
-
}
|
|
157
|
-
const filteredStatsFile = statsFile.filter((file) => supportedItemTypes.includes(file.item_type));
|
|
158
|
-
const orderedFiles = filteredStatsFile.sort((a, b) => {
|
|
159
|
-
const aIndex = supportedItemTypes.indexOf(a.item_type);
|
|
160
|
-
const bIndex = supportedItemTypes.indexOf(b.item_type);
|
|
161
|
-
return aIndex - bIndex;
|
|
162
|
-
});
|
|
163
|
-
for (const file of orderedFiles) {
|
|
164
|
-
filesToLoad.push({
|
|
165
|
-
id: file.id,
|
|
166
|
-
file_name: file.file_name,
|
|
167
|
-
itemType: file.item_type,
|
|
168
|
-
count: parseInt(file.count),
|
|
169
|
-
completed: false,
|
|
170
|
-
lineToProcess: 0,
|
|
171
|
-
});
|
|
172
|
-
}
|
|
173
|
-
return filesToLoad;
|
|
174
|
-
}
|
|
175
|
-
function addReportToLoaderReport({ loaderReports, report, }) {
|
|
176
|
-
const existingReport = loaderReports.find((loaderReport) => loaderReport.item_type === report.item_type);
|
|
177
|
-
if (existingReport) {
|
|
178
|
-
existingReport[loading_1.ActionType.CREATED] = existingReport[loading_1.ActionType.CREATED]
|
|
179
|
-
? report[loading_1.ActionType.CREATED]
|
|
180
|
-
? existingReport[loading_1.ActionType.CREATED] + report[loading_1.ActionType.CREATED]
|
|
181
|
-
: existingReport[loading_1.ActionType.CREATED]
|
|
182
|
-
: report[loading_1.ActionType.CREATED];
|
|
183
|
-
existingReport[loading_1.ActionType.UPDATED] = existingReport[loading_1.ActionType.UPDATED]
|
|
184
|
-
? report[loading_1.ActionType.UPDATED]
|
|
185
|
-
? existingReport[loading_1.ActionType.UPDATED] + report[loading_1.ActionType.UPDATED]
|
|
186
|
-
: existingReport[loading_1.ActionType.UPDATED]
|
|
187
|
-
: report[loading_1.ActionType.UPDATED];
|
|
188
|
-
existingReport[loading_1.ActionType.FAILED] = existingReport[loading_1.ActionType.FAILED]
|
|
189
|
-
? report[loading_1.ActionType.FAILED]
|
|
190
|
-
? existingReport[loading_1.ActionType.FAILED] + report[loading_1.ActionType.FAILED]
|
|
191
|
-
: existingReport[loading_1.ActionType.FAILED]
|
|
192
|
-
: report[loading_1.ActionType.FAILED];
|
|
193
|
-
}
|
|
194
|
-
else {
|
|
195
|
-
loaderReports.push(report);
|
|
196
|
-
}
|
|
197
|
-
return loaderReports;
|
|
198
|
-
}
|
|
199
|
-
// read adaas library version from package.json
|
|
44
|
+
/**
|
|
45
|
+
* Gets the library version from the package.json file.
|
|
46
|
+
* @returns {string} The library version
|
|
47
|
+
*/
|
|
200
48
|
function getLibraryVersion() {
|
|
201
49
|
var _a;
|
|
202
50
|
try {
|
|
@@ -211,10 +59,20 @@ function getLibraryVersion() {
|
|
|
211
59
|
return '';
|
|
212
60
|
}
|
|
213
61
|
}
|
|
62
|
+
/**
|
|
63
|
+
* Sleeps for a given number of milliseconds.
|
|
64
|
+
* @param {number} ms - The number of milliseconds to sleep
|
|
65
|
+
* @returns {Promise<void>} A promise that resolves after the given number of milliseconds
|
|
66
|
+
*/
|
|
214
67
|
async function sleep(ms) {
|
|
215
68
|
console.log(`Sleeping for ${ms}ms.`);
|
|
216
69
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
217
70
|
}
|
|
71
|
+
/**
|
|
72
|
+
* Truncates a filename if it exceeds the maximum allowed length.
|
|
73
|
+
* @param {string} filename - The filename to truncate
|
|
74
|
+
* @returns {string} The truncated filename
|
|
75
|
+
*/
|
|
218
76
|
function truncateFilename(filename) {
|
|
219
77
|
// If the filename is already within the limit, return it as is.
|
|
220
78
|
if (filename.length <= constants_1.MAX_DEVREV_FILENAME_LENGTH) {
|
|
@@ -228,6 +86,10 @@ function truncateFilename(filename) {
|
|
|
228
86
|
const truncatedFilename = filename.slice(0, availableNameLength);
|
|
229
87
|
return `${truncatedFilename}...${extension}`;
|
|
230
88
|
}
|
|
89
|
+
/**
|
|
90
|
+
* Gets the memory usage information.
|
|
91
|
+
* @returns {MemoryInfo} The memory usage information
|
|
92
|
+
*/
|
|
231
93
|
function getMemoryUsage() {
|
|
232
94
|
try {
|
|
233
95
|
const memUsage = process.memoryUsage();
|
package/dist/index.d.ts
CHANGED
|
@@ -5,9 +5,9 @@ export * from './deprecated/uploader';
|
|
|
5
5
|
export * from './http';
|
|
6
6
|
export * from './types';
|
|
7
7
|
export * from './common/install-initial-domain-mapping';
|
|
8
|
-
export { processTask } from './
|
|
9
|
-
export { spawn } from './
|
|
10
|
-
export { WorkerAdapter } from './
|
|
8
|
+
export { processTask } from './multithreading/process-task';
|
|
9
|
+
export { spawn } from './multithreading/spawn/spawn';
|
|
10
|
+
export { WorkerAdapter } from './multithreading/worker-adapter/worker-adapter';
|
|
11
11
|
export * from './types/workers';
|
|
12
12
|
export { formatAxiosError, serializeAxiosError } from './logger/logger';
|
|
13
13
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,0BAA0B,CAAC;AACzC,cAAc,uBAAuB,CAAC;AAEtC,cAAc,QAAQ,CAAC;AACvB,cAAc,SAAS,CAAC;AAExB,cAAc,yCAAyC,CAAC;AAExD,OAAO,EAAE,WAAW,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,0BAA0B,CAAC;AACzC,cAAc,uBAAuB,CAAC;AAEtC,cAAc,QAAQ,CAAC;AACvB,cAAc,SAAS,CAAC;AAExB,cAAc,yCAAyC,CAAC;AAExD,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,KAAK,EAAE,MAAM,8BAA8B,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,gDAAgD,CAAC;AAE/E,cAAc,iBAAiB,CAAC;AAEhC,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -22,11 +22,11 @@ __exportStar(require("./deprecated/uploader"), exports);
|
|
|
22
22
|
__exportStar(require("./http"), exports);
|
|
23
23
|
__exportStar(require("./types"), exports);
|
|
24
24
|
__exportStar(require("./common/install-initial-domain-mapping"), exports);
|
|
25
|
-
var process_task_1 = require("./
|
|
25
|
+
var process_task_1 = require("./multithreading/process-task");
|
|
26
26
|
Object.defineProperty(exports, "processTask", { enumerable: true, get: function () { return process_task_1.processTask; } });
|
|
27
|
-
var spawn_1 = require("./
|
|
27
|
+
var spawn_1 = require("./multithreading/spawn/spawn");
|
|
28
28
|
Object.defineProperty(exports, "spawn", { enumerable: true, get: function () { return spawn_1.spawn; } });
|
|
29
|
-
var worker_adapter_1 = require("./
|
|
29
|
+
var worker_adapter_1 = require("./multithreading/worker-adapter/worker-adapter");
|
|
30
30
|
Object.defineProperty(exports, "WorkerAdapter", { enumerable: true, get: function () { return worker_adapter_1.WorkerAdapter; } });
|
|
31
31
|
__exportStar(require("./types/workers"), exports);
|
|
32
32
|
var logger_1 = require("./logger/logger");
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"create-worker.d.ts","sourceRoot":"","sources":["../../src/multithreading/create-worker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAE3D,OAAO,EAAE,UAAU,EAAe,MAAM,kBAAkB,CAAC;AAE3D,iBAAe,YAAY,CAAC,cAAc,EACxC,UAAU,EAAE,UAAU,CAAC,cAAc,CAAC,GACrC,OAAO,CAAC,MAAM,CAAC,CAyBjB;AAED,OAAO,EAAE,YAAY,EAAE,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"create-worker.test.d.ts","sourceRoot":"","sources":["../../src/multithreading/create-worker.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"process-task.d.ts","sourceRoot":"","sources":["../../src/multithreading/process-task.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,oBAAoB,EAGrB,MAAM,kBAAkB,CAAC;AAG1B,wBAAgB,WAAW,CAAC,cAAc,EAAE,EAC1C,IAAI,EACJ,SAAS,GACV,EAAE,oBAAoB,CAAC,cAAc,CAAC,QA6DtC"}
|
|
@@ -6,7 +6,7 @@ const event_type_translation_1 = require("../common/event-type-translation");
|
|
|
6
6
|
const logger_1 = require("../logger/logger");
|
|
7
7
|
const state_1 = require("../state/state");
|
|
8
8
|
const workers_1 = require("../types/workers");
|
|
9
|
-
const worker_adapter_1 = require("./worker-adapter");
|
|
9
|
+
const worker_adapter_1 = require("./worker-adapter/worker-adapter");
|
|
10
10
|
function processTask({ task, onTimeout, }) {
|
|
11
11
|
if (!node_worker_threads_1.isMainThread) {
|
|
12
12
|
void (async () => {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { SpawnFactoryInterface, SpawnInterface } from '
|
|
1
|
+
import { SpawnFactoryInterface, SpawnInterface } from '../../types/workers';
|
|
2
2
|
/**
|
|
3
3
|
* Creates a new instance of Spawn class.
|
|
4
4
|
* Spawn class is responsible for spawning a new worker thread and managing the lifecycle of the worker.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"spawn.d.ts","sourceRoot":"","sources":["../../../src/multithreading/spawn/spawn.ts"],"names":[],"mappings":"AAQA,OAAO,EAEL,qBAAqB,EACrB,cAAc,EAGf,MAAM,qBAAqB,CAAC;AA+C7B;;;;;;;;;;;GAWG;AACH,wBAAsB,KAAK,CAAC,cAAc,EAAE,EAC1C,KAAK,EACL,YAAY,EACZ,UAAU,EACV,oBAAoB,EACpB,OAAO,EACP,cAAc,GACf,EAAE,qBAAqB,CAAC,cAAc,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAwFvD;AAED,qBAAa,KAAK;IAChB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,cAAc,CAAU;IAChC,OAAO,CAAC,oBAAoB,CAAkC;IAC9D,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,gBAAgB,CAA4C;IACpE,OAAO,CAAC,gBAAgB,CAA4C;IACpE,OAAO,CAAC,wBAAwB,CAA6C;IAC7E,OAAO,CAAC,OAAO,CAA4C;IAC3D,OAAO,CAAC,eAAe,CAAU;IACjC,OAAO,CAAC,MAAM,CAAS;gBACX,EACV,KAAK,EACL,MAAM,EACN,OAAO,EACP,OAAO,EACP,eAAe,GAChB,EAAE,cAAc;IAgGjB,OAAO,CAAC,aAAa;YAYP,kBAAkB;CAiCjC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { EventType, ExtractorEventType } from '../../types/extraction';
|
|
2
|
+
import { LoaderEventType } from '../../types/loading';
|
|
3
|
+
/**
|
|
4
|
+
* Gets the event type for the timeout error.
|
|
5
|
+
* @param {EventType} eventType - The event type to get the timeout error event type for
|
|
6
|
+
* @returns {ExtractorEventType | LoaderEventType} The event type for the timeout error
|
|
7
|
+
*/
|
|
8
|
+
export declare function getTimeoutErrorEventType(eventType: EventType): {
|
|
9
|
+
eventType: ExtractorEventType | LoaderEventType;
|
|
10
|
+
};
|
|
11
|
+
/**
|
|
12
|
+
* Gets the event type for the no script error.
|
|
13
|
+
* @param {EventType} eventType - The event type to get the no script error event type for
|
|
14
|
+
* @returns {ExtractorEventType | LoaderEventType} The event type for the no script error
|
|
15
|
+
*/
|
|
16
|
+
export declare function getNoScriptEventType(eventType: EventType): {
|
|
17
|
+
eventType: ExtractorEventType;
|
|
18
|
+
} | {
|
|
19
|
+
eventType: LoaderEventType;
|
|
20
|
+
};
|
|
21
|
+
//# sourceMappingURL=spawn.helpers.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"spawn.helpers.d.ts","sourceRoot":"","sources":["../../../src/multithreading/spawn/spawn.helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEtD;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,SAAS,EAAE,SAAS,GAAG;IAC9D,SAAS,EAAE,kBAAkB,GAAG,eAAe,CAAC;CACjD,CAmFA;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,SAAS;;;;EA2BxD"}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.getTimeoutErrorEventType = getTimeoutErrorEventType;
|
|
4
|
+
exports.getNoScriptEventType = getNoScriptEventType;
|
|
5
|
+
const extraction_1 = require("../../types/extraction");
|
|
6
|
+
const loading_1 = require("../../types/loading");
|
|
7
|
+
/**
|
|
8
|
+
* Gets the event type for the timeout error.
|
|
9
|
+
* @param {EventType} eventType - The event type to get the timeout error event type for
|
|
10
|
+
* @returns {ExtractorEventType | LoaderEventType} The event type for the timeout error
|
|
11
|
+
*/
|
|
12
|
+
function getTimeoutErrorEventType(eventType) {
|
|
13
|
+
switch (eventType) {
|
|
14
|
+
// Metadata extraction (handles both old and new enum members)
|
|
15
|
+
case extraction_1.EventType.StartExtractingMetadata:
|
|
16
|
+
case extraction_1.EventType.ExtractionMetadataStart:
|
|
17
|
+
return {
|
|
18
|
+
eventType: extraction_1.ExtractorEventType.MetadataExtractionError,
|
|
19
|
+
};
|
|
20
|
+
// Data extraction (handles both old and new enum members)
|
|
21
|
+
case extraction_1.EventType.StartExtractingData:
|
|
22
|
+
case extraction_1.EventType.ContinueExtractingData:
|
|
23
|
+
case extraction_1.EventType.ExtractionDataStart:
|
|
24
|
+
case extraction_1.EventType.ExtractionDataContinue:
|
|
25
|
+
return {
|
|
26
|
+
eventType: extraction_1.ExtractorEventType.DataExtractionError,
|
|
27
|
+
};
|
|
28
|
+
// Data deletion (handles both old and new enum members)
|
|
29
|
+
case extraction_1.EventType.StartDeletingExtractorState:
|
|
30
|
+
case extraction_1.EventType.ExtractionDataDelete:
|
|
31
|
+
return {
|
|
32
|
+
eventType: extraction_1.ExtractorEventType.ExtractorStateDeletionError,
|
|
33
|
+
};
|
|
34
|
+
// Attachments extraction (handles both old and new enum members)
|
|
35
|
+
case extraction_1.EventType.StartExtractingAttachments:
|
|
36
|
+
case extraction_1.EventType.ContinueExtractingAttachments:
|
|
37
|
+
case extraction_1.EventType.ExtractionAttachmentsStart:
|
|
38
|
+
case extraction_1.EventType.ExtractionAttachmentsContinue:
|
|
39
|
+
return {
|
|
40
|
+
eventType: extraction_1.ExtractorEventType.AttachmentExtractionError,
|
|
41
|
+
};
|
|
42
|
+
// Attachments deletion (handles both old and new enum members)
|
|
43
|
+
case extraction_1.EventType.StartDeletingExtractorAttachmentsState:
|
|
44
|
+
case extraction_1.EventType.ExtractionAttachmentsDelete:
|
|
45
|
+
return {
|
|
46
|
+
eventType: extraction_1.ExtractorEventType.ExtractorAttachmentsStateDeletionError,
|
|
47
|
+
};
|
|
48
|
+
// External sync units (handles both old and new enum members)
|
|
49
|
+
case extraction_1.EventType.StartExtractingExternalSyncUnits:
|
|
50
|
+
case extraction_1.EventType.ExtractionExternalSyncUnitsStart:
|
|
51
|
+
return {
|
|
52
|
+
eventType: extraction_1.ExtractorEventType.ExternalSyncUnitExtractionError,
|
|
53
|
+
};
|
|
54
|
+
// Loading data
|
|
55
|
+
case extraction_1.EventType.StartLoadingData:
|
|
56
|
+
case extraction_1.EventType.ContinueLoadingData:
|
|
57
|
+
return {
|
|
58
|
+
eventType: loading_1.LoaderEventType.DataLoadingError,
|
|
59
|
+
};
|
|
60
|
+
// Deleting loader state
|
|
61
|
+
case extraction_1.EventType.StartDeletingLoaderState:
|
|
62
|
+
return {
|
|
63
|
+
eventType: loading_1.LoaderEventType.LoaderStateDeletionError,
|
|
64
|
+
};
|
|
65
|
+
// Loading attachments
|
|
66
|
+
case extraction_1.EventType.StartLoadingAttachments:
|
|
67
|
+
case extraction_1.EventType.ContinueLoadingAttachments:
|
|
68
|
+
return {
|
|
69
|
+
eventType: loading_1.LoaderEventType.AttachmentLoadingError,
|
|
70
|
+
};
|
|
71
|
+
// Deleting loader attachment state
|
|
72
|
+
case extraction_1.EventType.StartDeletingLoaderAttachmentState:
|
|
73
|
+
return {
|
|
74
|
+
eventType: loading_1.LoaderEventType.LoaderAttachmentStateDeletionError,
|
|
75
|
+
};
|
|
76
|
+
default:
|
|
77
|
+
console.error('Event type not recognized in getTimeoutErrorEventType function: ' +
|
|
78
|
+
eventType);
|
|
79
|
+
return {
|
|
80
|
+
eventType: loading_1.LoaderEventType.UnknownEventType,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Gets the event type for the no script error.
|
|
86
|
+
* @param {EventType} eventType - The event type to get the no script error event type for
|
|
87
|
+
* @returns {ExtractorEventType | LoaderEventType} The event type for the no script error
|
|
88
|
+
*/
|
|
89
|
+
function getNoScriptEventType(eventType) {
|
|
90
|
+
switch (eventType) {
|
|
91
|
+
case extraction_1.EventType.StartDeletingExtractorState:
|
|
92
|
+
return {
|
|
93
|
+
eventType: extraction_1.ExtractorEventType.ExtractorStateDeletionDone,
|
|
94
|
+
};
|
|
95
|
+
case extraction_1.EventType.StartDeletingExtractorAttachmentsState:
|
|
96
|
+
return {
|
|
97
|
+
eventType: extraction_1.ExtractorEventType.ExtractorAttachmentsStateDeletionDone,
|
|
98
|
+
};
|
|
99
|
+
case extraction_1.EventType.StartDeletingLoaderState:
|
|
100
|
+
return {
|
|
101
|
+
eventType: loading_1.LoaderEventType.LoaderStateDeletionDone,
|
|
102
|
+
};
|
|
103
|
+
case extraction_1.EventType.StartDeletingLoaderAttachmentState:
|
|
104
|
+
return {
|
|
105
|
+
eventType: loading_1.LoaderEventType.LoaderAttachmentStateDeletionDone,
|
|
106
|
+
};
|
|
107
|
+
default:
|
|
108
|
+
console.error('Event type not recognized in getNoScriptEventType function: ' +
|
|
109
|
+
eventType);
|
|
110
|
+
return {
|
|
111
|
+
eventType: loading_1.LoaderEventType.UnknownEventType,
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"spawn.helpers.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/spawn/spawn.helpers.test.ts"],"names":[],"mappings":""}
|