@devrev/airsync-sdk 2.0.0-beta.0 → 2.0.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -4
- package/dist/airsync-sdk.d.ts +1802 -0
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts +7 -0
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
- package/dist/attachments-streaming/attachments-streaming-pool.js +44 -4
- package/dist/attachments-streaming/attachments-streaming-pool.test.js +259 -10
- package/dist/common/constants.d.ts +16 -5
- package/dist/common/constants.d.ts.map +1 -1
- package/dist/common/constants.js +20 -6
- package/dist/common/control-protocol.d.ts +2 -2
- package/dist/common/control-protocol.d.ts.map +1 -1
- package/dist/common/control-protocol.js +1 -5
- package/dist/common/helpers.d.ts +8 -0
- package/dist/common/helpers.d.ts.map +1 -1
- package/dist/common/helpers.js +15 -0
- package/dist/common/install-initial-domain-mapping.d.ts +2 -2
- package/dist/common/install-initial-domain-mapping.test.js +7 -3
- package/dist/common/test-utils.d.ts +19 -0
- package/dist/common/test-utils.d.ts.map +1 -0
- package/dist/common/test-utils.js +101 -0
- package/dist/common/time-value-resolver.d.ts +47 -0
- package/dist/common/time-value-resolver.d.ts.map +1 -0
- package/dist/common/time-value-resolver.js +141 -0
- package/dist/common/time-value-resolver.test.d.ts +2 -0
- package/dist/common/time-value-resolver.test.d.ts.map +1 -0
- package/dist/common/time-value-resolver.test.js +268 -0
- package/dist/http/axios-client-internal.d.ts.map +1 -1
- package/dist/http/axios-client-internal.js +7 -0
- package/dist/http/axios-client-internal.test.js +27 -6
- package/dist/http/axios-client.d.ts.map +1 -1
- package/dist/http/axios-client.js +6 -3
- package/dist/http/index.d.ts +0 -1
- package/dist/http/index.d.ts.map +1 -1
- package/dist/http/index.js +0 -1
- package/dist/index.d.ts +9 -8
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +12 -10
- package/dist/logger/logger.d.ts +1 -17
- package/dist/logger/logger.d.ts.map +1 -1
- package/dist/logger/logger.interfaces.d.ts +2 -2
- package/dist/logger/logger.js +35 -27
- package/dist/logger/logger.test.js +66 -19
- package/dist/mappers/mappers.interface.d.ts +5 -5
- package/dist/mappers/mappers.interface.js +3 -3
- package/dist/mappers/mappers.test.js +8 -10
- package/dist/mock-server/mock-server.d.ts +47 -0
- package/dist/mock-server/mock-server.d.ts.map +1 -0
- package/dist/mock-server/mock-server.interfaces.d.ts +73 -0
- package/dist/mock-server/mock-server.interfaces.d.ts.map +1 -0
- package/dist/mock-server/mock-server.interfaces.js +4 -0
- package/dist/mock-server/mock-server.js +278 -0
- package/dist/multithreading/create-worker.test.js +36 -17
- package/dist/multithreading/process-task.d.ts.map +1 -1
- package/dist/multithreading/process-task.js +46 -44
- package/dist/multithreading/process-task.test.d.ts +2 -0
- package/dist/multithreading/process-task.test.d.ts.map +1 -0
- package/dist/multithreading/process-task.test.js +146 -0
- package/dist/multithreading/spawn/spawn.d.ts +4 -3
- package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
- package/dist/multithreading/spawn/spawn.helpers.js +6 -14
- package/dist/multithreading/spawn/spawn.helpers.test.js +8 -8
- package/dist/multithreading/spawn/spawn.js +41 -31
- package/dist/multithreading/spawn/spawn.test.d.ts +2 -0
- package/dist/multithreading/spawn/spawn.test.d.ts.map +1 -0
- package/dist/multithreading/spawn/spawn.test.js +232 -0
- package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.js +49 -1
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts +11 -5
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +1 -1
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.emit.test.js +390 -0
- package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.js +799 -0
- package/dist/multithreading/worker-adapter/worker-adapter.js +233 -134
- package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.loading.test.js +596 -0
- package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.js +71 -0
- package/dist/repo/repo.d.ts.map +1 -1
- package/dist/repo/repo.interfaces.d.ts +4 -2
- package/dist/repo/repo.interfaces.d.ts.map +1 -1
- package/dist/repo/repo.js +4 -3
- package/dist/repo/repo.test.js +60 -7
- package/dist/state/state.d.ts +7 -1
- package/dist/state/state.d.ts.map +1 -1
- package/dist/state/state.extract-window.test.d.ts +2 -0
- package/dist/state/state.extract-window.test.d.ts.map +1 -0
- package/dist/state/state.extract-window.test.js +163 -0
- package/dist/state/state.interfaces.d.ts +25 -8
- package/dist/state/state.interfaces.d.ts.map +1 -1
- package/dist/state/state.interfaces.js +4 -2
- package/dist/state/state.js +111 -12
- package/dist/state/state.pending-boundaries.test.d.ts +2 -0
- package/dist/state/state.pending-boundaries.test.d.ts.map +1 -0
- package/dist/state/state.pending-boundaries.test.js +189 -0
- package/dist/state/state.post-state.test.d.ts +2 -0
- package/dist/state/state.post-state.test.d.ts.map +1 -0
- package/dist/state/state.post-state.test.js +77 -0
- package/dist/state/state.test.js +117 -73
- package/dist/state/state.time-value-resolution.test.d.ts +2 -0
- package/dist/state/state.time-value-resolution.test.d.ts.map +1 -0
- package/dist/state/state.time-value-resolution.test.js +175 -0
- package/dist/tsdoc-metadata.json +11 -0
- package/dist/types/common.d.ts +0 -25
- package/dist/types/common.d.ts.map +1 -1
- package/dist/types/common.js +1 -11
- package/dist/types/external-domain-metadata.d.ts +338 -0
- package/dist/types/external-domain-metadata.d.ts.map +1 -0
- package/dist/types/extraction.d.ts +89 -218
- package/dist/types/extraction.d.ts.map +1 -1
- package/dist/types/extraction.js +56 -124
- package/dist/types/extraction.test.js +69 -43
- package/dist/types/index.d.ts +4 -2
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js +6 -3
- package/dist/types/loading.d.ts +4 -24
- package/dist/types/loading.d.ts.map +1 -1
- package/dist/types/loading.js +1 -22
- package/dist/types/workers.d.ts +31 -14
- package/dist/types/workers.d.ts.map +1 -1
- package/dist/types/workers.js +1 -0
- package/dist/uploader/uploader.d.ts +1 -0
- package/dist/uploader/uploader.d.ts.map +1 -1
- package/dist/uploader/uploader.helpers.test.js +0 -11
- package/dist/uploader/uploader.interfaces.d.ts +3 -3
- package/dist/uploader/uploader.js +53 -20
- package/dist/uploader/uploader.test.js +22 -10
- package/package.json +12 -11
- package/dist/common/event-type-translation.d.ts +0 -24
- package/dist/common/event-type-translation.d.ts.map +0 -1
- package/dist/common/event-type-translation.js +0 -117
- package/dist/deprecated/adapter/index.d.ts +0 -62
- package/dist/deprecated/adapter/index.d.ts.map +0 -1
- package/dist/deprecated/adapter/index.js +0 -151
- package/dist/deprecated/common/helpers.d.ts +0 -7
- package/dist/deprecated/common/helpers.d.ts.map +0 -1
- package/dist/deprecated/common/helpers.js +0 -47
- package/dist/deprecated/demo-extractor/external_domain_metadata.json +0 -38
- package/dist/deprecated/demo-extractor/index.d.ts +0 -18
- package/dist/deprecated/demo-extractor/index.d.ts.map +0 -1
- package/dist/deprecated/demo-extractor/index.js +0 -161
- package/dist/deprecated/http/client.d.ts +0 -22
- package/dist/deprecated/http/client.d.ts.map +0 -1
- package/dist/deprecated/http/client.js +0 -161
- package/dist/deprecated/uploader/index.d.ts +0 -35
- package/dist/deprecated/uploader/index.d.ts.map +0 -1
- package/dist/deprecated/uploader/index.js +0 -161
- package/dist/http/types.d.ts +0 -17
- package/dist/http/types.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts +0 -2
- package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts.map +0 -1
- package/dist/multithreading/worker-adapter/worker-adapter.test.js +0 -483
- /package/dist/{http/types.js → types/external-domain-metadata.js} +0 -0
|
@@ -1,161 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
-
var ownKeys = function(o) {
|
|
20
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
-
var ar = [];
|
|
22
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
-
return ar;
|
|
24
|
-
};
|
|
25
|
-
return ownKeys(o);
|
|
26
|
-
};
|
|
27
|
-
return function (mod) {
|
|
28
|
-
if (mod && mod.__esModule) return mod;
|
|
29
|
-
var result = {};
|
|
30
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
-
__setModuleDefault(result, mod);
|
|
32
|
-
return result;
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.Uploader = void 0;
|
|
37
|
-
const typescript_sdk_1 = require("@devrev/typescript-sdk");
|
|
38
|
-
const fs_1 = __importStar(require("fs"));
|
|
39
|
-
const axios_client_internal_1 = require("../../http/axios-client-internal");
|
|
40
|
-
const helpers_1 = require("../common/helpers");
|
|
41
|
-
/**
|
|
42
|
-
* Uploader class is used to upload files to the DevRev platform.
|
|
43
|
-
* The class provides utilities to
|
|
44
|
-
* - prepare artifact
|
|
45
|
-
* - upload artifact
|
|
46
|
-
* - return the artifact information to the platform
|
|
47
|
-
*
|
|
48
|
-
* @class Uploader
|
|
49
|
-
* @constructor
|
|
50
|
-
* @param {string} endpoint - The endpoint of the DevRev platform
|
|
51
|
-
* @param {string} token - The token to authenticate with the DevRev platform
|
|
52
|
-
* @param {boolean} local - Flag to indicate if the uploader should upload to the file-system.
|
|
53
|
-
*/
|
|
54
|
-
class Uploader {
|
|
55
|
-
constructor(endpoint, token, local = false) {
|
|
56
|
-
this.betaDevrevSdk = typescript_sdk_1.client.setupBeta({
|
|
57
|
-
endpoint,
|
|
58
|
-
token,
|
|
59
|
-
});
|
|
60
|
-
this.local = local;
|
|
61
|
-
}
|
|
62
|
-
/**
|
|
63
|
-
*
|
|
64
|
-
* Uploads the file to the DevRev platform. The file is uploaded to the platform
|
|
65
|
-
* and the artifact information is returned.
|
|
66
|
-
*
|
|
67
|
-
* @param {string} filename - The name of the file to be uploaded
|
|
68
|
-
* @param {string} entity - The entity type of the file to be uploaded
|
|
69
|
-
* @param {object[] | object} fetchedObjects - The objects to be uploaded
|
|
70
|
-
* @param filetype - The type of the file to be uploaded
|
|
71
|
-
* @returns {Promise<UploadResponse>} - The response object containing the artifact information
|
|
72
|
-
*/
|
|
73
|
-
async upload(filename, entity, fetchedObjects, filetype = 'application/jsonl+json') {
|
|
74
|
-
if (this.local) {
|
|
75
|
-
await this.downloadToLocal(filename, fetchedObjects);
|
|
76
|
-
}
|
|
77
|
-
const preparedArtifact = await this.prepareArtifact(filename, filetype);
|
|
78
|
-
if (!preparedArtifact) {
|
|
79
|
-
return {
|
|
80
|
-
artifact: undefined,
|
|
81
|
-
error: { message: 'Error while preparing artifact' },
|
|
82
|
-
};
|
|
83
|
-
}
|
|
84
|
-
const uploadedArtifact = await this.uploadToArtifact(preparedArtifact, fetchedObjects);
|
|
85
|
-
if (!uploadedArtifact) {
|
|
86
|
-
return {
|
|
87
|
-
artifact: undefined,
|
|
88
|
-
error: { message: 'Error while uploading artifact' },
|
|
89
|
-
};
|
|
90
|
-
}
|
|
91
|
-
// If file was successfully uploaded we want to post data about that file when emitting
|
|
92
|
-
const itemCount = Array.isArray(fetchedObjects) ? fetchedObjects.length : 1;
|
|
93
|
-
const artifact = {
|
|
94
|
-
id: preparedArtifact.id,
|
|
95
|
-
item_type: entity,
|
|
96
|
-
item_count: itemCount,
|
|
97
|
-
};
|
|
98
|
-
console.log(`Artifact uploaded successfully: ${artifact.id}`);
|
|
99
|
-
return { artifact, error: undefined };
|
|
100
|
-
}
|
|
101
|
-
async prepareArtifact(filename, filetype) {
|
|
102
|
-
try {
|
|
103
|
-
const response = await this.betaDevrevSdk.artifactsPrepare({
|
|
104
|
-
file_name: filename,
|
|
105
|
-
file_type: filetype,
|
|
106
|
-
});
|
|
107
|
-
return response.data;
|
|
108
|
-
}
|
|
109
|
-
catch (error) {
|
|
110
|
-
console.error('Error while preparing artifact: ' + error);
|
|
111
|
-
return null;
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
async uploadToArtifact(
|
|
115
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
116
|
-
preparedArtifact, fetchedObjects
|
|
117
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
118
|
-
) {
|
|
119
|
-
const formData = (0, helpers_1.createFormData)(preparedArtifact, fetchedObjects);
|
|
120
|
-
try {
|
|
121
|
-
const response = await axios_client_internal_1.axiosClient.post(preparedArtifact.url, formData, {
|
|
122
|
-
headers: {
|
|
123
|
-
'Content-Type': 'multipart/form-data',
|
|
124
|
-
},
|
|
125
|
-
});
|
|
126
|
-
return response;
|
|
127
|
-
}
|
|
128
|
-
catch (error) {
|
|
129
|
-
console.error('Error while uploading artifact: ' + error);
|
|
130
|
-
return null;
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
async downloadToLocal(filePath, fetchedObjects) {
|
|
134
|
-
console.log(`Uploading ${filePath} to local file system`);
|
|
135
|
-
try {
|
|
136
|
-
if (!fs_1.default.existsSync('extracted_files')) {
|
|
137
|
-
fs_1.default.mkdirSync('extracted_files');
|
|
138
|
-
}
|
|
139
|
-
const timestamp = new Date().getTime();
|
|
140
|
-
const fileHandle = await fs_1.promises.open(`extracted_files/${timestamp}_${filePath}`, 'w');
|
|
141
|
-
let objArray = [];
|
|
142
|
-
if (!Array.isArray(fetchedObjects)) {
|
|
143
|
-
objArray.push(fetchedObjects);
|
|
144
|
-
}
|
|
145
|
-
else {
|
|
146
|
-
objArray = fetchedObjects;
|
|
147
|
-
}
|
|
148
|
-
for (const jsonObject of objArray) {
|
|
149
|
-
const jsonLine = JSON.stringify(jsonObject) + '\n';
|
|
150
|
-
await fileHandle.write(jsonLine);
|
|
151
|
-
}
|
|
152
|
-
await fileHandle.close();
|
|
153
|
-
console.log('Data successfully written to', filePath);
|
|
154
|
-
}
|
|
155
|
-
catch (error) {
|
|
156
|
-
console.error('Error writing data to file:', error);
|
|
157
|
-
return Promise.reject(error);
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
exports.Uploader = Uploader;
|
package/dist/http/types.d.ts
DELETED
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* HTTP Response type
|
|
3
|
-
* @deprecated
|
|
4
|
-
*/
|
|
5
|
-
export type HTTPResponse = {
|
|
6
|
-
success: boolean;
|
|
7
|
-
message: string;
|
|
8
|
-
data: Data;
|
|
9
|
-
};
|
|
10
|
-
interface Data {
|
|
11
|
-
records: object[];
|
|
12
|
-
delay: number;
|
|
13
|
-
nextPage?: number;
|
|
14
|
-
metadata?: object;
|
|
15
|
-
}
|
|
16
|
-
export {};
|
|
17
|
-
//# sourceMappingURL=types.d.ts.map
|
package/dist/http/types.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/http/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,IAAI,CAAC;CACZ,CAAC;AAEF,UAAU,IAAI;IACZ,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"worker-adapter.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.test.ts"],"names":[],"mappings":""}
|
|
@@ -1,483 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
const attachments_streaming_pool_1 = require("../../attachments-streaming/attachments-streaming-pool");
|
|
4
|
-
const state_1 = require("../../state/state");
|
|
5
|
-
const test_helpers_1 = require("../../tests/test-helpers");
|
|
6
|
-
const types_1 = require("../../types");
|
|
7
|
-
const worker_adapter_1 = require("./worker-adapter");
|
|
8
|
-
/* eslint-disable @typescript-eslint/no-require-imports */
|
|
9
|
-
// Mock dependencies
|
|
10
|
-
jest.mock('../../common/control-protocol', () => ({
|
|
11
|
-
emit: jest.fn().mockResolvedValue({}),
|
|
12
|
-
}));
|
|
13
|
-
// const mockPostState = jest.spyOn(State.prototype, 'postState').mockResolvedValue(); // Mock to resolve void
|
|
14
|
-
// const mockFetchState = jest.spyOn(State.prototype, 'fetchState').mockResolvedValue({}); // Mock to resolve a default state
|
|
15
|
-
jest.mock('../../mappers/mappers');
|
|
16
|
-
jest.mock('../../uploader/uploader');
|
|
17
|
-
// jest.mock('../../state/state');
|
|
18
|
-
jest.mock('../../repo/repo');
|
|
19
|
-
jest.mock('node:worker_threads', () => ({
|
|
20
|
-
parentPort: {
|
|
21
|
-
postMessage: jest.fn(),
|
|
22
|
-
},
|
|
23
|
-
}));
|
|
24
|
-
jest.mock('../../attachments-streaming/attachments-streaming-pool', () => {
|
|
25
|
-
return {
|
|
26
|
-
AttachmentsStreamingPool: jest.fn().mockImplementation(() => {
|
|
27
|
-
return {
|
|
28
|
-
streamAll: jest.fn().mockResolvedValue(undefined),
|
|
29
|
-
};
|
|
30
|
-
}),
|
|
31
|
-
};
|
|
32
|
-
});
|
|
33
|
-
describe(worker_adapter_1.WorkerAdapter.name, () => {
|
|
34
|
-
let adapter;
|
|
35
|
-
let mockEvent;
|
|
36
|
-
let mockAdapterState;
|
|
37
|
-
beforeEach(() => {
|
|
38
|
-
// Reset all mocks
|
|
39
|
-
jest.clearAllMocks();
|
|
40
|
-
// Create mock objects
|
|
41
|
-
mockEvent = (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.StartExtractingData });
|
|
42
|
-
const initialState = {
|
|
43
|
-
attachments: { completed: false },
|
|
44
|
-
lastSyncStarted: '',
|
|
45
|
-
lastSuccessfulSyncStarted: '',
|
|
46
|
-
snapInVersionId: '',
|
|
47
|
-
toDevRev: {
|
|
48
|
-
attachmentsMetadata: {
|
|
49
|
-
artifactIds: [],
|
|
50
|
-
lastProcessed: 0,
|
|
51
|
-
lastProcessedAttachmentsIdsList: [],
|
|
52
|
-
},
|
|
53
|
-
},
|
|
54
|
-
};
|
|
55
|
-
mockAdapterState = new state_1.State({
|
|
56
|
-
event: mockEvent,
|
|
57
|
-
initialState: initialState,
|
|
58
|
-
});
|
|
59
|
-
// Create the adapter instance
|
|
60
|
-
adapter = new worker_adapter_1.WorkerAdapter({
|
|
61
|
-
event: mockEvent,
|
|
62
|
-
adapterState: mockAdapterState,
|
|
63
|
-
});
|
|
64
|
-
});
|
|
65
|
-
describe(worker_adapter_1.WorkerAdapter.prototype.streamAttachments.name, () => {
|
|
66
|
-
it('should process all artifact batches successfully', async () => {
|
|
67
|
-
const mockStream = jest.fn();
|
|
68
|
-
// Set up adapter state with artifact IDs
|
|
69
|
-
adapter.state.toDevRev = {
|
|
70
|
-
attachmentsMetadata: {
|
|
71
|
-
artifactIds: ['artifact1', 'artifact2'],
|
|
72
|
-
lastProcessed: 0,
|
|
73
|
-
lastProcessedAttachmentsIdsList: [],
|
|
74
|
-
},
|
|
75
|
-
};
|
|
76
|
-
// Mock getting attachments from each artifact
|
|
77
|
-
adapter['uploader'].getAttachmentsFromArtifactId = jest
|
|
78
|
-
.fn()
|
|
79
|
-
.mockResolvedValueOnce({
|
|
80
|
-
attachments: [
|
|
81
|
-
{
|
|
82
|
-
url: 'http://example.com/file1.pdf',
|
|
83
|
-
id: 'attachment1',
|
|
84
|
-
file_name: 'file1.pdf',
|
|
85
|
-
parent_id: 'parent1',
|
|
86
|
-
},
|
|
87
|
-
{
|
|
88
|
-
url: 'http://example.com/file2.pdf',
|
|
89
|
-
id: 'attachment2',
|
|
90
|
-
file_name: 'file2.pdf',
|
|
91
|
-
parent_id: 'parent2',
|
|
92
|
-
},
|
|
93
|
-
],
|
|
94
|
-
})
|
|
95
|
-
.mockResolvedValueOnce({
|
|
96
|
-
attachments: [
|
|
97
|
-
{
|
|
98
|
-
url: 'http://example.com/file3.pdf',
|
|
99
|
-
id: 'attachment3',
|
|
100
|
-
file_name: 'file3.pdf',
|
|
101
|
-
parent_id: 'parent3',
|
|
102
|
-
},
|
|
103
|
-
],
|
|
104
|
-
});
|
|
105
|
-
// Mock the initializeRepos method
|
|
106
|
-
adapter.initializeRepos = jest.fn();
|
|
107
|
-
const result = await adapter.streamAttachments({
|
|
108
|
-
stream: mockStream,
|
|
109
|
-
});
|
|
110
|
-
expect(adapter.initializeRepos).toHaveBeenCalledWith([
|
|
111
|
-
{ itemType: 'ssor_attachment' },
|
|
112
|
-
]);
|
|
113
|
-
expect(adapter.initializeRepos).toHaveBeenCalledTimes(1);
|
|
114
|
-
expect(adapter['uploader'].getAttachmentsFromArtifactId).toHaveBeenCalledTimes(2);
|
|
115
|
-
// Verify state was updated correctly
|
|
116
|
-
expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([]);
|
|
117
|
-
expect(adapter.state.toDevRev.attachmentsMetadata.lastProcessed).toBe(0);
|
|
118
|
-
expect(result).toBeUndefined();
|
|
119
|
-
});
|
|
120
|
-
it('[edge] should handle invalid batch size by using 1 instead', async () => {
|
|
121
|
-
const mockStream = jest.fn();
|
|
122
|
-
// Set up adapter state with artifact IDs
|
|
123
|
-
adapter.state.toDevRev = {
|
|
124
|
-
attachmentsMetadata: {
|
|
125
|
-
artifactIds: ['artifact1'],
|
|
126
|
-
lastProcessed: 0,
|
|
127
|
-
lastProcessedAttachmentsIdsList: [],
|
|
128
|
-
},
|
|
129
|
-
};
|
|
130
|
-
// Mock getting attachments
|
|
131
|
-
adapter['uploader'].getAttachmentsFromArtifactId = jest
|
|
132
|
-
.fn()
|
|
133
|
-
.mockResolvedValue({
|
|
134
|
-
attachments: [
|
|
135
|
-
{
|
|
136
|
-
url: 'http://example.com/file1.pdf',
|
|
137
|
-
id: 'attachment1',
|
|
138
|
-
file_name: 'file1.pdf',
|
|
139
|
-
parent_id: 'parent1',
|
|
140
|
-
},
|
|
141
|
-
],
|
|
142
|
-
});
|
|
143
|
-
adapter.initializeRepos = jest.fn();
|
|
144
|
-
const result = await adapter.streamAttachments({
|
|
145
|
-
stream: mockStream,
|
|
146
|
-
batchSize: 0,
|
|
147
|
-
});
|
|
148
|
-
expect(result).toBeUndefined();
|
|
149
|
-
});
|
|
150
|
-
it('[edge] should cap batch size to 50 when batchSize is greater than 50', async () => {
|
|
151
|
-
const mockStream = jest.fn();
|
|
152
|
-
// Set up adapter state with artifact IDs
|
|
153
|
-
adapter.state.toDevRev = {
|
|
154
|
-
attachmentsMetadata: {
|
|
155
|
-
artifactIds: ['artifact1'],
|
|
156
|
-
lastProcessed: 0,
|
|
157
|
-
lastProcessedAttachmentsIdsList: [],
|
|
158
|
-
},
|
|
159
|
-
};
|
|
160
|
-
// Mock getting attachments
|
|
161
|
-
adapter['uploader'].getAttachmentsFromArtifactId = jest
|
|
162
|
-
.fn()
|
|
163
|
-
.mockResolvedValue({
|
|
164
|
-
attachments: [
|
|
165
|
-
{
|
|
166
|
-
url: 'http://example.com/file1.pdf',
|
|
167
|
-
id: 'attachment1',
|
|
168
|
-
file_name: 'file1.pdf',
|
|
169
|
-
parent_id: 'parent1',
|
|
170
|
-
},
|
|
171
|
-
],
|
|
172
|
-
});
|
|
173
|
-
// Mock the required methods
|
|
174
|
-
adapter.initializeRepos = jest.fn();
|
|
175
|
-
const result = await adapter.streamAttachments({
|
|
176
|
-
stream: mockStream,
|
|
177
|
-
batchSize: 100, // Set batch size greater than 50
|
|
178
|
-
});
|
|
179
|
-
expect(result).toBeUndefined();
|
|
180
|
-
});
|
|
181
|
-
it('[edge] should handle empty attachments metadata artifact IDs', async () => {
|
|
182
|
-
const mockStream = jest.fn();
|
|
183
|
-
// Set up adapter state with no artifact IDs
|
|
184
|
-
adapter.state.toDevRev = {
|
|
185
|
-
attachmentsMetadata: {
|
|
186
|
-
artifactIds: [],
|
|
187
|
-
lastProcessed: 0,
|
|
188
|
-
},
|
|
189
|
-
};
|
|
190
|
-
const result = await adapter.streamAttachments({
|
|
191
|
-
stream: mockStream,
|
|
192
|
-
});
|
|
193
|
-
expect(result).toBeUndefined();
|
|
194
|
-
});
|
|
195
|
-
it('[edge] should handle errors when getting attachments', async () => {
|
|
196
|
-
const mockStream = jest.fn();
|
|
197
|
-
// Set up adapter state with artifact IDs
|
|
198
|
-
adapter.state.toDevRev = {
|
|
199
|
-
attachmentsMetadata: {
|
|
200
|
-
artifactIds: ['artifact1'],
|
|
201
|
-
lastProcessed: 0,
|
|
202
|
-
lastProcessedAttachmentsIdsList: [],
|
|
203
|
-
},
|
|
204
|
-
};
|
|
205
|
-
// Mock error when getting attachments
|
|
206
|
-
const mockError = new Error('Failed to get attachments');
|
|
207
|
-
adapter['uploader'].getAttachmentsFromArtifactId = jest
|
|
208
|
-
.fn()
|
|
209
|
-
.mockResolvedValue({
|
|
210
|
-
error: mockError,
|
|
211
|
-
});
|
|
212
|
-
// Mock methods
|
|
213
|
-
adapter.initializeRepos = jest.fn();
|
|
214
|
-
const result = await adapter.streamAttachments({
|
|
215
|
-
stream: mockStream,
|
|
216
|
-
});
|
|
217
|
-
expect(result).toEqual({
|
|
218
|
-
error: mockError,
|
|
219
|
-
});
|
|
220
|
-
});
|
|
221
|
-
it('[edge] should handle empty attachments array from artifact', async () => {
|
|
222
|
-
const mockStream = jest.fn();
|
|
223
|
-
// Set up adapter state with artifact IDs
|
|
224
|
-
adapter.state.toDevRev = {
|
|
225
|
-
attachmentsMetadata: {
|
|
226
|
-
artifactIds: ['artifact1'],
|
|
227
|
-
lastProcessed: 0,
|
|
228
|
-
lastProcessedAttachmentsIdsList: [],
|
|
229
|
-
},
|
|
230
|
-
};
|
|
231
|
-
// Mock getting empty attachments
|
|
232
|
-
adapter['uploader'].getAttachmentsFromArtifactId = jest
|
|
233
|
-
.fn()
|
|
234
|
-
.mockResolvedValue({
|
|
235
|
-
attachments: [],
|
|
236
|
-
});
|
|
237
|
-
// Mock methods
|
|
238
|
-
adapter.initializeRepos = jest.fn();
|
|
239
|
-
const result = await adapter.streamAttachments({
|
|
240
|
-
stream: mockStream,
|
|
241
|
-
});
|
|
242
|
-
expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([]);
|
|
243
|
-
expect(result).toBeUndefined();
|
|
244
|
-
});
|
|
245
|
-
it('should use custom processors when provided', async () => {
|
|
246
|
-
const mockStream = jest.fn();
|
|
247
|
-
const mockReducer = jest.fn().mockReturnValue(['custom-reduced']);
|
|
248
|
-
const mockIterator = jest.fn().mockResolvedValue({});
|
|
249
|
-
// Set up adapter state with artifact IDs
|
|
250
|
-
adapter.state.toDevRev = {
|
|
251
|
-
attachmentsMetadata: {
|
|
252
|
-
artifactIds: ['artifact1'],
|
|
253
|
-
lastProcessed: 0,
|
|
254
|
-
lastProcessedAttachmentsIdsList: [],
|
|
255
|
-
},
|
|
256
|
-
};
|
|
257
|
-
// Mock getting attachments
|
|
258
|
-
adapter['uploader'].getAttachmentsFromArtifactId = jest
|
|
259
|
-
.fn()
|
|
260
|
-
.mockResolvedValue({
|
|
261
|
-
attachments: [{ id: 'attachment1' }],
|
|
262
|
-
});
|
|
263
|
-
// Mock methods
|
|
264
|
-
adapter.initializeRepos = jest.fn();
|
|
265
|
-
const result = await adapter.streamAttachments({
|
|
266
|
-
stream: mockStream,
|
|
267
|
-
processors: {
|
|
268
|
-
reducer: mockReducer,
|
|
269
|
-
iterator: mockIterator,
|
|
270
|
-
},
|
|
271
|
-
});
|
|
272
|
-
expect(mockReducer).toHaveBeenCalledWith({
|
|
273
|
-
attachments: [{ id: 'attachment1' }],
|
|
274
|
-
adapter: adapter,
|
|
275
|
-
batchSize: 1,
|
|
276
|
-
});
|
|
277
|
-
expect(mockIterator).toHaveBeenCalledWith({
|
|
278
|
-
reducedAttachments: ['custom-reduced'],
|
|
279
|
-
adapter: adapter,
|
|
280
|
-
stream: mockStream,
|
|
281
|
-
});
|
|
282
|
-
expect(result).toBeUndefined();
|
|
283
|
-
});
|
|
284
|
-
it('should handle rate limiting from iterator', async () => {
|
|
285
|
-
const mockStream = jest.fn();
|
|
286
|
-
attachments_streaming_pool_1.AttachmentsStreamingPool.mockImplementationOnce(() => {
|
|
287
|
-
return {
|
|
288
|
-
// Return an object with a `streamAll` method that resolves to your desired value.
|
|
289
|
-
streamAll: jest.fn().mockResolvedValue({ delay: 30 }),
|
|
290
|
-
};
|
|
291
|
-
});
|
|
292
|
-
// Set up adapter state with artifact IDs
|
|
293
|
-
adapter.state.toDevRev = {
|
|
294
|
-
attachmentsMetadata: {
|
|
295
|
-
artifactIds: ['artifact1'],
|
|
296
|
-
lastProcessed: 0,
|
|
297
|
-
lastProcessedAttachmentsIdsList: [],
|
|
298
|
-
},
|
|
299
|
-
};
|
|
300
|
-
// Mock getting attachments
|
|
301
|
-
adapter['uploader'].getAttachmentsFromArtifactId = jest
|
|
302
|
-
.fn()
|
|
303
|
-
.mockResolvedValue({
|
|
304
|
-
attachments: [{ id: 'attachment1' }],
|
|
305
|
-
});
|
|
306
|
-
// Mock methods
|
|
307
|
-
adapter.initializeRepos = jest.fn();
|
|
308
|
-
const result = await adapter.streamAttachments({
|
|
309
|
-
stream: mockStream,
|
|
310
|
-
});
|
|
311
|
-
expect(result).toEqual({
|
|
312
|
-
delay: 30,
|
|
313
|
-
});
|
|
314
|
-
// The artifactIds array should remain unchanged
|
|
315
|
-
expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([
|
|
316
|
-
'artifact1',
|
|
317
|
-
]);
|
|
318
|
-
});
|
|
319
|
-
it('should handle error from iterator', async () => {
|
|
320
|
-
const mockStream = jest.fn();
|
|
321
|
-
attachments_streaming_pool_1.AttachmentsStreamingPool.mockImplementationOnce(() => {
|
|
322
|
-
return {
|
|
323
|
-
// Return an object with a `streamAll` method that resolves to your desired value.
|
|
324
|
-
streamAll: jest.fn().mockResolvedValue({
|
|
325
|
-
error: 'Mock error',
|
|
326
|
-
}),
|
|
327
|
-
};
|
|
328
|
-
});
|
|
329
|
-
// Set up adapter state with artifact IDs
|
|
330
|
-
adapter.state.toDevRev = {
|
|
331
|
-
attachmentsMetadata: {
|
|
332
|
-
artifactIds: ['artifact1'],
|
|
333
|
-
lastProcessed: 0,
|
|
334
|
-
lastProcessedAttachmentsIdsList: [],
|
|
335
|
-
},
|
|
336
|
-
};
|
|
337
|
-
// Mock getting attachments
|
|
338
|
-
adapter['uploader'].getAttachmentsFromArtifactId = jest
|
|
339
|
-
.fn()
|
|
340
|
-
.mockResolvedValue({
|
|
341
|
-
attachments: [{ id: 'attachment1' }],
|
|
342
|
-
});
|
|
343
|
-
// Mock methods
|
|
344
|
-
adapter.initializeRepos = jest.fn();
|
|
345
|
-
const result = await adapter.streamAttachments({
|
|
346
|
-
stream: mockStream,
|
|
347
|
-
});
|
|
348
|
-
expect(result).toEqual({
|
|
349
|
-
error: 'Mock error',
|
|
350
|
-
});
|
|
351
|
-
// The artifactIds array should remain unchanged
|
|
352
|
-
expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([
|
|
353
|
-
'artifact1',
|
|
354
|
-
]);
|
|
355
|
-
});
|
|
356
|
-
it('should reset lastProcessed and attachment IDs list after processing all artifacts', async () => {
|
|
357
|
-
const mockStream = jest.fn();
|
|
358
|
-
adapter.state.toDevRev = {
|
|
359
|
-
attachmentsMetadata: {
|
|
360
|
-
artifactIds: ['artifact1'],
|
|
361
|
-
lastProcessed: 0,
|
|
362
|
-
lastProcessedAttachmentsIdsList: [],
|
|
363
|
-
},
|
|
364
|
-
};
|
|
365
|
-
adapter['uploader'].getAttachmentsFromArtifactId = jest
|
|
366
|
-
.fn()
|
|
367
|
-
.mockResolvedValueOnce({
|
|
368
|
-
attachments: [
|
|
369
|
-
{
|
|
370
|
-
url: 'http://example.com/file1.pdf',
|
|
371
|
-
id: 'attachment1',
|
|
372
|
-
file_name: 'file1.pdf',
|
|
373
|
-
parent_id: 'parent1',
|
|
374
|
-
},
|
|
375
|
-
{
|
|
376
|
-
url: 'http://example.com/file2.pdf',
|
|
377
|
-
id: 'attachment2',
|
|
378
|
-
file_name: 'file2.pdf',
|
|
379
|
-
parent_id: 'parent2',
|
|
380
|
-
},
|
|
381
|
-
{
|
|
382
|
-
url: 'http://example.com/file3.pdf',
|
|
383
|
-
id: 'attachment3',
|
|
384
|
-
file_name: 'file3.pdf',
|
|
385
|
-
parent_id: 'parent3',
|
|
386
|
-
},
|
|
387
|
-
],
|
|
388
|
-
});
|
|
389
|
-
adapter.processAttachment = jest.fn().mockResolvedValue(null);
|
|
390
|
-
await adapter.streamAttachments({
|
|
391
|
-
stream: mockStream,
|
|
392
|
-
});
|
|
393
|
-
expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toHaveLength(0);
|
|
394
|
-
expect(adapter.state.toDevRev.attachmentsMetadata.lastProcessed).toBe(0);
|
|
395
|
-
});
|
|
396
|
-
});
|
|
397
|
-
describe(worker_adapter_1.WorkerAdapter.prototype.emit.name, () => {
|
|
398
|
-
let counter;
|
|
399
|
-
let mockPostMessage;
|
|
400
|
-
beforeEach(() => {
|
|
401
|
-
counter = { counter: 0 };
|
|
402
|
-
// Import the worker_threads module and spy on parentPort.postMessage
|
|
403
|
-
const workerThreads = require('node:worker_threads');
|
|
404
|
-
mockPostMessage = jest.fn().mockImplementation(() => {
|
|
405
|
-
counter.counter += 1;
|
|
406
|
-
});
|
|
407
|
-
// Spy on the parentPort.postMessage method
|
|
408
|
-
if (workerThreads.parentPort) {
|
|
409
|
-
jest
|
|
410
|
-
.spyOn(workerThreads.parentPort, 'postMessage')
|
|
411
|
-
.mockImplementation(mockPostMessage);
|
|
412
|
-
}
|
|
413
|
-
else {
|
|
414
|
-
// If parentPort is null (not in worker context), create a mock
|
|
415
|
-
workerThreads.parentPort = {
|
|
416
|
-
postMessage: mockPostMessage,
|
|
417
|
-
};
|
|
418
|
-
}
|
|
419
|
-
});
|
|
420
|
-
afterEach(() => {
|
|
421
|
-
// Restore all mocks
|
|
422
|
-
jest.restoreAllMocks();
|
|
423
|
-
});
|
|
424
|
-
it('should emit only one event when multiple events of same type are sent', async () => {
|
|
425
|
-
adapter['adapterState'].postState = jest
|
|
426
|
-
.fn()
|
|
427
|
-
.mockResolvedValue(undefined);
|
|
428
|
-
adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
|
|
429
|
-
await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
|
|
430
|
-
reports: [],
|
|
431
|
-
processed_files: [],
|
|
432
|
-
});
|
|
433
|
-
await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
|
|
434
|
-
reports: [],
|
|
435
|
-
processed_files: [],
|
|
436
|
-
});
|
|
437
|
-
expect(counter.counter).toBe(1);
|
|
438
|
-
});
|
|
439
|
-
it('should emit event when different event type is sent after previous events', async () => {
|
|
440
|
-
adapter['adapterState'].postState = jest
|
|
441
|
-
.fn()
|
|
442
|
-
.mockResolvedValue(undefined);
|
|
443
|
-
adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
|
|
444
|
-
await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
|
|
445
|
-
reports: [],
|
|
446
|
-
processed_files: [],
|
|
447
|
-
});
|
|
448
|
-
await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
|
|
449
|
-
reports: [],
|
|
450
|
-
processed_files: [],
|
|
451
|
-
});
|
|
452
|
-
await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
|
|
453
|
-
reports: [],
|
|
454
|
-
processed_files: [],
|
|
455
|
-
});
|
|
456
|
-
expect(counter.counter).toBe(1);
|
|
457
|
-
});
|
|
458
|
-
it('should correctly emit one event even if postState errors', async () => {
|
|
459
|
-
adapter['adapterState'].postState = jest
|
|
460
|
-
.fn()
|
|
461
|
-
.mockRejectedValue(new Error('postState error'));
|
|
462
|
-
adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
|
|
463
|
-
await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
|
|
464
|
-
reports: [],
|
|
465
|
-
processed_files: [],
|
|
466
|
-
});
|
|
467
|
-
expect(counter.counter).toBe(1);
|
|
468
|
-
});
|
|
469
|
-
it('should correctly emit one event even if uploadAllRepos errors', async () => {
|
|
470
|
-
adapter['adapterState'].postState = jest
|
|
471
|
-
.fn()
|
|
472
|
-
.mockResolvedValue(undefined);
|
|
473
|
-
adapter.uploadAllRepos = jest
|
|
474
|
-
.fn()
|
|
475
|
-
.mockRejectedValue(new Error('uploadAllRepos error'));
|
|
476
|
-
await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
|
|
477
|
-
reports: [],
|
|
478
|
-
processed_files: [],
|
|
479
|
-
});
|
|
480
|
-
expect(counter.counter).toBe(1);
|
|
481
|
-
});
|
|
482
|
-
});
|
|
483
|
-
});
|
|
File without changes
|