@devrev/airsync-sdk 2.0.0-beta.1 → 2.0.0-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. package/README.md +0 -4
  2. package/dist/airsync-sdk.d.ts +1802 -0
  3. package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
  4. package/dist/attachments-streaming/attachments-streaming-pool.js +13 -2
  5. package/dist/attachments-streaming/attachments-streaming-pool.test.js +201 -6
  6. package/dist/common/constants.d.ts +16 -5
  7. package/dist/common/constants.d.ts.map +1 -1
  8. package/dist/common/constants.js +20 -6
  9. package/dist/common/control-protocol.d.ts +2 -2
  10. package/dist/common/control-protocol.d.ts.map +1 -1
  11. package/dist/common/control-protocol.js +1 -5
  12. package/dist/common/helpers.d.ts +8 -0
  13. package/dist/common/helpers.d.ts.map +1 -1
  14. package/dist/common/helpers.js +15 -0
  15. package/dist/common/install-initial-domain-mapping.d.ts +2 -2
  16. package/dist/common/install-initial-domain-mapping.test.js +5 -2
  17. package/dist/common/test-utils.d.ts +19 -0
  18. package/dist/common/test-utils.d.ts.map +1 -0
  19. package/dist/common/test-utils.js +101 -0
  20. package/dist/common/time-value-resolver.d.ts +47 -0
  21. package/dist/common/time-value-resolver.d.ts.map +1 -0
  22. package/dist/common/time-value-resolver.js +141 -0
  23. package/dist/common/time-value-resolver.test.d.ts +2 -0
  24. package/dist/common/time-value-resolver.test.d.ts.map +1 -0
  25. package/dist/common/time-value-resolver.test.js +268 -0
  26. package/dist/http/axios-client-internal.d.ts.map +1 -1
  27. package/dist/http/axios-client-internal.js +7 -0
  28. package/dist/http/axios-client-internal.test.js +21 -0
  29. package/dist/http/axios-client.d.ts.map +1 -1
  30. package/dist/http/axios-client.js +6 -3
  31. package/dist/http/index.d.ts +0 -1
  32. package/dist/http/index.d.ts.map +1 -1
  33. package/dist/http/index.js +0 -1
  34. package/dist/index.d.ts +9 -8
  35. package/dist/index.d.ts.map +1 -1
  36. package/dist/index.js +12 -10
  37. package/dist/logger/logger.d.ts +1 -17
  38. package/dist/logger/logger.d.ts.map +1 -1
  39. package/dist/logger/logger.interfaces.d.ts +2 -2
  40. package/dist/logger/logger.js +35 -27
  41. package/dist/logger/logger.test.js +66 -19
  42. package/dist/mappers/mappers.interface.d.ts +5 -5
  43. package/dist/mappers/mappers.interface.js +3 -3
  44. package/dist/mappers/mappers.test.js +8 -10
  45. package/dist/mock-server/mock-server.d.ts +47 -0
  46. package/dist/mock-server/mock-server.d.ts.map +1 -0
  47. package/dist/mock-server/mock-server.interfaces.d.ts +73 -0
  48. package/dist/mock-server/mock-server.interfaces.d.ts.map +1 -0
  49. package/dist/mock-server/mock-server.interfaces.js +4 -0
  50. package/dist/mock-server/mock-server.js +278 -0
  51. package/dist/multithreading/create-worker.test.js +36 -17
  52. package/dist/multithreading/process-task.d.ts.map +1 -1
  53. package/dist/multithreading/process-task.js +15 -48
  54. package/dist/multithreading/process-task.test.d.ts +2 -0
  55. package/dist/multithreading/process-task.test.d.ts.map +1 -0
  56. package/dist/multithreading/process-task.test.js +146 -0
  57. package/dist/multithreading/spawn/spawn.d.ts +3 -3
  58. package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
  59. package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
  60. package/dist/multithreading/spawn/spawn.helpers.js +6 -14
  61. package/dist/multithreading/spawn/spawn.helpers.test.js +8 -8
  62. package/dist/multithreading/spawn/spawn.js +20 -25
  63. package/dist/multithreading/spawn/spawn.test.d.ts +2 -0
  64. package/dist/multithreading/spawn/spawn.test.d.ts.map +1 -0
  65. package/dist/multithreading/spawn/spawn.test.js +232 -0
  66. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.js +49 -1
  67. package/dist/multithreading/worker-adapter/worker-adapter.d.ts +11 -5
  68. package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +1 -1
  69. package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts +2 -0
  70. package/dist/multithreading/worker-adapter/worker-adapter.emit.test.d.ts.map +1 -0
  71. package/dist/multithreading/worker-adapter/worker-adapter.emit.test.js +390 -0
  72. package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts +2 -0
  73. package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.d.ts.map +1 -0
  74. package/dist/multithreading/worker-adapter/worker-adapter.extraction.test.js +799 -0
  75. package/dist/multithreading/worker-adapter/worker-adapter.js +232 -131
  76. package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts +2 -0
  77. package/dist/multithreading/worker-adapter/worker-adapter.loading.test.d.ts.map +1 -0
  78. package/dist/multithreading/worker-adapter/worker-adapter.loading.test.js +596 -0
  79. package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts +2 -0
  80. package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.d.ts.map +1 -0
  81. package/dist/multithreading/worker-adapter/worker-adapter.serialization.test.js +71 -0
  82. package/dist/repo/repo.d.ts.map +1 -1
  83. package/dist/repo/repo.interfaces.d.ts +4 -2
  84. package/dist/repo/repo.interfaces.d.ts.map +1 -1
  85. package/dist/repo/repo.js +4 -3
  86. package/dist/repo/repo.test.js +60 -7
  87. package/dist/state/state.d.ts +7 -1
  88. package/dist/state/state.d.ts.map +1 -1
  89. package/dist/state/state.extract-window.test.d.ts +2 -0
  90. package/dist/state/state.extract-window.test.d.ts.map +1 -0
  91. package/dist/state/state.extract-window.test.js +163 -0
  92. package/dist/state/state.interfaces.d.ts +17 -7
  93. package/dist/state/state.interfaces.d.ts.map +1 -1
  94. package/dist/state/state.interfaces.js +4 -2
  95. package/dist/state/state.js +111 -12
  96. package/dist/state/state.pending-boundaries.test.d.ts +2 -0
  97. package/dist/state/state.pending-boundaries.test.d.ts.map +1 -0
  98. package/dist/state/state.pending-boundaries.test.js +189 -0
  99. package/dist/state/state.post-state.test.d.ts +2 -0
  100. package/dist/state/state.post-state.test.d.ts.map +1 -0
  101. package/dist/state/state.post-state.test.js +77 -0
  102. package/dist/state/state.test.js +117 -73
  103. package/dist/state/state.time-value-resolution.test.d.ts +2 -0
  104. package/dist/state/state.time-value-resolution.test.d.ts.map +1 -0
  105. package/dist/state/state.time-value-resolution.test.js +175 -0
  106. package/dist/tsdoc-metadata.json +11 -0
  107. package/dist/types/common.d.ts +0 -25
  108. package/dist/types/common.d.ts.map +1 -1
  109. package/dist/types/common.js +1 -11
  110. package/dist/types/external-domain-metadata.d.ts +20 -1
  111. package/dist/types/external-domain-metadata.d.ts.map +1 -1
  112. package/dist/types/extraction.d.ts +89 -218
  113. package/dist/types/extraction.d.ts.map +1 -1
  114. package/dist/types/extraction.js +56 -124
  115. package/dist/types/extraction.test.js +69 -43
  116. package/dist/types/index.d.ts +3 -2
  117. package/dist/types/index.d.ts.map +1 -1
  118. package/dist/types/index.js +6 -3
  119. package/dist/types/loading.d.ts +4 -24
  120. package/dist/types/loading.d.ts.map +1 -1
  121. package/dist/types/loading.js +1 -22
  122. package/dist/types/workers.d.ts +31 -14
  123. package/dist/types/workers.d.ts.map +1 -1
  124. package/dist/types/workers.js +1 -0
  125. package/dist/uploader/uploader.d.ts +1 -0
  126. package/dist/uploader/uploader.d.ts.map +1 -1
  127. package/dist/uploader/uploader.helpers.test.js +0 -11
  128. package/dist/uploader/uploader.interfaces.d.ts +3 -3
  129. package/dist/uploader/uploader.js +53 -20
  130. package/dist/uploader/uploader.test.js +22 -10
  131. package/package.json +10 -13
  132. package/dist/common/event-type-translation.d.ts +0 -24
  133. package/dist/common/event-type-translation.d.ts.map +0 -1
  134. package/dist/common/event-type-translation.js +0 -117
  135. package/dist/deprecated/adapter/index.d.ts +0 -62
  136. package/dist/deprecated/adapter/index.d.ts.map +0 -1
  137. package/dist/deprecated/adapter/index.js +0 -151
  138. package/dist/deprecated/common/helpers.d.ts +0 -7
  139. package/dist/deprecated/common/helpers.d.ts.map +0 -1
  140. package/dist/deprecated/common/helpers.js +0 -47
  141. package/dist/deprecated/demo-extractor/external_domain_metadata.json +0 -38
  142. package/dist/deprecated/demo-extractor/index.d.ts +0 -18
  143. package/dist/deprecated/demo-extractor/index.d.ts.map +0 -1
  144. package/dist/deprecated/demo-extractor/index.js +0 -161
  145. package/dist/deprecated/http/client.d.ts +0 -22
  146. package/dist/deprecated/http/client.d.ts.map +0 -1
  147. package/dist/deprecated/http/client.js +0 -161
  148. package/dist/deprecated/uploader/index.d.ts +0 -35
  149. package/dist/deprecated/uploader/index.d.ts.map +0 -1
  150. package/dist/deprecated/uploader/index.js +0 -161
  151. package/dist/http/types.d.ts +0 -17
  152. package/dist/http/types.d.ts.map +0 -1
  153. package/dist/http/types.js +0 -2
  154. package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts +0 -2
  155. package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts.map +0 -1
  156. package/dist/multithreading/worker-adapter/worker-adapter.test.js +0 -483
@@ -1,161 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || (function () {
19
- var ownKeys = function(o) {
20
- ownKeys = Object.getOwnPropertyNames || function (o) {
21
- var ar = [];
22
- for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
- return ar;
24
- };
25
- return ownKeys(o);
26
- };
27
- return function (mod) {
28
- if (mod && mod.__esModule) return mod;
29
- var result = {};
30
- if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
- __setModuleDefault(result, mod);
32
- return result;
33
- };
34
- })();
35
- Object.defineProperty(exports, "__esModule", { value: true });
36
- exports.Uploader = void 0;
37
- const typescript_sdk_1 = require("@devrev/typescript-sdk");
38
- const fs_1 = __importStar(require("fs"));
39
- const axios_client_internal_1 = require("../../http/axios-client-internal");
40
- const helpers_1 = require("../common/helpers");
41
- /**
42
- * Uploader class is used to upload files to the DevRev platform.
43
- * The class provides utilities to
44
- * - prepare artifact
45
- * - upload artifact
46
- * - return the artifact information to the platform
47
- *
48
- * @class Uploader
49
- * @constructor
50
- * @param {string} endpoint - The endpoint of the DevRev platform
51
- * @param {string} token - The token to authenticate with the DevRev platform
52
- * @param {boolean} local - Flag to indicate if the uploader should upload to the file-system.
53
- */
54
- class Uploader {
55
- constructor(endpoint, token, local = false) {
56
- this.betaDevrevSdk = typescript_sdk_1.client.setupBeta({
57
- endpoint,
58
- token,
59
- });
60
- this.local = local;
61
- }
62
- /**
63
- *
64
- * Uploads the file to the DevRev platform. The file is uploaded to the platform
65
- * and the artifact information is returned.
66
- *
67
- * @param {string} filename - The name of the file to be uploaded
68
- * @param {string} entity - The entity type of the file to be uploaded
69
- * @param {object[] | object} fetchedObjects - The objects to be uploaded
70
- * @param filetype - The type of the file to be uploaded
71
- * @returns {Promise<UploadResponse>} - The response object containing the artifact information
72
- */
73
- async upload(filename, entity, fetchedObjects, filetype = 'application/jsonl+json') {
74
- if (this.local) {
75
- await this.downloadToLocal(filename, fetchedObjects);
76
- }
77
- const preparedArtifact = await this.prepareArtifact(filename, filetype);
78
- if (!preparedArtifact) {
79
- return {
80
- artifact: undefined,
81
- error: { message: 'Error while preparing artifact' },
82
- };
83
- }
84
- const uploadedArtifact = await this.uploadToArtifact(preparedArtifact, fetchedObjects);
85
- if (!uploadedArtifact) {
86
- return {
87
- artifact: undefined,
88
- error: { message: 'Error while uploading artifact' },
89
- };
90
- }
91
- // If file was successfully uploaded we want to post data about that file when emitting
92
- const itemCount = Array.isArray(fetchedObjects) ? fetchedObjects.length : 1;
93
- const artifact = {
94
- id: preparedArtifact.id,
95
- item_type: entity,
96
- item_count: itemCount,
97
- };
98
- console.log(`Artifact uploaded successfully: ${artifact.id}`);
99
- return { artifact, error: undefined };
100
- }
101
- async prepareArtifact(filename, filetype) {
102
- try {
103
- const response = await this.betaDevrevSdk.artifactsPrepare({
104
- file_name: filename,
105
- file_type: filetype,
106
- });
107
- return response.data;
108
- }
109
- catch (error) {
110
- console.error('Error while preparing artifact: ' + error);
111
- return null;
112
- }
113
- }
114
- async uploadToArtifact(
115
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
116
- preparedArtifact, fetchedObjects
117
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
118
- ) {
119
- const formData = (0, helpers_1.createFormData)(preparedArtifact, fetchedObjects);
120
- try {
121
- const response = await axios_client_internal_1.axiosClient.post(preparedArtifact.url, formData, {
122
- headers: {
123
- 'Content-Type': 'multipart/form-data',
124
- },
125
- });
126
- return response;
127
- }
128
- catch (error) {
129
- console.error('Error while uploading artifact: ' + error);
130
- return null;
131
- }
132
- }
133
- async downloadToLocal(filePath, fetchedObjects) {
134
- console.log(`Uploading ${filePath} to local file system`);
135
- try {
136
- if (!fs_1.default.existsSync('extracted_files')) {
137
- fs_1.default.mkdirSync('extracted_files');
138
- }
139
- const timestamp = new Date().getTime();
140
- const fileHandle = await fs_1.promises.open(`extracted_files/${timestamp}_${filePath}`, 'w');
141
- let objArray = [];
142
- if (!Array.isArray(fetchedObjects)) {
143
- objArray.push(fetchedObjects);
144
- }
145
- else {
146
- objArray = fetchedObjects;
147
- }
148
- for (const jsonObject of objArray) {
149
- const jsonLine = JSON.stringify(jsonObject) + '\n';
150
- await fileHandle.write(jsonLine);
151
- }
152
- await fileHandle.close();
153
- console.log('Data successfully written to', filePath);
154
- }
155
- catch (error) {
156
- console.error('Error writing data to file:', error);
157
- return Promise.reject(error);
158
- }
159
- }
160
- }
161
- exports.Uploader = Uploader;
@@ -1,17 +0,0 @@
1
- /**
2
- * HTTP Response type
3
- * @deprecated
4
- */
5
- export type HTTPResponse = {
6
- success: boolean;
7
- message: string;
8
- data: Data;
9
- };
10
- interface Data {
11
- records: object[];
12
- delay: number;
13
- nextPage?: number;
14
- metadata?: object;
15
- }
16
- export {};
17
- //# sourceMappingURL=types.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/http/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,IAAI,CAAC;CACZ,CAAC;AAEF,UAAU,IAAI;IACZ,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB"}
@@ -1,2 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=worker-adapter.test.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"worker-adapter.test.d.ts","sourceRoot":"","sources":["../../../src/multithreading/worker-adapter/worker-adapter.test.ts"],"names":[],"mappings":""}
@@ -1,483 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- const attachments_streaming_pool_1 = require("../../attachments-streaming/attachments-streaming-pool");
4
- const state_1 = require("../../state/state");
5
- const test_helpers_1 = require("../../tests/test-helpers");
6
- const types_1 = require("../../types");
7
- const worker_adapter_1 = require("./worker-adapter");
8
- /* eslint-disable @typescript-eslint/no-require-imports */
9
- // Mock dependencies
10
- jest.mock('../../common/control-protocol', () => ({
11
- emit: jest.fn().mockResolvedValue({}),
12
- }));
13
- // const mockPostState = jest.spyOn(State.prototype, 'postState').mockResolvedValue(); // Mock to resolve void
14
- // const mockFetchState = jest.spyOn(State.prototype, 'fetchState').mockResolvedValue({}); // Mock to resolve a default state
15
- jest.mock('../../mappers/mappers');
16
- jest.mock('../../uploader/uploader');
17
- // jest.mock('../../state/state');
18
- jest.mock('../../repo/repo');
19
- jest.mock('node:worker_threads', () => ({
20
- parentPort: {
21
- postMessage: jest.fn(),
22
- },
23
- }));
24
- jest.mock('../../attachments-streaming/attachments-streaming-pool', () => {
25
- return {
26
- AttachmentsStreamingPool: jest.fn().mockImplementation(() => {
27
- return {
28
- streamAll: jest.fn().mockResolvedValue(undefined),
29
- };
30
- }),
31
- };
32
- });
33
- describe(worker_adapter_1.WorkerAdapter.name, () => {
34
- let adapter;
35
- let mockEvent;
36
- let mockAdapterState;
37
- beforeEach(() => {
38
- // Reset all mocks
39
- jest.clearAllMocks();
40
- // Create mock objects
41
- mockEvent = (0, test_helpers_1.createEvent)({ eventType: types_1.EventType.StartExtractingData });
42
- const initialState = {
43
- attachments: { completed: false },
44
- lastSyncStarted: '',
45
- lastSuccessfulSyncStarted: '',
46
- snapInVersionId: '',
47
- toDevRev: {
48
- attachmentsMetadata: {
49
- artifactIds: [],
50
- lastProcessed: 0,
51
- lastProcessedAttachmentsIdsList: [],
52
- },
53
- },
54
- };
55
- mockAdapterState = new state_1.State({
56
- event: mockEvent,
57
- initialState: initialState,
58
- });
59
- // Create the adapter instance
60
- adapter = new worker_adapter_1.WorkerAdapter({
61
- event: mockEvent,
62
- adapterState: mockAdapterState,
63
- });
64
- });
65
- describe(worker_adapter_1.WorkerAdapter.prototype.streamAttachments.name, () => {
66
- it('should process all artifact batches successfully', async () => {
67
- const mockStream = jest.fn();
68
- // Set up adapter state with artifact IDs
69
- adapter.state.toDevRev = {
70
- attachmentsMetadata: {
71
- artifactIds: ['artifact1', 'artifact2'],
72
- lastProcessed: 0,
73
- lastProcessedAttachmentsIdsList: [],
74
- },
75
- };
76
- // Mock getting attachments from each artifact
77
- adapter['uploader'].getAttachmentsFromArtifactId = jest
78
- .fn()
79
- .mockResolvedValueOnce({
80
- attachments: [
81
- {
82
- url: 'http://example.com/file1.pdf',
83
- id: 'attachment1',
84
- file_name: 'file1.pdf',
85
- parent_id: 'parent1',
86
- },
87
- {
88
- url: 'http://example.com/file2.pdf',
89
- id: 'attachment2',
90
- file_name: 'file2.pdf',
91
- parent_id: 'parent2',
92
- },
93
- ],
94
- })
95
- .mockResolvedValueOnce({
96
- attachments: [
97
- {
98
- url: 'http://example.com/file3.pdf',
99
- id: 'attachment3',
100
- file_name: 'file3.pdf',
101
- parent_id: 'parent3',
102
- },
103
- ],
104
- });
105
- // Mock the initializeRepos method
106
- adapter.initializeRepos = jest.fn();
107
- const result = await adapter.streamAttachments({
108
- stream: mockStream,
109
- });
110
- expect(adapter.initializeRepos).toHaveBeenCalledWith([
111
- { itemType: 'ssor_attachment' },
112
- ]);
113
- expect(adapter.initializeRepos).toHaveBeenCalledTimes(1);
114
- expect(adapter['uploader'].getAttachmentsFromArtifactId).toHaveBeenCalledTimes(2);
115
- // Verify state was updated correctly
116
- expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([]);
117
- expect(adapter.state.toDevRev.attachmentsMetadata.lastProcessed).toBe(0);
118
- expect(result).toBeUndefined();
119
- });
120
- it('[edge] should handle invalid batch size by using 1 instead', async () => {
121
- const mockStream = jest.fn();
122
- // Set up adapter state with artifact IDs
123
- adapter.state.toDevRev = {
124
- attachmentsMetadata: {
125
- artifactIds: ['artifact1'],
126
- lastProcessed: 0,
127
- lastProcessedAttachmentsIdsList: [],
128
- },
129
- };
130
- // Mock getting attachments
131
- adapter['uploader'].getAttachmentsFromArtifactId = jest
132
- .fn()
133
- .mockResolvedValue({
134
- attachments: [
135
- {
136
- url: 'http://example.com/file1.pdf',
137
- id: 'attachment1',
138
- file_name: 'file1.pdf',
139
- parent_id: 'parent1',
140
- },
141
- ],
142
- });
143
- adapter.initializeRepos = jest.fn();
144
- const result = await adapter.streamAttachments({
145
- stream: mockStream,
146
- batchSize: 0,
147
- });
148
- expect(result).toBeUndefined();
149
- });
150
- it('[edge] should cap batch size to 50 when batchSize is greater than 50', async () => {
151
- const mockStream = jest.fn();
152
- // Set up adapter state with artifact IDs
153
- adapter.state.toDevRev = {
154
- attachmentsMetadata: {
155
- artifactIds: ['artifact1'],
156
- lastProcessed: 0,
157
- lastProcessedAttachmentsIdsList: [],
158
- },
159
- };
160
- // Mock getting attachments
161
- adapter['uploader'].getAttachmentsFromArtifactId = jest
162
- .fn()
163
- .mockResolvedValue({
164
- attachments: [
165
- {
166
- url: 'http://example.com/file1.pdf',
167
- id: 'attachment1',
168
- file_name: 'file1.pdf',
169
- parent_id: 'parent1',
170
- },
171
- ],
172
- });
173
- // Mock the required methods
174
- adapter.initializeRepos = jest.fn();
175
- const result = await adapter.streamAttachments({
176
- stream: mockStream,
177
- batchSize: 100, // Set batch size greater than 50
178
- });
179
- expect(result).toBeUndefined();
180
- });
181
- it('[edge] should handle empty attachments metadata artifact IDs', async () => {
182
- const mockStream = jest.fn();
183
- // Set up adapter state with no artifact IDs
184
- adapter.state.toDevRev = {
185
- attachmentsMetadata: {
186
- artifactIds: [],
187
- lastProcessed: 0,
188
- },
189
- };
190
- const result = await adapter.streamAttachments({
191
- stream: mockStream,
192
- });
193
- expect(result).toBeUndefined();
194
- });
195
- it('[edge] should handle errors when getting attachments', async () => {
196
- const mockStream = jest.fn();
197
- // Set up adapter state with artifact IDs
198
- adapter.state.toDevRev = {
199
- attachmentsMetadata: {
200
- artifactIds: ['artifact1'],
201
- lastProcessed: 0,
202
- lastProcessedAttachmentsIdsList: [],
203
- },
204
- };
205
- // Mock error when getting attachments
206
- const mockError = new Error('Failed to get attachments');
207
- adapter['uploader'].getAttachmentsFromArtifactId = jest
208
- .fn()
209
- .mockResolvedValue({
210
- error: mockError,
211
- });
212
- // Mock methods
213
- adapter.initializeRepos = jest.fn();
214
- const result = await adapter.streamAttachments({
215
- stream: mockStream,
216
- });
217
- expect(result).toEqual({
218
- error: mockError,
219
- });
220
- });
221
- it('[edge] should handle empty attachments array from artifact', async () => {
222
- const mockStream = jest.fn();
223
- // Set up adapter state with artifact IDs
224
- adapter.state.toDevRev = {
225
- attachmentsMetadata: {
226
- artifactIds: ['artifact1'],
227
- lastProcessed: 0,
228
- lastProcessedAttachmentsIdsList: [],
229
- },
230
- };
231
- // Mock getting empty attachments
232
- adapter['uploader'].getAttachmentsFromArtifactId = jest
233
- .fn()
234
- .mockResolvedValue({
235
- attachments: [],
236
- });
237
- // Mock methods
238
- adapter.initializeRepos = jest.fn();
239
- const result = await adapter.streamAttachments({
240
- stream: mockStream,
241
- });
242
- expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([]);
243
- expect(result).toBeUndefined();
244
- });
245
- it('should use custom processors when provided', async () => {
246
- const mockStream = jest.fn();
247
- const mockReducer = jest.fn().mockReturnValue(['custom-reduced']);
248
- const mockIterator = jest.fn().mockResolvedValue({});
249
- // Set up adapter state with artifact IDs
250
- adapter.state.toDevRev = {
251
- attachmentsMetadata: {
252
- artifactIds: ['artifact1'],
253
- lastProcessed: 0,
254
- lastProcessedAttachmentsIdsList: [],
255
- },
256
- };
257
- // Mock getting attachments
258
- adapter['uploader'].getAttachmentsFromArtifactId = jest
259
- .fn()
260
- .mockResolvedValue({
261
- attachments: [{ id: 'attachment1' }],
262
- });
263
- // Mock methods
264
- adapter.initializeRepos = jest.fn();
265
- const result = await adapter.streamAttachments({
266
- stream: mockStream,
267
- processors: {
268
- reducer: mockReducer,
269
- iterator: mockIterator,
270
- },
271
- });
272
- expect(mockReducer).toHaveBeenCalledWith({
273
- attachments: [{ id: 'attachment1' }],
274
- adapter: adapter,
275
- batchSize: 1,
276
- });
277
- expect(mockIterator).toHaveBeenCalledWith({
278
- reducedAttachments: ['custom-reduced'],
279
- adapter: adapter,
280
- stream: mockStream,
281
- });
282
- expect(result).toBeUndefined();
283
- });
284
- it('should handle rate limiting from iterator', async () => {
285
- const mockStream = jest.fn();
286
- attachments_streaming_pool_1.AttachmentsStreamingPool.mockImplementationOnce(() => {
287
- return {
288
- // Return an object with a `streamAll` method that resolves to your desired value.
289
- streamAll: jest.fn().mockResolvedValue({ delay: 30 }),
290
- };
291
- });
292
- // Set up adapter state with artifact IDs
293
- adapter.state.toDevRev = {
294
- attachmentsMetadata: {
295
- artifactIds: ['artifact1'],
296
- lastProcessed: 0,
297
- lastProcessedAttachmentsIdsList: [],
298
- },
299
- };
300
- // Mock getting attachments
301
- adapter['uploader'].getAttachmentsFromArtifactId = jest
302
- .fn()
303
- .mockResolvedValue({
304
- attachments: [{ id: 'attachment1' }],
305
- });
306
- // Mock methods
307
- adapter.initializeRepos = jest.fn();
308
- const result = await adapter.streamAttachments({
309
- stream: mockStream,
310
- });
311
- expect(result).toEqual({
312
- delay: 30,
313
- });
314
- // The artifactIds array should remain unchanged
315
- expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([
316
- 'artifact1',
317
- ]);
318
- });
319
- it('should handle error from iterator', async () => {
320
- const mockStream = jest.fn();
321
- attachments_streaming_pool_1.AttachmentsStreamingPool.mockImplementationOnce(() => {
322
- return {
323
- // Return an object with a `streamAll` method that resolves to your desired value.
324
- streamAll: jest.fn().mockResolvedValue({
325
- error: 'Mock error',
326
- }),
327
- };
328
- });
329
- // Set up adapter state with artifact IDs
330
- adapter.state.toDevRev = {
331
- attachmentsMetadata: {
332
- artifactIds: ['artifact1'],
333
- lastProcessed: 0,
334
- lastProcessedAttachmentsIdsList: [],
335
- },
336
- };
337
- // Mock getting attachments
338
- adapter['uploader'].getAttachmentsFromArtifactId = jest
339
- .fn()
340
- .mockResolvedValue({
341
- attachments: [{ id: 'attachment1' }],
342
- });
343
- // Mock methods
344
- adapter.initializeRepos = jest.fn();
345
- const result = await adapter.streamAttachments({
346
- stream: mockStream,
347
- });
348
- expect(result).toEqual({
349
- error: 'Mock error',
350
- });
351
- // The artifactIds array should remain unchanged
352
- expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([
353
- 'artifact1',
354
- ]);
355
- });
356
- it('should reset lastProcessed and attachment IDs list after processing all artifacts', async () => {
357
- const mockStream = jest.fn();
358
- adapter.state.toDevRev = {
359
- attachmentsMetadata: {
360
- artifactIds: ['artifact1'],
361
- lastProcessed: 0,
362
- lastProcessedAttachmentsIdsList: [],
363
- },
364
- };
365
- adapter['uploader'].getAttachmentsFromArtifactId = jest
366
- .fn()
367
- .mockResolvedValueOnce({
368
- attachments: [
369
- {
370
- url: 'http://example.com/file1.pdf',
371
- id: 'attachment1',
372
- file_name: 'file1.pdf',
373
- parent_id: 'parent1',
374
- },
375
- {
376
- url: 'http://example.com/file2.pdf',
377
- id: 'attachment2',
378
- file_name: 'file2.pdf',
379
- parent_id: 'parent2',
380
- },
381
- {
382
- url: 'http://example.com/file3.pdf',
383
- id: 'attachment3',
384
- file_name: 'file3.pdf',
385
- parent_id: 'parent3',
386
- },
387
- ],
388
- });
389
- adapter.processAttachment = jest.fn().mockResolvedValue(null);
390
- await adapter.streamAttachments({
391
- stream: mockStream,
392
- });
393
- expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toHaveLength(0);
394
- expect(adapter.state.toDevRev.attachmentsMetadata.lastProcessed).toBe(0);
395
- });
396
- });
397
- describe(worker_adapter_1.WorkerAdapter.prototype.emit.name, () => {
398
- let counter;
399
- let mockPostMessage;
400
- beforeEach(() => {
401
- counter = { counter: 0 };
402
- // Import the worker_threads module and spy on parentPort.postMessage
403
- const workerThreads = require('node:worker_threads');
404
- mockPostMessage = jest.fn().mockImplementation(() => {
405
- counter.counter += 1;
406
- });
407
- // Spy on the parentPort.postMessage method
408
- if (workerThreads.parentPort) {
409
- jest
410
- .spyOn(workerThreads.parentPort, 'postMessage')
411
- .mockImplementation(mockPostMessage);
412
- }
413
- else {
414
- // If parentPort is null (not in worker context), create a mock
415
- workerThreads.parentPort = {
416
- postMessage: mockPostMessage,
417
- };
418
- }
419
- });
420
- afterEach(() => {
421
- // Restore all mocks
422
- jest.restoreAllMocks();
423
- });
424
- it('should emit only one event when multiple events of same type are sent', async () => {
425
- adapter['adapterState'].postState = jest
426
- .fn()
427
- .mockResolvedValue(undefined);
428
- adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
429
- await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
430
- reports: [],
431
- processed_files: [],
432
- });
433
- await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
434
- reports: [],
435
- processed_files: [],
436
- });
437
- expect(counter.counter).toBe(1);
438
- });
439
- it('should emit event when different event type is sent after previous events', async () => {
440
- adapter['adapterState'].postState = jest
441
- .fn()
442
- .mockResolvedValue(undefined);
443
- adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
444
- await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
445
- reports: [],
446
- processed_files: [],
447
- });
448
- await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
449
- reports: [],
450
- processed_files: [],
451
- });
452
- await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
453
- reports: [],
454
- processed_files: [],
455
- });
456
- expect(counter.counter).toBe(1);
457
- });
458
- it('should correctly emit one event even if postState errors', async () => {
459
- adapter['adapterState'].postState = jest
460
- .fn()
461
- .mockRejectedValue(new Error('postState error'));
462
- adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined);
463
- await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
464
- reports: [],
465
- processed_files: [],
466
- });
467
- expect(counter.counter).toBe(1);
468
- });
469
- it('should correctly emit one event even if uploadAllRepos errors', async () => {
470
- adapter['adapterState'].postState = jest
471
- .fn()
472
- .mockResolvedValue(undefined);
473
- adapter.uploadAllRepos = jest
474
- .fn()
475
- .mockRejectedValue(new Error('uploadAllRepos error'));
476
- await adapter.emit(types_1.ExtractorEventType.MetadataExtractionError, {
477
- reports: [],
478
- processed_files: [],
479
- });
480
- expect(counter.counter).toBe(1);
481
- });
482
- });
483
- });