@devrev/airsync-sdk 2.0.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/README.md +31 -0
  2. package/dist/attachments-streaming/attachments-streaming-pool.d.ts +16 -0
  3. package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -0
  4. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +9 -0
  5. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -0
  6. package/dist/attachments-streaming/attachments-streaming-pool.interfaces.js +2 -0
  7. package/dist/attachments-streaming/attachments-streaming-pool.js +97 -0
  8. package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts +2 -0
  9. package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts.map +1 -0
  10. package/dist/attachments-streaming/attachments-streaming-pool.test.js +267 -0
  11. package/dist/common/constants.d.ts +25 -0
  12. package/dist/common/constants.d.ts.map +1 -0
  13. package/dist/common/constants.js +58 -0
  14. package/dist/common/control-protocol.d.ts +10 -0
  15. package/dist/common/control-protocol.d.ts.map +1 -0
  16. package/dist/common/control-protocol.js +31 -0
  17. package/dist/common/errors.d.ts +6 -0
  18. package/dist/common/errors.d.ts.map +1 -0
  19. package/dist/common/errors.js +4 -0
  20. package/dist/common/event-type-translation.d.ts +24 -0
  21. package/dist/common/event-type-translation.d.ts.map +1 -0
  22. package/dist/common/event-type-translation.js +117 -0
  23. package/dist/common/helpers.d.ts +41 -0
  24. package/dist/common/helpers.d.ts.map +1 -0
  25. package/dist/common/helpers.js +124 -0
  26. package/dist/common/install-initial-domain-mapping.d.ts +4 -0
  27. package/dist/common/install-initial-domain-mapping.d.ts.map +1 -0
  28. package/dist/common/install-initial-domain-mapping.js +58 -0
  29. package/dist/common/install-initial-domain-mapping.test.d.ts +2 -0
  30. package/dist/common/install-initial-domain-mapping.test.d.ts.map +1 -0
  31. package/dist/common/install-initial-domain-mapping.test.js +207 -0
  32. package/dist/deprecated/adapter/index.d.ts +62 -0
  33. package/dist/deprecated/adapter/index.d.ts.map +1 -0
  34. package/dist/deprecated/adapter/index.js +151 -0
  35. package/dist/deprecated/common/helpers.d.ts +7 -0
  36. package/dist/deprecated/common/helpers.d.ts.map +1 -0
  37. package/dist/deprecated/common/helpers.js +47 -0
  38. package/dist/deprecated/demo-extractor/external_domain_metadata.json +38 -0
  39. package/dist/deprecated/demo-extractor/index.d.ts +18 -0
  40. package/dist/deprecated/demo-extractor/index.d.ts.map +1 -0
  41. package/dist/deprecated/demo-extractor/index.js +161 -0
  42. package/dist/deprecated/http/client.d.ts +22 -0
  43. package/dist/deprecated/http/client.d.ts.map +1 -0
  44. package/dist/deprecated/http/client.js +161 -0
  45. package/dist/deprecated/uploader/index.d.ts +35 -0
  46. package/dist/deprecated/uploader/index.d.ts.map +1 -0
  47. package/dist/deprecated/uploader/index.js +161 -0
  48. package/dist/http/axios-client-internal.d.ts +3 -0
  49. package/dist/http/axios-client-internal.d.ts.map +1 -0
  50. package/dist/http/axios-client-internal.js +66 -0
  51. package/dist/http/axios-client-internal.test.d.ts +2 -0
  52. package/dist/http/axios-client-internal.test.d.ts.map +1 -0
  53. package/dist/http/axios-client-internal.test.js +154 -0
  54. package/dist/http/axios-client.d.ts +27 -0
  55. package/dist/http/axios-client.d.ts.map +1 -0
  56. package/dist/http/axios-client.js +57 -0
  57. package/dist/http/constants.d.ts +4 -0
  58. package/dist/http/constants.d.ts.map +1 -0
  59. package/dist/http/constants.js +6 -0
  60. package/dist/http/index.d.ts +3 -0
  61. package/dist/http/index.d.ts.map +1 -0
  62. package/dist/http/index.js +18 -0
  63. package/dist/http/types.d.ts +17 -0
  64. package/dist/http/types.d.ts.map +1 -0
  65. package/dist/http/types.js +2 -0
  66. package/dist/index.d.ts +14 -0
  67. package/dist/index.d.ts.map +1 -0
  68. package/dist/index.js +34 -0
  69. package/dist/logger/logger.constants.d.ts +6 -0
  70. package/dist/logger/logger.constants.d.ts.map +1 -0
  71. package/dist/logger/logger.constants.js +13 -0
  72. package/dist/logger/logger.context.d.ts +58 -0
  73. package/dist/logger/logger.context.d.ts.map +1 -0
  74. package/dist/logger/logger.context.js +86 -0
  75. package/dist/logger/logger.d.ts +89 -0
  76. package/dist/logger/logger.d.ts.map +1 -0
  77. package/dist/logger/logger.interfaces.d.ts +43 -0
  78. package/dist/logger/logger.interfaces.d.ts.map +1 -0
  79. package/dist/logger/logger.interfaces.js +9 -0
  80. package/dist/logger/logger.js +196 -0
  81. package/dist/logger/logger.test.d.ts +2 -0
  82. package/dist/logger/logger.test.d.ts.map +1 -0
  83. package/dist/logger/logger.test.js +490 -0
  84. package/dist/mappers/mappers.d.ts +52 -0
  85. package/dist/mappers/mappers.d.ts.map +1 -0
  86. package/dist/mappers/mappers.interface.d.ts +294 -0
  87. package/dist/mappers/mappers.interface.d.ts.map +1 -0
  88. package/dist/mappers/mappers.interface.js +48 -0
  89. package/dist/mappers/mappers.js +83 -0
  90. package/dist/mappers/mappers.test.d.ts +2 -0
  91. package/dist/mappers/mappers.test.d.ts.map +1 -0
  92. package/dist/mappers/mappers.test.js +107 -0
  93. package/dist/multithreading/create-worker.d.ts +5 -0
  94. package/dist/multithreading/create-worker.d.ts.map +1 -0
  95. package/dist/multithreading/create-worker.js +28 -0
  96. package/dist/multithreading/create-worker.test.d.ts +2 -0
  97. package/dist/multithreading/create-worker.test.d.ts.map +1 -0
  98. package/dist/multithreading/create-worker.test.js +89 -0
  99. package/dist/multithreading/process-task.d.ts +3 -0
  100. package/dist/multithreading/process-task.d.ts.map +1 -0
  101. package/dist/multithreading/process-task.js +58 -0
  102. package/dist/multithreading/spawn/spawn.d.ts +30 -0
  103. package/dist/multithreading/spawn/spawn.d.ts.map +1 -0
  104. package/dist/multithreading/spawn/spawn.helpers.d.ts +21 -0
  105. package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -0
  106. package/dist/multithreading/spawn/spawn.helpers.js +114 -0
  107. package/dist/multithreading/spawn/spawn.helpers.test.d.ts +2 -0
  108. package/dist/multithreading/spawn/spawn.helpers.test.d.ts.map +1 -0
  109. package/dist/multithreading/spawn/spawn.helpers.test.js +293 -0
  110. package/dist/multithreading/spawn/spawn.js +249 -0
  111. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts +2 -0
  112. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts.map +1 -0
  113. package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.js +127 -0
  114. package/dist/multithreading/worker-adapter/worker-adapter.d.ts +91 -0
  115. package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +1 -0
  116. package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts +22 -0
  117. package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts.map +1 -0
  118. package/dist/multithreading/worker-adapter/worker-adapter.helpers.js +64 -0
  119. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts +2 -0
  120. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts.map +1 -0
  121. package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.js +514 -0
  122. package/dist/multithreading/worker-adapter/worker-adapter.js +747 -0
  123. package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts +2 -0
  124. package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts.map +1 -0
  125. package/dist/multithreading/worker-adapter/worker-adapter.test.js +483 -0
  126. package/dist/multithreading/worker.d.ts +2 -0
  127. package/dist/multithreading/worker.d.ts.map +1 -0
  128. package/dist/multithreading/worker.js +9 -0
  129. package/dist/repo/repo.d.ts +18 -0
  130. package/dist/repo/repo.d.ts.map +1 -0
  131. package/dist/repo/repo.interfaces.d.ts +46 -0
  132. package/dist/repo/repo.interfaces.d.ts.map +1 -0
  133. package/dist/repo/repo.interfaces.js +2 -0
  134. package/dist/repo/repo.js +75 -0
  135. package/dist/repo/repo.test.d.ts +2 -0
  136. package/dist/repo/repo.test.d.ts.map +1 -0
  137. package/dist/repo/repo.test.js +131 -0
  138. package/dist/state/state.d.ts +30 -0
  139. package/dist/state/state.d.ts.map +1 -0
  140. package/dist/state/state.interfaces.d.ts +51 -0
  141. package/dist/state/state.interfaces.d.ts.map +1 -0
  142. package/dist/state/state.interfaces.js +21 -0
  143. package/dist/state/state.js +166 -0
  144. package/dist/state/state.test.d.ts +2 -0
  145. package/dist/state/state.test.d.ts.map +1 -0
  146. package/dist/state/state.test.js +224 -0
  147. package/dist/types/common.d.ts +50 -0
  148. package/dist/types/common.d.ts.map +1 -0
  149. package/dist/types/common.js +25 -0
  150. package/dist/types/extraction.d.ts +417 -0
  151. package/dist/types/extraction.d.ts.map +1 -0
  152. package/dist/types/extraction.js +170 -0
  153. package/dist/types/extraction.test.d.ts +2 -0
  154. package/dist/types/extraction.test.d.ts.map +1 -0
  155. package/dist/types/extraction.test.js +70 -0
  156. package/dist/types/index.d.ts +9 -0
  157. package/dist/types/index.d.ts.map +1 -0
  158. package/dist/types/index.js +18 -0
  159. package/dist/types/loading.d.ts +147 -0
  160. package/dist/types/loading.d.ts.map +1 -0
  161. package/dist/types/loading.js +48 -0
  162. package/dist/types/workers.d.ts +161 -0
  163. package/dist/types/workers.d.ts.map +1 -0
  164. package/dist/types/workers.js +22 -0
  165. package/dist/uploader/uploader.d.ts +92 -0
  166. package/dist/uploader/uploader.d.ts.map +1 -0
  167. package/dist/uploader/uploader.helpers.d.ts +33 -0
  168. package/dist/uploader/uploader.helpers.d.ts.map +1 -0
  169. package/dist/uploader/uploader.helpers.js +139 -0
  170. package/dist/uploader/uploader.helpers.test.d.ts +2 -0
  171. package/dist/uploader/uploader.helpers.test.d.ts.map +1 -0
  172. package/dist/uploader/uploader.helpers.test.js +267 -0
  173. package/dist/uploader/uploader.interfaces.d.ts +95 -0
  174. package/dist/uploader/uploader.interfaces.d.ts.map +1 -0
  175. package/dist/uploader/uploader.interfaces.js +2 -0
  176. package/dist/uploader/uploader.js +305 -0
  177. package/dist/uploader/uploader.test.d.ts +2 -0
  178. package/dist/uploader/uploader.test.d.ts.map +1 -0
  179. package/dist/uploader/uploader.test.js +589 -0
  180. package/package.json +60 -0
@@ -0,0 +1,139 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || (function () {
19
+ var ownKeys = function(o) {
20
+ ownKeys = Object.getOwnPropertyNames || function (o) {
21
+ var ar = [];
22
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
+ return ar;
24
+ };
25
+ return ownKeys(o);
26
+ };
27
+ return function (mod) {
28
+ if (mod && mod.__esModule) return mod;
29
+ var result = {};
30
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
+ __setModuleDefault(result, mod);
32
+ return result;
33
+ };
34
+ })();
35
+ var __importDefault = (this && this.__importDefault) || function (mod) {
36
+ return (mod && mod.__esModule) ? mod : { "default": mod };
37
+ };
38
+ Object.defineProperty(exports, "__esModule", { value: true });
39
+ exports.compressGzip = compressGzip;
40
+ exports.decompressGzip = decompressGzip;
41
+ exports.parseJsonl = parseJsonl;
42
+ exports.downloadToLocal = downloadToLocal;
43
+ exports.truncateFilename = truncateFilename;
44
+ const fs_1 = __importStar(require("fs"));
45
+ const js_jsonl_1 = require("js-jsonl");
46
+ const zlib_1 = __importDefault(require("zlib"));
47
+ const constants_1 = require("../common/constants");
48
+ /**
49
+ * Compresses a JSONL string using gzip compression.
50
+ * @param {string} jsonlObject - The JSONL string to compress
51
+ * @returns {Buffer | void} The compressed buffer or undefined on error
52
+ */
53
+ function compressGzip(jsonlObject) {
54
+ try {
55
+ return { response: zlib_1.default.gzipSync(jsonlObject) };
56
+ }
57
+ catch (error) {
58
+ return { error };
59
+ }
60
+ }
61
+ /**
62
+ * Decompresses a gzipped buffer to a JSONL string.
63
+ * @param {Buffer} gzippedJsonlObject - The gzipped buffer to decompress
64
+ * @returns {string | void} The decompressed JSONL string or undefined on error
65
+ */
66
+ function decompressGzip(gzippedJsonlObject) {
67
+ try {
68
+ const jsonlObject = zlib_1.default.gunzipSync(gzippedJsonlObject);
69
+ return { response: jsonlObject.toString() };
70
+ }
71
+ catch (error) {
72
+ return { error };
73
+ }
74
+ }
75
+ /**
76
+ * Parses a JSONL string into an array of objects.
77
+ * @param {string} jsonlObject - The JSONL string to parse
78
+ * @returns {object[] | null} The parsed array of objects or null on error
79
+ */
80
+ function parseJsonl(jsonlObject) {
81
+ try {
82
+ return { response: js_jsonl_1.jsonl.parse(jsonlObject) };
83
+ }
84
+ catch (error) {
85
+ return { error };
86
+ }
87
+ }
88
+ /**
89
+ * Downloads fetched objects to the local file system (for local development).
90
+ * @param {string} itemType - The type of items being downloaded
91
+ * @param {object | object[]} fetchedObjects - The objects to write to file
92
+ * @returns {Promise<void>} Resolves when the file is written or rejects on error
93
+ */
94
+ async function downloadToLocal(itemType, fetchedObjects) {
95
+ console.log(`Downloading ${itemType} to local file system.`);
96
+ try {
97
+ if (!fs_1.default.existsSync('extracted_files')) {
98
+ fs_1.default.mkdirSync('extracted_files');
99
+ }
100
+ const timestamp = new Date().getTime();
101
+ const filePath = `extracted_files/extractor_${itemType}_${timestamp}.${itemType === 'external_domain_metadata' ? 'json' : 'jsonl'}`;
102
+ const fileHandle = await fs_1.promises.open(filePath, 'w');
103
+ let objArray = [];
104
+ if (!Array.isArray(fetchedObjects)) {
105
+ objArray.push(fetchedObjects);
106
+ }
107
+ else {
108
+ objArray = fetchedObjects;
109
+ }
110
+ for (const jsonObject of objArray) {
111
+ const jsonLine = JSON.stringify(jsonObject) + '\n';
112
+ await fileHandle.write(jsonLine);
113
+ }
114
+ await fileHandle.close();
115
+ console.log('Data successfully written to', filePath);
116
+ }
117
+ catch (error) {
118
+ console.error('Error writing data to file.', error);
119
+ return Promise.reject(error);
120
+ }
121
+ }
122
+ /**
123
+ * Truncates a filename if it exceeds the maximum allowed length.
124
+ * @param {string} filename - The filename to truncate
125
+ * @returns {string} The truncated filename
126
+ */
127
+ function truncateFilename(filename) {
128
+ // If the filename is already within the limit, return it as is.
129
+ if (filename.length <= constants_1.MAX_DEVREV_FILENAME_LENGTH) {
130
+ return filename;
131
+ }
132
+ console.warn(`Filename length exceeds the maximum limit of ${constants_1.MAX_DEVREV_FILENAME_LENGTH} characters. Truncating filename.`);
133
+ const extension = filename.slice(-constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH);
134
+ // Calculate how many characters are available for the name part after accounting for the extension and "..."
135
+ const availableNameLength = constants_1.MAX_DEVREV_FILENAME_LENGTH - constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH - 3; // -3 for "..."
136
+ // Truncate the name part and add an ellipsis
137
+ const truncatedFilename = filename.slice(0, availableNameLength);
138
+ return `${truncatedFilename}...${extension}`;
139
+ }
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=uploader.helpers.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"uploader.helpers.test.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.helpers.test.ts"],"names":[],"mappings":""}
@@ -0,0 +1,267 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ const fs_1 = __importDefault(require("fs"));
7
+ const fs_2 = require("fs");
8
+ const js_jsonl_1 = require("js-jsonl");
9
+ const zlib_1 = __importDefault(require("zlib"));
10
+ const constants_1 = require("../common/constants");
11
+ const uploader_helpers_1 = require("./uploader.helpers");
12
+ describe('uploader.helpers', () => {
13
+ let consoleErrorSpy;
14
+ let consoleLogSpy;
15
+ let consoleWarnSpy;
16
+ beforeEach(() => {
17
+ consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation();
18
+ consoleLogSpy = jest.spyOn(console, 'log').mockImplementation();
19
+ consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation();
20
+ });
21
+ afterEach(() => {
22
+ jest.clearAllMocks();
23
+ consoleErrorSpy.mockRestore();
24
+ consoleLogSpy.mockRestore();
25
+ consoleWarnSpy.mockRestore();
26
+ });
27
+ describe(uploader_helpers_1.compressGzip.name, () => {
28
+ it('should compress a string to gzip buffer', () => {
29
+ // Arrange
30
+ const input = '{"id":1}\n{"id":2}';
31
+ // Act
32
+ const result = (0, uploader_helpers_1.compressGzip)(input);
33
+ // Assert
34
+ expect(result.response).toBeInstanceOf(Buffer);
35
+ const decompressed = zlib_1.default
36
+ .gunzipSync(result.response)
37
+ .toString();
38
+ expect(decompressed).toBe(input);
39
+ });
40
+ it('[edge] should return undefined and log error when compression fails', () => {
41
+ // Arrange
42
+ const gzipSyncSpy = jest
43
+ .spyOn(zlib_1.default, 'gzipSync')
44
+ .mockImplementationOnce(() => {
45
+ throw new Error('Compression failed');
46
+ });
47
+ // Act
48
+ const result = (0, uploader_helpers_1.compressGzip)('test data');
49
+ // Assert
50
+ expect(result.response).toBeUndefined();
51
+ gzipSyncSpy.mockRestore();
52
+ });
53
+ });
54
+ describe(uploader_helpers_1.decompressGzip.name, () => {
55
+ it('should decompress a gzip buffer to string', () => {
56
+ // Arrange
57
+ const originalString = '{"id":1}\n{"id":2}';
58
+ const compressed = zlib_1.default.gzipSync(originalString);
59
+ // Act
60
+ const result = (0, uploader_helpers_1.decompressGzip)(compressed);
61
+ // Assert
62
+ expect(result.response).toBe(originalString);
63
+ });
64
+ it('[edge] should return undefined and log error when decompression fails', () => {
65
+ // Arrange
66
+ const invalidGzip = Buffer.from('not valid gzip data');
67
+ // Act
68
+ const result = (0, uploader_helpers_1.decompressGzip)(invalidGzip);
69
+ // Assert
70
+ expect(result.response).toBeUndefined();
71
+ });
72
+ });
73
+ describe(uploader_helpers_1.parseJsonl.name, () => {
74
+ it('should parse valid JSONL string to array of objects', () => {
75
+ // Arrange
76
+ const data = [
77
+ { id: 1, name: 'Item 1' },
78
+ { id: 2, name: 'Item 2' },
79
+ ];
80
+ const jsonlString = js_jsonl_1.jsonl.stringify(data);
81
+ // Act
82
+ const result = (0, uploader_helpers_1.parseJsonl)(jsonlString);
83
+ // Assert
84
+ expect(result.response).toEqual(data);
85
+ });
86
+ it('[edge] should return null and log error when parsing fails', () => {
87
+ // Arrange
88
+ const invalidJsonl = 'not valid jsonl {{{';
89
+ // Act
90
+ const result = (0, uploader_helpers_1.parseJsonl)(invalidJsonl);
91
+ // Assert
92
+ expect(result.response).toBeUndefined();
93
+ });
94
+ });
95
+ describe(uploader_helpers_1.downloadToLocal.name, () => {
96
+ const mockExistsSync = jest.spyOn(fs_1.default, 'existsSync');
97
+ const mockMkdirSync = jest.spyOn(fs_1.default, 'mkdirSync');
98
+ afterEach(() => {
99
+ mockExistsSync.mockReset();
100
+ mockMkdirSync.mockReset();
101
+ });
102
+ it('should create extracted_files directory if it does not exist', async () => {
103
+ // Arrange
104
+ const itemType = 'tasks';
105
+ const fetchedObjects = [{ id: 1 }];
106
+ mockExistsSync.mockReturnValueOnce(false);
107
+ mockMkdirSync.mockImplementationOnce(() => undefined);
108
+ const mockFileHandle = {
109
+ write: jest.fn().mockResolvedValue(undefined),
110
+ close: jest.fn().mockResolvedValue(undefined),
111
+ };
112
+ const fsPromisesOpenSpy = jest
113
+ .spyOn(fs_2.promises, 'open')
114
+ .mockResolvedValueOnce(mockFileHandle);
115
+ // Act
116
+ await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects);
117
+ // Assert
118
+ expect(mockExistsSync).toHaveBeenCalledWith('extracted_files');
119
+ expect(mockMkdirSync).toHaveBeenCalledWith('extracted_files');
120
+ fsPromisesOpenSpy.mockRestore();
121
+ });
122
+ it('should write objects as JSONL lines to file', async () => {
123
+ // Arrange
124
+ const itemType = 'tasks';
125
+ const fetchedObjects = [{ id: 1 }, { id: 2 }];
126
+ mockExistsSync.mockReturnValueOnce(true);
127
+ const mockFileHandle = {
128
+ write: jest.fn().mockResolvedValue(undefined),
129
+ close: jest.fn().mockResolvedValue(undefined),
130
+ };
131
+ const fsPromisesOpenSpy = jest
132
+ .spyOn(fs_2.promises, 'open')
133
+ .mockResolvedValueOnce(mockFileHandle);
134
+ // Act
135
+ await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects);
136
+ // Assert
137
+ expect(mockFileHandle.write).toHaveBeenCalledTimes(2);
138
+ expect(mockFileHandle.write).toHaveBeenCalledWith('{"id":1}\n');
139
+ expect(mockFileHandle.write).toHaveBeenCalledWith('{"id":2}\n');
140
+ expect(mockFileHandle.close).toHaveBeenCalled();
141
+ fsPromisesOpenSpy.mockRestore();
142
+ });
143
+ it('should handle single object (non-array) input', async () => {
144
+ // Arrange
145
+ const itemType = 'metadata';
146
+ const fetchedObject = { key: 'value' };
147
+ mockExistsSync.mockReturnValueOnce(true);
148
+ const mockFileHandle = {
149
+ write: jest.fn().mockResolvedValue(undefined),
150
+ close: jest.fn().mockResolvedValue(undefined),
151
+ };
152
+ const fsPromisesOpenSpy = jest
153
+ .spyOn(fs_2.promises, 'open')
154
+ .mockResolvedValueOnce(mockFileHandle);
155
+ // Act
156
+ await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObject);
157
+ // Assert
158
+ expect(mockFileHandle.write).toHaveBeenCalledTimes(1);
159
+ expect(mockFileHandle.write).toHaveBeenCalledWith('{"key":"value"}\n');
160
+ fsPromisesOpenSpy.mockRestore();
161
+ });
162
+ it('should use .json extension when itemType is external_domain_metadata', async () => {
163
+ // Arrange
164
+ const itemType = 'external_domain_metadata';
165
+ const fetchedObject = { domain: 'example.com' };
166
+ mockExistsSync.mockReturnValueOnce(true);
167
+ const mockFileHandle = {
168
+ write: jest.fn().mockResolvedValue(undefined),
169
+ close: jest.fn().mockResolvedValue(undefined),
170
+ };
171
+ const fsPromisesOpenSpy = jest
172
+ .spyOn(fs_2.promises, 'open')
173
+ .mockResolvedValueOnce(mockFileHandle);
174
+ // Act
175
+ await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObject);
176
+ // Assert
177
+ expect(fsPromisesOpenSpy).toHaveBeenCalledWith(expect.stringMatching(/extractor_external_domain_metadata_\d+\.json$/), 'w');
178
+ fsPromisesOpenSpy.mockRestore();
179
+ });
180
+ it('[edge] should reject and log error when file operations fail', async () => {
181
+ // Arrange
182
+ const itemType = 'tasks';
183
+ const fetchedObjects = [{ id: 1 }];
184
+ const fileError = new Error('File write failed');
185
+ mockExistsSync.mockReturnValueOnce(true);
186
+ const fsPromisesOpenSpy = jest
187
+ .spyOn(fs_2.promises, 'open')
188
+ .mockRejectedValueOnce(fileError);
189
+ // Act & Assert
190
+ await expect((0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects)).rejects.toThrow(fileError);
191
+ fsPromisesOpenSpy.mockRestore();
192
+ });
193
+ });
194
+ describe(uploader_helpers_1.truncateFilename.name, () => {
195
+ it('should return filename unchanged when within the limit', () => {
196
+ // Arrange
197
+ const filename = 'short-filename.txt';
198
+ // Act
199
+ const result = (0, uploader_helpers_1.truncateFilename)(filename);
200
+ // Assert
201
+ expect(result).toBe(filename);
202
+ });
203
+ it('should return filename unchanged when exactly at the limit', () => {
204
+ // Arrange
205
+ const filename = 'a'.repeat(constants_1.MAX_DEVREV_FILENAME_LENGTH);
206
+ // Act
207
+ const result = (0, uploader_helpers_1.truncateFilename)(filename);
208
+ // Assert
209
+ expect(result).toBe(filename);
210
+ expect(result.length).toBe(constants_1.MAX_DEVREV_FILENAME_LENGTH);
211
+ });
212
+ it('should truncate filename and preserve extension when exceeding the limit', () => {
213
+ // Arrange
214
+ const longName = 'a'.repeat(300);
215
+ const extension = '.txt';
216
+ const filename = longName + extension;
217
+ // Act
218
+ const result = (0, uploader_helpers_1.truncateFilename)(filename);
219
+ // Assert
220
+ expect(result.length).toBe(constants_1.MAX_DEVREV_FILENAME_LENGTH);
221
+ expect(result).toContain('...');
222
+ expect(result.endsWith(extension)).toBe(true);
223
+ });
224
+ it('should preserve the last MAX_DEVREV_FILENAME_EXTENSION_LENGTH characters as extension', () => {
225
+ // Arrange
226
+ const longName = 'document-'.repeat(50);
227
+ const extension = '.verylongextension';
228
+ const filename = longName + extension;
229
+ // Act
230
+ const result = (0, uploader_helpers_1.truncateFilename)(filename);
231
+ // Assert
232
+ expect(result.length).toBe(constants_1.MAX_DEVREV_FILENAME_LENGTH);
233
+ const expectedExtension = filename.slice(-constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH);
234
+ expect(result.endsWith(expectedExtension)).toBe(true);
235
+ });
236
+ it('should correctly format the truncated filename with ellipsis', () => {
237
+ // Arrange
238
+ const filename = 'x'.repeat(300) + '.pdf';
239
+ // Act
240
+ const result = (0, uploader_helpers_1.truncateFilename)(filename);
241
+ // Assert
242
+ const availableNameLength = constants_1.MAX_DEVREV_FILENAME_LENGTH - constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH - 3;
243
+ const expectedPrefix = 'x'.repeat(availableNameLength);
244
+ const expectedExtension = filename.slice(-constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH);
245
+ expect(result).toBe(`${expectedPrefix}...${expectedExtension}`);
246
+ });
247
+ it('[edge] should handle filename with no extension', () => {
248
+ // Arrange
249
+ const filename = 'a'.repeat(300);
250
+ // Act
251
+ const result = (0, uploader_helpers_1.truncateFilename)(filename);
252
+ // Assert
253
+ expect(result.length).toBe(constants_1.MAX_DEVREV_FILENAME_LENGTH);
254
+ expect(result).toContain('...');
255
+ // Last 20 chars are preserved as "extension"
256
+ expect(result.endsWith('a'.repeat(constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH))).toBe(true);
257
+ });
258
+ it('[edge] should handle filename that is just one character over the limit', () => {
259
+ // Arrange
260
+ const filename = 'a'.repeat(constants_1.MAX_DEVREV_FILENAME_LENGTH + 1);
261
+ // Act
262
+ const result = (0, uploader_helpers_1.truncateFilename)(filename);
263
+ // Assert
264
+ expect(result.length).toBe(constants_1.MAX_DEVREV_FILENAME_LENGTH);
265
+ });
266
+ });
267
+ });
@@ -0,0 +1,95 @@
1
+ import { ErrorRecord } from '../types/common';
2
+ import { AirdropEvent } from '../types/extraction';
3
+ import { ExternalSystemItem, StatsFileObject } from '../types/loading';
4
+ import { WorkerAdapterOptions } from '../types/workers';
5
+ export interface UploaderFactoryInterface {
6
+ event: AirdropEvent;
7
+ options?: WorkerAdapterOptions;
8
+ }
9
+ /**
10
+ * Generic result type for uploader operations that can either succeed with a response or fail with an error.
11
+ * @template T The type of the successful response data
12
+ */
13
+ export type UploaderResult<T> = {
14
+ response: T;
15
+ error?: never;
16
+ } | {
17
+ response?: never;
18
+ error: unknown;
19
+ };
20
+ /**
21
+ * Artifact is an interface that defines the structure of an artifact. Artifact is a file that is generated by the extractor and uploaded to ADaaS.
22
+ */
23
+ export interface Artifact {
24
+ id: string;
25
+ item_type: string;
26
+ item_count: number;
27
+ }
28
+ /**
29
+ * ArtifactsPrepareResponse is an interface that defines the structure of the response from the prepare artifacts endpoint.
30
+ */
31
+ export interface ArtifactsPrepareResponse {
32
+ url: string;
33
+ id: string;
34
+ form_data: {
35
+ key: string;
36
+ value: string;
37
+ }[];
38
+ }
39
+ /**
40
+ * ArtifactToUpload is an interface that defines the structure of the response from the get upload url endpoint.
41
+ */
42
+ export interface ArtifactToUpload {
43
+ upload_url: string;
44
+ artifact_id: string;
45
+ form_data: {
46
+ key: string;
47
+ value: string;
48
+ }[];
49
+ }
50
+ /**
51
+ * UploadResponse is an interface that defines the structure of the response from upload through Uploader.
52
+ */
53
+ export interface UploadResponse {
54
+ artifact?: Artifact;
55
+ error?: ErrorRecord;
56
+ }
57
+ /**
58
+ * StreamAttachmentsResponse is an interface that defines the structure of the response from the stream attachments through Uploader.
59
+ */
60
+ export interface StreamAttachmentsResponse {
61
+ ssorAttachments?: SsorAttachment[];
62
+ error?: ErrorRecord;
63
+ }
64
+ /**
65
+ * StreamResponse is an interface that defines the structure of the response from the stream of single attachment through Uploader.
66
+ */
67
+ export interface StreamResponse {
68
+ ssorAttachment?: SsorAttachment;
69
+ error?: ErrorRecord;
70
+ }
71
+ /**
72
+ * SsorAttachment is an interface that defines the structure of the SSOR attachment.
73
+ */
74
+ export interface SsorAttachment {
75
+ id: {
76
+ devrev: string;
77
+ external: string;
78
+ };
79
+ parent_id: {
80
+ external: string;
81
+ };
82
+ actor_id?: {
83
+ external: string;
84
+ };
85
+ inline?: boolean;
86
+ }
87
+ export interface StatsFileResponse {
88
+ error?: ErrorRecord;
89
+ statsFile?: StatsFileObject[];
90
+ }
91
+ export interface TransformerFileResponse {
92
+ error?: ErrorRecord;
93
+ transformerFile?: ExternalSystemItem[];
94
+ }
95
+ //# sourceMappingURL=uploader.interfaces.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;;GAGG;AACH,MAAM,MAAM,cAAc,CAAC,CAAC,IACxB;IAAE,QAAQ,EAAE,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAEzC;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,eAAe,CAAC,EAAE,cAAc,EAAE,CAAC;IACnC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE;QACF,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,QAAQ,CAAC,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,eAAe,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,uBAAuB;IACtC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,eAAe,CAAC,EAAE,kBAAkB,EAAE,CAAC;CACxC"}
@@ -0,0 +1,2 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });