@devrev/airsync-sdk 2.0.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -0
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts +16 -0
- package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -0
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts +9 -0
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.d.ts.map +1 -0
- package/dist/attachments-streaming/attachments-streaming-pool.interfaces.js +2 -0
- package/dist/attachments-streaming/attachments-streaming-pool.js +97 -0
- package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts +2 -0
- package/dist/attachments-streaming/attachments-streaming-pool.test.d.ts.map +1 -0
- package/dist/attachments-streaming/attachments-streaming-pool.test.js +267 -0
- package/dist/common/constants.d.ts +25 -0
- package/dist/common/constants.d.ts.map +1 -0
- package/dist/common/constants.js +58 -0
- package/dist/common/control-protocol.d.ts +10 -0
- package/dist/common/control-protocol.d.ts.map +1 -0
- package/dist/common/control-protocol.js +31 -0
- package/dist/common/errors.d.ts +6 -0
- package/dist/common/errors.d.ts.map +1 -0
- package/dist/common/errors.js +4 -0
- package/dist/common/event-type-translation.d.ts +24 -0
- package/dist/common/event-type-translation.d.ts.map +1 -0
- package/dist/common/event-type-translation.js +117 -0
- package/dist/common/helpers.d.ts +41 -0
- package/dist/common/helpers.d.ts.map +1 -0
- package/dist/common/helpers.js +124 -0
- package/dist/common/install-initial-domain-mapping.d.ts +4 -0
- package/dist/common/install-initial-domain-mapping.d.ts.map +1 -0
- package/dist/common/install-initial-domain-mapping.js +58 -0
- package/dist/common/install-initial-domain-mapping.test.d.ts +2 -0
- package/dist/common/install-initial-domain-mapping.test.d.ts.map +1 -0
- package/dist/common/install-initial-domain-mapping.test.js +207 -0
- package/dist/deprecated/adapter/index.d.ts +62 -0
- package/dist/deprecated/adapter/index.d.ts.map +1 -0
- package/dist/deprecated/adapter/index.js +151 -0
- package/dist/deprecated/common/helpers.d.ts +7 -0
- package/dist/deprecated/common/helpers.d.ts.map +1 -0
- package/dist/deprecated/common/helpers.js +47 -0
- package/dist/deprecated/demo-extractor/external_domain_metadata.json +38 -0
- package/dist/deprecated/demo-extractor/index.d.ts +18 -0
- package/dist/deprecated/demo-extractor/index.d.ts.map +1 -0
- package/dist/deprecated/demo-extractor/index.js +161 -0
- package/dist/deprecated/http/client.d.ts +22 -0
- package/dist/deprecated/http/client.d.ts.map +1 -0
- package/dist/deprecated/http/client.js +161 -0
- package/dist/deprecated/uploader/index.d.ts +35 -0
- package/dist/deprecated/uploader/index.d.ts.map +1 -0
- package/dist/deprecated/uploader/index.js +161 -0
- package/dist/http/axios-client-internal.d.ts +3 -0
- package/dist/http/axios-client-internal.d.ts.map +1 -0
- package/dist/http/axios-client-internal.js +66 -0
- package/dist/http/axios-client-internal.test.d.ts +2 -0
- package/dist/http/axios-client-internal.test.d.ts.map +1 -0
- package/dist/http/axios-client-internal.test.js +154 -0
- package/dist/http/axios-client.d.ts +27 -0
- package/dist/http/axios-client.d.ts.map +1 -0
- package/dist/http/axios-client.js +57 -0
- package/dist/http/constants.d.ts +4 -0
- package/dist/http/constants.d.ts.map +1 -0
- package/dist/http/constants.js +6 -0
- package/dist/http/index.d.ts +3 -0
- package/dist/http/index.d.ts.map +1 -0
- package/dist/http/index.js +18 -0
- package/dist/http/types.d.ts +17 -0
- package/dist/http/types.d.ts.map +1 -0
- package/dist/http/types.js +2 -0
- package/dist/index.d.ts +14 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +34 -0
- package/dist/logger/logger.constants.d.ts +6 -0
- package/dist/logger/logger.constants.d.ts.map +1 -0
- package/dist/logger/logger.constants.js +13 -0
- package/dist/logger/logger.context.d.ts +58 -0
- package/dist/logger/logger.context.d.ts.map +1 -0
- package/dist/logger/logger.context.js +86 -0
- package/dist/logger/logger.d.ts +89 -0
- package/dist/logger/logger.d.ts.map +1 -0
- package/dist/logger/logger.interfaces.d.ts +43 -0
- package/dist/logger/logger.interfaces.d.ts.map +1 -0
- package/dist/logger/logger.interfaces.js +9 -0
- package/dist/logger/logger.js +196 -0
- package/dist/logger/logger.test.d.ts +2 -0
- package/dist/logger/logger.test.d.ts.map +1 -0
- package/dist/logger/logger.test.js +490 -0
- package/dist/mappers/mappers.d.ts +52 -0
- package/dist/mappers/mappers.d.ts.map +1 -0
- package/dist/mappers/mappers.interface.d.ts +294 -0
- package/dist/mappers/mappers.interface.d.ts.map +1 -0
- package/dist/mappers/mappers.interface.js +48 -0
- package/dist/mappers/mappers.js +83 -0
- package/dist/mappers/mappers.test.d.ts +2 -0
- package/dist/mappers/mappers.test.d.ts.map +1 -0
- package/dist/mappers/mappers.test.js +107 -0
- package/dist/multithreading/create-worker.d.ts +5 -0
- package/dist/multithreading/create-worker.d.ts.map +1 -0
- package/dist/multithreading/create-worker.js +28 -0
- package/dist/multithreading/create-worker.test.d.ts +2 -0
- package/dist/multithreading/create-worker.test.d.ts.map +1 -0
- package/dist/multithreading/create-worker.test.js +89 -0
- package/dist/multithreading/process-task.d.ts +3 -0
- package/dist/multithreading/process-task.d.ts.map +1 -0
- package/dist/multithreading/process-task.js +58 -0
- package/dist/multithreading/spawn/spawn.d.ts +30 -0
- package/dist/multithreading/spawn/spawn.d.ts.map +1 -0
- package/dist/multithreading/spawn/spawn.helpers.d.ts +21 -0
- package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -0
- package/dist/multithreading/spawn/spawn.helpers.js +114 -0
- package/dist/multithreading/spawn/spawn.helpers.test.d.ts +2 -0
- package/dist/multithreading/spawn/spawn.helpers.test.d.ts.map +1 -0
- package/dist/multithreading/spawn/spawn.helpers.test.js +293 -0
- package/dist/multithreading/spawn/spawn.js +249 -0
- package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.artifacts.test.js +127 -0
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts +91 -0
- package/dist/multithreading/worker-adapter/worker-adapter.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts +22 -0
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.js +64 -0
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.helpers.test.js +514 -0
- package/dist/multithreading/worker-adapter/worker-adapter.js +747 -0
- package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts +2 -0
- package/dist/multithreading/worker-adapter/worker-adapter.test.d.ts.map +1 -0
- package/dist/multithreading/worker-adapter/worker-adapter.test.js +483 -0
- package/dist/multithreading/worker.d.ts +2 -0
- package/dist/multithreading/worker.d.ts.map +1 -0
- package/dist/multithreading/worker.js +9 -0
- package/dist/repo/repo.d.ts +18 -0
- package/dist/repo/repo.d.ts.map +1 -0
- package/dist/repo/repo.interfaces.d.ts +46 -0
- package/dist/repo/repo.interfaces.d.ts.map +1 -0
- package/dist/repo/repo.interfaces.js +2 -0
- package/dist/repo/repo.js +75 -0
- package/dist/repo/repo.test.d.ts +2 -0
- package/dist/repo/repo.test.d.ts.map +1 -0
- package/dist/repo/repo.test.js +131 -0
- package/dist/state/state.d.ts +30 -0
- package/dist/state/state.d.ts.map +1 -0
- package/dist/state/state.interfaces.d.ts +51 -0
- package/dist/state/state.interfaces.d.ts.map +1 -0
- package/dist/state/state.interfaces.js +21 -0
- package/dist/state/state.js +166 -0
- package/dist/state/state.test.d.ts +2 -0
- package/dist/state/state.test.d.ts.map +1 -0
- package/dist/state/state.test.js +224 -0
- package/dist/types/common.d.ts +50 -0
- package/dist/types/common.d.ts.map +1 -0
- package/dist/types/common.js +25 -0
- package/dist/types/extraction.d.ts +417 -0
- package/dist/types/extraction.d.ts.map +1 -0
- package/dist/types/extraction.js +170 -0
- package/dist/types/extraction.test.d.ts +2 -0
- package/dist/types/extraction.test.d.ts.map +1 -0
- package/dist/types/extraction.test.js +70 -0
- package/dist/types/index.d.ts +9 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +18 -0
- package/dist/types/loading.d.ts +147 -0
- package/dist/types/loading.d.ts.map +1 -0
- package/dist/types/loading.js +48 -0
- package/dist/types/workers.d.ts +161 -0
- package/dist/types/workers.d.ts.map +1 -0
- package/dist/types/workers.js +22 -0
- package/dist/uploader/uploader.d.ts +92 -0
- package/dist/uploader/uploader.d.ts.map +1 -0
- package/dist/uploader/uploader.helpers.d.ts +33 -0
- package/dist/uploader/uploader.helpers.d.ts.map +1 -0
- package/dist/uploader/uploader.helpers.js +139 -0
- package/dist/uploader/uploader.helpers.test.d.ts +2 -0
- package/dist/uploader/uploader.helpers.test.d.ts.map +1 -0
- package/dist/uploader/uploader.helpers.test.js +267 -0
- package/dist/uploader/uploader.interfaces.d.ts +95 -0
- package/dist/uploader/uploader.interfaces.d.ts.map +1 -0
- package/dist/uploader/uploader.interfaces.js +2 -0
- package/dist/uploader/uploader.js +305 -0
- package/dist/uploader/uploader.test.d.ts +2 -0
- package/dist/uploader/uploader.test.d.ts.map +1 -0
- package/dist/uploader/uploader.test.js +589 -0
- package/package.json +60 -0
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
36
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
|
+
};
|
|
38
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
+
exports.compressGzip = compressGzip;
|
|
40
|
+
exports.decompressGzip = decompressGzip;
|
|
41
|
+
exports.parseJsonl = parseJsonl;
|
|
42
|
+
exports.downloadToLocal = downloadToLocal;
|
|
43
|
+
exports.truncateFilename = truncateFilename;
|
|
44
|
+
const fs_1 = __importStar(require("fs"));
|
|
45
|
+
const js_jsonl_1 = require("js-jsonl");
|
|
46
|
+
const zlib_1 = __importDefault(require("zlib"));
|
|
47
|
+
const constants_1 = require("../common/constants");
|
|
48
|
+
/**
|
|
49
|
+
* Compresses a JSONL string using gzip compression.
|
|
50
|
+
* @param {string} jsonlObject - The JSONL string to compress
|
|
51
|
+
* @returns {Buffer | void} The compressed buffer or undefined on error
|
|
52
|
+
*/
|
|
53
|
+
function compressGzip(jsonlObject) {
|
|
54
|
+
try {
|
|
55
|
+
return { response: zlib_1.default.gzipSync(jsonlObject) };
|
|
56
|
+
}
|
|
57
|
+
catch (error) {
|
|
58
|
+
return { error };
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Decompresses a gzipped buffer to a JSONL string.
|
|
63
|
+
* @param {Buffer} gzippedJsonlObject - The gzipped buffer to decompress
|
|
64
|
+
* @returns {string | void} The decompressed JSONL string or undefined on error
|
|
65
|
+
*/
|
|
66
|
+
function decompressGzip(gzippedJsonlObject) {
|
|
67
|
+
try {
|
|
68
|
+
const jsonlObject = zlib_1.default.gunzipSync(gzippedJsonlObject);
|
|
69
|
+
return { response: jsonlObject.toString() };
|
|
70
|
+
}
|
|
71
|
+
catch (error) {
|
|
72
|
+
return { error };
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Parses a JSONL string into an array of objects.
|
|
77
|
+
* @param {string} jsonlObject - The JSONL string to parse
|
|
78
|
+
* @returns {object[] | null} The parsed array of objects or null on error
|
|
79
|
+
*/
|
|
80
|
+
function parseJsonl(jsonlObject) {
|
|
81
|
+
try {
|
|
82
|
+
return { response: js_jsonl_1.jsonl.parse(jsonlObject) };
|
|
83
|
+
}
|
|
84
|
+
catch (error) {
|
|
85
|
+
return { error };
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Downloads fetched objects to the local file system (for local development).
|
|
90
|
+
* @param {string} itemType - The type of items being downloaded
|
|
91
|
+
* @param {object | object[]} fetchedObjects - The objects to write to file
|
|
92
|
+
* @returns {Promise<void>} Resolves when the file is written or rejects on error
|
|
93
|
+
*/
|
|
94
|
+
async function downloadToLocal(itemType, fetchedObjects) {
|
|
95
|
+
console.log(`Downloading ${itemType} to local file system.`);
|
|
96
|
+
try {
|
|
97
|
+
if (!fs_1.default.existsSync('extracted_files')) {
|
|
98
|
+
fs_1.default.mkdirSync('extracted_files');
|
|
99
|
+
}
|
|
100
|
+
const timestamp = new Date().getTime();
|
|
101
|
+
const filePath = `extracted_files/extractor_${itemType}_${timestamp}.${itemType === 'external_domain_metadata' ? 'json' : 'jsonl'}`;
|
|
102
|
+
const fileHandle = await fs_1.promises.open(filePath, 'w');
|
|
103
|
+
let objArray = [];
|
|
104
|
+
if (!Array.isArray(fetchedObjects)) {
|
|
105
|
+
objArray.push(fetchedObjects);
|
|
106
|
+
}
|
|
107
|
+
else {
|
|
108
|
+
objArray = fetchedObjects;
|
|
109
|
+
}
|
|
110
|
+
for (const jsonObject of objArray) {
|
|
111
|
+
const jsonLine = JSON.stringify(jsonObject) + '\n';
|
|
112
|
+
await fileHandle.write(jsonLine);
|
|
113
|
+
}
|
|
114
|
+
await fileHandle.close();
|
|
115
|
+
console.log('Data successfully written to', filePath);
|
|
116
|
+
}
|
|
117
|
+
catch (error) {
|
|
118
|
+
console.error('Error writing data to file.', error);
|
|
119
|
+
return Promise.reject(error);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Truncates a filename if it exceeds the maximum allowed length.
|
|
124
|
+
* @param {string} filename - The filename to truncate
|
|
125
|
+
* @returns {string} The truncated filename
|
|
126
|
+
*/
|
|
127
|
+
function truncateFilename(filename) {
|
|
128
|
+
// If the filename is already within the limit, return it as is.
|
|
129
|
+
if (filename.length <= constants_1.MAX_DEVREV_FILENAME_LENGTH) {
|
|
130
|
+
return filename;
|
|
131
|
+
}
|
|
132
|
+
console.warn(`Filename length exceeds the maximum limit of ${constants_1.MAX_DEVREV_FILENAME_LENGTH} characters. Truncating filename.`);
|
|
133
|
+
const extension = filename.slice(-constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH);
|
|
134
|
+
// Calculate how many characters are available for the name part after accounting for the extension and "..."
|
|
135
|
+
const availableNameLength = constants_1.MAX_DEVREV_FILENAME_LENGTH - constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH - 3; // -3 for "..."
|
|
136
|
+
// Truncate the name part and add an ellipsis
|
|
137
|
+
const truncatedFilename = filename.slice(0, availableNameLength);
|
|
138
|
+
return `${truncatedFilename}...${extension}`;
|
|
139
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"uploader.helpers.test.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.helpers.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const fs_1 = __importDefault(require("fs"));
|
|
7
|
+
const fs_2 = require("fs");
|
|
8
|
+
const js_jsonl_1 = require("js-jsonl");
|
|
9
|
+
const zlib_1 = __importDefault(require("zlib"));
|
|
10
|
+
const constants_1 = require("../common/constants");
|
|
11
|
+
const uploader_helpers_1 = require("./uploader.helpers");
|
|
12
|
+
describe('uploader.helpers', () => {
|
|
13
|
+
let consoleErrorSpy;
|
|
14
|
+
let consoleLogSpy;
|
|
15
|
+
let consoleWarnSpy;
|
|
16
|
+
beforeEach(() => {
|
|
17
|
+
consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation();
|
|
18
|
+
consoleLogSpy = jest.spyOn(console, 'log').mockImplementation();
|
|
19
|
+
consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation();
|
|
20
|
+
});
|
|
21
|
+
afterEach(() => {
|
|
22
|
+
jest.clearAllMocks();
|
|
23
|
+
consoleErrorSpy.mockRestore();
|
|
24
|
+
consoleLogSpy.mockRestore();
|
|
25
|
+
consoleWarnSpy.mockRestore();
|
|
26
|
+
});
|
|
27
|
+
describe(uploader_helpers_1.compressGzip.name, () => {
|
|
28
|
+
it('should compress a string to gzip buffer', () => {
|
|
29
|
+
// Arrange
|
|
30
|
+
const input = '{"id":1}\n{"id":2}';
|
|
31
|
+
// Act
|
|
32
|
+
const result = (0, uploader_helpers_1.compressGzip)(input);
|
|
33
|
+
// Assert
|
|
34
|
+
expect(result.response).toBeInstanceOf(Buffer);
|
|
35
|
+
const decompressed = zlib_1.default
|
|
36
|
+
.gunzipSync(result.response)
|
|
37
|
+
.toString();
|
|
38
|
+
expect(decompressed).toBe(input);
|
|
39
|
+
});
|
|
40
|
+
it('[edge] should return undefined and log error when compression fails', () => {
|
|
41
|
+
// Arrange
|
|
42
|
+
const gzipSyncSpy = jest
|
|
43
|
+
.spyOn(zlib_1.default, 'gzipSync')
|
|
44
|
+
.mockImplementationOnce(() => {
|
|
45
|
+
throw new Error('Compression failed');
|
|
46
|
+
});
|
|
47
|
+
// Act
|
|
48
|
+
const result = (0, uploader_helpers_1.compressGzip)('test data');
|
|
49
|
+
// Assert
|
|
50
|
+
expect(result.response).toBeUndefined();
|
|
51
|
+
gzipSyncSpy.mockRestore();
|
|
52
|
+
});
|
|
53
|
+
});
|
|
54
|
+
describe(uploader_helpers_1.decompressGzip.name, () => {
|
|
55
|
+
it('should decompress a gzip buffer to string', () => {
|
|
56
|
+
// Arrange
|
|
57
|
+
const originalString = '{"id":1}\n{"id":2}';
|
|
58
|
+
const compressed = zlib_1.default.gzipSync(originalString);
|
|
59
|
+
// Act
|
|
60
|
+
const result = (0, uploader_helpers_1.decompressGzip)(compressed);
|
|
61
|
+
// Assert
|
|
62
|
+
expect(result.response).toBe(originalString);
|
|
63
|
+
});
|
|
64
|
+
it('[edge] should return undefined and log error when decompression fails', () => {
|
|
65
|
+
// Arrange
|
|
66
|
+
const invalidGzip = Buffer.from('not valid gzip data');
|
|
67
|
+
// Act
|
|
68
|
+
const result = (0, uploader_helpers_1.decompressGzip)(invalidGzip);
|
|
69
|
+
// Assert
|
|
70
|
+
expect(result.response).toBeUndefined();
|
|
71
|
+
});
|
|
72
|
+
});
|
|
73
|
+
describe(uploader_helpers_1.parseJsonl.name, () => {
|
|
74
|
+
it('should parse valid JSONL string to array of objects', () => {
|
|
75
|
+
// Arrange
|
|
76
|
+
const data = [
|
|
77
|
+
{ id: 1, name: 'Item 1' },
|
|
78
|
+
{ id: 2, name: 'Item 2' },
|
|
79
|
+
];
|
|
80
|
+
const jsonlString = js_jsonl_1.jsonl.stringify(data);
|
|
81
|
+
// Act
|
|
82
|
+
const result = (0, uploader_helpers_1.parseJsonl)(jsonlString);
|
|
83
|
+
// Assert
|
|
84
|
+
expect(result.response).toEqual(data);
|
|
85
|
+
});
|
|
86
|
+
it('[edge] should return null and log error when parsing fails', () => {
|
|
87
|
+
// Arrange
|
|
88
|
+
const invalidJsonl = 'not valid jsonl {{{';
|
|
89
|
+
// Act
|
|
90
|
+
const result = (0, uploader_helpers_1.parseJsonl)(invalidJsonl);
|
|
91
|
+
// Assert
|
|
92
|
+
expect(result.response).toBeUndefined();
|
|
93
|
+
});
|
|
94
|
+
});
|
|
95
|
+
describe(uploader_helpers_1.downloadToLocal.name, () => {
|
|
96
|
+
const mockExistsSync = jest.spyOn(fs_1.default, 'existsSync');
|
|
97
|
+
const mockMkdirSync = jest.spyOn(fs_1.default, 'mkdirSync');
|
|
98
|
+
afterEach(() => {
|
|
99
|
+
mockExistsSync.mockReset();
|
|
100
|
+
mockMkdirSync.mockReset();
|
|
101
|
+
});
|
|
102
|
+
it('should create extracted_files directory if it does not exist', async () => {
|
|
103
|
+
// Arrange
|
|
104
|
+
const itemType = 'tasks';
|
|
105
|
+
const fetchedObjects = [{ id: 1 }];
|
|
106
|
+
mockExistsSync.mockReturnValueOnce(false);
|
|
107
|
+
mockMkdirSync.mockImplementationOnce(() => undefined);
|
|
108
|
+
const mockFileHandle = {
|
|
109
|
+
write: jest.fn().mockResolvedValue(undefined),
|
|
110
|
+
close: jest.fn().mockResolvedValue(undefined),
|
|
111
|
+
};
|
|
112
|
+
const fsPromisesOpenSpy = jest
|
|
113
|
+
.spyOn(fs_2.promises, 'open')
|
|
114
|
+
.mockResolvedValueOnce(mockFileHandle);
|
|
115
|
+
// Act
|
|
116
|
+
await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects);
|
|
117
|
+
// Assert
|
|
118
|
+
expect(mockExistsSync).toHaveBeenCalledWith('extracted_files');
|
|
119
|
+
expect(mockMkdirSync).toHaveBeenCalledWith('extracted_files');
|
|
120
|
+
fsPromisesOpenSpy.mockRestore();
|
|
121
|
+
});
|
|
122
|
+
it('should write objects as JSONL lines to file', async () => {
|
|
123
|
+
// Arrange
|
|
124
|
+
const itemType = 'tasks';
|
|
125
|
+
const fetchedObjects = [{ id: 1 }, { id: 2 }];
|
|
126
|
+
mockExistsSync.mockReturnValueOnce(true);
|
|
127
|
+
const mockFileHandle = {
|
|
128
|
+
write: jest.fn().mockResolvedValue(undefined),
|
|
129
|
+
close: jest.fn().mockResolvedValue(undefined),
|
|
130
|
+
};
|
|
131
|
+
const fsPromisesOpenSpy = jest
|
|
132
|
+
.spyOn(fs_2.promises, 'open')
|
|
133
|
+
.mockResolvedValueOnce(mockFileHandle);
|
|
134
|
+
// Act
|
|
135
|
+
await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects);
|
|
136
|
+
// Assert
|
|
137
|
+
expect(mockFileHandle.write).toHaveBeenCalledTimes(2);
|
|
138
|
+
expect(mockFileHandle.write).toHaveBeenCalledWith('{"id":1}\n');
|
|
139
|
+
expect(mockFileHandle.write).toHaveBeenCalledWith('{"id":2}\n');
|
|
140
|
+
expect(mockFileHandle.close).toHaveBeenCalled();
|
|
141
|
+
fsPromisesOpenSpy.mockRestore();
|
|
142
|
+
});
|
|
143
|
+
it('should handle single object (non-array) input', async () => {
|
|
144
|
+
// Arrange
|
|
145
|
+
const itemType = 'metadata';
|
|
146
|
+
const fetchedObject = { key: 'value' };
|
|
147
|
+
mockExistsSync.mockReturnValueOnce(true);
|
|
148
|
+
const mockFileHandle = {
|
|
149
|
+
write: jest.fn().mockResolvedValue(undefined),
|
|
150
|
+
close: jest.fn().mockResolvedValue(undefined),
|
|
151
|
+
};
|
|
152
|
+
const fsPromisesOpenSpy = jest
|
|
153
|
+
.spyOn(fs_2.promises, 'open')
|
|
154
|
+
.mockResolvedValueOnce(mockFileHandle);
|
|
155
|
+
// Act
|
|
156
|
+
await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObject);
|
|
157
|
+
// Assert
|
|
158
|
+
expect(mockFileHandle.write).toHaveBeenCalledTimes(1);
|
|
159
|
+
expect(mockFileHandle.write).toHaveBeenCalledWith('{"key":"value"}\n');
|
|
160
|
+
fsPromisesOpenSpy.mockRestore();
|
|
161
|
+
});
|
|
162
|
+
it('should use .json extension when itemType is external_domain_metadata', async () => {
|
|
163
|
+
// Arrange
|
|
164
|
+
const itemType = 'external_domain_metadata';
|
|
165
|
+
const fetchedObject = { domain: 'example.com' };
|
|
166
|
+
mockExistsSync.mockReturnValueOnce(true);
|
|
167
|
+
const mockFileHandle = {
|
|
168
|
+
write: jest.fn().mockResolvedValue(undefined),
|
|
169
|
+
close: jest.fn().mockResolvedValue(undefined),
|
|
170
|
+
};
|
|
171
|
+
const fsPromisesOpenSpy = jest
|
|
172
|
+
.spyOn(fs_2.promises, 'open')
|
|
173
|
+
.mockResolvedValueOnce(mockFileHandle);
|
|
174
|
+
// Act
|
|
175
|
+
await (0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObject);
|
|
176
|
+
// Assert
|
|
177
|
+
expect(fsPromisesOpenSpy).toHaveBeenCalledWith(expect.stringMatching(/extractor_external_domain_metadata_\d+\.json$/), 'w');
|
|
178
|
+
fsPromisesOpenSpy.mockRestore();
|
|
179
|
+
});
|
|
180
|
+
it('[edge] should reject and log error when file operations fail', async () => {
|
|
181
|
+
// Arrange
|
|
182
|
+
const itemType = 'tasks';
|
|
183
|
+
const fetchedObjects = [{ id: 1 }];
|
|
184
|
+
const fileError = new Error('File write failed');
|
|
185
|
+
mockExistsSync.mockReturnValueOnce(true);
|
|
186
|
+
const fsPromisesOpenSpy = jest
|
|
187
|
+
.spyOn(fs_2.promises, 'open')
|
|
188
|
+
.mockRejectedValueOnce(fileError);
|
|
189
|
+
// Act & Assert
|
|
190
|
+
await expect((0, uploader_helpers_1.downloadToLocal)(itemType, fetchedObjects)).rejects.toThrow(fileError);
|
|
191
|
+
fsPromisesOpenSpy.mockRestore();
|
|
192
|
+
});
|
|
193
|
+
});
|
|
194
|
+
describe(uploader_helpers_1.truncateFilename.name, () => {
|
|
195
|
+
it('should return filename unchanged when within the limit', () => {
|
|
196
|
+
// Arrange
|
|
197
|
+
const filename = 'short-filename.txt';
|
|
198
|
+
// Act
|
|
199
|
+
const result = (0, uploader_helpers_1.truncateFilename)(filename);
|
|
200
|
+
// Assert
|
|
201
|
+
expect(result).toBe(filename);
|
|
202
|
+
});
|
|
203
|
+
it('should return filename unchanged when exactly at the limit', () => {
|
|
204
|
+
// Arrange
|
|
205
|
+
const filename = 'a'.repeat(constants_1.MAX_DEVREV_FILENAME_LENGTH);
|
|
206
|
+
// Act
|
|
207
|
+
const result = (0, uploader_helpers_1.truncateFilename)(filename);
|
|
208
|
+
// Assert
|
|
209
|
+
expect(result).toBe(filename);
|
|
210
|
+
expect(result.length).toBe(constants_1.MAX_DEVREV_FILENAME_LENGTH);
|
|
211
|
+
});
|
|
212
|
+
it('should truncate filename and preserve extension when exceeding the limit', () => {
|
|
213
|
+
// Arrange
|
|
214
|
+
const longName = 'a'.repeat(300);
|
|
215
|
+
const extension = '.txt';
|
|
216
|
+
const filename = longName + extension;
|
|
217
|
+
// Act
|
|
218
|
+
const result = (0, uploader_helpers_1.truncateFilename)(filename);
|
|
219
|
+
// Assert
|
|
220
|
+
expect(result.length).toBe(constants_1.MAX_DEVREV_FILENAME_LENGTH);
|
|
221
|
+
expect(result).toContain('...');
|
|
222
|
+
expect(result.endsWith(extension)).toBe(true);
|
|
223
|
+
});
|
|
224
|
+
it('should preserve the last MAX_DEVREV_FILENAME_EXTENSION_LENGTH characters as extension', () => {
|
|
225
|
+
// Arrange
|
|
226
|
+
const longName = 'document-'.repeat(50);
|
|
227
|
+
const extension = '.verylongextension';
|
|
228
|
+
const filename = longName + extension;
|
|
229
|
+
// Act
|
|
230
|
+
const result = (0, uploader_helpers_1.truncateFilename)(filename);
|
|
231
|
+
// Assert
|
|
232
|
+
expect(result.length).toBe(constants_1.MAX_DEVREV_FILENAME_LENGTH);
|
|
233
|
+
const expectedExtension = filename.slice(-constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH);
|
|
234
|
+
expect(result.endsWith(expectedExtension)).toBe(true);
|
|
235
|
+
});
|
|
236
|
+
it('should correctly format the truncated filename with ellipsis', () => {
|
|
237
|
+
// Arrange
|
|
238
|
+
const filename = 'x'.repeat(300) + '.pdf';
|
|
239
|
+
// Act
|
|
240
|
+
const result = (0, uploader_helpers_1.truncateFilename)(filename);
|
|
241
|
+
// Assert
|
|
242
|
+
const availableNameLength = constants_1.MAX_DEVREV_FILENAME_LENGTH - constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH - 3;
|
|
243
|
+
const expectedPrefix = 'x'.repeat(availableNameLength);
|
|
244
|
+
const expectedExtension = filename.slice(-constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH);
|
|
245
|
+
expect(result).toBe(`${expectedPrefix}...${expectedExtension}`);
|
|
246
|
+
});
|
|
247
|
+
it('[edge] should handle filename with no extension', () => {
|
|
248
|
+
// Arrange
|
|
249
|
+
const filename = 'a'.repeat(300);
|
|
250
|
+
// Act
|
|
251
|
+
const result = (0, uploader_helpers_1.truncateFilename)(filename);
|
|
252
|
+
// Assert
|
|
253
|
+
expect(result.length).toBe(constants_1.MAX_DEVREV_FILENAME_LENGTH);
|
|
254
|
+
expect(result).toContain('...');
|
|
255
|
+
// Last 20 chars are preserved as "extension"
|
|
256
|
+
expect(result.endsWith('a'.repeat(constants_1.MAX_DEVREV_FILENAME_EXTENSION_LENGTH))).toBe(true);
|
|
257
|
+
});
|
|
258
|
+
it('[edge] should handle filename that is just one character over the limit', () => {
|
|
259
|
+
// Arrange
|
|
260
|
+
const filename = 'a'.repeat(constants_1.MAX_DEVREV_FILENAME_LENGTH + 1);
|
|
261
|
+
// Act
|
|
262
|
+
const result = (0, uploader_helpers_1.truncateFilename)(filename);
|
|
263
|
+
// Assert
|
|
264
|
+
expect(result.length).toBe(constants_1.MAX_DEVREV_FILENAME_LENGTH);
|
|
265
|
+
});
|
|
266
|
+
});
|
|
267
|
+
});
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { ErrorRecord } from '../types/common';
|
|
2
|
+
import { AirdropEvent } from '../types/extraction';
|
|
3
|
+
import { ExternalSystemItem, StatsFileObject } from '../types/loading';
|
|
4
|
+
import { WorkerAdapterOptions } from '../types/workers';
|
|
5
|
+
export interface UploaderFactoryInterface {
|
|
6
|
+
event: AirdropEvent;
|
|
7
|
+
options?: WorkerAdapterOptions;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Generic result type for uploader operations that can either succeed with a response or fail with an error.
|
|
11
|
+
* @template T The type of the successful response data
|
|
12
|
+
*/
|
|
13
|
+
export type UploaderResult<T> = {
|
|
14
|
+
response: T;
|
|
15
|
+
error?: never;
|
|
16
|
+
} | {
|
|
17
|
+
response?: never;
|
|
18
|
+
error: unknown;
|
|
19
|
+
};
|
|
20
|
+
/**
|
|
21
|
+
* Artifact is an interface that defines the structure of an artifact. Artifact is a file that is generated by the extractor and uploaded to ADaaS.
|
|
22
|
+
*/
|
|
23
|
+
export interface Artifact {
|
|
24
|
+
id: string;
|
|
25
|
+
item_type: string;
|
|
26
|
+
item_count: number;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* ArtifactsPrepareResponse is an interface that defines the structure of the response from the prepare artifacts endpoint.
|
|
30
|
+
*/
|
|
31
|
+
export interface ArtifactsPrepareResponse {
|
|
32
|
+
url: string;
|
|
33
|
+
id: string;
|
|
34
|
+
form_data: {
|
|
35
|
+
key: string;
|
|
36
|
+
value: string;
|
|
37
|
+
}[];
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* ArtifactToUpload is an interface that defines the structure of the response from the get upload url endpoint.
|
|
41
|
+
*/
|
|
42
|
+
export interface ArtifactToUpload {
|
|
43
|
+
upload_url: string;
|
|
44
|
+
artifact_id: string;
|
|
45
|
+
form_data: {
|
|
46
|
+
key: string;
|
|
47
|
+
value: string;
|
|
48
|
+
}[];
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* UploadResponse is an interface that defines the structure of the response from upload through Uploader.
|
|
52
|
+
*/
|
|
53
|
+
export interface UploadResponse {
|
|
54
|
+
artifact?: Artifact;
|
|
55
|
+
error?: ErrorRecord;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* StreamAttachmentsResponse is an interface that defines the structure of the response from the stream attachments through Uploader.
|
|
59
|
+
*/
|
|
60
|
+
export interface StreamAttachmentsResponse {
|
|
61
|
+
ssorAttachments?: SsorAttachment[];
|
|
62
|
+
error?: ErrorRecord;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* StreamResponse is an interface that defines the structure of the response from the stream of single attachment through Uploader.
|
|
66
|
+
*/
|
|
67
|
+
export interface StreamResponse {
|
|
68
|
+
ssorAttachment?: SsorAttachment;
|
|
69
|
+
error?: ErrorRecord;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* SsorAttachment is an interface that defines the structure of the SSOR attachment.
|
|
73
|
+
*/
|
|
74
|
+
export interface SsorAttachment {
|
|
75
|
+
id: {
|
|
76
|
+
devrev: string;
|
|
77
|
+
external: string;
|
|
78
|
+
};
|
|
79
|
+
parent_id: {
|
|
80
|
+
external: string;
|
|
81
|
+
};
|
|
82
|
+
actor_id?: {
|
|
83
|
+
external: string;
|
|
84
|
+
};
|
|
85
|
+
inline?: boolean;
|
|
86
|
+
}
|
|
87
|
+
export interface StatsFileResponse {
|
|
88
|
+
error?: ErrorRecord;
|
|
89
|
+
statsFile?: StatsFileObject[];
|
|
90
|
+
}
|
|
91
|
+
export interface TransformerFileResponse {
|
|
92
|
+
error?: ErrorRecord;
|
|
93
|
+
transformerFile?: ExternalSystemItem[];
|
|
94
|
+
}
|
|
95
|
+
//# sourceMappingURL=uploader.interfaces.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"uploader.interfaces.d.ts","sourceRoot":"","sources":["../../src/uploader/uploader.interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,YAAY,CAAC;IACpB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED;;;GAGG;AACH,MAAM,MAAM,cAAc,CAAC,CAAC,IACxB;IAAE,QAAQ,EAAE,CAAC,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC9B;IAAE,QAAQ,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAC;AAEzC;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,EAAE,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,eAAe,CAAC,EAAE,cAAc,EAAE,CAAC;IACnC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE;QACF,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,QAAQ,CAAC,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,CAAC,EAAE,eAAe,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,uBAAuB;IACtC,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,eAAe,CAAC,EAAE,kBAAkB,EAAE,CAAC;CACxC"}
|