@aj-archipelago/cortex 1.3.57 → 1.3.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/config.js +22 -0
- package/helper-apps/cortex-file-handler/INTERFACE.md +20 -9
- package/helper-apps/cortex-file-handler/package-lock.json +2 -2
- package/helper-apps/cortex-file-handler/package.json +1 -1
- package/helper-apps/cortex-file-handler/scripts/setup-azure-container.js +17 -17
- package/helper-apps/cortex-file-handler/scripts/setup-test-containers.js +35 -35
- package/helper-apps/cortex-file-handler/src/blobHandler.js +1010 -909
- package/helper-apps/cortex-file-handler/src/constants.js +98 -98
- package/helper-apps/cortex-file-handler/src/docHelper.js +27 -27
- package/helper-apps/cortex-file-handler/src/fileChunker.js +224 -214
- package/helper-apps/cortex-file-handler/src/helper.js +93 -93
- package/helper-apps/cortex-file-handler/src/index.js +584 -550
- package/helper-apps/cortex-file-handler/src/localFileHandler.js +86 -86
- package/helper-apps/cortex-file-handler/src/redis.js +186 -90
- package/helper-apps/cortex-file-handler/src/services/ConversionService.js +301 -273
- package/helper-apps/cortex-file-handler/src/services/FileConversionService.js +55 -55
- package/helper-apps/cortex-file-handler/src/services/storage/AzureStorageProvider.js +174 -154
- package/helper-apps/cortex-file-handler/src/services/storage/GCSStorageProvider.js +239 -223
- package/helper-apps/cortex-file-handler/src/services/storage/LocalStorageProvider.js +161 -159
- package/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js +73 -71
- package/helper-apps/cortex-file-handler/src/services/storage/StorageProvider.js +46 -45
- package/helper-apps/cortex-file-handler/src/services/storage/StorageService.js +256 -213
- package/helper-apps/cortex-file-handler/src/start.js +4 -1
- package/helper-apps/cortex-file-handler/src/utils/filenameUtils.js +59 -25
- package/helper-apps/cortex-file-handler/tests/FileConversionService.test.js +119 -116
- package/helper-apps/cortex-file-handler/tests/blobHandler.test.js +257 -257
- package/helper-apps/cortex-file-handler/tests/cleanup.test.js +676 -0
- package/helper-apps/cortex-file-handler/tests/conversionResilience.test.js +124 -124
- package/helper-apps/cortex-file-handler/tests/fileChunker.test.js +249 -208
- package/helper-apps/cortex-file-handler/tests/fileUpload.test.js +439 -380
- package/helper-apps/cortex-file-handler/tests/getOperations.test.js +299 -263
- package/helper-apps/cortex-file-handler/tests/postOperations.test.js +265 -239
- package/helper-apps/cortex-file-handler/tests/start.test.js +1230 -1201
- package/helper-apps/cortex-file-handler/tests/storage/AzureStorageProvider.test.js +110 -105
- package/helper-apps/cortex-file-handler/tests/storage/GCSStorageProvider.test.js +201 -175
- package/helper-apps/cortex-file-handler/tests/storage/LocalStorageProvider.test.js +128 -125
- package/helper-apps/cortex-file-handler/tests/storage/StorageFactory.test.js +78 -73
- package/helper-apps/cortex-file-handler/tests/storage/StorageService.test.js +99 -99
- package/helper-apps/cortex-file-handler/tests/testUtils.helper.js +74 -70
- package/package.json +1 -1
- package/pathways/translate_apptek.js +33 -0
- package/pathways/translate_subtitle.js +15 -8
- package/server/plugins/apptekTranslatePlugin.js +46 -91
- package/tests/apptekTranslatePlugin.test.js +0 -2
- package/tests/integration/apptekTranslatePlugin.integration.test.js +159 -93
- package/tests/translate_apptek.test.js +16 -0
|
@@ -1,14 +1,17 @@
|
|
|
1
|
-
import test from
|
|
2
|
-
import fs from
|
|
3
|
-
import path from
|
|
4
|
-
import { fileURLToPath } from
|
|
5
|
-
import { v4 as uuidv4 } from
|
|
6
|
-
import axios from
|
|
7
|
-
import FormData from
|
|
8
|
-
import { port } from
|
|
9
|
-
import { gcs } from
|
|
10
|
-
import {
|
|
11
|
-
|
|
1
|
+
import test from "ava";
|
|
2
|
+
import fs from "fs";
|
|
3
|
+
import path from "path";
|
|
4
|
+
import { fileURLToPath } from "url";
|
|
5
|
+
import { v4 as uuidv4 } from "uuid";
|
|
6
|
+
import axios from "axios";
|
|
7
|
+
import FormData from "form-data";
|
|
8
|
+
import { port } from "../src/start.js";
|
|
9
|
+
import { gcs } from "../src/blobHandler.js";
|
|
10
|
+
import {
|
|
11
|
+
cleanupHashAndFile,
|
|
12
|
+
getFolderNameFromUrl,
|
|
13
|
+
} from "./testUtils.helper.js";
|
|
14
|
+
import XLSX from "xlsx";
|
|
12
15
|
|
|
13
16
|
const __filename = fileURLToPath(import.meta.url);
|
|
14
17
|
const __dirname = path.dirname(__filename);
|
|
@@ -16,407 +19,450 @@ const baseUrl = `http://localhost:${port}/api/CortexFileHandler`;
|
|
|
16
19
|
|
|
17
20
|
// Helper function to determine if GCS is configured
|
|
18
21
|
function isGCSConfigured() {
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
22
|
+
return (
|
|
23
|
+
process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 ||
|
|
24
|
+
process.env.GCP_SERVICE_ACCOUNT_KEY
|
|
25
|
+
);
|
|
23
26
|
}
|
|
24
27
|
|
|
25
28
|
// Helper function to create test files
|
|
26
29
|
async function createTestFile(content, extension) {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
30
|
+
const testDir = path.join(__dirname, "test-files");
|
|
31
|
+
if (!fs.existsSync(testDir)) {
|
|
32
|
+
fs.mkdirSync(testDir, { recursive: true });
|
|
33
|
+
}
|
|
34
|
+
const filename = path.join(testDir, `${uuidv4()}.${extension}`);
|
|
35
|
+
fs.writeFileSync(filename, content);
|
|
36
|
+
return filename;
|
|
34
37
|
}
|
|
35
38
|
|
|
36
39
|
// Helper function to upload file
|
|
37
40
|
async function uploadFile(filePath, requestId = null, hash = null) {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
41
|
+
const form = new FormData();
|
|
42
|
+
form.append("file", fs.createReadStream(filePath));
|
|
43
|
+
if (requestId) form.append("requestId", requestId);
|
|
44
|
+
if (hash) form.append("hash", hash);
|
|
45
|
+
|
|
46
|
+
const response = await axios.post(baseUrl, form, {
|
|
47
|
+
headers: {
|
|
48
|
+
...form.getHeaders(),
|
|
49
|
+
"Content-Type": "multipart/form-data",
|
|
50
|
+
},
|
|
51
|
+
validateStatus: (status) => true,
|
|
52
|
+
timeout: 30000,
|
|
53
|
+
maxContentLength: Infinity,
|
|
54
|
+
maxBodyLength: Infinity,
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
return response;
|
|
55
58
|
}
|
|
56
59
|
|
|
57
60
|
// Helper function to verify GCS file
|
|
58
61
|
async function verifyGCSFile(gcsUrl) {
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
62
|
+
if (!isGCSConfigured() || !gcs) return true;
|
|
63
|
+
|
|
64
|
+
try {
|
|
65
|
+
const bucket = gcsUrl.split("/")[2];
|
|
66
|
+
const filename = gcsUrl.split("/").slice(3).join("/");
|
|
67
|
+
const [exists] = await gcs.bucket(bucket).file(filename).exists();
|
|
68
|
+
return exists;
|
|
69
|
+
} catch (error) {
|
|
70
|
+
console.error("Error verifying GCS file:", error);
|
|
71
|
+
return false;
|
|
72
|
+
}
|
|
70
73
|
}
|
|
71
74
|
|
|
72
75
|
// Helper function to fetch file content from a URL
|
|
73
76
|
async function fetchFileContent(url) {
|
|
74
|
-
|
|
75
|
-
|
|
77
|
+
const response = await axios.get(url, { responseType: "arraybuffer" });
|
|
78
|
+
return Buffer.from(response.data);
|
|
76
79
|
}
|
|
77
80
|
|
|
78
81
|
// Setup: Create test directory
|
|
79
82
|
test.before(async (t) => {
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
+
const testDir = path.join(__dirname, "test-files");
|
|
84
|
+
await fs.promises.mkdir(testDir, { recursive: true });
|
|
85
|
+
t.context = { testDir };
|
|
83
86
|
});
|
|
84
87
|
|
|
85
88
|
// Cleanup
|
|
86
89
|
test.after.always(async (t) => {
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
}
|
|
90
|
+
// Clean up test directory
|
|
91
|
+
await fs.promises.rm(t.context.testDir, { recursive: true, force: true });
|
|
92
|
+
|
|
93
|
+
// Clean up any remaining files in the files directory
|
|
94
|
+
const filesDir = path.join(__dirname, "..", "files");
|
|
95
|
+
if (fs.existsSync(filesDir)) {
|
|
96
|
+
const dirs = await fs.promises.readdir(filesDir);
|
|
97
|
+
for (const dir of dirs) {
|
|
98
|
+
const dirPath = path.join(filesDir, dir);
|
|
99
|
+
try {
|
|
100
|
+
await fs.promises.rm(dirPath, { recursive: true, force: true });
|
|
101
|
+
} catch (e) {
|
|
102
|
+
console.error("Error cleaning up directory:", {
|
|
103
|
+
dir: dirPath,
|
|
104
|
+
error: e.message,
|
|
105
|
+
});
|
|
106
|
+
}
|
|
105
107
|
}
|
|
108
|
+
}
|
|
106
109
|
});
|
|
107
110
|
|
|
108
111
|
// Basic File Upload Tests
|
|
109
|
-
test.serial(
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
112
|
+
test.serial("should handle basic file upload", async (t) => {
|
|
113
|
+
const fileContent = "test content";
|
|
114
|
+
const filePath = await createTestFile(fileContent, "txt");
|
|
115
|
+
const requestId = uuidv4();
|
|
116
|
+
let response;
|
|
117
|
+
|
|
118
|
+
try {
|
|
119
|
+
response = await uploadFile(filePath, requestId);
|
|
120
|
+
|
|
121
|
+
t.is(response.status, 200);
|
|
122
|
+
t.truthy(response.data.url);
|
|
123
|
+
t.truthy(response.data.filename);
|
|
124
|
+
|
|
125
|
+
// Verify file content matches
|
|
126
|
+
const uploadedContent = await fetchFileContent(response.data.url);
|
|
127
|
+
t.deepEqual(
|
|
128
|
+
uploadedContent,
|
|
129
|
+
Buffer.from(fileContent),
|
|
130
|
+
"Uploaded file content should match",
|
|
131
|
+
);
|
|
132
|
+
} finally {
|
|
133
|
+
fs.unlinkSync(filePath);
|
|
134
|
+
if (response?.data?.url) {
|
|
135
|
+
await cleanupHashAndFile(null, response.data.url, baseUrl);
|
|
130
136
|
}
|
|
137
|
+
}
|
|
131
138
|
});
|
|
132
139
|
|
|
133
|
-
test.serial(
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
140
|
+
test.serial("should handle file upload with hash", async (t) => {
|
|
141
|
+
const fileContent = "test content";
|
|
142
|
+
const filePath = await createTestFile(fileContent, "txt");
|
|
143
|
+
const requestId = uuidv4();
|
|
144
|
+
const hash = "test-hash-" + uuidv4();
|
|
145
|
+
let uploadedUrl;
|
|
146
|
+
let convertedUrl;
|
|
147
|
+
let response;
|
|
148
|
+
|
|
149
|
+
try {
|
|
150
|
+
// First upload the file
|
|
151
|
+
response = await uploadFile(filePath, requestId, hash);
|
|
152
|
+
t.is(response.status, 200);
|
|
153
|
+
t.truthy(response.data.url);
|
|
154
|
+
uploadedUrl = response.data.url;
|
|
155
|
+
if (response.data.converted && response.data.converted.url) {
|
|
156
|
+
convertedUrl = response.data.converted.url;
|
|
157
|
+
}
|
|
158
|
+
console.log("Upload hash response.data", response.data);
|
|
159
|
+
|
|
160
|
+
// Wait for Redis operations to complete and verify storage
|
|
161
|
+
await new Promise((resolve) => setTimeout(resolve, 2000));
|
|
162
|
+
|
|
163
|
+
const checkResponse = await axios.get(baseUrl, {
|
|
164
|
+
params: {
|
|
165
|
+
hash,
|
|
166
|
+
checkHash: true,
|
|
167
|
+
},
|
|
168
|
+
validateStatus: (status) => true,
|
|
169
|
+
});
|
|
170
|
+
console.log("Upload hash checkResponse", checkResponse);
|
|
171
|
+
if (checkResponse.status !== 200) {
|
|
172
|
+
// Only log if not 200
|
|
173
|
+
console.error("Hash check failed:", {
|
|
174
|
+
status: checkResponse.status,
|
|
175
|
+
data: checkResponse.data,
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
// Hash should exist since we just uploaded it
|
|
179
|
+
t.is(checkResponse.status, 200);
|
|
180
|
+
t.truthy(checkResponse.data.hash);
|
|
181
|
+
|
|
182
|
+
// Verify file exists and content matches
|
|
183
|
+
const fileResponse = await axios.get(response.data.url, {
|
|
184
|
+
responseType: "arraybuffer",
|
|
185
|
+
});
|
|
186
|
+
t.is(fileResponse.status, 200);
|
|
187
|
+
t.deepEqual(
|
|
188
|
+
Buffer.from(fileResponse.data),
|
|
189
|
+
Buffer.from(fileContent),
|
|
190
|
+
"Uploaded file content should match",
|
|
191
|
+
);
|
|
192
|
+
} finally {
|
|
193
|
+
fs.unlinkSync(filePath);
|
|
194
|
+
if (uploadedUrl) {
|
|
195
|
+
await cleanupHashAndFile(hash, uploadedUrl, baseUrl);
|
|
187
196
|
}
|
|
197
|
+
if (convertedUrl) {
|
|
198
|
+
await cleanupHashAndFile(null, convertedUrl, baseUrl);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
188
201
|
});
|
|
189
202
|
|
|
190
203
|
// Document Processing Tests
|
|
191
|
-
test.serial(
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
204
|
+
test.serial("should handle PDF document upload and conversion", async (t) => {
|
|
205
|
+
// Create a simple PDF file
|
|
206
|
+
const fileContent = "%PDF-1.4\nTest PDF content";
|
|
207
|
+
const filePath = await createTestFile(fileContent, "pdf");
|
|
208
|
+
const requestId = uuidv4();
|
|
209
|
+
let response;
|
|
210
|
+
|
|
211
|
+
try {
|
|
212
|
+
response = await uploadFile(filePath, requestId);
|
|
213
|
+
t.is(response.status, 200);
|
|
214
|
+
t.truthy(response.data.url);
|
|
215
|
+
|
|
216
|
+
// Verify original PDF content matches
|
|
217
|
+
const uploadedContent = await fetchFileContent(response.data.url);
|
|
218
|
+
t.deepEqual(
|
|
219
|
+
uploadedContent,
|
|
220
|
+
Buffer.from(fileContent),
|
|
221
|
+
"Uploaded PDF content should match",
|
|
222
|
+
);
|
|
223
|
+
|
|
224
|
+
// Check if converted version exists
|
|
225
|
+
if (response.data.converted) {
|
|
226
|
+
t.truthy(response.data.converted.url);
|
|
227
|
+
const convertedResponse = await axios.get(response.data.converted.url, {
|
|
228
|
+
responseType: "arraybuffer",
|
|
229
|
+
});
|
|
230
|
+
t.is(convertedResponse.status, 200);
|
|
231
|
+
// For conversion, just check non-empty
|
|
232
|
+
t.true(
|
|
233
|
+
Buffer.from(convertedResponse.data).length > 0,
|
|
234
|
+
"Converted file should not be empty",
|
|
235
|
+
);
|
|
236
|
+
}
|
|
237
|
+
} finally {
|
|
238
|
+
fs.unlinkSync(filePath);
|
|
239
|
+
if (response?.data?.url) {
|
|
240
|
+
await cleanupHashAndFile(null, response.data.url, baseUrl);
|
|
223
241
|
}
|
|
242
|
+
if (response?.data?.converted?.url) {
|
|
243
|
+
await cleanupHashAndFile(null, response.data.converted.url, baseUrl);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
224
246
|
});
|
|
225
247
|
|
|
226
248
|
// Media Chunking Tests
|
|
227
|
-
test.serial(
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
249
|
+
test.serial("should handle media file chunking", async (t) => {
|
|
250
|
+
// Create a large test file to trigger chunking
|
|
251
|
+
const chunkContent = "x".repeat(1024 * 1024);
|
|
252
|
+
const filePath = await createTestFile(chunkContent, "mp4");
|
|
253
|
+
const requestId = uuidv4();
|
|
254
|
+
let response;
|
|
255
|
+
|
|
256
|
+
try {
|
|
257
|
+
response = await uploadFile(filePath, requestId);
|
|
258
|
+
t.is(response.status, 200);
|
|
259
|
+
t.truthy(response.data);
|
|
260
|
+
|
|
261
|
+
// For media files, we expect either an array of chunks or a single URL
|
|
262
|
+
if (Array.isArray(response.data)) {
|
|
263
|
+
t.true(response.data.length > 0);
|
|
264
|
+
|
|
265
|
+
// Verify each chunk
|
|
266
|
+
for (const chunk of response.data) {
|
|
267
|
+
t.truthy(chunk.uri);
|
|
268
|
+
t.truthy(chunk.offset);
|
|
269
|
+
|
|
270
|
+
// Verify chunk exists and content matches
|
|
271
|
+
const chunkResponse = await axios.get(chunk.uri, {
|
|
272
|
+
responseType: "arraybuffer",
|
|
273
|
+
});
|
|
274
|
+
t.is(chunkResponse.status, 200);
|
|
275
|
+
// Each chunk should be a slice of the original content
|
|
276
|
+
const expectedChunk = Buffer.from(chunkContent).slice(
|
|
277
|
+
chunk.offset,
|
|
278
|
+
chunk.offset + chunk.length || undefined,
|
|
279
|
+
);
|
|
280
|
+
t.deepEqual(
|
|
281
|
+
Buffer.from(chunkResponse.data),
|
|
282
|
+
expectedChunk,
|
|
283
|
+
"Chunk content should match original",
|
|
284
|
+
);
|
|
285
|
+
|
|
286
|
+
// If GCS is configured, verify backup
|
|
287
|
+
if (isGCSConfigured() && chunk.gcs) {
|
|
288
|
+
const exists = await verifyGCSFile(chunk.gcs);
|
|
289
|
+
t.true(exists, "GCS chunk should exist");
|
|
267
290
|
}
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
291
|
+
}
|
|
292
|
+
} else {
|
|
293
|
+
// Single file response
|
|
294
|
+
t.truthy(response.data.url);
|
|
295
|
+
const fileResponse = await axios.get(response.data.url, {
|
|
296
|
+
responseType: "arraybuffer",
|
|
297
|
+
});
|
|
298
|
+
t.is(fileResponse.status, 200);
|
|
299
|
+
t.deepEqual(
|
|
300
|
+
Buffer.from(fileResponse.data),
|
|
301
|
+
Buffer.from(chunkContent),
|
|
302
|
+
"Uploaded file content should match",
|
|
303
|
+
);
|
|
304
|
+
}
|
|
305
|
+
} finally {
|
|
306
|
+
fs.unlinkSync(filePath);
|
|
307
|
+
if (response?.data) {
|
|
308
|
+
if (Array.isArray(response.data)) {
|
|
309
|
+
for (const chunk of response.data) {
|
|
310
|
+
if (chunk.uri) {
|
|
311
|
+
await cleanupHashAndFile(null, chunk.uri, baseUrl);
|
|
312
|
+
}
|
|
280
313
|
}
|
|
314
|
+
} else if (response.data.url) {
|
|
315
|
+
await cleanupHashAndFile(null, response.data.url, baseUrl);
|
|
316
|
+
}
|
|
281
317
|
}
|
|
318
|
+
}
|
|
282
319
|
});
|
|
283
320
|
|
|
284
321
|
// Error Handling Tests
|
|
285
|
-
test.serial(
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
322
|
+
test.serial("should handle invalid file upload", async (t) => {
|
|
323
|
+
const requestId = uuidv4();
|
|
324
|
+
const form = new FormData();
|
|
325
|
+
// Send a file with no name and no content
|
|
326
|
+
form.append("file", Buffer.from(""), { filename: "" });
|
|
327
|
+
form.append("requestId", requestId);
|
|
328
|
+
|
|
329
|
+
const response = await axios.post(baseUrl, form, {
|
|
330
|
+
headers: {
|
|
331
|
+
...form.getHeaders(),
|
|
332
|
+
"Content-Type": "multipart/form-data",
|
|
333
|
+
},
|
|
334
|
+
validateStatus: (status) => true,
|
|
335
|
+
timeout: 30000,
|
|
336
|
+
});
|
|
337
|
+
|
|
338
|
+
// Log the response for debugging
|
|
339
|
+
console.log("Invalid file upload response:", {
|
|
340
|
+
status: response.status,
|
|
341
|
+
data: response.data,
|
|
342
|
+
});
|
|
343
|
+
|
|
344
|
+
t.is(response.status, 400, "Should reject invalid file with 400 status");
|
|
345
|
+
t.is(
|
|
346
|
+
response.data,
|
|
347
|
+
"Invalid file: missing filename",
|
|
348
|
+
"Should return correct error message",
|
|
349
|
+
);
|
|
309
350
|
});
|
|
310
351
|
|
|
311
352
|
// Cleanup Tests
|
|
312
|
-
test.serial(
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
}
|
|
348
|
-
} finally {
|
|
349
|
-
fs.unlinkSync(filePath);
|
|
353
|
+
test.serial("should handle file deletion", async (t) => {
|
|
354
|
+
const filePath = await createTestFile("test content", "txt");
|
|
355
|
+
const requestId = uuidv4();
|
|
356
|
+
|
|
357
|
+
try {
|
|
358
|
+
// Upload file
|
|
359
|
+
const uploadResponse = await uploadFile(filePath, requestId);
|
|
360
|
+
t.is(uploadResponse.status, 200);
|
|
361
|
+
|
|
362
|
+
// Wait a moment for file to be fully written
|
|
363
|
+
await new Promise((resolve) => setTimeout(resolve, 1000));
|
|
364
|
+
|
|
365
|
+
// Extract the file identifier from the URL
|
|
366
|
+
const fileIdentifier = getFolderNameFromUrl(uploadResponse.data.url);
|
|
367
|
+
console.log("File identifier for deletion:", fileIdentifier);
|
|
368
|
+
|
|
369
|
+
// Delete file using the correct identifier
|
|
370
|
+
const deleteUrl = `${baseUrl}?operation=delete&requestId=${fileIdentifier}`;
|
|
371
|
+
console.log("Deleting file with URL:", deleteUrl);
|
|
372
|
+
const deleteResponse = await axios.delete(deleteUrl);
|
|
373
|
+
t.is(deleteResponse.status, 200);
|
|
374
|
+
|
|
375
|
+
// Wait a moment for deletion to complete
|
|
376
|
+
await new Promise((resolve) => setTimeout(resolve, 1000));
|
|
377
|
+
|
|
378
|
+
// Verify file is gone
|
|
379
|
+
const verifyResponse = await axios.get(uploadResponse.data.url, {
|
|
380
|
+
validateStatus: (status) => true,
|
|
381
|
+
});
|
|
382
|
+
t.is(verifyResponse.status, 404, "File should be deleted");
|
|
383
|
+
|
|
384
|
+
// If GCS is configured, verify backup is gone
|
|
385
|
+
if (isGCSConfigured() && uploadResponse.data.gcs) {
|
|
386
|
+
const exists = await verifyGCSFile(uploadResponse.data.gcs);
|
|
387
|
+
t.false(exists, "GCS file should be deleted");
|
|
350
388
|
}
|
|
389
|
+
} finally {
|
|
390
|
+
fs.unlinkSync(filePath);
|
|
391
|
+
}
|
|
351
392
|
});
|
|
352
393
|
|
|
353
394
|
// Save Option Test
|
|
354
|
-
test.serial(
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
395
|
+
test.serial("should handle document upload with save option", async (t) => {
|
|
396
|
+
// Create a minimal XLSX workbook in-memory
|
|
397
|
+
const workbook = XLSX.utils.book_new();
|
|
398
|
+
const worksheet = XLSX.utils.aoa_to_sheet([
|
|
399
|
+
["Name", "Score"],
|
|
400
|
+
["Alice", 10],
|
|
401
|
+
["Bob", 8],
|
|
402
|
+
]);
|
|
403
|
+
XLSX.utils.book_append_sheet(workbook, worksheet, "Sheet1");
|
|
404
|
+
|
|
405
|
+
// Write it to a temp file inside the test directory
|
|
406
|
+
const filePath = path.join(t.context.testDir, `${uuidv4()}.xlsx`);
|
|
407
|
+
XLSX.writeFile(workbook, filePath);
|
|
408
|
+
|
|
409
|
+
const initialRequestId = uuidv4();
|
|
410
|
+
const saveRequestId = uuidv4();
|
|
411
|
+
|
|
412
|
+
let uploadedUrl;
|
|
413
|
+
let savedUrl;
|
|
414
|
+
|
|
415
|
+
try {
|
|
416
|
+
// First, upload the document so we have a publicly reachable URL
|
|
417
|
+
const uploadResponse = await uploadFile(filePath, initialRequestId);
|
|
418
|
+
t.is(uploadResponse.status, 200);
|
|
419
|
+
t.truthy(uploadResponse.data.url, "Upload should return a URL");
|
|
420
|
+
|
|
421
|
+
uploadedUrl = uploadResponse.data.url;
|
|
422
|
+
|
|
423
|
+
// Now call the handler again with the save flag
|
|
424
|
+
const saveResponse = await axios.get(baseUrl, {
|
|
425
|
+
params: {
|
|
426
|
+
uri: uploadedUrl,
|
|
427
|
+
requestId: saveRequestId,
|
|
428
|
+
save: true,
|
|
429
|
+
},
|
|
430
|
+
validateStatus: (status) => true,
|
|
431
|
+
});
|
|
391
432
|
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
433
|
+
// The save operation should return a 200 status with a result object
|
|
434
|
+
t.is(saveResponse.status, 200, "Save request should succeed");
|
|
435
|
+
t.truthy(saveResponse.data, "Response should have data");
|
|
436
|
+
t.truthy(saveResponse.data.url, "Response should include a URL");
|
|
437
|
+
t.true(
|
|
438
|
+
saveResponse.data.url.includes(".csv"),
|
|
439
|
+
"Response should include a CSV URL",
|
|
440
|
+
);
|
|
441
|
+
savedUrl = saveResponse.data.url;
|
|
442
|
+
} finally {
|
|
443
|
+
fs.unlinkSync(filePath);
|
|
444
|
+
// Clean up both URLs
|
|
445
|
+
if (uploadedUrl) {
|
|
446
|
+
await cleanupHashAndFile(null, uploadedUrl, baseUrl);
|
|
447
|
+
}
|
|
448
|
+
if (savedUrl && savedUrl !== uploadedUrl) {
|
|
449
|
+
await cleanupHashAndFile(null, savedUrl, baseUrl);
|
|
407
450
|
}
|
|
451
|
+
}
|
|
408
452
|
});
|
|
409
453
|
|
|
410
454
|
// Converted file persistence test – ensures needsConversion works for extension-only checks
|
|
411
|
-
test.serial(
|
|
455
|
+
test.serial(
|
|
456
|
+
"should preserve converted version when checking hash for convertible file",
|
|
457
|
+
async (t) => {
|
|
412
458
|
// Create a minimal XLSX workbook in-memory
|
|
413
459
|
const workbook = XLSX.utils.book_new();
|
|
414
460
|
const worksheet = XLSX.utils.aoa_to_sheet([
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
461
|
+
["Name", "Score"],
|
|
462
|
+
["Alice", 10],
|
|
463
|
+
["Bob", 8],
|
|
418
464
|
]);
|
|
419
|
-
XLSX.utils.book_append_sheet(workbook, worksheet,
|
|
465
|
+
XLSX.utils.book_append_sheet(workbook, worksheet, "Sheet1");
|
|
420
466
|
|
|
421
467
|
// Write it to a temp file inside the test directory
|
|
422
468
|
const filePath = path.join(t.context.testDir, `${uuidv4()}.xlsx`);
|
|
@@ -429,34 +475,47 @@ test.serial('should preserve converted version when checking hash for convertibl
|
|
|
429
475
|
let convertedUrl;
|
|
430
476
|
|
|
431
477
|
try {
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
478
|
+
// 1. Upload the XLSX file (conversion should run automatically)
|
|
479
|
+
const uploadResponse = await uploadFile(filePath, requestId, hash);
|
|
480
|
+
t.is(uploadResponse.status, 200, "Upload should succeed");
|
|
481
|
+
t.truthy(
|
|
482
|
+
uploadResponse.data.converted,
|
|
483
|
+
"Upload response must contain converted info",
|
|
484
|
+
);
|
|
485
|
+
t.truthy(
|
|
486
|
+
uploadResponse.data.converted.url,
|
|
487
|
+
"Converted URL should be present",
|
|
488
|
+
);
|
|
489
|
+
|
|
490
|
+
uploadedUrl = uploadResponse.data.url;
|
|
491
|
+
convertedUrl = uploadResponse.data.converted.url;
|
|
492
|
+
|
|
493
|
+
// 2. Give Redis a moment to persist
|
|
494
|
+
await new Promise((resolve) => setTimeout(resolve, 4000));
|
|
495
|
+
|
|
496
|
+
// 3. Ask the handler for the hash – it will invoke ensureConvertedVersion
|
|
497
|
+
const checkResponse = await axios.get(baseUrl, {
|
|
498
|
+
params: { hash, checkHash: true },
|
|
499
|
+
validateStatus: (status) => true,
|
|
500
|
+
timeout: 30000,
|
|
501
|
+
});
|
|
502
|
+
|
|
503
|
+
t.is(checkResponse.status, 200, "Hash check should succeed");
|
|
504
|
+
t.truthy(
|
|
505
|
+
checkResponse.data.converted,
|
|
506
|
+
"Hash response should include converted info",
|
|
507
|
+
);
|
|
508
|
+
t.truthy(
|
|
509
|
+
checkResponse.data.converted.url,
|
|
510
|
+
"Converted URL should still be present after hash check",
|
|
511
|
+
);
|
|
454
512
|
} finally {
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
513
|
+
// Clean up temp file and remote artifacts
|
|
514
|
+
fs.unlinkSync(filePath);
|
|
515
|
+
await cleanupHashAndFile(hash, uploadedUrl, baseUrl);
|
|
516
|
+
if (convertedUrl) {
|
|
517
|
+
await cleanupHashAndFile(null, convertedUrl, baseUrl);
|
|
518
|
+
}
|
|
461
519
|
}
|
|
462
|
-
}
|
|
520
|
+
},
|
|
521
|
+
);
|