@aj-archipelago/cortex 1.3.57 → 1.3.59

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +6 -0
  2. package/config.js +22 -0
  3. package/helper-apps/cortex-file-handler/INTERFACE.md +20 -9
  4. package/helper-apps/cortex-file-handler/package-lock.json +2 -2
  5. package/helper-apps/cortex-file-handler/package.json +1 -1
  6. package/helper-apps/cortex-file-handler/scripts/setup-azure-container.js +17 -17
  7. package/helper-apps/cortex-file-handler/scripts/setup-test-containers.js +35 -35
  8. package/helper-apps/cortex-file-handler/src/blobHandler.js +1010 -909
  9. package/helper-apps/cortex-file-handler/src/constants.js +98 -98
  10. package/helper-apps/cortex-file-handler/src/docHelper.js +27 -27
  11. package/helper-apps/cortex-file-handler/src/fileChunker.js +224 -214
  12. package/helper-apps/cortex-file-handler/src/helper.js +93 -93
  13. package/helper-apps/cortex-file-handler/src/index.js +584 -550
  14. package/helper-apps/cortex-file-handler/src/localFileHandler.js +86 -86
  15. package/helper-apps/cortex-file-handler/src/redis.js +186 -90
  16. package/helper-apps/cortex-file-handler/src/services/ConversionService.js +301 -273
  17. package/helper-apps/cortex-file-handler/src/services/FileConversionService.js +55 -55
  18. package/helper-apps/cortex-file-handler/src/services/storage/AzureStorageProvider.js +174 -154
  19. package/helper-apps/cortex-file-handler/src/services/storage/GCSStorageProvider.js +239 -223
  20. package/helper-apps/cortex-file-handler/src/services/storage/LocalStorageProvider.js +161 -159
  21. package/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js +73 -71
  22. package/helper-apps/cortex-file-handler/src/services/storage/StorageProvider.js +46 -45
  23. package/helper-apps/cortex-file-handler/src/services/storage/StorageService.js +256 -213
  24. package/helper-apps/cortex-file-handler/src/start.js +4 -1
  25. package/helper-apps/cortex-file-handler/src/utils/filenameUtils.js +59 -25
  26. package/helper-apps/cortex-file-handler/tests/FileConversionService.test.js +119 -116
  27. package/helper-apps/cortex-file-handler/tests/blobHandler.test.js +257 -257
  28. package/helper-apps/cortex-file-handler/tests/cleanup.test.js +676 -0
  29. package/helper-apps/cortex-file-handler/tests/conversionResilience.test.js +124 -124
  30. package/helper-apps/cortex-file-handler/tests/fileChunker.test.js +249 -208
  31. package/helper-apps/cortex-file-handler/tests/fileUpload.test.js +439 -380
  32. package/helper-apps/cortex-file-handler/tests/getOperations.test.js +299 -263
  33. package/helper-apps/cortex-file-handler/tests/postOperations.test.js +265 -239
  34. package/helper-apps/cortex-file-handler/tests/start.test.js +1230 -1201
  35. package/helper-apps/cortex-file-handler/tests/storage/AzureStorageProvider.test.js +110 -105
  36. package/helper-apps/cortex-file-handler/tests/storage/GCSStorageProvider.test.js +201 -175
  37. package/helper-apps/cortex-file-handler/tests/storage/LocalStorageProvider.test.js +128 -125
  38. package/helper-apps/cortex-file-handler/tests/storage/StorageFactory.test.js +78 -73
  39. package/helper-apps/cortex-file-handler/tests/storage/StorageService.test.js +99 -99
  40. package/helper-apps/cortex-file-handler/tests/testUtils.helper.js +74 -70
  41. package/package.json +1 -1
  42. package/pathways/translate_apptek.js +33 -0
  43. package/pathways/translate_subtitle.js +15 -8
  44. package/server/plugins/apptekTranslatePlugin.js +46 -91
  45. package/tests/apptekTranslatePlugin.test.js +0 -2
  46. package/tests/integration/apptekTranslatePlugin.integration.test.js +159 -93
  47. package/tests/translate_apptek.test.js +16 -0
@@ -1,14 +1,17 @@
1
- import test from 'ava';
2
- import fs from 'fs';
3
- import path from 'path';
4
- import { fileURLToPath } from 'url';
5
- import { v4 as uuidv4 } from 'uuid';
6
- import axios from 'axios';
7
- import FormData from 'form-data';
8
- import { port } from '../src/start.js';
9
- import { gcs } from '../src/blobHandler.js';
10
- import { cleanupHashAndFile, getFolderNameFromUrl } from './testUtils.helper.js';
11
- import XLSX from 'xlsx';
1
+ import test from "ava";
2
+ import fs from "fs";
3
+ import path from "path";
4
+ import { fileURLToPath } from "url";
5
+ import { v4 as uuidv4 } from "uuid";
6
+ import axios from "axios";
7
+ import FormData from "form-data";
8
+ import { port } from "../src/start.js";
9
+ import { gcs } from "../src/blobHandler.js";
10
+ import {
11
+ cleanupHashAndFile,
12
+ getFolderNameFromUrl,
13
+ } from "./testUtils.helper.js";
14
+ import XLSX from "xlsx";
12
15
 
13
16
  const __filename = fileURLToPath(import.meta.url);
14
17
  const __dirname = path.dirname(__filename);
@@ -16,407 +19,450 @@ const baseUrl = `http://localhost:${port}/api/CortexFileHandler`;
16
19
 
17
20
  // Helper function to determine if GCS is configured
18
21
  function isGCSConfigured() {
19
- return (
20
- process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 ||
21
- process.env.GCP_SERVICE_ACCOUNT_KEY
22
- );
22
+ return (
23
+ process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 ||
24
+ process.env.GCP_SERVICE_ACCOUNT_KEY
25
+ );
23
26
  }
24
27
 
25
28
  // Helper function to create test files
26
29
  async function createTestFile(content, extension) {
27
- const testDir = path.join(__dirname, 'test-files');
28
- if (!fs.existsSync(testDir)) {
29
- fs.mkdirSync(testDir, { recursive: true });
30
- }
31
- const filename = path.join(testDir, `${uuidv4()}.${extension}`);
32
- fs.writeFileSync(filename, content);
33
- return filename;
30
+ const testDir = path.join(__dirname, "test-files");
31
+ if (!fs.existsSync(testDir)) {
32
+ fs.mkdirSync(testDir, { recursive: true });
33
+ }
34
+ const filename = path.join(testDir, `${uuidv4()}.${extension}`);
35
+ fs.writeFileSync(filename, content);
36
+ return filename;
34
37
  }
35
38
 
36
39
  // Helper function to upload file
37
40
  async function uploadFile(filePath, requestId = null, hash = null) {
38
- const form = new FormData();
39
- form.append('file', fs.createReadStream(filePath));
40
- if (requestId) form.append('requestId', requestId);
41
- if (hash) form.append('hash', hash);
42
-
43
- const response = await axios.post(baseUrl, form, {
44
- headers: {
45
- ...form.getHeaders(),
46
- 'Content-Type': 'multipart/form-data',
47
- },
48
- validateStatus: (status) => true,
49
- timeout: 30000,
50
- maxContentLength: Infinity,
51
- maxBodyLength: Infinity,
52
- });
53
-
54
- return response;
41
+ const form = new FormData();
42
+ form.append("file", fs.createReadStream(filePath));
43
+ if (requestId) form.append("requestId", requestId);
44
+ if (hash) form.append("hash", hash);
45
+
46
+ const response = await axios.post(baseUrl, form, {
47
+ headers: {
48
+ ...form.getHeaders(),
49
+ "Content-Type": "multipart/form-data",
50
+ },
51
+ validateStatus: (status) => true,
52
+ timeout: 30000,
53
+ maxContentLength: Infinity,
54
+ maxBodyLength: Infinity,
55
+ });
56
+
57
+ return response;
55
58
  }
56
59
 
57
60
  // Helper function to verify GCS file
58
61
  async function verifyGCSFile(gcsUrl) {
59
- if (!isGCSConfigured() || !gcs) return true;
60
-
61
- try {
62
- const bucket = gcsUrl.split('/')[2];
63
- const filename = gcsUrl.split('/').slice(3).join('/');
64
- const [exists] = await gcs.bucket(bucket).file(filename).exists();
65
- return exists;
66
- } catch (error) {
67
- console.error('Error verifying GCS file:', error);
68
- return false;
69
- }
62
+ if (!isGCSConfigured() || !gcs) return true;
63
+
64
+ try {
65
+ const bucket = gcsUrl.split("/")[2];
66
+ const filename = gcsUrl.split("/").slice(3).join("/");
67
+ const [exists] = await gcs.bucket(bucket).file(filename).exists();
68
+ return exists;
69
+ } catch (error) {
70
+ console.error("Error verifying GCS file:", error);
71
+ return false;
72
+ }
70
73
  }
71
74
 
72
75
  // Helper function to fetch file content from a URL
73
76
  async function fetchFileContent(url) {
74
- const response = await axios.get(url, { responseType: 'arraybuffer' });
75
- return Buffer.from(response.data);
77
+ const response = await axios.get(url, { responseType: "arraybuffer" });
78
+ return Buffer.from(response.data);
76
79
  }
77
80
 
78
81
  // Setup: Create test directory
79
82
  test.before(async (t) => {
80
- const testDir = path.join(__dirname, 'test-files');
81
- await fs.promises.mkdir(testDir, { recursive: true });
82
- t.context = { testDir };
83
+ const testDir = path.join(__dirname, "test-files");
84
+ await fs.promises.mkdir(testDir, { recursive: true });
85
+ t.context = { testDir };
83
86
  });
84
87
 
85
88
  // Cleanup
86
89
  test.after.always(async (t) => {
87
- // Clean up test directory
88
- await fs.promises.rm(t.context.testDir, { recursive: true, force: true });
89
-
90
- // Clean up any remaining files in the files directory
91
- const filesDir = path.join(__dirname, '..', 'files');
92
- if (fs.existsSync(filesDir)) {
93
- const dirs = await fs.promises.readdir(filesDir);
94
- for (const dir of dirs) {
95
- const dirPath = path.join(filesDir, dir);
96
- try {
97
- await fs.promises.rm(dirPath, { recursive: true, force: true });
98
- } catch (e) {
99
- console.error('Error cleaning up directory:', {
100
- dir: dirPath,
101
- error: e.message
102
- });
103
- }
104
- }
90
+ // Clean up test directory
91
+ await fs.promises.rm(t.context.testDir, { recursive: true, force: true });
92
+
93
+ // Clean up any remaining files in the files directory
94
+ const filesDir = path.join(__dirname, "..", "files");
95
+ if (fs.existsSync(filesDir)) {
96
+ const dirs = await fs.promises.readdir(filesDir);
97
+ for (const dir of dirs) {
98
+ const dirPath = path.join(filesDir, dir);
99
+ try {
100
+ await fs.promises.rm(dirPath, { recursive: true, force: true });
101
+ } catch (e) {
102
+ console.error("Error cleaning up directory:", {
103
+ dir: dirPath,
104
+ error: e.message,
105
+ });
106
+ }
105
107
  }
108
+ }
106
109
  });
107
110
 
108
111
  // Basic File Upload Tests
109
- test.serial('should handle basic file upload', async (t) => {
110
- const fileContent = 'test content';
111
- const filePath = await createTestFile(fileContent, 'txt');
112
- const requestId = uuidv4();
113
- let response;
114
-
115
- try {
116
- response = await uploadFile(filePath, requestId);
117
-
118
- t.is(response.status, 200);
119
- t.truthy(response.data.url);
120
- t.truthy(response.data.filename);
121
-
122
- // Verify file content matches
123
- const uploadedContent = await fetchFileContent(response.data.url);
124
- t.deepEqual(uploadedContent, Buffer.from(fileContent), 'Uploaded file content should match');
125
- } finally {
126
- fs.unlinkSync(filePath);
127
- if (response?.data?.url) {
128
- await cleanupHashAndFile(null, response.data.url, baseUrl);
129
- }
112
+ test.serial("should handle basic file upload", async (t) => {
113
+ const fileContent = "test content";
114
+ const filePath = await createTestFile(fileContent, "txt");
115
+ const requestId = uuidv4();
116
+ let response;
117
+
118
+ try {
119
+ response = await uploadFile(filePath, requestId);
120
+
121
+ t.is(response.status, 200);
122
+ t.truthy(response.data.url);
123
+ t.truthy(response.data.filename);
124
+
125
+ // Verify file content matches
126
+ const uploadedContent = await fetchFileContent(response.data.url);
127
+ t.deepEqual(
128
+ uploadedContent,
129
+ Buffer.from(fileContent),
130
+ "Uploaded file content should match",
131
+ );
132
+ } finally {
133
+ fs.unlinkSync(filePath);
134
+ if (response?.data?.url) {
135
+ await cleanupHashAndFile(null, response.data.url, baseUrl);
130
136
  }
137
+ }
131
138
  });
132
139
 
133
- test.serial('should handle file upload with hash', async (t) => {
134
- const fileContent = 'test content';
135
- const filePath = await createTestFile(fileContent, 'txt');
136
- const requestId = uuidv4();
137
- const hash = 'test-hash-' + uuidv4();
138
- let uploadedUrl;
139
- let convertedUrl;
140
- let response;
141
-
142
- try {
143
- // First upload the file
144
- response = await uploadFile(filePath, requestId, hash);
145
- t.is(response.status, 200);
146
- t.truthy(response.data.url);
147
- uploadedUrl = response.data.url;
148
- if (response.data.converted && response.data.converted.url) {
149
- convertedUrl = response.data.converted.url;
150
- }
151
- console.log('Upload hash response.data', response.data)
152
-
153
- // Wait for Redis operations to complete and verify storage
154
- await new Promise(resolve => setTimeout(resolve, 2000));
155
-
156
- const checkResponse = await axios.get(baseUrl, {
157
- params: {
158
- hash,
159
- checkHash: true,
160
- },
161
- validateStatus: (status) => true,
162
- });
163
- console.log('Upload hash checkResponse', checkResponse)
164
- if (checkResponse.status !== 200) {
165
- // Only log if not 200
166
- console.error('Hash check failed:', {
167
- status: checkResponse.status,
168
- data: checkResponse.data
169
- });
170
- }
171
- // Hash should exist since we just uploaded it
172
- t.is(checkResponse.status, 200);
173
- t.truthy(checkResponse.data.hash);
174
-
175
- // Verify file exists and content matches
176
- const fileResponse = await axios.get(response.data.url, { responseType: 'arraybuffer' });
177
- t.is(fileResponse.status, 200);
178
- t.deepEqual(Buffer.from(fileResponse.data), Buffer.from(fileContent), 'Uploaded file content should match');
179
- } finally {
180
- fs.unlinkSync(filePath);
181
- if (uploadedUrl) {
182
- await cleanupHashAndFile(hash, uploadedUrl, baseUrl);
183
- }
184
- if (convertedUrl) {
185
- await cleanupHashAndFile(null, convertedUrl, baseUrl);
186
- }
140
+ test.serial("should handle file upload with hash", async (t) => {
141
+ const fileContent = "test content";
142
+ const filePath = await createTestFile(fileContent, "txt");
143
+ const requestId = uuidv4();
144
+ const hash = "test-hash-" + uuidv4();
145
+ let uploadedUrl;
146
+ let convertedUrl;
147
+ let response;
148
+
149
+ try {
150
+ // First upload the file
151
+ response = await uploadFile(filePath, requestId, hash);
152
+ t.is(response.status, 200);
153
+ t.truthy(response.data.url);
154
+ uploadedUrl = response.data.url;
155
+ if (response.data.converted && response.data.converted.url) {
156
+ convertedUrl = response.data.converted.url;
157
+ }
158
+ console.log("Upload hash response.data", response.data);
159
+
160
+ // Wait for Redis operations to complete and verify storage
161
+ await new Promise((resolve) => setTimeout(resolve, 2000));
162
+
163
+ const checkResponse = await axios.get(baseUrl, {
164
+ params: {
165
+ hash,
166
+ checkHash: true,
167
+ },
168
+ validateStatus: (status) => true,
169
+ });
170
+ console.log("Upload hash checkResponse", checkResponse);
171
+ if (checkResponse.status !== 200) {
172
+ // Only log if not 200
173
+ console.error("Hash check failed:", {
174
+ status: checkResponse.status,
175
+ data: checkResponse.data,
176
+ });
177
+ }
178
+ // Hash should exist since we just uploaded it
179
+ t.is(checkResponse.status, 200);
180
+ t.truthy(checkResponse.data.hash);
181
+
182
+ // Verify file exists and content matches
183
+ const fileResponse = await axios.get(response.data.url, {
184
+ responseType: "arraybuffer",
185
+ });
186
+ t.is(fileResponse.status, 200);
187
+ t.deepEqual(
188
+ Buffer.from(fileResponse.data),
189
+ Buffer.from(fileContent),
190
+ "Uploaded file content should match",
191
+ );
192
+ } finally {
193
+ fs.unlinkSync(filePath);
194
+ if (uploadedUrl) {
195
+ await cleanupHashAndFile(hash, uploadedUrl, baseUrl);
187
196
  }
197
+ if (convertedUrl) {
198
+ await cleanupHashAndFile(null, convertedUrl, baseUrl);
199
+ }
200
+ }
188
201
  });
189
202
 
190
203
  // Document Processing Tests
191
- test.serial('should handle PDF document upload and conversion', async (t) => {
192
- // Create a simple PDF file
193
- const fileContent = '%PDF-1.4\nTest PDF content';
194
- const filePath = await createTestFile(fileContent, 'pdf');
195
- const requestId = uuidv4();
196
- let response;
197
-
198
- try {
199
- response = await uploadFile(filePath, requestId);
200
- t.is(response.status, 200);
201
- t.truthy(response.data.url);
202
-
203
- // Verify original PDF content matches
204
- const uploadedContent = await fetchFileContent(response.data.url);
205
- t.deepEqual(uploadedContent, Buffer.from(fileContent), 'Uploaded PDF content should match');
206
-
207
- // Check if converted version exists
208
- if (response.data.converted) {
209
- t.truthy(response.data.converted.url);
210
- const convertedResponse = await axios.get(response.data.converted.url, { responseType: 'arraybuffer' });
211
- t.is(convertedResponse.status, 200);
212
- // For conversion, just check non-empty
213
- t.true(Buffer.from(convertedResponse.data).length > 0, 'Converted file should not be empty');
214
- }
215
- } finally {
216
- fs.unlinkSync(filePath);
217
- if (response?.data?.url) {
218
- await cleanupHashAndFile(null, response.data.url, baseUrl);
219
- }
220
- if (response?.data?.converted?.url) {
221
- await cleanupHashAndFile(null, response.data.converted.url, baseUrl);
222
- }
204
+ test.serial("should handle PDF document upload and conversion", async (t) => {
205
+ // Create a simple PDF file
206
+ const fileContent = "%PDF-1.4\nTest PDF content";
207
+ const filePath = await createTestFile(fileContent, "pdf");
208
+ const requestId = uuidv4();
209
+ let response;
210
+
211
+ try {
212
+ response = await uploadFile(filePath, requestId);
213
+ t.is(response.status, 200);
214
+ t.truthy(response.data.url);
215
+
216
+ // Verify original PDF content matches
217
+ const uploadedContent = await fetchFileContent(response.data.url);
218
+ t.deepEqual(
219
+ uploadedContent,
220
+ Buffer.from(fileContent),
221
+ "Uploaded PDF content should match",
222
+ );
223
+
224
+ // Check if converted version exists
225
+ if (response.data.converted) {
226
+ t.truthy(response.data.converted.url);
227
+ const convertedResponse = await axios.get(response.data.converted.url, {
228
+ responseType: "arraybuffer",
229
+ });
230
+ t.is(convertedResponse.status, 200);
231
+ // For conversion, just check non-empty
232
+ t.true(
233
+ Buffer.from(convertedResponse.data).length > 0,
234
+ "Converted file should not be empty",
235
+ );
236
+ }
237
+ } finally {
238
+ fs.unlinkSync(filePath);
239
+ if (response?.data?.url) {
240
+ await cleanupHashAndFile(null, response.data.url, baseUrl);
223
241
  }
242
+ if (response?.data?.converted?.url) {
243
+ await cleanupHashAndFile(null, response.data.converted.url, baseUrl);
244
+ }
245
+ }
224
246
  });
225
247
 
226
248
  // Media Chunking Tests
227
- test.serial('should handle media file chunking', async (t) => {
228
- // Create a large test file to trigger chunking
229
- const chunkContent = 'x'.repeat(1024 * 1024);
230
- const filePath = await createTestFile(chunkContent, 'mp4');
231
- const requestId = uuidv4();
232
- let response;
233
-
234
- try {
235
- response = await uploadFile(filePath, requestId);
236
- t.is(response.status, 200);
237
- t.truthy(response.data);
238
-
239
- // For media files, we expect either an array of chunks or a single URL
240
- if (Array.isArray(response.data)) {
241
- t.true(response.data.length > 0);
242
-
243
- // Verify each chunk
244
- for (const chunk of response.data) {
245
- t.truthy(chunk.uri);
246
- t.truthy(chunk.offset);
247
-
248
- // Verify chunk exists and content matches
249
- const chunkResponse = await axios.get(chunk.uri, { responseType: 'arraybuffer' });
250
- t.is(chunkResponse.status, 200);
251
- // Each chunk should be a slice of the original content
252
- const expectedChunk = Buffer.from(chunkContent).slice(chunk.offset, chunk.offset + chunk.length || undefined);
253
- t.deepEqual(Buffer.from(chunkResponse.data), expectedChunk, 'Chunk content should match original');
254
-
255
- // If GCS is configured, verify backup
256
- if (isGCSConfigured() && chunk.gcs) {
257
- const exists = await verifyGCSFile(chunk.gcs);
258
- t.true(exists, 'GCS chunk should exist');
259
- }
260
- }
261
- } else {
262
- // Single file response
263
- t.truthy(response.data.url);
264
- const fileResponse = await axios.get(response.data.url, { responseType: 'arraybuffer' });
265
- t.is(fileResponse.status, 200);
266
- t.deepEqual(Buffer.from(fileResponse.data), Buffer.from(chunkContent), 'Uploaded file content should match');
249
+ test.serial("should handle media file chunking", async (t) => {
250
+ // Create a large test file to trigger chunking
251
+ const chunkContent = "x".repeat(1024 * 1024);
252
+ const filePath = await createTestFile(chunkContent, "mp4");
253
+ const requestId = uuidv4();
254
+ let response;
255
+
256
+ try {
257
+ response = await uploadFile(filePath, requestId);
258
+ t.is(response.status, 200);
259
+ t.truthy(response.data);
260
+
261
+ // For media files, we expect either an array of chunks or a single URL
262
+ if (Array.isArray(response.data)) {
263
+ t.true(response.data.length > 0);
264
+
265
+ // Verify each chunk
266
+ for (const chunk of response.data) {
267
+ t.truthy(chunk.uri);
268
+ t.truthy(chunk.offset);
269
+
270
+ // Verify chunk exists and content matches
271
+ const chunkResponse = await axios.get(chunk.uri, {
272
+ responseType: "arraybuffer",
273
+ });
274
+ t.is(chunkResponse.status, 200);
275
+ // Each chunk should be a slice of the original content
276
+ const expectedChunk = Buffer.from(chunkContent).slice(
277
+ chunk.offset,
278
+ chunk.offset + chunk.length || undefined,
279
+ );
280
+ t.deepEqual(
281
+ Buffer.from(chunkResponse.data),
282
+ expectedChunk,
283
+ "Chunk content should match original",
284
+ );
285
+
286
+ // If GCS is configured, verify backup
287
+ if (isGCSConfigured() && chunk.gcs) {
288
+ const exists = await verifyGCSFile(chunk.gcs);
289
+ t.true(exists, "GCS chunk should exist");
267
290
  }
268
- } finally {
269
- fs.unlinkSync(filePath);
270
- if (response?.data) {
271
- if (Array.isArray(response.data)) {
272
- for (const chunk of response.data) {
273
- if (chunk.uri) {
274
- await cleanupHashAndFile(null, chunk.uri, baseUrl);
275
- }
276
- }
277
- } else if (response.data.url) {
278
- await cleanupHashAndFile(null, response.data.url, baseUrl);
279
- }
291
+ }
292
+ } else {
293
+ // Single file response
294
+ t.truthy(response.data.url);
295
+ const fileResponse = await axios.get(response.data.url, {
296
+ responseType: "arraybuffer",
297
+ });
298
+ t.is(fileResponse.status, 200);
299
+ t.deepEqual(
300
+ Buffer.from(fileResponse.data),
301
+ Buffer.from(chunkContent),
302
+ "Uploaded file content should match",
303
+ );
304
+ }
305
+ } finally {
306
+ fs.unlinkSync(filePath);
307
+ if (response?.data) {
308
+ if (Array.isArray(response.data)) {
309
+ for (const chunk of response.data) {
310
+ if (chunk.uri) {
311
+ await cleanupHashAndFile(null, chunk.uri, baseUrl);
312
+ }
280
313
  }
314
+ } else if (response.data.url) {
315
+ await cleanupHashAndFile(null, response.data.url, baseUrl);
316
+ }
281
317
  }
318
+ }
282
319
  });
283
320
 
284
321
  // Error Handling Tests
285
- test.serial('should handle invalid file upload', async (t) => {
286
- const requestId = uuidv4();
287
- const form = new FormData();
288
- // Send a file with no name and no content
289
- form.append('file', Buffer.from(''), { filename: '' });
290
- form.append('requestId', requestId);
291
-
292
- const response = await axios.post(baseUrl, form, {
293
- headers: {
294
- ...form.getHeaders(),
295
- 'Content-Type': 'multipart/form-data',
296
- },
297
- validateStatus: (status) => true,
298
- timeout: 30000,
299
- });
300
-
301
- // Log the response for debugging
302
- console.log('Invalid file upload response:', {
303
- status: response.status,
304
- data: response.data
305
- });
306
-
307
- t.is(response.status, 400, 'Should reject invalid file with 400 status');
308
- t.is(response.data, 'Invalid file: missing filename', 'Should return correct error message');
322
+ test.serial("should handle invalid file upload", async (t) => {
323
+ const requestId = uuidv4();
324
+ const form = new FormData();
325
+ // Send a file with no name and no content
326
+ form.append("file", Buffer.from(""), { filename: "" });
327
+ form.append("requestId", requestId);
328
+
329
+ const response = await axios.post(baseUrl, form, {
330
+ headers: {
331
+ ...form.getHeaders(),
332
+ "Content-Type": "multipart/form-data",
333
+ },
334
+ validateStatus: (status) => true,
335
+ timeout: 30000,
336
+ });
337
+
338
+ // Log the response for debugging
339
+ console.log("Invalid file upload response:", {
340
+ status: response.status,
341
+ data: response.data,
342
+ });
343
+
344
+ t.is(response.status, 400, "Should reject invalid file with 400 status");
345
+ t.is(
346
+ response.data,
347
+ "Invalid file: missing filename",
348
+ "Should return correct error message",
349
+ );
309
350
  });
310
351
 
311
352
  // Cleanup Tests
312
- test.serial('should handle file deletion', async (t) => {
313
- const filePath = await createTestFile('test content', 'txt');
314
- const requestId = uuidv4();
315
-
316
- try {
317
- // Upload file
318
- const uploadResponse = await uploadFile(filePath, requestId);
319
- t.is(uploadResponse.status, 200);
320
-
321
- // Wait a moment for file to be fully written
322
- await new Promise(resolve => setTimeout(resolve, 1000));
323
-
324
- // Extract the file identifier from the URL
325
- const fileIdentifier = getFolderNameFromUrl(uploadResponse.data.url);
326
- console.log('File identifier for deletion:', fileIdentifier);
327
-
328
- // Delete file using the correct identifier
329
- const deleteUrl = `${baseUrl}?operation=delete&requestId=${fileIdentifier}`;
330
- console.log('Deleting file with URL:', deleteUrl);
331
- const deleteResponse = await axios.delete(deleteUrl);
332
- t.is(deleteResponse.status, 200);
333
-
334
- // Wait a moment for deletion to complete
335
- await new Promise(resolve => setTimeout(resolve, 1000));
336
-
337
- // Verify file is gone
338
- const verifyResponse = await axios.get(uploadResponse.data.url, {
339
- validateStatus: (status) => true,
340
- });
341
- t.is(verifyResponse.status, 404, 'File should be deleted');
342
-
343
- // If GCS is configured, verify backup is gone
344
- if (isGCSConfigured() && uploadResponse.data.gcs) {
345
- const exists = await verifyGCSFile(uploadResponse.data.gcs);
346
- t.false(exists, 'GCS file should be deleted');
347
- }
348
- } finally {
349
- fs.unlinkSync(filePath);
353
+ test.serial("should handle file deletion", async (t) => {
354
+ const filePath = await createTestFile("test content", "txt");
355
+ const requestId = uuidv4();
356
+
357
+ try {
358
+ // Upload file
359
+ const uploadResponse = await uploadFile(filePath, requestId);
360
+ t.is(uploadResponse.status, 200);
361
+
362
+ // Wait a moment for file to be fully written
363
+ await new Promise((resolve) => setTimeout(resolve, 1000));
364
+
365
+ // Extract the file identifier from the URL
366
+ const fileIdentifier = getFolderNameFromUrl(uploadResponse.data.url);
367
+ console.log("File identifier for deletion:", fileIdentifier);
368
+
369
+ // Delete file using the correct identifier
370
+ const deleteUrl = `${baseUrl}?operation=delete&requestId=${fileIdentifier}`;
371
+ console.log("Deleting file with URL:", deleteUrl);
372
+ const deleteResponse = await axios.delete(deleteUrl);
373
+ t.is(deleteResponse.status, 200);
374
+
375
+ // Wait a moment for deletion to complete
376
+ await new Promise((resolve) => setTimeout(resolve, 1000));
377
+
378
+ // Verify file is gone
379
+ const verifyResponse = await axios.get(uploadResponse.data.url, {
380
+ validateStatus: (status) => true,
381
+ });
382
+ t.is(verifyResponse.status, 404, "File should be deleted");
383
+
384
+ // If GCS is configured, verify backup is gone
385
+ if (isGCSConfigured() && uploadResponse.data.gcs) {
386
+ const exists = await verifyGCSFile(uploadResponse.data.gcs);
387
+ t.false(exists, "GCS file should be deleted");
350
388
  }
389
+ } finally {
390
+ fs.unlinkSync(filePath);
391
+ }
351
392
  });
352
393
 
353
394
  // Save Option Test
354
- test.serial('should handle document upload with save option', async (t) => {
355
- // Create a minimal XLSX workbook in-memory
356
- const workbook = XLSX.utils.book_new();
357
- const worksheet = XLSX.utils.aoa_to_sheet([
358
- ['Name', 'Score'],
359
- ['Alice', 10],
360
- ['Bob', 8],
361
- ]);
362
- XLSX.utils.book_append_sheet(workbook, worksheet, 'Sheet1');
363
-
364
- // Write it to a temp file inside the test directory
365
- const filePath = path.join(t.context.testDir, `${uuidv4()}.xlsx`);
366
- XLSX.writeFile(workbook, filePath);
367
-
368
- const initialRequestId = uuidv4();
369
- const saveRequestId = uuidv4();
370
-
371
- let uploadedUrl;
372
- let savedUrl;
373
-
374
- try {
375
- // First, upload the document so we have a publicly reachable URL
376
- const uploadResponse = await uploadFile(filePath, initialRequestId);
377
- t.is(uploadResponse.status, 200);
378
- t.truthy(uploadResponse.data.url, 'Upload should return a URL');
379
-
380
- uploadedUrl = uploadResponse.data.url;
381
-
382
- // Now call the handler again with the save flag
383
- const saveResponse = await axios.get(baseUrl, {
384
- params: {
385
- uri: uploadedUrl,
386
- requestId: saveRequestId,
387
- save: true,
388
- },
389
- validateStatus: (status) => true,
390
- });
395
+ test.serial("should handle document upload with save option", async (t) => {
396
+ // Create a minimal XLSX workbook in-memory
397
+ const workbook = XLSX.utils.book_new();
398
+ const worksheet = XLSX.utils.aoa_to_sheet([
399
+ ["Name", "Score"],
400
+ ["Alice", 10],
401
+ ["Bob", 8],
402
+ ]);
403
+ XLSX.utils.book_append_sheet(workbook, worksheet, "Sheet1");
404
+
405
+ // Write it to a temp file inside the test directory
406
+ const filePath = path.join(t.context.testDir, `${uuidv4()}.xlsx`);
407
+ XLSX.writeFile(workbook, filePath);
408
+
409
+ const initialRequestId = uuidv4();
410
+ const saveRequestId = uuidv4();
411
+
412
+ let uploadedUrl;
413
+ let savedUrl;
414
+
415
+ try {
416
+ // First, upload the document so we have a publicly reachable URL
417
+ const uploadResponse = await uploadFile(filePath, initialRequestId);
418
+ t.is(uploadResponse.status, 200);
419
+ t.truthy(uploadResponse.data.url, "Upload should return a URL");
420
+
421
+ uploadedUrl = uploadResponse.data.url;
422
+
423
+ // Now call the handler again with the save flag
424
+ const saveResponse = await axios.get(baseUrl, {
425
+ params: {
426
+ uri: uploadedUrl,
427
+ requestId: saveRequestId,
428
+ save: true,
429
+ },
430
+ validateStatus: (status) => true,
431
+ });
391
432
 
392
- // The save operation should return a 200 status with a result object
393
- t.is(saveResponse.status, 200, 'Save request should succeed');
394
- t.truthy(saveResponse.data, 'Response should have data');
395
- t.truthy(saveResponse.data.url, 'Response should include a URL');
396
- t.true(saveResponse.data.url.includes('.csv'), 'Response should include a CSV URL');
397
- savedUrl = saveResponse.data.url;
398
- } finally {
399
- fs.unlinkSync(filePath);
400
- // Clean up both URLs
401
- if (uploadedUrl) {
402
- await cleanupHashAndFile(null, uploadedUrl, baseUrl);
403
- }
404
- if (savedUrl && savedUrl !== uploadedUrl) {
405
- await cleanupHashAndFile(null, savedUrl, baseUrl);
406
- }
433
+ // The save operation should return a 200 status with a result object
434
+ t.is(saveResponse.status, 200, "Save request should succeed");
435
+ t.truthy(saveResponse.data, "Response should have data");
436
+ t.truthy(saveResponse.data.url, "Response should include a URL");
437
+ t.true(
438
+ saveResponse.data.url.includes(".csv"),
439
+ "Response should include a CSV URL",
440
+ );
441
+ savedUrl = saveResponse.data.url;
442
+ } finally {
443
+ fs.unlinkSync(filePath);
444
+ // Clean up both URLs
445
+ if (uploadedUrl) {
446
+ await cleanupHashAndFile(null, uploadedUrl, baseUrl);
447
+ }
448
+ if (savedUrl && savedUrl !== uploadedUrl) {
449
+ await cleanupHashAndFile(null, savedUrl, baseUrl);
407
450
  }
451
+ }
408
452
  });
409
453
 
410
454
  // Converted file persistence test – ensures needsConversion works for extension-only checks
411
- test.serial('should preserve converted version when checking hash for convertible file', async (t) => {
455
+ test.serial(
456
+ "should preserve converted version when checking hash for convertible file",
457
+ async (t) => {
412
458
  // Create a minimal XLSX workbook in-memory
413
459
  const workbook = XLSX.utils.book_new();
414
460
  const worksheet = XLSX.utils.aoa_to_sheet([
415
- ['Name', 'Score'],
416
- ['Alice', 10],
417
- ['Bob', 8],
461
+ ["Name", "Score"],
462
+ ["Alice", 10],
463
+ ["Bob", 8],
418
464
  ]);
419
- XLSX.utils.book_append_sheet(workbook, worksheet, 'Sheet1');
465
+ XLSX.utils.book_append_sheet(workbook, worksheet, "Sheet1");
420
466
 
421
467
  // Write it to a temp file inside the test directory
422
468
  const filePath = path.join(t.context.testDir, `${uuidv4()}.xlsx`);
@@ -429,34 +475,47 @@ test.serial('should preserve converted version when checking hash for convertibl
429
475
  let convertedUrl;
430
476
 
431
477
  try {
432
- // 1. Upload the XLSX file (conversion should run automatically)
433
- const uploadResponse = await uploadFile(filePath, requestId, hash);
434
- t.is(uploadResponse.status, 200, 'Upload should succeed');
435
- t.truthy(uploadResponse.data.converted, 'Upload response must contain converted info');
436
- t.truthy(uploadResponse.data.converted.url, 'Converted URL should be present');
437
-
438
- uploadedUrl = uploadResponse.data.url;
439
- convertedUrl = uploadResponse.data.converted.url;
440
-
441
- // 2. Give Redis a moment to persist
442
- await new Promise((resolve) => setTimeout(resolve, 4000));
443
-
444
- // 3. Ask the handler for the hash – it will invoke ensureConvertedVersion
445
- const checkResponse = await axios.get(baseUrl, {
446
- params: { hash, checkHash: true },
447
- validateStatus: (status) => true,
448
- timeout: 30000,
449
- });
450
-
451
- t.is(checkResponse.status, 200, 'Hash check should succeed');
452
- t.truthy(checkResponse.data.converted, 'Hash response should include converted info');
453
- t.truthy(checkResponse.data.converted.url, 'Converted URL should still be present after hash check');
478
+ // 1. Upload the XLSX file (conversion should run automatically)
479
+ const uploadResponse = await uploadFile(filePath, requestId, hash);
480
+ t.is(uploadResponse.status, 200, "Upload should succeed");
481
+ t.truthy(
482
+ uploadResponse.data.converted,
483
+ "Upload response must contain converted info",
484
+ );
485
+ t.truthy(
486
+ uploadResponse.data.converted.url,
487
+ "Converted URL should be present",
488
+ );
489
+
490
+ uploadedUrl = uploadResponse.data.url;
491
+ convertedUrl = uploadResponse.data.converted.url;
492
+
493
+ // 2. Give Redis a moment to persist
494
+ await new Promise((resolve) => setTimeout(resolve, 4000));
495
+
496
+ // 3. Ask the handler for the hash – it will invoke ensureConvertedVersion
497
+ const checkResponse = await axios.get(baseUrl, {
498
+ params: { hash, checkHash: true },
499
+ validateStatus: (status) => true,
500
+ timeout: 30000,
501
+ });
502
+
503
+ t.is(checkResponse.status, 200, "Hash check should succeed");
504
+ t.truthy(
505
+ checkResponse.data.converted,
506
+ "Hash response should include converted info",
507
+ );
508
+ t.truthy(
509
+ checkResponse.data.converted.url,
510
+ "Converted URL should still be present after hash check",
511
+ );
454
512
  } finally {
455
- // Clean up temp file and remote artifacts
456
- fs.unlinkSync(filePath);
457
- await cleanupHashAndFile(hash, uploadedUrl, baseUrl);
458
- if (convertedUrl) {
459
- await cleanupHashAndFile(null, convertedUrl, baseUrl);
460
- }
513
+ // Clean up temp file and remote artifacts
514
+ fs.unlinkSync(filePath);
515
+ await cleanupHashAndFile(hash, uploadedUrl, baseUrl);
516
+ if (convertedUrl) {
517
+ await cleanupHashAndFile(null, convertedUrl, baseUrl);
518
+ }
461
519
  }
462
- });
520
+ },
521
+ );