@aj-archipelago/cortex 1.4.2 → 1.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/README.md +1 -0
  2. package/config.js +1 -1
  3. package/helper-apps/cortex-autogen2/.dockerignore +1 -0
  4. package/helper-apps/cortex-autogen2/Dockerfile +6 -10
  5. package/helper-apps/cortex-autogen2/Dockerfile.worker +2 -0
  6. package/helper-apps/cortex-autogen2/agents.py +203 -2
  7. package/helper-apps/cortex-autogen2/main.py +1 -1
  8. package/helper-apps/cortex-autogen2/pyproject.toml +12 -0
  9. package/helper-apps/cortex-autogen2/requirements.txt +14 -0
  10. package/helper-apps/cortex-autogen2/services/redis_publisher.py +1 -1
  11. package/helper-apps/cortex-autogen2/services/run_analyzer.py +1 -1
  12. package/helper-apps/cortex-autogen2/task_processor.py +431 -229
  13. package/helper-apps/cortex-autogen2/test_entity_fetcher.py +305 -0
  14. package/helper-apps/cortex-autogen2/tests/README.md +240 -0
  15. package/helper-apps/cortex-autogen2/tests/TEST_REPORT.md +342 -0
  16. package/helper-apps/cortex-autogen2/tests/__init__.py +8 -0
  17. package/helper-apps/cortex-autogen2/tests/analysis/__init__.py +1 -0
  18. package/helper-apps/cortex-autogen2/tests/analysis/improvement_suggester.py +224 -0
  19. package/helper-apps/cortex-autogen2/tests/analysis/trend_analyzer.py +211 -0
  20. package/helper-apps/cortex-autogen2/tests/cli/__init__.py +1 -0
  21. package/helper-apps/cortex-autogen2/tests/cli/run_tests.py +296 -0
  22. package/helper-apps/cortex-autogen2/tests/collectors/__init__.py +1 -0
  23. package/helper-apps/cortex-autogen2/tests/collectors/log_collector.py +252 -0
  24. package/helper-apps/cortex-autogen2/tests/collectors/progress_collector.py +182 -0
  25. package/helper-apps/cortex-autogen2/tests/conftest.py +15 -0
  26. package/helper-apps/cortex-autogen2/tests/database/__init__.py +1 -0
  27. package/helper-apps/cortex-autogen2/tests/database/repository.py +501 -0
  28. package/helper-apps/cortex-autogen2/tests/database/schema.sql +108 -0
  29. package/helper-apps/cortex-autogen2/tests/evaluators/__init__.py +1 -0
  30. package/helper-apps/cortex-autogen2/tests/evaluators/llm_scorer.py +294 -0
  31. package/helper-apps/cortex-autogen2/tests/evaluators/prompts.py +250 -0
  32. package/helper-apps/cortex-autogen2/tests/evaluators/wordcloud_validator.py +168 -0
  33. package/helper-apps/cortex-autogen2/tests/metrics/__init__.py +1 -0
  34. package/helper-apps/cortex-autogen2/tests/metrics/collector.py +155 -0
  35. package/helper-apps/cortex-autogen2/tests/orchestrator.py +576 -0
  36. package/helper-apps/cortex-autogen2/tests/test_cases.yaml +279 -0
  37. package/helper-apps/cortex-autogen2/tests/test_data.db +0 -0
  38. package/helper-apps/cortex-autogen2/tests/utils/__init__.py +3 -0
  39. package/helper-apps/cortex-autogen2/tests/utils/connectivity.py +112 -0
  40. package/helper-apps/cortex-autogen2/tools/azure_blob_tools.py +74 -24
  41. package/helper-apps/cortex-autogen2/tools/entity_api_registry.json +38 -0
  42. package/helper-apps/cortex-autogen2/tools/file_tools.py +1 -1
  43. package/helper-apps/cortex-autogen2/tools/search_tools.py +436 -238
  44. package/helper-apps/cortex-file-handler/package-lock.json +2 -2
  45. package/helper-apps/cortex-file-handler/package.json +1 -1
  46. package/helper-apps/cortex-file-handler/scripts/setup-test-containers.js +4 -5
  47. package/helper-apps/cortex-file-handler/src/blobHandler.js +36 -144
  48. package/helper-apps/cortex-file-handler/src/services/FileConversionService.js +5 -3
  49. package/helper-apps/cortex-file-handler/src/services/storage/AzureStorageProvider.js +34 -1
  50. package/helper-apps/cortex-file-handler/src/services/storage/GCSStorageProvider.js +22 -0
  51. package/helper-apps/cortex-file-handler/src/services/storage/LocalStorageProvider.js +28 -1
  52. package/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js +29 -4
  53. package/helper-apps/cortex-file-handler/src/services/storage/StorageProvider.js +11 -0
  54. package/helper-apps/cortex-file-handler/src/services/storage/StorageService.js +1 -1
  55. package/helper-apps/cortex-file-handler/tests/blobHandler.test.js +3 -2
  56. package/helper-apps/cortex-file-handler/tests/checkHashShortLived.test.js +8 -1
  57. package/helper-apps/cortex-file-handler/tests/containerConversionFlow.test.js +5 -2
  58. package/helper-apps/cortex-file-handler/tests/containerNameParsing.test.js +14 -7
  59. package/helper-apps/cortex-file-handler/tests/containerParameterFlow.test.js +5 -2
  60. package/helper-apps/cortex-file-handler/tests/storage/StorageFactory.test.js +31 -19
  61. package/package.json +1 -1
  62. package/server/modelExecutor.js +4 -0
  63. package/server/plugins/claude4VertexPlugin.js +540 -0
  64. package/server/plugins/openAiWhisperPlugin.js +43 -2
  65. package/tests/integration/rest/vendors/claude_streaming.test.js +121 -0
  66. package/tests/unit/plugins/claude4VertexPlugin.test.js +462 -0
  67. package/tests/unit/plugins/claude4VertexToolConversion.test.js +413 -0
  68. package/helper-apps/cortex-autogen/.funcignore +0 -8
  69. package/helper-apps/cortex-autogen/Dockerfile +0 -10
  70. package/helper-apps/cortex-autogen/OAI_CONFIG_LIST +0 -6
  71. package/helper-apps/cortex-autogen/agents.py +0 -493
  72. package/helper-apps/cortex-autogen/agents_extra.py +0 -14
  73. package/helper-apps/cortex-autogen/config.py +0 -18
  74. package/helper-apps/cortex-autogen/data_operations.py +0 -29
  75. package/helper-apps/cortex-autogen/function_app.py +0 -44
  76. package/helper-apps/cortex-autogen/host.json +0 -15
  77. package/helper-apps/cortex-autogen/main.py +0 -38
  78. package/helper-apps/cortex-autogen/prompts.py +0 -196
  79. package/helper-apps/cortex-autogen/prompts_extra.py +0 -5
  80. package/helper-apps/cortex-autogen/requirements.txt +0 -9
  81. package/helper-apps/cortex-autogen/search.py +0 -85
  82. package/helper-apps/cortex-autogen/test.sh +0 -40
  83. package/helper-apps/cortex-autogen/tools/sasfileuploader.py +0 -66
  84. package/helper-apps/cortex-autogen/utils.py +0 -88
  85. package/helper-apps/cortex-autogen2/DigiCertGlobalRootCA.crt.pem +0 -22
  86. package/helper-apps/cortex-autogen2/poetry.lock +0 -3652
@@ -1,12 +1,12 @@
1
1
  {
2
2
  "name": "@aj-archipelago/cortex-file-handler",
3
- "version": "2.6.1",
3
+ "version": "2.6.2",
4
4
  "lockfileVersion": 3,
5
5
  "requires": true,
6
6
  "packages": {
7
7
  "": {
8
8
  "name": "@aj-archipelago/cortex-file-handler",
9
- "version": "2.6.1",
9
+ "version": "2.6.2",
10
10
  "dependencies": {
11
11
  "@azure/storage-blob": "^12.13.0",
12
12
  "@distube/ytdl-core": "^4.14.3",
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@aj-archipelago/cortex-file-handler",
3
- "version": "2.6.1",
3
+ "version": "2.6.2",
4
4
  "description": "File handling service for Cortex - handles file uploads, media chunking, and document processing",
5
5
  "type": "module",
6
6
  "main": "src/index.js",
@@ -7,13 +7,12 @@ async function createAzureContainers() {
7
7
  "UseDevelopmentStorage=true",
8
8
  );
9
9
 
10
- // Get container names from environment variable
11
- const containerStr = process.env.AZURE_STORAGE_CONTAINER_NAME || "default,test-container,test1,test2,test3,container1,container2,container3";
12
- const containerNames = containerStr.split(',').map(name => name.trim()).filter(name => name.length > 0);
10
+ // Always create all possible test containers to support dynamic test environments
11
+ const allTestContainers = ["default", "test-container", "test1", "test2", "test3", "container1", "container2", "container3"];
13
12
 
14
- console.log(`Creating Azure containers: ${containerNames.join(', ')}`);
13
+ console.log(`Creating Azure containers: ${allTestContainers.join(', ')}`);
15
14
 
16
- for (const containerName of containerNames) {
15
+ for (const containerName of allTestContainers) {
17
16
  try {
18
17
  const containerClient = blobServiceClient.getContainerClient(containerName);
19
18
  await containerClient.create();
@@ -6,13 +6,8 @@ import { pipeline as _pipeline } from "stream";
6
6
  import { v4 as uuidv4 } from "uuid";
7
7
  import Busboy from "busboy";
8
8
  import { PassThrough } from "stream";
9
- import mime from "mime-types";
10
9
  import { Storage } from "@google-cloud/storage";
11
- import {
12
- generateBlobSASQueryParameters,
13
- StorageSharedKeyCredential,
14
- BlobServiceClient,
15
- } from "@azure/storage-blob";
10
+ import { BlobServiceClient } from "@azure/storage-blob";
16
11
  import axios from "axios";
17
12
 
18
13
  import {
@@ -23,6 +18,7 @@ import {
23
18
  import { publicFolder, port, ipAddress } from "./start.js";
24
19
  import { CONVERTED_EXTENSIONS, AZURITE_ACCOUNT_NAME } from "./constants.js";
25
20
  import { FileConversionService } from "./services/FileConversionService.js";
21
+ import { StorageFactory } from "./services/storage/StorageFactory.js";
26
22
 
27
23
  const pipeline = promisify(_pipeline);
28
24
 
@@ -80,13 +76,20 @@ const parseContainerNames = () => {
80
76
  return containerStr.split(',').map(name => name.trim());
81
77
  };
82
78
 
79
+ // Helper function to get current container names at runtime
80
+ export const getCurrentContainerNames = () => {
81
+ return parseContainerNames();
82
+ };
83
+
83
84
  export const AZURE_STORAGE_CONTAINER_NAMES = parseContainerNames();
84
85
  export const DEFAULT_AZURE_STORAGE_CONTAINER_NAME = AZURE_STORAGE_CONTAINER_NAMES[0];
85
86
  export const GCS_BUCKETNAME = process.env.GCS_BUCKETNAME || "cortextempfiles";
86
87
 
87
88
  // Validate if a container name is allowed
88
89
  export const isValidContainerName = (containerName) => {
89
- return AZURE_STORAGE_CONTAINER_NAMES.includes(containerName);
90
+ // Read from environment at runtime to support dynamically changing env in tests
91
+ const currentContainerNames = getCurrentContainerNames();
92
+ return currentContainerNames.includes(containerName);
90
93
  };
91
94
 
92
95
  function isEncoded(str) {
@@ -221,95 +224,12 @@ export const getBlobClient = async (containerName = null) => {
221
224
  };
222
225
 
223
226
  async function saveFileToBlob(chunkPath, requestId, filename = null, containerName = null) {
224
- const { containerClient } = await getBlobClient(containerName);
225
- // Use provided filename or generate LLM-friendly naming
226
- let blobName;
227
- if (filename) {
228
- blobName = generateBlobName(requestId, filename);
229
- } else {
230
- const fileExtension = path.extname(chunkPath);
231
- const shortId = generateShortId();
232
- blobName = generateBlobName(requestId, `${shortId}${fileExtension}`);
233
- }
234
-
235
- // Create a read stream for the chunk file
236
- const fileStream = fs.createReadStream(chunkPath);
237
-
238
- // Upload the chunk to Azure Blob Storage using the stream
239
- const blockBlobClient = containerClient.getBlockBlobClient(blobName);
240
- await blockBlobClient.uploadStream(fileStream);
241
-
242
- // Generate SAS token after successful upload
243
- const sasToken = generateSASToken(containerClient, blobName);
244
-
245
- // Return an object with the URL property
246
- return {
247
- url: `${blockBlobClient.url}?${sasToken}`,
248
- blobName: blobName,
249
- };
227
+ // Use provider for consistency with cache control headers
228
+ const storageFactory = StorageFactory.getInstance();
229
+ const provider = await storageFactory.getAzureProvider(containerName);
230
+ return await provider.uploadFile({}, chunkPath, requestId, null, filename);
250
231
  }
251
232
 
252
- const generateSASToken = (
253
- containerClient,
254
- blobName,
255
- options = {},
256
- ) => {
257
- // Handle Azurite (development storage) credentials with fallback
258
- let accountName, accountKey;
259
-
260
- if (containerClient.credential && containerClient.credential.accountName) {
261
- // Regular Azure Storage credentials
262
- accountName = containerClient.credential.accountName;
263
-
264
- // Handle Buffer case (Azurite) vs string case (real Azure)
265
- if (Buffer.isBuffer(containerClient.credential.accountKey)) {
266
- accountKey = containerClient.credential.accountKey.toString('base64');
267
- } else {
268
- accountKey = containerClient.credential.accountKey;
269
- }
270
- } else {
271
- // Azurite development storage fallback
272
- accountName = AZURITE_ACCOUNT_NAME;
273
- accountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";
274
- }
275
-
276
- const sharedKeyCredential = new StorageSharedKeyCredential(
277
- accountName,
278
- accountKey,
279
- );
280
-
281
- // Support custom duration: minutes, hours, or fall back to default
282
- let expirationTime;
283
- if (options.minutes) {
284
- expirationTime = new Date(new Date().valueOf() + options.minutes * 60 * 1000);
285
- } else if (options.hours) {
286
- expirationTime = new Date(new Date().valueOf() + options.hours * 60 * 60 * 1000);
287
- } else if (options.days) {
288
- expirationTime = new Date(new Date().valueOf() + options.days * 24 * 60 * 60 * 1000);
289
- } else if (options.expiryTimeSeconds !== undefined) {
290
- // Legacy support for existing parameter
291
- expirationTime = new Date(new Date().valueOf() + options.expiryTimeSeconds * 1000);
292
- } else {
293
- // Default to SAS_TOKEN_LIFE_DAYS environment variable
294
- const defaultExpirySeconds = parseInt(SAS_TOKEN_LIFE_DAYS) * 24 * 60 * 60;
295
- expirationTime = new Date(new Date().valueOf() + defaultExpirySeconds * 1000);
296
- }
297
-
298
- const sasOptions = {
299
- containerName: containerClient.containerName,
300
- blobName: blobName,
301
- permissions: options.permissions || "r", // Read permission
302
- startsOn: new Date(),
303
- expiresOn: expirationTime,
304
- };
305
-
306
- const sasToken = generateBlobSASQueryParameters(
307
- sasOptions,
308
- sharedKeyCredential,
309
- ).toString();
310
- return sasToken;
311
- };
312
-
313
233
  //deletes blob that has the requestId
314
234
  async function deleteBlob(requestId, containerName = null) {
315
235
  if (!requestId) throw new Error("Missing requestId parameter");
@@ -396,8 +316,10 @@ function uploadBlob(
396
316
  hash = value;
397
317
  } else if (fieldname === "container") {
398
318
  if (value && !isValidContainerName(value)) {
319
+ // Read current containers from env for error message
320
+ const currentContainerNames = getCurrentContainerNames();
399
321
  errorOccurred = true;
400
- const err = new Error(`Invalid container name '${value}'. Allowed containers: ${AZURE_STORAGE_CONTAINER_NAMES.join(', ')}`);
322
+ const err = new Error(`Invalid container name '${value}'. Allowed containers: ${currentContainerNames.join(', ')}`);
401
323
  err.status = 400;
402
324
  reject(err);
403
325
  return;
@@ -767,57 +689,17 @@ function uploadBlob(
767
689
 
768
690
  // Helper function to handle local file storage
769
691
  async function saveToLocalStorage(context, requestId, encodedFilename, file) {
770
- const localPath = path.join(publicFolder, requestId);
771
- fs.mkdirSync(localPath, { recursive: true });
772
-
773
- // Sanitize filename by removing invalid characters
774
- const sanitizedFilename = sanitizeFilename(encodedFilename);
775
- const destinationPath = `${localPath}/${sanitizedFilename}`;
776
-
777
- await pipeline(file, fs.createWriteStream(destinationPath));
778
- return `http://${ipAddress}:${port}/files/${requestId}/${sanitizedFilename}`;
692
+ const storageFactory = StorageFactory.getInstance();
693
+ const localProvider = storageFactory.getLocalProvider();
694
+ const contextWithRequestId = { ...context, requestId };
695
+ return await localProvider.uploadStream(contextWithRequestId, encodedFilename, file);
779
696
  }
780
697
 
781
698
  // Helper function to handle Azure blob storage
782
699
  async function saveToAzureStorage(context, encodedFilename, file, containerName = null) {
783
- const { containerClient } = await getBlobClient(containerName);
784
- const contentType = mime.lookup(encodedFilename);
785
-
786
- // Create a safe blob name that is URI-encoded once (no double encoding)
787
- let blobName = sanitizeFilename(encodedFilename);
788
- blobName = encodeURIComponent(blobName);
789
-
790
- const options = {
791
- blobHTTPHeaders: contentType ? { blobContentType: contentType } : {},
792
- maxConcurrency: 50,
793
- blockSize: 8 * 1024 * 1024,
794
- };
795
-
796
- const blockBlobClient = containerClient.getBlockBlobClient(blobName);
797
- console.log("Uploading to container:", containerName || "default");
798
- context.log(`Uploading to Azure... ${blobName}`);
799
- await blockBlobClient.uploadStream(file, undefined, undefined, options);
800
- const sasToken = generateSASToken(containerClient, blobName);
801
- return `${blockBlobClient.url}?${sasToken}`;
802
- }
803
-
804
- // Helper function to upload a file to Google Cloud Storage
805
- async function uploadToGCS(context, file, filename) {
806
- const objectName = sanitizeFilename(filename);
807
- const gcsFile = gcs.bucket(GCS_BUCKETNAME).file(objectName);
808
- const writeStream = gcsFile.createWriteStream({
809
- resumable: true,
810
- validation: false,
811
- metadata: {
812
- contentType: mime.lookup(objectName) || "application/octet-stream",
813
- },
814
- chunkSize: 8 * 1024 * 1024,
815
- numRetries: 3,
816
- retryDelay: 1000,
817
- });
818
- context.log(`Uploading to GCS... ${objectName}`);
819
- await pipeline(file, writeStream);
820
- return `gs://${GCS_BUCKETNAME}/${objectName}`;
700
+ const storageFactory = StorageFactory.getInstance();
701
+ const provider = await storageFactory.getAzureProvider(containerName);
702
+ return await provider.uploadStream(context, encodedFilename, file);
821
703
  }
822
704
 
823
705
  // Wrapper that checks if GCS is configured
@@ -825,7 +707,12 @@ async function saveToGoogleStorage(context, encodedFilename, file) {
825
707
  if (!gcs) {
826
708
  throw new Error("Google Cloud Storage is not initialized");
827
709
  }
828
- return uploadToGCS(context, file, encodedFilename);
710
+ const storageFactory = StorageFactory.getInstance();
711
+ const gcsProvider = storageFactory.getGCSProvider();
712
+ if (!gcsProvider) {
713
+ throw new Error("GCS provider not available");
714
+ }
715
+ return await gcsProvider.uploadStream(context, encodedFilename, file);
829
716
  }
830
717
 
831
718
  async function uploadFile(
@@ -1229,7 +1116,12 @@ async function ensureGCSUpload(context, existingFile) {
1229
1116
  url: existingFile.url,
1230
1117
  responseType: "stream",
1231
1118
  });
1232
- existingFile.gcs = await uploadToGCS(context, response.data, fileName);
1119
+
1120
+ const storageFactory = StorageFactory.getInstance();
1121
+ const gcsProvider = storageFactory.getGCSProvider();
1122
+ if (gcsProvider) {
1123
+ existingFile.gcs = await gcsProvider.uploadStream(context, fileName, response.data);
1124
+ }
1233
1125
  }
1234
1126
  return existingFile;
1235
1127
  }
@@ -3,7 +3,7 @@ import { getFileStoreMap, setFileStoreMap } from "../redis.js";
3
3
  import { urlExists } from "../helper.js";
4
4
  import { gcsUrlExists, uploadChunkToGCS, gcs } from "../blobHandler.js";
5
5
  import { downloadFile } from "../fileChunker.js";
6
- import { saveFileToBlob } from "../blobHandler.js";
6
+ import { StorageFactory } from "./storage/StorageFactory.js";
7
7
  import { moveFileToPublicFolder } from "../localFileHandler.js";
8
8
  import { v4 as uuidv4 } from "uuid";
9
9
 
@@ -11,6 +11,7 @@ export class FileConversionService extends ConversionService {
11
11
  constructor(context, useAzure = true) {
12
12
  super(context);
13
13
  this.useAzure = useAzure;
14
+ this.storageFactory = StorageFactory.getInstance();
14
15
  }
15
16
 
16
17
  async _getFileStoreMap(key) {
@@ -39,8 +40,9 @@ export class FileConversionService extends ConversionService {
39
40
 
40
41
  let fileUrl;
41
42
  if (this.useAzure) {
42
- const savedBlob = await saveFileToBlob(filePath, reqId, filename, containerName);
43
- fileUrl = savedBlob.url;
43
+ const provider = await this.storageFactory.getAzureProvider(containerName);
44
+ const result = await provider.uploadFile({}, filePath, reqId, null, filename);
45
+ fileUrl = result.url;
44
46
  } else {
45
47
  fileUrl = await moveFileToPublicFolder(filePath, reqId);
46
48
  }
@@ -5,12 +5,14 @@ import {
5
5
  } from "@azure/storage-blob";
6
6
  import fs from "fs";
7
7
  import path from "path";
8
+ import mime from "mime-types";
8
9
 
9
10
  import { StorageProvider } from "./StorageProvider.js";
10
11
  import { AZURITE_ACCOUNT_NAME } from "../../constants.js";
11
12
  import {
12
13
  generateShortId,
13
14
  generateBlobName,
15
+ sanitizeFilename,
14
16
  } from "../../utils/filenameUtils.js";
15
17
 
16
18
  export class AzureStorageProvider extends StorageProvider {
@@ -122,7 +124,12 @@ export class AzureStorageProvider extends StorageProvider {
122
124
 
123
125
  // Upload the file to Azure Blob Storage using the stream
124
126
  const blockBlobClient = containerClient.getBlockBlobClient(blobName);
125
- await blockBlobClient.uploadStream(fileStream);
127
+ const uploadOptions = {
128
+ blobHTTPHeaders: {
129
+ blobCacheControl: 'public, max-age=2592000, immutable',
130
+ },
131
+ };
132
+ await blockBlobClient.uploadStream(fileStream, undefined, undefined, uploadOptions);
126
133
 
127
134
  // Generate SAS token after successful upload
128
135
  const sasToken = this.generateSASToken(containerClient, blobName);
@@ -133,6 +140,32 @@ export class AzureStorageProvider extends StorageProvider {
133
140
  };
134
141
  }
135
142
 
143
+ async uploadStream(context, encodedFilename, stream) {
144
+ const { containerClient } = await this.getBlobClient();
145
+ const contentType = mime.lookup(encodedFilename);
146
+
147
+ // Normalize the blob name: sanitizeFilename decodes, cleans, then we encode for Azure
148
+ let blobName = sanitizeFilename(encodedFilename);
149
+ blobName = encodeURIComponent(blobName);
150
+
151
+ const options = {
152
+ blobHTTPHeaders: {
153
+ ...(contentType ? { blobContentType: contentType } : {}),
154
+ blobCacheControl: 'public, max-age=2592000, immutable',
155
+ },
156
+ maxConcurrency: 50,
157
+ blockSize: 8 * 1024 * 1024,
158
+ };
159
+
160
+ const blockBlobClient = containerClient.getBlockBlobClient(blobName);
161
+ if (context.log) context.log(`Uploading to Azure... ${blobName}`);
162
+
163
+ await blockBlobClient.uploadStream(stream, undefined, undefined, options);
164
+ const sasToken = this.generateSASToken(containerClient, blobName);
165
+
166
+ return `${blockBlobClient.url}?${sasToken}`;
167
+ }
168
+
136
169
  async deleteFiles(requestId) {
137
170
  if (!requestId) throw new Error("Missing requestId parameter");
138
171
  const { containerClient } = await this.getBlobClient();
@@ -4,6 +4,7 @@ import path from "path";
4
4
  import {
5
5
  generateShortId,
6
6
  generateBlobName,
7
+ sanitizeFilename,
7
8
  } from "../../utils/filenameUtils.js";
8
9
  import axios from "axios";
9
10
 
@@ -72,6 +73,27 @@ export class GCSStorageProvider extends StorageProvider {
72
73
  };
73
74
  }
74
75
 
76
+ async uploadStream(context, encodedFilename, stream) {
77
+ const bucket = this.storage.bucket(this.bucketName);
78
+ const blobName = sanitizeFilename(encodedFilename);
79
+
80
+ const file = bucket.file(blobName);
81
+ const writeStream = file.createWriteStream({
82
+ metadata: {
83
+ contentType: this.getContentType(encodedFilename) || "application/octet-stream",
84
+ },
85
+ resumable: false,
86
+ });
87
+
88
+ await new Promise((resolve, reject) => {
89
+ stream.pipe(writeStream)
90
+ .on('finish', resolve)
91
+ .on('error', reject);
92
+ });
93
+
94
+ return `gs://${this.bucketName}/${blobName}`;
95
+ }
96
+
75
97
  async deleteFiles(requestId) {
76
98
  if (!requestId) throw new Error("Missing requestId parameter");
77
99
 
@@ -1,10 +1,14 @@
1
1
  import fs from "fs";
2
2
  import path from "path";
3
- import { generateShortId } from "../../utils/filenameUtils.js";
3
+ import { promisify } from "util";
4
+ import { pipeline as _pipeline } from "stream";
5
+ import { generateShortId, sanitizeFilename } from "../../utils/filenameUtils.js";
4
6
  import { ipAddress, port } from "../../start.js";
5
7
 
6
8
  import { StorageProvider } from "./StorageProvider.js";
7
9
 
10
+ const pipeline = promisify(_pipeline);
11
+
8
12
  export class LocalStorageProvider extends StorageProvider {
9
13
  constructor(publicFolder) {
10
14
  super();
@@ -52,6 +56,29 @@ export class LocalStorageProvider extends StorageProvider {
52
56
  };
53
57
  }
54
58
 
59
+ async uploadStream(context, encodedFilename, stream) {
60
+ // For local storage, we need a requestId to create a folder
61
+ // Extract from context or generate one
62
+ const requestId = context.requestId || "default";
63
+
64
+ // Create request folder if it doesn't exist
65
+ const requestFolder = path.join(this.publicFolder, requestId);
66
+ if (!fs.existsSync(requestFolder)) {
67
+ fs.mkdirSync(requestFolder, { recursive: true });
68
+ }
69
+
70
+ const sanitizedFilename = sanitizeFilename(encodedFilename);
71
+ const destinationPath = path.join(requestFolder, sanitizedFilename);
72
+ const writeStream = fs.createWriteStream(destinationPath);
73
+
74
+ await pipeline(stream, writeStream);
75
+
76
+ // Generate full URL
77
+ const url = `http://${ipAddress}:${port}/files/${requestId}/${sanitizedFilename}`;
78
+
79
+ return url;
80
+ }
81
+
55
82
  async deleteFiles(requestId) {
56
83
  if (!requestId) throw new Error("Missing requestId parameter");
57
84
 
@@ -13,11 +13,33 @@ async function getBlobHandlerConstants() {
13
13
  return blobHandlerConstants;
14
14
  }
15
15
 
16
+ // Singleton instance for provider caching across the application
17
+ let storageFactoryInstance = null;
18
+
16
19
  export class StorageFactory {
17
20
  constructor() {
18
21
  this.providers = new Map();
19
22
  }
20
23
 
24
+ /**
25
+ * Get the singleton instance of StorageFactory
26
+ * This ensures provider caching works across the entire application
27
+ */
28
+ static getInstance() {
29
+ if (!storageFactoryInstance) {
30
+ storageFactoryInstance = new StorageFactory();
31
+ }
32
+ return storageFactoryInstance;
33
+ }
34
+
35
+ /**
36
+ * Reset the singleton instance (useful for testing)
37
+ * @internal
38
+ */
39
+ static resetInstance() {
40
+ storageFactoryInstance = null;
41
+ }
42
+
21
43
  async getPrimaryProvider(containerName = null) {
22
44
  if (process.env.AZURE_STORAGE_CONNECTION_STRING) {
23
45
  return await this.getAzureProvider(containerName);
@@ -26,14 +48,17 @@ export class StorageFactory {
26
48
  }
27
49
 
28
50
  async getAzureProvider(containerName = null) {
29
- const { AZURE_STORAGE_CONTAINER_NAMES, DEFAULT_AZURE_STORAGE_CONTAINER_NAME, isValidContainerName } = await getBlobHandlerConstants();
51
+ // Read container names from environment directly to get current values
52
+ const { getCurrentContainerNames } = await getBlobHandlerConstants();
53
+ const azureStorageContainerNames = getCurrentContainerNames();
54
+ const defaultAzureStorageContainerName = azureStorageContainerNames[0];
30
55
 
31
56
  // Use provided container name or default to first in whitelist
32
- const finalContainerName = containerName || DEFAULT_AZURE_STORAGE_CONTAINER_NAME;
57
+ const finalContainerName = containerName || defaultAzureStorageContainerName;
33
58
 
34
59
  // Validate container name
35
- if (!isValidContainerName(finalContainerName)) {
36
- throw new Error(`Invalid container name '${finalContainerName}'. Allowed containers: ${AZURE_STORAGE_CONTAINER_NAMES.join(', ')}`);
60
+ if (!azureStorageContainerNames.includes(finalContainerName)) {
61
+ throw new Error(`Invalid container name '${finalContainerName}'. Allowed containers: ${azureStorageContainerNames.join(', ')}`);
37
62
  }
38
63
 
39
64
  // Create unique key for each container
@@ -15,6 +15,17 @@ export class StorageProvider {
15
15
  throw new Error("Method not implemented");
16
16
  }
17
17
 
18
+ /**
19
+ * Upload a stream to storage
20
+ * @param {Object} context - The context object
21
+ * @param {string} encodedFilename - The filename for the uploaded file
22
+ * @param {stream.Readable} stream - The stream to upload
23
+ * @returns {Promise<string>} The URL of the uploaded file with SAS token
24
+ */
25
+ async uploadStream(context, encodedFilename, stream) {
26
+ throw new Error("Method not implemented");
27
+ }
28
+
18
29
  /**
19
30
  * Delete files associated with a request ID
20
31
  * @param {string} requestId - The request ID to delete files for
@@ -6,7 +6,7 @@ import { generateShortId } from "../../utils/filenameUtils.js";
6
6
 
7
7
  export class StorageService {
8
8
  constructor(factory) {
9
- this.factory = factory || new StorageFactory();
9
+ this.factory = factory || StorageFactory.getInstance();
10
10
  this.primaryProvider = null;
11
11
  this.backupProvider = null;
12
12
  this._initialized = false;
@@ -337,8 +337,9 @@ test("DEFAULT_AZURE_STORAGE_CONTAINER_NAME should be the first container", (t) =
337
337
  });
338
338
 
339
339
  test("isValidContainerName should validate container names correctly", (t) => {
340
- // All configured containers should be valid
341
- AZURE_STORAGE_CONTAINER_NAMES.forEach(containerName => {
340
+ // All configured containers should be valid (getCurrentContainerNames reads from env at runtime)
341
+ const currentContainers = AZURE_STORAGE_CONTAINER_NAMES;
342
+ currentContainers.forEach(containerName => {
342
343
  t.true(isValidContainerName(containerName), `${containerName} should be valid`);
343
344
  });
344
345
 
@@ -404,6 +404,12 @@ test.serial("checkHash should maintain consistent behavior across multiple calls
404
404
 
405
405
  t.is(checkResponse.status, 200, `checkHash call ${i + 1} should succeed`);
406
406
  responses.push(checkResponse.data);
407
+
408
+ // Add small delay to ensure different timestamps in SAS tokens
409
+ // Azure SAS tokens have second-level precision
410
+ if (i < 2 && isUsingAzureStorage()) {
411
+ await new Promise(resolve => setTimeout(resolve, 1100));
412
+ }
407
413
  }
408
414
 
409
415
  // All responses should have the required fields
@@ -415,8 +421,9 @@ test.serial("checkHash should maintain consistent behavior across multiple calls
415
421
  t.truthy(response.url, `Response ${index + 1} should have original URL`);
416
422
  }
417
423
 
418
- if (isAzureConfigured()) {
424
+ if (isUsingAzureStorage()) {
419
425
  // All shortLivedUrls should be different (new SAS tokens each time)
426
+ // Azure (including Azurite) generates new SAS tokens with current timestamp
420
427
  const shortLivedUrls = responses.map(r => r.shortLivedUrl);
421
428
  const uniqueUrls = new Set(shortLivedUrls);
422
429
  t.is(uniqueUrls.size, shortLivedUrls.length, "Each call should generate unique short-lived URL with Azure");
@@ -394,8 +394,11 @@ test("Conversion should use default container when no container specified", asyn
394
394
 
395
395
  // Verify the URL indicates it was uploaded to the default container
396
396
  const containerFromUrl = getContainerFromUrl(result.url);
397
- t.is(containerFromUrl, AZURE_STORAGE_CONTAINER_NAMES[0],
398
- `File should be uploaded to default container ${AZURE_STORAGE_CONTAINER_NAMES[0]}, but was uploaded to ${containerFromUrl}`);
397
+ // Read current default from environment (not the cached module value)
398
+ const currentContainerStr = process.env.AZURE_STORAGE_CONTAINER_NAME || "whispertempfiles";
399
+ const currentDefaultContainer = currentContainerStr.split(',').map(name => name.trim())[0];
400
+ t.is(containerFromUrl, currentDefaultContainer,
401
+ `File should be uploaded to default container ${currentDefaultContainer}, but was uploaded to ${containerFromUrl}`);
399
402
 
400
403
  // Cleanup
401
404
  await cleanupHashAndFile(null, result.url, baseUrl);
@@ -2,7 +2,8 @@ import test from "ava";
2
2
  import {
3
3
  AZURE_STORAGE_CONTAINER_NAMES,
4
4
  DEFAULT_AZURE_STORAGE_CONTAINER_NAME,
5
- isValidContainerName
5
+ isValidContainerName,
6
+ getCurrentContainerNames
6
7
  } from "../src/blobHandler.js";
7
8
 
8
9
  // Mock environment variables for testing
@@ -130,8 +131,11 @@ test("DEFAULT_AZURE_STORAGE_CONTAINER_NAME should be the first container in the
130
131
  });
131
132
 
132
133
  test("isValidContainerName should return true for valid container names", (t) => {
134
+ // Get current container names at runtime (not cached)
135
+ const currentContainers = getCurrentContainerNames();
136
+
133
137
  // Test with each container name from the current configuration
134
- AZURE_STORAGE_CONTAINER_NAMES.forEach(containerName => {
138
+ currentContainers.forEach(containerName => {
135
139
  t.true(isValidContainerName(containerName), `Container name '${containerName}' should be valid`);
136
140
  });
137
141
  });
@@ -163,12 +167,14 @@ test("isValidContainerName should handle edge cases", (t) => {
163
167
  });
164
168
 
165
169
  test("container configuration should have at least one container", (t) => {
166
- t.true(AZURE_STORAGE_CONTAINER_NAMES.length > 0, "Should have at least one container configured");
167
- t.truthy(AZURE_STORAGE_CONTAINER_NAMES[0], "First container should not be empty");
170
+ const currentContainers = getCurrentContainerNames();
171
+ t.true(currentContainers.length > 0, "Should have at least one container configured");
172
+ t.truthy(currentContainers[0], "First container should not be empty");
168
173
  });
169
174
 
170
175
  test("all configured container names should be non-empty strings", (t) => {
171
- AZURE_STORAGE_CONTAINER_NAMES.forEach((containerName, index) => {
176
+ const currentContainers = getCurrentContainerNames();
177
+ currentContainers.forEach((containerName, index) => {
172
178
  t.is(typeof containerName, 'string', `Container at index ${index} should be a string`);
173
179
  t.true(containerName.length > 0, `Container at index ${index} should not be empty`);
174
180
  t.true(containerName.trim().length > 0, `Container at index ${index} should not be only whitespace`);
@@ -176,8 +182,9 @@ test("all configured container names should be non-empty strings", (t) => {
176
182
  });
177
183
 
178
184
  test("container names should not contain duplicates", (t) => {
179
- const uniqueNames = new Set(AZURE_STORAGE_CONTAINER_NAMES);
180
- t.is(uniqueNames.size, AZURE_STORAGE_CONTAINER_NAMES.length, "Container names should be unique");
185
+ const currentContainers = getCurrentContainerNames();
186
+ const uniqueNames = new Set(currentContainers);
187
+ t.is(uniqueNames.size, currentContainers.length, "Container names should be unique");
181
188
  });
182
189
 
183
190
  // Integration test with actual environment simulation