@aj-archipelago/cortex 1.4.2 → 1.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/config.js +1 -1
- package/helper-apps/cortex-autogen2/.dockerignore +1 -0
- package/helper-apps/cortex-autogen2/Dockerfile +6 -10
- package/helper-apps/cortex-autogen2/Dockerfile.worker +2 -0
- package/helper-apps/cortex-autogen2/agents.py +203 -2
- package/helper-apps/cortex-autogen2/main.py +1 -1
- package/helper-apps/cortex-autogen2/pyproject.toml +12 -0
- package/helper-apps/cortex-autogen2/requirements.txt +14 -0
- package/helper-apps/cortex-autogen2/services/redis_publisher.py +1 -1
- package/helper-apps/cortex-autogen2/services/run_analyzer.py +1 -1
- package/helper-apps/cortex-autogen2/task_processor.py +431 -229
- package/helper-apps/cortex-autogen2/test_entity_fetcher.py +305 -0
- package/helper-apps/cortex-autogen2/tests/README.md +240 -0
- package/helper-apps/cortex-autogen2/tests/TEST_REPORT.md +342 -0
- package/helper-apps/cortex-autogen2/tests/__init__.py +8 -0
- package/helper-apps/cortex-autogen2/tests/analysis/__init__.py +1 -0
- package/helper-apps/cortex-autogen2/tests/analysis/improvement_suggester.py +224 -0
- package/helper-apps/cortex-autogen2/tests/analysis/trend_analyzer.py +211 -0
- package/helper-apps/cortex-autogen2/tests/cli/__init__.py +1 -0
- package/helper-apps/cortex-autogen2/tests/cli/run_tests.py +296 -0
- package/helper-apps/cortex-autogen2/tests/collectors/__init__.py +1 -0
- package/helper-apps/cortex-autogen2/tests/collectors/log_collector.py +252 -0
- package/helper-apps/cortex-autogen2/tests/collectors/progress_collector.py +182 -0
- package/helper-apps/cortex-autogen2/tests/conftest.py +15 -0
- package/helper-apps/cortex-autogen2/tests/database/__init__.py +1 -0
- package/helper-apps/cortex-autogen2/tests/database/repository.py +501 -0
- package/helper-apps/cortex-autogen2/tests/database/schema.sql +108 -0
- package/helper-apps/cortex-autogen2/tests/evaluators/__init__.py +1 -0
- package/helper-apps/cortex-autogen2/tests/evaluators/llm_scorer.py +294 -0
- package/helper-apps/cortex-autogen2/tests/evaluators/prompts.py +250 -0
- package/helper-apps/cortex-autogen2/tests/evaluators/wordcloud_validator.py +168 -0
- package/helper-apps/cortex-autogen2/tests/metrics/__init__.py +1 -0
- package/helper-apps/cortex-autogen2/tests/metrics/collector.py +155 -0
- package/helper-apps/cortex-autogen2/tests/orchestrator.py +576 -0
- package/helper-apps/cortex-autogen2/tests/test_cases.yaml +279 -0
- package/helper-apps/cortex-autogen2/tests/test_data.db +0 -0
- package/helper-apps/cortex-autogen2/tests/utils/__init__.py +3 -0
- package/helper-apps/cortex-autogen2/tests/utils/connectivity.py +112 -0
- package/helper-apps/cortex-autogen2/tools/azure_blob_tools.py +74 -24
- package/helper-apps/cortex-autogen2/tools/entity_api_registry.json +38 -0
- package/helper-apps/cortex-autogen2/tools/file_tools.py +1 -1
- package/helper-apps/cortex-autogen2/tools/search_tools.py +436 -238
- package/helper-apps/cortex-file-handler/package-lock.json +2 -2
- package/helper-apps/cortex-file-handler/package.json +1 -1
- package/helper-apps/cortex-file-handler/scripts/setup-test-containers.js +4 -5
- package/helper-apps/cortex-file-handler/src/blobHandler.js +36 -144
- package/helper-apps/cortex-file-handler/src/services/FileConversionService.js +5 -3
- package/helper-apps/cortex-file-handler/src/services/storage/AzureStorageProvider.js +34 -1
- package/helper-apps/cortex-file-handler/src/services/storage/GCSStorageProvider.js +22 -0
- package/helper-apps/cortex-file-handler/src/services/storage/LocalStorageProvider.js +28 -1
- package/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js +29 -4
- package/helper-apps/cortex-file-handler/src/services/storage/StorageProvider.js +11 -0
- package/helper-apps/cortex-file-handler/src/services/storage/StorageService.js +1 -1
- package/helper-apps/cortex-file-handler/tests/blobHandler.test.js +3 -2
- package/helper-apps/cortex-file-handler/tests/checkHashShortLived.test.js +8 -1
- package/helper-apps/cortex-file-handler/tests/containerConversionFlow.test.js +5 -2
- package/helper-apps/cortex-file-handler/tests/containerNameParsing.test.js +14 -7
- package/helper-apps/cortex-file-handler/tests/containerParameterFlow.test.js +5 -2
- package/helper-apps/cortex-file-handler/tests/storage/StorageFactory.test.js +31 -19
- package/package.json +1 -1
- package/server/modelExecutor.js +4 -0
- package/server/plugins/claude4VertexPlugin.js +540 -0
- package/server/plugins/openAiWhisperPlugin.js +43 -2
- package/tests/integration/rest/vendors/claude_streaming.test.js +121 -0
- package/tests/unit/plugins/claude4VertexPlugin.test.js +462 -0
- package/tests/unit/plugins/claude4VertexToolConversion.test.js +413 -0
- package/helper-apps/cortex-autogen/.funcignore +0 -8
- package/helper-apps/cortex-autogen/Dockerfile +0 -10
- package/helper-apps/cortex-autogen/OAI_CONFIG_LIST +0 -6
- package/helper-apps/cortex-autogen/agents.py +0 -493
- package/helper-apps/cortex-autogen/agents_extra.py +0 -14
- package/helper-apps/cortex-autogen/config.py +0 -18
- package/helper-apps/cortex-autogen/data_operations.py +0 -29
- package/helper-apps/cortex-autogen/function_app.py +0 -44
- package/helper-apps/cortex-autogen/host.json +0 -15
- package/helper-apps/cortex-autogen/main.py +0 -38
- package/helper-apps/cortex-autogen/prompts.py +0 -196
- package/helper-apps/cortex-autogen/prompts_extra.py +0 -5
- package/helper-apps/cortex-autogen/requirements.txt +0 -9
- package/helper-apps/cortex-autogen/search.py +0 -85
- package/helper-apps/cortex-autogen/test.sh +0 -40
- package/helper-apps/cortex-autogen/tools/sasfileuploader.py +0 -66
- package/helper-apps/cortex-autogen/utils.py +0 -88
- package/helper-apps/cortex-autogen2/DigiCertGlobalRootCA.crt.pem +0 -22
- package/helper-apps/cortex-autogen2/poetry.lock +0 -3652
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aj-archipelago/cortex-file-handler",
|
|
3
|
-
"version": "2.6.
|
|
3
|
+
"version": "2.6.2",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@aj-archipelago/cortex-file-handler",
|
|
9
|
-
"version": "2.6.
|
|
9
|
+
"version": "2.6.2",
|
|
10
10
|
"dependencies": {
|
|
11
11
|
"@azure/storage-blob": "^12.13.0",
|
|
12
12
|
"@distube/ytdl-core": "^4.14.3",
|
|
@@ -7,13 +7,12 @@ async function createAzureContainers() {
|
|
|
7
7
|
"UseDevelopmentStorage=true",
|
|
8
8
|
);
|
|
9
9
|
|
|
10
|
-
//
|
|
11
|
-
const
|
|
12
|
-
const containerNames = containerStr.split(',').map(name => name.trim()).filter(name => name.length > 0);
|
|
10
|
+
// Always create all possible test containers to support dynamic test environments
|
|
11
|
+
const allTestContainers = ["default", "test-container", "test1", "test2", "test3", "container1", "container2", "container3"];
|
|
13
12
|
|
|
14
|
-
console.log(`Creating Azure containers: ${
|
|
13
|
+
console.log(`Creating Azure containers: ${allTestContainers.join(', ')}`);
|
|
15
14
|
|
|
16
|
-
for (const containerName of
|
|
15
|
+
for (const containerName of allTestContainers) {
|
|
17
16
|
try {
|
|
18
17
|
const containerClient = blobServiceClient.getContainerClient(containerName);
|
|
19
18
|
await containerClient.create();
|
|
@@ -6,13 +6,8 @@ import { pipeline as _pipeline } from "stream";
|
|
|
6
6
|
import { v4 as uuidv4 } from "uuid";
|
|
7
7
|
import Busboy from "busboy";
|
|
8
8
|
import { PassThrough } from "stream";
|
|
9
|
-
import mime from "mime-types";
|
|
10
9
|
import { Storage } from "@google-cloud/storage";
|
|
11
|
-
import {
|
|
12
|
-
generateBlobSASQueryParameters,
|
|
13
|
-
StorageSharedKeyCredential,
|
|
14
|
-
BlobServiceClient,
|
|
15
|
-
} from "@azure/storage-blob";
|
|
10
|
+
import { BlobServiceClient } from "@azure/storage-blob";
|
|
16
11
|
import axios from "axios";
|
|
17
12
|
|
|
18
13
|
import {
|
|
@@ -23,6 +18,7 @@ import {
|
|
|
23
18
|
import { publicFolder, port, ipAddress } from "./start.js";
|
|
24
19
|
import { CONVERTED_EXTENSIONS, AZURITE_ACCOUNT_NAME } from "./constants.js";
|
|
25
20
|
import { FileConversionService } from "./services/FileConversionService.js";
|
|
21
|
+
import { StorageFactory } from "./services/storage/StorageFactory.js";
|
|
26
22
|
|
|
27
23
|
const pipeline = promisify(_pipeline);
|
|
28
24
|
|
|
@@ -80,13 +76,20 @@ const parseContainerNames = () => {
|
|
|
80
76
|
return containerStr.split(',').map(name => name.trim());
|
|
81
77
|
};
|
|
82
78
|
|
|
79
|
+
// Helper function to get current container names at runtime
|
|
80
|
+
export const getCurrentContainerNames = () => {
|
|
81
|
+
return parseContainerNames();
|
|
82
|
+
};
|
|
83
|
+
|
|
83
84
|
export const AZURE_STORAGE_CONTAINER_NAMES = parseContainerNames();
|
|
84
85
|
export const DEFAULT_AZURE_STORAGE_CONTAINER_NAME = AZURE_STORAGE_CONTAINER_NAMES[0];
|
|
85
86
|
export const GCS_BUCKETNAME = process.env.GCS_BUCKETNAME || "cortextempfiles";
|
|
86
87
|
|
|
87
88
|
// Validate if a container name is allowed
|
|
88
89
|
export const isValidContainerName = (containerName) => {
|
|
89
|
-
|
|
90
|
+
// Read from environment at runtime to support dynamically changing env in tests
|
|
91
|
+
const currentContainerNames = getCurrentContainerNames();
|
|
92
|
+
return currentContainerNames.includes(containerName);
|
|
90
93
|
};
|
|
91
94
|
|
|
92
95
|
function isEncoded(str) {
|
|
@@ -221,95 +224,12 @@ export const getBlobClient = async (containerName = null) => {
|
|
|
221
224
|
};
|
|
222
225
|
|
|
223
226
|
async function saveFileToBlob(chunkPath, requestId, filename = null, containerName = null) {
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
blobName = generateBlobName(requestId, filename);
|
|
229
|
-
} else {
|
|
230
|
-
const fileExtension = path.extname(chunkPath);
|
|
231
|
-
const shortId = generateShortId();
|
|
232
|
-
blobName = generateBlobName(requestId, `${shortId}${fileExtension}`);
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
// Create a read stream for the chunk file
|
|
236
|
-
const fileStream = fs.createReadStream(chunkPath);
|
|
237
|
-
|
|
238
|
-
// Upload the chunk to Azure Blob Storage using the stream
|
|
239
|
-
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
|
|
240
|
-
await blockBlobClient.uploadStream(fileStream);
|
|
241
|
-
|
|
242
|
-
// Generate SAS token after successful upload
|
|
243
|
-
const sasToken = generateSASToken(containerClient, blobName);
|
|
244
|
-
|
|
245
|
-
// Return an object with the URL property
|
|
246
|
-
return {
|
|
247
|
-
url: `${blockBlobClient.url}?${sasToken}`,
|
|
248
|
-
blobName: blobName,
|
|
249
|
-
};
|
|
227
|
+
// Use provider for consistency with cache control headers
|
|
228
|
+
const storageFactory = StorageFactory.getInstance();
|
|
229
|
+
const provider = await storageFactory.getAzureProvider(containerName);
|
|
230
|
+
return await provider.uploadFile({}, chunkPath, requestId, null, filename);
|
|
250
231
|
}
|
|
251
232
|
|
|
252
|
-
const generateSASToken = (
|
|
253
|
-
containerClient,
|
|
254
|
-
blobName,
|
|
255
|
-
options = {},
|
|
256
|
-
) => {
|
|
257
|
-
// Handle Azurite (development storage) credentials with fallback
|
|
258
|
-
let accountName, accountKey;
|
|
259
|
-
|
|
260
|
-
if (containerClient.credential && containerClient.credential.accountName) {
|
|
261
|
-
// Regular Azure Storage credentials
|
|
262
|
-
accountName = containerClient.credential.accountName;
|
|
263
|
-
|
|
264
|
-
// Handle Buffer case (Azurite) vs string case (real Azure)
|
|
265
|
-
if (Buffer.isBuffer(containerClient.credential.accountKey)) {
|
|
266
|
-
accountKey = containerClient.credential.accountKey.toString('base64');
|
|
267
|
-
} else {
|
|
268
|
-
accountKey = containerClient.credential.accountKey;
|
|
269
|
-
}
|
|
270
|
-
} else {
|
|
271
|
-
// Azurite development storage fallback
|
|
272
|
-
accountName = AZURITE_ACCOUNT_NAME;
|
|
273
|
-
accountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";
|
|
274
|
-
}
|
|
275
|
-
|
|
276
|
-
const sharedKeyCredential = new StorageSharedKeyCredential(
|
|
277
|
-
accountName,
|
|
278
|
-
accountKey,
|
|
279
|
-
);
|
|
280
|
-
|
|
281
|
-
// Support custom duration: minutes, hours, or fall back to default
|
|
282
|
-
let expirationTime;
|
|
283
|
-
if (options.minutes) {
|
|
284
|
-
expirationTime = new Date(new Date().valueOf() + options.minutes * 60 * 1000);
|
|
285
|
-
} else if (options.hours) {
|
|
286
|
-
expirationTime = new Date(new Date().valueOf() + options.hours * 60 * 60 * 1000);
|
|
287
|
-
} else if (options.days) {
|
|
288
|
-
expirationTime = new Date(new Date().valueOf() + options.days * 24 * 60 * 60 * 1000);
|
|
289
|
-
} else if (options.expiryTimeSeconds !== undefined) {
|
|
290
|
-
// Legacy support for existing parameter
|
|
291
|
-
expirationTime = new Date(new Date().valueOf() + options.expiryTimeSeconds * 1000);
|
|
292
|
-
} else {
|
|
293
|
-
// Default to SAS_TOKEN_LIFE_DAYS environment variable
|
|
294
|
-
const defaultExpirySeconds = parseInt(SAS_TOKEN_LIFE_DAYS) * 24 * 60 * 60;
|
|
295
|
-
expirationTime = new Date(new Date().valueOf() + defaultExpirySeconds * 1000);
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
const sasOptions = {
|
|
299
|
-
containerName: containerClient.containerName,
|
|
300
|
-
blobName: blobName,
|
|
301
|
-
permissions: options.permissions || "r", // Read permission
|
|
302
|
-
startsOn: new Date(),
|
|
303
|
-
expiresOn: expirationTime,
|
|
304
|
-
};
|
|
305
|
-
|
|
306
|
-
const sasToken = generateBlobSASQueryParameters(
|
|
307
|
-
sasOptions,
|
|
308
|
-
sharedKeyCredential,
|
|
309
|
-
).toString();
|
|
310
|
-
return sasToken;
|
|
311
|
-
};
|
|
312
|
-
|
|
313
233
|
//deletes blob that has the requestId
|
|
314
234
|
async function deleteBlob(requestId, containerName = null) {
|
|
315
235
|
if (!requestId) throw new Error("Missing requestId parameter");
|
|
@@ -396,8 +316,10 @@ function uploadBlob(
|
|
|
396
316
|
hash = value;
|
|
397
317
|
} else if (fieldname === "container") {
|
|
398
318
|
if (value && !isValidContainerName(value)) {
|
|
319
|
+
// Read current containers from env for error message
|
|
320
|
+
const currentContainerNames = getCurrentContainerNames();
|
|
399
321
|
errorOccurred = true;
|
|
400
|
-
const err = new Error(`Invalid container name '${value}'. Allowed containers: ${
|
|
322
|
+
const err = new Error(`Invalid container name '${value}'. Allowed containers: ${currentContainerNames.join(', ')}`);
|
|
401
323
|
err.status = 400;
|
|
402
324
|
reject(err);
|
|
403
325
|
return;
|
|
@@ -767,57 +689,17 @@ function uploadBlob(
|
|
|
767
689
|
|
|
768
690
|
// Helper function to handle local file storage
|
|
769
691
|
async function saveToLocalStorage(context, requestId, encodedFilename, file) {
|
|
770
|
-
const
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
const sanitizedFilename = sanitizeFilename(encodedFilename);
|
|
775
|
-
const destinationPath = `${localPath}/${sanitizedFilename}`;
|
|
776
|
-
|
|
777
|
-
await pipeline(file, fs.createWriteStream(destinationPath));
|
|
778
|
-
return `http://${ipAddress}:${port}/files/${requestId}/${sanitizedFilename}`;
|
|
692
|
+
const storageFactory = StorageFactory.getInstance();
|
|
693
|
+
const localProvider = storageFactory.getLocalProvider();
|
|
694
|
+
const contextWithRequestId = { ...context, requestId };
|
|
695
|
+
return await localProvider.uploadStream(contextWithRequestId, encodedFilename, file);
|
|
779
696
|
}
|
|
780
697
|
|
|
781
698
|
// Helper function to handle Azure blob storage
|
|
782
699
|
async function saveToAzureStorage(context, encodedFilename, file, containerName = null) {
|
|
783
|
-
const
|
|
784
|
-
const
|
|
785
|
-
|
|
786
|
-
// Create a safe blob name that is URI-encoded once (no double encoding)
|
|
787
|
-
let blobName = sanitizeFilename(encodedFilename);
|
|
788
|
-
blobName = encodeURIComponent(blobName);
|
|
789
|
-
|
|
790
|
-
const options = {
|
|
791
|
-
blobHTTPHeaders: contentType ? { blobContentType: contentType } : {},
|
|
792
|
-
maxConcurrency: 50,
|
|
793
|
-
blockSize: 8 * 1024 * 1024,
|
|
794
|
-
};
|
|
795
|
-
|
|
796
|
-
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
|
|
797
|
-
console.log("Uploading to container:", containerName || "default");
|
|
798
|
-
context.log(`Uploading to Azure... ${blobName}`);
|
|
799
|
-
await blockBlobClient.uploadStream(file, undefined, undefined, options);
|
|
800
|
-
const sasToken = generateSASToken(containerClient, blobName);
|
|
801
|
-
return `${blockBlobClient.url}?${sasToken}`;
|
|
802
|
-
}
|
|
803
|
-
|
|
804
|
-
// Helper function to upload a file to Google Cloud Storage
|
|
805
|
-
async function uploadToGCS(context, file, filename) {
|
|
806
|
-
const objectName = sanitizeFilename(filename);
|
|
807
|
-
const gcsFile = gcs.bucket(GCS_BUCKETNAME).file(objectName);
|
|
808
|
-
const writeStream = gcsFile.createWriteStream({
|
|
809
|
-
resumable: true,
|
|
810
|
-
validation: false,
|
|
811
|
-
metadata: {
|
|
812
|
-
contentType: mime.lookup(objectName) || "application/octet-stream",
|
|
813
|
-
},
|
|
814
|
-
chunkSize: 8 * 1024 * 1024,
|
|
815
|
-
numRetries: 3,
|
|
816
|
-
retryDelay: 1000,
|
|
817
|
-
});
|
|
818
|
-
context.log(`Uploading to GCS... ${objectName}`);
|
|
819
|
-
await pipeline(file, writeStream);
|
|
820
|
-
return `gs://${GCS_BUCKETNAME}/${objectName}`;
|
|
700
|
+
const storageFactory = StorageFactory.getInstance();
|
|
701
|
+
const provider = await storageFactory.getAzureProvider(containerName);
|
|
702
|
+
return await provider.uploadStream(context, encodedFilename, file);
|
|
821
703
|
}
|
|
822
704
|
|
|
823
705
|
// Wrapper that checks if GCS is configured
|
|
@@ -825,7 +707,12 @@ async function saveToGoogleStorage(context, encodedFilename, file) {
|
|
|
825
707
|
if (!gcs) {
|
|
826
708
|
throw new Error("Google Cloud Storage is not initialized");
|
|
827
709
|
}
|
|
828
|
-
|
|
710
|
+
const storageFactory = StorageFactory.getInstance();
|
|
711
|
+
const gcsProvider = storageFactory.getGCSProvider();
|
|
712
|
+
if (!gcsProvider) {
|
|
713
|
+
throw new Error("GCS provider not available");
|
|
714
|
+
}
|
|
715
|
+
return await gcsProvider.uploadStream(context, encodedFilename, file);
|
|
829
716
|
}
|
|
830
717
|
|
|
831
718
|
async function uploadFile(
|
|
@@ -1229,7 +1116,12 @@ async function ensureGCSUpload(context, existingFile) {
|
|
|
1229
1116
|
url: existingFile.url,
|
|
1230
1117
|
responseType: "stream",
|
|
1231
1118
|
});
|
|
1232
|
-
|
|
1119
|
+
|
|
1120
|
+
const storageFactory = StorageFactory.getInstance();
|
|
1121
|
+
const gcsProvider = storageFactory.getGCSProvider();
|
|
1122
|
+
if (gcsProvider) {
|
|
1123
|
+
existingFile.gcs = await gcsProvider.uploadStream(context, fileName, response.data);
|
|
1124
|
+
}
|
|
1233
1125
|
}
|
|
1234
1126
|
return existingFile;
|
|
1235
1127
|
}
|
|
@@ -3,7 +3,7 @@ import { getFileStoreMap, setFileStoreMap } from "../redis.js";
|
|
|
3
3
|
import { urlExists } from "../helper.js";
|
|
4
4
|
import { gcsUrlExists, uploadChunkToGCS, gcs } from "../blobHandler.js";
|
|
5
5
|
import { downloadFile } from "../fileChunker.js";
|
|
6
|
-
import {
|
|
6
|
+
import { StorageFactory } from "./storage/StorageFactory.js";
|
|
7
7
|
import { moveFileToPublicFolder } from "../localFileHandler.js";
|
|
8
8
|
import { v4 as uuidv4 } from "uuid";
|
|
9
9
|
|
|
@@ -11,6 +11,7 @@ export class FileConversionService extends ConversionService {
|
|
|
11
11
|
constructor(context, useAzure = true) {
|
|
12
12
|
super(context);
|
|
13
13
|
this.useAzure = useAzure;
|
|
14
|
+
this.storageFactory = StorageFactory.getInstance();
|
|
14
15
|
}
|
|
15
16
|
|
|
16
17
|
async _getFileStoreMap(key) {
|
|
@@ -39,8 +40,9 @@ export class FileConversionService extends ConversionService {
|
|
|
39
40
|
|
|
40
41
|
let fileUrl;
|
|
41
42
|
if (this.useAzure) {
|
|
42
|
-
const
|
|
43
|
-
|
|
43
|
+
const provider = await this.storageFactory.getAzureProvider(containerName);
|
|
44
|
+
const result = await provider.uploadFile({}, filePath, reqId, null, filename);
|
|
45
|
+
fileUrl = result.url;
|
|
44
46
|
} else {
|
|
45
47
|
fileUrl = await moveFileToPublicFolder(filePath, reqId);
|
|
46
48
|
}
|
|
@@ -5,12 +5,14 @@ import {
|
|
|
5
5
|
} from "@azure/storage-blob";
|
|
6
6
|
import fs from "fs";
|
|
7
7
|
import path from "path";
|
|
8
|
+
import mime from "mime-types";
|
|
8
9
|
|
|
9
10
|
import { StorageProvider } from "./StorageProvider.js";
|
|
10
11
|
import { AZURITE_ACCOUNT_NAME } from "../../constants.js";
|
|
11
12
|
import {
|
|
12
13
|
generateShortId,
|
|
13
14
|
generateBlobName,
|
|
15
|
+
sanitizeFilename,
|
|
14
16
|
} from "../../utils/filenameUtils.js";
|
|
15
17
|
|
|
16
18
|
export class AzureStorageProvider extends StorageProvider {
|
|
@@ -122,7 +124,12 @@ export class AzureStorageProvider extends StorageProvider {
|
|
|
122
124
|
|
|
123
125
|
// Upload the file to Azure Blob Storage using the stream
|
|
124
126
|
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
|
|
125
|
-
|
|
127
|
+
const uploadOptions = {
|
|
128
|
+
blobHTTPHeaders: {
|
|
129
|
+
blobCacheControl: 'public, max-age=2592000, immutable',
|
|
130
|
+
},
|
|
131
|
+
};
|
|
132
|
+
await blockBlobClient.uploadStream(fileStream, undefined, undefined, uploadOptions);
|
|
126
133
|
|
|
127
134
|
// Generate SAS token after successful upload
|
|
128
135
|
const sasToken = this.generateSASToken(containerClient, blobName);
|
|
@@ -133,6 +140,32 @@ export class AzureStorageProvider extends StorageProvider {
|
|
|
133
140
|
};
|
|
134
141
|
}
|
|
135
142
|
|
|
143
|
+
async uploadStream(context, encodedFilename, stream) {
|
|
144
|
+
const { containerClient } = await this.getBlobClient();
|
|
145
|
+
const contentType = mime.lookup(encodedFilename);
|
|
146
|
+
|
|
147
|
+
// Normalize the blob name: sanitizeFilename decodes, cleans, then we encode for Azure
|
|
148
|
+
let blobName = sanitizeFilename(encodedFilename);
|
|
149
|
+
blobName = encodeURIComponent(blobName);
|
|
150
|
+
|
|
151
|
+
const options = {
|
|
152
|
+
blobHTTPHeaders: {
|
|
153
|
+
...(contentType ? { blobContentType: contentType } : {}),
|
|
154
|
+
blobCacheControl: 'public, max-age=2592000, immutable',
|
|
155
|
+
},
|
|
156
|
+
maxConcurrency: 50,
|
|
157
|
+
blockSize: 8 * 1024 * 1024,
|
|
158
|
+
};
|
|
159
|
+
|
|
160
|
+
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
|
|
161
|
+
if (context.log) context.log(`Uploading to Azure... ${blobName}`);
|
|
162
|
+
|
|
163
|
+
await blockBlobClient.uploadStream(stream, undefined, undefined, options);
|
|
164
|
+
const sasToken = this.generateSASToken(containerClient, blobName);
|
|
165
|
+
|
|
166
|
+
return `${blockBlobClient.url}?${sasToken}`;
|
|
167
|
+
}
|
|
168
|
+
|
|
136
169
|
async deleteFiles(requestId) {
|
|
137
170
|
if (!requestId) throw new Error("Missing requestId parameter");
|
|
138
171
|
const { containerClient } = await this.getBlobClient();
|
|
@@ -4,6 +4,7 @@ import path from "path";
|
|
|
4
4
|
import {
|
|
5
5
|
generateShortId,
|
|
6
6
|
generateBlobName,
|
|
7
|
+
sanitizeFilename,
|
|
7
8
|
} from "../../utils/filenameUtils.js";
|
|
8
9
|
import axios from "axios";
|
|
9
10
|
|
|
@@ -72,6 +73,27 @@ export class GCSStorageProvider extends StorageProvider {
|
|
|
72
73
|
};
|
|
73
74
|
}
|
|
74
75
|
|
|
76
|
+
async uploadStream(context, encodedFilename, stream) {
|
|
77
|
+
const bucket = this.storage.bucket(this.bucketName);
|
|
78
|
+
const blobName = sanitizeFilename(encodedFilename);
|
|
79
|
+
|
|
80
|
+
const file = bucket.file(blobName);
|
|
81
|
+
const writeStream = file.createWriteStream({
|
|
82
|
+
metadata: {
|
|
83
|
+
contentType: this.getContentType(encodedFilename) || "application/octet-stream",
|
|
84
|
+
},
|
|
85
|
+
resumable: false,
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
await new Promise((resolve, reject) => {
|
|
89
|
+
stream.pipe(writeStream)
|
|
90
|
+
.on('finish', resolve)
|
|
91
|
+
.on('error', reject);
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
return `gs://${this.bucketName}/${blobName}`;
|
|
95
|
+
}
|
|
96
|
+
|
|
75
97
|
async deleteFiles(requestId) {
|
|
76
98
|
if (!requestId) throw new Error("Missing requestId parameter");
|
|
77
99
|
|
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
import fs from "fs";
|
|
2
2
|
import path from "path";
|
|
3
|
-
import {
|
|
3
|
+
import { promisify } from "util";
|
|
4
|
+
import { pipeline as _pipeline } from "stream";
|
|
5
|
+
import { generateShortId, sanitizeFilename } from "../../utils/filenameUtils.js";
|
|
4
6
|
import { ipAddress, port } from "../../start.js";
|
|
5
7
|
|
|
6
8
|
import { StorageProvider } from "./StorageProvider.js";
|
|
7
9
|
|
|
10
|
+
const pipeline = promisify(_pipeline);
|
|
11
|
+
|
|
8
12
|
export class LocalStorageProvider extends StorageProvider {
|
|
9
13
|
constructor(publicFolder) {
|
|
10
14
|
super();
|
|
@@ -52,6 +56,29 @@ export class LocalStorageProvider extends StorageProvider {
|
|
|
52
56
|
};
|
|
53
57
|
}
|
|
54
58
|
|
|
59
|
+
async uploadStream(context, encodedFilename, stream) {
|
|
60
|
+
// For local storage, we need a requestId to create a folder
|
|
61
|
+
// Extract from context or generate one
|
|
62
|
+
const requestId = context.requestId || "default";
|
|
63
|
+
|
|
64
|
+
// Create request folder if it doesn't exist
|
|
65
|
+
const requestFolder = path.join(this.publicFolder, requestId);
|
|
66
|
+
if (!fs.existsSync(requestFolder)) {
|
|
67
|
+
fs.mkdirSync(requestFolder, { recursive: true });
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const sanitizedFilename = sanitizeFilename(encodedFilename);
|
|
71
|
+
const destinationPath = path.join(requestFolder, sanitizedFilename);
|
|
72
|
+
const writeStream = fs.createWriteStream(destinationPath);
|
|
73
|
+
|
|
74
|
+
await pipeline(stream, writeStream);
|
|
75
|
+
|
|
76
|
+
// Generate full URL
|
|
77
|
+
const url = `http://${ipAddress}:${port}/files/${requestId}/${sanitizedFilename}`;
|
|
78
|
+
|
|
79
|
+
return url;
|
|
80
|
+
}
|
|
81
|
+
|
|
55
82
|
async deleteFiles(requestId) {
|
|
56
83
|
if (!requestId) throw new Error("Missing requestId parameter");
|
|
57
84
|
|
|
@@ -13,11 +13,33 @@ async function getBlobHandlerConstants() {
|
|
|
13
13
|
return blobHandlerConstants;
|
|
14
14
|
}
|
|
15
15
|
|
|
16
|
+
// Singleton instance for provider caching across the application
|
|
17
|
+
let storageFactoryInstance = null;
|
|
18
|
+
|
|
16
19
|
export class StorageFactory {
|
|
17
20
|
constructor() {
|
|
18
21
|
this.providers = new Map();
|
|
19
22
|
}
|
|
20
23
|
|
|
24
|
+
/**
|
|
25
|
+
* Get the singleton instance of StorageFactory
|
|
26
|
+
* This ensures provider caching works across the entire application
|
|
27
|
+
*/
|
|
28
|
+
static getInstance() {
|
|
29
|
+
if (!storageFactoryInstance) {
|
|
30
|
+
storageFactoryInstance = new StorageFactory();
|
|
31
|
+
}
|
|
32
|
+
return storageFactoryInstance;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Reset the singleton instance (useful for testing)
|
|
37
|
+
* @internal
|
|
38
|
+
*/
|
|
39
|
+
static resetInstance() {
|
|
40
|
+
storageFactoryInstance = null;
|
|
41
|
+
}
|
|
42
|
+
|
|
21
43
|
async getPrimaryProvider(containerName = null) {
|
|
22
44
|
if (process.env.AZURE_STORAGE_CONNECTION_STRING) {
|
|
23
45
|
return await this.getAzureProvider(containerName);
|
|
@@ -26,14 +48,17 @@ export class StorageFactory {
|
|
|
26
48
|
}
|
|
27
49
|
|
|
28
50
|
async getAzureProvider(containerName = null) {
|
|
29
|
-
|
|
51
|
+
// Read container names from environment directly to get current values
|
|
52
|
+
const { getCurrentContainerNames } = await getBlobHandlerConstants();
|
|
53
|
+
const azureStorageContainerNames = getCurrentContainerNames();
|
|
54
|
+
const defaultAzureStorageContainerName = azureStorageContainerNames[0];
|
|
30
55
|
|
|
31
56
|
// Use provided container name or default to first in whitelist
|
|
32
|
-
const finalContainerName = containerName ||
|
|
57
|
+
const finalContainerName = containerName || defaultAzureStorageContainerName;
|
|
33
58
|
|
|
34
59
|
// Validate container name
|
|
35
|
-
if (!
|
|
36
|
-
throw new Error(`Invalid container name '${finalContainerName}'. Allowed containers: ${
|
|
60
|
+
if (!azureStorageContainerNames.includes(finalContainerName)) {
|
|
61
|
+
throw new Error(`Invalid container name '${finalContainerName}'. Allowed containers: ${azureStorageContainerNames.join(', ')}`);
|
|
37
62
|
}
|
|
38
63
|
|
|
39
64
|
// Create unique key for each container
|
|
@@ -15,6 +15,17 @@ export class StorageProvider {
|
|
|
15
15
|
throw new Error("Method not implemented");
|
|
16
16
|
}
|
|
17
17
|
|
|
18
|
+
/**
|
|
19
|
+
* Upload a stream to storage
|
|
20
|
+
* @param {Object} context - The context object
|
|
21
|
+
* @param {string} encodedFilename - The filename for the uploaded file
|
|
22
|
+
* @param {stream.Readable} stream - The stream to upload
|
|
23
|
+
* @returns {Promise<string>} The URL of the uploaded file with SAS token
|
|
24
|
+
*/
|
|
25
|
+
async uploadStream(context, encodedFilename, stream) {
|
|
26
|
+
throw new Error("Method not implemented");
|
|
27
|
+
}
|
|
28
|
+
|
|
18
29
|
/**
|
|
19
30
|
* Delete files associated with a request ID
|
|
20
31
|
* @param {string} requestId - The request ID to delete files for
|
|
@@ -6,7 +6,7 @@ import { generateShortId } from "../../utils/filenameUtils.js";
|
|
|
6
6
|
|
|
7
7
|
export class StorageService {
|
|
8
8
|
constructor(factory) {
|
|
9
|
-
this.factory = factory ||
|
|
9
|
+
this.factory = factory || StorageFactory.getInstance();
|
|
10
10
|
this.primaryProvider = null;
|
|
11
11
|
this.backupProvider = null;
|
|
12
12
|
this._initialized = false;
|
|
@@ -337,8 +337,9 @@ test("DEFAULT_AZURE_STORAGE_CONTAINER_NAME should be the first container", (t) =
|
|
|
337
337
|
});
|
|
338
338
|
|
|
339
339
|
test("isValidContainerName should validate container names correctly", (t) => {
|
|
340
|
-
// All configured containers should be valid
|
|
341
|
-
|
|
340
|
+
// All configured containers should be valid (getCurrentContainerNames reads from env at runtime)
|
|
341
|
+
const currentContainers = AZURE_STORAGE_CONTAINER_NAMES;
|
|
342
|
+
currentContainers.forEach(containerName => {
|
|
342
343
|
t.true(isValidContainerName(containerName), `${containerName} should be valid`);
|
|
343
344
|
});
|
|
344
345
|
|
|
@@ -404,6 +404,12 @@ test.serial("checkHash should maintain consistent behavior across multiple calls
|
|
|
404
404
|
|
|
405
405
|
t.is(checkResponse.status, 200, `checkHash call ${i + 1} should succeed`);
|
|
406
406
|
responses.push(checkResponse.data);
|
|
407
|
+
|
|
408
|
+
// Add small delay to ensure different timestamps in SAS tokens
|
|
409
|
+
// Azure SAS tokens have second-level precision
|
|
410
|
+
if (i < 2 && isUsingAzureStorage()) {
|
|
411
|
+
await new Promise(resolve => setTimeout(resolve, 1100));
|
|
412
|
+
}
|
|
407
413
|
}
|
|
408
414
|
|
|
409
415
|
// All responses should have the required fields
|
|
@@ -415,8 +421,9 @@ test.serial("checkHash should maintain consistent behavior across multiple calls
|
|
|
415
421
|
t.truthy(response.url, `Response ${index + 1} should have original URL`);
|
|
416
422
|
}
|
|
417
423
|
|
|
418
|
-
if (
|
|
424
|
+
if (isUsingAzureStorage()) {
|
|
419
425
|
// All shortLivedUrls should be different (new SAS tokens each time)
|
|
426
|
+
// Azure (including Azurite) generates new SAS tokens with current timestamp
|
|
420
427
|
const shortLivedUrls = responses.map(r => r.shortLivedUrl);
|
|
421
428
|
const uniqueUrls = new Set(shortLivedUrls);
|
|
422
429
|
t.is(uniqueUrls.size, shortLivedUrls.length, "Each call should generate unique short-lived URL with Azure");
|
|
@@ -394,8 +394,11 @@ test("Conversion should use default container when no container specified", asyn
|
|
|
394
394
|
|
|
395
395
|
// Verify the URL indicates it was uploaded to the default container
|
|
396
396
|
const containerFromUrl = getContainerFromUrl(result.url);
|
|
397
|
-
|
|
398
|
-
|
|
397
|
+
// Read current default from environment (not the cached module value)
|
|
398
|
+
const currentContainerStr = process.env.AZURE_STORAGE_CONTAINER_NAME || "whispertempfiles";
|
|
399
|
+
const currentDefaultContainer = currentContainerStr.split(',').map(name => name.trim())[0];
|
|
400
|
+
t.is(containerFromUrl, currentDefaultContainer,
|
|
401
|
+
`File should be uploaded to default container ${currentDefaultContainer}, but was uploaded to ${containerFromUrl}`);
|
|
399
402
|
|
|
400
403
|
// Cleanup
|
|
401
404
|
await cleanupHashAndFile(null, result.url, baseUrl);
|
|
@@ -2,7 +2,8 @@ import test from "ava";
|
|
|
2
2
|
import {
|
|
3
3
|
AZURE_STORAGE_CONTAINER_NAMES,
|
|
4
4
|
DEFAULT_AZURE_STORAGE_CONTAINER_NAME,
|
|
5
|
-
isValidContainerName
|
|
5
|
+
isValidContainerName,
|
|
6
|
+
getCurrentContainerNames
|
|
6
7
|
} from "../src/blobHandler.js";
|
|
7
8
|
|
|
8
9
|
// Mock environment variables for testing
|
|
@@ -130,8 +131,11 @@ test("DEFAULT_AZURE_STORAGE_CONTAINER_NAME should be the first container in the
|
|
|
130
131
|
});
|
|
131
132
|
|
|
132
133
|
test("isValidContainerName should return true for valid container names", (t) => {
|
|
134
|
+
// Get current container names at runtime (not cached)
|
|
135
|
+
const currentContainers = getCurrentContainerNames();
|
|
136
|
+
|
|
133
137
|
// Test with each container name from the current configuration
|
|
134
|
-
|
|
138
|
+
currentContainers.forEach(containerName => {
|
|
135
139
|
t.true(isValidContainerName(containerName), `Container name '${containerName}' should be valid`);
|
|
136
140
|
});
|
|
137
141
|
});
|
|
@@ -163,12 +167,14 @@ test("isValidContainerName should handle edge cases", (t) => {
|
|
|
163
167
|
});
|
|
164
168
|
|
|
165
169
|
test("container configuration should have at least one container", (t) => {
|
|
166
|
-
|
|
167
|
-
t.
|
|
170
|
+
const currentContainers = getCurrentContainerNames();
|
|
171
|
+
t.true(currentContainers.length > 0, "Should have at least one container configured");
|
|
172
|
+
t.truthy(currentContainers[0], "First container should not be empty");
|
|
168
173
|
});
|
|
169
174
|
|
|
170
175
|
test("all configured container names should be non-empty strings", (t) => {
|
|
171
|
-
|
|
176
|
+
const currentContainers = getCurrentContainerNames();
|
|
177
|
+
currentContainers.forEach((containerName, index) => {
|
|
172
178
|
t.is(typeof containerName, 'string', `Container at index ${index} should be a string`);
|
|
173
179
|
t.true(containerName.length > 0, `Container at index ${index} should not be empty`);
|
|
174
180
|
t.true(containerName.trim().length > 0, `Container at index ${index} should not be only whitespace`);
|
|
@@ -176,8 +182,9 @@ test("all configured container names should be non-empty strings", (t) => {
|
|
|
176
182
|
});
|
|
177
183
|
|
|
178
184
|
test("container names should not contain duplicates", (t) => {
|
|
179
|
-
const
|
|
180
|
-
|
|
185
|
+
const currentContainers = getCurrentContainerNames();
|
|
186
|
+
const uniqueNames = new Set(currentContainers);
|
|
187
|
+
t.is(uniqueNames.size, currentContainers.length, "Container names should be unique");
|
|
181
188
|
});
|
|
182
189
|
|
|
183
190
|
// Integration test with actual environment simulation
|