@aj-archipelago/cortex 1.3.50 → 1.3.52

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/config.js +1 -1
  2. package/helper-apps/cortex-browser/Dockerfile +19 -31
  3. package/helper-apps/cortex-browser/function_app.py +708 -181
  4. package/helper-apps/cortex-browser/requirements.txt +4 -4
  5. package/helper-apps/cortex-file-handler/{.env.test.azure → .env.test.azure.sample} +2 -1
  6. package/helper-apps/cortex-file-handler/{.env.test.gcs → .env.test.gcs.sample} +2 -1
  7. package/helper-apps/cortex-file-handler/{.env.test → .env.test.sample} +2 -1
  8. package/helper-apps/cortex-file-handler/Dockerfile +1 -1
  9. package/helper-apps/cortex-file-handler/INTERFACE.md +178 -0
  10. package/helper-apps/cortex-file-handler/function.json +2 -6
  11. package/helper-apps/cortex-file-handler/package-lock.json +6065 -5964
  12. package/helper-apps/cortex-file-handler/package.json +11 -6
  13. package/helper-apps/cortex-file-handler/scripts/setup-azure-container.js +12 -9
  14. package/helper-apps/cortex-file-handler/scripts/setup-test-containers.js +21 -18
  15. package/helper-apps/cortex-file-handler/scripts/test-azure.sh +4 -1
  16. package/helper-apps/cortex-file-handler/scripts/test-gcs.sh +1 -1
  17. package/helper-apps/cortex-file-handler/src/blobHandler.js +1056 -0
  18. package/helper-apps/cortex-file-handler/{constants.js → src/constants.js} +64 -48
  19. package/helper-apps/cortex-file-handler/src/docHelper.js +37 -0
  20. package/helper-apps/cortex-file-handler/{fileChunker.js → src/fileChunker.js} +97 -65
  21. package/helper-apps/cortex-file-handler/{helper.js → src/helper.js} +34 -25
  22. package/helper-apps/cortex-file-handler/src/index.js +608 -0
  23. package/helper-apps/cortex-file-handler/src/localFileHandler.js +107 -0
  24. package/helper-apps/cortex-file-handler/{redis.js → src/redis.js} +23 -17
  25. package/helper-apps/cortex-file-handler/src/services/ConversionService.js +309 -0
  26. package/helper-apps/cortex-file-handler/src/services/FileConversionService.js +57 -0
  27. package/helper-apps/cortex-file-handler/src/services/storage/AzureStorageProvider.js +177 -0
  28. package/helper-apps/cortex-file-handler/src/services/storage/GCSStorageProvider.js +258 -0
  29. package/helper-apps/cortex-file-handler/src/services/storage/LocalStorageProvider.js +182 -0
  30. package/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js +86 -0
  31. package/helper-apps/cortex-file-handler/src/services/storage/StorageProvider.js +53 -0
  32. package/helper-apps/cortex-file-handler/src/services/storage/StorageService.js +259 -0
  33. package/helper-apps/cortex-file-handler/src/start.js +88 -0
  34. package/helper-apps/cortex-file-handler/src/utils/filenameUtils.js +28 -0
  35. package/helper-apps/cortex-file-handler/tests/FileConversionService.test.js +144 -0
  36. package/helper-apps/cortex-file-handler/tests/blobHandler.test.js +90 -66
  37. package/helper-apps/cortex-file-handler/tests/conversionResilience.test.js +152 -0
  38. package/helper-apps/cortex-file-handler/tests/fileChunker.test.js +105 -108
  39. package/helper-apps/cortex-file-handler/tests/fileUpload.test.js +462 -0
  40. package/helper-apps/cortex-file-handler/tests/files/DOCX_TestPage.docx +0 -0
  41. package/helper-apps/cortex-file-handler/tests/files/tests-example.xls +0 -0
  42. package/helper-apps/cortex-file-handler/tests/getOperations.test.js +307 -0
  43. package/helper-apps/cortex-file-handler/tests/postOperations.test.js +291 -0
  44. package/helper-apps/cortex-file-handler/tests/start.test.js +984 -647
  45. package/helper-apps/cortex-file-handler/tests/storage/AzureStorageProvider.test.js +120 -0
  46. package/helper-apps/cortex-file-handler/tests/storage/GCSStorageProvider.test.js +193 -0
  47. package/helper-apps/cortex-file-handler/tests/storage/LocalStorageProvider.test.js +148 -0
  48. package/helper-apps/cortex-file-handler/tests/storage/StorageFactory.test.js +100 -0
  49. package/helper-apps/cortex-file-handler/tests/storage/StorageService.test.js +113 -0
  50. package/helper-apps/cortex-file-handler/tests/testUtils.helper.js +85 -0
  51. package/helper-apps/cortex-markitdown/.funcignore +1 -0
  52. package/helper-apps/cortex-markitdown/MarkitdownConverterFunction/__init__.py +64 -0
  53. package/helper-apps/cortex-markitdown/MarkitdownConverterFunction/function.json +21 -0
  54. package/helper-apps/cortex-markitdown/README.md +94 -0
  55. package/helper-apps/cortex-markitdown/host.json +15 -0
  56. package/helper-apps/cortex-markitdown/requirements.txt +2 -0
  57. package/lib/entityConstants.js +1 -1
  58. package/lib/requestExecutor.js +44 -36
  59. package/package.json +1 -1
  60. package/pathways/system/entity/tools/sys_tool_readfile.js +24 -2
  61. package/server/plugins/openAiWhisperPlugin.js +59 -87
  62. package/helper-apps/cortex-file-handler/blobHandler.js +0 -567
  63. package/helper-apps/cortex-file-handler/docHelper.js +0 -144
  64. package/helper-apps/cortex-file-handler/index.js +0 -440
  65. package/helper-apps/cortex-file-handler/localFileHandler.js +0 -108
  66. package/helper-apps/cortex-file-handler/start.js +0 -63
  67. package/helper-apps/cortex-file-handler/tests/docHelper.test.js +0 -148
@@ -1,11 +1,12 @@
1
1
  {
2
2
  "name": "@aj-archipelago/cortex-file-handler",
3
- "version": "1.0.17",
3
+ "version": "2.0.02",
4
4
  "description": "File handling service for Cortex - handles file uploads, media chunking, and document processing",
5
5
  "type": "module",
6
+ "main": "src/index.js",
6
7
  "scripts": {
7
- "start": "node start.js",
8
- "dev": "node -r dotenv/config start.js",
8
+ "start": "node src/start.js",
9
+ "dev": "node -r dotenv/config src/start.js",
9
10
  "test": "DOTENV_CONFIG_PATH=.env.test NODE_ENV=test node -r dotenv/config node_modules/ava/entrypoints/cli.mjs",
10
11
  "test:azure": "DOTENV_CONFIG_PATH=.env.test.azure NODE_ENV=test ./scripts/test-azure.sh",
11
12
  "test:watch": "DOTENV_CONFIG_PATH=.env.test NODE_ENV=test node -r dotenv/config node_modules/ava/entrypoints/cli.mjs --watch",
@@ -21,7 +22,7 @@
21
22
  "express": "^4.21.1",
22
23
  "fluent-ffmpeg": "^2.1.3",
23
24
  "ioredis": "^5.3.1",
24
- "mammoth": "^1.6.0",
25
+ "mime-types": "^3.0.1",
25
26
  "papaparse": "^5.4.1",
26
27
  "pdfjs-dist": "^4.2.67",
27
28
  "public-ip": "^6.0.1",
@@ -29,13 +30,17 @@
29
30
  "xlsx": "^0.18.5"
30
31
  },
31
32
  "devDependencies": {
33
+ "@eslint/js": "^9.26.0",
32
34
  "ava": "^5.3.1",
33
35
  "dotenv": "^16.3.1",
34
- "nock": "^13.3.0"
36
+ "eslint-plugin-import": "^2.31.0",
37
+ "globals": "^16.1.0",
38
+ "nock": "^13.3.0",
39
+ "typescript-eslint": "^8.32.1"
35
40
  },
36
41
  "ava": {
37
42
  "files": [
38
- "tests/**/*",
43
+ "tests/**/*.test.js",
39
44
  "!tests/test-files/**/*",
40
45
  "!tests/test-docs/**/*",
41
46
  "!tests/mocks/**/*"
@@ -2,21 +2,24 @@ import { BlobServiceClient } from '@azure/storage-blob';
2
2
 
3
3
  async function createContainer() {
4
4
  try {
5
- const blobServiceClient = BlobServiceClient.fromConnectionString("UseDevelopmentStorage=true");
6
- const containerClient = blobServiceClient.getContainerClient("test-container");
7
-
8
- console.log("Creating container...");
5
+ const blobServiceClient = BlobServiceClient.fromConnectionString(
6
+ 'UseDevelopmentStorage=true',
7
+ );
8
+ const containerClient =
9
+ blobServiceClient.getContainerClient('test-container');
10
+
11
+ console.log('Creating container...');
9
12
  await containerClient.create();
10
- console.log("Container created successfully");
13
+ console.log('Container created successfully');
11
14
  } catch (error) {
12
- // Ignore if container already exists
15
+ // Ignore if container already exists
13
16
  if (error.statusCode === 409) {
14
- console.log("Container already exists");
17
+ console.log('Container already exists');
15
18
  } else {
16
- console.error("Error creating container:", error);
19
+ console.error('Error creating container:', error);
17
20
  process.exit(1);
18
21
  }
19
22
  }
20
23
  }
21
24
 
22
- createContainer();
25
+ createContainer();
@@ -3,18 +3,21 @@ import { Storage } from '@google-cloud/storage';
3
3
 
4
4
  async function createAzureContainer() {
5
5
  try {
6
- const blobServiceClient = BlobServiceClient.fromConnectionString("UseDevelopmentStorage=true");
7
- const containerClient = blobServiceClient.getContainerClient("test-container");
8
-
9
- console.log("Creating Azure container...");
6
+ const blobServiceClient = BlobServiceClient.fromConnectionString(
7
+ 'UseDevelopmentStorage=true',
8
+ );
9
+ const containerClient =
10
+ blobServiceClient.getContainerClient('test-container');
11
+
12
+ console.log('Creating Azure container...');
10
13
  await containerClient.create();
11
- console.log("Azure container created successfully");
14
+ console.log('Azure container created successfully');
12
15
  } catch (error) {
13
- // Ignore if container already exists
16
+ // Ignore if container already exists
14
17
  if (error.statusCode === 409) {
15
- console.log("Azure container already exists");
18
+ console.log('Azure container already exists');
16
19
  } else {
17
- console.error("Error creating Azure container:", error);
20
+ console.error('Error creating Azure container:', error);
18
21
  process.exit(1);
19
22
  }
20
23
  }
@@ -23,19 +26,19 @@ async function createAzureContainer() {
23
26
  async function createGCSBucket() {
24
27
  try {
25
28
  const storage = new Storage({
26
- projectId: "test-project",
27
- apiEndpoint: "http://localhost:4443",
29
+ projectId: 'test-project',
30
+ apiEndpoint: 'http://localhost:4443',
28
31
  });
29
-
30
- console.log("Creating GCS bucket...");
31
- await storage.createBucket("cortextempfiles");
32
- console.log("GCS bucket created successfully");
32
+
33
+ console.log('Creating GCS bucket...');
34
+ await storage.createBucket('cortextempfiles');
35
+ console.log('GCS bucket created successfully');
33
36
  } catch (error) {
34
- // Ignore if bucket already exists
37
+ // Ignore if bucket already exists
35
38
  if (error.code === 409) {
36
- console.log("GCS bucket already exists");
39
+ console.log('GCS bucket already exists');
37
40
  } else {
38
- console.error("Error creating GCS bucket:", error);
41
+ console.error('Error creating GCS bucket:', error);
39
42
  process.exit(1);
40
43
  }
41
44
  }
@@ -46,4 +49,4 @@ async function setup() {
46
49
  await createGCSBucket();
47
50
  }
48
51
 
49
- setup();
52
+ setup();
@@ -6,7 +6,7 @@ mkdir -p $AZURITE_DIR
6
6
 
7
7
  # Start Azurite in background
8
8
  echo "Starting Azurite..."
9
- azurite --silent --location $AZURITE_DIR &
9
+ azurite --silent --skipApiVersionCheck --location $AZURITE_DIR &
10
10
  AZURITE_PID=$!
11
11
 
12
12
  # Wait for Azurite to start
@@ -27,6 +27,9 @@ TEST_RESULT=$?
27
27
  echo "Cleaning up..."
28
28
  kill $AZURITE_PID
29
29
 
30
+ # Wait for Azurite to finish cleanup
31
+ sleep 2
32
+
30
33
  # Clean up Azurite directory
31
34
  rm -rf $AZURITE_DIR
32
35
 
@@ -20,7 +20,7 @@ echo "Starting test environment..."
20
20
  # Start Azurite if not running
21
21
  if ! nc -z localhost 10000; then
22
22
  echo "Starting Azurite..."
23
- azurite --silent --location .azurite --debug .azurite/debug.log &
23
+ azurite --silent --skipApiVersionCheck --location .azurite --debug .azurite/debug.log &
24
24
  AZURITE_PID=$!
25
25
  # Wait for Azurite to be ready
26
26
  until nc -z localhost 10000; do