@aj-archipelago/cortex 1.3.50 → 1.3.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config.js +1 -1
- package/helper-apps/cortex-browser/Dockerfile +19 -31
- package/helper-apps/cortex-browser/function_app.py +708 -181
- package/helper-apps/cortex-browser/requirements.txt +4 -4
- package/helper-apps/cortex-file-handler/{.env.test.azure → .env.test.azure.sample} +2 -1
- package/helper-apps/cortex-file-handler/{.env.test.gcs → .env.test.gcs.sample} +2 -1
- package/helper-apps/cortex-file-handler/{.env.test → .env.test.sample} +2 -1
- package/helper-apps/cortex-file-handler/Dockerfile +1 -1
- package/helper-apps/cortex-file-handler/INTERFACE.md +178 -0
- package/helper-apps/cortex-file-handler/function.json +2 -6
- package/helper-apps/cortex-file-handler/package-lock.json +6065 -5964
- package/helper-apps/cortex-file-handler/package.json +11 -6
- package/helper-apps/cortex-file-handler/scripts/setup-azure-container.js +12 -9
- package/helper-apps/cortex-file-handler/scripts/setup-test-containers.js +21 -18
- package/helper-apps/cortex-file-handler/scripts/test-azure.sh +4 -1
- package/helper-apps/cortex-file-handler/scripts/test-gcs.sh +1 -1
- package/helper-apps/cortex-file-handler/src/blobHandler.js +1056 -0
- package/helper-apps/cortex-file-handler/{constants.js → src/constants.js} +64 -48
- package/helper-apps/cortex-file-handler/src/docHelper.js +37 -0
- package/helper-apps/cortex-file-handler/{fileChunker.js → src/fileChunker.js} +97 -65
- package/helper-apps/cortex-file-handler/{helper.js → src/helper.js} +34 -25
- package/helper-apps/cortex-file-handler/src/index.js +608 -0
- package/helper-apps/cortex-file-handler/src/localFileHandler.js +107 -0
- package/helper-apps/cortex-file-handler/{redis.js → src/redis.js} +23 -17
- package/helper-apps/cortex-file-handler/src/services/ConversionService.js +309 -0
- package/helper-apps/cortex-file-handler/src/services/FileConversionService.js +57 -0
- package/helper-apps/cortex-file-handler/src/services/storage/AzureStorageProvider.js +177 -0
- package/helper-apps/cortex-file-handler/src/services/storage/GCSStorageProvider.js +258 -0
- package/helper-apps/cortex-file-handler/src/services/storage/LocalStorageProvider.js +182 -0
- package/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js +86 -0
- package/helper-apps/cortex-file-handler/src/services/storage/StorageProvider.js +53 -0
- package/helper-apps/cortex-file-handler/src/services/storage/StorageService.js +259 -0
- package/helper-apps/cortex-file-handler/src/start.js +88 -0
- package/helper-apps/cortex-file-handler/src/utils/filenameUtils.js +28 -0
- package/helper-apps/cortex-file-handler/tests/FileConversionService.test.js +144 -0
- package/helper-apps/cortex-file-handler/tests/blobHandler.test.js +90 -66
- package/helper-apps/cortex-file-handler/tests/conversionResilience.test.js +152 -0
- package/helper-apps/cortex-file-handler/tests/fileChunker.test.js +105 -108
- package/helper-apps/cortex-file-handler/tests/fileUpload.test.js +462 -0
- package/helper-apps/cortex-file-handler/tests/files/DOCX_TestPage.docx +0 -0
- package/helper-apps/cortex-file-handler/tests/files/tests-example.xls +0 -0
- package/helper-apps/cortex-file-handler/tests/getOperations.test.js +307 -0
- package/helper-apps/cortex-file-handler/tests/postOperations.test.js +291 -0
- package/helper-apps/cortex-file-handler/tests/start.test.js +984 -647
- package/helper-apps/cortex-file-handler/tests/storage/AzureStorageProvider.test.js +120 -0
- package/helper-apps/cortex-file-handler/tests/storage/GCSStorageProvider.test.js +193 -0
- package/helper-apps/cortex-file-handler/tests/storage/LocalStorageProvider.test.js +148 -0
- package/helper-apps/cortex-file-handler/tests/storage/StorageFactory.test.js +100 -0
- package/helper-apps/cortex-file-handler/tests/storage/StorageService.test.js +113 -0
- package/helper-apps/cortex-file-handler/tests/testUtils.helper.js +85 -0
- package/helper-apps/cortex-markitdown/.funcignore +1 -0
- package/helper-apps/cortex-markitdown/MarkitdownConverterFunction/__init__.py +64 -0
- package/helper-apps/cortex-markitdown/MarkitdownConverterFunction/function.json +21 -0
- package/helper-apps/cortex-markitdown/README.md +94 -0
- package/helper-apps/cortex-markitdown/host.json +15 -0
- package/helper-apps/cortex-markitdown/requirements.txt +2 -0
- package/lib/entityConstants.js +1 -1
- package/lib/requestExecutor.js +44 -36
- package/package.json +1 -1
- package/pathways/system/entity/tools/sys_tool_readfile.js +24 -2
- package/server/plugins/openAiWhisperPlugin.js +59 -87
- package/helper-apps/cortex-file-handler/blobHandler.js +0 -567
- package/helper-apps/cortex-file-handler/docHelper.js +0 -144
- package/helper-apps/cortex-file-handler/index.js +0 -440
- package/helper-apps/cortex-file-handler/localFileHandler.js +0 -108
- package/helper-apps/cortex-file-handler/start.js +0 -63
- package/helper-apps/cortex-file-handler/tests/docHelper.test.js +0 -148
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aj-archipelago/cortex-file-handler",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "2.0.02",
|
|
4
4
|
"description": "File handling service for Cortex - handles file uploads, media chunking, and document processing",
|
|
5
5
|
"type": "module",
|
|
6
|
+
"main": "src/index.js",
|
|
6
7
|
"scripts": {
|
|
7
|
-
"start": "node start.js",
|
|
8
|
-
"dev": "node -r dotenv/config start.js",
|
|
8
|
+
"start": "node src/start.js",
|
|
9
|
+
"dev": "node -r dotenv/config src/start.js",
|
|
9
10
|
"test": "DOTENV_CONFIG_PATH=.env.test NODE_ENV=test node -r dotenv/config node_modules/ava/entrypoints/cli.mjs",
|
|
10
11
|
"test:azure": "DOTENV_CONFIG_PATH=.env.test.azure NODE_ENV=test ./scripts/test-azure.sh",
|
|
11
12
|
"test:watch": "DOTENV_CONFIG_PATH=.env.test NODE_ENV=test node -r dotenv/config node_modules/ava/entrypoints/cli.mjs --watch",
|
|
@@ -21,7 +22,7 @@
|
|
|
21
22
|
"express": "^4.21.1",
|
|
22
23
|
"fluent-ffmpeg": "^2.1.3",
|
|
23
24
|
"ioredis": "^5.3.1",
|
|
24
|
-
"
|
|
25
|
+
"mime-types": "^3.0.1",
|
|
25
26
|
"papaparse": "^5.4.1",
|
|
26
27
|
"pdfjs-dist": "^4.2.67",
|
|
27
28
|
"public-ip": "^6.0.1",
|
|
@@ -29,13 +30,17 @@
|
|
|
29
30
|
"xlsx": "^0.18.5"
|
|
30
31
|
},
|
|
31
32
|
"devDependencies": {
|
|
33
|
+
"@eslint/js": "^9.26.0",
|
|
32
34
|
"ava": "^5.3.1",
|
|
33
35
|
"dotenv": "^16.3.1",
|
|
34
|
-
"
|
|
36
|
+
"eslint-plugin-import": "^2.31.0",
|
|
37
|
+
"globals": "^16.1.0",
|
|
38
|
+
"nock": "^13.3.0",
|
|
39
|
+
"typescript-eslint": "^8.32.1"
|
|
35
40
|
},
|
|
36
41
|
"ava": {
|
|
37
42
|
"files": [
|
|
38
|
-
"tests
|
|
43
|
+
"tests/**/*.test.js",
|
|
39
44
|
"!tests/test-files/**/*",
|
|
40
45
|
"!tests/test-docs/**/*",
|
|
41
46
|
"!tests/mocks/**/*"
|
|
@@ -2,21 +2,24 @@ import { BlobServiceClient } from '@azure/storage-blob';
|
|
|
2
2
|
|
|
3
3
|
async function createContainer() {
|
|
4
4
|
try {
|
|
5
|
-
const blobServiceClient = BlobServiceClient.fromConnectionString(
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
5
|
+
const blobServiceClient = BlobServiceClient.fromConnectionString(
|
|
6
|
+
'UseDevelopmentStorage=true',
|
|
7
|
+
);
|
|
8
|
+
const containerClient =
|
|
9
|
+
blobServiceClient.getContainerClient('test-container');
|
|
10
|
+
|
|
11
|
+
console.log('Creating container...');
|
|
9
12
|
await containerClient.create();
|
|
10
|
-
console.log(
|
|
13
|
+
console.log('Container created successfully');
|
|
11
14
|
} catch (error) {
|
|
12
|
-
|
|
15
|
+
// Ignore if container already exists
|
|
13
16
|
if (error.statusCode === 409) {
|
|
14
|
-
console.log(
|
|
17
|
+
console.log('Container already exists');
|
|
15
18
|
} else {
|
|
16
|
-
console.error(
|
|
19
|
+
console.error('Error creating container:', error);
|
|
17
20
|
process.exit(1);
|
|
18
21
|
}
|
|
19
22
|
}
|
|
20
23
|
}
|
|
21
24
|
|
|
22
|
-
createContainer();
|
|
25
|
+
createContainer();
|
|
@@ -3,18 +3,21 @@ import { Storage } from '@google-cloud/storage';
|
|
|
3
3
|
|
|
4
4
|
async function createAzureContainer() {
|
|
5
5
|
try {
|
|
6
|
-
const blobServiceClient = BlobServiceClient.fromConnectionString(
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
6
|
+
const blobServiceClient = BlobServiceClient.fromConnectionString(
|
|
7
|
+
'UseDevelopmentStorage=true',
|
|
8
|
+
);
|
|
9
|
+
const containerClient =
|
|
10
|
+
blobServiceClient.getContainerClient('test-container');
|
|
11
|
+
|
|
12
|
+
console.log('Creating Azure container...');
|
|
10
13
|
await containerClient.create();
|
|
11
|
-
console.log(
|
|
14
|
+
console.log('Azure container created successfully');
|
|
12
15
|
} catch (error) {
|
|
13
|
-
|
|
16
|
+
// Ignore if container already exists
|
|
14
17
|
if (error.statusCode === 409) {
|
|
15
|
-
console.log(
|
|
18
|
+
console.log('Azure container already exists');
|
|
16
19
|
} else {
|
|
17
|
-
console.error(
|
|
20
|
+
console.error('Error creating Azure container:', error);
|
|
18
21
|
process.exit(1);
|
|
19
22
|
}
|
|
20
23
|
}
|
|
@@ -23,19 +26,19 @@ async function createAzureContainer() {
|
|
|
23
26
|
async function createGCSBucket() {
|
|
24
27
|
try {
|
|
25
28
|
const storage = new Storage({
|
|
26
|
-
projectId:
|
|
27
|
-
apiEndpoint:
|
|
29
|
+
projectId: 'test-project',
|
|
30
|
+
apiEndpoint: 'http://localhost:4443',
|
|
28
31
|
});
|
|
29
|
-
|
|
30
|
-
console.log(
|
|
31
|
-
await storage.createBucket(
|
|
32
|
-
console.log(
|
|
32
|
+
|
|
33
|
+
console.log('Creating GCS bucket...');
|
|
34
|
+
await storage.createBucket('cortextempfiles');
|
|
35
|
+
console.log('GCS bucket created successfully');
|
|
33
36
|
} catch (error) {
|
|
34
|
-
|
|
37
|
+
// Ignore if bucket already exists
|
|
35
38
|
if (error.code === 409) {
|
|
36
|
-
console.log(
|
|
39
|
+
console.log('GCS bucket already exists');
|
|
37
40
|
} else {
|
|
38
|
-
console.error(
|
|
41
|
+
console.error('Error creating GCS bucket:', error);
|
|
39
42
|
process.exit(1);
|
|
40
43
|
}
|
|
41
44
|
}
|
|
@@ -46,4 +49,4 @@ async function setup() {
|
|
|
46
49
|
await createGCSBucket();
|
|
47
50
|
}
|
|
48
51
|
|
|
49
|
-
setup();
|
|
52
|
+
setup();
|
|
@@ -6,7 +6,7 @@ mkdir -p $AZURITE_DIR
|
|
|
6
6
|
|
|
7
7
|
# Start Azurite in background
|
|
8
8
|
echo "Starting Azurite..."
|
|
9
|
-
azurite --silent --location $AZURITE_DIR &
|
|
9
|
+
azurite --silent --skipApiVersionCheck --location $AZURITE_DIR &
|
|
10
10
|
AZURITE_PID=$!
|
|
11
11
|
|
|
12
12
|
# Wait for Azurite to start
|
|
@@ -27,6 +27,9 @@ TEST_RESULT=$?
|
|
|
27
27
|
echo "Cleaning up..."
|
|
28
28
|
kill $AZURITE_PID
|
|
29
29
|
|
|
30
|
+
# Wait for Azurite to finish cleanup
|
|
31
|
+
sleep 2
|
|
32
|
+
|
|
30
33
|
# Clean up Azurite directory
|
|
31
34
|
rm -rf $AZURITE_DIR
|
|
32
35
|
|
|
@@ -20,7 +20,7 @@ echo "Starting test environment..."
|
|
|
20
20
|
# Start Azurite if not running
|
|
21
21
|
if ! nc -z localhost 10000; then
|
|
22
22
|
echo "Starting Azurite..."
|
|
23
|
-
azurite --silent --location .azurite --debug .azurite/debug.log &
|
|
23
|
+
azurite --silent --skipApiVersionCheck --location .azurite --debug .azurite/debug.log &
|
|
24
24
|
AZURITE_PID=$!
|
|
25
25
|
# Wait for Azurite to be ready
|
|
26
26
|
until nc -z localhost 10000; do
|