@aj-archipelago/cortex 1.3.49 → 1.3.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config.js +1 -1
- package/helper-apps/cortex-browser/Dockerfile +19 -31
- package/helper-apps/cortex-browser/function_app.py +708 -181
- package/helper-apps/cortex-browser/requirements.txt +4 -4
- package/helper-apps/cortex-file-handler/blobHandler.js +850 -429
- package/helper-apps/cortex-file-handler/constants.js +64 -48
- package/helper-apps/cortex-file-handler/docHelper.js +7 -114
- package/helper-apps/cortex-file-handler/fileChunker.js +96 -51
- package/helper-apps/cortex-file-handler/function.json +2 -6
- package/helper-apps/cortex-file-handler/helper.js +34 -25
- package/helper-apps/cortex-file-handler/index.js +324 -136
- package/helper-apps/cortex-file-handler/localFileHandler.js +56 -57
- package/helper-apps/cortex-file-handler/package-lock.json +6065 -5964
- package/helper-apps/cortex-file-handler/package.json +8 -4
- package/helper-apps/cortex-file-handler/redis.js +23 -17
- package/helper-apps/cortex-file-handler/scripts/setup-azure-container.js +12 -9
- package/helper-apps/cortex-file-handler/scripts/setup-test-containers.js +21 -18
- package/helper-apps/cortex-file-handler/scripts/test-azure.sh +1 -1
- package/helper-apps/cortex-file-handler/scripts/test-gcs.sh +1 -1
- package/helper-apps/cortex-file-handler/services/ConversionService.js +288 -0
- package/helper-apps/cortex-file-handler/services/FileConversionService.js +53 -0
- package/helper-apps/cortex-file-handler/start.js +63 -38
- package/helper-apps/cortex-file-handler/tests/FileConversionService.test.js +144 -0
- package/helper-apps/cortex-file-handler/tests/blobHandler.test.js +88 -64
- package/helper-apps/cortex-file-handler/tests/fileChunker.test.js +114 -91
- package/helper-apps/cortex-file-handler/tests/fileUpload.test.js +351 -0
- package/helper-apps/cortex-file-handler/tests/files/DOCX_TestPage.docx +0 -0
- package/helper-apps/cortex-file-handler/tests/files/tests-example.xls +0 -0
- package/helper-apps/cortex-file-handler/tests/start.test.js +943 -642
- package/helper-apps/cortex-file-handler/tests/testUtils.helper.js +31 -0
- package/helper-apps/cortex-markitdown/.funcignore +1 -0
- package/helper-apps/cortex-markitdown/MarkitdownConverterFunction/__init__.py +64 -0
- package/helper-apps/cortex-markitdown/MarkitdownConverterFunction/function.json +21 -0
- package/helper-apps/cortex-markitdown/README.md +94 -0
- package/helper-apps/cortex-markitdown/host.json +15 -0
- package/helper-apps/cortex-markitdown/requirements.txt +2 -0
- package/lib/requestExecutor.js +44 -36
- package/package.json +1 -1
- package/pathways/system/entity/tools/sys_tool_cognitive_search.js +1 -1
- package/pathways/system/entity/tools/sys_tool_readfile.js +24 -2
- package/server/plugins/openAiWhisperPlugin.js +59 -87
- package/helper-apps/cortex-file-handler/tests/docHelper.test.js +0 -148
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { promises as fs } from 'fs';
|
|
2
2
|
import { join, basename } from 'path';
|
|
3
|
+
|
|
3
4
|
import { v4 as uuidv4 } from 'uuid';
|
|
4
5
|
|
|
5
|
-
import { publicFolder, port, ipAddress } from
|
|
6
|
+
import { publicFolder, port, ipAddress } from './start.js';
|
|
6
7
|
|
|
7
8
|
async function moveFileToPublicFolder(chunkPath, requestId) {
|
|
8
9
|
// Use the filename with a UUID as the blob name
|
|
@@ -26,12 +27,12 @@ async function deleteFolder(requestId) {
|
|
|
26
27
|
if (!requestId) throw new Error('Missing requestId parameter');
|
|
27
28
|
const targetFolder = join(publicFolder, requestId);
|
|
28
29
|
try {
|
|
29
|
-
|
|
30
|
+
// Check if folder exists first
|
|
30
31
|
const stats = await fs.stat(targetFolder);
|
|
31
32
|
if (stats.isDirectory()) {
|
|
32
33
|
// Get list of files before deleting
|
|
33
34
|
const files = await fs.readdir(targetFolder);
|
|
34
|
-
const deletedFiles = files.map(file => join(requestId, file));
|
|
35
|
+
const deletedFiles = files.map((file) => join(requestId, file));
|
|
35
36
|
// Delete the folder
|
|
36
37
|
await fs.rm(targetFolder, { recursive: true });
|
|
37
38
|
console.log(`Cleaned folder: ${targetFolder}`);
|
|
@@ -47,62 +48,60 @@ async function deleteFolder(requestId) {
|
|
|
47
48
|
}
|
|
48
49
|
}
|
|
49
50
|
|
|
50
|
-
async function cleanupLocal(urls=null) {
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
51
|
+
async function cleanupLocal(urls = null) {
|
|
52
|
+
const cleanedUrls = [];
|
|
53
|
+
if (!urls) {
|
|
54
|
+
try {
|
|
55
|
+
// Read the directory
|
|
56
|
+
const items = await fs.readdir(publicFolder);
|
|
57
|
+
|
|
58
|
+
// Calculate the date that is x months ago
|
|
59
|
+
const monthsAgo = new Date();
|
|
60
|
+
monthsAgo.setMonth(monthsAgo.getMonth() - 1);
|
|
61
|
+
|
|
62
|
+
// Iterate through the items
|
|
63
|
+
for (const item of items) {
|
|
64
|
+
const itemPath = join(publicFolder, item);
|
|
65
|
+
|
|
66
|
+
// Get the stats of the item
|
|
67
|
+
const stats = await fs.stat(itemPath);
|
|
68
|
+
|
|
69
|
+
// Check if the item is a file or a directory
|
|
70
|
+
const isDirectory = stats.isDirectory();
|
|
71
|
+
|
|
72
|
+
// Compare the last modified date with three months ago
|
|
73
|
+
if (stats.mtime < monthsAgo) {
|
|
74
|
+
if (isDirectory) {
|
|
75
|
+
// If it's a directory, delete it recursively
|
|
76
|
+
await fs.rm(itemPath, { recursive: true });
|
|
77
|
+
console.log(`Cleaned directory: ${item}`);
|
|
78
|
+
} else {
|
|
79
|
+
// If it's a file, delete it
|
|
80
|
+
await fs.unlink(itemPath);
|
|
81
|
+
console.log(`Cleaned file: ${item}`);
|
|
82
|
+
|
|
83
|
+
// Add the URL of the cleaned file to cleanedUrls array
|
|
84
|
+
cleanedUrls.push(`http://${ipAddress}:${port}/files/${item}`);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
} catch (error) {
|
|
89
|
+
console.error(`Error cleaning up files: ${error}`);
|
|
90
|
+
}
|
|
91
|
+
} else {
|
|
92
|
+
try {
|
|
93
|
+
for (const url of urls) {
|
|
94
|
+
const filename = url.split('/').pop();
|
|
95
|
+
const itemPath = join(publicFolder, filename);
|
|
96
|
+
await fs.unlink(itemPath);
|
|
97
|
+
}
|
|
98
|
+
} catch (error) {
|
|
99
|
+
console.error(`Error cleaning up files: ${error}`);
|
|
85
100
|
}
|
|
86
|
-
}
|
|
87
|
-
} catch (error) {
|
|
88
|
-
console.error(`Error cleaning up files: ${error}`);
|
|
89
|
-
}
|
|
90
|
-
}else{
|
|
91
|
-
try{
|
|
92
|
-
for (const url of urls) {
|
|
93
|
-
const filename = url.split('/').pop();
|
|
94
|
-
const itemPath = join(publicFolder, filename);
|
|
95
|
-
await fs.unlink(itemPath);
|
|
96
|
-
}
|
|
97
|
-
}catch(error){
|
|
98
|
-
console.error(`Error cleaning up files: ${error}`);
|
|
99
101
|
}
|
|
100
|
-
}
|
|
101
102
|
|
|
102
|
-
|
|
103
|
-
|
|
103
|
+
// Return the array of cleaned file URLs
|
|
104
|
+
return cleanedUrls;
|
|
104
105
|
}
|
|
105
106
|
|
|
106
|
-
export {
|
|
107
|
-
moveFileToPublicFolder, deleteFolder, cleanupLocal
|
|
108
|
-
};
|
|
107
|
+
export { moveFileToPublicFolder, deleteFolder, cleanupLocal };
|