@aj-archipelago/cortex 1.3.49 → 1.3.51

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/config.js +1 -1
  2. package/helper-apps/cortex-browser/Dockerfile +19 -31
  3. package/helper-apps/cortex-browser/function_app.py +708 -181
  4. package/helper-apps/cortex-browser/requirements.txt +4 -4
  5. package/helper-apps/cortex-file-handler/blobHandler.js +850 -429
  6. package/helper-apps/cortex-file-handler/constants.js +64 -48
  7. package/helper-apps/cortex-file-handler/docHelper.js +7 -114
  8. package/helper-apps/cortex-file-handler/fileChunker.js +96 -51
  9. package/helper-apps/cortex-file-handler/function.json +2 -6
  10. package/helper-apps/cortex-file-handler/helper.js +34 -25
  11. package/helper-apps/cortex-file-handler/index.js +324 -136
  12. package/helper-apps/cortex-file-handler/localFileHandler.js +56 -57
  13. package/helper-apps/cortex-file-handler/package-lock.json +6065 -5964
  14. package/helper-apps/cortex-file-handler/package.json +8 -4
  15. package/helper-apps/cortex-file-handler/redis.js +23 -17
  16. package/helper-apps/cortex-file-handler/scripts/setup-azure-container.js +12 -9
  17. package/helper-apps/cortex-file-handler/scripts/setup-test-containers.js +21 -18
  18. package/helper-apps/cortex-file-handler/scripts/test-azure.sh +1 -1
  19. package/helper-apps/cortex-file-handler/scripts/test-gcs.sh +1 -1
  20. package/helper-apps/cortex-file-handler/services/ConversionService.js +288 -0
  21. package/helper-apps/cortex-file-handler/services/FileConversionService.js +53 -0
  22. package/helper-apps/cortex-file-handler/start.js +63 -38
  23. package/helper-apps/cortex-file-handler/tests/FileConversionService.test.js +144 -0
  24. package/helper-apps/cortex-file-handler/tests/blobHandler.test.js +88 -64
  25. package/helper-apps/cortex-file-handler/tests/fileChunker.test.js +114 -91
  26. package/helper-apps/cortex-file-handler/tests/fileUpload.test.js +351 -0
  27. package/helper-apps/cortex-file-handler/tests/files/DOCX_TestPage.docx +0 -0
  28. package/helper-apps/cortex-file-handler/tests/files/tests-example.xls +0 -0
  29. package/helper-apps/cortex-file-handler/tests/start.test.js +943 -642
  30. package/helper-apps/cortex-file-handler/tests/testUtils.helper.js +31 -0
  31. package/helper-apps/cortex-markitdown/.funcignore +1 -0
  32. package/helper-apps/cortex-markitdown/MarkitdownConverterFunction/__init__.py +64 -0
  33. package/helper-apps/cortex-markitdown/MarkitdownConverterFunction/function.json +21 -0
  34. package/helper-apps/cortex-markitdown/README.md +94 -0
  35. package/helper-apps/cortex-markitdown/host.json +15 -0
  36. package/helper-apps/cortex-markitdown/requirements.txt +2 -0
  37. package/lib/requestExecutor.js +44 -36
  38. package/package.json +1 -1
  39. package/pathways/system/entity/tools/sys_tool_cognitive_search.js +1 -1
  40. package/pathways/system/entity/tools/sys_tool_readfile.js +24 -2
  41. package/server/plugins/openAiWhisperPlugin.js +59 -87
  42. package/helper-apps/cortex-file-handler/tests/docHelper.test.js +0 -148
@@ -1,8 +1,9 @@
1
1
  import { promises as fs } from 'fs';
2
2
  import { join, basename } from 'path';
3
+
3
4
  import { v4 as uuidv4 } from 'uuid';
4
5
 
5
- import { publicFolder, port, ipAddress } from "./start.js";
6
+ import { publicFolder, port, ipAddress } from './start.js';
6
7
 
7
8
  async function moveFileToPublicFolder(chunkPath, requestId) {
8
9
  // Use the filename with a UUID as the blob name
@@ -26,12 +27,12 @@ async function deleteFolder(requestId) {
26
27
  if (!requestId) throw new Error('Missing requestId parameter');
27
28
  const targetFolder = join(publicFolder, requestId);
28
29
  try {
29
- // Check if folder exists first
30
+ // Check if folder exists first
30
31
  const stats = await fs.stat(targetFolder);
31
32
  if (stats.isDirectory()) {
32
33
  // Get list of files before deleting
33
34
  const files = await fs.readdir(targetFolder);
34
- const deletedFiles = files.map(file => join(requestId, file));
35
+ const deletedFiles = files.map((file) => join(requestId, file));
35
36
  // Delete the folder
36
37
  await fs.rm(targetFolder, { recursive: true });
37
38
  console.log(`Cleaned folder: ${targetFolder}`);
@@ -47,62 +48,60 @@ async function deleteFolder(requestId) {
47
48
  }
48
49
  }
49
50
 
50
- async function cleanupLocal(urls=null) {
51
- const cleanedUrls = [];
52
- if(!urls){
53
- try {
54
- // Read the directory
55
- const items = await fs.readdir(publicFolder);
56
-
57
- // Calculate the date that is x months ago
58
- const monthsAgo = new Date();
59
- monthsAgo.setMonth(monthsAgo.getMonth() - 1);
60
-
61
- // Iterate through the items
62
- for (const item of items) {
63
- const itemPath = join(publicFolder, item);
64
-
65
- // Get the stats of the item
66
- const stats = await fs.stat(itemPath);
67
-
68
- // Check if the item is a file or a directory
69
- const isDirectory = stats.isDirectory();
70
-
71
- // Compare the last modified date with three months ago
72
- if (stats.mtime < monthsAgo) {
73
- if (isDirectory) {
74
- // If it's a directory, delete it recursively
75
- await fs.rm(itemPath, { recursive: true });
76
- console.log(`Cleaned directory: ${item}`);
77
- } else {
78
- // If it's a file, delete it
79
- await fs.unlink(itemPath);
80
- console.log(`Cleaned file: ${item}`);
81
-
82
- // Add the URL of the cleaned file to cleanedUrls array
83
- cleanedUrls.push(`http://${ipAddress}:${port}/files/${item}`);
84
- }
51
+ async function cleanupLocal(urls = null) {
52
+ const cleanedUrls = [];
53
+ if (!urls) {
54
+ try {
55
+ // Read the directory
56
+ const items = await fs.readdir(publicFolder);
57
+
58
+ // Calculate the date that is x months ago
59
+ const monthsAgo = new Date();
60
+ monthsAgo.setMonth(monthsAgo.getMonth() - 1);
61
+
62
+ // Iterate through the items
63
+ for (const item of items) {
64
+ const itemPath = join(publicFolder, item);
65
+
66
+ // Get the stats of the item
67
+ const stats = await fs.stat(itemPath);
68
+
69
+ // Check if the item is a file or a directory
70
+ const isDirectory = stats.isDirectory();
71
+
72
+ // Compare the last modified date with three months ago
73
+ if (stats.mtime < monthsAgo) {
74
+ if (isDirectory) {
75
+ // If it's a directory, delete it recursively
76
+ await fs.rm(itemPath, { recursive: true });
77
+ console.log(`Cleaned directory: ${item}`);
78
+ } else {
79
+ // If it's a file, delete it
80
+ await fs.unlink(itemPath);
81
+ console.log(`Cleaned file: ${item}`);
82
+
83
+ // Add the URL of the cleaned file to cleanedUrls array
84
+ cleanedUrls.push(`http://${ipAddress}:${port}/files/${item}`);
85
+ }
86
+ }
87
+ }
88
+ } catch (error) {
89
+ console.error(`Error cleaning up files: ${error}`);
90
+ }
91
+ } else {
92
+ try {
93
+ for (const url of urls) {
94
+ const filename = url.split('/').pop();
95
+ const itemPath = join(publicFolder, filename);
96
+ await fs.unlink(itemPath);
97
+ }
98
+ } catch (error) {
99
+ console.error(`Error cleaning up files: ${error}`);
85
100
  }
86
- }
87
- } catch (error) {
88
- console.error(`Error cleaning up files: ${error}`);
89
- }
90
- }else{
91
- try{
92
- for (const url of urls) {
93
- const filename = url.split('/').pop();
94
- const itemPath = join(publicFolder, filename);
95
- await fs.unlink(itemPath);
96
- }
97
- }catch(error){
98
- console.error(`Error cleaning up files: ${error}`);
99
101
  }
100
- }
101
102
 
102
- // Return the array of cleaned file URLs
103
- return cleanedUrls;
103
+ // Return the array of cleaned file URLs
104
+ return cleanedUrls;
104
105
  }
105
106
 
106
- export {
107
- moveFileToPublicFolder, deleteFolder, cleanupLocal
108
- };
107
+ export { moveFileToPublicFolder, deleteFolder, cleanupLocal };