data-primals-engine 1.7.1 → 1.7.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/README.md +9 -9
  2. package/client/package-lock.json +8430 -8121
  3. package/client/package.json +7 -4
  4. package/client/src/APIInfo.jsx +1 -1
  5. package/client/src/App.jsx +2 -1
  6. package/client/src/App.scss +1635 -1626
  7. package/client/src/AssistantChat.jsx +2 -3
  8. package/client/src/CalendarView.jsx +1 -0
  9. package/client/src/ContentView.jsx +3 -3
  10. package/client/src/Dashboard.jsx +5 -2
  11. package/client/src/DashboardChart.jsx +1 -0
  12. package/client/src/DashboardFlexViewItem.jsx +1 -0
  13. package/client/src/DashboardHtmlViewItem.jsx +1 -0
  14. package/client/src/DashboardView.jsx +2 -0
  15. package/client/src/DataEditor.jsx +0 -1
  16. package/client/src/DataImporter.jsx +489 -468
  17. package/client/src/DataLayout.jsx +25 -23
  18. package/client/src/DataTable.jsx +6 -5
  19. package/client/src/Dialog.jsx +92 -90
  20. package/client/src/Dialog.scss +122 -116
  21. package/client/src/DisplayFlexNodeRenderer.jsx +1 -0
  22. package/client/src/DocumentationPageLayout.scss +1 -1
  23. package/client/src/FlexBuilderControls.jsx +1 -0
  24. package/client/src/FlexBuilderPreview.jsx +1 -1
  25. package/client/src/FlexNode.jsx +1 -1
  26. package/client/src/HistoryDialog.jsx +3 -2
  27. package/client/src/KPIWidget.jsx +1 -1
  28. package/client/src/KanbanView.jsx +1 -0
  29. package/client/src/ModelCreator.jsx +4 -0
  30. package/client/src/ModelList.jsx +1 -0
  31. package/client/src/PackGallery.jsx +5 -4
  32. package/client/src/RTETrans.jsx +1 -1
  33. package/client/src/RelationField.jsx +2 -0
  34. package/client/src/RelationSelectorWidget.jsx +2 -0
  35. package/client/src/RelationValue.jsx +1 -0
  36. package/client/src/RestoreDialog.jsx +1 -0
  37. package/client/src/ViewSwitcher.jsx +1 -1
  38. package/client/src/WorkflowEditor.jsx +3 -0
  39. package/client/src/contexts/CommandContext.jsx +2 -1
  40. package/client/src/contexts/ModelContext.jsx +3 -3
  41. package/client/src/hooks/data.js +1 -0
  42. package/client/src/hooks/useValidation.js +1 -0
  43. package/client/src/translations.js +24 -24
  44. package/client/vite.config.js +31 -30
  45. package/doc/AI-assistance.md +87 -63
  46. package/doc/Concepts.md +122 -0
  47. package/doc/Custom-Endpoints.md +31 -0
  48. package/doc/Event-system.md +13 -14
  49. package/doc/Home.md +33 -0
  50. package/doc/Modules.md +83 -0
  51. package/doc/Packs-gallery.md +8 -23
  52. package/doc/Workflows.md +32 -0
  53. package/doc/automation-workflows.md +141 -102
  54. package/doc/dashboards-kpis-charts.md +47 -49
  55. package/doc/data-management.md +126 -120
  56. package/doc/data-models.md +68 -75
  57. package/doc/roles-permissions.md +144 -43
  58. package/doc/sharding-replication.md +158 -0
  59. package/doc/users.md +54 -30
  60. package/package.json +27 -17
  61. package/server.js +37 -37
  62. package/src/ai.jobs.js +135 -0
  63. package/src/constants.js +560 -545
  64. package/src/data.js +521 -518
  65. package/src/email.js +157 -154
  66. package/src/engine.js +167 -49
  67. package/src/gameObject.js +6 -0
  68. package/src/i18n.js +0 -1
  69. package/src/modules/assistant/assistant.js +782 -763
  70. package/src/modules/assistant/constants.js +23 -16
  71. package/src/modules/assistant/providers.js +77 -37
  72. package/src/modules/auth-google/index.js +53 -50
  73. package/src/modules/bucket.js +346 -335
  74. package/src/modules/data/data.backup.js +400 -376
  75. package/src/modules/data/data.cluster.js +559 -0
  76. package/src/modules/data/data.core.js +11 -8
  77. package/src/modules/data/data.js +311 -311
  78. package/src/modules/data/data.operations.js +3666 -3555
  79. package/src/modules/data/data.relations.js +1 -0
  80. package/src/modules/data/data.replication.js +125 -0
  81. package/src/modules/data/data.routes.js +2206 -1879
  82. package/src/modules/data/data.scheduling.js +2 -1
  83. package/src/modules/file.js +248 -247
  84. package/src/modules/mongodb.js +76 -73
  85. package/src/modules/user.js +18 -2
  86. package/src/modules/worker-script-runner.js +97 -0
  87. package/src/modules/workflow.js +177 -52
  88. package/src/packs.js +5701 -5701
  89. package/src/providers.js +298 -297
  90. package/src/sso.js +91 -25
  91. package/test/assistant.test.js +207 -206
  92. package/test/cluster.test.js +221 -0
  93. package/test/core.test.js +0 -2
  94. package/test/data.integration.test.js +1425 -1416
  95. package/test/import_export.integration.test.js +210 -210
  96. package/test/replication.test.js +163 -0
  97. package/test/workflow.actions.integration.test.js +487 -475
  98. package/test/workflow.integration.test.js +332 -329
  99. package/doc/core-concepts.md +0 -33
@@ -1,17 +1,24 @@
1
-
2
- export const providers = {
3
- "OpenAI" : {
4
- key:"OPENAI_API_KEY",
5
- defaultModel: 'gpt-3.5-turbo'
6
- },
7
- "Google": {
8
- key:"GOOGLE_API_KEY"
9
- },
10
- "DeepSeek": {
11
- key: "DEEPSEEK_API_KEY",
12
- defaultModel: 'deepseek-chat'
13
- },
14
- "Anthropic": {
15
- key:"ANTHROPIC_API_KEY"
16
- }
1
+
2
+ export const providers = {
3
+ "OpenAI" : {
4
+ key:"OPENAI_API_KEY",
5
+ defaultModel: 'gpt-4o-mini',
6
+ generationModel: 'gpt-4o-mini' // Excellent for JSON generation
7
+ },
8
+ "Google": {
9
+ key:"GOOGLE_API_KEY",
10
+ defaultModel: 'gemini-1.5-flash',
11
+ generationModel: 'gemini-1.5-flash'
12
+ },
13
+ "DeepSeek": {
14
+ key: "DEEPSEEK_API_KEY",
15
+ defaultModel: 'deepseek-chat',
16
+ // As per the error message, 'deepseek-v2-coder' is a good candidate for code/JSON generation
17
+ generationModel: 'deepseek-coder'
18
+ },
19
+ "Anthropic": {
20
+ key:"ANTHROPIC_API_KEY",
21
+ defaultModel: 'claude-3-haiku-20240307',
22
+ generationModel: 'claude-3-haiku-20240307' // Haiku is fast and good with JSON
23
+ }
17
24
  };
@@ -1,38 +1,78 @@
1
- import { Config } from "../../config.js";
2
- import { assistantConfig } from "../../constants.js";
3
- import { Logger } from "../../gameObject.js";
4
-
5
- const logger = new Logger("AssistantProviders");
6
-
7
- export const getAIProvider = async (aiProvider, aiModel, apiKey) => {
8
- const maxTokens = Config.Get('assistant.maxTokens', assistantConfig.maxTokens);
9
- try {
10
- switch (aiProvider) {
11
- case 'OpenAI': {
12
- const { ChatOpenAI } = await import("@langchain/openai");
13
- return new ChatOpenAI({ apiKey, model: aiModel, temperature: 0.7, maxTokens });
14
- }
15
- case 'Google': {
16
- const { ChatGoogleGenerativeAI } = await import("@langchain/google-genai");
17
- return new ChatGoogleGenerativeAI({ apiKey, model: aiModel, temperature: 0.7, maxTokens });
18
- }
19
- case 'DeepSeek': {
20
- const { ChatDeepSeek } = await import("@langchain/deepseek");
21
- return new ChatDeepSeek({ apiKey, model: aiModel, temperature: 0.7, maxTokens });
22
- }
23
- case 'Anthropic': {
24
- const { ChatAnthropic } = await import("@langchain/anthropic");
25
- return new ChatAnthropic({ apiKey, model: aiModel, temperature: 0.7, maxTokens });
26
- }
27
- default:
28
- throw new Error(`Unsupported AI provider: ${aiProvider}`);
29
- }
30
- } catch (e) {
31
- if (e.code === 'ERR_MODULE_NOT_FOUND') {
32
- logger.error(`[Assistant] The package for the '${aiProvider}' provider is not installed. Please run 'npm install @langchain/${aiProvider.toLowerCase()}' to use this provider.`);
33
- throw new Error(`The AI provider '${aiProvider}' is not installed. Please ask the administrator to install the corresponding package.`);
34
- }
35
- logger.error(`[Assistant] Error initializing AI provider '${aiProvider}': ${e.message}`);
36
- throw e; // Re-throw other errors
37
- }
1
+ import { Config } from "../../config.js";
2
+ import {assistantConfig} from "../../constants.js";
3
+ import { Logger } from "../../gameObject.js";
4
+ import { getCollectionForUser } from "../mongodb.js";
5
+ import { providers } from "./constants.js";
6
+ import process from "node:process";
7
+
8
+ const logger = new Logger("AssistantProviders");
9
+
10
+ /**
11
+ * Finds the first available AI provider based on configured API keys for a given user.
12
+ * It checks the user's specific environment variables first, then falls back to the machine's process.env.
13
+ * @param {object} user - The user object.
14
+ * @param {string|null} [preferredProvider=null] - If specified, will only check for this provider.
15
+ * @returns {Promise<{provider: string, apiKey: string}|null>} An object with the provider name and API key, or null if none are found.
16
+ */
17
+ export async function findFirstAvailableProvider(user, preferredProvider = null) {
18
+ const envCollection = await getCollectionForUser(user);
19
+ const providerNames = preferredProvider ? [preferredProvider] : Object.keys(providers);
20
+
21
+ for (const pName of providerNames) {
22
+ const providerInfo = providers[pName];
23
+ if (!providerInfo) continue;
24
+
25
+ const envKeyName = providerInfo.key;
26
+ const userEnvVar = await envCollection.findOne({ _model: 'env', name: envKeyName, _user: user.username });
27
+ const key = userEnvVar?.value || process.env[envKeyName];
28
+
29
+ if (key) {
30
+ return { provider: pName, apiKey: key };
31
+ }
32
+ }
33
+ return null;
34
+ }
35
+
36
+ export const getAIProvider = async (aiProvider, aiModel, apiKey, isJsonMode = false) => {
37
+ const maxTokens = Config.Get('assistant.maxTokens', assistantConfig.maxTokens);
38
+ const modelOptions = {
39
+ apiKey,
40
+ model: aiModel,
41
+ temperature: 0.7,
42
+ maxTokens
43
+ };
44
+
45
+ if (isJsonMode) {
46
+ modelOptions.response_format = { type: "json_object" };
47
+ }
48
+
49
+ try {
50
+ switch (aiProvider) {
51
+ case 'OpenAI': {
52
+ const { ChatOpenAI } = await import("@langchain/openai");
53
+ return new ChatOpenAI(modelOptions);
54
+ }
55
+ case 'Google': {
56
+ const { ChatGoogleGenerativeAI } = await import("@langchain/google-genai");
57
+ return new ChatGoogleGenerativeAI(modelOptions);
58
+ }
59
+ case 'DeepSeek': {
60
+ const { ChatDeepSeek } = await import("@langchain/deepseek");
61
+ return new ChatDeepSeek(modelOptions);
62
+ }
63
+ case 'Anthropic': {
64
+ const { ChatAnthropic } = await import("@langchain/anthropic");
65
+ return new ChatAnthropic(modelOptions);
66
+ }
67
+ default:
68
+ throw new Error(`Unsupported AI provider: ${aiProvider}`);
69
+ }
70
+ } catch (e) {
71
+ if (e.code === 'ERR_MODULE_NOT_FOUND') {
72
+ logger.error(`[Assistant] The package for the '${aiProvider}' provider is not installed. Please run 'npm install @langchain/${aiProvider.toLowerCase()}' to use this provider.`);
73
+ throw new Error(`The AI provider '${aiProvider}' is not installed. Please ask the administrator to install the corresponding package.`);
74
+ }
75
+ logger.error(`[Assistant] Error initializing AI provider '${aiProvider}': ${e.message}`);
76
+ throw e; // Re-throw other errors
77
+ }
38
78
  }
@@ -1,51 +1,54 @@
1
- import { Strategy as GoogleStrategy } from 'passport-google-oauth20';
2
- import { Sso, SSOUserProvider } from '../../sso.js';
3
- import {Logger} from "../../gameObject.js";
4
-
5
- let logger;
6
-
7
- export async function onInit(engine) {
8
- logger = engine.getComponent(Logger);
9
-
10
- // Le SSOUserProvider est nécessaire pour l'initialisation et pour la stratégie.
11
- // On le crée donc en premier.
12
- const ssoUserProvider = new SSOUserProvider(engine);
13
-
14
- // 1. Récupérer le composant PassportAuth central.
15
- let passportAuth = engine.getComponent(Sso);
16
- if (!passportAuth) {
17
- passportAuth = engine.addComponent(Sso);
18
- // C'EST ICI QUE L'INITIALISATION DOIT AVOIR LIEU, UNE SEULE FOIS.
19
- passportAuth.initialize({ ssoUserProvider });
20
- passportAuth.addLogoutRoute();
21
- logger.info("[auth-google] Sso component was not found, created and initialized a new one.");
22
- }
23
-
24
- // 2. Vérifier que les variables d'environnement nécessaires sont présentes.
25
- if (!process.env.GOOGLE_CLIENT_ID || !process.env.GOOGLE_CLIENT_SECRET) {
26
- logger.warn("[auth-google] GOOGLE_CLIENT_ID or GOOGLE_CLIENT_SECRET are not set. Google SSO will be disabled.");
27
- return;
28
- }
29
-
30
- // 3. Créer l'instance de la stratégie Google.
31
- const googleStrategy = new GoogleStrategy({
32
- clientID: process.env.GOOGLE_CLIENT_ID,
33
- clientSecret: process.env.GOOGLE_CLIENT_SECRET,
34
- callbackURL: "/api/auth/google/callback"
35
- },
36
- async (accessToken, refreshToken, profile, done) => {
37
- try {
38
- // On utilise le SSOUserProvider pour trouver ou créer l'utilisateur.
39
- const user = await ssoUserProvider.findOrCreate(profile);
40
- return done(null, user);
41
- } catch (err) {
42
- return done(err);
43
- }
44
- });
45
-
46
- // 4. Enregistrer la stratégie auprès du composant PassportAuth.
47
- passportAuth.addStrategy('google', googleStrategy,
48
- { authPath: '/api/auth/google', callbackPath: '/api/auth/google/callback' },
49
- { scope: ['profile', 'email'] }
50
- );
1
+ import { Strategy as GoogleStrategy } from 'passport-google-oauth20';
2
+ import { Sso, SSOUserProvider } from '../../sso.js';
3
+ import {Logger} from "../../gameObject.js";
4
+ import passport from "passport";
5
+
6
+ let logger;
7
+
8
+ export async function onInit(engine) {
9
+ logger = engine.getComponent(Logger);
10
+
11
+ // Le SSOUserProvider est nécessaire pour l'initialisation et pour la stratégie.
12
+ // On le crée donc en premier.
13
+ const ssoUserProvider = new SSOUserProvider(engine);
14
+
15
+ // 1. Récupérer le composant PassportAuth central.
16
+ let ssoComponent = engine.getComponent(Sso);
17
+ if (!ssoComponent) {
18
+ ssoComponent = engine.addComponent(Sso);
19
+ // C'EST ICI QUE L'INITIALISATION DOIT AVOIR LIEU, UNE SEULE FOIS.
20
+ ssoComponent.initialize({ ssoUserProvider });
21
+ ssoComponent.addLogoutRoute();
22
+ logger.info("[auth-google] Sso component was not found, created and initialized a new one.");
23
+ }
24
+
25
+ // 2. Vérifier que les variables d'environnement nécessaires sont présentes.
26
+ if (!process.env.GOOGLE_CLIENT_ID || !process.env.GOOGLE_CLIENT_SECRET) {
27
+ logger.warn("[auth-google] GOOGLE_CLIENT_ID or GOOGLE_CLIENT_SECRET are not set. Google SSO will be disabled.");
28
+ return;
29
+ }
30
+
31
+ // 3. Créer l'instance de la stratégie Google.
32
+ const googleStrategy = new GoogleStrategy({
33
+ clientID: process.env.GOOGLE_CLIENT_ID,
34
+ clientSecret: process.env.GOOGLE_CLIENT_SECRET,
35
+ callbackURL: "/api/auth/google/callback"
36
+ },
37
+ async (accessToken, refreshToken, profile, done) => {
38
+ try {
39
+ const user = await ssoUserProvider.findOrCreate(profile);
40
+ // On passe le profil brut dans le 3ème argument pour le récupérer dans le callback de `authenticate`
41
+ // et le passer à l'événement OnSsoLogin.
42
+ return done(null, user, { profile });
43
+ } catch (err) {
44
+ return done(err);
45
+ }
46
+ });
47
+
48
+ // 4. Enregistrer la stratégie auprès du composant SSO.
49
+ // Le composant Sso gère lui-même les routes, le callback, req.logIn() et le déclenchement de l'événement.
50
+ ssoComponent.addStrategy('google', googleStrategy,
51
+ { authPath: '/api/auth/google', callbackPath: '/api/auth/google/callback' },
52
+ { scope: ['profile', 'email'] }
53
+ );
51
54
  }