@smythos/sre 1.7.5 → 1.7.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +28 -9
- package/dist/index.js.map +1 -1
- package/dist/types/Core/ConnectorsService.d.ts +15 -16
- package/dist/types/Core/SmythRuntime.class.d.ts +3 -0
- package/dist/types/subsystems/IO/Storage.service/SmythFS.class.d.ts +2 -2
- package/dist/types/subsystems/Security/Vault.service/connectors/JSONFileVault.class.d.ts +2 -0
- package/dist/types/utils/cli.utils.d.ts +36 -0
- package/package.json +1 -1
- package/src/Core/ConnectorsService.ts +51 -22
- package/src/Core/SmythRuntime.class.ts +42 -34
- package/src/subsystems/IO/Storage.service/SmythFS.class.ts +5 -6
- package/src/subsystems/IO/VectorDB.service/connectors/MilvusVectorDB.class.ts +2 -2
- package/src/subsystems/IO/VectorDB.service/connectors/PineconeVectorDB.class.ts +2 -2
- package/src/subsystems/IO/VectorDB.service/connectors/RAMVecrtorDB.class.ts +2 -2
- package/src/subsystems/IO/VectorDB.service/embed/BaseEmbedding.ts +2 -2
- package/src/subsystems/IO/VectorDB.service/embed/OpenAIEmbedding.ts +1 -1
- package/src/subsystems/Security/Vault.service/connectors/JSONFileVault.class.ts +60 -2
- package/src/utils/cli.utils.ts +161 -0
- package/dist/bundle-analysis-lazy.html +0 -4949
- package/dist/bundle-analysis.html +0 -4949
|
@@ -1,24 +1,23 @@
|
|
|
1
|
-
import { TConnectorService, TServiceRegistry } from '@sre/types/SRE.types';
|
|
2
|
-
import { Connector } from './Connector.class';
|
|
3
|
-
import { StorageConnector } from '@sre/IO/Storage.service/StorageConnector';
|
|
4
|
-
import { CacheConnector } from '@sre/MemoryManager/Cache.service/CacheConnector';
|
|
5
|
-
import { LLMConnector } from '@sre/LLMManager/LLM.service/LLMConnector';
|
|
6
|
-
import { VaultConnector } from '@sre/Security/Vault.service/VaultConnector';
|
|
7
|
-
import { AccountConnector } from '@sre/Security/Account.service/AccountConnector';
|
|
8
1
|
import { AgentDataConnector } from '@sre/AgentManager/AgentData.service/AgentDataConnector';
|
|
9
|
-
import {
|
|
2
|
+
import { ComponentConnector } from '@sre/AgentManager/Component.service/ComponentConnector';
|
|
3
|
+
import { SchedulerConnector } from '@sre/AgentManager/Scheduler.service/SchedulerConnector';
|
|
4
|
+
import { CodeConnector } from '@sre/ComputeManager/Code.service/CodeConnector';
|
|
10
5
|
import { CLIConnector } from '@sre/IO/CLI.service/CLIConnector';
|
|
6
|
+
import { LogConnector } from '@sre/IO/Log.service/LogConnector';
|
|
11
7
|
import { NKVConnector } from '@sre/IO/NKV.service/NKVConnector';
|
|
12
8
|
import { RouterConnector } from '@sre/IO/Router.service/RouterConnector';
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
9
|
+
import { StorageConnector } from '@sre/IO/Storage.service/StorageConnector';
|
|
10
|
+
import { VectorDBConnector } from '@sre/IO/VectorDB.service/VectorDBConnector';
|
|
11
|
+
import { LLMConnector } from '@sre/LLMManager/LLM.service/LLMConnector';
|
|
16
12
|
import { ModelsProviderConnector } from '@sre/LLMManager/ModelsProvider.service/ModelsProviderConnector';
|
|
17
|
-
import {
|
|
18
|
-
import {
|
|
13
|
+
import { CacheConnector } from '@sre/MemoryManager/Cache.service/CacheConnector';
|
|
14
|
+
import { AccountConnector } from '@sre/Security/Account.service/AccountConnector';
|
|
15
|
+
import { ManagedVaultConnector } from '@sre/Security/ManagedVault.service/ManagedVaultConnector';
|
|
16
|
+
import { VaultConnector } from '@sre/Security/Vault.service/VaultConnector';
|
|
17
|
+
import { TConnectorService, TServiceRegistry } from '@sre/types/SRE.types';
|
|
19
18
|
export declare class ConnectorService {
|
|
20
|
-
static Connectors:
|
|
21
|
-
static ConnectorInstances: any;
|
|
19
|
+
static get Connectors(): any;
|
|
20
|
+
static get ConnectorInstances(): any;
|
|
22
21
|
static get ready(): boolean;
|
|
23
22
|
static get service(): TServiceRegistry;
|
|
24
23
|
/**
|
|
@@ -40,7 +39,7 @@ export declare class ConnectorService {
|
|
|
40
39
|
* @param isDefault
|
|
41
40
|
* @returns
|
|
42
41
|
*/
|
|
43
|
-
static init(connectorType: TConnectorService, connectorName: string, connectorId?: string, settings?: any, isDefault?: boolean):
|
|
42
|
+
static init(connectorType: TConnectorService, connectorName: string, connectorId?: string, settings?: any, isDefault?: boolean): any;
|
|
44
43
|
static _stop(): Promise<void>;
|
|
45
44
|
static getInstance<T>(connectorType: TConnectorService, connectorName?: string): T;
|
|
46
45
|
static getStorageConnector(name?: string): StorageConnector;
|
|
@@ -7,6 +7,8 @@ export declare class SmythRuntime {
|
|
|
7
7
|
private _readyResolve;
|
|
8
8
|
get version(): string;
|
|
9
9
|
private defaultConfig;
|
|
10
|
+
connectors: any;
|
|
11
|
+
connectorInstances: any;
|
|
10
12
|
protected constructor();
|
|
11
13
|
protected static instance?: SmythRuntime;
|
|
12
14
|
static get Instance(): SmythRuntime;
|
|
@@ -25,5 +27,6 @@ export declare class SmythRuntime {
|
|
|
25
27
|
ready(): Promise<boolean>;
|
|
26
28
|
private _stopping;
|
|
27
29
|
_stop(): Promise<void>;
|
|
30
|
+
private setupShutdownHandlers;
|
|
28
31
|
}
|
|
29
32
|
export declare const SRE: SmythRuntime;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
+
import { CacheConnector } from '@sre/MemoryManager/Cache.service/CacheConnector';
|
|
1
2
|
import { IAccessCandidate } from '@sre/types/ACL.types';
|
|
2
3
|
import { StorageData, StorageMetadata } from '@sre/types/Storage.types';
|
|
3
|
-
import { StorageConnector } from '
|
|
4
|
-
import { CacheConnector } from '@sre/MemoryManager/Cache.service/CacheConnector';
|
|
4
|
+
import { StorageConnector } from '@sre/IO/Storage.service/StorageConnector';
|
|
5
5
|
export type TSmythFSURI = {
|
|
6
6
|
hash: string;
|
|
7
7
|
team: string;
|
|
@@ -15,9 +15,11 @@ export declare class JSONFileVault extends VaultConnector {
|
|
|
15
15
|
private shared;
|
|
16
16
|
private vaultFile;
|
|
17
17
|
private watcher;
|
|
18
|
+
private _interactiveVaultCreation;
|
|
18
19
|
constructor(_settings: JSONFileVaultConfig);
|
|
19
20
|
private findVaultFile;
|
|
20
21
|
private getMasterKeyInteractive;
|
|
22
|
+
private createDefaultVaultInteractive;
|
|
21
23
|
/**
|
|
22
24
|
* Resolves environment variable references in vault values.
|
|
23
25
|
* Supports syntax: $env(VARIABLE_NAME)
|
|
@@ -18,3 +18,39 @@ export declare function parseCLIArgs(argList: string | Array<string>, argv?: Arr
|
|
|
18
18
|
* calling getMainArgs(process.argv) will return ['file', 'settings']
|
|
19
19
|
*/
|
|
20
20
|
export declare function getMainArgs(argv?: Array<string>): Array<string>;
|
|
21
|
+
export declare const colors: {
|
|
22
|
+
reset: string;
|
|
23
|
+
bright: string;
|
|
24
|
+
dim: string;
|
|
25
|
+
underscore: string;
|
|
26
|
+
blink: string;
|
|
27
|
+
reverse: string;
|
|
28
|
+
hidden: string;
|
|
29
|
+
black: string;
|
|
30
|
+
red: string;
|
|
31
|
+
green: string;
|
|
32
|
+
orange: string;
|
|
33
|
+
yellow: string;
|
|
34
|
+
blue: string;
|
|
35
|
+
magenta: string;
|
|
36
|
+
cyan: string;
|
|
37
|
+
white: string;
|
|
38
|
+
bgBlack: string;
|
|
39
|
+
bgRed: string;
|
|
40
|
+
bgGreen: string;
|
|
41
|
+
bgYellow: string;
|
|
42
|
+
bgBlue: string;
|
|
43
|
+
bgMagenta: string;
|
|
44
|
+
bgCyan: string;
|
|
45
|
+
bgWhite: string;
|
|
46
|
+
};
|
|
47
|
+
/**
|
|
48
|
+
* Prompt the user for a yes/no question synchronously.
|
|
49
|
+
* Returns true for Yes, false for No.
|
|
50
|
+
* Keeps asking until a valid response is entered.
|
|
51
|
+
*
|
|
52
|
+
* @param {string} question - The question to ask (no need to add "(Y/N)").
|
|
53
|
+
* @returns {boolean} - True for yes, false for no.
|
|
54
|
+
*/
|
|
55
|
+
export declare function askYesNo(question: any): boolean;
|
|
56
|
+
export declare function askForValues(message: any, prompts: any, options?: {}): {};
|
package/package.json
CHANGED
|
@@ -1,26 +1,28 @@
|
|
|
1
|
-
import { SREConnectorConfig, TConnectorService, TServiceRegistry } from '@sre/types/SRE.types';
|
|
2
|
-
import { DummyConnector } from './DummyConnector';
|
|
3
|
-
import { Logger } from '../helpers/Log.helper';
|
|
4
|
-
import { Connector } from './Connector.class';
|
|
5
|
-
import { getFormattedStackTrace, isSubclassOf, printStackTrace } from '@sre/utils';
|
|
6
|
-
import { SystemEvents } from './SystemEvents';
|
|
7
|
-
import { StorageConnector } from '@sre/IO/Storage.service/StorageConnector';
|
|
8
|
-
import { CacheConnector } from '@sre/MemoryManager/Cache.service/CacheConnector';
|
|
9
|
-
import { LLMConnector } from '@sre/LLMManager/LLM.service/LLMConnector';
|
|
10
|
-
import { VaultConnector } from '@sre/Security/Vault.service/VaultConnector';
|
|
11
|
-
import { AccountConnector } from '@sre/Security/Account.service/AccountConnector';
|
|
12
1
|
import { AgentDataConnector } from '@sre/AgentManager/AgentData.service/AgentDataConnector';
|
|
13
|
-
import {
|
|
2
|
+
import { ComponentConnector } from '@sre/AgentManager/Component.service/ComponentConnector';
|
|
3
|
+
import { SchedulerConnector } from '@sre/AgentManager/Scheduler.service/SchedulerConnector';
|
|
4
|
+
import { CodeConnector } from '@sre/ComputeManager/Code.service/CodeConnector';
|
|
14
5
|
import { CLIConnector } from '@sre/IO/CLI.service/CLIConnector';
|
|
6
|
+
import { LogConnector } from '@sre/IO/Log.service/LogConnector';
|
|
15
7
|
import { NKVConnector } from '@sre/IO/NKV.service/NKVConnector';
|
|
16
8
|
import { RouterConnector } from '@sre/IO/Router.service/RouterConnector';
|
|
17
|
-
import {
|
|
18
|
-
import {
|
|
19
|
-
import {
|
|
9
|
+
import { StorageConnector } from '@sre/IO/Storage.service/StorageConnector';
|
|
10
|
+
import { VectorDBConnector } from '@sre/IO/VectorDB.service/VectorDBConnector';
|
|
11
|
+
import { LLMConnector } from '@sre/LLMManager/LLM.service/LLMConnector';
|
|
20
12
|
import { ModelsProviderConnector } from '@sre/LLMManager/ModelsProvider.service/ModelsProviderConnector';
|
|
21
|
-
import {
|
|
22
|
-
import {
|
|
13
|
+
import { CacheConnector } from '@sre/MemoryManager/Cache.service/CacheConnector';
|
|
14
|
+
import { AccountConnector } from '@sre/Security/Account.service/AccountConnector';
|
|
15
|
+
import { ManagedVaultConnector } from '@sre/Security/ManagedVault.service/ManagedVaultConnector';
|
|
16
|
+
import { VaultConnector } from '@sre/Security/Vault.service/VaultConnector';
|
|
17
|
+
import { TConnectorService, TServiceRegistry } from '@sre/types/SRE.types';
|
|
18
|
+
import { isSubclassOf, printStackTrace } from '@sre/utils';
|
|
19
|
+
import { Logger } from '../helpers/Log.helper';
|
|
20
|
+
import { Connector } from './Connector.class';
|
|
21
|
+
import { DummyConnector } from './DummyConnector';
|
|
22
|
+
import { SystemEvents } from './SystemEvents';
|
|
23
23
|
const console = Logger('ConnectorService');
|
|
24
|
+
const SRE_CONNECTORS_INSTANCE_SYMBOL = Symbol.for('SRE:ConnectorInstances');
|
|
25
|
+
const SRE_CONNECTORS_SYMBOL = Symbol.for('SRE:Connector');
|
|
24
26
|
|
|
25
27
|
let ServiceRegistry: TServiceRegistry = {};
|
|
26
28
|
let _ready = false;
|
|
@@ -29,9 +31,23 @@ SystemEvents.on('SRE:Booted', (services) => {
|
|
|
29
31
|
_ready = true;
|
|
30
32
|
});
|
|
31
33
|
export class ConnectorService {
|
|
32
|
-
public static Connectors = {};
|
|
34
|
+
//public static Connectors = {};
|
|
35
|
+
public static get Connectors() {
|
|
36
|
+
if (!global[SRE_CONNECTORS_SYMBOL]) {
|
|
37
|
+
global[SRE_CONNECTORS_SYMBOL] = {};
|
|
38
|
+
}
|
|
39
|
+
return global[SRE_CONNECTORS_SYMBOL];
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
public static get ConnectorInstances() {
|
|
43
|
+
if (!global[SRE_CONNECTORS_INSTANCE_SYMBOL]) {
|
|
44
|
+
global[SRE_CONNECTORS_INSTANCE_SYMBOL] = {};
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
return global[SRE_CONNECTORS_INSTANCE_SYMBOL];
|
|
48
|
+
}
|
|
49
|
+
//public static ConnectorInstances: any = {};
|
|
33
50
|
|
|
34
|
-
public static ConnectorInstances: any = {};
|
|
35
51
|
public static get ready() {
|
|
36
52
|
return _ready;
|
|
37
53
|
}
|
|
@@ -54,6 +70,10 @@ export class ConnectorService {
|
|
|
54
70
|
if (!ConnectorService.Connectors[connectorType]) {
|
|
55
71
|
ConnectorService.Connectors[connectorType] = {};
|
|
56
72
|
}
|
|
73
|
+
if (ConnectorService.Connectors[connectorType][connectorName]) {
|
|
74
|
+
console.warn(`Connector ${connectorType}:${connectorName} already registered ... skipping`);
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
57
77
|
ConnectorService.Connectors[connectorType][connectorName] = connectorConstructor;
|
|
58
78
|
}
|
|
59
79
|
|
|
@@ -70,7 +90,10 @@ export class ConnectorService {
|
|
|
70
90
|
*/
|
|
71
91
|
static init(connectorType: TConnectorService, connectorName: string, connectorId?: string, settings: any = {}, isDefault = false) {
|
|
72
92
|
if (ConnectorService.ConnectorInstances[connectorType]?.[connectorName]) {
|
|
73
|
-
throw new Error(`Connector ${connectorType}:${connectorName} already initialized`);
|
|
93
|
+
//throw new Error(`Connector ${connectorType}:${connectorName} already initialized`);
|
|
94
|
+
console.warn(`Connector ${connectorType}:${connectorName} already initialized ... skipping`);
|
|
95
|
+
|
|
96
|
+
return ConnectorService.ConnectorInstances[connectorType]?.[connectorName];
|
|
74
97
|
}
|
|
75
98
|
|
|
76
99
|
const entry = ConnectorService.Connectors[connectorType];
|
|
@@ -104,6 +127,8 @@ export class ConnectorService {
|
|
|
104
127
|
connector.stop();
|
|
105
128
|
}
|
|
106
129
|
}
|
|
130
|
+
delete global[SRE_CONNECTORS_INSTANCE_SYMBOL];
|
|
131
|
+
delete global[SRE_CONNECTORS_SYMBOL];
|
|
107
132
|
}
|
|
108
133
|
static getInstance<T>(connectorType: TConnectorService, connectorName: string = 'default'): T {
|
|
109
134
|
const instance = ConnectorService.ConnectorInstances[connectorType]?.[connectorName || 'default'] as T;
|
|
@@ -113,10 +138,14 @@ export class ConnectorService {
|
|
|
113
138
|
// //return the first instance
|
|
114
139
|
// return ConnectorService.ConnectorInstances[connectorType][Object.keys(ConnectorService.ConnectorInstances[connectorType])[0]] as T;
|
|
115
140
|
// }
|
|
116
|
-
console.warn(
|
|
141
|
+
console.warn(
|
|
142
|
+
`Connector ${connectorType}:${
|
|
143
|
+
typeof connectorName === 'string' ? connectorName : JSON.stringify(connectorName)
|
|
144
|
+
} not initialized returning DummyConnector`
|
|
145
|
+
);
|
|
117
146
|
//print stack trace
|
|
118
147
|
|
|
119
|
-
printStackTrace(console,
|
|
148
|
+
printStackTrace(console, 10);
|
|
120
149
|
|
|
121
150
|
return DummyConnector(connectorType) as T;
|
|
122
151
|
}
|
|
@@ -9,7 +9,7 @@ import pkg from '../../package.json';
|
|
|
9
9
|
|
|
10
10
|
const logger = Logger('SRE');
|
|
11
11
|
|
|
12
|
-
const
|
|
12
|
+
const SRE_INSTANCE_SYMBOL = Symbol.for('SRE:Instance');
|
|
13
13
|
|
|
14
14
|
export class SmythRuntime {
|
|
15
15
|
public started = false;
|
|
@@ -78,6 +78,8 @@ export class SmythRuntime {
|
|
|
78
78
|
},
|
|
79
79
|
};
|
|
80
80
|
|
|
81
|
+
public connectors: any = {};
|
|
82
|
+
public connectorInstances: any = {};
|
|
81
83
|
protected constructor() {
|
|
82
84
|
this.started = true;
|
|
83
85
|
this._readyPromise = new Promise((resolve) => {
|
|
@@ -86,22 +88,22 @@ export class SmythRuntime {
|
|
|
86
88
|
}
|
|
87
89
|
|
|
88
90
|
protected static instance?: SmythRuntime;
|
|
89
|
-
// public static get Instance(): SmythRuntime {
|
|
90
|
-
// if (!SmythRuntime.instance) {
|
|
91
|
-
// SmythRuntime.instance = new SmythRuntime();
|
|
92
|
-
// }
|
|
93
|
-
// return SmythRuntime.instance;
|
|
94
|
-
// }
|
|
95
|
-
|
|
96
91
|
public static get Instance(): SmythRuntime {
|
|
97
|
-
if (
|
|
98
|
-
|
|
92
|
+
if (!SmythRuntime.instance) {
|
|
93
|
+
SmythRuntime.instance = new SmythRuntime();
|
|
99
94
|
}
|
|
100
|
-
SmythRuntime.instance = new SmythRuntime();
|
|
101
|
-
global[SRE_GLOBAL_KEY] = SmythRuntime.instance;
|
|
102
95
|
return SmythRuntime.instance;
|
|
103
96
|
}
|
|
104
97
|
|
|
98
|
+
// public static get Instance(): SmythRuntime {
|
|
99
|
+
// if (global[SRE_INSTANCE_SYMBOL]) {
|
|
100
|
+
// return global[SRE_INSTANCE_SYMBOL];
|
|
101
|
+
// }
|
|
102
|
+
// global[SRE_INSTANCE_SYMBOL] = new SmythRuntime();
|
|
103
|
+
// //global[SRE_GLOBAL_KEY] = SmythRuntime.instance;
|
|
104
|
+
// return global[SRE_INSTANCE_SYMBOL];
|
|
105
|
+
// }
|
|
106
|
+
|
|
105
107
|
private _initializing = false;
|
|
106
108
|
|
|
107
109
|
public get initializing() {
|
|
@@ -112,18 +114,20 @@ export class SmythRuntime {
|
|
|
112
114
|
|
|
113
115
|
public init(_config?: SREConfig): SmythRuntime {
|
|
114
116
|
logger.info(`SRE v${this.version} initializing...`);
|
|
115
|
-
if (
|
|
116
|
-
|
|
117
|
-
|
|
117
|
+
if (this._initialized) {
|
|
118
|
+
console.warn('SRE already initialized ... skipping');
|
|
119
|
+
return SmythRuntime.Instance;
|
|
118
120
|
}
|
|
119
121
|
|
|
120
122
|
if (this._initializing) {
|
|
121
123
|
console.warn('You tried to initialize SRE while it is already initializing ... skipping');
|
|
122
|
-
return;
|
|
124
|
+
return SmythRuntime.Instance;
|
|
123
125
|
}
|
|
124
|
-
if (
|
|
125
|
-
|
|
126
|
+
if (!_config || JSON.stringify(_config) === '{}') {
|
|
127
|
+
this._smythDir = findSmythPath();
|
|
128
|
+
logger.info('.smyth directory found in:', this._smythDir);
|
|
126
129
|
}
|
|
130
|
+
|
|
127
131
|
this._initializing = true;
|
|
128
132
|
SystemEvents.on('SRE:Booted', () => {
|
|
129
133
|
this._readyResolve(true);
|
|
@@ -150,6 +154,8 @@ export class SmythRuntime {
|
|
|
150
154
|
this._initialized = true;
|
|
151
155
|
SystemEvents.emit('SRE:Initialized', SmythRuntime.Instance);
|
|
152
156
|
|
|
157
|
+
this.setupShutdownHandlers();
|
|
158
|
+
|
|
153
159
|
return SmythRuntime.Instance as SmythRuntime;
|
|
154
160
|
}
|
|
155
161
|
|
|
@@ -216,9 +222,26 @@ export class SmythRuntime {
|
|
|
216
222
|
this._stopping = true;
|
|
217
223
|
logger.info('Sending Shutdown Signals To All Subsystems...');
|
|
218
224
|
await ConnectorService._stop();
|
|
219
|
-
|
|
225
|
+
//delete global[SRE_INSTANCE_SYMBOL];
|
|
220
226
|
this.started = false;
|
|
221
227
|
}
|
|
228
|
+
|
|
229
|
+
private setupShutdownHandlers() {
|
|
230
|
+
['SIGINT', 'SIGTERM'].forEach((signal) => {
|
|
231
|
+
process.on(signal, async () => {
|
|
232
|
+
await shutdown(signal);
|
|
233
|
+
process.exit(0); // Required after async
|
|
234
|
+
});
|
|
235
|
+
});
|
|
236
|
+
|
|
237
|
+
process.on('beforeExit', (code) => {
|
|
238
|
+
shutdown('beforeExit');
|
|
239
|
+
});
|
|
240
|
+
|
|
241
|
+
process.on('exit', (code) => {
|
|
242
|
+
logger.info(`Goodbye!`);
|
|
243
|
+
});
|
|
244
|
+
}
|
|
222
245
|
}
|
|
223
246
|
|
|
224
247
|
export const SRE = SmythRuntime.Instance;
|
|
@@ -239,21 +262,6 @@ async function shutdown(reason) {
|
|
|
239
262
|
}
|
|
240
263
|
}
|
|
241
264
|
|
|
242
|
-
['SIGINT', 'SIGTERM'].forEach((signal) => {
|
|
243
|
-
process.on(signal, async () => {
|
|
244
|
-
await shutdown(signal);
|
|
245
|
-
process.exit(0); // Required after async
|
|
246
|
-
});
|
|
247
|
-
});
|
|
248
|
-
|
|
249
|
-
process.on('beforeExit', (code) => {
|
|
250
|
-
shutdown('beforeExit');
|
|
251
|
-
});
|
|
252
|
-
|
|
253
|
-
process.on('exit', (code) => {
|
|
254
|
-
logger.info(`Goodbye!`);
|
|
255
|
-
});
|
|
256
|
-
|
|
257
265
|
// process.on('uncaughtException', (err) => {
|
|
258
266
|
|
|
259
267
|
// });
|
|
@@ -1,17 +1,15 @@
|
|
|
1
1
|
import { ConnectorService } from '@sre/Core/ConnectorsService';
|
|
2
|
+
import { JSONContentHelper } from '@sre/helpers/JsonContent.helper';
|
|
3
|
+
import { CacheConnector } from '@sre/MemoryManager/Cache.service/CacheConnector';
|
|
2
4
|
import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
|
|
3
5
|
import { ACL } from '@sre/Security/AccessControl/ACL.class';
|
|
4
6
|
import { DEFAULT_TEAM_ID, IAccessCandidate, TAccessLevel, TAccessRole } from '@sre/types/ACL.types';
|
|
5
7
|
import { StorageData, StorageMetadata } from '@sre/types/Storage.types';
|
|
6
8
|
import { getMimeType } from '@sre/utils';
|
|
9
|
+
import crypto from 'crypto';
|
|
7
10
|
import mime from 'mime';
|
|
8
11
|
import { Readable } from 'stream';
|
|
9
|
-
import { StorageConnector } from '
|
|
10
|
-
import { SmythRuntime } from '@sre/Core/SmythRuntime.class';
|
|
11
|
-
import { CacheConnector } from '@sre/MemoryManager/Cache.service/CacheConnector';
|
|
12
|
-
import crypto from 'crypto';
|
|
13
|
-
import { JSONContentHelper } from '@sre/helpers/JsonContent.helper';
|
|
14
|
-
import { SystemEvents } from '@sre/Core/SystemEvents';
|
|
12
|
+
import { StorageConnector } from '@sre/IO/Storage.service/StorageConnector';
|
|
15
13
|
|
|
16
14
|
export type TSmythFSURI = {
|
|
17
15
|
hash: string;
|
|
@@ -46,6 +44,7 @@ export class SmythFS {
|
|
|
46
44
|
// Multiton pattern - get instance based on storage and cache provider combination
|
|
47
45
|
public static getInstance(storageProvider: string | StorageConnector = '', cacheProvider: string | CacheConnector = ''): SmythFS {
|
|
48
46
|
// First get the actual connector names to calculate the correct hash
|
|
47
|
+
|
|
49
48
|
const storage = storageProvider instanceof StorageConnector ? storageProvider : ConnectorService.getStorageConnector(storageProvider);
|
|
50
49
|
const cache = cacheProvider instanceof CacheConnector ? cacheProvider : ConnectorService.getCacheConnector(cacheProvider);
|
|
51
50
|
const hash = SmythFS.generateInstanceHash(storage.name, cache.name);
|
|
@@ -78,10 +78,10 @@ export class MilvusVectorDB extends VectorDBConnector {
|
|
|
78
78
|
this.cache = ConnectorService.getCacheConnector();
|
|
79
79
|
|
|
80
80
|
if (!_settings.embeddings) {
|
|
81
|
-
_settings.embeddings = { provider: 'OpenAI', model: 'text-embedding-3-large', dimensions:
|
|
81
|
+
_settings.embeddings = { provider: 'OpenAI', model: 'text-embedding-3-large', dimensions: 3072 };
|
|
82
82
|
}
|
|
83
83
|
|
|
84
|
-
if (!_settings.embeddings.dimensions) _settings.embeddings.dimensions =
|
|
84
|
+
if (!_settings.embeddings.dimensions) _settings.embeddings.dimensions = 3072;
|
|
85
85
|
|
|
86
86
|
this.embedder = EmbeddingsFactory.create(_settings.embeddings.provider, _settings.embeddings);
|
|
87
87
|
|
|
@@ -74,9 +74,9 @@ export class PineconeVectorDB extends VectorDBConnector {
|
|
|
74
74
|
this.cache = ConnectorService.getCacheConnector();
|
|
75
75
|
this.nkvConnector = ConnectorService.getNKVConnector();
|
|
76
76
|
if (!_settings.embeddings) {
|
|
77
|
-
_settings.embeddings = { provider: 'OpenAI', model: 'text-embedding-3-large', dimensions:
|
|
77
|
+
_settings.embeddings = { provider: 'OpenAI', model: 'text-embedding-3-large', dimensions: 3072 };
|
|
78
78
|
}
|
|
79
|
-
if (!_settings.embeddings.dimensions) _settings.embeddings.dimensions =
|
|
79
|
+
if (!_settings.embeddings.dimensions) _settings.embeddings.dimensions = 3072;
|
|
80
80
|
|
|
81
81
|
this.embedder = EmbeddingsFactory.create(_settings.embeddings.provider, _settings.embeddings);
|
|
82
82
|
}
|
|
@@ -72,10 +72,10 @@ export class RAMVectorDB extends VectorDBConnector {
|
|
|
72
72
|
this.accountConnector = ConnectorService.getAccountConnector();
|
|
73
73
|
|
|
74
74
|
if (!_settings.embeddings) {
|
|
75
|
-
_settings.embeddings = { provider: 'OpenAI', model: 'text-embedding-3-large', dimensions:
|
|
75
|
+
_settings.embeddings = { provider: 'OpenAI', model: 'text-embedding-3-large', dimensions: 3072 };
|
|
76
76
|
}
|
|
77
77
|
|
|
78
|
-
if (!_settings.embeddings.dimensions) _settings.embeddings.dimensions =
|
|
78
|
+
if (!_settings.embeddings.dimensions) _settings.embeddings.dimensions = 3072;
|
|
79
79
|
|
|
80
80
|
this.embedder = EmbeddingsFactory.create(_settings.embeddings.provider, _settings.embeddings);
|
|
81
81
|
}
|
|
@@ -66,8 +66,8 @@ export abstract class BaseEmbedding {
|
|
|
66
66
|
|
|
67
67
|
public chunkText(text: string, { chunkSize, chunkOverlap }: { chunkSize?: number; chunkOverlap?: number }): string[] {
|
|
68
68
|
const textSplitter = new TextSplitter({
|
|
69
|
-
chunkSize: chunkSize
|
|
70
|
-
chunkOverlap: chunkOverlap
|
|
69
|
+
chunkSize: chunkSize ?? this.chunkSize,
|
|
70
|
+
chunkOverlap: chunkOverlap ?? this.chunkOverlap,
|
|
71
71
|
});
|
|
72
72
|
return textSplitter.splitText(text);
|
|
73
73
|
}
|
|
@@ -18,7 +18,7 @@ const createOpenAIError = (statusCode: number, error: any) => {
|
|
|
18
18
|
);
|
|
19
19
|
};
|
|
20
20
|
|
|
21
|
-
const DEFAULT_MODEL = 'text-embedding-
|
|
21
|
+
const DEFAULT_MODEL = 'text-embedding-3-large';
|
|
22
22
|
|
|
23
23
|
export class OpenAIEmbeds extends BaseEmbedding {
|
|
24
24
|
protected client: OpenAIClient;
|
|
@@ -10,6 +10,9 @@ import crypto from 'crypto';
|
|
|
10
10
|
import fs from 'fs';
|
|
11
11
|
import * as readlineSync from 'readline-sync';
|
|
12
12
|
import { VaultConnector } from '../VaultConnector';
|
|
13
|
+
import { askForValues, colors } from '@sre/utils/index';
|
|
14
|
+
import os from 'os';
|
|
15
|
+
import path from 'path';
|
|
13
16
|
|
|
14
17
|
const logger = Logger('JSONFileVault');
|
|
15
18
|
|
|
@@ -27,6 +30,8 @@ export class JSONFileVault extends VaultConnector {
|
|
|
27
30
|
private vaultFile: string;
|
|
28
31
|
private watcher: chokidar.FSWatcher | null = null;
|
|
29
32
|
|
|
33
|
+
private _interactiveVaultCreation: boolean = false;
|
|
34
|
+
|
|
30
35
|
constructor(protected _settings: JSONFileVaultConfig) {
|
|
31
36
|
super(_settings);
|
|
32
37
|
//if (!SmythRuntime.Instance) throw new Error('SRE not initialized');
|
|
@@ -61,8 +66,15 @@ export class JSONFileVault extends VaultConnector {
|
|
|
61
66
|
}
|
|
62
67
|
|
|
63
68
|
console.warn('!!! All attempts to find the vault file failed !!!');
|
|
64
|
-
|
|
65
|
-
|
|
69
|
+
if (!this._interactiveVaultCreation) {
|
|
70
|
+
this.createDefaultVaultInteractive();
|
|
71
|
+
this._interactiveVaultCreation = true;
|
|
72
|
+
return this.findVaultFile(this.vaultFile);
|
|
73
|
+
} else {
|
|
74
|
+
process.stdout.write(colors.red + '[ERR] Could not find or create a valid vault file.\n' + colors.reset);
|
|
75
|
+
console.warn('!!! SRE Will continue without vault !!!');
|
|
76
|
+
console.warn('!!! Many features might not work !!!');
|
|
77
|
+
}
|
|
66
78
|
|
|
67
79
|
return null;
|
|
68
80
|
}
|
|
@@ -79,6 +91,52 @@ export class JSONFileVault extends VaultConnector {
|
|
|
79
91
|
return masterKey;
|
|
80
92
|
}
|
|
81
93
|
|
|
94
|
+
private createDefaultVaultInteractive(): boolean {
|
|
95
|
+
const userVaultDir = path.resolve(os.homedir(), '.smyth');
|
|
96
|
+
|
|
97
|
+
const userVaultPath = path.resolve(userVaultDir, 'vault.json');
|
|
98
|
+
|
|
99
|
+
process.stdout.write(colors.red + '\n\n\n\n SRE Initialization was interrupted because no valid vault file was found.\n' + colors.reset);
|
|
100
|
+
process.stdout.write(
|
|
101
|
+
colors.red + 'I will help you create a default vault file here : ' + path.relative(process.cwd(), userVaultPath) + '\n' + colors.reset
|
|
102
|
+
);
|
|
103
|
+
process.stdout.write(colors.red + '\nUse Ctrl+C to cancel.\n' + colors.reset);
|
|
104
|
+
process.stdout.write(colors.bright + '===[ SRE : JSON Vault Creation ]=========================' + colors.reset + '\n');
|
|
105
|
+
|
|
106
|
+
const apiKeys = askForValues('Please enter the API keys for your LLM providers (Press Enter to skip any key):', {
|
|
107
|
+
openai: 'OpenAI : ',
|
|
108
|
+
anthropic: 'Anthropic : ',
|
|
109
|
+
googleai: 'Google AI : ',
|
|
110
|
+
xai: 'xAI : ',
|
|
111
|
+
groq: 'Groq : ',
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
const defaultVault = {
|
|
115
|
+
default: {
|
|
116
|
+
echo: '',
|
|
117
|
+
openai: '',
|
|
118
|
+
anthropic: '',
|
|
119
|
+
googleai: '',
|
|
120
|
+
groq: '',
|
|
121
|
+
togetherai: '',
|
|
122
|
+
xai: '',
|
|
123
|
+
deepseek: '',
|
|
124
|
+
tavily: '',
|
|
125
|
+
scrapfly: '',
|
|
126
|
+
...apiKeys,
|
|
127
|
+
},
|
|
128
|
+
};
|
|
129
|
+
fs.mkdirSync(userVaultDir, { recursive: true });
|
|
130
|
+
fs.writeFileSync(userVaultPath, JSON.stringify(defaultVault, null, 2));
|
|
131
|
+
|
|
132
|
+
process.stdout.write(colors.bright + colors.green + '\nThe vault file is now created at : ' + userVaultPath + colors.reset);
|
|
133
|
+
process.stdout.write(colors.bright + '\nYou can edit it later if you want to add/update keys.' + colors.reset);
|
|
134
|
+
|
|
135
|
+
process.stdout.write(colors.bright + '\n================================================\n\n\n' + colors.reset);
|
|
136
|
+
|
|
137
|
+
return true;
|
|
138
|
+
}
|
|
139
|
+
|
|
82
140
|
/**
|
|
83
141
|
* Resolves environment variable references in vault values.
|
|
84
142
|
* Supports syntax: $env(VARIABLE_NAME)
|