@smythos/sre 1.5.1 → 1.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +18 -0
- package/dist/index.js +22329 -4
- package/dist/index.js.map +1 -1
- package/dist/types/subsystems/IO/VectorDB.service/connectors/PineconeVectorDB.class.d.ts +2 -2
- package/dist/types/subsystems/IO/VectorDB.service/embed/BaseEmbedding.d.ts +11 -7
- package/dist/types/types/VectorDB.types.d.ts +13 -11
- package/package.json +102 -127
- package/src/Components/APICall/APICall.class.ts +155 -0
- package/src/Components/APICall/AccessTokenManager.ts +130 -0
- package/src/Components/APICall/ArrayBufferResponse.helper.ts +58 -0
- package/src/Components/APICall/OAuth.helper.ts +294 -0
- package/src/Components/APICall/mimeTypeCategories.ts +46 -0
- package/src/Components/APICall/parseData.ts +167 -0
- package/src/Components/APICall/parseHeaders.ts +41 -0
- package/src/Components/APICall/parseProxy.ts +68 -0
- package/src/Components/APICall/parseUrl.ts +91 -0
- package/src/Components/APIEndpoint.class.ts +234 -0
- package/src/Components/APIOutput.class.ts +58 -0
- package/src/Components/AgentPlugin.class.ts +102 -0
- package/src/Components/Async.class.ts +155 -0
- package/src/Components/Await.class.ts +90 -0
- package/src/Components/Classifier.class.ts +158 -0
- package/src/Components/Component.class.ts +94 -0
- package/src/Components/ComponentHost.class.ts +38 -0
- package/src/Components/DataSourceCleaner.class.ts +92 -0
- package/src/Components/DataSourceIndexer.class.ts +181 -0
- package/src/Components/DataSourceLookup.class.ts +141 -0
- package/src/Components/FEncDec.class.ts +29 -0
- package/src/Components/FHash.class.ts +33 -0
- package/src/Components/FSign.class.ts +80 -0
- package/src/Components/FSleep.class.ts +25 -0
- package/src/Components/FTimestamp.class.ts +25 -0
- package/src/Components/FileStore.class.ts +75 -0
- package/src/Components/ForEach.class.ts +97 -0
- package/src/Components/GPTPlugin.class.ts +70 -0
- package/src/Components/GenAILLM.class.ts +395 -0
- package/src/Components/HuggingFace.class.ts +314 -0
- package/src/Components/Image/imageSettings.config.ts +70 -0
- package/src/Components/ImageGenerator.class.ts +407 -0
- package/src/Components/JSONFilter.class.ts +54 -0
- package/src/Components/LLMAssistant.class.ts +213 -0
- package/src/Components/LogicAND.class.ts +28 -0
- package/src/Components/LogicAtLeast.class.ts +85 -0
- package/src/Components/LogicAtMost.class.ts +86 -0
- package/src/Components/LogicOR.class.ts +29 -0
- package/src/Components/LogicXOR.class.ts +34 -0
- package/src/Components/MCPClient.class.ts +112 -0
- package/src/Components/PromptGenerator.class.ts +122 -0
- package/src/Components/ScrapflyWebScrape.class.ts +159 -0
- package/src/Components/TavilyWebSearch.class.ts +98 -0
- package/src/Components/index.ts +77 -0
- package/src/Core/AgentProcess.helper.ts +240 -0
- package/src/Core/Connector.class.ts +123 -0
- package/src/Core/ConnectorsService.ts +192 -0
- package/src/Core/DummyConnector.ts +49 -0
- package/src/Core/HookService.ts +105 -0
- package/src/Core/SmythRuntime.class.ts +292 -0
- package/src/Core/SystemEvents.ts +15 -0
- package/src/Core/boot.ts +55 -0
- package/src/config.ts +15 -0
- package/src/constants.ts +125 -0
- package/src/data/hugging-face.params.json +580 -0
- package/src/helpers/BinaryInput.helper.ts +324 -0
- package/src/helpers/Conversation.helper.ts +1094 -0
- package/src/helpers/JsonContent.helper.ts +97 -0
- package/src/helpers/LocalCache.helper.ts +97 -0
- package/src/helpers/Log.helper.ts +234 -0
- package/src/helpers/OpenApiParser.helper.ts +150 -0
- package/src/helpers/S3Cache.helper.ts +129 -0
- package/src/helpers/SmythURI.helper.ts +5 -0
- package/src/helpers/TemplateString.helper.ts +243 -0
- package/src/helpers/TypeChecker.helper.ts +329 -0
- package/src/index.ts +179 -0
- package/src/index.ts.bak +179 -0
- package/src/subsystems/AgentManager/Agent.class.ts +1108 -0
- package/src/subsystems/AgentManager/Agent.helper.ts +3 -0
- package/src/subsystems/AgentManager/AgentData.service/AgentDataConnector.ts +230 -0
- package/src/subsystems/AgentManager/AgentData.service/connectors/CLIAgentDataConnector.class.ts +66 -0
- package/src/subsystems/AgentManager/AgentData.service/connectors/LocalAgentDataConnector.class.ts +142 -0
- package/src/subsystems/AgentManager/AgentData.service/connectors/NullAgentData.class.ts +39 -0
- package/src/subsystems/AgentManager/AgentData.service/index.ts +18 -0
- package/src/subsystems/AgentManager/AgentLogger.class.ts +297 -0
- package/src/subsystems/AgentManager/AgentRequest.class.ts +51 -0
- package/src/subsystems/AgentManager/AgentRuntime.class.ts +559 -0
- package/src/subsystems/AgentManager/AgentSSE.class.ts +101 -0
- package/src/subsystems/AgentManager/AgentSettings.class.ts +52 -0
- package/src/subsystems/AgentManager/Component.service/ComponentConnector.ts +32 -0
- package/src/subsystems/AgentManager/Component.service/connectors/LocalComponentConnector.class.ts +59 -0
- package/src/subsystems/AgentManager/Component.service/index.ts +11 -0
- package/src/subsystems/AgentManager/EmbodimentSettings.class.ts +47 -0
- package/src/subsystems/AgentManager/ForkedAgent.class.ts +153 -0
- package/src/subsystems/AgentManager/OSResourceMonitor.ts +77 -0
- package/src/subsystems/ComputeManager/Code.service/CodeConnector.ts +99 -0
- package/src/subsystems/ComputeManager/Code.service/connectors/AWSLambdaCode.class.ts +63 -0
- package/src/subsystems/ComputeManager/Code.service/index.ts +11 -0
- package/src/subsystems/IO/CLI.service/CLIConnector.ts +47 -0
- package/src/subsystems/IO/CLI.service/index.ts +9 -0
- package/src/subsystems/IO/Log.service/LogConnector.ts +32 -0
- package/src/subsystems/IO/Log.service/connectors/ConsoleLog.class.ts +28 -0
- package/src/subsystems/IO/Log.service/index.ts +13 -0
- package/src/subsystems/IO/NKV.service/NKVConnector.ts +41 -0
- package/src/subsystems/IO/NKV.service/connectors/NKVRAM.class.ts +204 -0
- package/src/subsystems/IO/NKV.service/connectors/NKVRedis.class.ts +182 -0
- package/src/subsystems/IO/NKV.service/index.ts +12 -0
- package/src/subsystems/IO/Router.service/RouterConnector.ts +21 -0
- package/src/subsystems/IO/Router.service/connectors/ExpressRouter.class.ts +48 -0
- package/src/subsystems/IO/Router.service/connectors/NullRouter.class.ts +40 -0
- package/src/subsystems/IO/Router.service/index.ts +11 -0
- package/src/subsystems/IO/Storage.service/SmythFS.class.ts +472 -0
- package/src/subsystems/IO/Storage.service/StorageConnector.ts +66 -0
- package/src/subsystems/IO/Storage.service/connectors/LocalStorage.class.ts +305 -0
- package/src/subsystems/IO/Storage.service/connectors/S3Storage.class.ts +418 -0
- package/src/subsystems/IO/Storage.service/index.ts +13 -0
- package/src/subsystems/IO/VectorDB.service/VectorDBConnector.ts +108 -0
- package/src/subsystems/IO/VectorDB.service/connectors/MilvusVectorDB.class.ts +454 -0
- package/src/subsystems/IO/VectorDB.service/connectors/PineconeVectorDB.class.ts +384 -0
- package/src/subsystems/IO/VectorDB.service/connectors/RAMVecrtorDB.class.ts +421 -0
- package/src/subsystems/IO/VectorDB.service/embed/BaseEmbedding.ts +107 -0
- package/src/subsystems/IO/VectorDB.service/embed/OpenAIEmbedding.ts +109 -0
- package/src/subsystems/IO/VectorDB.service/embed/index.ts +21 -0
- package/src/subsystems/IO/VectorDB.service/index.ts +14 -0
- package/src/subsystems/LLMManager/LLM.helper.ts +221 -0
- package/src/subsystems/LLMManager/LLM.inference.ts +335 -0
- package/src/subsystems/LLMManager/LLM.service/LLMConnector.ts +375 -0
- package/src/subsystems/LLMManager/LLM.service/LLMCredentials.helper.ts +145 -0
- package/src/subsystems/LLMManager/LLM.service/connectors/Anthropic.class.ts +632 -0
- package/src/subsystems/LLMManager/LLM.service/connectors/Bedrock.class.ts +405 -0
- package/src/subsystems/LLMManager/LLM.service/connectors/Echo.class.ts +81 -0
- package/src/subsystems/LLMManager/LLM.service/connectors/GoogleAI.class.ts +689 -0
- package/src/subsystems/LLMManager/LLM.service/connectors/Groq.class.ts +257 -0
- package/src/subsystems/LLMManager/LLM.service/connectors/OpenAI.class.ts +848 -0
- package/src/subsystems/LLMManager/LLM.service/connectors/Perplexity.class.ts +255 -0
- package/src/subsystems/LLMManager/LLM.service/connectors/VertexAI.class.ts +193 -0
- package/src/subsystems/LLMManager/LLM.service/index.ts +43 -0
- package/src/subsystems/LLMManager/ModelsProvider.service/ModelsProviderConnector.ts +281 -0
- package/src/subsystems/LLMManager/ModelsProvider.service/connectors/SmythModelsProvider.class.ts +229 -0
- package/src/subsystems/LLMManager/ModelsProvider.service/index.ts +11 -0
- package/src/subsystems/LLMManager/custom-models.ts +854 -0
- package/src/subsystems/LLMManager/models.ts +2539 -0
- package/src/subsystems/LLMManager/paramMappings.ts +69 -0
- package/src/subsystems/MemoryManager/Cache.service/CacheConnector.ts +86 -0
- package/src/subsystems/MemoryManager/Cache.service/connectors/LocalStorageCache.class.ts +297 -0
- package/src/subsystems/MemoryManager/Cache.service/connectors/RAMCache.class.ts +201 -0
- package/src/subsystems/MemoryManager/Cache.service/connectors/RedisCache.class.ts +252 -0
- package/src/subsystems/MemoryManager/Cache.service/connectors/S3Cache.class.ts +373 -0
- package/src/subsystems/MemoryManager/Cache.service/index.ts +15 -0
- package/src/subsystems/MemoryManager/LLMCache.ts +72 -0
- package/src/subsystems/MemoryManager/LLMContext.ts +125 -0
- package/src/subsystems/MemoryManager/RuntimeContext.ts +249 -0
- package/src/subsystems/Security/AccessControl/ACL.class.ts +208 -0
- package/src/subsystems/Security/AccessControl/AccessCandidate.class.ts +76 -0
- package/src/subsystems/Security/AccessControl/AccessRequest.class.ts +52 -0
- package/src/subsystems/Security/Account.service/AccountConnector.ts +41 -0
- package/src/subsystems/Security/Account.service/connectors/AWSAccount.class.ts +76 -0
- package/src/subsystems/Security/Account.service/connectors/DummyAccount.class.ts +130 -0
- package/src/subsystems/Security/Account.service/connectors/JSONFileAccount.class.ts +159 -0
- package/src/subsystems/Security/Account.service/index.ts +14 -0
- package/src/subsystems/Security/Credentials.helper.ts +62 -0
- package/src/subsystems/Security/ManagedVault.service/ManagedVaultConnector.ts +34 -0
- package/src/subsystems/Security/ManagedVault.service/connectors/NullManagedVault.class.ts +57 -0
- package/src/subsystems/Security/ManagedVault.service/connectors/SecretManagerManagedVault.ts +154 -0
- package/src/subsystems/Security/ManagedVault.service/index.ts +12 -0
- package/src/subsystems/Security/SecureConnector.class.ts +110 -0
- package/src/subsystems/Security/Vault.service/Vault.helper.ts +30 -0
- package/src/subsystems/Security/Vault.service/VaultConnector.ts +26 -0
- package/src/subsystems/Security/Vault.service/connectors/HashicorpVault.class.ts +46 -0
- package/src/subsystems/Security/Vault.service/connectors/JSONFileVault.class.ts +166 -0
- package/src/subsystems/Security/Vault.service/connectors/NullVault.class.ts +54 -0
- package/src/subsystems/Security/Vault.service/connectors/SecretsManager.class.ts +140 -0
- package/src/subsystems/Security/Vault.service/index.ts +12 -0
- package/src/types/ACL.types.ts +104 -0
- package/src/types/AWS.types.ts +9 -0
- package/src/types/Agent.types.ts +61 -0
- package/src/types/AgentLogger.types.ts +17 -0
- package/src/types/Cache.types.ts +1 -0
- package/src/types/Common.types.ts +3 -0
- package/src/types/LLM.types.ts +419 -0
- package/src/types/Redis.types.ts +8 -0
- package/src/types/SRE.types.ts +64 -0
- package/src/types/Security.types.ts +18 -0
- package/src/types/Storage.types.ts +5 -0
- package/src/types/VectorDB.types.ts +86 -0
- package/src/utils/base64.utils.ts +275 -0
- package/src/utils/cli.utils.ts +68 -0
- package/src/utils/data.utils.ts +263 -0
- package/src/utils/date-time.utils.ts +22 -0
- package/src/utils/general.utils.ts +238 -0
- package/src/utils/index.ts +12 -0
- package/src/utils/numbers.utils.ts +13 -0
- package/src/utils/oauth.utils.ts +35 -0
- package/src/utils/string.utils.ts +414 -0
- package/src/utils/url.utils.ts +19 -0
- package/src/utils/validation.utils.ts +74 -0
|
@@ -2,7 +2,7 @@ import { ACL } from '@sre/Security/AccessControl/ACL.class';
|
|
|
2
2
|
import { IAccessCandidate } from '@sre/types/ACL.types';
|
|
3
3
|
import { AccessRequest } from '@sre/Security/AccessControl/AccessRequest.class';
|
|
4
4
|
import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
|
|
5
|
-
import { VectorDBConnector } from '../VectorDBConnector';
|
|
5
|
+
import { VectorDBConnector, DeleteTarget } from '../VectorDBConnector';
|
|
6
6
|
import { DatasourceDto, IStorageVectorDataSource, IVectorDataSourceDto, QueryOptions, VectorsResultData } from '@sre/types/VectorDB.types';
|
|
7
7
|
import { BaseEmbedding, TEmbeddings } from '../embed/BaseEmbedding';
|
|
8
8
|
export type PineconeConfig = {
|
|
@@ -38,7 +38,7 @@ export declare class PineconeVectorDB extends VectorDBConnector {
|
|
|
38
38
|
protected deleteNamespace(acRequest: AccessRequest, namespace: string): Promise<void>;
|
|
39
39
|
protected search(acRequest: AccessRequest, namespace: string, query: string | number[], options?: QueryOptions): Promise<VectorsResultData>;
|
|
40
40
|
protected insert(acRequest: AccessRequest, namespace: string, sourceWrapper: IVectorDataSourceDto | IVectorDataSourceDto[]): Promise<string[]>;
|
|
41
|
-
protected delete(acRequest: AccessRequest, namespace: string,
|
|
41
|
+
protected delete(acRequest: AccessRequest, namespace: string, deleteTarget: DeleteTarget): Promise<void>;
|
|
42
42
|
protected createDatasource(acRequest: AccessRequest, namespace: string, datasource: DatasourceDto): Promise<IStorageVectorDataSource>;
|
|
43
43
|
protected deleteDatasource(acRequest: AccessRequest, namespace: string, datasourceId: string): Promise<void>;
|
|
44
44
|
protected listDatasources(acRequest: AccessRequest, namespace: string): Promise<IStorageVectorDataSource[]>;
|
|
@@ -4,14 +4,15 @@ import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.cla
|
|
|
4
4
|
export type TEmbeddings = {
|
|
5
5
|
provider: SupportedProviders;
|
|
6
6
|
model: SupportedModels[SupportedProviders];
|
|
7
|
-
dimensions?: number;
|
|
8
|
-
timeout?: number;
|
|
9
|
-
chunkSize?: number;
|
|
10
|
-
stripNewLines?: boolean;
|
|
11
|
-
maxConcurrency?: number;
|
|
12
7
|
credentials?: {
|
|
13
8
|
apiKey: string;
|
|
14
9
|
};
|
|
10
|
+
params?: {
|
|
11
|
+
dimensions?: number;
|
|
12
|
+
timeout?: number;
|
|
13
|
+
chunkSize?: number;
|
|
14
|
+
stripNewLines?: boolean;
|
|
15
|
+
};
|
|
15
16
|
};
|
|
16
17
|
type SupportedSources = 'text' | 'vector' | 'url';
|
|
17
18
|
export declare abstract class BaseEmbedding {
|
|
@@ -21,7 +22,6 @@ export declare abstract class BaseEmbedding {
|
|
|
21
22
|
stripNewLines: boolean;
|
|
22
23
|
dimensions?: number;
|
|
23
24
|
timeout?: number;
|
|
24
|
-
maxConcurrency?: number;
|
|
25
25
|
constructor(fields?: Partial<TEmbeddings>);
|
|
26
26
|
/**
|
|
27
27
|
* Embed multiple texts and return their vector representations
|
|
@@ -44,7 +44,11 @@ export declare abstract class BaseEmbedding {
|
|
|
44
44
|
source: number[];
|
|
45
45
|
metadata: {
|
|
46
46
|
text: string;
|
|
47
|
-
|
|
47
|
+
namespaceId: string;
|
|
48
|
+
datasourceId: string;
|
|
49
|
+
datasourceLabel: string;
|
|
50
|
+
acl: string;
|
|
51
|
+
user_metadata?: string;
|
|
48
52
|
};
|
|
49
53
|
id: string;
|
|
50
54
|
}[]>;
|
|
@@ -1,15 +1,17 @@
|
|
|
1
|
-
export type VectorDBMetadata =
|
|
1
|
+
export type VectorDBMetadata = {
|
|
2
|
+
namespaceId: string;
|
|
3
|
+
datasourceId: string;
|
|
4
|
+
datasourceLabel: string;
|
|
5
|
+
acl: string;
|
|
6
|
+
user_metadata?: string;
|
|
2
7
|
text?: string;
|
|
3
|
-
|
|
4
|
-
} & Record<string, string>) | undefined;
|
|
8
|
+
};
|
|
5
9
|
export type VectorsResultData = {
|
|
6
10
|
id: string;
|
|
7
11
|
score?: number;
|
|
8
12
|
values: number[];
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
user?: Record<string, string>;
|
|
12
|
-
} & Record<string, any>) | undefined;
|
|
13
|
+
text: string;
|
|
14
|
+
metadata?: Record<string, any>;
|
|
13
15
|
}[];
|
|
14
16
|
export interface NsKnownMetadata {
|
|
15
17
|
isOnCustomStorage?: boolean;
|
|
@@ -31,20 +33,20 @@ export interface IVectorDataSourceDto {
|
|
|
31
33
|
}
|
|
32
34
|
export interface IStorageVectorDataSource {
|
|
33
35
|
namespaceId: string;
|
|
34
|
-
candidateId: string;
|
|
35
|
-
candidateRole: string;
|
|
36
36
|
name: string;
|
|
37
37
|
metadata: string;
|
|
38
|
-
text
|
|
38
|
+
text?: string;
|
|
39
39
|
vectorIds: string[];
|
|
40
40
|
id: string;
|
|
41
|
+
candidateId: string;
|
|
42
|
+
candidateRole: string;
|
|
41
43
|
}
|
|
42
44
|
export interface IStorageVectorNamespace {
|
|
43
45
|
namespace: string;
|
|
44
46
|
displayName: string;
|
|
47
|
+
metadata?: StorageVectorNamespaceMetadata;
|
|
45
48
|
candidateId: string;
|
|
46
49
|
candidateRole: string;
|
|
47
|
-
metadata?: StorageVectorNamespaceMetadata;
|
|
48
50
|
}
|
|
49
51
|
export type StorageVectorNamespaceMetadata = Partial<PineconeNamespaceMetadata> & {
|
|
50
52
|
isOnCustomStorage?: boolean;
|
package/package.json
CHANGED
|
@@ -1,128 +1,103 @@
|
|
|
1
1
|
{
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
"
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
"
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
"oauth-1.0a": "^2.2.6",
|
|
105
|
-
"openai": "^4.103.0",
|
|
106
|
-
"p-limit": "^6.1.0",
|
|
107
|
-
"pkce-challenge": "^5.0.0",
|
|
108
|
-
"qs": "^6.13.0",
|
|
109
|
-
"rate-limiter-flexible": "^5.0.3",
|
|
110
|
-
"readline-sync": "^1.4.10",
|
|
111
|
-
"socket.io-client": "^4.8.1",
|
|
112
|
-
"socks-proxy-agent": "^8.0.4",
|
|
113
|
-
"uuid": "^10.0.0",
|
|
114
|
-
"winston": "^3.13.0",
|
|
115
|
-
"winston-transport": "^4.7.0",
|
|
116
|
-
"xxhashjs": "^0.2.2",
|
|
117
|
-
"zip-lib": "^1.0.5"
|
|
118
|
-
},
|
|
119
|
-
"repository": {
|
|
120
|
-
"type": "git",
|
|
121
|
-
"url": "https://github.com/smythos/sre.git",
|
|
122
|
-
"directory": "packages/core"
|
|
123
|
-
},
|
|
124
|
-
"bugs": {
|
|
125
|
-
"url": "https://github.com/smythos/sre/issues"
|
|
126
|
-
},
|
|
127
|
-
"homepage": "https://github.com/smythos/sre/tree/main/packages/core#readme"
|
|
128
|
-
}
|
|
2
|
+
"name": "@smythos/sre",
|
|
3
|
+
"version": "1.5.4",
|
|
4
|
+
"description": "Smyth Runtime Environment",
|
|
5
|
+
"author": "Alaa-eddine KADDOURI",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"main": "dist/index.js",
|
|
8
|
+
"types": "dist/types/index.d.ts",
|
|
9
|
+
"exports": {
|
|
10
|
+
".": {
|
|
11
|
+
"import": "./dist/index.js",
|
|
12
|
+
"types": "./dist/types/index.d.ts"
|
|
13
|
+
}
|
|
14
|
+
},
|
|
15
|
+
"files": [
|
|
16
|
+
"dist",
|
|
17
|
+
"src",
|
|
18
|
+
"README.md",
|
|
19
|
+
"CHANGELOG"
|
|
20
|
+
],
|
|
21
|
+
"type": "module",
|
|
22
|
+
"devDependencies": {
|
|
23
|
+
"@istanbuljs/nyc-config-typescript": "^1.0.2",
|
|
24
|
+
"@rollup/plugin-json": "^6.1.0",
|
|
25
|
+
"@rollup/pluginutils": "^5.1.0",
|
|
26
|
+
"@types/express": "^4.17.23",
|
|
27
|
+
"@types/lodash": "^4.17.10",
|
|
28
|
+
"@types/node": "^20.19.0",
|
|
29
|
+
"cross-env": "^7.0.3",
|
|
30
|
+
"ctix": "^2.7.1",
|
|
31
|
+
"dependency-cruiser": "^16.3.3",
|
|
32
|
+
"esbuild": "^0.25.0",
|
|
33
|
+
"knip": "^5.23.1",
|
|
34
|
+
"nyc": "^17.0.0",
|
|
35
|
+
"rollup-plugin-esbuild": "^6.1.1",
|
|
36
|
+
"rollup-plugin-sourcemaps": "^0.6.3",
|
|
37
|
+
"rollup-plugin-terser": "^7.0.2",
|
|
38
|
+
"rollup-plugin-typescript-paths": "^1.5.0",
|
|
39
|
+
"typedoc": "^0.28.5",
|
|
40
|
+
"typescript": "^5.4.5"
|
|
41
|
+
},
|
|
42
|
+
"dependencies": {
|
|
43
|
+
"@anthropic-ai/sdk": "^0.39.0",
|
|
44
|
+
"@aws-sdk/client-bedrock-runtime": "^3.826.0",
|
|
45
|
+
"@aws-sdk/client-s3": "^3.826.0",
|
|
46
|
+
"@aws-sdk/client-secrets-manager": "^3.826.0",
|
|
47
|
+
"@google-cloud/vertexai": "^1.7.0",
|
|
48
|
+
"@google/generative-ai": "^0.14.1",
|
|
49
|
+
"@huggingface/inference": "^2.8.0",
|
|
50
|
+
"@modelcontextprotocol/sdk": "^1.10.1",
|
|
51
|
+
"@pinecone-database/pinecone": "^3.0.0",
|
|
52
|
+
"@runware/sdk-js": "^1.1.36",
|
|
53
|
+
"@smithy/smithy-client": "^4.4.3",
|
|
54
|
+
"@zilliz/milvus2-sdk-node": "^2.5.11",
|
|
55
|
+
"acorn": "^8.14.1",
|
|
56
|
+
"axios": "^1.7.2",
|
|
57
|
+
"chokidar": "^4.0.3",
|
|
58
|
+
"dayjs": "^1.11.11",
|
|
59
|
+
"dotenv": "^16.4.5",
|
|
60
|
+
"eventsource": "^3.0.2",
|
|
61
|
+
"express": "^4.21.2",
|
|
62
|
+
"file-type": "^19.0.0",
|
|
63
|
+
"form-data": "^4.0.3",
|
|
64
|
+
"gpt-tokenizer": "^2.2.1",
|
|
65
|
+
"groq-sdk": "^0.6.1",
|
|
66
|
+
"image-size": "^1.1.1",
|
|
67
|
+
"ioredis": "^5.4.1",
|
|
68
|
+
"isbinaryfile": "^5.0.2",
|
|
69
|
+
"joi": "^17.13.1",
|
|
70
|
+
"js-yaml": "^4.1.0",
|
|
71
|
+
"jsonrepair": "^3.8.0",
|
|
72
|
+
"lodash": "^4.17.21",
|
|
73
|
+
"mime": "^4.0.3",
|
|
74
|
+
"mysql2": "^3.11.3",
|
|
75
|
+
"oauth-1.0a": "^2.2.6",
|
|
76
|
+
"openai": "^4.103.0",
|
|
77
|
+
"p-limit": "^6.1.0",
|
|
78
|
+
"qs": "^6.13.0",
|
|
79
|
+
"readline-sync": "^1.4.10",
|
|
80
|
+
"socks-proxy-agent": "^8.0.4",
|
|
81
|
+
"winston": "^3.13.0",
|
|
82
|
+
"winston-transport": "^4.7.0",
|
|
83
|
+
"xxhashjs": "^0.2.2"
|
|
84
|
+
},
|
|
85
|
+
"repository": {
|
|
86
|
+
"type": "git",
|
|
87
|
+
"url": "https://github.com/smythos/sre.git",
|
|
88
|
+
"directory": "packages/core"
|
|
89
|
+
},
|
|
90
|
+
"bugs": {
|
|
91
|
+
"url": "https://github.com/smythos/sre/issues"
|
|
92
|
+
},
|
|
93
|
+
"homepage": "https://github.com/smythos/sre/tree/main/packages/core#readme",
|
|
94
|
+
"scripts": {
|
|
95
|
+
"gen:barrel": "ctix build",
|
|
96
|
+
"build:types": "tsc --emitDeclarationOnly --declaration --outDir dist/types -p tsconfig.dts.json",
|
|
97
|
+
"build:jsbundle": "cross-env rollup -c",
|
|
98
|
+
"build": "pnpm run build:jsbundle && pnpm run build:types",
|
|
99
|
+
"doc:gen": "typedoc",
|
|
100
|
+
"doc:graphgen": "npx depcruise src --config ./doc/.dep-minimal.json --output-type dot > ./doc/dep-graph.dot && dot -Tpng ./doc/dep-graph.dot -o ./doc/dep-graph.png",
|
|
101
|
+
"knip": "knip"
|
|
102
|
+
}
|
|
103
|
+
}
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import axios, { AxiosRequestConfig } from 'axios';
|
|
2
|
+
import Joi from 'joi';
|
|
3
|
+
|
|
4
|
+
import { Agent } from '@sre/AgentManager/Agent.class';
|
|
5
|
+
import { Component } from '../Component.class';
|
|
6
|
+
import { parseHeaders } from './parseHeaders';
|
|
7
|
+
import { parseUrl, parseSmythFsUrl, destroyPublicUrls } from './parseUrl';
|
|
8
|
+
import { parseData } from './parseData';
|
|
9
|
+
import { parseProxy } from './parseProxy';
|
|
10
|
+
import { parseArrayBufferResponse } from './ArrayBufferResponse.helper';
|
|
11
|
+
import { extractAdditionalParamsForOAuth1, handleOAuthHeaders as generateOAuthHeaders } from './OAuth.helper';
|
|
12
|
+
import { SocksProxyAgent } from 'socks-proxy-agent';
|
|
13
|
+
import { formatDataForDebug } from '@sre/utils/data.utils';
|
|
14
|
+
|
|
15
|
+
export class APICall extends Component {
|
|
16
|
+
protected schema = {
|
|
17
|
+
name: 'APICall',
|
|
18
|
+
description: 'Use this component to make an API call',
|
|
19
|
+
inputs: {},
|
|
20
|
+
outputs: {
|
|
21
|
+
Headers: {
|
|
22
|
+
description: 'The headers of the API call response',
|
|
23
|
+
default: true,
|
|
24
|
+
},
|
|
25
|
+
Response: {
|
|
26
|
+
description: 'The response of the API call',
|
|
27
|
+
default: true,
|
|
28
|
+
},
|
|
29
|
+
},
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
protected configSchema = Joi.object({
|
|
33
|
+
method: Joi.string().valid('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS').required().label('Method'),
|
|
34
|
+
url: Joi.string()
|
|
35
|
+
.max(8192) /*.custom(isUrlValid, 'URL validation')*/
|
|
36
|
+
.required()
|
|
37
|
+
.label('URL'),
|
|
38
|
+
headers: Joi.any().allow('').label('Headers'),
|
|
39
|
+
contentType: Joi.string()
|
|
40
|
+
.valid('none', 'application/json', 'multipart/form-data', 'binary', 'application/x-www-form-urlencoded', 'text/plain', 'application/xml')
|
|
41
|
+
.label('Content-Type'),
|
|
42
|
+
body: Joi.any().allow('').label('Body'),
|
|
43
|
+
_templateSettings: Joi.object().allow(null).label('Template Settings'),
|
|
44
|
+
_templateVars: Joi.object().allow(null).label('Template Variables'),
|
|
45
|
+
proxy: Joi.string().allow('').label('Proxy'),
|
|
46
|
+
oauthService: Joi.string().allow('').label('OAuth Service'),
|
|
47
|
+
scope: Joi.string().allow('').label('Scope'),
|
|
48
|
+
authorizationURL: Joi.string().allow('').label('Authorization URL'),
|
|
49
|
+
tokenURL: Joi.string().allow('').label('Token URL'),
|
|
50
|
+
clientID: Joi.string().allow('').label('Client ID'),
|
|
51
|
+
clientSecret: Joi.string().allow('').label('Client Secret'),
|
|
52
|
+
oauth2CallbackURL: Joi.string().allow('').label('OAuth2 Callback URL'),
|
|
53
|
+
callbackURL: Joi.string().allow('').label('Callback URL'), // !TEMP: prevent validation error
|
|
54
|
+
requestTokenURL: Joi.string().allow('').label('Request Token URL'),
|
|
55
|
+
accessTokenURL: Joi.string().allow('').label('Access Token URL'),
|
|
56
|
+
userAuthorizationURL: Joi.string().allow('').label('User Authorization URL'),
|
|
57
|
+
consumerKey: Joi.string().allow('').label('Consumer Key'),
|
|
58
|
+
consumerSecret: Joi.string().allow('').label('Consumer Secret'),
|
|
59
|
+
oauth1CallbackURL: Joi.string().allow('').label('OAuth1 Callback URL'),
|
|
60
|
+
authenticate: Joi.string().allow('').label('Authenticate'),
|
|
61
|
+
});
|
|
62
|
+
constructor() {
|
|
63
|
+
super();
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
init() {}
|
|
67
|
+
|
|
68
|
+
async process(input, config, agent: Agent) {
|
|
69
|
+
await super.process(input, config, agent);
|
|
70
|
+
|
|
71
|
+
const logger = this.createComponentLogger(agent, config);
|
|
72
|
+
|
|
73
|
+
let publicUrls: string[] = [];
|
|
74
|
+
|
|
75
|
+
try {
|
|
76
|
+
logger.debug(`=== API Call Log ===`);
|
|
77
|
+
|
|
78
|
+
const method = config?.data?.method || 'get';
|
|
79
|
+
|
|
80
|
+
const reqConfig: AxiosRequestConfig = {};
|
|
81
|
+
reqConfig.method = method;
|
|
82
|
+
|
|
83
|
+
reqConfig.url = await parseUrl(input, config, agent);
|
|
84
|
+
|
|
85
|
+
// We generate public URLs for any resources specified with the smythfs protocol in the request URL.
|
|
86
|
+
({ url: reqConfig.url, publicUrls } = await parseSmythFsUrl(reqConfig.url, agent));
|
|
87
|
+
|
|
88
|
+
const { data, headers } = await parseData(input, config, agent);
|
|
89
|
+
|
|
90
|
+
// If the data is null, the request may fail. We encountered an issue where a request failed due to null data being provided.
|
|
91
|
+
let dataForDebug;
|
|
92
|
+
if (data) {
|
|
93
|
+
reqConfig.data = data;
|
|
94
|
+
|
|
95
|
+
dataForDebug = await formatDataForDebug(data);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
reqConfig.headers = (await parseHeaders(input, config, agent)).concat({ ...headers });
|
|
99
|
+
|
|
100
|
+
const proxyConfig = await parseProxy(input, config, agent);
|
|
101
|
+
|
|
102
|
+
if (proxyConfig) {
|
|
103
|
+
if (proxyConfig instanceof SocksProxyAgent) {
|
|
104
|
+
const isSecureEndpoint = reqConfig.url?.startsWith('https://');
|
|
105
|
+
reqConfig[isSecureEndpoint ? 'httpsAgent' : 'httpAgent'] = proxyConfig;
|
|
106
|
+
} else {
|
|
107
|
+
reqConfig.proxy = proxyConfig;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
let Response: any = {};
|
|
112
|
+
let Headers: any = {};
|
|
113
|
+
let _error: any = undefined;
|
|
114
|
+
try {
|
|
115
|
+
if (config?.data?.oauthService && config?.data?.oauthService !== 'None') {
|
|
116
|
+
const rootUrl = new URL(reqConfig.url).origin;
|
|
117
|
+
const additionalParams = extractAdditionalParamsForOAuth1(reqConfig);
|
|
118
|
+
const oauthHeaders = await generateOAuthHeaders(agent, config, reqConfig, logger, additionalParams, rootUrl);
|
|
119
|
+
//reqConfig.headers = { ...reqConfig.headers, ...oauthHeaders };
|
|
120
|
+
reqConfig.headers = reqConfig.headers.concat({ ...oauthHeaders });
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// in order to handle binary data automatically, we need to set responseType to 'arraybuffer' for all requests, then parse the response data based on content-type
|
|
124
|
+
reqConfig.responseType = 'arraybuffer';
|
|
125
|
+
|
|
126
|
+
logger.debug('Making API call', { ...reqConfig, data: dataForDebug });
|
|
127
|
+
|
|
128
|
+
const response = await axios.request(reqConfig);
|
|
129
|
+
|
|
130
|
+
const parsedRes = await parseArrayBufferResponse(response, agent);
|
|
131
|
+
|
|
132
|
+
// log response headers
|
|
133
|
+
logger.debug('API call Response Headers', response.headers);
|
|
134
|
+
Response = parsedRes;
|
|
135
|
+
|
|
136
|
+
logger.debug('API call Response\n', Response);
|
|
137
|
+
|
|
138
|
+
Headers = Object.fromEntries(Object.entries(response.headers));
|
|
139
|
+
} catch (error) {
|
|
140
|
+
logger.debug(`Error making API call: ${error.message}`);
|
|
141
|
+
Headers = error?.response?.headers ? Object.fromEntries(Object.entries(error.response.headers)) : {};
|
|
142
|
+
Response = await parseArrayBufferResponse(error.response, agent);
|
|
143
|
+
_error = error.message;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
return { Response, Headers, _error, _debug: logger.output };
|
|
147
|
+
} catch (error) {
|
|
148
|
+
return { _error: error.message, _debug: logger.output };
|
|
149
|
+
} finally {
|
|
150
|
+
if (publicUrls.length > 0) {
|
|
151
|
+
await destroyPublicUrls(publicUrls);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
// accessTokenManager.ts
|
|
2
|
+
import { Agent } from '@sre/AgentManager/Agent.class';
|
|
3
|
+
import { ConnectorService } from '@sre/Core/ConnectorsService';
|
|
4
|
+
import { SystemEvents } from '@sre/Core/SystemEvents';
|
|
5
|
+
import { Logger } from '@sre/helpers/Log.helper';
|
|
6
|
+
import { AccessCandidate } from '@sre/Security/AccessControl/AccessCandidate.class';
|
|
7
|
+
import axios from 'axios';
|
|
8
|
+
|
|
9
|
+
const console = Logger('AccessTokenManager');
|
|
10
|
+
let managedVault: any;
|
|
11
|
+
|
|
12
|
+
SystemEvents.on('SRE:Booted', () => {
|
|
13
|
+
managedVault = ConnectorService.getManagedVaultConnector();
|
|
14
|
+
});
|
|
15
|
+
class AccessTokenManager {
|
|
16
|
+
private clientId: string;
|
|
17
|
+
private clientSecret: string;
|
|
18
|
+
private primaryToken: string; // accessToken || token
|
|
19
|
+
private secondaryToken: string; // refreshToken || tokenSecret
|
|
20
|
+
private tokenUrl: string; // tokenURL to refresh accessToken
|
|
21
|
+
private expires_in: any;
|
|
22
|
+
private data: any; // value of key(keyId) in teamSettings that needs to be updated if required
|
|
23
|
+
private keyId: any; // key of object in teamSettings
|
|
24
|
+
private logger: any; // Use to log console in debugger
|
|
25
|
+
private agent: Agent;
|
|
26
|
+
constructor(
|
|
27
|
+
clientId: string,
|
|
28
|
+
clientSecret: string,
|
|
29
|
+
secondaryToken: string,
|
|
30
|
+
tokenUrl: string,
|
|
31
|
+
expires_in: any,
|
|
32
|
+
primaryToken: string,
|
|
33
|
+
data: any,
|
|
34
|
+
keyId: any,
|
|
35
|
+
logger: any,
|
|
36
|
+
agent: Agent,
|
|
37
|
+
) {
|
|
38
|
+
this.clientId = clientId;
|
|
39
|
+
this.clientSecret = clientSecret;
|
|
40
|
+
this.primaryToken = primaryToken;
|
|
41
|
+
this.secondaryToken = secondaryToken;
|
|
42
|
+
this.tokenUrl = tokenUrl;
|
|
43
|
+
this.expires_in = expires_in;
|
|
44
|
+
this.data = data;
|
|
45
|
+
this.keyId = keyId;
|
|
46
|
+
this.logger = logger;
|
|
47
|
+
this.agent = agent;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
async getAccessToken(): Promise<string> {
|
|
51
|
+
try {
|
|
52
|
+
const currentTime: any = new Date().getTime();
|
|
53
|
+
|
|
54
|
+
// If there's no secondaryToken (refresh token) and no expires_in,
|
|
55
|
+
// assume it's a long-lived token and return the primaryToken directly
|
|
56
|
+
if (!this.secondaryToken && !this.expires_in) {
|
|
57
|
+
console.log('Using long-lived access token');
|
|
58
|
+
this.logger.debug('Using long-lived access token. If authentication failes, please re-authenticate and try again');
|
|
59
|
+
return this.primaryToken;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// Regular token expiration check for tokens with expiration
|
|
63
|
+
// should be alway currentTime >= Number(this.expires_in)
|
|
64
|
+
if (!this.expires_in || currentTime >= Number(this.expires_in)) {
|
|
65
|
+
if (!this.secondaryToken) {
|
|
66
|
+
this.logger.debug('Refresh token is missing. Please re authenticate');
|
|
67
|
+
console.log('Refresh token is missing. Please re authenticate...');
|
|
68
|
+
// Redirect the user to the OAuth authorization URL or initiate the reauthentication flow
|
|
69
|
+
throw new Error('Reauthentication required');
|
|
70
|
+
}
|
|
71
|
+
this.logger.debug('Access token is expired or missing. Refreshing access token...');
|
|
72
|
+
console.log('Access token is expired or missing. Refreshing access token...');
|
|
73
|
+
return await this.refreshAccessToken();
|
|
74
|
+
} else {
|
|
75
|
+
console.log('Access token is still valid');
|
|
76
|
+
this.logger.debug('Access token is still valid.');
|
|
77
|
+
return this.primaryToken;
|
|
78
|
+
}
|
|
79
|
+
} catch (error) {
|
|
80
|
+
console.error('Error fetching access token:', error);
|
|
81
|
+
this.logger.debug('Error fetching access token');
|
|
82
|
+
throw error;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
async refreshAccessToken(): Promise<string> {
|
|
87
|
+
try {
|
|
88
|
+
const response = await axios.post(
|
|
89
|
+
this.tokenUrl,
|
|
90
|
+
new URLSearchParams({
|
|
91
|
+
client_id: this.clientId,
|
|
92
|
+
client_secret: this.clientSecret,
|
|
93
|
+
refresh_token: this.secondaryToken,
|
|
94
|
+
grant_type: 'refresh_token',
|
|
95
|
+
}).toString(),
|
|
96
|
+
{
|
|
97
|
+
headers: {
|
|
98
|
+
'Content-Type': 'application/x-www-form-urlencoded',
|
|
99
|
+
},
|
|
100
|
+
},
|
|
101
|
+
);
|
|
102
|
+
|
|
103
|
+
const newAccessToken: string = response?.data?.access_token;
|
|
104
|
+
console.log('Access token refreshed successfully.');
|
|
105
|
+
this.logger.debug('Access token refreshed successfully.');
|
|
106
|
+
const expiresInMilliseconds: number = response?.data?.expires_in ? response?.data?.expires_in * 1000 : response?.data?.expires_in;
|
|
107
|
+
const expirationTimestamp: number = expiresInMilliseconds ? new Date().getTime() + expiresInMilliseconds : expiresInMilliseconds;
|
|
108
|
+
this.data.primary = newAccessToken;
|
|
109
|
+
this.data.expires_in = expirationTimestamp ? expirationTimestamp?.toString() : expirationTimestamp;
|
|
110
|
+
//const oauthTeamSettings = new OauthTeamSettings();
|
|
111
|
+
//const save: any = await oauthTeamSettings.update({ keyId: this.keyId, data: this.data });
|
|
112
|
+
|
|
113
|
+
const save: any = await managedVault.user(AccessCandidate.agent(this.agent.id)).set(this.keyId, JSON.stringify(this.data));
|
|
114
|
+
if (save && save.status === 200) {
|
|
115
|
+
console.log('Access token value is updated successfully.');
|
|
116
|
+
this.logger.debug('Access token value is updated successfully.');
|
|
117
|
+
} else {
|
|
118
|
+
console.log('Warning: new access token value is not updated.');
|
|
119
|
+
this.logger.debug('Warning: new access token value is not updated.');
|
|
120
|
+
}
|
|
121
|
+
return newAccessToken;
|
|
122
|
+
} catch (error) {
|
|
123
|
+
console.error('Failed to refresh access token:', error);
|
|
124
|
+
this.logger.debug(`Failed to refresh access token: ${error}`);
|
|
125
|
+
throw new Error('Failed to refresh access token.');
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
export default AccessTokenManager;
|