@smythos/sre 1.5.46 → 1.5.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG +98 -90
- package/LICENSE +18 -18
- package/README.md +135 -135
- package/dist/bundle-analysis-lazy.html +4949 -0
- package/dist/bundle-analysis.html +4949 -0
- package/dist/index.js +3 -3
- package/dist/index.js.map +1 -1
- package/dist/types/Components/MCPClient.class.d.ts +1 -0
- package/dist/types/subsystems/LLMManager/LLM.service/connectors/openai/OpenAIConnector.class.d.ts +1 -6
- package/dist/types/utils/package-manager.utils.d.ts +26 -0
- package/package.json +1 -1
- package/src/Components/APICall/APICall.class.ts +156 -156
- package/src/Components/APICall/AccessTokenManager.ts +130 -130
- package/src/Components/APICall/ArrayBufferResponse.helper.ts +58 -58
- package/src/Components/APICall/OAuth.helper.ts +294 -294
- package/src/Components/APICall/mimeTypeCategories.ts +46 -46
- package/src/Components/APICall/parseData.ts +167 -167
- package/src/Components/APICall/parseHeaders.ts +41 -41
- package/src/Components/APICall/parseProxy.ts +68 -68
- package/src/Components/APICall/parseUrl.ts +91 -91
- package/src/Components/APIEndpoint.class.ts +234 -234
- package/src/Components/APIOutput.class.ts +58 -58
- package/src/Components/AgentPlugin.class.ts +102 -102
- package/src/Components/Async.class.ts +155 -155
- package/src/Components/Await.class.ts +90 -90
- package/src/Components/Classifier.class.ts +158 -158
- package/src/Components/Component.class.ts +132 -132
- package/src/Components/ComponentHost.class.ts +38 -38
- package/src/Components/DataSourceCleaner.class.ts +92 -92
- package/src/Components/DataSourceIndexer.class.ts +181 -181
- package/src/Components/DataSourceLookup.class.ts +161 -161
- package/src/Components/ECMASandbox.class.ts +71 -71
- package/src/Components/FEncDec.class.ts +29 -29
- package/src/Components/FHash.class.ts +33 -33
- package/src/Components/FSign.class.ts +80 -80
- package/src/Components/FSleep.class.ts +25 -25
- package/src/Components/FTimestamp.class.ts +25 -25
- package/src/Components/FileStore.class.ts +78 -78
- package/src/Components/ForEach.class.ts +97 -97
- package/src/Components/GPTPlugin.class.ts +70 -70
- package/src/Components/GenAILLM.class.ts +586 -586
- package/src/Components/HuggingFace.class.ts +314 -314
- package/src/Components/Image/imageSettings.config.ts +70 -70
- package/src/Components/ImageGenerator.class.ts +502 -502
- package/src/Components/JSONFilter.class.ts +54 -54
- package/src/Components/LLMAssistant.class.ts +213 -213
- package/src/Components/LogicAND.class.ts +28 -28
- package/src/Components/LogicAtLeast.class.ts +85 -85
- package/src/Components/LogicAtMost.class.ts +86 -86
- package/src/Components/LogicOR.class.ts +29 -29
- package/src/Components/LogicXOR.class.ts +34 -34
- package/src/Components/MCPClient.class.ts +138 -112
- package/src/Components/MemoryDeleteKeyVal.class.ts +70 -70
- package/src/Components/MemoryReadKeyVal.class.ts +66 -66
- package/src/Components/MemoryWriteKeyVal.class.ts +62 -62
- package/src/Components/MemoryWriteObject.class.ts +97 -97
- package/src/Components/MultimodalLLM.class.ts +128 -128
- package/src/Components/OpenAPI.class.ts +72 -72
- package/src/Components/PromptGenerator.class.ts +122 -122
- package/src/Components/ScrapflyWebScrape.class.ts +159 -159
- package/src/Components/ServerlessCode.class.ts +123 -123
- package/src/Components/TavilyWebSearch.class.ts +98 -98
- package/src/Components/VisionLLM.class.ts +104 -104
- package/src/Components/ZapierAction.class.ts +127 -127
- package/src/Components/index.ts +97 -97
- package/src/Core/AgentProcess.helper.ts +240 -240
- package/src/Core/Connector.class.ts +123 -123
- package/src/Core/ConnectorsService.ts +197 -197
- package/src/Core/DummyConnector.ts +49 -49
- package/src/Core/HookService.ts +105 -105
- package/src/Core/SmythRuntime.class.ts +235 -235
- package/src/Core/SystemEvents.ts +16 -16
- package/src/Core/boot.ts +56 -56
- package/src/config.ts +15 -15
- package/src/constants.ts +126 -126
- package/src/data/hugging-face.params.json +579 -579
- package/src/helpers/AWSLambdaCode.helper.ts +587 -587
- package/src/helpers/BinaryInput.helper.ts +331 -331
- package/src/helpers/Conversation.helper.ts +1119 -1119
- package/src/helpers/ECMASandbox.helper.ts +54 -54
- package/src/helpers/JsonContent.helper.ts +97 -97
- package/src/helpers/LocalCache.helper.ts +97 -97
- package/src/helpers/Log.helper.ts +274 -274
- package/src/helpers/OpenApiParser.helper.ts +150 -150
- package/src/helpers/S3Cache.helper.ts +147 -147
- package/src/helpers/SmythURI.helper.ts +5 -5
- package/src/helpers/Sysconfig.helper.ts +77 -77
- package/src/helpers/TemplateString.helper.ts +243 -243
- package/src/helpers/TypeChecker.helper.ts +329 -329
- package/src/index.ts +3 -3
- package/src/index.ts.bak +3 -3
- package/src/subsystems/AgentManager/Agent.class.ts +1114 -1114
- package/src/subsystems/AgentManager/Agent.helper.ts +3 -3
- package/src/subsystems/AgentManager/AgentData.service/AgentDataConnector.ts +230 -230
- package/src/subsystems/AgentManager/AgentData.service/connectors/CLIAgentDataConnector.class.ts +66 -66
- package/src/subsystems/AgentManager/AgentData.service/connectors/LocalAgentDataConnector.class.ts +142 -142
- package/src/subsystems/AgentManager/AgentData.service/connectors/NullAgentData.class.ts +39 -39
- package/src/subsystems/AgentManager/AgentData.service/index.ts +18 -18
- package/src/subsystems/AgentManager/AgentLogger.class.ts +297 -297
- package/src/subsystems/AgentManager/AgentRequest.class.ts +51 -51
- package/src/subsystems/AgentManager/AgentRuntime.class.ts +559 -559
- package/src/subsystems/AgentManager/AgentSSE.class.ts +101 -101
- package/src/subsystems/AgentManager/AgentSettings.class.ts +52 -52
- package/src/subsystems/AgentManager/Component.service/ComponentConnector.ts +32 -32
- package/src/subsystems/AgentManager/Component.service/connectors/LocalComponentConnector.class.ts +60 -60
- package/src/subsystems/AgentManager/Component.service/index.ts +11 -11
- package/src/subsystems/AgentManager/EmbodimentSettings.class.ts +47 -47
- package/src/subsystems/AgentManager/ForkedAgent.class.ts +154 -154
- package/src/subsystems/AgentManager/OSResourceMonitor.ts +77 -77
- package/src/subsystems/ComputeManager/Code.service/CodeConnector.ts +98 -98
- package/src/subsystems/ComputeManager/Code.service/connectors/AWSLambdaCode.class.ts +172 -172
- package/src/subsystems/ComputeManager/Code.service/connectors/ECMASandbox.class.ts +131 -131
- package/src/subsystems/ComputeManager/Code.service/index.ts +13 -13
- package/src/subsystems/IO/CLI.service/CLIConnector.ts +47 -47
- package/src/subsystems/IO/CLI.service/index.ts +9 -9
- package/src/subsystems/IO/Log.service/LogConnector.ts +32 -32
- package/src/subsystems/IO/Log.service/connectors/ConsoleLog.class.ts +28 -28
- package/src/subsystems/IO/Log.service/index.ts +13 -13
- package/src/subsystems/IO/NKV.service/NKVConnector.ts +43 -43
- package/src/subsystems/IO/NKV.service/connectors/NKVLocalStorage.class.ts +234 -234
- package/src/subsystems/IO/NKV.service/connectors/NKVRAM.class.ts +204 -204
- package/src/subsystems/IO/NKV.service/connectors/NKVRedis.class.ts +182 -182
- package/src/subsystems/IO/NKV.service/index.ts +14 -14
- package/src/subsystems/IO/Router.service/RouterConnector.ts +21 -21
- package/src/subsystems/IO/Router.service/connectors/ExpressRouter.class.ts +48 -48
- package/src/subsystems/IO/Router.service/connectors/NullRouter.class.ts +40 -40
- package/src/subsystems/IO/Router.service/index.ts +11 -11
- package/src/subsystems/IO/Storage.service/SmythFS.class.ts +489 -489
- package/src/subsystems/IO/Storage.service/StorageConnector.ts +66 -66
- package/src/subsystems/IO/Storage.service/connectors/LocalStorage.class.ts +327 -327
- package/src/subsystems/IO/Storage.service/connectors/S3Storage.class.ts +482 -482
- package/src/subsystems/IO/Storage.service/index.ts +13 -13
- package/src/subsystems/IO/VectorDB.service/VectorDBConnector.ts +108 -108
- package/src/subsystems/IO/VectorDB.service/connectors/MilvusVectorDB.class.ts +454 -454
- package/src/subsystems/IO/VectorDB.service/connectors/PineconeVectorDB.class.ts +384 -384
- package/src/subsystems/IO/VectorDB.service/connectors/RAMVecrtorDB.class.ts +421 -421
- package/src/subsystems/IO/VectorDB.service/embed/BaseEmbedding.ts +107 -107
- package/src/subsystems/IO/VectorDB.service/embed/OpenAIEmbedding.ts +109 -109
- package/src/subsystems/IO/VectorDB.service/embed/index.ts +21 -21
- package/src/subsystems/IO/VectorDB.service/index.ts +14 -14
- package/src/subsystems/LLMManager/LLM.helper.ts +251 -251
- package/src/subsystems/LLMManager/LLM.inference.ts +339 -339
- package/src/subsystems/LLMManager/LLM.service/LLMConnector.ts +489 -489
- package/src/subsystems/LLMManager/LLM.service/LLMCredentials.helper.ts +171 -171
- package/src/subsystems/LLMManager/LLM.service/connectors/Anthropic.class.ts +659 -659
- package/src/subsystems/LLMManager/LLM.service/connectors/Bedrock.class.ts +400 -400
- package/src/subsystems/LLMManager/LLM.service/connectors/Echo.class.ts +77 -77
- package/src/subsystems/LLMManager/LLM.service/connectors/GoogleAI.class.ts +757 -757
- package/src/subsystems/LLMManager/LLM.service/connectors/Groq.class.ts +304 -304
- package/src/subsystems/LLMManager/LLM.service/connectors/Perplexity.class.ts +250 -250
- package/src/subsystems/LLMManager/LLM.service/connectors/VertexAI.class.ts +423 -423
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/OpenAIConnector.class.ts +488 -488
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/ChatCompletionsApiInterface.ts +524 -524
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/OpenAIApiInterface.ts +100 -100
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/OpenAIApiInterfaceFactory.ts +81 -81
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/ResponsesApiInterface.ts +1145 -1145
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/constants.ts +13 -13
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/index.ts +4 -4
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/utils.ts +11 -11
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/types.ts +32 -32
- package/src/subsystems/LLMManager/LLM.service/connectors/xAI.class.ts +471 -471
- package/src/subsystems/LLMManager/LLM.service/index.ts +44 -44
- package/src/subsystems/LLMManager/ModelsProvider.service/ModelsProviderConnector.ts +300 -300
- package/src/subsystems/LLMManager/ModelsProvider.service/connectors/JSONModelsProvider.class.ts +252 -252
- package/src/subsystems/LLMManager/ModelsProvider.service/index.ts +11 -11
- package/src/subsystems/LLMManager/custom-models.ts +854 -854
- package/src/subsystems/LLMManager/models.ts +2540 -2540
- package/src/subsystems/LLMManager/paramMappings.ts +69 -69
- package/src/subsystems/MemoryManager/Cache.service/CacheConnector.ts +86 -86
- package/src/subsystems/MemoryManager/Cache.service/connectors/LocalStorageCache.class.ts +297 -297
- package/src/subsystems/MemoryManager/Cache.service/connectors/RAMCache.class.ts +201 -201
- package/src/subsystems/MemoryManager/Cache.service/connectors/RedisCache.class.ts +252 -252
- package/src/subsystems/MemoryManager/Cache.service/connectors/S3Cache.class.ts +373 -373
- package/src/subsystems/MemoryManager/Cache.service/index.ts +15 -15
- package/src/subsystems/MemoryManager/LLMCache.ts +72 -72
- package/src/subsystems/MemoryManager/LLMContext.ts +124 -124
- package/src/subsystems/MemoryManager/LLMMemory.service/LLMMemoryConnector.ts +26 -26
- package/src/subsystems/MemoryManager/RuntimeContext.ts +266 -266
- package/src/subsystems/Security/AccessControl/ACL.class.ts +208 -208
- package/src/subsystems/Security/AccessControl/AccessCandidate.class.ts +82 -82
- package/src/subsystems/Security/AccessControl/AccessRequest.class.ts +52 -52
- package/src/subsystems/Security/Account.service/AccountConnector.ts +44 -44
- package/src/subsystems/Security/Account.service/connectors/AWSAccount.class.ts +76 -76
- package/src/subsystems/Security/Account.service/connectors/DummyAccount.class.ts +130 -130
- package/src/subsystems/Security/Account.service/connectors/JSONFileAccount.class.ts +159 -159
- package/src/subsystems/Security/Account.service/index.ts +14 -14
- package/src/subsystems/Security/Credentials.helper.ts +62 -62
- package/src/subsystems/Security/ManagedVault.service/ManagedVaultConnector.ts +38 -38
- package/src/subsystems/Security/ManagedVault.service/connectors/NullManagedVault.class.ts +53 -53
- package/src/subsystems/Security/ManagedVault.service/connectors/SecretManagerManagedVault.ts +154 -154
- package/src/subsystems/Security/ManagedVault.service/index.ts +12 -12
- package/src/subsystems/Security/SecureConnector.class.ts +110 -110
- package/src/subsystems/Security/Vault.service/Vault.helper.ts +30 -30
- package/src/subsystems/Security/Vault.service/VaultConnector.ts +29 -29
- package/src/subsystems/Security/Vault.service/connectors/HashicorpVault.class.ts +46 -46
- package/src/subsystems/Security/Vault.service/connectors/JSONFileVault.class.ts +221 -221
- package/src/subsystems/Security/Vault.service/connectors/NullVault.class.ts +54 -54
- package/src/subsystems/Security/Vault.service/connectors/SecretsManager.class.ts +140 -140
- package/src/subsystems/Security/Vault.service/index.ts +12 -12
- package/src/types/ACL.types.ts +104 -104
- package/src/types/AWS.types.ts +10 -10
- package/src/types/Agent.types.ts +61 -61
- package/src/types/AgentLogger.types.ts +17 -17
- package/src/types/Cache.types.ts +1 -1
- package/src/types/Common.types.ts +2 -2
- package/src/types/LLM.types.ts +496 -496
- package/src/types/Redis.types.ts +8 -8
- package/src/types/SRE.types.ts +64 -64
- package/src/types/Security.types.ts +14 -14
- package/src/types/Storage.types.ts +5 -5
- package/src/types/VectorDB.types.ts +86 -86
- package/src/utils/base64.utils.ts +275 -275
- package/src/utils/cli.utils.ts +68 -68
- package/src/utils/data.utils.ts +322 -322
- package/src/utils/date-time.utils.ts +22 -22
- package/src/utils/general.utils.ts +238 -238
- package/src/utils/index.ts +12 -12
- package/src/utils/lazy-client.ts +261 -261
- package/src/utils/numbers.utils.ts +13 -13
- package/src/utils/oauth.utils.ts +35 -35
- package/src/utils/string.utils.ts +414 -414
- package/src/utils/url.utils.ts +19 -19
- package/src/utils/validation.utils.ts +74 -74
- package/dist/types/subsystems/LLMManager/ModelsProvider.service/connectors/SmythModelsProvider.class.d.ts +0 -39
|
@@ -1,252 +1,252 @@
|
|
|
1
|
-
import { Logger } from '@sre/helpers/Log.helper';
|
|
2
|
-
import { IAccessCandidate, IACL, TAccessLevel } from '@sre/types/ACL.types';
|
|
3
|
-
import { CacheMetadata } from '@sre/types/Cache.types';
|
|
4
|
-
import IORedis from 'ioredis';
|
|
5
|
-
import { CacheConnector } from '../CacheConnector';
|
|
6
|
-
|
|
7
|
-
import { ACL } from '@sre/Security/AccessControl/ACL.class';
|
|
8
|
-
import { RedisConfig } from '@sre/types/Redis.types';
|
|
9
|
-
|
|
10
|
-
import { AccessRequest } from '@sre/Security/AccessControl/AccessRequest.class';
|
|
11
|
-
import { SecureConnector } from '@sre/Security/SecureConnector.class';
|
|
12
|
-
import { AccountConnector } from '@sre/Security/Account.service/AccountConnector';
|
|
13
|
-
import { ConnectorService } from '@sre/Core/ConnectorsService';
|
|
14
|
-
|
|
15
|
-
const console = Logger('RedisCache');
|
|
16
|
-
|
|
17
|
-
export class RedisCache extends CacheConnector {
|
|
18
|
-
public name: string = 'RedisCache';
|
|
19
|
-
private redis: IORedis;
|
|
20
|
-
private _prefix: string = 'smyth:cache';
|
|
21
|
-
private _mdPrefix: string = 'smyth:metadata';
|
|
22
|
-
|
|
23
|
-
constructor(protected _settings: RedisConfig) {
|
|
24
|
-
super(_settings);
|
|
25
|
-
const sentinels = parseSentinelHosts(_settings.hosts || process.env.REDIS_HOSTS);
|
|
26
|
-
let host = sentinels.length === 1 ? sentinels[0].host : null;
|
|
27
|
-
let port = sentinels.length === 1 ? sentinels[0].port : null;
|
|
28
|
-
|
|
29
|
-
this.redis = new IORedis({
|
|
30
|
-
...(host ? { host, port } : { sentinels, name: _settings.name || process.env.REDIS_MASTER_NAME }),
|
|
31
|
-
password: _settings.password || process.env.REDIS_PASSWORD,
|
|
32
|
-
});
|
|
33
|
-
|
|
34
|
-
this.redis.on('error', (error) => {
|
|
35
|
-
console.error('Redis Error:', error);
|
|
36
|
-
});
|
|
37
|
-
|
|
38
|
-
this.redis.on('connect', () => {
|
|
39
|
-
console.log('Redis connected!');
|
|
40
|
-
});
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
public get client() {
|
|
44
|
-
return this.redis;
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
public get prefix() {
|
|
48
|
-
return this._prefix;
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
public get mdPrefix() {
|
|
52
|
-
return this._mdPrefix;
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
@SecureConnector.AccessControl
|
|
56
|
-
public async get(acRequest: AccessRequest, key: string): Promise<string | null> {
|
|
57
|
-
const value = await this.redis.get(`${this._prefix}:${key}`);
|
|
58
|
-
return value;
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
@SecureConnector.AccessControl
|
|
62
|
-
public async set(acRequest: AccessRequest, key: string, data: any, acl?: IACL, metadata?: CacheMetadata, ttl?: number): Promise<boolean> {
|
|
63
|
-
const accessCandidate = acRequest.candidate;
|
|
64
|
-
const promises: any[] = [];
|
|
65
|
-
|
|
66
|
-
const newMetadata: CacheMetadata = metadata || {};
|
|
67
|
-
newMetadata.acl = ACL.from(acl).addAccess(accessCandidate.role, accessCandidate.id, TAccessLevel.Owner).ACL;
|
|
68
|
-
|
|
69
|
-
if (ttl && ttl > 0) {
|
|
70
|
-
promises.push(this.redis.set(`${this._prefix}:${key}`, data, 'EX', ttl));
|
|
71
|
-
promises.push(this.setMetadataWithTTL(acRequest, key, newMetadata, ttl));
|
|
72
|
-
} else {
|
|
73
|
-
promises.push(this.redis.set(`${this._prefix}:${key}`, data));
|
|
74
|
-
promises.push(this.setMetadata(acRequest, key, newMetadata));
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
await Promise.all(promises);
|
|
78
|
-
|
|
79
|
-
// if (ttl) {
|
|
80
|
-
// try {
|
|
81
|
-
// await this.updateTTL(acRequest, key, ttl);
|
|
82
|
-
// } catch (error) {
|
|
83
|
-
// console.error(`Error setting TTL for key ${key}`, error);
|
|
84
|
-
// }
|
|
85
|
-
// }
|
|
86
|
-
|
|
87
|
-
return true;
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
@SecureConnector.AccessControl
|
|
91
|
-
public async delete(acRequest: AccessRequest, key: string): Promise<void> {
|
|
92
|
-
//delete both the key and its metadata
|
|
93
|
-
await Promise.all([this.redis.del(`${this._prefix}:${key}`), this.redis.del(`${this._mdPrefix}:${key}`)]);
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
@SecureConnector.AccessControl
|
|
97
|
-
public async exists(acRequest: AccessRequest, key: string): Promise<boolean> {
|
|
98
|
-
return !!(await this.redis.exists(`${this._prefix}:${key}`));
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
@SecureConnector.AccessControl
|
|
102
|
-
public async getMetadata(acRequest: AccessRequest, key: string): Promise<CacheMetadata> {
|
|
103
|
-
if (!this.exists(acRequest, key)) return undefined;
|
|
104
|
-
try {
|
|
105
|
-
const metadata = await this.redis.get(`${this._mdPrefix}:${key}`);
|
|
106
|
-
return metadata ? (this.deserializeRedisMetadata(metadata) as CacheMetadata) : {};
|
|
107
|
-
} catch (error) {
|
|
108
|
-
return {};
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
@SecureConnector.AccessControl
|
|
113
|
-
public async setMetadata(acRequest: AccessRequest, key: string, metadata: CacheMetadata): Promise<void> {
|
|
114
|
-
await this.setMetadataWithTTL(acRequest, key, metadata);
|
|
115
|
-
}
|
|
116
|
-
private async setMetadataWithTTL(acRequest: AccessRequest, key: string, metadata: CacheMetadata, ttl?: number): Promise<void> {
|
|
117
|
-
if (ttl && ttl > 0) {
|
|
118
|
-
await this.redis.set(`${this._mdPrefix}:${key}`, this.serializeRedisMetadata(metadata), 'EX', ttl);
|
|
119
|
-
} else {
|
|
120
|
-
await this.redis.set(`${this._mdPrefix}:${key}`, this.serializeRedisMetadata(metadata));
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
@SecureConnector.AccessControl
|
|
125
|
-
public async updateTTL(acRequest: AccessRequest, key: string, ttl?: number): Promise<void> {
|
|
126
|
-
if (ttl) {
|
|
127
|
-
await Promise.all([this.redis.expire(`${this._prefix}:${key}`, ttl), this.redis.expire(`${this._mdPrefix}:${key}`, ttl)]);
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
@SecureConnector.AccessControl
|
|
132
|
-
public async getTTL(acRequest: AccessRequest, key: string): Promise<number> {
|
|
133
|
-
return this.redis.ttl(`${this._prefix}:${key}`);
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
public async getResourceACL(resourceId: string, candidate: IAccessCandidate): Promise<ACL> {
|
|
137
|
-
const _metadata: any = await this.redis.get(`${this._mdPrefix}:${resourceId}`).catch((error) => {});
|
|
138
|
-
const exists = _metadata !== undefined && _metadata !== null; //null or undefined metadata means the resource does not exist
|
|
139
|
-
const metadata = exists ? this.deserializeRedisMetadata(_metadata) : {};
|
|
140
|
-
|
|
141
|
-
if (!exists) {
|
|
142
|
-
//the resource does not exist yet, we grant write access to the candidate in order to allow the resource creation
|
|
143
|
-
return new ACL().addAccess(candidate.role, candidate.id, TAccessLevel.Owner);
|
|
144
|
-
}
|
|
145
|
-
return ACL.from(metadata?.acl as IACL);
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
// async hasAccess(request: IAccessRequest): Promise<boolean> {
|
|
149
|
-
// try {
|
|
150
|
-
// const metadata = await this.getMetadata(request.resourceId);
|
|
151
|
-
// const acl: IACL = metadata?.acl as IACL;
|
|
152
|
-
// return ACL.from(acl).checkExactAccess(request);
|
|
153
|
-
// } catch (error) {
|
|
154
|
-
// if (error.name === 'NotFound') {
|
|
155
|
-
// return false;
|
|
156
|
-
// }
|
|
157
|
-
// console.error(`Error checking access rights in S3`, error.name, error.message);
|
|
158
|
-
// throw error;
|
|
159
|
-
// }
|
|
160
|
-
// }
|
|
161
|
-
|
|
162
|
-
@SecureConnector.AccessControl
|
|
163
|
-
async getACL(acRequest: AccessRequest, key: string): Promise<IACL> {
|
|
164
|
-
try {
|
|
165
|
-
const metadata = await this.getMetadata(acRequest, key);
|
|
166
|
-
return (metadata?.acl as IACL) || {};
|
|
167
|
-
} catch (error) {
|
|
168
|
-
console.error(`Error getting access rights in S3`, error.name, error.message);
|
|
169
|
-
throw error;
|
|
170
|
-
}
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
@SecureConnector.AccessControl
|
|
174
|
-
async setACL(acRequest: AccessRequest, key: string, acl: IACL) {
|
|
175
|
-
try {
|
|
176
|
-
let metadata = await this.getMetadata(acRequest, key);
|
|
177
|
-
if (!metadata) metadata = {};
|
|
178
|
-
//when setting ACL make sure to not lose ownership
|
|
179
|
-
metadata.acl = ACL.from(acl).addAccess(acRequest.candidate.role, acRequest.candidate.id, TAccessLevel.Owner).ACL;
|
|
180
|
-
await this.setMetadata(acRequest, key, metadata);
|
|
181
|
-
} catch (error) {
|
|
182
|
-
console.error(`Error setting access rights in S3`, error);
|
|
183
|
-
throw error;
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
private serializeRedisMetadata(redisMetadata: Record<string, any> | undefined): string {
|
|
188
|
-
if (!redisMetadata) return '';
|
|
189
|
-
if (redisMetadata.acl) {
|
|
190
|
-
const acl: IACL = redisMetadata.acl;
|
|
191
|
-
if (acl) {
|
|
192
|
-
redisMetadata.acl = ACL.from(acl).serializedACL;
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
return JSON.stringify(redisMetadata);
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
private deserializeRedisMetadata(strMetadata: string): Record<string, any> {
|
|
200
|
-
try {
|
|
201
|
-
const redisMetadata = JSON.parse(strMetadata);
|
|
202
|
-
if (redisMetadata.acl) {
|
|
203
|
-
const acl: IACL = ACL.from(redisMetadata.acl).ACL;
|
|
204
|
-
redisMetadata.acl = acl;
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
return redisMetadata;
|
|
208
|
-
} catch (error) {
|
|
209
|
-
console.warn(`Error deserializing metadata`, strMetadata);
|
|
210
|
-
return {};
|
|
211
|
-
}
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
public async stop() {
|
|
215
|
-
super.stop();
|
|
216
|
-
await this.redis.quit();
|
|
217
|
-
}
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
/**
|
|
221
|
-
* hosts can take any of the following formats:
|
|
222
|
-
* 1. A string with comma-separated host:port pairs
|
|
223
|
-
* 2. An array of strings with host:port pairs
|
|
224
|
-
* 3. An array of objects with host and port properties
|
|
225
|
-
* @param hosts
|
|
226
|
-
*/
|
|
227
|
-
function parseSentinelHosts(hosts: string | string[] | any[]) {
|
|
228
|
-
//handle all possible formats of hosts
|
|
229
|
-
if (typeof hosts === 'string') {
|
|
230
|
-
return hosts.split(',').map((host) => {
|
|
231
|
-
const [hostName, port] = host.split(':');
|
|
232
|
-
return {
|
|
233
|
-
host: hostName,
|
|
234
|
-
port: Number(port),
|
|
235
|
-
};
|
|
236
|
-
});
|
|
237
|
-
} else if (Array.isArray(hosts)) {
|
|
238
|
-
return hosts.map((host) => {
|
|
239
|
-
if (typeof host === 'string') {
|
|
240
|
-
const [hostName, port] = host.split(':');
|
|
241
|
-
return {
|
|
242
|
-
host: hostName,
|
|
243
|
-
port: Number(port),
|
|
244
|
-
};
|
|
245
|
-
} else {
|
|
246
|
-
return host;
|
|
247
|
-
}
|
|
248
|
-
});
|
|
249
|
-
} else {
|
|
250
|
-
return [];
|
|
251
|
-
}
|
|
252
|
-
}
|
|
1
|
+
import { Logger } from '@sre/helpers/Log.helper';
|
|
2
|
+
import { IAccessCandidate, IACL, TAccessLevel } from '@sre/types/ACL.types';
|
|
3
|
+
import { CacheMetadata } from '@sre/types/Cache.types';
|
|
4
|
+
import IORedis from 'ioredis';
|
|
5
|
+
import { CacheConnector } from '../CacheConnector';
|
|
6
|
+
|
|
7
|
+
import { ACL } from '@sre/Security/AccessControl/ACL.class';
|
|
8
|
+
import { RedisConfig } from '@sre/types/Redis.types';
|
|
9
|
+
|
|
10
|
+
import { AccessRequest } from '@sre/Security/AccessControl/AccessRequest.class';
|
|
11
|
+
import { SecureConnector } from '@sre/Security/SecureConnector.class';
|
|
12
|
+
import { AccountConnector } from '@sre/Security/Account.service/AccountConnector';
|
|
13
|
+
import { ConnectorService } from '@sre/Core/ConnectorsService';
|
|
14
|
+
|
|
15
|
+
const console = Logger('RedisCache');
|
|
16
|
+
|
|
17
|
+
export class RedisCache extends CacheConnector {
|
|
18
|
+
public name: string = 'RedisCache';
|
|
19
|
+
private redis: IORedis;
|
|
20
|
+
private _prefix: string = 'smyth:cache';
|
|
21
|
+
private _mdPrefix: string = 'smyth:metadata';
|
|
22
|
+
|
|
23
|
+
constructor(protected _settings: RedisConfig) {
|
|
24
|
+
super(_settings);
|
|
25
|
+
const sentinels = parseSentinelHosts(_settings.hosts || process.env.REDIS_HOSTS);
|
|
26
|
+
let host = sentinels.length === 1 ? sentinels[0].host : null;
|
|
27
|
+
let port = sentinels.length === 1 ? sentinels[0].port : null;
|
|
28
|
+
|
|
29
|
+
this.redis = new IORedis({
|
|
30
|
+
...(host ? { host, port } : { sentinels, name: _settings.name || process.env.REDIS_MASTER_NAME }),
|
|
31
|
+
password: _settings.password || process.env.REDIS_PASSWORD,
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
this.redis.on('error', (error) => {
|
|
35
|
+
console.error('Redis Error:', error);
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
this.redis.on('connect', () => {
|
|
39
|
+
console.log('Redis connected!');
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
public get client() {
|
|
44
|
+
return this.redis;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
public get prefix() {
|
|
48
|
+
return this._prefix;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
public get mdPrefix() {
|
|
52
|
+
return this._mdPrefix;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
@SecureConnector.AccessControl
|
|
56
|
+
public async get(acRequest: AccessRequest, key: string): Promise<string | null> {
|
|
57
|
+
const value = await this.redis.get(`${this._prefix}:${key}`);
|
|
58
|
+
return value;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
@SecureConnector.AccessControl
|
|
62
|
+
public async set(acRequest: AccessRequest, key: string, data: any, acl?: IACL, metadata?: CacheMetadata, ttl?: number): Promise<boolean> {
|
|
63
|
+
const accessCandidate = acRequest.candidate;
|
|
64
|
+
const promises: any[] = [];
|
|
65
|
+
|
|
66
|
+
const newMetadata: CacheMetadata = metadata || {};
|
|
67
|
+
newMetadata.acl = ACL.from(acl).addAccess(accessCandidate.role, accessCandidate.id, TAccessLevel.Owner).ACL;
|
|
68
|
+
|
|
69
|
+
if (ttl && ttl > 0) {
|
|
70
|
+
promises.push(this.redis.set(`${this._prefix}:${key}`, data, 'EX', ttl));
|
|
71
|
+
promises.push(this.setMetadataWithTTL(acRequest, key, newMetadata, ttl));
|
|
72
|
+
} else {
|
|
73
|
+
promises.push(this.redis.set(`${this._prefix}:${key}`, data));
|
|
74
|
+
promises.push(this.setMetadata(acRequest, key, newMetadata));
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
await Promise.all(promises);
|
|
78
|
+
|
|
79
|
+
// if (ttl) {
|
|
80
|
+
// try {
|
|
81
|
+
// await this.updateTTL(acRequest, key, ttl);
|
|
82
|
+
// } catch (error) {
|
|
83
|
+
// console.error(`Error setting TTL for key ${key}`, error);
|
|
84
|
+
// }
|
|
85
|
+
// }
|
|
86
|
+
|
|
87
|
+
return true;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
@SecureConnector.AccessControl
|
|
91
|
+
public async delete(acRequest: AccessRequest, key: string): Promise<void> {
|
|
92
|
+
//delete both the key and its metadata
|
|
93
|
+
await Promise.all([this.redis.del(`${this._prefix}:${key}`), this.redis.del(`${this._mdPrefix}:${key}`)]);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
@SecureConnector.AccessControl
|
|
97
|
+
public async exists(acRequest: AccessRequest, key: string): Promise<boolean> {
|
|
98
|
+
return !!(await this.redis.exists(`${this._prefix}:${key}`));
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
@SecureConnector.AccessControl
|
|
102
|
+
public async getMetadata(acRequest: AccessRequest, key: string): Promise<CacheMetadata> {
|
|
103
|
+
if (!this.exists(acRequest, key)) return undefined;
|
|
104
|
+
try {
|
|
105
|
+
const metadata = await this.redis.get(`${this._mdPrefix}:${key}`);
|
|
106
|
+
return metadata ? (this.deserializeRedisMetadata(metadata) as CacheMetadata) : {};
|
|
107
|
+
} catch (error) {
|
|
108
|
+
return {};
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
@SecureConnector.AccessControl
|
|
113
|
+
public async setMetadata(acRequest: AccessRequest, key: string, metadata: CacheMetadata): Promise<void> {
|
|
114
|
+
await this.setMetadataWithTTL(acRequest, key, metadata);
|
|
115
|
+
}
|
|
116
|
+
private async setMetadataWithTTL(acRequest: AccessRequest, key: string, metadata: CacheMetadata, ttl?: number): Promise<void> {
|
|
117
|
+
if (ttl && ttl > 0) {
|
|
118
|
+
await this.redis.set(`${this._mdPrefix}:${key}`, this.serializeRedisMetadata(metadata), 'EX', ttl);
|
|
119
|
+
} else {
|
|
120
|
+
await this.redis.set(`${this._mdPrefix}:${key}`, this.serializeRedisMetadata(metadata));
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
@SecureConnector.AccessControl
|
|
125
|
+
public async updateTTL(acRequest: AccessRequest, key: string, ttl?: number): Promise<void> {
|
|
126
|
+
if (ttl) {
|
|
127
|
+
await Promise.all([this.redis.expire(`${this._prefix}:${key}`, ttl), this.redis.expire(`${this._mdPrefix}:${key}`, ttl)]);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
@SecureConnector.AccessControl
|
|
132
|
+
public async getTTL(acRequest: AccessRequest, key: string): Promise<number> {
|
|
133
|
+
return this.redis.ttl(`${this._prefix}:${key}`);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
public async getResourceACL(resourceId: string, candidate: IAccessCandidate): Promise<ACL> {
|
|
137
|
+
const _metadata: any = await this.redis.get(`${this._mdPrefix}:${resourceId}`).catch((error) => {});
|
|
138
|
+
const exists = _metadata !== undefined && _metadata !== null; //null or undefined metadata means the resource does not exist
|
|
139
|
+
const metadata = exists ? this.deserializeRedisMetadata(_metadata) : {};
|
|
140
|
+
|
|
141
|
+
if (!exists) {
|
|
142
|
+
//the resource does not exist yet, we grant write access to the candidate in order to allow the resource creation
|
|
143
|
+
return new ACL().addAccess(candidate.role, candidate.id, TAccessLevel.Owner);
|
|
144
|
+
}
|
|
145
|
+
return ACL.from(metadata?.acl as IACL);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// async hasAccess(request: IAccessRequest): Promise<boolean> {
|
|
149
|
+
// try {
|
|
150
|
+
// const metadata = await this.getMetadata(request.resourceId);
|
|
151
|
+
// const acl: IACL = metadata?.acl as IACL;
|
|
152
|
+
// return ACL.from(acl).checkExactAccess(request);
|
|
153
|
+
// } catch (error) {
|
|
154
|
+
// if (error.name === 'NotFound') {
|
|
155
|
+
// return false;
|
|
156
|
+
// }
|
|
157
|
+
// console.error(`Error checking access rights in S3`, error.name, error.message);
|
|
158
|
+
// throw error;
|
|
159
|
+
// }
|
|
160
|
+
// }
|
|
161
|
+
|
|
162
|
+
@SecureConnector.AccessControl
|
|
163
|
+
async getACL(acRequest: AccessRequest, key: string): Promise<IACL> {
|
|
164
|
+
try {
|
|
165
|
+
const metadata = await this.getMetadata(acRequest, key);
|
|
166
|
+
return (metadata?.acl as IACL) || {};
|
|
167
|
+
} catch (error) {
|
|
168
|
+
console.error(`Error getting access rights in S3`, error.name, error.message);
|
|
169
|
+
throw error;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
@SecureConnector.AccessControl
|
|
174
|
+
async setACL(acRequest: AccessRequest, key: string, acl: IACL) {
|
|
175
|
+
try {
|
|
176
|
+
let metadata = await this.getMetadata(acRequest, key);
|
|
177
|
+
if (!metadata) metadata = {};
|
|
178
|
+
//when setting ACL make sure to not lose ownership
|
|
179
|
+
metadata.acl = ACL.from(acl).addAccess(acRequest.candidate.role, acRequest.candidate.id, TAccessLevel.Owner).ACL;
|
|
180
|
+
await this.setMetadata(acRequest, key, metadata);
|
|
181
|
+
} catch (error) {
|
|
182
|
+
console.error(`Error setting access rights in S3`, error);
|
|
183
|
+
throw error;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
private serializeRedisMetadata(redisMetadata: Record<string, any> | undefined): string {
|
|
188
|
+
if (!redisMetadata) return '';
|
|
189
|
+
if (redisMetadata.acl) {
|
|
190
|
+
const acl: IACL = redisMetadata.acl;
|
|
191
|
+
if (acl) {
|
|
192
|
+
redisMetadata.acl = ACL.from(acl).serializedACL;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
return JSON.stringify(redisMetadata);
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
private deserializeRedisMetadata(strMetadata: string): Record<string, any> {
|
|
200
|
+
try {
|
|
201
|
+
const redisMetadata = JSON.parse(strMetadata);
|
|
202
|
+
if (redisMetadata.acl) {
|
|
203
|
+
const acl: IACL = ACL.from(redisMetadata.acl).ACL;
|
|
204
|
+
redisMetadata.acl = acl;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
return redisMetadata;
|
|
208
|
+
} catch (error) {
|
|
209
|
+
console.warn(`Error deserializing metadata`, strMetadata);
|
|
210
|
+
return {};
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
public async stop() {
|
|
215
|
+
super.stop();
|
|
216
|
+
await this.redis.quit();
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* hosts can take any of the following formats:
|
|
222
|
+
* 1. A string with comma-separated host:port pairs
|
|
223
|
+
* 2. An array of strings with host:port pairs
|
|
224
|
+
* 3. An array of objects with host and port properties
|
|
225
|
+
* @param hosts
|
|
226
|
+
*/
|
|
227
|
+
function parseSentinelHosts(hosts: string | string[] | any[]) {
|
|
228
|
+
//handle all possible formats of hosts
|
|
229
|
+
if (typeof hosts === 'string') {
|
|
230
|
+
return hosts.split(',').map((host) => {
|
|
231
|
+
const [hostName, port] = host.split(':');
|
|
232
|
+
return {
|
|
233
|
+
host: hostName,
|
|
234
|
+
port: Number(port),
|
|
235
|
+
};
|
|
236
|
+
});
|
|
237
|
+
} else if (Array.isArray(hosts)) {
|
|
238
|
+
return hosts.map((host) => {
|
|
239
|
+
if (typeof host === 'string') {
|
|
240
|
+
const [hostName, port] = host.split(':');
|
|
241
|
+
return {
|
|
242
|
+
host: hostName,
|
|
243
|
+
port: Number(port),
|
|
244
|
+
};
|
|
245
|
+
} else {
|
|
246
|
+
return host;
|
|
247
|
+
}
|
|
248
|
+
});
|
|
249
|
+
} else {
|
|
250
|
+
return [];
|
|
251
|
+
}
|
|
252
|
+
}
|