@smythos/sre 1.5.0 → 1.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (189) hide show
  1. package/CHANGELOG +62 -0
  2. package/LICENSE +18 -0
  3. package/package.json +127 -115
  4. package/src/Components/APICall/APICall.class.ts +155 -0
  5. package/src/Components/APICall/AccessTokenManager.ts +130 -0
  6. package/src/Components/APICall/ArrayBufferResponse.helper.ts +58 -0
  7. package/src/Components/APICall/OAuth.helper.ts +294 -0
  8. package/src/Components/APICall/mimeTypeCategories.ts +46 -0
  9. package/src/Components/APICall/parseData.ts +167 -0
  10. package/src/Components/APICall/parseHeaders.ts +41 -0
  11. package/src/Components/APICall/parseProxy.ts +68 -0
  12. package/src/Components/APICall/parseUrl.ts +91 -0
  13. package/src/Components/APIEndpoint.class.ts +234 -0
  14. package/src/Components/APIOutput.class.ts +58 -0
  15. package/src/Components/AgentPlugin.class.ts +102 -0
  16. package/src/Components/Async.class.ts +155 -0
  17. package/src/Components/Await.class.ts +90 -0
  18. package/src/Components/Classifier.class.ts +158 -0
  19. package/src/Components/Component.class.ts +94 -0
  20. package/src/Components/ComponentHost.class.ts +38 -0
  21. package/src/Components/DataSourceCleaner.class.ts +92 -0
  22. package/src/Components/DataSourceIndexer.class.ts +181 -0
  23. package/src/Components/DataSourceLookup.class.ts +141 -0
  24. package/src/Components/FEncDec.class.ts +29 -0
  25. package/src/Components/FHash.class.ts +33 -0
  26. package/src/Components/FSign.class.ts +80 -0
  27. package/src/Components/FSleep.class.ts +25 -0
  28. package/src/Components/FTimestamp.class.ts +25 -0
  29. package/src/Components/FileStore.class.ts +75 -0
  30. package/src/Components/ForEach.class.ts +97 -0
  31. package/src/Components/GPTPlugin.class.ts +70 -0
  32. package/src/Components/GenAILLM.class.ts +395 -0
  33. package/src/Components/HuggingFace.class.ts +314 -0
  34. package/src/Components/Image/imageSettings.config.ts +70 -0
  35. package/src/Components/ImageGenerator.class.ts +407 -0
  36. package/src/Components/JSONFilter.class.ts +54 -0
  37. package/src/Components/LLMAssistant.class.ts +213 -0
  38. package/src/Components/LogicAND.class.ts +28 -0
  39. package/src/Components/LogicAtLeast.class.ts +85 -0
  40. package/src/Components/LogicAtMost.class.ts +86 -0
  41. package/src/Components/LogicOR.class.ts +29 -0
  42. package/src/Components/LogicXOR.class.ts +34 -0
  43. package/src/Components/MCPClient.class.ts +112 -0
  44. package/src/Components/PromptGenerator.class.ts +122 -0
  45. package/src/Components/ScrapflyWebScrape.class.ts +159 -0
  46. package/src/Components/TavilyWebSearch.class.ts +98 -0
  47. package/src/Components/index.ts +77 -0
  48. package/src/Core/AgentProcess.helper.ts +240 -0
  49. package/src/Core/Connector.class.ts +123 -0
  50. package/src/Core/ConnectorsService.ts +192 -0
  51. package/src/Core/DummyConnector.ts +49 -0
  52. package/src/Core/HookService.ts +105 -0
  53. package/src/Core/SmythRuntime.class.ts +292 -0
  54. package/src/Core/SystemEvents.ts +15 -0
  55. package/src/Core/boot.ts +55 -0
  56. package/src/config.ts +15 -0
  57. package/src/constants.ts +125 -0
  58. package/src/data/hugging-face.params.json +580 -0
  59. package/src/helpers/BinaryInput.helper.ts +324 -0
  60. package/src/helpers/Conversation.helper.ts +1094 -0
  61. package/src/helpers/JsonContent.helper.ts +97 -0
  62. package/src/helpers/LocalCache.helper.ts +97 -0
  63. package/src/helpers/Log.helper.ts +234 -0
  64. package/src/helpers/OpenApiParser.helper.ts +150 -0
  65. package/src/helpers/S3Cache.helper.ts +129 -0
  66. package/src/helpers/SmythURI.helper.ts +5 -0
  67. package/src/helpers/TemplateString.helper.ts +243 -0
  68. package/src/helpers/TypeChecker.helper.ts +329 -0
  69. package/src/index.ts +179 -0
  70. package/src/index.ts.bak +179 -0
  71. package/src/subsystems/AgentManager/Agent.class.ts +1108 -0
  72. package/src/subsystems/AgentManager/Agent.helper.ts +3 -0
  73. package/src/subsystems/AgentManager/AgentData.service/AgentDataConnector.ts +230 -0
  74. package/src/subsystems/AgentManager/AgentData.service/connectors/CLIAgentDataConnector.class.ts +66 -0
  75. package/src/subsystems/AgentManager/AgentData.service/connectors/LocalAgentDataConnector.class.ts +142 -0
  76. package/src/subsystems/AgentManager/AgentData.service/connectors/NullAgentData.class.ts +39 -0
  77. package/src/subsystems/AgentManager/AgentData.service/index.ts +18 -0
  78. package/src/subsystems/AgentManager/AgentLogger.class.ts +297 -0
  79. package/src/subsystems/AgentManager/AgentRequest.class.ts +51 -0
  80. package/src/subsystems/AgentManager/AgentRuntime.class.ts +559 -0
  81. package/src/subsystems/AgentManager/AgentSSE.class.ts +101 -0
  82. package/src/subsystems/AgentManager/AgentSettings.class.ts +52 -0
  83. package/src/subsystems/AgentManager/Component.service/ComponentConnector.ts +32 -0
  84. package/src/subsystems/AgentManager/Component.service/connectors/LocalComponentConnector.class.ts +59 -0
  85. package/src/subsystems/AgentManager/Component.service/index.ts +11 -0
  86. package/src/subsystems/AgentManager/EmbodimentSettings.class.ts +47 -0
  87. package/src/subsystems/AgentManager/ForkedAgent.class.ts +153 -0
  88. package/src/subsystems/AgentManager/OSResourceMonitor.ts +77 -0
  89. package/src/subsystems/ComputeManager/Code.service/CodeConnector.ts +99 -0
  90. package/src/subsystems/ComputeManager/Code.service/connectors/AWSLambdaCode.class.ts +63 -0
  91. package/src/subsystems/ComputeManager/Code.service/index.ts +11 -0
  92. package/src/subsystems/IO/CLI.service/CLIConnector.ts +47 -0
  93. package/src/subsystems/IO/CLI.service/index.ts +9 -0
  94. package/src/subsystems/IO/Log.service/LogConnector.ts +32 -0
  95. package/src/subsystems/IO/Log.service/connectors/ConsoleLog.class.ts +28 -0
  96. package/src/subsystems/IO/Log.service/index.ts +13 -0
  97. package/src/subsystems/IO/NKV.service/NKVConnector.ts +41 -0
  98. package/src/subsystems/IO/NKV.service/connectors/NKVRAM.class.ts +204 -0
  99. package/src/subsystems/IO/NKV.service/connectors/NKVRedis.class.ts +182 -0
  100. package/src/subsystems/IO/NKV.service/index.ts +12 -0
  101. package/src/subsystems/IO/Router.service/RouterConnector.ts +21 -0
  102. package/src/subsystems/IO/Router.service/connectors/ExpressRouter.class.ts +48 -0
  103. package/src/subsystems/IO/Router.service/connectors/NullRouter.class.ts +40 -0
  104. package/src/subsystems/IO/Router.service/index.ts +11 -0
  105. package/src/subsystems/IO/Storage.service/SmythFS.class.ts +472 -0
  106. package/src/subsystems/IO/Storage.service/StorageConnector.ts +66 -0
  107. package/src/subsystems/IO/Storage.service/connectors/LocalStorage.class.ts +305 -0
  108. package/src/subsystems/IO/Storage.service/connectors/S3Storage.class.ts +418 -0
  109. package/src/subsystems/IO/Storage.service/index.ts +13 -0
  110. package/src/subsystems/IO/VectorDB.service/VectorDBConnector.ts +108 -0
  111. package/src/subsystems/IO/VectorDB.service/connectors/MilvusVectorDB.class.ts +450 -0
  112. package/src/subsystems/IO/VectorDB.service/connectors/PineconeVectorDB.class.ts +373 -0
  113. package/src/subsystems/IO/VectorDB.service/connectors/RAMVecrtorDB.class.ts +420 -0
  114. package/src/subsystems/IO/VectorDB.service/embed/BaseEmbedding.ts +106 -0
  115. package/src/subsystems/IO/VectorDB.service/embed/OpenAIEmbedding.ts +109 -0
  116. package/src/subsystems/IO/VectorDB.service/embed/index.ts +21 -0
  117. package/src/subsystems/IO/VectorDB.service/index.ts +14 -0
  118. package/src/subsystems/LLMManager/LLM.helper.ts +221 -0
  119. package/src/subsystems/LLMManager/LLM.inference.ts +335 -0
  120. package/src/subsystems/LLMManager/LLM.service/LLMConnector.ts +374 -0
  121. package/src/subsystems/LLMManager/LLM.service/LLMCredentials.helper.ts +145 -0
  122. package/src/subsystems/LLMManager/LLM.service/connectors/Anthropic.class.ts +632 -0
  123. package/src/subsystems/LLMManager/LLM.service/connectors/Bedrock.class.ts +405 -0
  124. package/src/subsystems/LLMManager/LLM.service/connectors/Echo.class.ts +81 -0
  125. package/src/subsystems/LLMManager/LLM.service/connectors/GoogleAI.class.ts +689 -0
  126. package/src/subsystems/LLMManager/LLM.service/connectors/Groq.class.ts +257 -0
  127. package/src/subsystems/LLMManager/LLM.service/connectors/OpenAI.class.ts +848 -0
  128. package/src/subsystems/LLMManager/LLM.service/connectors/Perplexity.class.ts +255 -0
  129. package/src/subsystems/LLMManager/LLM.service/connectors/VertexAI.class.ts +193 -0
  130. package/src/subsystems/LLMManager/LLM.service/index.ts +43 -0
  131. package/src/subsystems/LLMManager/ModelsProvider.service/ModelsProviderConnector.ts +281 -0
  132. package/src/subsystems/LLMManager/ModelsProvider.service/connectors/SmythModelsProvider.class.ts +229 -0
  133. package/src/subsystems/LLMManager/ModelsProvider.service/index.ts +11 -0
  134. package/src/subsystems/LLMManager/custom-models.ts +854 -0
  135. package/src/subsystems/LLMManager/models.ts +2539 -0
  136. package/src/subsystems/LLMManager/paramMappings.ts +69 -0
  137. package/src/subsystems/MemoryManager/Cache.service/CacheConnector.ts +86 -0
  138. package/src/subsystems/MemoryManager/Cache.service/connectors/LocalStorageCache.class.ts +297 -0
  139. package/src/subsystems/MemoryManager/Cache.service/connectors/RAMCache.class.ts +201 -0
  140. package/src/subsystems/MemoryManager/Cache.service/connectors/RedisCache.class.ts +252 -0
  141. package/src/subsystems/MemoryManager/Cache.service/connectors/S3Cache.class.ts +373 -0
  142. package/src/subsystems/MemoryManager/Cache.service/index.ts +15 -0
  143. package/src/subsystems/MemoryManager/LLMCache.ts +72 -0
  144. package/src/subsystems/MemoryManager/LLMContext.ts +125 -0
  145. package/src/subsystems/MemoryManager/RuntimeContext.ts +249 -0
  146. package/src/subsystems/Security/AccessControl/ACL.class.ts +208 -0
  147. package/src/subsystems/Security/AccessControl/AccessCandidate.class.ts +76 -0
  148. package/src/subsystems/Security/AccessControl/AccessRequest.class.ts +52 -0
  149. package/src/subsystems/Security/Account.service/AccountConnector.ts +41 -0
  150. package/src/subsystems/Security/Account.service/connectors/AWSAccount.class.ts +76 -0
  151. package/src/subsystems/Security/Account.service/connectors/DummyAccount.class.ts +130 -0
  152. package/src/subsystems/Security/Account.service/connectors/JSONFileAccount.class.ts +159 -0
  153. package/src/subsystems/Security/Account.service/index.ts +14 -0
  154. package/src/subsystems/Security/Credentials.helper.ts +62 -0
  155. package/src/subsystems/Security/ManagedVault.service/ManagedVaultConnector.ts +34 -0
  156. package/src/subsystems/Security/ManagedVault.service/connectors/NullManagedVault.class.ts +57 -0
  157. package/src/subsystems/Security/ManagedVault.service/connectors/SecretManagerManagedVault.ts +154 -0
  158. package/src/subsystems/Security/ManagedVault.service/index.ts +12 -0
  159. package/src/subsystems/Security/SecureConnector.class.ts +110 -0
  160. package/src/subsystems/Security/Vault.service/Vault.helper.ts +30 -0
  161. package/src/subsystems/Security/Vault.service/VaultConnector.ts +26 -0
  162. package/src/subsystems/Security/Vault.service/connectors/HashicorpVault.class.ts +46 -0
  163. package/src/subsystems/Security/Vault.service/connectors/JSONFileVault.class.ts +166 -0
  164. package/src/subsystems/Security/Vault.service/connectors/NullVault.class.ts +54 -0
  165. package/src/subsystems/Security/Vault.service/connectors/SecretsManager.class.ts +140 -0
  166. package/src/subsystems/Security/Vault.service/index.ts +12 -0
  167. package/src/types/ACL.types.ts +104 -0
  168. package/src/types/AWS.types.ts +9 -0
  169. package/src/types/Agent.types.ts +61 -0
  170. package/src/types/AgentLogger.types.ts +17 -0
  171. package/src/types/Cache.types.ts +1 -0
  172. package/src/types/Common.types.ts +3 -0
  173. package/src/types/LLM.types.ts +419 -0
  174. package/src/types/Redis.types.ts +8 -0
  175. package/src/types/SRE.types.ts +64 -0
  176. package/src/types/Security.types.ts +18 -0
  177. package/src/types/Storage.types.ts +5 -0
  178. package/src/types/VectorDB.types.ts +78 -0
  179. package/src/utils/base64.utils.ts +275 -0
  180. package/src/utils/cli.utils.ts +68 -0
  181. package/src/utils/data.utils.ts +263 -0
  182. package/src/utils/date-time.utils.ts +22 -0
  183. package/src/utils/general.utils.ts +238 -0
  184. package/src/utils/index.ts +12 -0
  185. package/src/utils/numbers.utils.ts +13 -0
  186. package/src/utils/oauth.utils.ts +35 -0
  187. package/src/utils/string.utils.ts +414 -0
  188. package/src/utils/url.utils.ts +19 -0
  189. package/src/utils/validation.utils.ts +74 -0
@@ -0,0 +1,252 @@
1
+ import { Logger } from '@sre/helpers/Log.helper';
2
+ import { IAccessCandidate, IACL, TAccessLevel } from '@sre/types/ACL.types';
3
+ import { CacheMetadata } from '@sre/types/Cache.types';
4
+ import IORedis from 'ioredis';
5
+ import { CacheConnector } from '../CacheConnector';
6
+
7
+ import { ACL } from '@sre/Security/AccessControl/ACL.class';
8
+ import { RedisConfig } from '@sre/types/Redis.types';
9
+
10
+ import { AccessRequest } from '@sre/Security/AccessControl/AccessRequest.class';
11
+ import { SecureConnector } from '@sre/Security/SecureConnector.class';
12
+ import { AccountConnector } from '@sre/Security/Account.service/AccountConnector';
13
+ import { ConnectorService } from '@sre/Core/ConnectorsService';
14
+
15
+ const console = Logger('RedisCache');
16
+
17
+ export class RedisCache extends CacheConnector {
18
+ public name: string = 'RedisCache';
19
+ private redis: IORedis;
20
+ private _prefix: string = 'smyth:cache';
21
+ private _mdPrefix: string = 'smyth:metadata';
22
+
23
+ constructor(protected _settings: RedisConfig) {
24
+ super(_settings);
25
+ const sentinels = parseSentinelHosts(_settings.hosts || process.env.REDIS_HOSTS);
26
+ let host = sentinels.length === 1 ? sentinels[0].host : null;
27
+ let port = sentinels.length === 1 ? sentinels[0].port : null;
28
+
29
+ this.redis = new IORedis({
30
+ ...(host ? { host, port } : { sentinels, name: _settings.name || process.env.REDIS_MASTER_NAME }),
31
+ password: _settings.password || process.env.REDIS_PASSWORD,
32
+ });
33
+
34
+ this.redis.on('error', (error) => {
35
+ console.error('Redis Error:', error);
36
+ });
37
+
38
+ this.redis.on('connect', () => {
39
+ console.log('Redis connected!');
40
+ });
41
+ }
42
+
43
+ public get client() {
44
+ return this.redis;
45
+ }
46
+
47
+ public get prefix() {
48
+ return this._prefix;
49
+ }
50
+
51
+ public get mdPrefix() {
52
+ return this._mdPrefix;
53
+ }
54
+
55
+ @SecureConnector.AccessControl
56
+ public async get(acRequest: AccessRequest, key: string): Promise<string | null> {
57
+ const value = await this.redis.get(`${this._prefix}:${key}`);
58
+ return value;
59
+ }
60
+
61
+ @SecureConnector.AccessControl
62
+ public async set(acRequest: AccessRequest, key: string, data: any, acl?: IACL, metadata?: CacheMetadata, ttl?: number): Promise<boolean> {
63
+ const accessCandidate = acRequest.candidate;
64
+ const promises: any[] = [];
65
+
66
+ const newMetadata: CacheMetadata = metadata || {};
67
+ newMetadata.acl = ACL.from(acl).addAccess(accessCandidate.role, accessCandidate.id, TAccessLevel.Owner).ACL;
68
+
69
+ if (ttl && ttl > 0) {
70
+ promises.push(this.redis.set(`${this._prefix}:${key}`, data, 'EX', ttl));
71
+ promises.push(this.setMetadataWithTTL(acRequest, key, newMetadata, ttl));
72
+ } else {
73
+ promises.push(this.redis.set(`${this._prefix}:${key}`, data));
74
+ promises.push(this.setMetadata(acRequest, key, newMetadata));
75
+ }
76
+
77
+ await Promise.all(promises);
78
+
79
+ // if (ttl) {
80
+ // try {
81
+ // await this.updateTTL(acRequest, key, ttl);
82
+ // } catch (error) {
83
+ // console.error(`Error setting TTL for key ${key}`, error);
84
+ // }
85
+ // }
86
+
87
+ return true;
88
+ }
89
+
90
+ @SecureConnector.AccessControl
91
+ public async delete(acRequest: AccessRequest, key: string): Promise<void> {
92
+ //delete both the key and its metadata
93
+ await Promise.all([this.redis.del(`${this._prefix}:${key}`), this.redis.del(`${this._mdPrefix}:${key}`)]);
94
+ }
95
+
96
+ @SecureConnector.AccessControl
97
+ public async exists(acRequest: AccessRequest, key: string): Promise<boolean> {
98
+ return !!(await this.redis.exists(`${this._prefix}:${key}`));
99
+ }
100
+
101
+ @SecureConnector.AccessControl
102
+ public async getMetadata(acRequest: AccessRequest, key: string): Promise<CacheMetadata> {
103
+ if (!this.exists(acRequest, key)) return undefined;
104
+ try {
105
+ const metadata = await this.redis.get(`${this._mdPrefix}:${key}`);
106
+ return metadata ? (this.deserializeRedisMetadata(metadata) as CacheMetadata) : {};
107
+ } catch (error) {
108
+ return {};
109
+ }
110
+ }
111
+
112
+ @SecureConnector.AccessControl
113
+ public async setMetadata(acRequest: AccessRequest, key: string, metadata: CacheMetadata): Promise<void> {
114
+ await this.setMetadataWithTTL(acRequest, key, metadata);
115
+ }
116
+ private async setMetadataWithTTL(acRequest: AccessRequest, key: string, metadata: CacheMetadata, ttl?: number): Promise<void> {
117
+ if (ttl && ttl > 0) {
118
+ await this.redis.set(`${this._mdPrefix}:${key}`, this.serializeRedisMetadata(metadata), 'EX', ttl);
119
+ } else {
120
+ await this.redis.set(`${this._mdPrefix}:${key}`, this.serializeRedisMetadata(metadata));
121
+ }
122
+ }
123
+
124
+ @SecureConnector.AccessControl
125
+ public async updateTTL(acRequest: AccessRequest, key: string, ttl?: number): Promise<void> {
126
+ if (ttl) {
127
+ await Promise.all([this.redis.expire(`${this._prefix}:${key}`, ttl), this.redis.expire(`${this._mdPrefix}:${key}`, ttl)]);
128
+ }
129
+ }
130
+
131
+ @SecureConnector.AccessControl
132
+ public async getTTL(acRequest: AccessRequest, key: string): Promise<number> {
133
+ return this.redis.ttl(`${this._prefix}:${key}`);
134
+ }
135
+
136
+ public async getResourceACL(resourceId: string, candidate: IAccessCandidate): Promise<ACL> {
137
+ const _metadata: any = await this.redis.get(`${this._mdPrefix}:${resourceId}`).catch((error) => {});
138
+ const exists = _metadata !== undefined && _metadata !== null; //null or undefined metadata means the resource does not exist
139
+ const metadata = exists ? this.deserializeRedisMetadata(_metadata) : {};
140
+
141
+ if (!exists) {
142
+ //the resource does not exist yet, we grant write access to the candidate in order to allow the resource creation
143
+ return new ACL().addAccess(candidate.role, candidate.id, TAccessLevel.Owner);
144
+ }
145
+ return ACL.from(metadata?.acl as IACL);
146
+ }
147
+
148
+ // async hasAccess(request: IAccessRequest): Promise<boolean> {
149
+ // try {
150
+ // const metadata = await this.getMetadata(request.resourceId);
151
+ // const acl: IACL = metadata?.acl as IACL;
152
+ // return ACL.from(acl).checkExactAccess(request);
153
+ // } catch (error) {
154
+ // if (error.name === 'NotFound') {
155
+ // return false;
156
+ // }
157
+ // console.error(`Error checking access rights in S3`, error.name, error.message);
158
+ // throw error;
159
+ // }
160
+ // }
161
+
162
+ @SecureConnector.AccessControl
163
+ async getACL(acRequest: AccessRequest, key: string): Promise<IACL> {
164
+ try {
165
+ const metadata = await this.getMetadata(acRequest, key);
166
+ return (metadata?.acl as IACL) || {};
167
+ } catch (error) {
168
+ console.error(`Error getting access rights in S3`, error.name, error.message);
169
+ throw error;
170
+ }
171
+ }
172
+
173
+ @SecureConnector.AccessControl
174
+ async setACL(acRequest: AccessRequest, key: string, acl: IACL) {
175
+ try {
176
+ let metadata = await this.getMetadata(acRequest, key);
177
+ if (!metadata) metadata = {};
178
+ //when setting ACL make sure to not lose ownership
179
+ metadata.acl = ACL.from(acl).addAccess(acRequest.candidate.role, acRequest.candidate.id, TAccessLevel.Owner).ACL;
180
+ await this.setMetadata(acRequest, key, metadata);
181
+ } catch (error) {
182
+ console.error(`Error setting access rights in S3`, error);
183
+ throw error;
184
+ }
185
+ }
186
+
187
+ private serializeRedisMetadata(redisMetadata: Record<string, any> | undefined): string {
188
+ if (!redisMetadata) return '';
189
+ if (redisMetadata.acl) {
190
+ const acl: IACL = redisMetadata.acl;
191
+ if (acl) {
192
+ redisMetadata.acl = ACL.from(acl).serializedACL;
193
+ }
194
+ }
195
+
196
+ return JSON.stringify(redisMetadata);
197
+ }
198
+
199
+ private deserializeRedisMetadata(strMetadata: string): Record<string, any> {
200
+ try {
201
+ const redisMetadata = JSON.parse(strMetadata);
202
+ if (redisMetadata.acl) {
203
+ const acl: IACL = ACL.from(redisMetadata.acl).ACL;
204
+ redisMetadata.acl = acl;
205
+ }
206
+
207
+ return redisMetadata;
208
+ } catch (error) {
209
+ console.warn(`Error deserializing metadata`, strMetadata);
210
+ return {};
211
+ }
212
+ }
213
+
214
+ public async stop() {
215
+ super.stop();
216
+ await this.redis.quit();
217
+ }
218
+ }
219
+
220
+ /**
221
+ * hosts can take any of the following formats:
222
+ * 1. A string with comma-separated host:port pairs
223
+ * 2. An array of strings with host:port pairs
224
+ * 3. An array of objects with host and port properties
225
+ * @param hosts
226
+ */
227
+ function parseSentinelHosts(hosts: string | string[] | any[]) {
228
+ //handle all possible formats of hosts
229
+ if (typeof hosts === 'string') {
230
+ return hosts.split(',').map((host) => {
231
+ const [hostName, port] = host.split(':');
232
+ return {
233
+ host: hostName,
234
+ port: Number(port),
235
+ };
236
+ });
237
+ } else if (Array.isArray(hosts)) {
238
+ return hosts.map((host) => {
239
+ if (typeof host === 'string') {
240
+ const [hostName, port] = host.split(':');
241
+ return {
242
+ host: hostName,
243
+ port: Number(port),
244
+ };
245
+ } else {
246
+ return host;
247
+ }
248
+ });
249
+ } else {
250
+ return [];
251
+ }
252
+ }
@@ -0,0 +1,373 @@
1
+ import { Logger } from '@sre/helpers/Log.helper';
2
+ import { IAccessCandidate, IACL, TAccessLevel, TAccessRole } from '@sre/types/ACL.types';
3
+ import { CacheMetadata } from '@sre/types/Cache.types';
4
+ import { CacheConnector } from '../CacheConnector';
5
+
6
+ import { ACL } from '@sre/Security/AccessControl/ACL.class';
7
+ import { AccessRequest } from '@sre/Security/AccessControl/AccessRequest.class';
8
+ import { SecureConnector } from '@sre/Security/SecureConnector.class';
9
+
10
+ import {
11
+ S3Client,
12
+ GetObjectCommand,
13
+ PutObjectCommand,
14
+ PutObjectCommandInput,
15
+ DeleteObjectCommand,
16
+ HeadObjectCommand,
17
+ CopyObjectCommand,
18
+ GetObjectTaggingCommand,
19
+ PutObjectTaggingCommand,
20
+ HeadObjectCommandOutput,
21
+ GetObjectTaggingCommandOutput,
22
+ GetObjectCommandOutput,
23
+ } from '@aws-sdk/client-s3';
24
+ import { checkAndInstallLifecycleRules, generateExpiryMetadata, ttlToExpiryDays } from '@sre/helpers/S3Cache.helper';
25
+
26
+ const console = Logger('S3Cache');
27
+ export type S3CacheConfig = {
28
+ bucketName: string;
29
+ region: string;
30
+ accessKeyId: string;
31
+ secretAccessKey: string;
32
+ };
33
+
34
+ export class S3Cache extends CacheConnector {
35
+ public name: string = 'S3Cache';
36
+ private s3Client: S3Client;
37
+ private bucketName: string;
38
+ private isInitialized: boolean = false;
39
+ private cachePrefix: string = '_smyth_cache';
40
+
41
+ constructor(protected _settings: S3CacheConfig) {
42
+ super(_settings);
43
+ this.s3Client = new S3Client({
44
+ region: _settings.region,
45
+ credentials: {
46
+ accessKeyId: _settings.accessKeyId,
47
+ secretAccessKey: _settings.secretAccessKey,
48
+ },
49
+ });
50
+ this.bucketName = _settings.bucketName;
51
+ }
52
+
53
+ public get client() {
54
+ return this.s3Client;
55
+ }
56
+
57
+ @SecureConnector.AccessControl
58
+ public async get(acRequest: AccessRequest, key: string): Promise<string | null> {
59
+ const candidateId = acRequest.candidate.id;
60
+ if (!this.isInitialized) {
61
+ await this.initialize();
62
+ }
63
+ try {
64
+ const params = {
65
+ Bucket: this.bucketName,
66
+ Key: `${this.cachePrefix}/${candidateId}/${key}`,
67
+ };
68
+
69
+ const s3HeadCommand = new HeadObjectCommand(params);
70
+ const headData: HeadObjectCommandOutput = await this.s3Client.send(s3HeadCommand);
71
+
72
+ const expirationHeader = headData?.Expiration;
73
+ if (expirationHeader) {
74
+ const expirationDateMatch = expirationHeader.match(/expiry-date="([^"]+)"/);
75
+ if (expirationDateMatch) {
76
+ const expirationDate = new Date(expirationDateMatch[1]);
77
+ const currentDate = new Date();
78
+
79
+ if (currentDate > expirationDate) {
80
+ const s3DeleteCommand = new DeleteObjectCommand(params);
81
+ await this.s3Client.send(s3DeleteCommand);
82
+ console.log(`Object ${key} expired and deleted.`);
83
+ return null;
84
+ }
85
+ }
86
+ }
87
+
88
+ const s3GetCommand = new GetObjectCommand(params);
89
+ const objectData: GetObjectCommandOutput = await this.s3Client.send(s3GetCommand);
90
+ return objectData.Body.transformToString();
91
+ } catch (error) {
92
+ console.error(`Error reading object ${key}:`, error);
93
+ throw null;
94
+ }
95
+ }
96
+
97
+ @SecureConnector.AccessControl
98
+ public async set(acRequest: AccessRequest, key: string, data: any, acl?: IACL, metadata?: CacheMetadata, ttl?: number): Promise<boolean> {
99
+ const accessCandidate = acRequest.candidate;
100
+ const candidateId = accessCandidate.id;
101
+
102
+ const newMetadata: CacheMetadata = metadata || {};
103
+ newMetadata['acl'] = ACL.from(acl).addAccess(accessCandidate.role, accessCandidate.id, TAccessLevel.Owner).ACL;
104
+ const serializedMetadata = this.serializeS3Metadata(newMetadata);
105
+
106
+ const s3PutCommandConfig: PutObjectCommandInput = {
107
+ Bucket: this.bucketName,
108
+ Key: `${this.cachePrefix}/${candidateId}/${key}`,
109
+ Body: data,
110
+ Metadata: serializedMetadata,
111
+ };
112
+ if (ttl) {
113
+ const expiryMetadata = generateExpiryMetadata(ttlToExpiryDays(ttl)); // seconds to days
114
+ s3PutCommandConfig.Tagging = `${expiryMetadata.Key}=${expiryMetadata.Value}`;
115
+ }
116
+
117
+ const s3PutCommand = new PutObjectCommand(s3PutCommandConfig);
118
+ await this.s3Client.send(s3PutCommand);
119
+
120
+ return true;
121
+ }
122
+
123
+ @SecureConnector.AccessControl
124
+ public async delete(acRequest: AccessRequest, key: string): Promise<void> {
125
+ try {
126
+ const candidateId = acRequest.candidate.id;
127
+ const deleteCommand = new DeleteObjectCommand({ Bucket: this.bucketName, Key: `${this.cachePrefix}/${candidateId}/${key}` });
128
+ await this.s3Client.send(deleteCommand);
129
+ } catch (error) {
130
+ console.log(`Error deleting object ${key}:`, error);
131
+ return;
132
+ }
133
+ }
134
+
135
+ @SecureConnector.AccessControl
136
+ public async exists(acRequest: AccessRequest, key: string): Promise<boolean> {
137
+ const candidateId = acRequest.candidate.id;
138
+ try {
139
+ const params = {
140
+ Bucket: this.bucketName,
141
+ Key: `${this.cachePrefix}/${candidateId}/${key}`,
142
+ };
143
+ const s3HeadCommand = new HeadObjectCommand(params);
144
+ const headData: HeadObjectCommandOutput = await this.s3Client.send(s3HeadCommand);
145
+
146
+ const expirationHeader = headData?.Expiration;
147
+ if (expirationHeader) {
148
+ const expirationDateMatch = expirationHeader.match(/expiry-date="([^"]+)"/);
149
+ if (expirationDateMatch) {
150
+ const expirationDate = new Date(expirationDateMatch[1]);
151
+ const currentDate = new Date();
152
+
153
+ if (currentDate > expirationDate) {
154
+ await this.delete(acRequest, key);
155
+ console.log(`Object ${key} expired and deleted.`);
156
+ return false;
157
+ }
158
+ }
159
+ }
160
+
161
+ return true;
162
+ } catch (error) {
163
+ console.error(`Error reading object ${key}:`, error);
164
+ return false;
165
+ }
166
+ }
167
+
168
+ @SecureConnector.AccessControl
169
+ public async getMetadata(acRequest: AccessRequest, key: string): Promise<CacheMetadata> {
170
+ const candidateId = acRequest.candidate.id;
171
+
172
+ try {
173
+ const s3Metadata = await this.getS3Metadata(`${this.cachePrefix}/${candidateId}/${key}`);
174
+ return s3Metadata as CacheMetadata;
175
+ } catch (error) {
176
+ console.error(`Error getting access rights in S3`, error.name, error.message);
177
+ throw error;
178
+ }
179
+ }
180
+
181
+ @SecureConnector.AccessControl
182
+ public async setMetadata(acRequest: AccessRequest, key: string, metadata: CacheMetadata): Promise<void> {
183
+ const candidateId = acRequest.candidate.id;
184
+
185
+ try {
186
+ let s3Metadata = await this.getS3Metadata(`${this.cachePrefix}/${candidateId}/${key}`);
187
+ if (!s3Metadata) s3Metadata = {};
188
+ //s3Metadata['x-amz-meta-data'] = metadata;
189
+ s3Metadata = { ...s3Metadata, ...metadata };
190
+ await this.setS3Metadata(`${this.cachePrefix}/${candidateId}/${key}`, s3Metadata);
191
+ } catch (error) {
192
+ console.error(`Error setting access rights in S3`, error);
193
+ throw error;
194
+ }
195
+ }
196
+
197
+ @SecureConnector.AccessControl
198
+ public async updateTTL(acRequest: AccessRequest, key: string, ttl?: number): Promise<void> {
199
+ if (ttl) {
200
+ const candidateId = acRequest.candidate.id;
201
+ const expiryMetadata = generateExpiryMetadata(ttlToExpiryDays(ttl)); // seconds to days
202
+ const s3PutObjectTaggingCommand = new PutObjectTaggingCommand({
203
+ Bucket: this.bucketName,
204
+ Key: `${this.cachePrefix}/${candidateId}/${key}`,
205
+ Tagging: { TagSet: [{ Key: expiryMetadata.Key, Value: expiryMetadata.Value }] },
206
+ });
207
+ await this.s3Client.send(s3PutObjectTaggingCommand);
208
+ }
209
+ }
210
+
211
+ @SecureConnector.AccessControl
212
+ public async getTTL(acRequest: AccessRequest, key: string): Promise<number> {
213
+ const candidateId = acRequest.candidate.id;
214
+ const s3HeadCommand = new HeadObjectCommand({ Bucket: this.bucketName, Key: `${this.cachePrefix}/${candidateId}/${key}` });
215
+ const s3HeadObjectResponse: HeadObjectCommandOutput = await this.s3Client.send(s3HeadCommand);
216
+ const expirationHeader = s3HeadObjectResponse?.Expiration;
217
+ if (expirationHeader) {
218
+ const expirationDateMatch = expirationHeader.match(/expiry-date="([^"]+)"/);
219
+ if (expirationDateMatch) {
220
+ const expirationDate = new Date(expirationDateMatch[1]);
221
+ const currentDate = new Date();
222
+ const timeDifference = expirationDate.getTime() - currentDate.getTime();
223
+ return Math.floor(timeDifference / (1000 * 60 * 60 * 24)); // Convert to days
224
+ }
225
+ }
226
+ return -1; // Return -1 if no expiration date is found
227
+ }
228
+
229
+ public async getResourceACL(resourceId: string, candidate: IAccessCandidate): Promise<ACL> {
230
+ try {
231
+ const s3HeadCommand = new HeadObjectCommand({ Bucket: this.bucketName, Key: `${this.cachePrefix}/${candidate.id}/${resourceId}` });
232
+ const s3HeadObjectResponse: HeadObjectCommandOutput = await this.s3Client.send(s3HeadCommand);
233
+
234
+ const metadata = s3HeadObjectResponse.Metadata;
235
+ if (!metadata.acl) {
236
+ //the resource does not exist yet, we grant write access to the candidate in order to allow the resource creation
237
+ return new ACL().addAccess(candidate.role, candidate.id, TAccessLevel.Owner);
238
+ }
239
+ return ACL.from(metadata?.acl as string);
240
+ } catch (error) {
241
+ if (error.name === 'NotFound') {
242
+ //the resource does not exist yet, we grant write access to the candidate in order to allow the resource creation
243
+ return new ACL().addAccess(candidate.role, candidate.id, TAccessLevel.Owner);
244
+ }
245
+ throw error;
246
+ }
247
+ }
248
+
249
+ @SecureConnector.AccessControl
250
+ async getACL(acRequest: AccessRequest, key: string): Promise<IACL> {
251
+ try {
252
+ const metadata = await this.getMetadata(acRequest, key);
253
+ return (metadata?.acl as IACL) || {};
254
+ } catch (error) {
255
+ throw error;
256
+ }
257
+ }
258
+
259
+ @SecureConnector.AccessControl
260
+ async setACL(acRequest: AccessRequest, key: string, acl: IACL) {
261
+ try {
262
+ let metadata = await this.getMetadata(acRequest, key);
263
+ if (!metadata) metadata = {};
264
+ //when setting ACL make sure to not lose ownership
265
+ metadata.acl = ACL.from(acl).addAccess(acRequest.candidate.role, acRequest.candidate.id, TAccessLevel.Owner).ACL;
266
+ await this.setMetadata(acRequest, key, metadata);
267
+ } catch (error) {
268
+ console.error(`Error setting access rights in S3`, error);
269
+ throw error;
270
+ }
271
+ }
272
+
273
+ private async getS3Metadata(resourceId: string): Promise<Record<string, any> | undefined> {
274
+ try {
275
+ const command = new HeadObjectCommand({
276
+ Bucket: this.bucketName,
277
+ Key: resourceId,
278
+ });
279
+ const response: HeadObjectCommandOutput = await this.client.send(command);
280
+ const s3RawMetadata = response.Metadata;
281
+ if (!s3RawMetadata || Object.keys(s3RawMetadata).length === 0) return {};
282
+
283
+ let metadata: Record<string, any> = this.deserializeS3Metadata(s3RawMetadata);
284
+
285
+ if (!metadata['ContentType']) metadata['ContentType'] = response.ContentType ? response.ContentType : 'application/octet-stream';
286
+ return metadata;
287
+ } catch (error) {
288
+ if (error.name === 'NotFound' || error.name === 'NoSuchKey') {
289
+ return undefined;
290
+ }
291
+ console.error(`Error reading object metadata from S3`, error.name, error.message);
292
+ throw error;
293
+ }
294
+ }
295
+
296
+ private async setS3Metadata(resourceId: string, metadata: Record<string, any>): Promise<void> {
297
+ try {
298
+ // Get the current object content
299
+ const getObjectTaggingCommand = new GetObjectTaggingCommand({
300
+ Bucket: this.bucketName,
301
+ Key: resourceId,
302
+ });
303
+ const objectTagging: GetObjectTaggingCommandOutput = await this.client.send(getObjectTaggingCommand);
304
+ const serializedMetadata = this.serializeS3Metadata(metadata);
305
+ const copyObjectCommand = new CopyObjectCommand({
306
+ Bucket: this.bucketName,
307
+ CopySource: `${this.bucketName}/${resourceId}`,
308
+ Key: resourceId,
309
+ Metadata: serializedMetadata,
310
+ MetadataDirective: 'REPLACE',
311
+ Tagging: objectTagging.TagSet.map((tag) => `${tag.Key}=${tag.Value}`).join('&'),
312
+ });
313
+
314
+ await this.client.send(copyObjectCommand);
315
+ } catch (error) {
316
+ console.error(`Error setting object metadata in S3`, error.name, error.message);
317
+ throw error;
318
+ }
319
+ }
320
+
321
+ private async initialize() {
322
+ await checkAndInstallLifecycleRules(this.bucketName, this.s3Client);
323
+ this.isInitialized = true;
324
+ }
325
+
326
+ private serializeS3Metadata(s3Metadata: Record<string, any>): Record<string, string> {
327
+ let amzMetadata = {};
328
+ if (s3Metadata['acl']) {
329
+ amzMetadata['acl'] = typeof s3Metadata['acl'] == 'string' ? s3Metadata['acl'] : ACL.from(s3Metadata['acl']).serializedACL;
330
+ delete s3Metadata['acl'];
331
+ }
332
+
333
+ for (let key in s3Metadata) {
334
+ if (key == 'ContentType') continue; //skip ContentType as it can only be set when writing the object
335
+ amzMetadata[key] = typeof s3Metadata[key] === 'string' ? s3Metadata[key] : JSON.stringify(s3Metadata[key]);
336
+ }
337
+
338
+ return amzMetadata;
339
+ }
340
+
341
+ private deserializeS3Metadata(amzMetadata: Record<string, string>): Record<string, any> {
342
+ let metadata: Record<string, any> = {};
343
+
344
+ for (let key in amzMetadata) {
345
+ if (key === 'acl') {
346
+ metadata[key] = ACL.from(amzMetadata[key]).ACL;
347
+ continue;
348
+ }
349
+
350
+ try {
351
+ metadata[key] = JSON.parse(amzMetadata[key]);
352
+ } catch (error) {
353
+ metadata[key] = amzMetadata[key];
354
+ }
355
+ }
356
+
357
+ return metadata;
358
+ }
359
+
360
+ // async hasAccess(request: IAccessRequest): Promise<boolean> {
361
+ // try {
362
+ // const metadata = await this.getMetadata(request.resourceId);
363
+ // const acl: IACL = metadata?.acl as IACL;
364
+ // return ACL.from(acl).checkExactAccess(request);
365
+ // } catch (error) {
366
+ // if (error.name === 'NotFound') {
367
+ // return false;
368
+ // }
369
+ // console.error(`Error checking access rights in S3`, error.name, error.message);
370
+ // throw error;
371
+ // }
372
+ // }
373
+ }
@@ -0,0 +1,15 @@
1
+ //==[ SRE: Cache ]======================
2
+ import { ConnectorServiceProvider, ConnectorService } from '@sre/Core/ConnectorsService';
3
+ import { TConnectorService } from '@sre/types/SRE.types';
4
+ import { RedisCache } from './connectors/RedisCache.class';
5
+ import { S3Cache } from './connectors/S3Cache.class';
6
+ import { LocalStorageCache } from './connectors/LocalStorageCache.class';
7
+ import { RAMCache } from './connectors/RAMCache.class';
8
+ export class CacheService extends ConnectorServiceProvider {
9
+ public register() {
10
+ ConnectorService.register(TConnectorService.Cache, 'Redis', RedisCache);
11
+ ConnectorService.register(TConnectorService.Cache, 'S3', S3Cache);
12
+ ConnectorService.register(TConnectorService.Cache, 'LocalStorage', LocalStorageCache);
13
+ ConnectorService.register(TConnectorService.Cache, 'RAM', RAMCache);
14
+ }
15
+ }