@smythos/sre 1.6.8 → 1.6.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG +111 -111
- package/LICENSE +18 -18
- package/README.md +135 -135
- package/dist/bundle-analysis-lazy.html +4949 -0
- package/dist/bundle-analysis.html +4949 -0
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/types/Components/Triggers/Gmail.trigger.d.ts +58 -0
- package/dist/types/Components/Triggers/GmailTrigger.class.d.ts +44 -0
- package/dist/types/Components/Triggers/Trigger.class.d.ts +21 -0
- package/dist/types/Components/Triggers/WhatsApp.trigger.d.ts +22 -0
- package/dist/types/helpers/AIPerformanceAnalyzer.helper.d.ts +45 -0
- package/dist/types/helpers/AIPerformanceCollector.helper.d.ts +111 -0
- package/dist/types/subsystems/IO/Storage.service/connectors/AzureBlobStorage.class.d.ts +211 -0
- package/dist/types/subsystems/IO/VectorDB.service/connectors/WeaviateVectorDB.class.d.ts +187 -0
- package/dist/types/subsystems/PerformanceManager/Performance.service/PerformanceConnector.d.ts +102 -0
- package/dist/types/subsystems/PerformanceManager/Performance.service/connectors/LocalPerformanceConnector.class.d.ts +100 -0
- package/dist/types/subsystems/PerformanceManager/Performance.service/index.d.ts +22 -0
- package/dist/types/subsystems/Security/Credentials/Credentials.class.d.ts +2 -0
- package/dist/types/subsystems/Security/Credentials/ManagedOAuth2Credentials.class.d.ts +18 -0
- package/dist/types/subsystems/Security/Credentials/OAuth2Credentials.class.d.ts +14 -0
- package/dist/types/types/Performance.types.d.ts +468 -0
- package/dist/types/utils/package-manager.utils.d.ts +26 -0
- package/package.json +1 -1
- package/src/Components/APICall/APICall.class.ts +161 -161
- package/src/Components/APICall/AccessTokenManager.ts +166 -166
- package/src/Components/APICall/ArrayBufferResponse.helper.ts +58 -58
- package/src/Components/APICall/OAuth.helper.ts +447 -447
- package/src/Components/APICall/mimeTypeCategories.ts +46 -46
- package/src/Components/APICall/parseData.ts +167 -167
- package/src/Components/APICall/parseHeaders.ts +41 -41
- package/src/Components/APICall/parseProxy.ts +68 -68
- package/src/Components/APICall/parseUrl.ts +91 -91
- package/src/Components/APIEndpoint.class.ts +234 -234
- package/src/Components/APIOutput.class.ts +58 -58
- package/src/Components/AgentPlugin.class.ts +102 -102
- package/src/Components/Async.class.ts +155 -155
- package/src/Components/Await.class.ts +90 -90
- package/src/Components/Classifier.class.ts +158 -158
- package/src/Components/Component.class.ts +147 -147
- package/src/Components/ComponentHost.class.ts +38 -38
- package/src/Components/DataSourceCleaner.class.ts +92 -92
- package/src/Components/DataSourceIndexer.class.ts +181 -181
- package/src/Components/DataSourceLookup.class.ts +161 -161
- package/src/Components/ECMASandbox.class.ts +72 -72
- package/src/Components/FEncDec.class.ts +29 -29
- package/src/Components/FHash.class.ts +33 -33
- package/src/Components/FSign.class.ts +80 -80
- package/src/Components/FSleep.class.ts +25 -25
- package/src/Components/FTimestamp.class.ts +66 -66
- package/src/Components/FileStore.class.ts +78 -78
- package/src/Components/ForEach.class.ts +97 -97
- package/src/Components/GPTPlugin.class.ts +70 -70
- package/src/Components/GenAILLM.class.ts +586 -586
- package/src/Components/HuggingFace.class.ts +313 -313
- package/src/Components/Image/imageSettings.config.ts +70 -70
- package/src/Components/ImageGenerator.class.ts +483 -483
- package/src/Components/JSONFilter.class.ts +54 -54
- package/src/Components/LLMAssistant.class.ts +213 -213
- package/src/Components/LogicAND.class.ts +28 -28
- package/src/Components/LogicAtLeast.class.ts +85 -85
- package/src/Components/LogicAtMost.class.ts +86 -86
- package/src/Components/LogicOR.class.ts +29 -29
- package/src/Components/LogicXOR.class.ts +34 -34
- package/src/Components/MCPClient.class.ts +137 -137
- package/src/Components/MemoryDeleteKeyVal.class.ts +70 -70
- package/src/Components/MemoryReadKeyVal.class.ts +67 -67
- package/src/Components/MemoryWriteKeyVal.class.ts +62 -62
- package/src/Components/MemoryWriteObject.class.ts +97 -97
- package/src/Components/MultimodalLLM.class.ts +128 -128
- package/src/Components/OpenAPI.class.ts +72 -72
- package/src/Components/PromptGenerator.class.ts +122 -122
- package/src/Components/ScrapflyWebScrape.class.ts +183 -183
- package/src/Components/ServerlessCode.class.ts +123 -123
- package/src/Components/TavilyWebSearch.class.ts +103 -103
- package/src/Components/VisionLLM.class.ts +104 -104
- package/src/Components/ZapierAction.class.ts +127 -127
- package/src/Components/index.ts +97 -97
- package/src/Core/AgentProcess.helper.ts +240 -240
- package/src/Core/Connector.class.ts +123 -123
- package/src/Core/ConnectorsService.ts +197 -197
- package/src/Core/DummyConnector.ts +49 -49
- package/src/Core/HookService.ts +105 -105
- package/src/Core/SmythRuntime.class.ts +241 -241
- package/src/Core/SystemEvents.ts +16 -16
- package/src/Core/boot.ts +56 -56
- package/src/config.ts +15 -15
- package/src/constants.ts +126 -126
- package/src/data/hugging-face.params.json +579 -579
- package/src/helpers/AWSLambdaCode.helper.ts +624 -624
- package/src/helpers/BinaryInput.helper.ts +331 -331
- package/src/helpers/Conversation.helper.ts +1157 -1157
- package/src/helpers/ECMASandbox.helper.ts +64 -64
- package/src/helpers/JsonContent.helper.ts +97 -97
- package/src/helpers/LocalCache.helper.ts +97 -97
- package/src/helpers/Log.helper.ts +274 -274
- package/src/helpers/OpenApiParser.helper.ts +150 -150
- package/src/helpers/S3Cache.helper.ts +147 -147
- package/src/helpers/SmythURI.helper.ts +5 -5
- package/src/helpers/Sysconfig.helper.ts +95 -95
- package/src/helpers/TemplateString.helper.ts +243 -243
- package/src/helpers/TypeChecker.helper.ts +329 -329
- package/src/index.ts +3 -3
- package/src/index.ts.bak +3 -3
- package/src/subsystems/AgentManager/Agent.class.ts +1114 -1114
- package/src/subsystems/AgentManager/Agent.helper.ts +3 -3
- package/src/subsystems/AgentManager/AgentData.service/AgentDataConnector.ts +230 -230
- package/src/subsystems/AgentManager/AgentData.service/connectors/CLIAgentDataConnector.class.ts +66 -66
- package/src/subsystems/AgentManager/AgentData.service/connectors/LocalAgentDataConnector.class.ts +145 -145
- package/src/subsystems/AgentManager/AgentData.service/connectors/NullAgentData.class.ts +39 -39
- package/src/subsystems/AgentManager/AgentData.service/index.ts +18 -18
- package/src/subsystems/AgentManager/AgentLogger.class.ts +301 -301
- package/src/subsystems/AgentManager/AgentRequest.class.ts +51 -51
- package/src/subsystems/AgentManager/AgentRuntime.class.ts +557 -557
- package/src/subsystems/AgentManager/AgentSSE.class.ts +101 -101
- package/src/subsystems/AgentManager/AgentSettings.class.ts +52 -52
- package/src/subsystems/AgentManager/Component.service/ComponentConnector.ts +32 -32
- package/src/subsystems/AgentManager/Component.service/connectors/LocalComponentConnector.class.ts +60 -60
- package/src/subsystems/AgentManager/Component.service/index.ts +11 -11
- package/src/subsystems/AgentManager/EmbodimentSettings.class.ts +47 -47
- package/src/subsystems/AgentManager/ForkedAgent.class.ts +154 -154
- package/src/subsystems/AgentManager/OSResourceMonitor.ts +77 -77
- package/src/subsystems/ComputeManager/Code.service/CodeConnector.ts +98 -98
- package/src/subsystems/ComputeManager/Code.service/connectors/AWSLambdaCode.class.ts +171 -171
- package/src/subsystems/ComputeManager/Code.service/connectors/ECMASandbox.class.ts +131 -131
- package/src/subsystems/ComputeManager/Code.service/index.ts +13 -13
- package/src/subsystems/IO/CLI.service/CLIConnector.ts +47 -47
- package/src/subsystems/IO/CLI.service/index.ts +9 -9
- package/src/subsystems/IO/Log.service/LogConnector.ts +32 -32
- package/src/subsystems/IO/Log.service/connectors/ConsoleLog.class.ts +28 -28
- package/src/subsystems/IO/Log.service/index.ts +13 -13
- package/src/subsystems/IO/NKV.service/NKVConnector.ts +43 -43
- package/src/subsystems/IO/NKV.service/connectors/NKVLocalStorage.class.ts +234 -234
- package/src/subsystems/IO/NKV.service/connectors/NKVRAM.class.ts +204 -204
- package/src/subsystems/IO/NKV.service/connectors/NKVRedis.class.ts +182 -182
- package/src/subsystems/IO/NKV.service/index.ts +14 -14
- package/src/subsystems/IO/Router.service/RouterConnector.ts +21 -21
- package/src/subsystems/IO/Router.service/connectors/ExpressRouter.class.ts +48 -48
- package/src/subsystems/IO/Router.service/connectors/NullRouter.class.ts +40 -40
- package/src/subsystems/IO/Router.service/index.ts +11 -11
- package/src/subsystems/IO/Storage.service/SmythFS.class.ts +488 -488
- package/src/subsystems/IO/Storage.service/StorageConnector.ts +66 -66
- package/src/subsystems/IO/Storage.service/connectors/LocalStorage.class.ts +327 -327
- package/src/subsystems/IO/Storage.service/connectors/S3Storage.class.ts +482 -482
- package/src/subsystems/IO/Storage.service/index.ts +13 -13
- package/src/subsystems/IO/VectorDB.service/VectorDBConnector.ts +108 -108
- package/src/subsystems/IO/VectorDB.service/connectors/MilvusVectorDB.class.ts +465 -465
- package/src/subsystems/IO/VectorDB.service/connectors/PineconeVectorDB.class.ts +387 -387
- package/src/subsystems/IO/VectorDB.service/connectors/RAMVecrtorDB.class.ts +408 -408
- package/src/subsystems/IO/VectorDB.service/embed/BaseEmbedding.ts +107 -107
- package/src/subsystems/IO/VectorDB.service/embed/GoogleEmbedding.ts +118 -118
- package/src/subsystems/IO/VectorDB.service/embed/OpenAIEmbedding.ts +109 -109
- package/src/subsystems/IO/VectorDB.service/embed/index.ts +26 -26
- package/src/subsystems/IO/VectorDB.service/index.ts +14 -14
- package/src/subsystems/LLMManager/LLM.helper.ts +251 -251
- package/src/subsystems/LLMManager/LLM.inference.ts +345 -345
- package/src/subsystems/LLMManager/LLM.service/LLMConnector.ts +492 -492
- package/src/subsystems/LLMManager/LLM.service/LLMCredentials.helper.ts +171 -171
- package/src/subsystems/LLMManager/LLM.service/connectors/Anthropic.class.ts +666 -666
- package/src/subsystems/LLMManager/LLM.service/connectors/Bedrock.class.ts +407 -407
- package/src/subsystems/LLMManager/LLM.service/connectors/Echo.class.ts +92 -92
- package/src/subsystems/LLMManager/LLM.service/connectors/GoogleAI.class.ts +983 -983
- package/src/subsystems/LLMManager/LLM.service/connectors/Groq.class.ts +319 -319
- package/src/subsystems/LLMManager/LLM.service/connectors/Ollama.class.ts +361 -361
- package/src/subsystems/LLMManager/LLM.service/connectors/Perplexity.class.ts +257 -257
- package/src/subsystems/LLMManager/LLM.service/connectors/VertexAI.class.ts +430 -430
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/OpenAIConnector.class.ts +503 -503
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/ChatCompletionsApiInterface.ts +524 -524
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/OpenAIApiInterface.ts +100 -100
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/OpenAIApiInterfaceFactory.ts +81 -81
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/ResponsesApiInterface.ts +1145 -1145
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/constants.ts +13 -13
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/index.ts +4 -4
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/utils.ts +11 -11
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/types.ts +32 -32
- package/src/subsystems/LLMManager/LLM.service/connectors/xAI.class.ts +478 -478
- package/src/subsystems/LLMManager/LLM.service/index.ts +47 -47
- package/src/subsystems/LLMManager/ModelsProvider.service/ModelsProviderConnector.ts +303 -303
- package/src/subsystems/LLMManager/ModelsProvider.service/connectors/JSONModelsProvider.class.ts +280 -271
- package/src/subsystems/LLMManager/ModelsProvider.service/index.ts +11 -11
- package/src/subsystems/LLMManager/custom-models.ts +854 -854
- package/src/subsystems/LLMManager/models.ts +2540 -2540
- package/src/subsystems/LLMManager/paramMappings.ts +69 -69
- package/src/subsystems/MemoryManager/Cache.service/CacheConnector.ts +86 -86
- package/src/subsystems/MemoryManager/Cache.service/connectors/LocalStorageCache.class.ts +297 -297
- package/src/subsystems/MemoryManager/Cache.service/connectors/RAMCache.class.ts +214 -214
- package/src/subsystems/MemoryManager/Cache.service/connectors/RedisCache.class.ts +252 -252
- package/src/subsystems/MemoryManager/Cache.service/connectors/S3Cache.class.ts +373 -373
- package/src/subsystems/MemoryManager/Cache.service/index.ts +15 -15
- package/src/subsystems/MemoryManager/LLMCache.ts +72 -72
- package/src/subsystems/MemoryManager/LLMContext.ts +124 -124
- package/src/subsystems/MemoryManager/LLMMemory.service/LLMMemoryConnector.ts +26 -26
- package/src/subsystems/MemoryManager/RuntimeContext.ts +277 -277
- package/src/subsystems/Security/AccessControl/ACL.class.ts +208 -208
- package/src/subsystems/Security/AccessControl/AccessCandidate.class.ts +82 -82
- package/src/subsystems/Security/AccessControl/AccessRequest.class.ts +52 -52
- package/src/subsystems/Security/Account.service/AccountConnector.ts +44 -44
- package/src/subsystems/Security/Account.service/connectors/DummyAccount.class.ts +130 -130
- package/src/subsystems/Security/Account.service/connectors/JSONFileAccount.class.ts +170 -170
- package/src/subsystems/Security/Account.service/connectors/MySQLAccount.class.ts +76 -76
- package/src/subsystems/Security/Account.service/index.ts +14 -14
- package/src/subsystems/Security/Credentials.helper.ts +62 -62
- package/src/subsystems/Security/ManagedVault.service/ManagedVaultConnector.ts +38 -38
- package/src/subsystems/Security/ManagedVault.service/connectors/NullManagedVault.class.ts +53 -53
- package/src/subsystems/Security/ManagedVault.service/connectors/SecretManagerManagedVault.ts +154 -154
- package/src/subsystems/Security/ManagedVault.service/index.ts +12 -12
- package/src/subsystems/Security/SecureConnector.class.ts +110 -110
- package/src/subsystems/Security/Vault.service/Vault.helper.ts +30 -30
- package/src/subsystems/Security/Vault.service/VaultConnector.ts +29 -29
- package/src/subsystems/Security/Vault.service/connectors/HashicorpVault.class.ts +46 -46
- package/src/subsystems/Security/Vault.service/connectors/JSONFileVault.class.ts +221 -221
- package/src/subsystems/Security/Vault.service/connectors/NullVault.class.ts +54 -54
- package/src/subsystems/Security/Vault.service/connectors/SecretsManager.class.ts +140 -140
- package/src/subsystems/Security/Vault.service/index.ts +12 -12
- package/src/types/ACL.types.ts +104 -104
- package/src/types/AWS.types.ts +10 -10
- package/src/types/Agent.types.ts +61 -61
- package/src/types/AgentLogger.types.ts +17 -17
- package/src/types/Cache.types.ts +1 -1
- package/src/types/Common.types.ts +2 -2
- package/src/types/LLM.types.ts +520 -520
- package/src/types/Redis.types.ts +8 -8
- package/src/types/SRE.types.ts +64 -64
- package/src/types/Security.types.ts +14 -14
- package/src/types/Storage.types.ts +5 -5
- package/src/types/VectorDB.types.ts +86 -86
- package/src/utils/base64.utils.ts +275 -275
- package/src/utils/cli.utils.ts +68 -68
- package/src/utils/data.utils.ts +322 -322
- package/src/utils/date-time.utils.ts +22 -22
- package/src/utils/general.utils.ts +238 -238
- package/src/utils/index.ts +12 -12
- package/src/utils/lazy-client.ts +261 -261
- package/src/utils/numbers.utils.ts +13 -13
- package/src/utils/oauth.utils.ts +35 -35
- package/src/utils/string.utils.ts +414 -414
- package/src/utils/url.utils.ts +19 -19
- package/src/utils/validation.utils.ts +74 -74
- package/dist/types/subsystems/LLMManager/ModelsProvider.service/connectors/SmythModelsProvider.class.d.ts +0 -39
|
@@ -1,373 +1,373 @@
|
|
|
1
|
-
import { Logger } from '@sre/helpers/Log.helper';
|
|
2
|
-
import { IAccessCandidate, IACL, TAccessLevel, TAccessRole } from '@sre/types/ACL.types';
|
|
3
|
-
import { CacheMetadata } from '@sre/types/Cache.types';
|
|
4
|
-
import { CacheConnector } from '../CacheConnector';
|
|
5
|
-
|
|
6
|
-
import { ACL } from '@sre/Security/AccessControl/ACL.class';
|
|
7
|
-
import { AccessRequest } from '@sre/Security/AccessControl/AccessRequest.class';
|
|
8
|
-
import { SecureConnector } from '@sre/Security/SecureConnector.class';
|
|
9
|
-
|
|
10
|
-
import {
|
|
11
|
-
S3Client,
|
|
12
|
-
GetObjectCommand,
|
|
13
|
-
PutObjectCommand,
|
|
14
|
-
PutObjectCommandInput,
|
|
15
|
-
DeleteObjectCommand,
|
|
16
|
-
HeadObjectCommand,
|
|
17
|
-
CopyObjectCommand,
|
|
18
|
-
GetObjectTaggingCommand,
|
|
19
|
-
PutObjectTaggingCommand,
|
|
20
|
-
HeadObjectCommandOutput,
|
|
21
|
-
GetObjectTaggingCommandOutput,
|
|
22
|
-
GetObjectCommandOutput,
|
|
23
|
-
} from '@aws-sdk/client-s3';
|
|
24
|
-
import { checkAndInstallLifecycleRules, generateExpiryMetadata, ttlToExpiryDays } from '@sre/helpers/S3Cache.helper';
|
|
25
|
-
|
|
26
|
-
const console = Logger('S3Cache');
|
|
27
|
-
export type S3CacheConfig = {
|
|
28
|
-
bucketName: string;
|
|
29
|
-
region: string;
|
|
30
|
-
accessKeyId: string;
|
|
31
|
-
secretAccessKey: string;
|
|
32
|
-
};
|
|
33
|
-
|
|
34
|
-
export class S3Cache extends CacheConnector {
|
|
35
|
-
public name: string = 'S3Cache';
|
|
36
|
-
private s3Client: S3Client;
|
|
37
|
-
private bucketName: string;
|
|
38
|
-
private isInitialized: boolean = false;
|
|
39
|
-
private cachePrefix: string = '_smyth_cache';
|
|
40
|
-
|
|
41
|
-
constructor(protected _settings: S3CacheConfig) {
|
|
42
|
-
super(_settings);
|
|
43
|
-
this.s3Client = new S3Client({
|
|
44
|
-
region: _settings.region,
|
|
45
|
-
credentials: {
|
|
46
|
-
accessKeyId: _settings.accessKeyId,
|
|
47
|
-
secretAccessKey: _settings.secretAccessKey,
|
|
48
|
-
},
|
|
49
|
-
});
|
|
50
|
-
this.bucketName = _settings.bucketName;
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
public get client() {
|
|
54
|
-
return this.s3Client;
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
@SecureConnector.AccessControl
|
|
58
|
-
public async get(acRequest: AccessRequest, key: string): Promise<string | null> {
|
|
59
|
-
const candidateId = acRequest.candidate.id;
|
|
60
|
-
if (!this.isInitialized) {
|
|
61
|
-
await this.initialize();
|
|
62
|
-
}
|
|
63
|
-
try {
|
|
64
|
-
const params = {
|
|
65
|
-
Bucket: this.bucketName,
|
|
66
|
-
Key: `${this.cachePrefix}/${candidateId}/${key}`,
|
|
67
|
-
};
|
|
68
|
-
|
|
69
|
-
const s3HeadCommand = new HeadObjectCommand(params);
|
|
70
|
-
const headData: HeadObjectCommandOutput = await this.s3Client.send(s3HeadCommand);
|
|
71
|
-
|
|
72
|
-
const expirationHeader = headData?.Expiration;
|
|
73
|
-
if (expirationHeader) {
|
|
74
|
-
const expirationDateMatch = expirationHeader.match(/expiry-date="([^"]+)"/);
|
|
75
|
-
if (expirationDateMatch) {
|
|
76
|
-
const expirationDate = new Date(expirationDateMatch[1]);
|
|
77
|
-
const currentDate = new Date();
|
|
78
|
-
|
|
79
|
-
if (currentDate > expirationDate) {
|
|
80
|
-
const s3DeleteCommand = new DeleteObjectCommand(params);
|
|
81
|
-
await this.s3Client.send(s3DeleteCommand);
|
|
82
|
-
console.log(`Object ${key} expired and deleted.`);
|
|
83
|
-
return null;
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
const s3GetCommand = new GetObjectCommand(params);
|
|
89
|
-
const objectData: GetObjectCommandOutput = await this.s3Client.send(s3GetCommand);
|
|
90
|
-
return objectData.Body.transformToString();
|
|
91
|
-
} catch (error) {
|
|
92
|
-
console.error(`Error reading object ${key}:`, error);
|
|
93
|
-
throw null;
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
@SecureConnector.AccessControl
|
|
98
|
-
public async set(acRequest: AccessRequest, key: string, data: any, acl?: IACL, metadata?: CacheMetadata, ttl?: number): Promise<boolean> {
|
|
99
|
-
const accessCandidate = acRequest.candidate;
|
|
100
|
-
const candidateId = accessCandidate.id;
|
|
101
|
-
|
|
102
|
-
const newMetadata: CacheMetadata = metadata || {};
|
|
103
|
-
newMetadata['acl'] = ACL.from(acl).addAccess(accessCandidate.role, accessCandidate.id, TAccessLevel.Owner).ACL;
|
|
104
|
-
const serializedMetadata = this.serializeS3Metadata(newMetadata);
|
|
105
|
-
|
|
106
|
-
const s3PutCommandConfig: PutObjectCommandInput = {
|
|
107
|
-
Bucket: this.bucketName,
|
|
108
|
-
Key: `${this.cachePrefix}/${candidateId}/${key}`,
|
|
109
|
-
Body: data,
|
|
110
|
-
Metadata: serializedMetadata,
|
|
111
|
-
};
|
|
112
|
-
if (ttl) {
|
|
113
|
-
const expiryMetadata = generateExpiryMetadata(ttlToExpiryDays(ttl)); // seconds to days
|
|
114
|
-
s3PutCommandConfig.Tagging = `${expiryMetadata.Key}=${expiryMetadata.Value}`;
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
const s3PutCommand = new PutObjectCommand(s3PutCommandConfig);
|
|
118
|
-
await this.s3Client.send(s3PutCommand);
|
|
119
|
-
|
|
120
|
-
return true;
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
@SecureConnector.AccessControl
|
|
124
|
-
public async delete(acRequest: AccessRequest, key: string): Promise<void> {
|
|
125
|
-
try {
|
|
126
|
-
const candidateId = acRequest.candidate.id;
|
|
127
|
-
const deleteCommand = new DeleteObjectCommand({ Bucket: this.bucketName, Key: `${this.cachePrefix}/${candidateId}/${key}` });
|
|
128
|
-
await this.s3Client.send(deleteCommand);
|
|
129
|
-
} catch (error) {
|
|
130
|
-
console.log(`Error deleting object ${key}:`, error);
|
|
131
|
-
return;
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
@SecureConnector.AccessControl
|
|
136
|
-
public async exists(acRequest: AccessRequest, key: string): Promise<boolean> {
|
|
137
|
-
const candidateId = acRequest.candidate.id;
|
|
138
|
-
try {
|
|
139
|
-
const params = {
|
|
140
|
-
Bucket: this.bucketName,
|
|
141
|
-
Key: `${this.cachePrefix}/${candidateId}/${key}`,
|
|
142
|
-
};
|
|
143
|
-
const s3HeadCommand = new HeadObjectCommand(params);
|
|
144
|
-
const headData: HeadObjectCommandOutput = await this.s3Client.send(s3HeadCommand);
|
|
145
|
-
|
|
146
|
-
const expirationHeader = headData?.Expiration;
|
|
147
|
-
if (expirationHeader) {
|
|
148
|
-
const expirationDateMatch = expirationHeader.match(/expiry-date="([^"]+)"/);
|
|
149
|
-
if (expirationDateMatch) {
|
|
150
|
-
const expirationDate = new Date(expirationDateMatch[1]);
|
|
151
|
-
const currentDate = new Date();
|
|
152
|
-
|
|
153
|
-
if (currentDate > expirationDate) {
|
|
154
|
-
await this.delete(acRequest, key);
|
|
155
|
-
console.log(`Object ${key} expired and deleted.`);
|
|
156
|
-
return false;
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
return true;
|
|
162
|
-
} catch (error) {
|
|
163
|
-
console.error(`Error reading object ${key}:`, error);
|
|
164
|
-
return false;
|
|
165
|
-
}
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
@SecureConnector.AccessControl
|
|
169
|
-
public async getMetadata(acRequest: AccessRequest, key: string): Promise<CacheMetadata> {
|
|
170
|
-
const candidateId = acRequest.candidate.id;
|
|
171
|
-
|
|
172
|
-
try {
|
|
173
|
-
const s3Metadata = await this.getS3Metadata(`${this.cachePrefix}/${candidateId}/${key}`);
|
|
174
|
-
return s3Metadata as CacheMetadata;
|
|
175
|
-
} catch (error) {
|
|
176
|
-
console.error(`Error getting access rights in S3`, error.name, error.message);
|
|
177
|
-
throw error;
|
|
178
|
-
}
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
@SecureConnector.AccessControl
|
|
182
|
-
public async setMetadata(acRequest: AccessRequest, key: string, metadata: CacheMetadata): Promise<void> {
|
|
183
|
-
const candidateId = acRequest.candidate.id;
|
|
184
|
-
|
|
185
|
-
try {
|
|
186
|
-
let s3Metadata = await this.getS3Metadata(`${this.cachePrefix}/${candidateId}/${key}`);
|
|
187
|
-
if (!s3Metadata) s3Metadata = {};
|
|
188
|
-
//s3Metadata['x-amz-meta-data'] = metadata;
|
|
189
|
-
s3Metadata = { ...s3Metadata, ...metadata };
|
|
190
|
-
await this.setS3Metadata(`${this.cachePrefix}/${candidateId}/${key}`, s3Metadata);
|
|
191
|
-
} catch (error) {
|
|
192
|
-
console.error(`Error setting access rights in S3`, error);
|
|
193
|
-
throw error;
|
|
194
|
-
}
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
@SecureConnector.AccessControl
|
|
198
|
-
public async updateTTL(acRequest: AccessRequest, key: string, ttl?: number): Promise<void> {
|
|
199
|
-
if (ttl) {
|
|
200
|
-
const candidateId = acRequest.candidate.id;
|
|
201
|
-
const expiryMetadata = generateExpiryMetadata(ttlToExpiryDays(ttl)); // seconds to days
|
|
202
|
-
const s3PutObjectTaggingCommand = new PutObjectTaggingCommand({
|
|
203
|
-
Bucket: this.bucketName,
|
|
204
|
-
Key: `${this.cachePrefix}/${candidateId}/${key}`,
|
|
205
|
-
Tagging: { TagSet: [{ Key: expiryMetadata.Key, Value: expiryMetadata.Value }] },
|
|
206
|
-
});
|
|
207
|
-
await this.s3Client.send(s3PutObjectTaggingCommand);
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
|
|
211
|
-
@SecureConnector.AccessControl
|
|
212
|
-
public async getTTL(acRequest: AccessRequest, key: string): Promise<number> {
|
|
213
|
-
const candidateId = acRequest.candidate.id;
|
|
214
|
-
const s3HeadCommand = new HeadObjectCommand({ Bucket: this.bucketName, Key: `${this.cachePrefix}/${candidateId}/${key}` });
|
|
215
|
-
const s3HeadObjectResponse: HeadObjectCommandOutput = await this.s3Client.send(s3HeadCommand);
|
|
216
|
-
const expirationHeader = s3HeadObjectResponse?.Expiration;
|
|
217
|
-
if (expirationHeader) {
|
|
218
|
-
const expirationDateMatch = expirationHeader.match(/expiry-date="([^"]+)"/);
|
|
219
|
-
if (expirationDateMatch) {
|
|
220
|
-
const expirationDate = new Date(expirationDateMatch[1]);
|
|
221
|
-
const currentDate = new Date();
|
|
222
|
-
const timeDifference = expirationDate.getTime() - currentDate.getTime();
|
|
223
|
-
return Math.floor(timeDifference / (1000 * 60 * 60 * 24)); // Convert to days
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
return -1; // Return -1 if no expiration date is found
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
public async getResourceACL(resourceId: string, candidate: IAccessCandidate): Promise<ACL> {
|
|
230
|
-
try {
|
|
231
|
-
const s3HeadCommand = new HeadObjectCommand({ Bucket: this.bucketName, Key: `${this.cachePrefix}/${candidate.id}/${resourceId}` });
|
|
232
|
-
const s3HeadObjectResponse: HeadObjectCommandOutput = await this.s3Client.send(s3HeadCommand);
|
|
233
|
-
|
|
234
|
-
const metadata = s3HeadObjectResponse.Metadata;
|
|
235
|
-
if (!metadata.acl) {
|
|
236
|
-
//the resource does not exist yet, we grant write access to the candidate in order to allow the resource creation
|
|
237
|
-
return new ACL().addAccess(candidate.role, candidate.id, TAccessLevel.Owner);
|
|
238
|
-
}
|
|
239
|
-
return ACL.from(metadata?.acl as string);
|
|
240
|
-
} catch (error) {
|
|
241
|
-
if (error.name === 'NotFound') {
|
|
242
|
-
//the resource does not exist yet, we grant write access to the candidate in order to allow the resource creation
|
|
243
|
-
return new ACL().addAccess(candidate.role, candidate.id, TAccessLevel.Owner);
|
|
244
|
-
}
|
|
245
|
-
throw error;
|
|
246
|
-
}
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
@SecureConnector.AccessControl
|
|
250
|
-
async getACL(acRequest: AccessRequest, key: string): Promise<IACL> {
|
|
251
|
-
try {
|
|
252
|
-
const metadata = await this.getMetadata(acRequest, key);
|
|
253
|
-
return (metadata?.acl as IACL) || {};
|
|
254
|
-
} catch (error) {
|
|
255
|
-
throw error;
|
|
256
|
-
}
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
@SecureConnector.AccessControl
|
|
260
|
-
async setACL(acRequest: AccessRequest, key: string, acl: IACL) {
|
|
261
|
-
try {
|
|
262
|
-
let metadata = await this.getMetadata(acRequest, key);
|
|
263
|
-
if (!metadata) metadata = {};
|
|
264
|
-
//when setting ACL make sure to not lose ownership
|
|
265
|
-
metadata.acl = ACL.from(acl).addAccess(acRequest.candidate.role, acRequest.candidate.id, TAccessLevel.Owner).ACL;
|
|
266
|
-
await this.setMetadata(acRequest, key, metadata);
|
|
267
|
-
} catch (error) {
|
|
268
|
-
console.error(`Error setting access rights in S3`, error);
|
|
269
|
-
throw error;
|
|
270
|
-
}
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
private async getS3Metadata(resourceId: string): Promise<Record<string, any> | undefined> {
|
|
274
|
-
try {
|
|
275
|
-
const command = new HeadObjectCommand({
|
|
276
|
-
Bucket: this.bucketName,
|
|
277
|
-
Key: resourceId,
|
|
278
|
-
});
|
|
279
|
-
const response: HeadObjectCommandOutput = await this.client.send(command);
|
|
280
|
-
const s3RawMetadata = response.Metadata;
|
|
281
|
-
if (!s3RawMetadata || Object.keys(s3RawMetadata).length === 0) return {};
|
|
282
|
-
|
|
283
|
-
let metadata: Record<string, any> = this.deserializeS3Metadata(s3RawMetadata);
|
|
284
|
-
|
|
285
|
-
if (!metadata['ContentType']) metadata['ContentType'] = response.ContentType ? response.ContentType : 'application/octet-stream';
|
|
286
|
-
return metadata;
|
|
287
|
-
} catch (error) {
|
|
288
|
-
if (error.name === 'NotFound' || error.name === 'NoSuchKey') {
|
|
289
|
-
return undefined;
|
|
290
|
-
}
|
|
291
|
-
console.error(`Error reading object metadata from S3`, error.name, error.message);
|
|
292
|
-
throw error;
|
|
293
|
-
}
|
|
294
|
-
}
|
|
295
|
-
|
|
296
|
-
private async setS3Metadata(resourceId: string, metadata: Record<string, any>): Promise<void> {
|
|
297
|
-
try {
|
|
298
|
-
// Get the current object content
|
|
299
|
-
const getObjectTaggingCommand = new GetObjectTaggingCommand({
|
|
300
|
-
Bucket: this.bucketName,
|
|
301
|
-
Key: resourceId,
|
|
302
|
-
});
|
|
303
|
-
const objectTagging: GetObjectTaggingCommandOutput = await this.client.send(getObjectTaggingCommand);
|
|
304
|
-
const serializedMetadata = this.serializeS3Metadata(metadata);
|
|
305
|
-
const copyObjectCommand = new CopyObjectCommand({
|
|
306
|
-
Bucket: this.bucketName,
|
|
307
|
-
CopySource: `${this.bucketName}/${resourceId}`,
|
|
308
|
-
Key: resourceId,
|
|
309
|
-
Metadata: serializedMetadata,
|
|
310
|
-
MetadataDirective: 'REPLACE',
|
|
311
|
-
Tagging: objectTagging.TagSet.map((tag) => `${tag.Key}=${tag.Value}`).join('&'),
|
|
312
|
-
});
|
|
313
|
-
|
|
314
|
-
await this.client.send(copyObjectCommand);
|
|
315
|
-
} catch (error) {
|
|
316
|
-
console.error(`Error setting object metadata in S3`, error.name, error.message);
|
|
317
|
-
throw error;
|
|
318
|
-
}
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
private async initialize() {
|
|
322
|
-
await checkAndInstallLifecycleRules(this.bucketName, this.s3Client);
|
|
323
|
-
this.isInitialized = true;
|
|
324
|
-
}
|
|
325
|
-
|
|
326
|
-
private serializeS3Metadata(s3Metadata: Record<string, any>): Record<string, string> {
|
|
327
|
-
let amzMetadata = {};
|
|
328
|
-
if (s3Metadata['acl']) {
|
|
329
|
-
amzMetadata['acl'] = typeof s3Metadata['acl'] == 'string' ? s3Metadata['acl'] : ACL.from(s3Metadata['acl']).serializedACL;
|
|
330
|
-
delete s3Metadata['acl'];
|
|
331
|
-
}
|
|
332
|
-
|
|
333
|
-
for (let key in s3Metadata) {
|
|
334
|
-
if (key == 'ContentType') continue; //skip ContentType as it can only be set when writing the object
|
|
335
|
-
amzMetadata[key] = typeof s3Metadata[key] === 'string' ? s3Metadata[key] : JSON.stringify(s3Metadata[key]);
|
|
336
|
-
}
|
|
337
|
-
|
|
338
|
-
return amzMetadata;
|
|
339
|
-
}
|
|
340
|
-
|
|
341
|
-
private deserializeS3Metadata(amzMetadata: Record<string, string>): Record<string, any> {
|
|
342
|
-
let metadata: Record<string, any> = {};
|
|
343
|
-
|
|
344
|
-
for (let key in amzMetadata) {
|
|
345
|
-
if (key === 'acl') {
|
|
346
|
-
metadata[key] = ACL.from(amzMetadata[key]).ACL;
|
|
347
|
-
continue;
|
|
348
|
-
}
|
|
349
|
-
|
|
350
|
-
try {
|
|
351
|
-
metadata[key] = JSON.parse(amzMetadata[key]);
|
|
352
|
-
} catch (error) {
|
|
353
|
-
metadata[key] = amzMetadata[key];
|
|
354
|
-
}
|
|
355
|
-
}
|
|
356
|
-
|
|
357
|
-
return metadata;
|
|
358
|
-
}
|
|
359
|
-
|
|
360
|
-
// async hasAccess(request: IAccessRequest): Promise<boolean> {
|
|
361
|
-
// try {
|
|
362
|
-
// const metadata = await this.getMetadata(request.resourceId);
|
|
363
|
-
// const acl: IACL = metadata?.acl as IACL;
|
|
364
|
-
// return ACL.from(acl).checkExactAccess(request);
|
|
365
|
-
// } catch (error) {
|
|
366
|
-
// if (error.name === 'NotFound') {
|
|
367
|
-
// return false;
|
|
368
|
-
// }
|
|
369
|
-
// console.error(`Error checking access rights in S3`, error.name, error.message);
|
|
370
|
-
// throw error;
|
|
371
|
-
// }
|
|
372
|
-
// }
|
|
373
|
-
}
|
|
1
|
+
import { Logger } from '@sre/helpers/Log.helper';
|
|
2
|
+
import { IAccessCandidate, IACL, TAccessLevel, TAccessRole } from '@sre/types/ACL.types';
|
|
3
|
+
import { CacheMetadata } from '@sre/types/Cache.types';
|
|
4
|
+
import { CacheConnector } from '../CacheConnector';
|
|
5
|
+
|
|
6
|
+
import { ACL } from '@sre/Security/AccessControl/ACL.class';
|
|
7
|
+
import { AccessRequest } from '@sre/Security/AccessControl/AccessRequest.class';
|
|
8
|
+
import { SecureConnector } from '@sre/Security/SecureConnector.class';
|
|
9
|
+
|
|
10
|
+
import {
|
|
11
|
+
S3Client,
|
|
12
|
+
GetObjectCommand,
|
|
13
|
+
PutObjectCommand,
|
|
14
|
+
PutObjectCommandInput,
|
|
15
|
+
DeleteObjectCommand,
|
|
16
|
+
HeadObjectCommand,
|
|
17
|
+
CopyObjectCommand,
|
|
18
|
+
GetObjectTaggingCommand,
|
|
19
|
+
PutObjectTaggingCommand,
|
|
20
|
+
HeadObjectCommandOutput,
|
|
21
|
+
GetObjectTaggingCommandOutput,
|
|
22
|
+
GetObjectCommandOutput,
|
|
23
|
+
} from '@aws-sdk/client-s3';
|
|
24
|
+
import { checkAndInstallLifecycleRules, generateExpiryMetadata, ttlToExpiryDays } from '@sre/helpers/S3Cache.helper';
|
|
25
|
+
|
|
26
|
+
const console = Logger('S3Cache');
|
|
27
|
+
export type S3CacheConfig = {
|
|
28
|
+
bucketName: string;
|
|
29
|
+
region: string;
|
|
30
|
+
accessKeyId: string;
|
|
31
|
+
secretAccessKey: string;
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
export class S3Cache extends CacheConnector {
|
|
35
|
+
public name: string = 'S3Cache';
|
|
36
|
+
private s3Client: S3Client;
|
|
37
|
+
private bucketName: string;
|
|
38
|
+
private isInitialized: boolean = false;
|
|
39
|
+
private cachePrefix: string = '_smyth_cache';
|
|
40
|
+
|
|
41
|
+
constructor(protected _settings: S3CacheConfig) {
|
|
42
|
+
super(_settings);
|
|
43
|
+
this.s3Client = new S3Client({
|
|
44
|
+
region: _settings.region,
|
|
45
|
+
credentials: {
|
|
46
|
+
accessKeyId: _settings.accessKeyId,
|
|
47
|
+
secretAccessKey: _settings.secretAccessKey,
|
|
48
|
+
},
|
|
49
|
+
});
|
|
50
|
+
this.bucketName = _settings.bucketName;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
public get client() {
|
|
54
|
+
return this.s3Client;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
@SecureConnector.AccessControl
|
|
58
|
+
public async get(acRequest: AccessRequest, key: string): Promise<string | null> {
|
|
59
|
+
const candidateId = acRequest.candidate.id;
|
|
60
|
+
if (!this.isInitialized) {
|
|
61
|
+
await this.initialize();
|
|
62
|
+
}
|
|
63
|
+
try {
|
|
64
|
+
const params = {
|
|
65
|
+
Bucket: this.bucketName,
|
|
66
|
+
Key: `${this.cachePrefix}/${candidateId}/${key}`,
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
const s3HeadCommand = new HeadObjectCommand(params);
|
|
70
|
+
const headData: HeadObjectCommandOutput = await this.s3Client.send(s3HeadCommand);
|
|
71
|
+
|
|
72
|
+
const expirationHeader = headData?.Expiration;
|
|
73
|
+
if (expirationHeader) {
|
|
74
|
+
const expirationDateMatch = expirationHeader.match(/expiry-date="([^"]+)"/);
|
|
75
|
+
if (expirationDateMatch) {
|
|
76
|
+
const expirationDate = new Date(expirationDateMatch[1]);
|
|
77
|
+
const currentDate = new Date();
|
|
78
|
+
|
|
79
|
+
if (currentDate > expirationDate) {
|
|
80
|
+
const s3DeleteCommand = new DeleteObjectCommand(params);
|
|
81
|
+
await this.s3Client.send(s3DeleteCommand);
|
|
82
|
+
console.log(`Object ${key} expired and deleted.`);
|
|
83
|
+
return null;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const s3GetCommand = new GetObjectCommand(params);
|
|
89
|
+
const objectData: GetObjectCommandOutput = await this.s3Client.send(s3GetCommand);
|
|
90
|
+
return objectData.Body.transformToString();
|
|
91
|
+
} catch (error) {
|
|
92
|
+
console.error(`Error reading object ${key}:`, error);
|
|
93
|
+
throw null;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
@SecureConnector.AccessControl
|
|
98
|
+
public async set(acRequest: AccessRequest, key: string, data: any, acl?: IACL, metadata?: CacheMetadata, ttl?: number): Promise<boolean> {
|
|
99
|
+
const accessCandidate = acRequest.candidate;
|
|
100
|
+
const candidateId = accessCandidate.id;
|
|
101
|
+
|
|
102
|
+
const newMetadata: CacheMetadata = metadata || {};
|
|
103
|
+
newMetadata['acl'] = ACL.from(acl).addAccess(accessCandidate.role, accessCandidate.id, TAccessLevel.Owner).ACL;
|
|
104
|
+
const serializedMetadata = this.serializeS3Metadata(newMetadata);
|
|
105
|
+
|
|
106
|
+
const s3PutCommandConfig: PutObjectCommandInput = {
|
|
107
|
+
Bucket: this.bucketName,
|
|
108
|
+
Key: `${this.cachePrefix}/${candidateId}/${key}`,
|
|
109
|
+
Body: data,
|
|
110
|
+
Metadata: serializedMetadata,
|
|
111
|
+
};
|
|
112
|
+
if (ttl) {
|
|
113
|
+
const expiryMetadata = generateExpiryMetadata(ttlToExpiryDays(ttl)); // seconds to days
|
|
114
|
+
s3PutCommandConfig.Tagging = `${expiryMetadata.Key}=${expiryMetadata.Value}`;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const s3PutCommand = new PutObjectCommand(s3PutCommandConfig);
|
|
118
|
+
await this.s3Client.send(s3PutCommand);
|
|
119
|
+
|
|
120
|
+
return true;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
@SecureConnector.AccessControl
|
|
124
|
+
public async delete(acRequest: AccessRequest, key: string): Promise<void> {
|
|
125
|
+
try {
|
|
126
|
+
const candidateId = acRequest.candidate.id;
|
|
127
|
+
const deleteCommand = new DeleteObjectCommand({ Bucket: this.bucketName, Key: `${this.cachePrefix}/${candidateId}/${key}` });
|
|
128
|
+
await this.s3Client.send(deleteCommand);
|
|
129
|
+
} catch (error) {
|
|
130
|
+
console.log(`Error deleting object ${key}:`, error);
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
@SecureConnector.AccessControl
|
|
136
|
+
public async exists(acRequest: AccessRequest, key: string): Promise<boolean> {
|
|
137
|
+
const candidateId = acRequest.candidate.id;
|
|
138
|
+
try {
|
|
139
|
+
const params = {
|
|
140
|
+
Bucket: this.bucketName,
|
|
141
|
+
Key: `${this.cachePrefix}/${candidateId}/${key}`,
|
|
142
|
+
};
|
|
143
|
+
const s3HeadCommand = new HeadObjectCommand(params);
|
|
144
|
+
const headData: HeadObjectCommandOutput = await this.s3Client.send(s3HeadCommand);
|
|
145
|
+
|
|
146
|
+
const expirationHeader = headData?.Expiration;
|
|
147
|
+
if (expirationHeader) {
|
|
148
|
+
const expirationDateMatch = expirationHeader.match(/expiry-date="([^"]+)"/);
|
|
149
|
+
if (expirationDateMatch) {
|
|
150
|
+
const expirationDate = new Date(expirationDateMatch[1]);
|
|
151
|
+
const currentDate = new Date();
|
|
152
|
+
|
|
153
|
+
if (currentDate > expirationDate) {
|
|
154
|
+
await this.delete(acRequest, key);
|
|
155
|
+
console.log(`Object ${key} expired and deleted.`);
|
|
156
|
+
return false;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
return true;
|
|
162
|
+
} catch (error) {
|
|
163
|
+
console.error(`Error reading object ${key}:`, error);
|
|
164
|
+
return false;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
@SecureConnector.AccessControl
|
|
169
|
+
public async getMetadata(acRequest: AccessRequest, key: string): Promise<CacheMetadata> {
|
|
170
|
+
const candidateId = acRequest.candidate.id;
|
|
171
|
+
|
|
172
|
+
try {
|
|
173
|
+
const s3Metadata = await this.getS3Metadata(`${this.cachePrefix}/${candidateId}/${key}`);
|
|
174
|
+
return s3Metadata as CacheMetadata;
|
|
175
|
+
} catch (error) {
|
|
176
|
+
console.error(`Error getting access rights in S3`, error.name, error.message);
|
|
177
|
+
throw error;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
@SecureConnector.AccessControl
|
|
182
|
+
public async setMetadata(acRequest: AccessRequest, key: string, metadata: CacheMetadata): Promise<void> {
|
|
183
|
+
const candidateId = acRequest.candidate.id;
|
|
184
|
+
|
|
185
|
+
try {
|
|
186
|
+
let s3Metadata = await this.getS3Metadata(`${this.cachePrefix}/${candidateId}/${key}`);
|
|
187
|
+
if (!s3Metadata) s3Metadata = {};
|
|
188
|
+
//s3Metadata['x-amz-meta-data'] = metadata;
|
|
189
|
+
s3Metadata = { ...s3Metadata, ...metadata };
|
|
190
|
+
await this.setS3Metadata(`${this.cachePrefix}/${candidateId}/${key}`, s3Metadata);
|
|
191
|
+
} catch (error) {
|
|
192
|
+
console.error(`Error setting access rights in S3`, error);
|
|
193
|
+
throw error;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
@SecureConnector.AccessControl
|
|
198
|
+
public async updateTTL(acRequest: AccessRequest, key: string, ttl?: number): Promise<void> {
|
|
199
|
+
if (ttl) {
|
|
200
|
+
const candidateId = acRequest.candidate.id;
|
|
201
|
+
const expiryMetadata = generateExpiryMetadata(ttlToExpiryDays(ttl)); // seconds to days
|
|
202
|
+
const s3PutObjectTaggingCommand = new PutObjectTaggingCommand({
|
|
203
|
+
Bucket: this.bucketName,
|
|
204
|
+
Key: `${this.cachePrefix}/${candidateId}/${key}`,
|
|
205
|
+
Tagging: { TagSet: [{ Key: expiryMetadata.Key, Value: expiryMetadata.Value }] },
|
|
206
|
+
});
|
|
207
|
+
await this.s3Client.send(s3PutObjectTaggingCommand);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
@SecureConnector.AccessControl
|
|
212
|
+
public async getTTL(acRequest: AccessRequest, key: string): Promise<number> {
|
|
213
|
+
const candidateId = acRequest.candidate.id;
|
|
214
|
+
const s3HeadCommand = new HeadObjectCommand({ Bucket: this.bucketName, Key: `${this.cachePrefix}/${candidateId}/${key}` });
|
|
215
|
+
const s3HeadObjectResponse: HeadObjectCommandOutput = await this.s3Client.send(s3HeadCommand);
|
|
216
|
+
const expirationHeader = s3HeadObjectResponse?.Expiration;
|
|
217
|
+
if (expirationHeader) {
|
|
218
|
+
const expirationDateMatch = expirationHeader.match(/expiry-date="([^"]+)"/);
|
|
219
|
+
if (expirationDateMatch) {
|
|
220
|
+
const expirationDate = new Date(expirationDateMatch[1]);
|
|
221
|
+
const currentDate = new Date();
|
|
222
|
+
const timeDifference = expirationDate.getTime() - currentDate.getTime();
|
|
223
|
+
return Math.floor(timeDifference / (1000 * 60 * 60 * 24)); // Convert to days
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
return -1; // Return -1 if no expiration date is found
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
public async getResourceACL(resourceId: string, candidate: IAccessCandidate): Promise<ACL> {
|
|
230
|
+
try {
|
|
231
|
+
const s3HeadCommand = new HeadObjectCommand({ Bucket: this.bucketName, Key: `${this.cachePrefix}/${candidate.id}/${resourceId}` });
|
|
232
|
+
const s3HeadObjectResponse: HeadObjectCommandOutput = await this.s3Client.send(s3HeadCommand);
|
|
233
|
+
|
|
234
|
+
const metadata = s3HeadObjectResponse.Metadata;
|
|
235
|
+
if (!metadata.acl) {
|
|
236
|
+
//the resource does not exist yet, we grant write access to the candidate in order to allow the resource creation
|
|
237
|
+
return new ACL().addAccess(candidate.role, candidate.id, TAccessLevel.Owner);
|
|
238
|
+
}
|
|
239
|
+
return ACL.from(metadata?.acl as string);
|
|
240
|
+
} catch (error) {
|
|
241
|
+
if (error.name === 'NotFound') {
|
|
242
|
+
//the resource does not exist yet, we grant write access to the candidate in order to allow the resource creation
|
|
243
|
+
return new ACL().addAccess(candidate.role, candidate.id, TAccessLevel.Owner);
|
|
244
|
+
}
|
|
245
|
+
throw error;
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
@SecureConnector.AccessControl
|
|
250
|
+
async getACL(acRequest: AccessRequest, key: string): Promise<IACL> {
|
|
251
|
+
try {
|
|
252
|
+
const metadata = await this.getMetadata(acRequest, key);
|
|
253
|
+
return (metadata?.acl as IACL) || {};
|
|
254
|
+
} catch (error) {
|
|
255
|
+
throw error;
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
@SecureConnector.AccessControl
|
|
260
|
+
async setACL(acRequest: AccessRequest, key: string, acl: IACL) {
|
|
261
|
+
try {
|
|
262
|
+
let metadata = await this.getMetadata(acRequest, key);
|
|
263
|
+
if (!metadata) metadata = {};
|
|
264
|
+
//when setting ACL make sure to not lose ownership
|
|
265
|
+
metadata.acl = ACL.from(acl).addAccess(acRequest.candidate.role, acRequest.candidate.id, TAccessLevel.Owner).ACL;
|
|
266
|
+
await this.setMetadata(acRequest, key, metadata);
|
|
267
|
+
} catch (error) {
|
|
268
|
+
console.error(`Error setting access rights in S3`, error);
|
|
269
|
+
throw error;
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
private async getS3Metadata(resourceId: string): Promise<Record<string, any> | undefined> {
|
|
274
|
+
try {
|
|
275
|
+
const command = new HeadObjectCommand({
|
|
276
|
+
Bucket: this.bucketName,
|
|
277
|
+
Key: resourceId,
|
|
278
|
+
});
|
|
279
|
+
const response: HeadObjectCommandOutput = await this.client.send(command);
|
|
280
|
+
const s3RawMetadata = response.Metadata;
|
|
281
|
+
if (!s3RawMetadata || Object.keys(s3RawMetadata).length === 0) return {};
|
|
282
|
+
|
|
283
|
+
let metadata: Record<string, any> = this.deserializeS3Metadata(s3RawMetadata);
|
|
284
|
+
|
|
285
|
+
if (!metadata['ContentType']) metadata['ContentType'] = response.ContentType ? response.ContentType : 'application/octet-stream';
|
|
286
|
+
return metadata;
|
|
287
|
+
} catch (error) {
|
|
288
|
+
if (error.name === 'NotFound' || error.name === 'NoSuchKey') {
|
|
289
|
+
return undefined;
|
|
290
|
+
}
|
|
291
|
+
console.error(`Error reading object metadata from S3`, error.name, error.message);
|
|
292
|
+
throw error;
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
private async setS3Metadata(resourceId: string, metadata: Record<string, any>): Promise<void> {
|
|
297
|
+
try {
|
|
298
|
+
// Get the current object content
|
|
299
|
+
const getObjectTaggingCommand = new GetObjectTaggingCommand({
|
|
300
|
+
Bucket: this.bucketName,
|
|
301
|
+
Key: resourceId,
|
|
302
|
+
});
|
|
303
|
+
const objectTagging: GetObjectTaggingCommandOutput = await this.client.send(getObjectTaggingCommand);
|
|
304
|
+
const serializedMetadata = this.serializeS3Metadata(metadata);
|
|
305
|
+
const copyObjectCommand = new CopyObjectCommand({
|
|
306
|
+
Bucket: this.bucketName,
|
|
307
|
+
CopySource: `${this.bucketName}/${resourceId}`,
|
|
308
|
+
Key: resourceId,
|
|
309
|
+
Metadata: serializedMetadata,
|
|
310
|
+
MetadataDirective: 'REPLACE',
|
|
311
|
+
Tagging: objectTagging.TagSet.map((tag) => `${tag.Key}=${tag.Value}`).join('&'),
|
|
312
|
+
});
|
|
313
|
+
|
|
314
|
+
await this.client.send(copyObjectCommand);
|
|
315
|
+
} catch (error) {
|
|
316
|
+
console.error(`Error setting object metadata in S3`, error.name, error.message);
|
|
317
|
+
throw error;
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
private async initialize() {
|
|
322
|
+
await checkAndInstallLifecycleRules(this.bucketName, this.s3Client);
|
|
323
|
+
this.isInitialized = true;
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
private serializeS3Metadata(s3Metadata: Record<string, any>): Record<string, string> {
|
|
327
|
+
let amzMetadata = {};
|
|
328
|
+
if (s3Metadata['acl']) {
|
|
329
|
+
amzMetadata['acl'] = typeof s3Metadata['acl'] == 'string' ? s3Metadata['acl'] : ACL.from(s3Metadata['acl']).serializedACL;
|
|
330
|
+
delete s3Metadata['acl'];
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
for (let key in s3Metadata) {
|
|
334
|
+
if (key == 'ContentType') continue; //skip ContentType as it can only be set when writing the object
|
|
335
|
+
amzMetadata[key] = typeof s3Metadata[key] === 'string' ? s3Metadata[key] : JSON.stringify(s3Metadata[key]);
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
return amzMetadata;
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
private deserializeS3Metadata(amzMetadata: Record<string, string>): Record<string, any> {
|
|
342
|
+
let metadata: Record<string, any> = {};
|
|
343
|
+
|
|
344
|
+
for (let key in amzMetadata) {
|
|
345
|
+
if (key === 'acl') {
|
|
346
|
+
metadata[key] = ACL.from(amzMetadata[key]).ACL;
|
|
347
|
+
continue;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
try {
|
|
351
|
+
metadata[key] = JSON.parse(amzMetadata[key]);
|
|
352
|
+
} catch (error) {
|
|
353
|
+
metadata[key] = amzMetadata[key];
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
return metadata;
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
// async hasAccess(request: IAccessRequest): Promise<boolean> {
|
|
361
|
+
// try {
|
|
362
|
+
// const metadata = await this.getMetadata(request.resourceId);
|
|
363
|
+
// const acl: IACL = metadata?.acl as IACL;
|
|
364
|
+
// return ACL.from(acl).checkExactAccess(request);
|
|
365
|
+
// } catch (error) {
|
|
366
|
+
// if (error.name === 'NotFound') {
|
|
367
|
+
// return false;
|
|
368
|
+
// }
|
|
369
|
+
// console.error(`Error checking access rights in S3`, error.name, error.message);
|
|
370
|
+
// throw error;
|
|
371
|
+
// }
|
|
372
|
+
// }
|
|
373
|
+
}
|