@smythos/sre 1.5.50 → 1.5.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG +98 -98
- package/LICENSE +18 -18
- package/README.md +135 -135
- package/dist/index.js +3 -3
- package/dist/index.js.map +1 -1
- package/dist/types/Components/APICall/AccessTokenManager.d.ts +3 -2
- package/dist/types/Components/APICall/OAuth.helper.d.ts +3 -2
- package/dist/types/subsystems/LLMManager/LLM.service/connectors/openai/OpenAIConnector.class.d.ts +6 -1
- package/dist/types/subsystems/LLMManager/ModelsProvider.service/connectors/SmythModelsProvider.class.d.ts +39 -0
- package/package.json +1 -1
- package/src/Components/APICall/APICall.class.ts +156 -156
- package/src/Components/APICall/AccessTokenManager.ts +166 -130
- package/src/Components/APICall/ArrayBufferResponse.helper.ts +58 -58
- package/src/Components/APICall/OAuth.helper.ts +446 -294
- package/src/Components/APICall/mimeTypeCategories.ts +46 -46
- package/src/Components/APICall/parseData.ts +167 -167
- package/src/Components/APICall/parseHeaders.ts +41 -41
- package/src/Components/APICall/parseProxy.ts +68 -68
- package/src/Components/APICall/parseUrl.ts +91 -91
- package/src/Components/APIEndpoint.class.ts +234 -234
- package/src/Components/APIOutput.class.ts +58 -58
- package/src/Components/AgentPlugin.class.ts +102 -102
- package/src/Components/Async.class.ts +155 -155
- package/src/Components/Await.class.ts +90 -90
- package/src/Components/Classifier.class.ts +158 -158
- package/src/Components/Component.class.ts +132 -132
- package/src/Components/ComponentHost.class.ts +38 -38
- package/src/Components/DataSourceCleaner.class.ts +92 -92
- package/src/Components/DataSourceIndexer.class.ts +181 -181
- package/src/Components/DataSourceLookup.class.ts +161 -161
- package/src/Components/ECMASandbox.class.ts +71 -71
- package/src/Components/FEncDec.class.ts +29 -29
- package/src/Components/FHash.class.ts +33 -33
- package/src/Components/FSign.class.ts +80 -80
- package/src/Components/FSleep.class.ts +25 -25
- package/src/Components/FTimestamp.class.ts +25 -25
- package/src/Components/FileStore.class.ts +78 -78
- package/src/Components/ForEach.class.ts +97 -97
- package/src/Components/GPTPlugin.class.ts +70 -70
- package/src/Components/GenAILLM.class.ts +586 -586
- package/src/Components/HuggingFace.class.ts +314 -314
- package/src/Components/Image/imageSettings.config.ts +70 -70
- package/src/Components/ImageGenerator.class.ts +502 -502
- package/src/Components/JSONFilter.class.ts +54 -54
- package/src/Components/LLMAssistant.class.ts +213 -213
- package/src/Components/LogicAND.class.ts +28 -28
- package/src/Components/LogicAtLeast.class.ts +85 -85
- package/src/Components/LogicAtMost.class.ts +86 -86
- package/src/Components/LogicOR.class.ts +29 -29
- package/src/Components/LogicXOR.class.ts +34 -34
- package/src/Components/MCPClient.class.ts +138 -138
- package/src/Components/MemoryDeleteKeyVal.class.ts +70 -70
- package/src/Components/MemoryReadKeyVal.class.ts +66 -66
- package/src/Components/MemoryWriteKeyVal.class.ts +62 -62
- package/src/Components/MemoryWriteObject.class.ts +97 -97
- package/src/Components/MultimodalLLM.class.ts +128 -128
- package/src/Components/OpenAPI.class.ts +72 -72
- package/src/Components/PromptGenerator.class.ts +122 -122
- package/src/Components/ScrapflyWebScrape.class.ts +159 -159
- package/src/Components/ServerlessCode.class.ts +123 -123
- package/src/Components/TavilyWebSearch.class.ts +98 -98
- package/src/Components/VisionLLM.class.ts +104 -104
- package/src/Components/ZapierAction.class.ts +127 -127
- package/src/Components/index.ts +97 -97
- package/src/Core/AgentProcess.helper.ts +240 -240
- package/src/Core/Connector.class.ts +123 -123
- package/src/Core/ConnectorsService.ts +197 -197
- package/src/Core/DummyConnector.ts +49 -49
- package/src/Core/HookService.ts +105 -105
- package/src/Core/SmythRuntime.class.ts +235 -235
- package/src/Core/SystemEvents.ts +16 -16
- package/src/Core/boot.ts +56 -56
- package/src/config.ts +15 -15
- package/src/constants.ts +126 -126
- package/src/data/hugging-face.params.json +579 -579
- package/src/helpers/AWSLambdaCode.helper.ts +590 -587
- package/src/helpers/BinaryInput.helper.ts +331 -331
- package/src/helpers/Conversation.helper.ts +1119 -1119
- package/src/helpers/ECMASandbox.helper.ts +54 -54
- package/src/helpers/JsonContent.helper.ts +97 -97
- package/src/helpers/LocalCache.helper.ts +97 -97
- package/src/helpers/Log.helper.ts +274 -274
- package/src/helpers/OpenApiParser.helper.ts +150 -150
- package/src/helpers/S3Cache.helper.ts +147 -147
- package/src/helpers/SmythURI.helper.ts +5 -5
- package/src/helpers/Sysconfig.helper.ts +77 -77
- package/src/helpers/TemplateString.helper.ts +243 -243
- package/src/helpers/TypeChecker.helper.ts +329 -329
- package/src/index.ts +3 -3
- package/src/index.ts.bak +3 -3
- package/src/subsystems/AgentManager/Agent.class.ts +1114 -1114
- package/src/subsystems/AgentManager/Agent.helper.ts +3 -3
- package/src/subsystems/AgentManager/AgentData.service/AgentDataConnector.ts +230 -230
- package/src/subsystems/AgentManager/AgentData.service/connectors/CLIAgentDataConnector.class.ts +66 -66
- package/src/subsystems/AgentManager/AgentData.service/connectors/LocalAgentDataConnector.class.ts +142 -142
- package/src/subsystems/AgentManager/AgentData.service/connectors/NullAgentData.class.ts +39 -39
- package/src/subsystems/AgentManager/AgentData.service/index.ts +18 -18
- package/src/subsystems/AgentManager/AgentLogger.class.ts +297 -297
- package/src/subsystems/AgentManager/AgentRequest.class.ts +51 -51
- package/src/subsystems/AgentManager/AgentRuntime.class.ts +559 -559
- package/src/subsystems/AgentManager/AgentSSE.class.ts +101 -101
- package/src/subsystems/AgentManager/AgentSettings.class.ts +52 -52
- package/src/subsystems/AgentManager/Component.service/ComponentConnector.ts +32 -32
- package/src/subsystems/AgentManager/Component.service/connectors/LocalComponentConnector.class.ts +60 -60
- package/src/subsystems/AgentManager/Component.service/index.ts +11 -11
- package/src/subsystems/AgentManager/EmbodimentSettings.class.ts +47 -47
- package/src/subsystems/AgentManager/ForkedAgent.class.ts +154 -154
- package/src/subsystems/AgentManager/OSResourceMonitor.ts +77 -77
- package/src/subsystems/ComputeManager/Code.service/CodeConnector.ts +98 -98
- package/src/subsystems/ComputeManager/Code.service/connectors/AWSLambdaCode.class.ts +172 -172
- package/src/subsystems/ComputeManager/Code.service/connectors/ECMASandbox.class.ts +131 -131
- package/src/subsystems/ComputeManager/Code.service/index.ts +13 -13
- package/src/subsystems/IO/CLI.service/CLIConnector.ts +47 -47
- package/src/subsystems/IO/CLI.service/index.ts +9 -9
- package/src/subsystems/IO/Log.service/LogConnector.ts +32 -32
- package/src/subsystems/IO/Log.service/connectors/ConsoleLog.class.ts +28 -28
- package/src/subsystems/IO/Log.service/index.ts +13 -13
- package/src/subsystems/IO/NKV.service/NKVConnector.ts +43 -43
- package/src/subsystems/IO/NKV.service/connectors/NKVLocalStorage.class.ts +234 -234
- package/src/subsystems/IO/NKV.service/connectors/NKVRAM.class.ts +204 -204
- package/src/subsystems/IO/NKV.service/connectors/NKVRedis.class.ts +182 -182
- package/src/subsystems/IO/NKV.service/index.ts +14 -14
- package/src/subsystems/IO/Router.service/RouterConnector.ts +21 -21
- package/src/subsystems/IO/Router.service/connectors/ExpressRouter.class.ts +48 -48
- package/src/subsystems/IO/Router.service/connectors/NullRouter.class.ts +40 -40
- package/src/subsystems/IO/Router.service/index.ts +11 -11
- package/src/subsystems/IO/Storage.service/SmythFS.class.ts +489 -489
- package/src/subsystems/IO/Storage.service/StorageConnector.ts +66 -66
- package/src/subsystems/IO/Storage.service/connectors/LocalStorage.class.ts +327 -327
- package/src/subsystems/IO/Storage.service/connectors/S3Storage.class.ts +482 -482
- package/src/subsystems/IO/Storage.service/index.ts +13 -13
- package/src/subsystems/IO/VectorDB.service/VectorDBConnector.ts +108 -108
- package/src/subsystems/IO/VectorDB.service/connectors/MilvusVectorDB.class.ts +454 -454
- package/src/subsystems/IO/VectorDB.service/connectors/PineconeVectorDB.class.ts +384 -384
- package/src/subsystems/IO/VectorDB.service/connectors/RAMVecrtorDB.class.ts +421 -421
- package/src/subsystems/IO/VectorDB.service/embed/BaseEmbedding.ts +107 -107
- package/src/subsystems/IO/VectorDB.service/embed/OpenAIEmbedding.ts +109 -109
- package/src/subsystems/IO/VectorDB.service/embed/index.ts +21 -21
- package/src/subsystems/IO/VectorDB.service/index.ts +14 -14
- package/src/subsystems/LLMManager/LLM.helper.ts +251 -251
- package/src/subsystems/LLMManager/LLM.inference.ts +339 -339
- package/src/subsystems/LLMManager/LLM.service/LLMConnector.ts +489 -489
- package/src/subsystems/LLMManager/LLM.service/LLMCredentials.helper.ts +171 -171
- package/src/subsystems/LLMManager/LLM.service/connectors/Anthropic.class.ts +659 -659
- package/src/subsystems/LLMManager/LLM.service/connectors/Bedrock.class.ts +400 -400
- package/src/subsystems/LLMManager/LLM.service/connectors/Echo.class.ts +77 -77
- package/src/subsystems/LLMManager/LLM.service/connectors/GoogleAI.class.ts +757 -757
- package/src/subsystems/LLMManager/LLM.service/connectors/Groq.class.ts +304 -304
- package/src/subsystems/LLMManager/LLM.service/connectors/Perplexity.class.ts +250 -250
- package/src/subsystems/LLMManager/LLM.service/connectors/VertexAI.class.ts +423 -423
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/OpenAIConnector.class.ts +488 -488
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/ChatCompletionsApiInterface.ts +524 -524
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/OpenAIApiInterface.ts +100 -100
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/OpenAIApiInterfaceFactory.ts +81 -81
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/ResponsesApiInterface.ts +1145 -1145
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/constants.ts +13 -13
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/index.ts +4 -4
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/apiInterfaces/utils.ts +11 -11
- package/src/subsystems/LLMManager/LLM.service/connectors/openai/types.ts +32 -32
- package/src/subsystems/LLMManager/LLM.service/connectors/xAI.class.ts +471 -471
- package/src/subsystems/LLMManager/LLM.service/index.ts +44 -44
- package/src/subsystems/LLMManager/ModelsProvider.service/ModelsProviderConnector.ts +300 -300
- package/src/subsystems/LLMManager/ModelsProvider.service/connectors/JSONModelsProvider.class.ts +252 -252
- package/src/subsystems/LLMManager/ModelsProvider.service/index.ts +11 -11
- package/src/subsystems/LLMManager/custom-models.ts +854 -854
- package/src/subsystems/LLMManager/models.ts +2540 -2540
- package/src/subsystems/LLMManager/paramMappings.ts +69 -69
- package/src/subsystems/MemoryManager/Cache.service/CacheConnector.ts +86 -86
- package/src/subsystems/MemoryManager/Cache.service/connectors/LocalStorageCache.class.ts +297 -297
- package/src/subsystems/MemoryManager/Cache.service/connectors/RAMCache.class.ts +201 -201
- package/src/subsystems/MemoryManager/Cache.service/connectors/RedisCache.class.ts +252 -252
- package/src/subsystems/MemoryManager/Cache.service/connectors/S3Cache.class.ts +373 -373
- package/src/subsystems/MemoryManager/Cache.service/index.ts +15 -15
- package/src/subsystems/MemoryManager/LLMCache.ts +72 -72
- package/src/subsystems/MemoryManager/LLMContext.ts +124 -124
- package/src/subsystems/MemoryManager/LLMMemory.service/LLMMemoryConnector.ts +26 -26
- package/src/subsystems/MemoryManager/RuntimeContext.ts +266 -266
- package/src/subsystems/Security/AccessControl/ACL.class.ts +208 -208
- package/src/subsystems/Security/AccessControl/AccessCandidate.class.ts +82 -82
- package/src/subsystems/Security/AccessControl/AccessRequest.class.ts +52 -52
- package/src/subsystems/Security/Account.service/AccountConnector.ts +44 -44
- package/src/subsystems/Security/Account.service/connectors/AWSAccount.class.ts +76 -76
- package/src/subsystems/Security/Account.service/connectors/DummyAccount.class.ts +130 -130
- package/src/subsystems/Security/Account.service/connectors/JSONFileAccount.class.ts +159 -159
- package/src/subsystems/Security/Account.service/index.ts +14 -14
- package/src/subsystems/Security/Credentials.helper.ts +62 -62
- package/src/subsystems/Security/ManagedVault.service/ManagedVaultConnector.ts +38 -38
- package/src/subsystems/Security/ManagedVault.service/connectors/NullManagedVault.class.ts +53 -53
- package/src/subsystems/Security/ManagedVault.service/connectors/SecretManagerManagedVault.ts +154 -154
- package/src/subsystems/Security/ManagedVault.service/index.ts +12 -12
- package/src/subsystems/Security/SecureConnector.class.ts +110 -110
- package/src/subsystems/Security/Vault.service/Vault.helper.ts +30 -30
- package/src/subsystems/Security/Vault.service/VaultConnector.ts +29 -29
- package/src/subsystems/Security/Vault.service/connectors/HashicorpVault.class.ts +46 -46
- package/src/subsystems/Security/Vault.service/connectors/JSONFileVault.class.ts +221 -221
- package/src/subsystems/Security/Vault.service/connectors/NullVault.class.ts +54 -54
- package/src/subsystems/Security/Vault.service/connectors/SecretsManager.class.ts +140 -140
- package/src/subsystems/Security/Vault.service/index.ts +12 -12
- package/src/types/ACL.types.ts +104 -104
- package/src/types/AWS.types.ts +10 -10
- package/src/types/Agent.types.ts +61 -61
- package/src/types/AgentLogger.types.ts +17 -17
- package/src/types/Cache.types.ts +1 -1
- package/src/types/Common.types.ts +2 -2
- package/src/types/LLM.types.ts +496 -496
- package/src/types/Redis.types.ts +8 -8
- package/src/types/SRE.types.ts +64 -64
- package/src/types/Security.types.ts +14 -14
- package/src/types/Storage.types.ts +5 -5
- package/src/types/VectorDB.types.ts +86 -86
- package/src/utils/base64.utils.ts +275 -275
- package/src/utils/cli.utils.ts +68 -68
- package/src/utils/data.utils.ts +322 -322
- package/src/utils/date-time.utils.ts +22 -22
- package/src/utils/general.utils.ts +238 -238
- package/src/utils/index.ts +12 -12
- package/src/utils/lazy-client.ts +261 -261
- package/src/utils/numbers.utils.ts +13 -13
- package/src/utils/oauth.utils.ts +35 -35
- package/src/utils/string.utils.ts +414 -414
- package/src/utils/url.utils.ts +19 -19
- package/src/utils/validation.utils.ts +74 -74
- package/dist/bundle-analysis-lazy.html +0 -4949
- package/dist/bundle-analysis.html +0 -4949
- package/dist/types/utils/package-manager.utils.d.ts +0 -26
|
@@ -1,150 +1,150 @@
|
|
|
1
|
-
import axios from 'axios';
|
|
2
|
-
import yaml from 'js-yaml';
|
|
3
|
-
|
|
4
|
-
import { REQUEST_METHODS } from '../constants';
|
|
5
|
-
|
|
6
|
-
// * We need a couple of packages to resolve references in the OpenAPI specification
|
|
7
|
-
//FIXME : YAML support temporarly disabled to allow executable packaging
|
|
8
|
-
//import SwaggerParser from '@apidevtools/swagger-parser';
|
|
9
|
-
//import $RefParser from '@apidevtools/json-schema-ref-parser';
|
|
10
|
-
//import { JSONSchema } from '@apidevtools/json-schema-ref-parser/dist/lib/types';
|
|
11
|
-
|
|
12
|
-
//const swaggerParser = new SwaggerParser();
|
|
13
|
-
|
|
14
|
-
type JSONSchema = Record<string, any>;
|
|
15
|
-
|
|
16
|
-
/**
|
|
17
|
-
* Utility function to resolve $ref values within a schema.
|
|
18
|
-
* @param schema The schema containing potential $ref values.
|
|
19
|
-
* @param root The root schema for reference resolution.
|
|
20
|
-
* @returns The dereferenced schema.
|
|
21
|
-
*/
|
|
22
|
-
const dereferenceSchema = async (schema: any, root: any = schema): Promise<any> => {
|
|
23
|
-
if (typeof schema !== 'object' || schema === null) {
|
|
24
|
-
return schema;
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
if (schema.$ref) {
|
|
28
|
-
const ref = schema.$ref;
|
|
29
|
-
const path = ref.replace(/^#\//, '').split('/'); // Remove the initial '#/' and split the path
|
|
30
|
-
let resolved = root;
|
|
31
|
-
for (const segment of path) {
|
|
32
|
-
if (resolved[segment] === undefined) {
|
|
33
|
-
throw new Error(`Could not resolve $ref: ${ref}`);
|
|
34
|
-
}
|
|
35
|
-
resolved = resolved[segment];
|
|
36
|
-
}
|
|
37
|
-
return dereferenceSchema(resolved, root); // Recursively dereference
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
// Recursively resolve all properties and items
|
|
41
|
-
if (Array.isArray(schema)) {
|
|
42
|
-
return Promise.all(schema.map((item) => dereferenceSchema(item, root)));
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
const result: any = {};
|
|
46
|
-
for (const key of Object.keys(schema)) {
|
|
47
|
-
result[key] = await dereferenceSchema(schema[key], root);
|
|
48
|
-
}
|
|
49
|
-
return result;
|
|
50
|
-
};
|
|
51
|
-
|
|
52
|
-
export class OpenAPIParser {
|
|
53
|
-
static mapReqMethods(paths: Array<Record<string, any>>): Map<string, any> {
|
|
54
|
-
const methods = new Map();
|
|
55
|
-
|
|
56
|
-
for (const path in paths) {
|
|
57
|
-
const pathData = paths[path];
|
|
58
|
-
|
|
59
|
-
for (const method in pathData) {
|
|
60
|
-
const data = pathData[method];
|
|
61
|
-
|
|
62
|
-
if (REQUEST_METHODS.includes(method.toUpperCase())) {
|
|
63
|
-
methods.set(data?.operationId, method);
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
return methods;
|
|
69
|
-
}
|
|
70
|
-
static mapEndpoints(paths: Array<Record<string, any>>): Map<string, any> {
|
|
71
|
-
const operationIds = new Map();
|
|
72
|
-
|
|
73
|
-
for (const path in paths) {
|
|
74
|
-
const pathData = paths[path];
|
|
75
|
-
|
|
76
|
-
// it's possible we have multiple methods for a single path
|
|
77
|
-
for (const method in pathData) {
|
|
78
|
-
const data = pathData[method];
|
|
79
|
-
|
|
80
|
-
if (REQUEST_METHODS.includes(method.toUpperCase())) {
|
|
81
|
-
operationIds.set(data?.operationId, path);
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
return operationIds;
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
// static async yamlToJson(yamlData: string): Promise<JSONSchema> {
|
|
90
|
-
// const data = yaml.load(yamlData);
|
|
91
|
-
// //FIXME : YAML support temporarly disabled to allow executable packaging
|
|
92
|
-
// const schema = await $RefParser.dereference(data);
|
|
93
|
-
// //const schema: any = {};
|
|
94
|
-
|
|
95
|
-
// return schema;
|
|
96
|
-
// }
|
|
97
|
-
|
|
98
|
-
static async yamlToJson(yamlData: string): Promise<JSONSchema> {
|
|
99
|
-
const data = yaml.load(yamlData);
|
|
100
|
-
const schema = await dereferenceSchema(data);
|
|
101
|
-
return schema;
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
// static async getJson(data: string | Record<string, any>): Promise<Record<string, any>> {
|
|
105
|
-
// try {
|
|
106
|
-
// let _data = data;
|
|
107
|
-
// if (typeof data === 'string') {
|
|
108
|
-
// _data = JSON.parse(_data as string);
|
|
109
|
-
// }
|
|
110
|
-
// //FIXME : YAML support temporarly disabled to allow executable packaging
|
|
111
|
-
// const result = swaggerParser.dereference(_data as any);
|
|
112
|
-
// //const result: any = {};
|
|
113
|
-
// return result;
|
|
114
|
-
// } catch (error) {
|
|
115
|
-
// try {
|
|
116
|
-
// return OpenAPIParser.yamlToJson(data as string);
|
|
117
|
-
// } catch (error) {
|
|
118
|
-
// throw new Error('Invalid OpenAPI specification data format');
|
|
119
|
-
// }
|
|
120
|
-
// }
|
|
121
|
-
// }
|
|
122
|
-
|
|
123
|
-
static async getJson(data: string | Record<string, any>): Promise<Record<string, any>> {
|
|
124
|
-
try {
|
|
125
|
-
let parsedData = data;
|
|
126
|
-
if (typeof data === 'string') {
|
|
127
|
-
parsedData = JSON.parse(data);
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
return await dereferenceSchema(parsedData);
|
|
131
|
-
} catch (error) {
|
|
132
|
-
try {
|
|
133
|
-
// Fallback to handle YAML data
|
|
134
|
-
return await OpenAPIParser.yamlToJson(data as string);
|
|
135
|
-
} catch (innerError) {
|
|
136
|
-
throw new Error('Invalid OpenAPI specification or JSON data format');
|
|
137
|
-
}
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
static async getJsonFromUrl(url: string): Promise<Record<string, any>> {
|
|
141
|
-
const response = await axios.get(url);
|
|
142
|
-
const data = response.data;
|
|
143
|
-
|
|
144
|
-
return OpenAPIParser.getJson(data);
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
static isValidOpenAPI(data: Record<string, any>): boolean {
|
|
148
|
-
return data?.openapi && data?.paths && data?.servers;
|
|
149
|
-
}
|
|
150
|
-
}
|
|
1
|
+
import axios from 'axios';
|
|
2
|
+
import yaml from 'js-yaml';
|
|
3
|
+
|
|
4
|
+
import { REQUEST_METHODS } from '../constants';
|
|
5
|
+
|
|
6
|
+
// * We need a couple of packages to resolve references in the OpenAPI specification
|
|
7
|
+
//FIXME : YAML support temporarly disabled to allow executable packaging
|
|
8
|
+
//import SwaggerParser from '@apidevtools/swagger-parser';
|
|
9
|
+
//import $RefParser from '@apidevtools/json-schema-ref-parser';
|
|
10
|
+
//import { JSONSchema } from '@apidevtools/json-schema-ref-parser/dist/lib/types';
|
|
11
|
+
|
|
12
|
+
//const swaggerParser = new SwaggerParser();
|
|
13
|
+
|
|
14
|
+
type JSONSchema = Record<string, any>;
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Utility function to resolve $ref values within a schema.
|
|
18
|
+
* @param schema The schema containing potential $ref values.
|
|
19
|
+
* @param root The root schema for reference resolution.
|
|
20
|
+
* @returns The dereferenced schema.
|
|
21
|
+
*/
|
|
22
|
+
const dereferenceSchema = async (schema: any, root: any = schema): Promise<any> => {
|
|
23
|
+
if (typeof schema !== 'object' || schema === null) {
|
|
24
|
+
return schema;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
if (schema.$ref) {
|
|
28
|
+
const ref = schema.$ref;
|
|
29
|
+
const path = ref.replace(/^#\//, '').split('/'); // Remove the initial '#/' and split the path
|
|
30
|
+
let resolved = root;
|
|
31
|
+
for (const segment of path) {
|
|
32
|
+
if (resolved[segment] === undefined) {
|
|
33
|
+
throw new Error(`Could not resolve $ref: ${ref}`);
|
|
34
|
+
}
|
|
35
|
+
resolved = resolved[segment];
|
|
36
|
+
}
|
|
37
|
+
return dereferenceSchema(resolved, root); // Recursively dereference
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// Recursively resolve all properties and items
|
|
41
|
+
if (Array.isArray(schema)) {
|
|
42
|
+
return Promise.all(schema.map((item) => dereferenceSchema(item, root)));
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const result: any = {};
|
|
46
|
+
for (const key of Object.keys(schema)) {
|
|
47
|
+
result[key] = await dereferenceSchema(schema[key], root);
|
|
48
|
+
}
|
|
49
|
+
return result;
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
export class OpenAPIParser {
|
|
53
|
+
static mapReqMethods(paths: Array<Record<string, any>>): Map<string, any> {
|
|
54
|
+
const methods = new Map();
|
|
55
|
+
|
|
56
|
+
for (const path in paths) {
|
|
57
|
+
const pathData = paths[path];
|
|
58
|
+
|
|
59
|
+
for (const method in pathData) {
|
|
60
|
+
const data = pathData[method];
|
|
61
|
+
|
|
62
|
+
if (REQUEST_METHODS.includes(method.toUpperCase())) {
|
|
63
|
+
methods.set(data?.operationId, method);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
return methods;
|
|
69
|
+
}
|
|
70
|
+
static mapEndpoints(paths: Array<Record<string, any>>): Map<string, any> {
|
|
71
|
+
const operationIds = new Map();
|
|
72
|
+
|
|
73
|
+
for (const path in paths) {
|
|
74
|
+
const pathData = paths[path];
|
|
75
|
+
|
|
76
|
+
// it's possible we have multiple methods for a single path
|
|
77
|
+
for (const method in pathData) {
|
|
78
|
+
const data = pathData[method];
|
|
79
|
+
|
|
80
|
+
if (REQUEST_METHODS.includes(method.toUpperCase())) {
|
|
81
|
+
operationIds.set(data?.operationId, path);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
return operationIds;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// static async yamlToJson(yamlData: string): Promise<JSONSchema> {
|
|
90
|
+
// const data = yaml.load(yamlData);
|
|
91
|
+
// //FIXME : YAML support temporarly disabled to allow executable packaging
|
|
92
|
+
// const schema = await $RefParser.dereference(data);
|
|
93
|
+
// //const schema: any = {};
|
|
94
|
+
|
|
95
|
+
// return schema;
|
|
96
|
+
// }
|
|
97
|
+
|
|
98
|
+
static async yamlToJson(yamlData: string): Promise<JSONSchema> {
|
|
99
|
+
const data = yaml.load(yamlData);
|
|
100
|
+
const schema = await dereferenceSchema(data);
|
|
101
|
+
return schema;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// static async getJson(data: string | Record<string, any>): Promise<Record<string, any>> {
|
|
105
|
+
// try {
|
|
106
|
+
// let _data = data;
|
|
107
|
+
// if (typeof data === 'string') {
|
|
108
|
+
// _data = JSON.parse(_data as string);
|
|
109
|
+
// }
|
|
110
|
+
// //FIXME : YAML support temporarly disabled to allow executable packaging
|
|
111
|
+
// const result = swaggerParser.dereference(_data as any);
|
|
112
|
+
// //const result: any = {};
|
|
113
|
+
// return result;
|
|
114
|
+
// } catch (error) {
|
|
115
|
+
// try {
|
|
116
|
+
// return OpenAPIParser.yamlToJson(data as string);
|
|
117
|
+
// } catch (error) {
|
|
118
|
+
// throw new Error('Invalid OpenAPI specification data format');
|
|
119
|
+
// }
|
|
120
|
+
// }
|
|
121
|
+
// }
|
|
122
|
+
|
|
123
|
+
static async getJson(data: string | Record<string, any>): Promise<Record<string, any>> {
|
|
124
|
+
try {
|
|
125
|
+
let parsedData = data;
|
|
126
|
+
if (typeof data === 'string') {
|
|
127
|
+
parsedData = JSON.parse(data);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
return await dereferenceSchema(parsedData);
|
|
131
|
+
} catch (error) {
|
|
132
|
+
try {
|
|
133
|
+
// Fallback to handle YAML data
|
|
134
|
+
return await OpenAPIParser.yamlToJson(data as string);
|
|
135
|
+
} catch (innerError) {
|
|
136
|
+
throw new Error('Invalid OpenAPI specification or JSON data format');
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
static async getJsonFromUrl(url: string): Promise<Record<string, any>> {
|
|
141
|
+
const response = await axios.get(url);
|
|
142
|
+
const data = response.data;
|
|
143
|
+
|
|
144
|
+
return OpenAPIParser.getJson(data);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
static isValidOpenAPI(data: Record<string, any>): boolean {
|
|
148
|
+
return data?.openapi && data?.paths && data?.servers;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
@@ -1,147 +1,147 @@
|
|
|
1
|
-
import { GetBucketLifecycleConfigurationCommandOutput, PutBucketLifecycleConfigurationCommand } from '@aws-sdk/client-s3';
|
|
2
|
-
|
|
3
|
-
import { GetBucketLifecycleConfigurationCommand } from '@aws-sdk/client-s3';
|
|
4
|
-
import { S3Client } from '@aws-sdk/client-s3';
|
|
5
|
-
import { Logger } from '@sre/helpers/Log.helper';
|
|
6
|
-
const console = Logger('S3Cache');
|
|
7
|
-
|
|
8
|
-
export function generateLifecycleRules() {
|
|
9
|
-
const rules = [];
|
|
10
|
-
|
|
11
|
-
// Add rules for 1-100 days
|
|
12
|
-
for (let i = 1; i < 100; i++) {
|
|
13
|
-
rules.push({
|
|
14
|
-
ID: `ExpireAfter${i}Days`,
|
|
15
|
-
Filter: {
|
|
16
|
-
// Prefix: '',
|
|
17
|
-
Tag: {
|
|
18
|
-
Key: 'Expiry',
|
|
19
|
-
Value: 'ExpireAfter' + i + 'Days',
|
|
20
|
-
},
|
|
21
|
-
},
|
|
22
|
-
Status: 'Enabled',
|
|
23
|
-
Expiration: { Days: i },
|
|
24
|
-
});
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
// Add rules for 110-1000 days with 10-day steps
|
|
28
|
-
for (let i = 100; i < 1000; i += 10) {
|
|
29
|
-
rules.push({
|
|
30
|
-
ID: `ExpireAfter${i}Days`,
|
|
31
|
-
Filter: {
|
|
32
|
-
// Prefix: '',
|
|
33
|
-
Tag: {
|
|
34
|
-
Key: 'Expiry',
|
|
35
|
-
Value: 'ExpireAfter' + i + 'Days',
|
|
36
|
-
},
|
|
37
|
-
},
|
|
38
|
-
Status: 'Enabled',
|
|
39
|
-
Expiration: { Days: i },
|
|
40
|
-
});
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
// Add rules for 1000-10000 days with 100-day steps
|
|
44
|
-
for (let i = 1000; i <= 10000; i += 100) {
|
|
45
|
-
rules.push({
|
|
46
|
-
ID: `ExpireAfter${i}Days`,
|
|
47
|
-
Filter: {
|
|
48
|
-
// Prefix: '',
|
|
49
|
-
Tag: {
|
|
50
|
-
Key: 'Expiry',
|
|
51
|
-
Value: 'ExpireAfter' + i + 'Days',
|
|
52
|
-
},
|
|
53
|
-
},
|
|
54
|
-
Status: 'Enabled',
|
|
55
|
-
Expiration: { Days: i },
|
|
56
|
-
});
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
return rules;
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
export function generateExpiryMetadata(expiryDays) {
|
|
63
|
-
let metadataValue;
|
|
64
|
-
|
|
65
|
-
if (expiryDays >= 1 && expiryDays < 100) {
|
|
66
|
-
metadataValue = `ExpireAfter${expiryDays}Days`;
|
|
67
|
-
} else if (expiryDays >= 100 && expiryDays < 1000) {
|
|
68
|
-
const roundedUpDays = Math.ceil(expiryDays / 10) * 10;
|
|
69
|
-
metadataValue = `ExpireAfter${roundedUpDays}Days`;
|
|
70
|
-
} else if (expiryDays >= 1000 && expiryDays <= 10000) {
|
|
71
|
-
const roundedUpDays = Math.ceil(expiryDays / 100) * 100;
|
|
72
|
-
metadataValue = `ExpireAfter${roundedUpDays}Days`;
|
|
73
|
-
} else {
|
|
74
|
-
throw new Error('Invalid expiry days. Please provide a valid expiry days value.');
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
return {
|
|
78
|
-
Key: 'Expiry',
|
|
79
|
-
Value: metadataValue,
|
|
80
|
-
};
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
export function getNonExistingRules(existingRules: any[], newRules: any[]) {
|
|
84
|
-
return newRules.filter((rule) => !existingRules.some((existingRule) => existingRule.ID === rule.ID));
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
export function ttlToExpiryDays(ttl: number) {
|
|
88
|
-
// seconds
|
|
89
|
-
return Math.ceil(ttl / (60 * 60 * 24));
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
export async function checkAndInstallLifecycleRules(bucketName: string, s3Client: S3Client) {
|
|
93
|
-
// Validate inputs
|
|
94
|
-
if (!bucketName || bucketName.trim() === '') {
|
|
95
|
-
throw new Error('Bucket name is required and cannot be empty');
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
if (!s3Client) {
|
|
99
|
-
throw new Error('S3Client is required');
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
console.log(`Checking lifecycle rules for bucket: ${bucketName}`);
|
|
103
|
-
|
|
104
|
-
try {
|
|
105
|
-
// Check existing lifecycle configuration
|
|
106
|
-
const getLifecycleCommand = new GetBucketLifecycleConfigurationCommand({ Bucket: bucketName });
|
|
107
|
-
const existingLifecycle: GetBucketLifecycleConfigurationCommandOutput = await s3Client.send(getLifecycleCommand);
|
|
108
|
-
const existingRules = existingLifecycle.Rules;
|
|
109
|
-
const newRules = generateLifecycleRules();
|
|
110
|
-
const nonExistingNewRules = getNonExistingRules(existingRules, newRules);
|
|
111
|
-
if (nonExistingNewRules.length > 0) {
|
|
112
|
-
const params = {
|
|
113
|
-
Bucket: bucketName,
|
|
114
|
-
LifecycleConfiguration: { Rules: [...existingRules, ...nonExistingNewRules] },
|
|
115
|
-
};
|
|
116
|
-
const putLifecycleCommand = new PutBucketLifecycleConfigurationCommand(params);
|
|
117
|
-
// Put the new lifecycle configuration
|
|
118
|
-
await s3Client.send(putLifecycleCommand);
|
|
119
|
-
console.log(`Added ${nonExistingNewRules.length} new lifecycle rules to bucket: ${bucketName}`);
|
|
120
|
-
} else {
|
|
121
|
-
console.log('Lifecycle configuration already exists');
|
|
122
|
-
}
|
|
123
|
-
} catch (error) {
|
|
124
|
-
if (error.code === 'NoSuchLifecycleConfiguration') {
|
|
125
|
-
console.log('No lifecycle configuration found. Creating new configuration...');
|
|
126
|
-
|
|
127
|
-
const lifecycleRules = generateLifecycleRules();
|
|
128
|
-
|
|
129
|
-
const params = {
|
|
130
|
-
Bucket: bucketName,
|
|
131
|
-
LifecycleConfiguration: { Rules: lifecycleRules },
|
|
132
|
-
};
|
|
133
|
-
const putLifecycleCommand = new PutBucketLifecycleConfigurationCommand(params);
|
|
134
|
-
// Put the new lifecycle configuration
|
|
135
|
-
await s3Client.send(putLifecycleCommand);
|
|
136
|
-
console.log('Lifecycle configuration created successfully.');
|
|
137
|
-
} else {
|
|
138
|
-
console.error('Error checking lifecycle configuration:', error);
|
|
139
|
-
console.error('Bucket name provided:', bucketName);
|
|
140
|
-
console.error('Error details:', {
|
|
141
|
-
name: error.name,
|
|
142
|
-
message: error.message,
|
|
143
|
-
code: error.code,
|
|
144
|
-
});
|
|
145
|
-
}
|
|
146
|
-
}
|
|
147
|
-
}
|
|
1
|
+
import { GetBucketLifecycleConfigurationCommandOutput, PutBucketLifecycleConfigurationCommand } from '@aws-sdk/client-s3';
|
|
2
|
+
|
|
3
|
+
import { GetBucketLifecycleConfigurationCommand } from '@aws-sdk/client-s3';
|
|
4
|
+
import { S3Client } from '@aws-sdk/client-s3';
|
|
5
|
+
import { Logger } from '@sre/helpers/Log.helper';
|
|
6
|
+
const console = Logger('S3Cache');
|
|
7
|
+
|
|
8
|
+
export function generateLifecycleRules() {
|
|
9
|
+
const rules = [];
|
|
10
|
+
|
|
11
|
+
// Add rules for 1-100 days
|
|
12
|
+
for (let i = 1; i < 100; i++) {
|
|
13
|
+
rules.push({
|
|
14
|
+
ID: `ExpireAfter${i}Days`,
|
|
15
|
+
Filter: {
|
|
16
|
+
// Prefix: '',
|
|
17
|
+
Tag: {
|
|
18
|
+
Key: 'Expiry',
|
|
19
|
+
Value: 'ExpireAfter' + i + 'Days',
|
|
20
|
+
},
|
|
21
|
+
},
|
|
22
|
+
Status: 'Enabled',
|
|
23
|
+
Expiration: { Days: i },
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// Add rules for 110-1000 days with 10-day steps
|
|
28
|
+
for (let i = 100; i < 1000; i += 10) {
|
|
29
|
+
rules.push({
|
|
30
|
+
ID: `ExpireAfter${i}Days`,
|
|
31
|
+
Filter: {
|
|
32
|
+
// Prefix: '',
|
|
33
|
+
Tag: {
|
|
34
|
+
Key: 'Expiry',
|
|
35
|
+
Value: 'ExpireAfter' + i + 'Days',
|
|
36
|
+
},
|
|
37
|
+
},
|
|
38
|
+
Status: 'Enabled',
|
|
39
|
+
Expiration: { Days: i },
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// Add rules for 1000-10000 days with 100-day steps
|
|
44
|
+
for (let i = 1000; i <= 10000; i += 100) {
|
|
45
|
+
rules.push({
|
|
46
|
+
ID: `ExpireAfter${i}Days`,
|
|
47
|
+
Filter: {
|
|
48
|
+
// Prefix: '',
|
|
49
|
+
Tag: {
|
|
50
|
+
Key: 'Expiry',
|
|
51
|
+
Value: 'ExpireAfter' + i + 'Days',
|
|
52
|
+
},
|
|
53
|
+
},
|
|
54
|
+
Status: 'Enabled',
|
|
55
|
+
Expiration: { Days: i },
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
return rules;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export function generateExpiryMetadata(expiryDays) {
|
|
63
|
+
let metadataValue;
|
|
64
|
+
|
|
65
|
+
if (expiryDays >= 1 && expiryDays < 100) {
|
|
66
|
+
metadataValue = `ExpireAfter${expiryDays}Days`;
|
|
67
|
+
} else if (expiryDays >= 100 && expiryDays < 1000) {
|
|
68
|
+
const roundedUpDays = Math.ceil(expiryDays / 10) * 10;
|
|
69
|
+
metadataValue = `ExpireAfter${roundedUpDays}Days`;
|
|
70
|
+
} else if (expiryDays >= 1000 && expiryDays <= 10000) {
|
|
71
|
+
const roundedUpDays = Math.ceil(expiryDays / 100) * 100;
|
|
72
|
+
metadataValue = `ExpireAfter${roundedUpDays}Days`;
|
|
73
|
+
} else {
|
|
74
|
+
throw new Error('Invalid expiry days. Please provide a valid expiry days value.');
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
return {
|
|
78
|
+
Key: 'Expiry',
|
|
79
|
+
Value: metadataValue,
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export function getNonExistingRules(existingRules: any[], newRules: any[]) {
|
|
84
|
+
return newRules.filter((rule) => !existingRules.some((existingRule) => existingRule.ID === rule.ID));
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export function ttlToExpiryDays(ttl: number) {
|
|
88
|
+
// seconds
|
|
89
|
+
return Math.ceil(ttl / (60 * 60 * 24));
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export async function checkAndInstallLifecycleRules(bucketName: string, s3Client: S3Client) {
|
|
93
|
+
// Validate inputs
|
|
94
|
+
if (!bucketName || bucketName.trim() === '') {
|
|
95
|
+
throw new Error('Bucket name is required and cannot be empty');
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
if (!s3Client) {
|
|
99
|
+
throw new Error('S3Client is required');
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
console.log(`Checking lifecycle rules for bucket: ${bucketName}`);
|
|
103
|
+
|
|
104
|
+
try {
|
|
105
|
+
// Check existing lifecycle configuration
|
|
106
|
+
const getLifecycleCommand = new GetBucketLifecycleConfigurationCommand({ Bucket: bucketName });
|
|
107
|
+
const existingLifecycle: GetBucketLifecycleConfigurationCommandOutput = await s3Client.send(getLifecycleCommand);
|
|
108
|
+
const existingRules = existingLifecycle.Rules;
|
|
109
|
+
const newRules = generateLifecycleRules();
|
|
110
|
+
const nonExistingNewRules = getNonExistingRules(existingRules, newRules);
|
|
111
|
+
if (nonExistingNewRules.length > 0) {
|
|
112
|
+
const params = {
|
|
113
|
+
Bucket: bucketName,
|
|
114
|
+
LifecycleConfiguration: { Rules: [...existingRules, ...nonExistingNewRules] },
|
|
115
|
+
};
|
|
116
|
+
const putLifecycleCommand = new PutBucketLifecycleConfigurationCommand(params);
|
|
117
|
+
// Put the new lifecycle configuration
|
|
118
|
+
await s3Client.send(putLifecycleCommand);
|
|
119
|
+
console.log(`Added ${nonExistingNewRules.length} new lifecycle rules to bucket: ${bucketName}`);
|
|
120
|
+
} else {
|
|
121
|
+
console.log('Lifecycle configuration already exists');
|
|
122
|
+
}
|
|
123
|
+
} catch (error) {
|
|
124
|
+
if (error.code === 'NoSuchLifecycleConfiguration') {
|
|
125
|
+
console.log('No lifecycle configuration found. Creating new configuration...');
|
|
126
|
+
|
|
127
|
+
const lifecycleRules = generateLifecycleRules();
|
|
128
|
+
|
|
129
|
+
const params = {
|
|
130
|
+
Bucket: bucketName,
|
|
131
|
+
LifecycleConfiguration: { Rules: lifecycleRules },
|
|
132
|
+
};
|
|
133
|
+
const putLifecycleCommand = new PutBucketLifecycleConfigurationCommand(params);
|
|
134
|
+
// Put the new lifecycle configuration
|
|
135
|
+
await s3Client.send(putLifecycleCommand);
|
|
136
|
+
console.log('Lifecycle configuration created successfully.');
|
|
137
|
+
} else {
|
|
138
|
+
console.error('Error checking lifecycle configuration:', error);
|
|
139
|
+
console.error('Bucket name provided:', bucketName);
|
|
140
|
+
console.error('Error details:', {
|
|
141
|
+
name: error.name,
|
|
142
|
+
message: error.message,
|
|
143
|
+
code: error.code,
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
export class SmythURI {
|
|
2
|
-
static agent(id: string): string {
|
|
3
|
-
return `smyth:agent:${id}`;
|
|
4
|
-
}
|
|
5
|
-
}
|
|
1
|
+
export class SmythURI {
|
|
2
|
+
static agent(id: string): string {
|
|
3
|
+
return `smyth:agent:${id}`;
|
|
4
|
+
}
|
|
5
|
+
}
|