@smythos/sre 1.5.0

package/dist/types/types/SRE.types.d.ts

@@ -0,0 +1,60 @@
+import { AgentDataService } from '@sre/AgentManager/AgentData.service';
+import { CLIService } from '@sre/IO/CLI.service';
+import { NKVService } from '@sre/IO/NKV.service';
+import { StorageService } from '@sre/IO/Storage.service';
+import { VectorDBService } from '@sre/IO/VectorDB.service';
+import { LLMService } from '@sre/LLMManager/LLM.service';
+import { CacheService } from '@sre/MemoryManager/Cache.service';
+import { AccountService } from '@sre/Security/Account.service';
+import { VaultService } from '@sre/Security/Vault.service';
+import { RouterService } from '@sre/IO/Router.service';
+import { ManagedVaultService } from '@sre/Security/ManagedVault.service';
+import { LogService } from '@sre/IO/Log.service';
+import { ComponentService } from '@sre/AgentManager/Component.service';
+import { ModelsProviderService } from '@sre/LLMManager/ModelsProvider.service';
+import { CodeService } from '@sre/ComputeManager/Code.service';
+export type TServiceRegistry = {
+    Storage?: StorageService;
+    VectorDB?: VectorDBService;
+    Cache?: CacheService;
+    LLM?: LLMService;
+    Vault?: VaultService;
+    Account?: AccountService;
+    AgentData?: AgentDataService;
+    CLI?: CLIService;
+    NKV?: NKVService;
+    Router?: RouterService;
+    ManagedVault?: ManagedVaultService;
+    Log?: LogService;
+    Component?: ComponentService;
+    ModelsProvider?: ModelsProviderService;
+    Code?: CodeService;
+};
+export declare enum TConnectorService {
+    Storage = "Storage",
+    VectorDB = "VectorDB",
+    Cache = "Cache",
+    LLM = "LLM",
+    Vault = "Vault",
+    Account = "Account",
+    AgentData = "AgentData",
+    CLI = "CLI",
+    NKV = "NKV",
+    Router = "Router",
+    ManagedVault = "ManagedVault",
+    Log = "Log",
+    Component = "Component",
+    ModelsProvider = "ModelsProvider",
+    Code = "Code"
+}
+export type SREConnectorConfig = {
+    Connector: string;
+    Id?: string;
+    Default?: boolean;
+    Settings?: {
+        [hashedOwnerKey: string]: any;
+    };
+};
+export type SREConfig = {
+    [key in TConnectorService]?: SREConnectorConfig[] | SREConnectorConfig | {};
+};

package/dist/types/types/Security.types.d.ts

@@ -0,0 +1,16 @@
+export type OAuthConfig = {
+    oAuthAppID: string;
+    oAuthAppSecret: string;
+    oAuthBaseUrl: string;
+    oAuthResource?: string;
+    oAuthScope?: string;
+};
+export type EncryptionSettings = {
+    encryption?: {
+        key?: string;
+        algorithm?: string;
+    };
+};
+export type SmythConfigs = {
+    smythAPIBaseUrl: string;
+};

package/dist/types/types/Storage.types.d.ts

@@ -0,0 +1,3 @@
+import { Readable } from 'stream';
+export type StorageMetadata = Record<string, any> | undefined;
+export type StorageData = string | Uint8Array | Buffer | Readable;

package/dist/types/types/VectorDB.types.d.ts

@@ -0,0 +1,66 @@
+export type VectorDBMetadata = ({
+    text?: string;
+    user?: string;
+} & Record<string, string>) | undefined;
+export type VectorsResultData = {
+    id: string;
+    score?: number;
+    values: number[];
+    metadata?: ({
+        text?: string;
+        user?: Record<string, string>;
+    } & Record<string, any>) | undefined;
+}[];
+export interface NsKnownMetadata {
+    isOnCustomStorage?: boolean;
+    [key: string]: any;
+}
+export interface QueryOptions {
+    topK?: number;
+    includeMetadata?: boolean;
+}
+export interface SourceTypes {
+    Vector: number[];
+    Text: string;
+}
+export type Source = SourceTypes[keyof SourceTypes];
+export interface IVectorDataSourceDto {
+    id: string;
+    source: number[] | string;
+    metadata?: VectorDBMetadata;
+}
+export interface IStorageVectorDataSource {
+    namespaceId: string;
+    candidateId: string;
+    candidateRole: string;
+    name: string;
+    metadata: string;
+    text: string;
+    vectorIds: string[];
+    id: string;
+}
+export interface IStorageVectorNamespace {
+    namespace: string;
+    displayName: string;
+    candidateId: string;
+    candidateRole: string;
+    metadata?: StorageVectorNamespaceMetadata;
+}
+export type StorageVectorNamespaceMetadata = Partial<PineconeNamespaceMetadata> & {
+    isOnCustomStorage?: boolean;
+} & {
+    [key: string]: any;
+};
+export interface PineconeNamespaceMetadata {
+    indexName: string;
+}
+export interface DatasourceDto {
+    text: string;
+    metadata?: Record<string, string> & {
+        smyth_metadata?: Record<string, string>;
+    };
+    chunkSize?: number;
+    chunkOverlap?: number;
+    label?: string;
+    id?: string;
+}

package/dist/types/utils/base64.utils.d.ts

@@ -0,0 +1,86 @@
+/**
+ * This function converts a text string to a base64 URL.
+ * @param text
+ * @returns
+ */
+export declare function textToBase64Url(text: any): string;
+export declare const isBase64FileUrl: (url: string) => boolean;
+export declare const getMimetypeFromBase64Data: (data: string) => Promise<"" | import("file-type").MimeType>;
+export declare function getBase64FileInfo(data: string): Promise<{
+    data: string;
+    mimetype: string;
+    size: number;
+} | null>;
+/**
+ * Checks if the input is a data URL.
+ *
+ * @param {string} input - The input string.
+ * @returns {boolean} True if the input is a data URL, false otherwise.
+ */
+export declare function isBase64DataUrl(input: string): boolean;
+/**
+ * Checks if the given string is a valid Base64-encoded string.
+ *
+ * @param {string} str - The string to check.
+ * @returns {boolean} True if the string is a valid Base64-encoded string, false otherwise.
+ */
+export declare function isBase64(str: string): boolean;
+/**
+ * Parses a Base64-encoded string and extracts the MIME type and cleaned data.
+ *
+ * @param {string} input - The Base64-encoded string.
+ * @returns {Promise<{ mimetype: string; data: string }>} An object containing the MIME type and the cleaned Base64 data.
+ */
+export declare function parseBase64(input: string): Promise<{
+    mimetype: string;
+    data: string;
+}>;
+/**
+ * Identifies the MIME type from a Base64-encoded string.
+ *
+ * This function cleans the input Base64 string, converts it to a buffer, and then identifies the MIME type
+ * using the `fileTypeFromBuffer` function.
+ *
+ * @param {string} data - The Base64-encoded string from which to identify the MIME type.
+ * @returns {Promise<string>} A promise that resolves to the MIME type of the data, or an empty string if the MIME type cannot be determined.
+ *
+ * @throws {Error} If an error occurs during the process, it logs the error and returns an empty string.
+ */
+export declare function identifyMimetypeFromBase64(data: string): Promise<string>;
+/**
+ * Identifies the MIME type from a Base64-encoded string.
+ *
+ * This function cleans the input Base64 string, converts it to a buffer, and then identifies the MIME type
+ * using the `fileTypeFromBuffer` function.
+ *
+ * @param {string} data - The Base64-encoded string from which to identify the MIME type.
+ * @returns {Promise<string>} A promise that resolves to the MIME type of the data, or an empty string if the MIME type cannot be determined.
+ *
+ * @throws {Error} If an error occurs during the process, it logs the error and returns an empty string.
+ */
+export declare function identifyMimeTypeFromBase64DataUrl(input: string): Promise<string>;
+/**
+ * Calculates the size of a Base64-encoded string in bytes.
+ *
+ * This function validates the input string to ensure it is a valid Base64-encoded string,
+ * converts it to a buffer, and then returns the byte length of the buffer.
+ *
+ * @param {string} str - The Base64-encoded string whose size is to be calculated.
+ * @returns {number} The size of the Base64-encoded string in bytes.
+ *
+ * @throws {Error} If the input string is not a valid Base64-encoded string or if an error occurs during conversion.
+ */
+export declare function getSizeOfBase64(str: string): number;
+/**
+ * Generates a Base64 Data URL from a Base64-encoded string.
+ *
+ * This function validates the input Base64 string, removes any newline characters,
+ * and constructs a Data URL with the specified MIME type.
+ *
+ * @param {string} data - The Base64-encoded string to be converted into a Data URL.
+ * @param {string} [mimetype='application/octet-stream'] - The MIME type of the data. Defaults to 'application/octet-stream'.
+ * @returns {string} The generated Base64 Data URL.
+ *
+ * @throws {Error} If the input string is not a valid Base64-encoded string.
+ */
+export declare function makeBase64Url(data: string, mimetype?: string): string;

package/dist/types/utils/cli.utils.d.ts

@@ -0,0 +1,20 @@
+/**
+ * This function parses the command line arguments and returns an object with the parsed values.
+ * The expected format is --file ./path/to/file.txt or --settings key1=value1 key2=value2
+ * Examples:
+ *  --file ./path/to/file.txt : calling parseCLIArgs('file', process.argv) will return {file: './path/to/file.txt'}
+ *  --settings key1=value1 key2=value2 : calling parseCLIArgs('settings', process.argv) will return {settings: {key1: 'value1', key2: 'value2'}}
+ *  it can also parse multiple arguments at once, for example:
+ *      parseCLIArgs(['file', 'settings'], process.argv) will return {file: './path/to/file.txt', settings: {key1: 'value1', key2: 'value2'}}
+ *
+ * @param argList the argument to parse
+ * @param argv the command line arguments, usually process.argv
+ * @returns parsed arguments object
+ */
+export declare function parseCLIArgs(argList: string | Array<string>, argv?: Array<string>): Record<string, any>;
+/**
+ * List all cli main arguments
+ * example : node index.js --file ./path/to/file.txt --settings key1=value1 key2=value2
+ * calling getMainArgs(process.argv) will return ['file', 'settings']
+ */
+export declare function getMainArgs(argv?: Array<string>): Array<string>;

package/dist/types/utils/data.utils.d.ts

@@ -0,0 +1,17 @@
+import { Readable } from 'stream';
+import { BinaryInput } from '@sre/helpers/BinaryInput.helper';
+export declare function streamToBuffer(stream: Readable): Promise<Buffer>;
+export declare function dataToBuffer(data: any): Buffer | null;
+export declare const getSizeFromBinary: (data: any) => number;
+export declare const isPlainObject: (data: any) => boolean;
+export declare const isBuffer: (data: any) => boolean;
+export declare const isBinaryMimeType: (mimetype: any) => boolean;
+export declare const isBinaryData: (data: any) => boolean;
+export declare function isUrl(str: string): boolean;
+export declare function isSmythFsUrl(str: string): boolean;
+export declare const isSmythFileObject: (data: any) => boolean;
+export declare const isBufferObject: (data: Record<string, any>) => boolean;
+export declare const isBase64Object: (data: Record<string, any>) => boolean;
+export declare function getMimeType(data: any): Promise<string>;
+export declare function formatDataForDebug(data: any): Promise<any>;
+export declare function normalizeImageInput(inputImage: string | BinaryInput): Promise<string>;

package/dist/types/utils/date-time.utils.d.ts

@@ -0,0 +1,3 @@
+export declare function getCurrentFormattedDate(): string;
+export declare function getDayFormattedDate(): string;
+export declare function delay(ms: any): Promise<unknown>;

package/dist/types/utils/general.utils.d.ts

@@ -0,0 +1,75 @@
+import { LogHelper } from '@sre/helpers/Log.helper';
+export declare function uid(): string;
+/**
+ * this function is used to check if a class is a subclass of another class
+ * @param subClass
+ * @param superClass
+ * @returns
+ */
+export declare function isSubclassOf(subClass: any, superClass: any): boolean;
+/**
+ * Processes an array of tasks concurrently with a specified concurrency limit.
+ *
+ * @template T - The type of the result returned by each task.
+ *
+ * @param {(() => Promise<T>)[]} tasks - An array of functions that return promises.
+ * Each function represents a task to be processed.
+ * @param {number} [maxConcurrentTasks=10] - The maximum number of concurrent tasks.
+ *
+ * @returns {Promise<T[]>} - A promise that resolves to an array of results.
+ * Only successfully fulfilled promises are included in the result array.
+ *
+ * @throws {TypeError} - Throws an error if the tasks parameter is not an array of functions.
+ *
+ * @example
+ * const tasks = [
+ *     () => await processFile('file1.txt'),
+ *     () => await processFile('file2.txt'),
+ *     () => await processFile('file3.txt'),
+ * ];
+ *
+ * const maxConcurrentTasks = 2;
+ *
+ * processWithConcurrencyLimit(tasks, maxConcurrentTasks)
+ *     .then(results => console.log(results)) // Array of results from the fulfilled promises
+ *     .catch(error => console.error(error));
+ *
+ * @note Currently, this function ignores tasks that fail to process.
+ *       Only successfully fulfilled promises are included in the result array.
+ *       To improve this behavior, we could add an option to control whether to exit the function if a task fails.
+ */
+export declare function processWithConcurrencyLimit<T>(tasks: (() => Promise<T>)[], maxConcurrentTasks?: number): Promise<T[]>;
+export declare const detectURLSourceType: (url: string) => "PDF" | "SITEMAP" | "WEBPAGE" | "WORD";
+/**
+ * This function checks if a string is a valid JSON string.
+ * @param str
+ * @returns
+ */
+export declare const isJSONString: (str: string) => boolean;
+export declare class ControlledPromise<T> extends Promise<T> {
+    private _isSettled;
+    readonly isSettled: () => boolean;
+    resolve: (value: T) => void;
+    reject: (reason?: any) => void;
+    constructor(executor: (resolve: (value: T) => void, reject: (reason?: any) => void, isSettled: () => boolean) => void);
+}
+/**
+ * This function ensures that a function is called at most once per wait time.
+ * @param func - The function to debounce
+ * @param wait - The wait time in milliseconds
+ * @param options - The options object : leading means the function will be called immediately if it's called within the wait time,
+ * trailing means the function will be called after the wait time has passed. maxWait is the maximum time to wait before calling the function.
+ * if maxWait is provided, the function will be called after the maxWait time has passed.
+ *
+ * @returns
+ */
+export declare function debounce(func: Function, wait: number, options: {
+    leading: boolean;
+    trailing: boolean;
+    maxWait?: number;
+}): (this: any, ...args: any[]) => void;
+/**
+ * Extracts and formats the last N calls from the stack trace in a user-friendly way
+ */
+export declare function getFormattedStackTrace(limit?: number, skip?: number): string[];
+export declare function printStackTrace(logger: LogHelper | Console, limit?: number, skip?: number): void;

package/dist/types/utils/index.d.ts

@@ -0,0 +1,9 @@
+export * from './general.utils';
+export * from './date-time.utils';
+export * from './data.utils';
+export * from './base64.utils';
+export * from './cli.utils';
+export * from './numbers.utils';
+export * from './string.utils';
+export * from './url.utils';
+export * from './validation.utils';

package/dist/types/utils/numbers.utils.d.ts

@@ -0,0 +1,2 @@
+export declare function isDigits(str: string): boolean;
+export declare function isSafeNumber(str: string): boolean;

package/dist/types/utils/oauth.utils.d.ts

@@ -0,0 +1,7 @@
+export declare function getM2MToken(configs: {
+    oauthAppId: string;
+    oauthAppSecret: string;
+    resource: string;
+    scope: string;
+    baseUrl: string;
+}): Promise<string>;

package/dist/types/utils/string.utils.d.ts

@@ -0,0 +1,28 @@
+/**
+ * perform a replace operation on a string asynchronously
+ * @param str
+ * @param regex
+ * @param asyncFn
+ * @returns
+ */
+export declare function asyncReplace(str: any, regex: any, asyncFn: any): Promise<string>;
+export declare function isValidString(str: string): boolean;
+/**
+ * The function parseJson() won't parse the data for property values.
+ * For instance, if you have '{"a": "1","b": "true"}', it will be parsed as {a: '1', b: 'true'}. That's why we parse the appropriate data type for property values
+ * so that the data will be parsed as {a: 1, b: true}
+ * @param data
+ * @returns
+ */
+export declare function convertStringToRespectiveType(data: any): any;
+export declare const kebabToCamel: (input: any) => any;
+export declare const kebabToCapitalize: (input: any) => any;
+/**
+ * This function reads a string and tries to identify the mimetype (e.g. text/plain, application/json, application/xml ...)
+ * @param input
+ */
+export declare const identifyMimetypeFromString: (input: string) => "application/json" | "text/plain" | "application/xml" | "text/html" | "text/css" | "text/csv" | "text/markdown" | "" | "image/svg+xml" | "application/javascript" | "application/yaml" | "application/sql";
+export declare function chunkText(text: string, { chunkSize, chunkOverlap, }?: {
+    chunkSize?: number;
+    chunkOverlap?: number;
+}): string[];

package/dist/types/utils/url.utils.d.ts

@@ -0,0 +1 @@
+export declare function getMimeTypeFromUrl(url: string): Promise<string>;

package/dist/types/utils/validation.utils.d.ts

@@ -0,0 +1,25 @@
+interface RangeValidationArgs {
+    min?: number;
+    max?: number;
+}
+/**
+ * Custom validation function to check if a string contains only specified characters.
+ * @param {string} value - The string to validate.
+ * @returns {string} - The validated string.
+ */
+export declare function validateCharacterSet(value: string): boolean;
+/**
+ * Validates whether a given string value can be converted to an integer that falls within a specified range.
+ * This function is designed to be used as a custom validator in Joi schemas.
+ *
+ * @param {RangeValidationArgs} args - An object containing optional 'min' and 'max' properties to define the range.
+ * @returns {Function} A function that takes a string value and a Joi helper object, and performs the validation.
+ *
+ * The validation function first converts the string value to a number. It then checks if the number is within the
+ * specified range (if provided). If the value is not a number or falls outside the range, it throws an error with a
+ * descriptive message.
+ *
+ * The error message includes the field name for clarity, using the label from the Joi schema if available.
+ */
+export declare const validateInteger: (args: RangeValidationArgs) => (value: string, helpers: any) => string;
+export {};

package/package.json

@@ -0,0 +1,117 @@
+{
+    "name": "@smythos/sre",
+    "version": "1.5.0",
+    "description": "Smyth Runtime Environment",
+    "author": "Alaa-eddine KADDOURI",
+    "license": "MIT",
+    "main": "dist/index.js",
+    "types": "dist/types/index.d.ts",
+    "exports": {
+        ".": {
+            "import": "./dist/index.js",
+            "types": "./dist/types/index.d.ts"
+        }
+    },
+    "files": [
+        "dist"
+    ],
+    "type": "module",
+    "scripts": {
+        "gen:barrel": "ctix build",
+        "build:types": "tsc --emitDeclarationOnly --declaration --outDir dist/types -p tsconfig.dts.json",
+        "build:jsbundle": "cross-env rollup -c",
+        "build": "pnpm run build:jsbundle && pnpm run build:types",
+        "doc:gen": "typedoc",
+        "doc:graphgen": "npx depcruise src --config ./doc/.dep-minimal.json --output-type dot > ./doc/dep-graph.dot && dot -Tpng ./doc/dep-graph.dot -o ./doc/dep-graph.png",
+        "knip": "knip"
+    },
+    "devDependencies": {
+        "@faker-js/faker": "^8.4.1",
+        "@istanbuljs/nyc-config-typescript": "^1.0.2",
+        "@rollup/plugin-json": "^6.1.0",
+        "@rollup/plugin-node-resolve": "^15.2.3",
+        "@rollup/pluginutils": "^5.1.0",
+        "@types/cookie-parser": "^1.4.7",
+        "@types/express": "^4.17.23",
+        "@types/express-session": "^1.18.0",
+        "@types/lodash": "^4.17.10",
+        "@types/node": "^20.19.0",
+        "cross-env": "^7.0.3",
+        "ctix": "^2.7.1",
+        "dependency-cruiser": "^16.3.3",
+        "esbuild": "^0.25.0",
+        "knip": "^5.23.1",
+        "nyc": "^17.0.0",
+        "pkg": "^5.8.1",
+        "rollup-plugin-copy": "^3.5.0",
+        "rollup-plugin-esbuild": "^6.1.1",
+        "rollup-plugin-sourcemaps": "^0.6.3",
+        "rollup-plugin-terser": "^7.0.2",
+        "rollup-plugin-typescript-paths": "^1.5.0",
+        "rollup-plugin-typescript2": "^0.36.0",
+        "source-map-support": "^0.5.21",
+        "typedoc": "^0.28.5",
+        "typedoc-github-theme": "^0.3.0",
+        "typescript": "^5.4.5"
+    },
+    "dependencies": {
+        "@anthropic-ai/sdk": "^0.39.0",
+        "@apidevtools/json-schema-ref-parser": "^11.6.4",
+        "@aws-sdk/client-bedrock-runtime": "^3.826.0",
+        "@aws-sdk/client-iam": "^3.826.0",
+        "@aws-sdk/client-lambda": "^3.826.0",
+        "@aws-sdk/client-s3": "^3.826.0",
+        "@aws-sdk/client-secrets-manager": "^3.826.0",
+        "@google-cloud/vertexai": "^1.7.0",
+        "@google/generative-ai": "^0.14.1",
+        "@huggingface/inference": "^2.8.0",
+        "@modelcontextprotocol/sdk": "^1.10.1",
+        "@pinecone-database/pinecone": "^3.0.0",
+        "@runware/sdk-js": "^1.1.36",
+        "@smithy/smithy-client": "^4.4.3",
+        "@zilliz/milvus2-sdk-node": "^2.5.11",
+        "acorn": "^8.14.1",
+        "axios": "^1.7.2",
+        "chokidar": "^4.0.3",
+        "commander": "^11.1.0",
+        "cookie-parser": "^1.4.7",
+        "cors": "^2.8.5",
+        "dayjs": "^1.11.11",
+        "dotenv": "^16.4.5",
+        "eventsource": "^3.0.2",
+        "express": "^4.21.2",
+        "express-async-handler": "^1.2.0",
+        "express-session": "^1.18.1",
+        "file-type": "^19.0.0",
+        "form-data": "^4.0.3",
+        "gpt-tokenizer": "^2.2.1",
+        "groq-sdk": "^0.6.1",
+        "image-size": "^1.1.1",
+        "ioredis": "^5.4.1",
+        "isbinaryfile": "^5.0.2",
+        "joi": "^17.13.1",
+        "js-yaml": "^4.1.0",
+        "jsonrepair": "^3.8.0",
+        "jsonwebtoken": "^9.0.2",
+        "jwks-rsa": "^3.1.0",
+        "lodash": "^4.17.21",
+        "memorystore": "^1.6.7",
+        "mime": "^4.0.3",
+        "multer": "^1.4.5-lts.1",
+        "mysql2": "^3.11.3",
+        "oauth-1.0a": "^2.2.6",
+        "openai": "^4.103.0",
+        "p-limit": "^6.1.0",
+        "pkce-challenge": "^5.0.0",
+        "qs": "^6.13.0",
+        "rate-limiter-flexible": "^5.0.3",
+        "readline-sync": "^1.4.10",
+        "socket.io-client": "^4.8.1",
+        "socks-proxy-agent": "^8.0.4",
+        "uuid": "^10.0.0",
+        "winston": "^3.13.0",
+        "winston-transport": "^4.7.0",
+        "xxhashjs": "^0.2.2",
+        "zip-lib": "^1.0.5"
+    }
+}