@smythos/sre 1.5.1 → 1.5.4

package/src/helpers/JsonContent.helper.ts

@@ -0,0 +1,97 @@
+import { jsonrepair } from 'jsonrepair';
+import { isDigits, isSafeNumber, isValidString } from '@sre/utils';
+
+export class JSONContentHelper {
+    private _current: string;
+
+    public get result() {
+        return this._current;
+    }
+    private constructor(private dataString: string) {
+        this._current = dataString;
+    }
+
+    public static create(dataString: string) {
+        return new JSONContentHelper(dataString);
+    }
+
+    /**
+     * This a permissive json parsing function : It tries to extract and parse a JSON object from a string. If it fails, it returns the original string.
+     * if the string is not a JSON representation, but contains a JSON object, it will extract and parse it.
+     * @returns
+     */
+    public tryParse() {
+        const strInput = this._current;
+
+        // is it an object ? a digit ? a safe number ?
+        if (!isValidString(strInput) || isDigits(strInput) || isSafeNumber(strInput)) return strInput;
+
+        let str = strInput.trim();
+
+        // the string seems to be a json object
+        if ((str.startsWith('{') && str.endsWith('}')) || (str.startsWith('[') && str.endsWith(']'))) {
+            try {
+                return JSON.parse(str);
+            } catch (e) {
+                try {
+                    const repairedJson = jsonrepair(str);
+                    return JSON.parse(repairedJson);
+                } catch (e: any) {}
+            }
+        }
+
+        // the string does not seem to be a json object, so we try to extract a json object from it
+        str = (this.extractJsonFromString(strInput) || strInput).trim();
+
+        if ((isDigits(str) && !isSafeNumber(str)) || (!str.startsWith('{') && !str.startsWith('['))) return str;
+
+        try {
+            return JSON.parse(str);
+        } catch (e) {
+            try {
+                const repairedJson = jsonrepair(str);
+                return JSON.parse(repairedJson);
+            } catch (e: any) {
+                //console.warn('Error on parseJson: ', e.toString());
+                //console.warn('   Tried to parse: ', str);
+                return strInput;
+            }
+        }
+    }
+
+    // Same as tryParse but it does not extract JSON from string
+    public tryFullParse() {
+        const str = this._current;
+        if (!str) return str;
+
+        if ((isDigits(str) && !isSafeNumber(str)) || (!str.startsWith('{') && !str.startsWith('['))) return str;
+
+        try {
+            return JSON.parse(str);
+        } catch (e) {
+            try {
+                return JSON.parse(jsonrepair(str));
+            } catch (e: any) {
+                console.warn('Error on parseJson: ', e.toString());
+                console.warn('   Tried to parse: ', str);
+                return { result: str, error: e.toString() };
+            }
+        }
+    }
+
+    private extractJsonFromString(str) {
+        try {
+            const regex = /(\{.*\})/s; // LLMs in smythOS are expected to generate json between curly brackets only
+
+            const match = str.match(regex);
+
+            return match?.[1];
+        } catch {
+            return null;
+        }
+    }
+}
+
+export function JSONContent(dataString: string) {
+    return JSONContentHelper.create(dataString);
+}

package/src/helpers/LocalCache.helper.ts

@@ -0,0 +1,97 @@
+interface CacheItem<T> {
+    value: T;
+    expiry: number;
+}
+
+export class LocalCache<K, V> {
+    private cache: Map<K, V>;
+    private expiryMap: Map<K, number>;
+    private timeouts: Map<K, NodeJS.Timeout>;
+    private defaultTTL: number = 60 * 60 * 1000;
+
+    constructor(defaultTTL: number = 60 * 60 * 1000) {
+        this.defaultTTL = defaultTTL;
+        this.cache = new Map<K, V>();
+        this.expiryMap = new Map<K, number>();
+        this.timeouts = new Map<K, NodeJS.Timeout>();
+    }
+
+    set(key: K, value: V, ttlMs: number = this.defaultTTL): void {
+        this.cache.set(key, value);
+        const expiry = Date.now() + ttlMs;
+        this.expiryMap.set(key, expiry);
+
+        // Clear any existing timeout for this key
+        this.clearTimeout(key);
+
+        // Set a new timeout to remove the item when it expires
+        const timeout = setTimeout(() => {
+            this.delete(key);
+        }, ttlMs);
+        this.timeouts.set(key, timeout);
+        timeout.unref(); //unblock the event loop
+    }
+
+    updateTTL(key: K, ttlMs: number = this.defaultTTL): void {
+        if (!this.has(key)) {
+            return;
+        }
+        const expiry = Date.now() + ttlMs;
+        this.expiryMap.set(key, expiry);
+
+        // Clear existing timeout and set a new one
+        this.clearTimeout(key);
+        const timeout = setTimeout(() => {
+            this.delete(key);
+        }, ttlMs);
+        this.timeouts.set(key, timeout);
+        timeout.unref(); //unblock the event loop
+    }
+
+    get(key: K, ttlMs?: number): V | undefined {
+        if (!this.has(key)) {
+            return undefined;
+        }
+        const value = this.cache.get(key);
+        if (value === undefined) {
+            return undefined;
+        }
+        this.updateTTL(key, ttlMs);
+        return value;
+    }
+
+    has(key: K): boolean {
+        if (!this.cache.has(key)) {
+            return false;
+        }
+        const expiry = this.expiryMap.get(key);
+        if (expiry && Date.now() > expiry) {
+            this.delete(key);
+            return false;
+        }
+        return true;
+    }
+
+    delete(key: K): boolean {
+        this.clearTimeout(key);
+        this.expiryMap.delete(key);
+        return this.cache.delete(key);
+    }
+
+    clear(): void {
+        for (const key of this.cache.keys()) {
+            this.clearTimeout(key);
+        }
+        this.cache.clear();
+        this.expiryMap.clear();
+        this.timeouts.clear();
+    }
+
+    private clearTimeout(key: K): void {
+        const timeout = this.timeouts.get(key);
+        if (timeout) {
+            clearTimeout(timeout);
+            this.timeouts.delete(key);
+        }
+    }
+}

package/src/helpers/Log.helper.ts

@@ -0,0 +1,234 @@
+import 'dotenv/config';
+import winston from 'winston';
+import Transport from 'winston-transport';
+import { getFormattedStackTrace, parseCLIArgs } from '../utils';
+import config from '@sre/config';
+import { EventEmitter } from 'events';
+winston.addColors({
+    error: 'red',
+    warn: 'yellow',
+    info: 'green',
+    debug: 'blue',
+});
+
+const logLevelMap = {
+    min: 'info',
+    full: 'debug',
+};
+
+let logLevel = () => {
+    let val = parseCLIArgs('debug')?.debug || config?.env?.LOG_LEVEL || 'none';
+    if (logLevelMap[val]) val = logLevelMap[val];
+    return !['none', 'error', 'warn', 'info', 'debug'].includes(val) ? 'none' : val;
+};
+
+// Retrieve the DEBUG environment variable and split it into an array of namespaces
+const namespaces = (config.env.LOG_FILTER || '').split(',');
+
+// Create a Winston format that filters messages based on namespaces
+const namespaceFilter = winston.format((info: any) => {
+    // If DEBUG is not set, log everything
+    if (!config.env.LOG_FILTER || namespaces.some((ns) => info.module?.includes(ns))) {
+        return info;
+    }
+    return false; // Filter out messages that do not match the namespace
+})();
+
+// Custom stream for your transport
+class ArrayTransport extends Transport {
+    private logs: any[];
+    constructor(opts) {
+        super(opts);
+        // Configure your storage array
+        this.logs = opts.logs;
+    }
+
+    log(info, callback) {
+        setImmediate(() => {
+            this.emit('logged', info);
+        });
+
+        // Perform the writing to the array storage
+        this.logs.push(`${info.level}: ${info.message}`);
+
+        // Perform the writing to the remote service
+        callback();
+    }
+}
+
+export class LogHelper extends EventEmitter {
+    public startTime = Date.now();
+    public get output() {
+        return Array.isArray(this.data) ? this.data.join('\n') : undefined;
+    }
+    public get elapsedTime() {
+        return Date.now() - this.startTime;
+    }
+    constructor(
+        private _logger: winston.Logger,
+        public data,
+        private labels: { [key: string]: any },
+    ) {
+        super();
+    }
+
+    public log(...args) {
+        this._logger.log('info', formatLogMessage(...args), this.labels);
+        this.emit('logged', { level: 'info', message: formatLogMessage(...args) });
+    }
+    public warn(...args) {
+        this._logger.log('warn', formatLogMessage(...args), this.labels);
+        this.emit('logged', { level: 'warn', message: formatLogMessage(...args) });
+    }
+    public debug(...args) {
+        this._logger.log('debug', formatLogMessage(...args), this.labels);
+        this.emit('logged', { level: 'debug', message: formatLogMessage(...args) });
+    }
+    public info(...args) {
+        this._logger.log('info', formatLogMessage(...args), this.labels);
+        this.emit('logged', { level: 'info', message: formatLogMessage(...args) });
+    }
+    public verbose(...args) {
+        this._logger.log('verbose', formatLogMessage(...args), this.labels);
+        this.emit('logged', { level: 'verbose', message: formatLogMessage(...args) });
+    }
+
+    public error(...args) {
+        const stack = '\nCall Stack:\n' + getFormattedStackTrace(10).join('\n');
+
+        this._logger.log('error', formatLogMessage(...args), { ...this.labels, stack });
+
+        this.emit('logged', { level: 'error', message: formatLogMessage(...args) });
+    }
+
+    public close() {
+        this._logger.clear();
+        this._logger.close();
+    }
+}
+
+const colorizedFormat = winston.format.printf((info) => {
+    return `${info.timestamp} ${winston.format.colorize().colorize(info.level, `${info.level}: ${info.message}`)}`;
+});
+
+const MAX_LOG_MESSAGE_LENGTH = 500;
+
+function redactLogMessage(logMessage: string) {
+    if (config.env.NODE_ENV !== 'PROD') return logMessage; //only redact logs in PROD
+    if (logMessage.length > 500) {
+        return logMessage;
+    }
+
+    const sensitiveWords = ['password', 'eyJ', 'token', 'email', 'secret', 'key', 'apikey', 'api_key', 'auth', 'credential'];
+    const obfuscatedString = ' [!! SmythOS::REDACTED_DATA !!] ';
+
+    // Iterate through the sensitive words list and replace sensitive data in the log message
+
+    for (const sensitiveWord of sensitiveWords) {
+        // Create a regular expression to find the sensitive word followed by any character (non-greedy) until a space, newline, or separator is found.
+        const regex = new RegExp(`(${sensitiveWord})((?:[^\\n]{0,29}(?=\\n))|(?:[^\\n]{30}\\S*))`, 'gmi');
+
+        // Replace sensitive data with the obfuscated string
+        logMessage = logMessage.replace(regex, `$1${obfuscatedString}`);
+    }
+
+    return logMessage;
+}
+
+function createBaseLogger(memoryStore?: any[]) {
+    const logger = winston.createLogger({
+        //level: 'info', // log level
+
+        format: winston.format.combine(
+            winston.format((info: any) => {
+                if (config.env.LOG_LEVEL == 'none' || logLevel() == 'none' || logLevel() == '') return false; // skip logging if log level is none
+
+                // Apply redaction to the log message
+                //info.message = redactSecrets(info.message, sensitiveOptions);
+
+                info.message = redactLogMessage(info.message);
+                return info;
+            })(),
+            winston.format.timestamp(),
+            winston.format.errors({
+                stack: true,
+            }),
+            winston.format.splat(),
+            winston.format.json(),
+        ),
+
+        transports: [
+            new winston.transports.Console({
+                level: 'error',
+                //handleExceptions: true,
+                format: winston.format.combine(
+                    winston.format.printf((info: any) => {
+                        let message = info.message;
+                        message = message?.length > MAX_LOG_MESSAGE_LENGTH ? message.substring(0, MAX_LOG_MESSAGE_LENGTH) + '...' : message;
+                        return `${info.level}:${info.module || ''} ${message} ${info.stack || ''}`;
+                    }),
+                ),
+                stderrLevels: ['error'], // Define levels that should be logged to stderr
+            }),
+            new winston.transports.Console({
+                level: logLevel(),
+                format: winston.format.combine(
+                    namespaceFilter,
+                    winston.format.printf((info: any) => {
+                        const module = info.module ? winston.format.colorize().colorize(info.level, ` [${info.module}]`) : '';
+                        const ns = winston.format.colorize().colorize(info.level, `${info.level}${module}`);
+
+                        let message = info.message;
+                        message = message?.length > MAX_LOG_MESSAGE_LENGTH ? message.substring(0, MAX_LOG_MESSAGE_LENGTH) + '...' : message;
+
+                        return `${ns} - ${message}`;
+                    }),
+                ),
+
+                //handleExceptions: true,
+            }),
+        ],
+    });
+
+    if (Array.isArray(memoryStore)) {
+        logger.add(
+            new ArrayTransport({
+                level: 'debug',
+                logs: memoryStore,
+            }),
+        );
+    }
+
+    return logger;
+}
+
+function formatLogMessage(...args) {
+    return args
+        .map((arg) => {
+            // If the argument is an object (and not null), serialize it to JSON
+            if (typeof arg === 'object' && arg !== null && !(arg instanceof Error)) {
+                try {
+                    return JSON.stringify(arg, null, 2); // set the space to 2 for better readability
+                } catch (error) {
+                    return String(arg);
+                }
+            }
+            // Otherwise, just convert it to a string in case it's not
+            return String(arg);
+        })
+        .join(' '); // Concatenate all arguments with a space
+}
+
+function createLabeledLogger(labels: { [key: string]: any }, memoryStore?: any[]) {
+    const _logger = createBaseLogger(memoryStore);
+
+    _logger.defaultMeta = labels;
+
+    const logger = new LogHelper(_logger, memoryStore, labels);
+
+    return logger;
+}
+
+export function Logger(module: string, withMemoryStore = false) {
+    return createLabeledLogger({ module }, withMemoryStore ? [] : undefined);
+}

package/src/helpers/OpenApiParser.helper.ts

@@ -0,0 +1,150 @@
+import axios from 'axios';
+import yaml from 'js-yaml';
+
+import { REQUEST_METHODS } from '../constants';
+
+// * We need a couple of packages to resolve references in the OpenAPI specification
+//FIXME : YAML support temporarly disabled to allow executable packaging
+//import SwaggerParser from '@apidevtools/swagger-parser';
+//import $RefParser from '@apidevtools/json-schema-ref-parser';
+//import { JSONSchema } from '@apidevtools/json-schema-ref-parser/dist/lib/types';
+
+//const swaggerParser = new SwaggerParser();
+
+type JSONSchema = Record<string, any>;
+
+/**
+ * Utility function to resolve $ref values within a schema.
+ * @param schema The schema containing potential $ref values.
+ * @param root The root schema for reference resolution.
+ * @returns The dereferenced schema.
+ */
+const dereferenceSchema = async (schema: any, root: any = schema): Promise<any> => {
+    if (typeof schema !== 'object' || schema === null) {
+        return schema;
+    }
+
+    if (schema.$ref) {
+        const ref = schema.$ref;
+        const path = ref.replace(/^#\//, '').split('/'); // Remove the initial '#/' and split the path
+        let resolved = root;
+        for (const segment of path) {
+            if (resolved[segment] === undefined) {
+                throw new Error(`Could not resolve $ref: ${ref}`);
+            }
+            resolved = resolved[segment];
+        }
+        return dereferenceSchema(resolved, root); // Recursively dereference
+    }
+
+    // Recursively resolve all properties and items
+    if (Array.isArray(schema)) {
+        return Promise.all(schema.map((item) => dereferenceSchema(item, root)));
+    }
+
+    const result: any = {};
+    for (const key of Object.keys(schema)) {
+        result[key] = await dereferenceSchema(schema[key], root);
+    }
+    return result;
+};
+
+export class OpenAPIParser {
+    static mapReqMethods(paths: Array<Record<string, any>>): Map<string, any> {
+        const methods = new Map();
+
+        for (const path in paths) {
+            const pathData = paths[path];
+
+            for (const method in pathData) {
+                const data = pathData[method];
+
+                if (REQUEST_METHODS.includes(method.toUpperCase())) {
+                    methods.set(data?.operationId, method);
+                }
+            }
+        }
+
+        return methods;
+    }
+    static mapEndpoints(paths: Array<Record<string, any>>): Map<string, any> {
+        const operationIds = new Map();
+
+        for (const path in paths) {
+            const pathData = paths[path];
+
+            // it's possible we have multiple methods for a single path
+            for (const method in pathData) {
+                const data = pathData[method];
+
+                if (REQUEST_METHODS.includes(method.toUpperCase())) {
+                    operationIds.set(data?.operationId, path);
+                }
+            }
+        }
+
+        return operationIds;
+    }
+
+    // static async yamlToJson(yamlData: string): Promise<JSONSchema> {
+    //     const data = yaml.load(yamlData);
+    //     //FIXME : YAML support temporarly disabled to allow executable packaging
+    //     const schema = await $RefParser.dereference(data);
+    //     //const schema: any = {};
+
+    //     return schema;
+    // }
+
+    static async yamlToJson(yamlData: string): Promise<JSONSchema> {
+        const data = yaml.load(yamlData);
+        const schema = await dereferenceSchema(data);
+        return schema;
+    }
+
+    // static async getJson(data: string | Record<string, any>): Promise<Record<string, any>> {
+    //     try {
+    //         let _data = data;
+    //         if (typeof data === 'string') {
+    //             _data = JSON.parse(_data as string);
+    //         }
+    //         //FIXME : YAML support temporarly disabled to allow executable packaging
+    //         const result = swaggerParser.dereference(_data as any);
+    //         //const result: any = {};
+    //         return result;
+    //     } catch (error) {
+    //         try {
+    //             return OpenAPIParser.yamlToJson(data as string);
+    //         } catch (error) {
+    //             throw new Error('Invalid OpenAPI specification data format');
+    //         }
+    //     }
+    // }
+
+    static async getJson(data: string | Record<string, any>): Promise<Record<string, any>> {
+        try {
+            let parsedData = data;
+            if (typeof data === 'string') {
+                parsedData = JSON.parse(data);
+            }
+
+            return await dereferenceSchema(parsedData);
+        } catch (error) {
+            try {
+                // Fallback to handle YAML data
+                return await OpenAPIParser.yamlToJson(data as string);
+            } catch (innerError) {
+                throw new Error('Invalid OpenAPI specification or JSON data format');
+            }
+        }
+    }
+    static async getJsonFromUrl(url: string): Promise<Record<string, any>> {
+        const response = await axios.get(url);
+        const data = response.data;
+
+        return OpenAPIParser.getJson(data);
+    }
+
+    static isValidOpenAPI(data: Record<string, any>): boolean {
+        return data?.openapi && data?.paths && data?.servers;
+    }
+}