akeyless-server-commons 1.0.203 → 1.0.204

package/dist/helpers/redis/helpers.js

@@ -0,0 +1,215 @@
+import { logger } from "../../managers";
+import { get_redis_commander, redis_commander_connected } from "./initialize";
+import { get_doc_key, get_collection_keys, scan_redis_keys } from "./keys";
+import { get_nx_settings, get_all_documents, query_documents, query_documents_by_conditions, query_document_by_conditions, query_document, query_document_optional, get_document_by_id, get_document_by_id_optional, } from "../firebase_helpers";
+const should_use_redis = async (collection_path) => {
+    if (!redis_commander_connected) {
+        logger.warn(`⚠️ Redis commander not connected, falling back to Firebase for collection: "${collection_path}"`);
+        return false;
+    }
+    try {
+        const nx_settings = await get_nx_settings();
+        const cache_collections_config = nx_settings.cache_collections_config || {};
+        const config = cache_collections_config[collection_path];
+        if (!config) {
+            logger.warn(`⚠️ Collection "${collection_path}" not found in cache_collections_config, falling back to Firebase`);
+            return false;
+        }
+    }
+    catch {
+        logger.warn(`⚠️ Failed to validate Redis config for collection: "${collection_path}", falling back to Firebase`);
+        return false;
+    }
+    return true;
+};
+const parse_redis_value = (raw) => {
+    const parsed = JSON.parse(raw);
+    return parsed.data ?? parsed;
+};
+const get_all_collection_docs = async (collection_path) => {
+    const commander = get_redis_commander();
+    const keys = await scan_redis_keys(get_collection_keys(collection_path), commander);
+    if (keys.length === 0)
+        return [];
+    const values = await commander.mget(keys);
+    return values.filter(Boolean).map((v) => {
+        const data = parse_redis_value(v);
+        const key = keys[values.indexOf(v)];
+        /// example key: "nx-users(collection):123(doc_id)"
+        const id = key.split(":").slice(1).join(":");
+        return { ...data, id };
+    });
+};
+const apply_operator = (doc_value, operator, value) => {
+    switch (operator) {
+        case "==":
+            return doc_value === value;
+        case "!=":
+            return doc_value !== value;
+        case "<":
+            return doc_value < value;
+        case "<=":
+            return doc_value <= value;
+        case ">":
+            return doc_value > value;
+        case ">=":
+            return doc_value >= value;
+        case "in":
+            return Array.isArray(value) && value.includes(doc_value);
+        case "not-in":
+            return Array.isArray(value) && !value.includes(doc_value);
+        case "array-contains":
+            return Array.isArray(doc_value) && doc_value.includes(value);
+        case "array-contains-any":
+            return Array.isArray(doc_value) && Array.isArray(value) && value.some((v) => doc_value.includes(v));
+        default:
+            return false;
+    }
+};
+const filter_by_condition = (docs, field_name, operator, value) => {
+    return docs.filter((doc) => apply_operator(doc[field_name], operator, value));
+};
+const filter_by_conditions = (docs, conditions) => {
+    return conditions.reduce((filtered, { field_name, operator, value }) => {
+        return filter_by_condition(filtered, field_name, operator, value);
+    }, docs);
+};
+// ── extract ──
+export const redis_simple_extract_data = (raw) => {
+    const data = parse_redis_value(raw);
+    return data;
+};
+// ── documents ──
+export const redis_get_all_documents = async (collection_path) => {
+    if (!(await should_use_redis(collection_path))) {
+        return get_all_documents(collection_path);
+    }
+    try {
+        return await get_all_collection_docs(collection_path);
+    }
+    catch (error) {
+        logger.error("Error fetching documents from Redis:", error);
+        throw error;
+    }
+};
+export const redis_query_documents = async (collection_path, field_name, operator, value) => {
+    if (!(await should_use_redis(collection_path))) {
+        return query_documents(collection_path, field_name, operator, value);
+    }
+    try {
+        const all_docs = await get_all_collection_docs(collection_path);
+        return filter_by_condition(all_docs, field_name, operator, value);
+    }
+    catch (error) {
+        logger.error(`Error querying documents from Redis: ${collection_path} - ${field_name} - ${operator} - ${value} `, error);
+        throw error;
+    }
+};
+export const redis_query_documents_by_conditions = async (collection_path, where_conditions) => {
+    if (!(await should_use_redis(collection_path))) {
+        return query_documents_by_conditions(collection_path, where_conditions);
+    }
+    try {
+        const all_docs = await get_all_collection_docs(collection_path);
+        return filter_by_conditions(all_docs, where_conditions);
+    }
+    catch (error) {
+        logger.error(`Error querying documents from Redis: ${collection_path} - ${JSON.stringify(where_conditions)} `, error);
+        throw error;
+    }
+};
+export const redis_query_document_by_conditions = async (collection_path, where_conditions, ignore_logs = false) => {
+    if (!(await should_use_redis(collection_path))) {
+        return query_document_by_conditions(collection_path, where_conditions, !ignore_logs);
+    }
+    try {
+        const all_docs = await get_all_collection_docs(collection_path);
+        const documents = filter_by_conditions(all_docs, where_conditions);
+        if (!documents[0]) {
+            throw "no data returned from Redis";
+        }
+        return documents[0];
+    }
+    catch (error) {
+        if (!ignore_logs) {
+            logger.error(`Error querying document from Redis: ${collection_path} - ${JSON.stringify(where_conditions)} `, error);
+        }
+        throw error;
+    }
+};
+// ── document ──
+export const redis_query_document = async (collection_path, field_name, operator, value, ignore_log = false) => {
+    if (!(await should_use_redis(collection_path))) {
+        return query_document(collection_path, field_name, operator, value, ignore_log);
+    }
+    try {
+        const all_docs = await get_all_collection_docs(collection_path);
+        const documents = filter_by_condition(all_docs, field_name, operator, value);
+        const base_error = `collection: ${collection_path}, field_name: ${field_name}, operator: ${operator}, value:${value}`;
+        if (documents.length < 1) {
+            throw `[Redis] No data to return from: ${base_error}`;
+        }
+        if (documents.length > 1) {
+            throw `[Redis] Multiple documents found in: ${base_error}`;
+        }
+        return documents[0];
+    }
+    catch (error) {
+        if (!ignore_log) {
+            logger.error("Error querying document from Redis: " + JSON.stringify({ collection_path, field_name, operator, value }), error);
+        }
+        throw error;
+    }
+};
+export const redis_query_document_optional = async (collection_path, field_name, operator, value, ignore_log = true) => {
+    if (!(await should_use_redis(collection_path))) {
+        return query_document_optional(collection_path, field_name, operator, value, ignore_log);
+    }
+    try {
+        const all_docs = await get_all_collection_docs(collection_path);
+        const documents = filter_by_condition(all_docs, field_name, operator, value);
+        return documents[0] || null;
+    }
+    catch (error) {
+        if (!ignore_log) {
+            logger.error("Error querying optional document from Redis: " + JSON.stringify({ collection_path, field_name, operator, value }), error);
+        }
+        return null;
+    }
+};
+export const redis_get_document_by_id = async (collection_path, doc_id) => {
+    if (!(await should_use_redis(collection_path))) {
+        return get_document_by_id(collection_path, doc_id);
+    }
+    try {
+        const commander = get_redis_commander();
+        const key = get_doc_key(collection_path, doc_id);
+        const raw = await commander.get(key);
+        if (!raw) {
+            throw "Document not found in Redis, document id: " + doc_id;
+        }
+        return redis_simple_extract_data(raw);
+    }
+    catch (error) {
+        logger.error("error from redis_get_document_by_id", error);
+        throw error;
+    }
+};
+export const redis_get_document_by_id_optional = async (collection_path, doc_id) => {
+    if (!(await should_use_redis(collection_path))) {
+        return get_document_by_id_optional(collection_path, doc_id);
+    }
+    try {
+        const commander = get_redis_commander();
+        const key = get_doc_key(collection_path, doc_id);
+        const raw = await commander.get(key);
+        if (!raw) {
+            return null;
+        }
+        return redis_simple_extract_data(raw);
+    }
+    catch (error) {
+        logger.error("error from redis_get_document_by_id_optional", error);
+        return null;
+    }
+};

package/dist/helpers/redis/index.js

@@ -0,0 +1,4 @@
+export * from "./initialize";
+export * from "./keys";
+export * from "./snapshot";
+export * from "./helpers";

package/dist/helpers/redis/initialize.js

@@ -0,0 +1,139 @@
+import Redis from "ioredis";
+import { REDIS_UPDATES_PREFIX } from "akeyless-types-commons";
+import { get_collection_keys } from "./keys";
+import { init_env_variables } from "../global_helpers";
+import { logger } from "../../managers";
+import dotenv from "dotenv";
+dotenv.config();
+export let redis_commander_connected = false;
+export let redis_listener_connected = false;
+const MAX_RETRY_ATTEMPTS = 2;
+let redis_commander;
+let redis_listener;
+let redis_initialized = false;
+function create_redis_instance(role) {
+    let retry_attempts = 0;
+    const base_config = {
+        host: init_env_variables(["redis_ip"]).redis_ip,
+        lazyConnect: true,
+        enableOfflineQueue: false,
+        maxRetriesPerRequest: 0,
+        reconnectOnError: () => false,
+        retryStrategy(times) {
+            retry_attempts = times;
+            if (retry_attempts >= MAX_RETRY_ATTEMPTS) {
+                logger.error(`❌ Redis ${role} failed after ${MAX_RETRY_ATTEMPTS} attempts. Stopping retries.`);
+                return null;
+            }
+            const delay = 1000;
+            logger.warn(`⚠️ Redis ${role} retry attempt ${retry_attempts}/${MAX_RETRY_ATTEMPTS} in ${delay}ms...`);
+            return delay;
+        },
+    };
+    // Listener needs enableReadyCheck: false because it will be in subscriber mode
+    // and cannot respond to PING commands
+    const config = role === "listener"
+        ? { ...base_config, enableReadyCheck: false }
+        : base_config;
+    const client = new Redis(config);
+    client.on("connect", () => {
+        logger.log(`✅ Redis ${role} connected`);
+        if (role === "commander")
+            redis_commander_connected = true;
+        if (role === "listener")
+            redis_listener_connected = true;
+    });
+    client.on("error", (err) => logger.error(`❌ Redis ${role} error`, err));
+    client.on("close", () => {
+        logger.warn(`ℹ️ Redis ${role} connection closed`);
+        if (role === "commander")
+            redis_commander_connected = false;
+        if (role === "listener")
+            redis_listener_connected = false;
+    });
+    client.on("end", () => {
+        logger.warn(`ℹ️ Redis ${role} connection ended`);
+        if (role === "commander")
+            redis_commander_connected = false;
+        if (role === "listener")
+            redis_listener_connected = false;
+    });
+    client.connect().catch((err) => {
+        logger.error(`❌ Redis ${role} initial connect failed`, err);
+    });
+    return client;
+}
+export const init_redis = () => {
+    return new Promise((resolve, reject) => {
+        if (redis_initialized) {
+            resolve();
+            return;
+        }
+        redis_commander = create_redis_instance("commander");
+        redis_listener = create_redis_instance("listener");
+        redis_initialized = true;
+        let commander_ready = false;
+        let listener_ready = false;
+        let listener_subscribed = false;
+        const check_and_resolve = () => {
+            if (commander_ready && listener_ready) {
+                resolve();
+            }
+        };
+        redis_commander.on("connect", () => {
+            commander_ready = true;
+            check_and_resolve();
+        });
+        redis_listener.on("connect", () => {
+            listener_ready = true;
+            // Only subscribe once, not on every reconnect
+            // ioredis will automatically resubscribe on reconnection
+            if (!listener_subscribed) {
+                listener_subscribed = true;
+                const redis_pattern = get_collection_keys(REDIS_UPDATES_PREFIX);
+                redis_listener
+                    .psubscribe(redis_pattern)
+                    .then(() => logger.log(`✅ Subscribed to Redis pattern: ${redis_pattern}`))
+                    .catch((err) => logger.error(`❌ Failed to psubscribe to ${redis_pattern}`, err));
+            }
+            check_and_resolve();
+        });
+        // Handle case where Redis connections fail
+        const connection_timeout = setTimeout(() => {
+            if (!commander_ready || !listener_ready) {
+                logger.warn("⚠️ Redis connection timeout, proceeding without Redis");
+                resolve();
+            }
+        }, 5000);
+        redis_commander.on("error", () => {
+            if (!commander_ready && !listener_ready) {
+                clearTimeout(connection_timeout);
+                reject();
+            }
+        });
+        redis_listener.on("error", () => {
+            if (!commander_ready && !listener_ready) {
+                clearTimeout(connection_timeout);
+                reject();
+            }
+        });
+        process.on("unhandledRejection", (err) => {
+            logger.error("❌ Unhandled promise rejection", err);
+        });
+        process.on("uncaughtException", (err) => {
+            logger.error("❌ Uncaught exception", err);
+        });
+    });
+};
+export const get_redis_commander = () => {
+    if (!redis_commander) {
+        throw new Error("Redis commander not initialized. Call init_redis() first.");
+    }
+    return redis_commander;
+};
+export const get_redis_listener = () => {
+    if (!redis_listener) {
+        throw new Error("Redis listener not initialized. Call init_redis() first.");
+    }
+    return redis_listener;
+};

package/dist/helpers/redis/keys.js

@@ -0,0 +1,23 @@
+import { REDIS_UPDATES_PREFIX } from "akeyless-types-commons";
+const get_key = (...args) => {
+    return args.join(":");
+};
+export const get_doc_key = (collection, doc_id) => {
+    return get_key(collection, doc_id);
+};
+export const get_collection_keys = (collection) => {
+    return get_key(collection, "*");
+};
+export const get_channel = (...args) => {
+    return get_key(REDIS_UPDATES_PREFIX, ...args);
+};
+export const scan_redis_keys = async (pattern, redis_publisher) => {
+    const found_keys = [];
+    let cursor = "0";
+    do {
+        const [next_cursor, keys] = await redis_publisher.scan(cursor, "MATCH", pattern, "COUNT", 100);
+        cursor = next_cursor;
+        found_keys.push(...keys);
+    } while (cursor !== "0");
+    return found_keys;
+};

package/dist/helpers/redis/snapshot.js

@@ -0,0 +1,166 @@
+import { get_redis_commander, get_redis_listener, redis_commander_connected, redis_listener_connected } from "./initialize";
+import { get_nx_settings, parse_add_update_as_array, parse_add_update_as_object, parse_delete_as_array, parse_delete_as_object, snapshot, } from "../firebase_helpers";
+import { get_collection_keys, scan_redis_keys } from "./keys";
+import { Timestamp } from "firebase-admin/firestore";
+import { logger } from "../../managers";
+const subscription_collections = new Set();
+export const redis_snapshots_bulk = async (configs) => {
+    if (configs.length === 0) {
+        return;
+    }
+    /// on first time
+    for (const config of configs) {
+        const validation = await validate_config(config);
+        if (!validation.success) {
+            logger.warn(validation.message, config);
+            if (validation.trigger_firebase_snapshot) {
+                await snapshot(config);
+            }
+            continue;
+        }
+        await parse_redis_snapshot(config);
+    }
+    /// on update
+    get_redis_listener().on("pmessage", async (pattern, channel, message) => {
+        const { collection_name, update_type, data: doc } = JSON.parse(message);
+        const update = [convert_object_timestamps(doc)];
+        const relevant_configs = configs.filter((config) => config.collection_name === collection_name);
+        for (const config of relevant_configs) {
+            await parse_redis_snapshot(config, { update_type, update });
+        }
+    });
+};
+const validate_config = async (config) => {
+    const nx_settings = await get_nx_settings();
+    const cache_collections_config = nx_settings.cache_collections_config || {};
+    const { collection_name, cache_name = collection_name, subscription_type = "firebase" } = config;
+    if (!redis_commander_connected || !redis_listener_connected) {
+        return {
+            success: false,
+            trigger_firebase_snapshot: true,
+            message: `⚠️ Redis commander or listener not connected, triggered firebase snapshot for collection: "${collection_name}"`,
+        };
+    }
+    if (subscription_type === "firebase") {
+        return {
+            success: false,
+            trigger_firebase_snapshot: true,
+            message: `⚠️ subscription type is "firebase", triggered firebase snapshot for collection: "${collection_name}"`,
+        };
+    }
+    if (!cache_collections_config[collection_name] || cache_collections_config[collection_name].sync_direction === "redis_to_firebase") {
+        return {
+            success: false,
+            trigger_firebase_snapshot: true,
+            message: `⚠️ collection: "${collection_name}" not found in cache_collections_config in "nx-settings" or sync_direction is "redis_to_firebase", triggered firebase snapshot for collection: "${collection_name}"`,
+        };
+    }
+    if (subscription_collections.has(cache_name)) {
+        return {
+            success: false,
+            trigger_firebase_snapshot: false,
+            message: `⚠️ collection: "${collection_name}" already has a subscription, skipped redis snapshot for collection: "${collection_name}"`,
+        };
+    }
+    subscription_collections.add(cache_name);
+    return { success: true };
+};
+const parse_redis_snapshot = async (config, redis_update) => {
+    const { collection_name: config_collection_name, cache_name = config_collection_name, extra_parsers, parse_as, debug, on_add, on_remove, on_modify, } = config;
+    if (!redis_update) {
+        const init_data = await get_collection_data(config_collection_name);
+        default_parsers(parse_as, init_data, config);
+        if (debug?.on_first_time) {
+            logger.log(`${cache_name} => Redis snapshot on first time: `, debug.on_first_time === "documents" ? init_data : { length: init_data.length });
+        }
+        config.on_first_time?.(init_data, config);
+        extra_parsers?.forEach((parser) => {
+            if (debug?.extra_parsers?.on_first_time) {
+                logger.log(`${cache_name} => Redis snapshot extra parsers on first time: `, debug.extra_parsers.on_first_time === "documents" ? init_data : { length: init_data.length });
+            }
+            parser?.on_first_time?.(init_data, config);
+        });
+        return;
+    }
+    if (!subscription_collections.has(cache_name)) {
+        return;
+    }
+    const { update_type, update } = redis_update;
+    switch (update_type) {
+        case "add":
+            if (debug?.on_add) {
+                logger.log(`${cache_name} => Redis snapshot on add: `, update);
+            }
+            default_parsers(parse_as, update, config);
+            on_add?.(update, config);
+            extra_parsers?.forEach((parser) => {
+                if (debug?.extra_parsers?.on_add) {
+                    logger.log(`${cache_name} => Redis snapshot extra parsers on add: `, update);
+                }
+                parser?.on_add?.(update, config);
+            });
+            return;
+        case "update":
+            if (debug?.on_modify) {
+                logger.log(`${cache_name} => Redis snapshot on modify: `, update);
+            }
+            default_parsers(parse_as, update, config);
+            on_modify?.(update, config);
+            extra_parsers?.forEach((parser) => {
+                if (debug?.extra_parsers?.on_modify) {
+                    logger.log(`${cache_name} => Redis snapshot extra parsers on modify: `, update);
+                }
+                parser?.on_modify?.(update, config);
+            });
+            return;
+        case "delete":
+            if (debug?.on_remove) {
+                logger.log(`${cache_name} => Redis snapshot on remove: `, update);
+            }
+            if (parse_as === "object") {
+                parse_delete_as_object(update, config);
+            }
+            if (parse_as === "array") {
+                parse_delete_as_array(update, config);
+            }
+            on_remove?.(update, config);
+            extra_parsers?.forEach((parser) => {
+                if (debug?.extra_parsers?.on_remove) {
+                    logger.log(`${cache_name} => Redis snapshot extra parsers on remove: `, update);
+                }
+                parser?.on_remove?.(update, config);
+            });
+            return;
+        default:
+            return;
+    }
+};
+const default_parsers = (parse_as, update, config) => {
+    if (parse_as === "object") {
+        parse_add_update_as_object(update, config);
+    }
+    if (parse_as === "array") {
+        parse_add_update_as_array(update, config);
+    }
+};
+const get_collection_data = async (collection_name) => {
+    const redis_commander = get_redis_commander();
+    const keys = await scan_redis_keys(get_collection_keys(collection_name), redis_commander);
+    let collection_data = [];
+    if (keys.length > 0) {
+        const values = await redis_commander.mget(keys);
+        collection_data = values.filter(Boolean).map((v) => JSON.parse(v).data);
+    }
+    return collection_data.map((data) => convert_object_timestamps(data));
+};
+const convert_object_timestamps = (data) => {
+    Object.entries(data).forEach(([key, value]) => {
+        if (typeof value === "object" && value && value._seconds !== undefined) {
+            data[key] = new Timestamp(value._seconds, value._nanoseconds);
+        }
+        if (typeof value === "object" && value && value._seconds === undefined) {
+            convert_object_timestamps(value);
+        }
+    });
+    return data;
+};

package/dist/helpers/start.js

@@ -0,0 +1,83 @@
+import express from "express";
+import cors from "cors";
+import { logger } from "../managers";
+import { init_env_variables, init_snapshots } from "./";
+import { error_handler } from "../middlewares/error_handling";
+import { request_logger, trim_body_middleware } from "../middlewares";
+import { init_redis } from "./redis/initialize";
+import axios from "axios";
+import https from "https";
+import http from "http";
+import { rabbitmq } from "../managers/rabbitmq_manager";
+export const start_server = async (main_router, project_name, version, options = {}) => {
+    let env_data = init_env_variables(["mode"]);
+    const { port = Number(env_data.port), log_requests, initialize_redis = true, enable_axios_keep_alive = true } = options;
+    const app = express();
+    app.use(cors());
+    app.use(express.json({ limit: "10mb" }));
+    app.use(express.urlencoded({ limit: "10mb", extended: true }));
+    app.use(trim_body_middleware());
+    app.use(request_logger(log_requests || {}));
+    main_router(app);
+    app.use(error_handler);
+    if (enable_axios_keep_alive) {
+        keep_axios_alive();
+    }
+    return new Promise((resolve) => {
+        app.listen(port, async () => {
+            logger.log(`Server is running at http://localhost:${port}`);
+            logger.log("project status", { project_name, version, environment: env_data.mode });
+            if (initialize_redis) {
+                try {
+                    logger.log("Initializing Redis...");
+                    await init_redis();
+                }
+                catch (err) {
+                    logger.warn("Redis unavailable, continuing without it", { err });
+                }
+            }
+            resolve(app);
+        });
+    });
+};
+export const basic_init = async (main_router, project_name, version, options = {}) => {
+    const { init_snapshot_options, on_shutdown } = options;
+    try {
+        const app = await start_server(main_router, project_name, version, options);
+        await init_snapshots(init_snapshot_options);
+        const shutdown = async (signal) => {
+            logger.log(`Received ${signal}, shutting down...`);
+            try {
+                await rabbitmq.cancel_subscriptions();
+                await rabbitmq.close();
+                await on_shutdown?.();
+            }
+            catch (err) {
+                logger.error("Shutdown error", err);
+                process.exit(1);
+            }
+            process.exit(0);
+        };
+        process.once("SIGINT", () => void shutdown("SIGINT"));
+        process.once("SIGTERM", () => void shutdown("SIGTERM"));
+        return app;
+    }
+    catch (error) {
+        logger.error("Error from init function: ", error);
+        process.exit(1);
+    }
+};
+const keep_axios_alive = () => {
+    axios.defaults.httpsAgent = new https.Agent({
+        keepAlive: true,
+        maxSockets: 100,
+        maxFreeSockets: 20,
+        timeout: 60000,
+    });
+    axios.defaults.httpAgent = new http.Agent({
+        keepAlive: true,
+        maxSockets: 100,
+        maxFreeSockets: 20,
+        timeout: 60000,
+    });
+};