akeyless-server-commons 1.0.205 → 1.0.207

package/dist/helpers/redis/initialize.js

@@ -1,139 +0,0 @@
-import Redis from "ioredis";
-import { REDIS_UPDATES_PREFIX } from "akeyless-types-commons";
-import { get_collection_keys } from "./keys";
-import { init_env_variables } from "../global_helpers";
-import { logger } from "../../managers";
-import dotenv from "dotenv";
-dotenv.config();
-export let redis_commander_connected = false;
-export let redis_listener_connected = false;
-const MAX_RETRY_ATTEMPTS = 2;
-let redis_commander;
-let redis_listener;
-let redis_initialized = false;
-function create_redis_instance(role) {
-    let retry_attempts = 0;
-    const base_config = {
-        host: init_env_variables(["redis_ip"]).redis_ip,
-        lazyConnect: true,
-        enableOfflineQueue: false,
-        maxRetriesPerRequest: 0,
-        reconnectOnError: () => false,
-        retryStrategy(times) {
-            retry_attempts = times;
-            if (retry_attempts >= MAX_RETRY_ATTEMPTS) {
-                logger.error(`❌ Redis ${role} failed after ${MAX_RETRY_ATTEMPTS} attempts. Stopping retries.`);
-                return null;
-            }
-            const delay = 1000;
-            logger.warn(`⚠️ Redis ${role} retry attempt ${retry_attempts}/${MAX_RETRY_ATTEMPTS} in ${delay}ms...`);
-            return delay;
-        },
-    };
-    // Listener needs enableReadyCheck: false because it will be in subscriber mode
-    // and cannot respond to PING commands
-    const config = role === "listener"
-        ? { ...base_config, enableReadyCheck: false }
-        : base_config;
-    const client = new Redis(config);
-    client.on("connect", () => {
-        logger.log(`✅ Redis ${role} connected`);
-        if (role === "commander")
-            redis_commander_connected = true;
-        if (role === "listener")
-            redis_listener_connected = true;
-    });
-    client.on("error", (err) => logger.error(`❌ Redis ${role} error`, err));
-    client.on("close", () => {
-        logger.warn(`ℹ️ Redis ${role} connection closed`);
-        if (role === "commander")
-            redis_commander_connected = false;
-        if (role === "listener")
-            redis_listener_connected = false;
-    });
-    client.on("end", () => {
-        logger.warn(`ℹ️ Redis ${role} connection ended`);
-        if (role === "commander")
-            redis_commander_connected = false;
-        if (role === "listener")
-            redis_listener_connected = false;
-    });
-    client.connect().catch((err) => {
-        logger.error(`❌ Redis ${role} initial connect failed`, err);
-    });
-    return client;
-}
-export const init_redis = () => {
-    return new Promise((resolve, reject) => {
-        if (redis_initialized) {
-            resolve();
-            return;
-        }
-        redis_commander = create_redis_instance("commander");
-        redis_listener = create_redis_instance("listener");
-        redis_initialized = true;
-        let commander_ready = false;
-        let listener_ready = false;
-        let listener_subscribed = false;
-        const check_and_resolve = () => {
-            if (commander_ready && listener_ready) {
-                resolve();
-            }
-        };
-        redis_commander.on("connect", () => {
-            commander_ready = true;
-            check_and_resolve();
-        });
-        redis_listener.on("connect", () => {
-            listener_ready = true;
-            // Only subscribe once, not on every reconnect
-            // ioredis will automatically resubscribe on reconnection
-            if (!listener_subscribed) {
-                listener_subscribed = true;
-                const redis_pattern = get_collection_keys(REDIS_UPDATES_PREFIX);
-                redis_listener
-                    .psubscribe(redis_pattern)
-                    .then(() => logger.log(`✅ Subscribed to Redis pattern: ${redis_pattern}`))
-                    .catch((err) => logger.error(`❌ Failed to psubscribe to ${redis_pattern}`, err));
-            }
-            check_and_resolve();
-        });
-        // Handle case where Redis connections fail
-        const connection_timeout = setTimeout(() => {
-            if (!commander_ready || !listener_ready) {
-                logger.warn("⚠️ Redis connection timeout, proceeding without Redis");
-                resolve();
-            }
-        }, 5000);
-        redis_commander.on("error", () => {
-            if (!commander_ready && !listener_ready) {
-                clearTimeout(connection_timeout);
-                reject();
-            }
-        });
-        redis_listener.on("error", () => {
-            if (!commander_ready && !listener_ready) {
-                clearTimeout(connection_timeout);
-                reject();
-            }
-        });
-        process.on("unhandledRejection", (err) => {
-            logger.error("❌ Unhandled promise rejection", err);
-        });
-        process.on("uncaughtException", (err) => {
-            logger.error("❌ Uncaught exception", err);
-        });
-    });
-};
-export const get_redis_commander = () => {
-    if (!redis_commander) {
-        throw new Error("Redis commander not initialized. Call init_redis() first.");
-    }
-    return redis_commander;
-};
-export const get_redis_listener = () => {
-    if (!redis_listener) {
-        throw new Error("Redis listener not initialized. Call init_redis() first.");
-    }
-    return redis_listener;
-};

package/dist/helpers/redis/keys.js

@@ -1,23 +0,0 @@
-import { REDIS_UPDATES_PREFIX } from "akeyless-types-commons";
-const get_key = (...args) => {
-    return args.join(":");
-};
-export const get_doc_key = (collection, doc_id) => {
-    return get_key(collection, doc_id);
-};
-export const get_collection_keys = (collection) => {
-    return get_key(collection, "*");
-};
-export const get_channel = (...args) => {
-    return get_key(REDIS_UPDATES_PREFIX, ...args);
-};
-export const scan_redis_keys = async (pattern, redis_publisher) => {
-    const found_keys = [];
-    let cursor = "0";
-    do {
-        const [next_cursor, keys] = await redis_publisher.scan(cursor, "MATCH", pattern, "COUNT", 100);
-        cursor = next_cursor;
-        found_keys.push(...keys);
-    } while (cursor !== "0");
-    return found_keys;
-};

package/dist/helpers/redis/snapshot.js

@@ -1,166 +0,0 @@
-import { get_redis_commander, get_redis_listener, redis_commander_connected, redis_listener_connected } from "./initialize";
-import { get_nx_settings, parse_add_update_as_array, parse_add_update_as_object, parse_delete_as_array, parse_delete_as_object, snapshot, } from "../firebase_helpers";
-import { get_collection_keys, scan_redis_keys } from "./keys";
-import { Timestamp } from "firebase-admin/firestore";
-import { logger } from "../../managers";
-const subscription_collections = new Set();
-export const redis_snapshots_bulk = async (configs) => {
-    if (configs.length === 0) {
-        return;
-    }
-    /// on first time
-    for (const config of configs) {
-        const validation = await validate_config(config);
-        if (!validation.success) {
-            logger.warn(validation.message, config);
-            if (validation.trigger_firebase_snapshot) {
-                await snapshot(config);
-            }
-            continue;
-        }
-        await parse_redis_snapshot(config);
-    }
-    /// on update
-    get_redis_listener().on("pmessage", async (pattern, channel, message) => {
-        const { collection_name, update_type, data: doc } = JSON.parse(message);
-        const update = [convert_object_timestamps(doc)];
-        const relevant_configs = configs.filter((config) => config.collection_name === collection_name);
-        for (const config of relevant_configs) {
-            await parse_redis_snapshot(config, { update_type, update });
-        }
-    });
-};
-const validate_config = async (config) => {
-    const nx_settings = await get_nx_settings();
-    const cache_collections_config = nx_settings.cache_collections_config || {};
-    const { collection_name, cache_name = collection_name, subscription_type = "firebase" } = config;
-    if (!redis_commander_connected || !redis_listener_connected) {
-        return {
-            success: false,
-            trigger_firebase_snapshot: true,
-            message: `⚠️ Redis commander or listener not connected, triggered firebase snapshot for collection: "${collection_name}"`,
-        };
-    }
-    if (subscription_type === "firebase") {
-        return {
-            success: false,
-            trigger_firebase_snapshot: true,
-            message: `⚠️ subscription type is "firebase", triggered firebase snapshot for collection: "${collection_name}"`,
-        };
-    }
-    if (!cache_collections_config[collection_name] || cache_collections_config[collection_name].sync_direction === "redis_to_firebase") {
-        return {
-            success: false,
-            trigger_firebase_snapshot: true,
-            message: `⚠️ collection: "${collection_name}" not found in cache_collections_config in "nx-settings" or sync_direction is "redis_to_firebase", triggered firebase snapshot for collection: "${collection_name}"`,
-        };
-    }
-    if (subscription_collections.has(cache_name)) {
-        return {
-            success: false,
-            trigger_firebase_snapshot: false,
-            message: `⚠️ collection: "${collection_name}" already has a subscription, skipped redis snapshot for collection: "${collection_name}"`,
-        };
-    }
-    subscription_collections.add(cache_name);
-    return { success: true };
-};
-const parse_redis_snapshot = async (config, redis_update) => {
-    const { collection_name: config_collection_name, cache_name = config_collection_name, extra_parsers, parse_as, debug, on_add, on_remove, on_modify, } = config;
-    if (!redis_update) {
-        const init_data = await get_collection_data(config_collection_name);
-        default_parsers(parse_as, init_data, config);
-        if (debug?.on_first_time) {
-            logger.log(`${cache_name} => Redis snapshot on first time: `, debug.on_first_time === "documents" ? init_data : { length: init_data.length });
-        }
-        config.on_first_time?.(init_data, config);
-        extra_parsers?.forEach((parser) => {
-            if (debug?.extra_parsers?.on_first_time) {
-                logger.log(`${cache_name} => Redis snapshot extra parsers on first time: `, debug.extra_parsers.on_first_time === "documents" ? init_data : { length: init_data.length });
-            }
-            parser?.on_first_time?.(init_data, config);
-        });
-        return;
-    }
-    if (!subscription_collections.has(cache_name)) {
-        return;
-    }
-    const { update_type, update } = redis_update;
-    switch (update_type) {
-        case "add":
-            if (debug?.on_add) {
-                logger.log(`${cache_name} => Redis snapshot on add: `, update);
-            }
-            default_parsers(parse_as, update, config);
-            on_add?.(update, config);
-            extra_parsers?.forEach((parser) => {
-                if (debug?.extra_parsers?.on_add) {
-                    logger.log(`${cache_name} => Redis snapshot extra parsers on add: `, update);
-                }
-                parser?.on_add?.(update, config);
-            });
-            return;
-        case "update":
-            if (debug?.on_modify) {
-                logger.log(`${cache_name} => Redis snapshot on modify: `, update);
-            }
-            default_parsers(parse_as, update, config);
-            on_modify?.(update, config);
-            extra_parsers?.forEach((parser) => {
-                if (debug?.extra_parsers?.on_modify) {
-                    logger.log(`${cache_name} => Redis snapshot extra parsers on modify: `, update);
-                }
-                parser?.on_modify?.(update, config);
-            });
-            return;
-        case "delete":
-            if (debug?.on_remove) {
-                logger.log(`${cache_name} => Redis snapshot on remove: `, update);
-            }
-            if (parse_as === "object") {
-                parse_delete_as_object(update, config);
-            }
-            if (parse_as === "array") {
-                parse_delete_as_array(update, config);
-            }
-            on_remove?.(update, config);
-            extra_parsers?.forEach((parser) => {
-                if (debug?.extra_parsers?.on_remove) {
-                    logger.log(`${cache_name} => Redis snapshot extra parsers on remove: `, update);
-                }
-                parser?.on_remove?.(update, config);
-            });
-            return;
-        default:
-            return;
-    }
-};
-const default_parsers = (parse_as, update, config) => {
-    if (parse_as === "object") {
-        parse_add_update_as_object(update, config);
-    }
-    if (parse_as === "array") {
-        parse_add_update_as_array(update, config);
-    }
-};
-const get_collection_data = async (collection_name) => {
-    const redis_commander = get_redis_commander();
-    const keys = await scan_redis_keys(get_collection_keys(collection_name), redis_commander);
-    let collection_data = [];
-    if (keys.length > 0) {
-        const values = await redis_commander.mget(keys);
-        collection_data = values.filter(Boolean).map((v) => JSON.parse(v).data);
-    }
-    return collection_data.map((data) => convert_object_timestamps(data));
-};
-const convert_object_timestamps = (data) => {
-    Object.entries(data).forEach(([key, value]) => {
-        if (typeof value === "object" && value && value._seconds !== undefined) {
-            data[key] = new Timestamp(value._seconds, value._nanoseconds);
-        }
-        if (typeof value === "object" && value && value._seconds === undefined) {
-            convert_object_timestamps(value);
-        }
-    });
-    return data;
-};

package/dist/helpers/start.js

@@ -1,83 +0,0 @@
-import express from "express";
-import cors from "cors";
-import { logger } from "../managers";
-import { init_env_variables, init_snapshots } from "./";
-import { error_handler } from "../middlewares/error_handling";
-import { request_logger, trim_body_middleware } from "../middlewares";
-import { init_redis } from "./redis/initialize";
-import axios from "axios";
-import https from "https";
-import http from "http";
-import { rabbitmq } from "../managers/rabbitmq_manager";
-export const start_server = async (main_router, project_name, version, options = {}) => {
-    let env_data = init_env_variables(["mode"]);
-    const { port = Number(env_data.port), log_requests, initialize_redis = true, enable_axios_keep_alive = true } = options;
-    const app = express();
-    app.use(cors());
-    app.use(express.json({ limit: "10mb" }));
-    app.use(express.urlencoded({ limit: "10mb", extended: true }));
-    app.use(trim_body_middleware());
-    app.use(request_logger(log_requests || {}));
-    main_router(app);
-    app.use(error_handler);
-    if (enable_axios_keep_alive) {
-        keep_axios_alive();
-    }
-    return new Promise((resolve) => {
-        app.listen(port, async () => {
-            logger.log(`Server is running at http://localhost:${port}`);
-            logger.log("project status", { project_name, version, environment: env_data.mode });
-            if (initialize_redis) {
-                try {
-                    logger.log("Initializing Redis...");
-                    await init_redis();
-                }
-                catch (err) {
-                    logger.warn("Redis unavailable, continuing without it", { err });
-                }
-            }
-            resolve(app);
-        });
-    });
-};
-export const basic_init = async (main_router, project_name, version, options = {}) => {
-    const { init_snapshot_options, on_shutdown } = options;
-    try {
-        const app = await start_server(main_router, project_name, version, options);
-        await init_snapshots(init_snapshot_options);
-        const shutdown = async (signal) => {
-            logger.log(`Received ${signal}, shutting down...`);
-            try {
-                await rabbitmq.cancel_subscriptions();
-                await rabbitmq.close();
-                await on_shutdown?.();
-            }
-            catch (err) {
-                logger.error("Shutdown error", err);
-                process.exit(1);
-            }
-            process.exit(0);
-        };
-        process.once("SIGINT", () => void shutdown("SIGINT"));
-        process.once("SIGTERM", () => void shutdown("SIGTERM"));
-        return app;
-    }
-    catch (error) {
-        logger.error("Error from init function: ", error);
-        process.exit(1);
-    }
-};
-const keep_axios_alive = () => {
-    axios.defaults.httpsAgent = new https.Agent({
-        keepAlive: true,
-        maxSockets: 100,
-        maxFreeSockets: 20,
-        timeout: 60000,
-    });
-    axios.defaults.httpAgent = new http.Agent({
-        keepAlive: true,
-        maxSockets: 100,
-        maxFreeSockets: 20,
-        timeout: 60000,
-    });
-};

package/dist/helpers/tasks_helpers.js

@@ -1,139 +0,0 @@
-import { cache_manager, logger } from "../managers";
-import { get_document_by_id_optional, set_document } from "./firebase_helpers";
-import admin from "firebase-admin";
-export var TaskName;
-(function (TaskName) {
-    TaskName["send_reset_sms"] = "send_reset_sms";
-    TaskName["collect_devices_health"] = "collect_devices_health";
-    TaskName["collect_charge_locations"] = "collect_charge_locations";
-    TaskName["collect_charge_cdrs"] = "collect_charge_cdrs";
-    TaskName["auto_connect_to_call_center"] = "auto_connect_to_call_center";
-    TaskName["make_erm_ev_events"] = "make_erm_ev_events";
-    TaskName["collect_jimi_cars_odometer"] = "collect_jimi_cars_odometer";
-})(TaskName || (TaskName = {}));
-export var TaskStatus;
-(function (TaskStatus) {
-    TaskStatus["running"] = "running";
-    TaskStatus["completed"] = "completed";
-    TaskStatus["failed"] = "failed";
-    TaskStatus["suspeneded"] = "suspeneded";
-})(TaskStatus || (TaskStatus = {}));
-const save_task_data_in_cache = (task_name, data) => {
-    if (Array.isArray(data)) {
-        cache_manager.setArrayData(task_name, data);
-    }
-    else {
-        cache_manager.setObjectData(task_name, data);
-    }
-};
-export const execute_task = async (source, task_name, task, options) => {
-    const { debug_logs = true, save_in } = options || {};
-    try {
-        await set_document("nx-tasks", task_name, {
-            source,
-            status: TaskStatus.running,
-            started: new Date(),
-            timestamp: new Date(),
-            error: "",
-        }, false);
-        if (debug_logs) {
-            logger.log(`Task [${task_name}] started`);
-        }
-        const start = performance.now();
-        const data = await task();
-        const update = {
-            status: TaskStatus.completed,
-            completed: new Date(),
-            timestamp: new Date(),
-        };
-        if (data) {
-            if (save_in === "none") {
-                update.data = "no data to save";
-            }
-            else if (save_in === "storage" && typeof data === "object") {
-                save_task_data_in_cache(task_name, data);
-                const url = await keep_task_data_in_storage(task_name, data);
-                update.data = url;
-            }
-            else {
-                save_task_data_in_cache(task_name, data);
-                update.data = data;
-            }
-        }
-        else {
-            update.data = "no data to save";
-        }
-        await set_document("nx-tasks", task_name, update);
-        if (debug_logs) {
-            logger.log(`Task [${task_name}] ended. It took ${Math.round(performance.now() - start)} ms`);
-        }
-    }
-    catch (exception) {
-        const error_for_db = exception instanceof Error ? exception.message : exception;
-        logger.error(`Task [${task_name}] error`, error_for_db);
-        await set_document("nx-tasks", task_name, {
-            status: TaskStatus.failed,
-            completed: new Date(),
-            timestamp: new Date(),
-            error: error_for_db,
-        });
-    }
-};
-export const get_task_data = async (task_name) => {
-    const cached_data = cache_manager.getArrayData(task_name);
-    const cached_data_object = cache_manager.getObjectData(task_name, null);
-    if (cached_data.length > 0) {
-        return cached_data;
-    }
-    if (cached_data_object) {
-        return cached_data;
-    }
-    const task_data = await get_document_by_id_optional("nx-tasks", task_name);
-    if (typeof task_data?.data === "string" && task_data.data.startsWith("http")) {
-        const storage_data = await get_task_data_from_storage(task_name);
-        if (storage_data) {
-            const value = storage_data;
-            save_task_data_in_cache(task_name, storage_data);
-            return value;
-        }
-    }
-    const value = typeof task_data?.data === "object" ? task_data.data : null;
-    if (value) {
-        save_task_data_in_cache(task_name, value);
-    }
-    return value || [];
-};
-export const get_task_data_from_storage = async (task_name) => {
-    const bucket = admin.storage().bucket();
-    const file = bucket.file(`tasks_data/${task_name}.json`);
-    try {
-        const [contents] = await file.download();
-        return JSON.parse(contents.toString("utf-8"));
-    }
-    catch (error) {
-        logger.error("Task error reading file from Firebase Storage:", error);
-        return null;
-    }
-};
-export const keep_task_data_in_storage = async (task_name, data) => {
-    const bucket = admin.storage().bucket();
-    const file = bucket.file(`tasks_data/${task_name}.json`);
-    try {
-        if (typeof data !== "object") {
-            throw new Error("Only arrays or objects can be written");
-        }
-        const json_string = JSON.stringify(data, null);
-        await file.save(json_string, {
-            contentType: "application/json",
-        });
-        const [signed_url] = await file.getSignedUrl({
-            action: "read",
-            expires: Date.now() + 1000 * 60 * 60 * 24 * 7,
-        });
-        return signed_url;
-    }
-    catch (error) {
-        logger.error("Error writing file to Firebase Storage:", error);
-        return "";
-    }
-};