@pioneer-platform/markets 8.12.0 → 8.13.0

package/lib/metrics-logger.d.ts

@@ -0,0 +1,72 @@
+import { Db } from 'mongodb';
+export interface APICallLog {
+    apiName: string;
+    endpoint?: string;
+    method?: string;
+    caip?: string;
+    symbol?: string;
+    tokenTier?: number;
+    success: boolean;
+    responseTime?: number;
+    statusCode?: number;
+    price?: number;
+    error?: string;
+    tokensUsed?: number;
+    tokensRemaining?: number;
+    costUSD?: number;
+    wasRateLimited?: boolean;
+    wasThrottled?: boolean;
+    cacheHit?: boolean;
+    timestamp: Date;
+    requestId?: string;
+}
+export declare class MetricsLogger {
+    private db;
+    private logsCollection;
+    private enabled;
+    constructor(db: Db);
+    /**
+     * Enable/disable logging (for performance testing)
+     */
+    setEnabled(enabled: boolean): void;
+    /**
+     * Log API call
+     */
+    logAPICall(logData: APICallLog): Promise<void>;
+    /**
+     * Get hourly stats
+     */
+    getHourlyStats(hour: Date): Promise<any[]>;
+    /**
+     * Get daily stats
+     */
+    getDailyStats(date: Date): Promise<any[]>;
+    /**
+     * Get stats for date range
+     */
+    getStatsForRange(startDate: Date, endDate: Date): Promise<any[]>;
+    /**
+     * Get top requested tokens
+     */
+    getTopTokens(limit?: number, hours?: number): Promise<any[]>;
+    /**
+     * Get error rate for API
+     */
+    getErrorRate(apiName: string, hours?: number): Promise<number>;
+    /**
+     * Get rate limit events
+     */
+    getRateLimitEvents(apiName?: string, hours?: number): Promise<any[]>;
+    /**
+     * Get cost summary for month
+     */
+    getMonthlyCostSummary(year: number, month: number): Promise<any>;
+    /**
+     * Clean up old logs (run periodically)
+     */
+    cleanupOldLogs(daysToKeep?: number): Promise<number>;
+    /**
+     * Get cache hit rate
+     */
+    getCacheHitRate(hours?: number): Promise<number>;
+}

package/lib/metrics-logger.js

@@ -0,0 +1,435 @@
+"use strict";
+/*
+    Metrics Logger
+
+    Logs all API calls to MongoDB for analytics and cost tracking
+*/
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
+    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MetricsLogger = void 0;
+var log = require('@pioneer-platform/loggerdog')();
+var TAG = ' | MetricsLogger | ';
+var MetricsLogger = /** @class */ (function () {
+    function MetricsLogger(db) {
+        this.enabled = true;
+        this.db = db;
+        this.logsCollection = db.collection('api_usage_logs');
+    }
+    /**
+     * Enable/disable logging (for performance testing)
+     */
+    MetricsLogger.prototype.setEnabled = function (enabled) {
+        this.enabled = enabled;
+    };
+    /**
+     * Log API call
+     */
+    MetricsLogger.prototype.logAPICall = function (logData) {
+        return __awaiter(this, void 0, void 0, function () {
+            var error_1;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        if (!this.enabled)
+                            return [2 /*return*/];
+                        _a.label = 1;
+                    case 1:
+                        _a.trys.push([1, 3, , 4]);
+                        return [4 /*yield*/, this.logsCollection.insertOne(__assign(__assign({}, logData), { timestamp: logData.timestamp || new Date() }))];
+                    case 2:
+                        _a.sent();
+                        return [3 /*break*/, 4];
+                    case 3:
+                        error_1 = _a.sent();
+                        // Don't block on metrics errors, but log them
+                        log.debug(TAG, "Failed to log API metrics: ".concat(error_1.message));
+                        return [3 /*break*/, 4];
+                    case 4: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    /**
+     * Get hourly stats
+     */
+    MetricsLogger.prototype.getHourlyStats = function (hour) {
+        return __awaiter(this, void 0, void 0, function () {
+            var hourStart, hourEnd, pipeline;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        hourStart = new Date(hour);
+                        hourStart.setMinutes(0, 0, 0);
+                        hourEnd = new Date(hourStart);
+                        hourEnd.setHours(hourStart.getHours() + 1);
+                        pipeline = [
+                            {
+                                $match: {
+                                    timestamp: { $gte: hourStart, $lt: hourEnd }
+                                }
+                            },
+                            {
+                                $group: {
+                                    _id: '$apiName',
+                                    requestCount: { $sum: 1 },
+                                    successCount: { $sum: { $cond: ['$success', 1, 0] } },
+                                    errorCount: { $sum: { $cond: ['$success', 0, 1] } },
+                                    rateLimitCount: { $sum: { $cond: ['$wasRateLimited', 1, 0] } },
+                                    tokensUsed: { $sum: { $ifNull: ['$tokensUsed', 0] } },
+                                    totalCost: { $sum: { $ifNull: ['$costUSD', 0] } },
+                                    avgResponseTime: { $avg: '$responseTime' }
+                                }
+                            }
+                        ];
+                        return [4 /*yield*/, this.logsCollection.aggregate(pipeline).toArray()];
+                    case 1: return [2 /*return*/, _a.sent()];
+                }
+            });
+        });
+    };
+    /**
+     * Get daily stats
+     */
+    MetricsLogger.prototype.getDailyStats = function (date) {
+        return __awaiter(this, void 0, void 0, function () {
+            var dayStart, dayEnd, pipeline;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        dayStart = new Date(date);
+                        dayStart.setHours(0, 0, 0, 0);
+                        dayEnd = new Date(dayStart);
+                        dayEnd.setDate(dayStart.getDate() + 1);
+                        pipeline = [
+                            {
+                                $match: {
+                                    timestamp: { $gte: dayStart, $lt: dayEnd }
+                                }
+                            },
+                            {
+                                $group: {
+                                    _id: '$apiName',
+                                    requestCount: { $sum: 1 },
+                                    successCount: { $sum: { $cond: ['$success', 1, 0] } },
+                                    errorCount: { $sum: { $cond: ['$success', 0, 1] } },
+                                    rateLimitCount: { $sum: { $cond: ['$wasRateLimited', 1, 0] } },
+                                    tokensUsed: { $sum: { $ifNull: ['$tokensUsed', 0] } },
+                                    totalCost: { $sum: { $ifNull: ['$costUSD', 0] } },
+                                    avgResponseTime: { $avg: '$responseTime' }
+                                }
+                            }
+                        ];
+                        return [4 /*yield*/, this.logsCollection.aggregate(pipeline).toArray()];
+                    case 1: return [2 /*return*/, _a.sent()];
+                }
+            });
+        });
+    };
+    /**
+     * Get stats for date range
+     */
+    MetricsLogger.prototype.getStatsForRange = function (startDate, endDate) {
+        return __awaiter(this, void 0, void 0, function () {
+            var pipeline;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        pipeline = [
+                            {
+                                $match: {
+                                    timestamp: { $gte: startDate, $lt: endDate }
+                                }
+                            },
+                            {
+                                $group: {
+                                    _id: {
+                                        apiName: '$apiName',
+                                        date: { $dateToString: { format: '%Y-%m-%d', date: '$timestamp' } }
+                                    },
+                                    requestCount: { $sum: 1 },
+                                    successCount: { $sum: { $cond: ['$success', 1, 0] } },
+                                    tokensUsed: { $sum: { $ifNull: ['$tokensUsed', 0] } },
+                                    totalCost: { $sum: { $ifNull: ['$costUSD', 0] } }
+                                }
+                            },
+                            {
+                                $sort: { '_id.date': -1, '_id.apiName': 1 }
+                            }
+                        ];
+                        return [4 /*yield*/, this.logsCollection.aggregate(pipeline).toArray()];
+                    case 1: return [2 /*return*/, _a.sent()];
+                }
+            });
+        });
+    };
+    /**
+     * Get top requested tokens
+     */
+    MetricsLogger.prototype.getTopTokens = function () {
+        return __awaiter(this, arguments, void 0, function (limit, hours) {
+            var since, pipeline;
+            if (limit === void 0) { limit = 10; }
+            if (hours === void 0) { hours = 24; }
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        since = new Date(Date.now() - hours * 60 * 60 * 1000);
+                        pipeline = [
+                            {
+                                $match: {
+                                    caip: { $exists: true },
+                                    timestamp: { $gte: since }
+                                }
+                            },
+                            {
+                                $group: {
+                                    _id: '$caip',
+                                    symbol: { $first: '$symbol' },
+                                    requestCount: { $sum: 1 },
+                                    tokensUsed: { $sum: { $ifNull: ['$tokensUsed', 0] } }
+                                }
+                            },
+                            {
+                                $sort: { requestCount: -1 }
+                            },
+                            {
+                                $limit: limit
+                            }
+                        ];
+                        return [4 /*yield*/, this.logsCollection.aggregate(pipeline).toArray()];
+                    case 1: return [2 /*return*/, _a.sent()];
+                }
+            });
+        });
+    };
+    /**
+     * Get error rate for API
+     */
+    MetricsLogger.prototype.getErrorRate = function (apiName_1) {
+        return __awaiter(this, arguments, void 0, function (apiName, hours) {
+            var since, pipeline, result, _a, total, errors;
+            if (hours === void 0) { hours = 24; }
+            return __generator(this, function (_b) {
+                switch (_b.label) {
+                    case 0:
+                        since = new Date(Date.now() - hours * 60 * 60 * 1000);
+                        pipeline = [
+                            {
+                                $match: {
+                                    apiName: apiName,
+                                    timestamp: { $gte: since }
+                                }
+                            },
+                            {
+                                $group: {
+                                    _id: null,
+                                    total: { $sum: 1 },
+                                    errors: { $sum: { $cond: ['$success', 0, 1] } }
+                                }
+                            }
+                        ];
+                        return [4 /*yield*/, this.logsCollection.aggregate(pipeline).toArray()];
+                    case 1:
+                        result = _b.sent();
+                        if (result.length === 0)
+                            return [2 /*return*/, 0];
+                        _a = result[0], total = _a.total, errors = _a.errors;
+                        return [2 /*return*/, errors / total];
+                }
+            });
+        });
+    };
+    /**
+     * Get rate limit events
+     */
+    MetricsLogger.prototype.getRateLimitEvents = function (apiName_1) {
+        return __awaiter(this, arguments, void 0, function (apiName, hours) {
+            var since, match;
+            if (hours === void 0) { hours = 24; }
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        since = new Date(Date.now() - hours * 60 * 60 * 1000);
+                        match = {
+                            wasRateLimited: true,
+                            timestamp: { $gte: since }
+                        };
+                        if (apiName) {
+                            match.apiName = apiName;
+                        }
+                        return [4 /*yield*/, this.logsCollection
+                                .find(match)
+                                .sort({ timestamp: -1 })
+                                .limit(100)
+                                .toArray()];
+                    case 1: return [2 /*return*/, _a.sent()];
+                }
+            });
+        });
+    };
+    /**
+     * Get cost summary for month
+     */
+    MetricsLogger.prototype.getMonthlyCostSummary = function (year, month) {
+        return __awaiter(this, void 0, void 0, function () {
+            var startDate, endDate, pipeline, results, totalCost, totalRequests, totalTokens, byAPI, _i, results_1, result, apiName;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        startDate = new Date(year, month - 1, 1);
+                        endDate = new Date(year, month, 1);
+                        pipeline = [
+                            {
+                                $match: {
+                                    timestamp: { $gte: startDate, $lt: endDate }
+                                }
+                            },
+                            {
+                                $group: {
+                                    _id: '$apiName',
+                                    totalCost: { $sum: { $ifNull: ['$costUSD', 0] } },
+                                    requestCount: { $sum: 1 },
+                                    tokensUsed: { $sum: { $ifNull: ['$tokensUsed', 0] } }
+                                }
+                            }
+                        ];
+                        return [4 /*yield*/, this.logsCollection.aggregate(pipeline).toArray()];
+                    case 1:
+                        results = _a.sent();
+                        totalCost = 0;
+                        totalRequests = 0;
+                        totalTokens = 0;
+                        byAPI = {};
+                        for (_i = 0, results_1 = results; _i < results_1.length; _i++) {
+                            result = results_1[_i];
+                            apiName = result._id;
+                            byAPI[apiName] = {
+                                cost: result.totalCost,
+                                requests: result.requestCount,
+                                tokens: result.tokensUsed
+                            };
+                            totalCost += result.totalCost;
+                            totalRequests += result.requestCount;
+                            totalTokens += result.tokensUsed;
+                        }
+                        return [2 /*return*/, {
+                                month: "".concat(year, "-").concat(String(month).padStart(2, '0')),
+                                byAPI: byAPI,
+                                totals: {
+                                    cost: totalCost,
+                                    requests: totalRequests,
+                                    tokens: totalTokens
+                                }
+                            }];
+                }
+            });
+        });
+    };
+    /**
+     * Clean up old logs (run periodically)
+     */
+    MetricsLogger.prototype.cleanupOldLogs = function () {
+        return __awaiter(this, arguments, void 0, function (daysToKeep) {
+            var tag, cutoffDate, result;
+            if (daysToKeep === void 0) { daysToKeep = 90; }
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        tag = TAG + ' | cleanupOldLogs | ';
+                        cutoffDate = new Date(Date.now() - daysToKeep * 24 * 60 * 60 * 1000);
+                        return [4 /*yield*/, this.logsCollection.deleteMany({
+                                timestamp: { $lt: cutoffDate }
+                            })];
+                    case 1:
+                        result = _a.sent();
+                        log.info(tag, "Deleted ".concat(result.deletedCount, " logs older than ").concat(daysToKeep, " days"));
+                        return [2 /*return*/, result.deletedCount];
+                }
+            });
+        });
+    };
+    /**
+     * Get cache hit rate
+     */
+    MetricsLogger.prototype.getCacheHitRate = function () {
+        return __awaiter(this, arguments, void 0, function (hours) {
+            var since, pipeline, result, _a, total, cacheHits;
+            if (hours === void 0) { hours = 24; }
+            return __generator(this, function (_b) {
+                switch (_b.label) {
+                    case 0:
+                        since = new Date(Date.now() - hours * 60 * 60 * 1000);
+                        pipeline = [
+                            {
+                                $match: {
+                                    timestamp: { $gte: since }
+                                }
+                            },
+                            {
+                                $group: {
+                                    _id: null,
+                                    total: { $sum: 1 },
+                                    cacheHits: { $sum: { $cond: ['$cacheHit', 1, 0] } }
+                                }
+                            }
+                        ];
+                        return [4 /*yield*/, this.logsCollection.aggregate(pipeline).toArray()];
+                    case 1:
+                        result = _b.sent();
+                        if (result.length === 0)
+                            return [2 /*return*/, 0];
+                        _a = result[0], total = _a.total, cacheHits = _a.cacheHits;
+                        return [2 /*return*/, cacheHits / total];
+                }
+            });
+        });
+    };
+    return MetricsLogger;
+}());
+exports.MetricsLogger = MetricsLogger;

package/lib/refill-scheduler.d.ts

@@ -0,0 +1,22 @@
+import { TokenBucketManager } from './token-bucket-manager';
+import { MetricsLogger } from './metrics-logger';
+/**
+ * Schedule daily token bucket refill at midnight UTC
+ */
+export declare function scheduleDailyRefill(manager: TokenBucketManager, metricsLogger?: MetricsLogger): void;
+/**
+ * Manual refill trigger (for admin use)
+ */
+export declare function manualRefill(manager: TokenBucketManager, metricsLogger?: MetricsLogger): Promise<void>;
+/**
+ * Cancel scheduled refill job
+ */
+export declare function cancelRefillSchedule(): void;
+/**
+ * Get next refill time
+ */
+export declare function getNextRefillTime(): Date | null;
+/**
+ * Check if refill is scheduled
+ */
+export declare function isRefillScheduled(): boolean;