@pioneer-platform/markets 8.12.0 → 8.15.0

package/lib/token-bucket.d.ts

@@ -0,0 +1,87 @@
+import { Db } from 'mongodb';
+export interface TokenBucketConfig {
+    apiName: string;
+    capacity: number;
+    refillAmount: number;
+    refillInterval: number;
+    tokensPerRequest: number;
+    costPerToken: number;
+    monthlyBudget: number;
+}
+export interface TokenBucketDocument {
+    apiName: string;
+    capacity: number;
+    currentTokens: number;
+    lastRefill: Date;
+    nextRefill: Date;
+    refillAmount: number;
+    refillInterval: number;
+    tokensPerRequest: number;
+    costPerToken: number;
+    monthlyCost: number;
+    monthlyBudget: number;
+    enabled: boolean;
+    exhausted: boolean;
+    exhaustedAt?: Date;
+    createdAt: Date;
+    updatedAt: Date;
+}
+export declare class TokenBucket {
+    private config;
+    private db;
+    private collection;
+    private cachedTokens;
+    private lastSync;
+    private syncInterval;
+    constructor(config: TokenBucketConfig, db: Db);
+    /**
+     * Initialize bucket (call on startup)
+     */
+    initialize(): Promise<void>;
+    /**
+     * Try to consume tokens (returns true if successful)
+     */
+    tryConsume(count?: number): Promise<boolean>;
+    /**
+     * Get current token count
+     */
+    getTokenCount(): Promise<number>;
+    /**
+     * Get bucket status
+     */
+    getStatus(): Promise<{
+        available: number;
+        capacity: number;
+        exhausted: boolean;
+        nextRefill: Date;
+        utilizationRate: number;
+    }>;
+    /**
+     * Force refill (admin use or scheduled job)
+     */
+    refill(): Promise<void>;
+    /**
+     * Check if refill is needed (call on startup and periodically)
+     */
+    private checkAndRefill;
+    /**
+     * Sync cached tokens to database
+     */
+    private syncToDB;
+    /**
+     * Sync from database to cache
+     */
+    private syncFromDB;
+    /**
+     * Get next midnight UTC
+     */
+    private getNextMidnightUTC;
+    /**
+     * Add tokens (for testing or manual adjustments)
+     */
+    addTokens(count: number): Promise<void>;
+    /**
+     * Get remaining tokens without consuming
+     */
+    getRemainingTokens(): Promise<number>;
+}

package/lib/token-bucket.js

@@ -0,0 +1,341 @@
+"use strict";
+/*
+    Token Bucket Rate Limiting System
+
+    Implements per-API token bucket rate limiting with:
+    - Daily token budgets (refill at midnight UTC)
+    - MongoDB persistence (survives restarts)
+    - In-memory caching (fast checks)
+    - Cost tracking
+*/
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
+    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TokenBucket = void 0;
+var log = require('@pioneer-platform/loggerdog')();
+var TAG = ' | TokenBucket | ';
+var TokenBucket = /** @class */ (function () {
+    function TokenBucket(config, db) {
+        this.syncInterval = 5000; // Sync to DB every 5 seconds
+        this.config = config;
+        this.db = db;
+        this.collection = db.collection('api_token_buckets');
+        this.cachedTokens = 0;
+        this.lastSync = new Date();
+    }
+    /**
+     * Initialize bucket (call on startup)
+     */
+    TokenBucket.prototype.initialize = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var tag, bucket, newBucket;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        tag = TAG + ' | initialize | ';
+                        return [4 /*yield*/, this.collection.findOne({ apiName: this.config.apiName })];
+                    case 1:
+                        bucket = _a.sent();
+                        if (!!bucket) return [3 /*break*/, 3];
+                        // Create new bucket
+                        log.info(tag, "Creating new token bucket for ".concat(this.config.apiName));
+                        newBucket = {
+                            apiName: this.config.apiName,
+                            capacity: this.config.capacity,
+                            currentTokens: this.config.capacity, // Start full
+                            lastRefill: new Date(),
+                            nextRefill: this.getNextMidnightUTC(),
+                            refillAmount: this.config.refillAmount,
+                            refillInterval: this.config.refillInterval,
+                            tokensPerRequest: this.config.tokensPerRequest,
+                            costPerToken: this.config.costPerToken,
+                            monthlyCost: 0,
+                            monthlyBudget: this.config.monthlyBudget,
+                            enabled: true,
+                            exhausted: false,
+                            createdAt: new Date(),
+                            updatedAt: new Date()
+                        };
+                        return [4 /*yield*/, this.collection.insertOne(newBucket)];
+                    case 2:
+                        _a.sent();
+                        return [3 /*break*/, 5];
+                    case 3: 
+                    // Check if refill is needed
+                    return [4 /*yield*/, this.checkAndRefill()];
+                    case 4:
+                        // Check if refill is needed
+                        _a.sent();
+                        _a.label = 5;
+                    case 5: return [4 /*yield*/, this.collection.findOne({ apiName: this.config.apiName })];
+                    case 6:
+                        // Load current tokens into memory
+                        bucket = _a.sent();
+                        this.cachedTokens = bucket.currentTokens;
+                        log.info(tag, "Initialized ".concat(this.config.apiName, ": ").concat(this.cachedTokens, "/").concat(this.config.capacity, " tokens"));
+                        return [2 /*return*/];
+                }
+            });
+        });
+    };
+    /**
+     * Try to consume tokens (returns true if successful)
+     */
+    TokenBucket.prototype.tryConsume = function () {
+        return __awaiter(this, arguments, void 0, function (count) {
+            var tag;
+            if (count === void 0) { count = 1; }
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        tag = TAG + ' | tryConsume | ';
+                        if (!(this.cachedTokens >= count)) return [3 /*break*/, 3];
+                        this.cachedTokens -= count;
+                        if (!(Date.now() - this.lastSync.getTime() > this.syncInterval)) return [3 /*break*/, 2];
+                        return [4 /*yield*/, this.syncToDB()];
+                    case 1:
+                        _a.sent();
+                        _a.label = 2;
+                    case 2: return [2 /*return*/, true];
+                    case 3: 
+                    // Cache might be stale, check DB
+                    return [4 /*yield*/, this.syncFromDB()];
+                    case 4:
+                        // Cache might be stale, check DB
+                        _a.sent();
+                        if (!(this.cachedTokens >= count)) return [3 /*break*/, 6];
+                        this.cachedTokens -= count;
+                        return [4 /*yield*/, this.syncToDB()];
+                    case 5:
+                        _a.sent();
+                        return [2 /*return*/, true];
+                    case 6:
+                        // Not enough tokens
+                        log.warn(tag, "".concat(this.config.apiName, " bucket exhausted! ").concat(this.cachedTokens, "/").concat(count, " needed"));
+                        // Mark as exhausted in DB
+                        return [4 /*yield*/, this.collection.updateOne({ apiName: this.config.apiName }, {
+                                $set: {
+                                    exhausted: true,
+                                    exhaustedAt: new Date(),
+                                    updatedAt: new Date()
+                                }
+                            })];
+                    case 7:
+                        // Mark as exhausted in DB
+                        _a.sent();
+                        return [2 /*return*/, false];
+                }
+            });
+        });
+    };
+    /**
+     * Get current token count
+     */
+    TokenBucket.prototype.getTokenCount = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0: return [4 /*yield*/, this.syncFromDB()];
+                    case 1:
+                        _a.sent();
+                        return [2 /*return*/, this.cachedTokens];
+                }
+            });
+        });
+    };
+    /**
+     * Get bucket status
+     */
+    TokenBucket.prototype.getStatus = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var bucket;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0: return [4 /*yield*/, this.collection.findOne({ apiName: this.config.apiName })];
+                    case 1:
+                        bucket = _a.sent();
+                        if (!bucket) {
+                            throw new Error("Bucket not found: ".concat(this.config.apiName));
+                        }
+                        return [2 /*return*/, {
+                                available: bucket.currentTokens,
+                                capacity: bucket.capacity,
+                                exhausted: bucket.exhausted,
+                                nextRefill: bucket.nextRefill,
+                                utilizationRate: 1 - (bucket.currentTokens / bucket.capacity)
+                            }];
+                }
+            });
+        });
+    };
+    /**
+     * Force refill (admin use or scheduled job)
+     */
+    TokenBucket.prototype.refill = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var tag;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        tag = TAG + ' | refill | ';
+                        return [4 /*yield*/, this.collection.updateOne({ apiName: this.config.apiName }, {
+                                $set: {
+                                    currentTokens: this.config.refillAmount,
+                                    lastRefill: new Date(),
+                                    nextRefill: this.getNextMidnightUTC(),
+                                    exhausted: false,
+                                    exhaustedAt: undefined,
+                                    updatedAt: new Date()
+                                }
+                            })];
+                    case 1:
+                        _a.sent();
+                        this.cachedTokens = this.config.refillAmount;
+                        log.info(tag, "Refilled ".concat(this.config.apiName, " to ").concat(this.config.refillAmount, " tokens"));
+                        return [2 /*return*/];
+                }
+            });
+        });
+    };
+    /**
+     * Check if refill is needed (call on startup and periodically)
+     */
+    TokenBucket.prototype.checkAndRefill = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var bucket;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0: return [4 /*yield*/, this.collection.findOne({ apiName: this.config.apiName })];
+                    case 1:
+                        bucket = _a.sent();
+                        if (!bucket)
+                            return [2 /*return*/];
+                        if (!(new Date() >= bucket.nextRefill)) return [3 /*break*/, 3];
+                        return [4 /*yield*/, this.refill()];
+                    case 2:
+                        _a.sent();
+                        _a.label = 3;
+                    case 3: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    /**
+     * Sync cached tokens to database
+     */
+    TokenBucket.prototype.syncToDB = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0: return [4 /*yield*/, this.collection.updateOne({ apiName: this.config.apiName }, {
+                            $set: {
+                                currentTokens: this.cachedTokens,
+                                updatedAt: new Date()
+                            }
+                        })];
+                    case 1:
+                        _a.sent();
+                        this.lastSync = new Date();
+                        return [2 /*return*/];
+                }
+            });
+        });
+    };
+    /**
+     * Sync from database to cache
+     */
+    TokenBucket.prototype.syncFromDB = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var bucket;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0: return [4 /*yield*/, this.collection.findOne({ apiName: this.config.apiName })];
+                    case 1:
+                        bucket = _a.sent();
+                        if (bucket) {
+                            this.cachedTokens = bucket.currentTokens;
+                            this.lastSync = new Date();
+                        }
+                        return [2 /*return*/];
+                }
+            });
+        });
+    };
+    /**
+     * Get next midnight UTC
+     */
+    TokenBucket.prototype.getNextMidnightUTC = function () {
+        var now = new Date();
+        var midnight = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate() + 1, // Next day
+        0, 0, 0, 0));
+        return midnight;
+    };
+    /**
+     * Add tokens (for testing or manual adjustments)
+     */
+    TokenBucket.prototype.addTokens = function (count) {
+        return __awaiter(this, void 0, void 0, function () {
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        this.cachedTokens = Math.min(this.cachedTokens + count, this.config.capacity);
+                        return [4 /*yield*/, this.syncToDB()];
+                    case 1:
+                        _a.sent();
+                        return [2 /*return*/];
+                }
+            });
+        });
+    };
+    /**
+     * Get remaining tokens without consuming
+     */
+    TokenBucket.prototype.getRemainingTokens = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0: return [4 /*yield*/, this.syncFromDB()];
+                    case 1:
+                        _a.sent();
+                        return [2 /*return*/, this.cachedTokens];
+                }
+            });
+        });
+    };
+    return TokenBucket;
+}());
+exports.TokenBucket = TokenBucket;

package/package.json

@@ -1,20 +1,22 @@
 {
   "name": "@pioneer-platform/markets",
-  "version": "8.12.0",
+  "version": "8.15.0",
   "main": "./lib/index.js",
   "types": "./lib/index.d.ts",
   "dependencies": {
     "@pioneer-platform/default-redis": "^8.11.7",
     "@pioneer-platform/loggerdog": "^8.11.0",
     "@pioneer-platform/pioneer-coins": "^9.11.0",
-    "@pioneer-platform/pioneer-discovery": "^8.12.0",
+    "@pioneer-platform/pioneer-discovery": "^8.14.1",
     "@pioneer-platform/pioneer-types": "^8.11.0",
     "@pioneer-platform/pro-token": "^0.9.0",
     "@shapeshiftoss/caip": "^9.0.0-alpha.0",
     "axios": "^1.6.0",
     "axios-retry": "^3.2.0",
     "bottleneck": "^2.19.5",
-    "dotenv": "^8.2.0"
+    "ccxt": "^4.5.18",
+    "dotenv": "^8.2.0",
+    "node-schedule": "^2.1.1"
   },
   "scripts": {
     "npm": "pnpm i",
@@ -27,6 +29,7 @@
   "devDependencies": {
     "@types/jest": "^25.2.3",
     "@types/node": "^18.16.0",
+    "@types/node-schedule": "^2.1.8",
     "@types/source-map-support": "^0.5.3",
     "jest": "^26.4.2",
     "onchange": "^7.0.2",