@queuedash/sdk 0.2.0 → 2.0.0

package/dist/client.js

@@ -1,7 +1,41 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.QueueDashClient = void 0;
-class QueueDashClient {
+exports.Queuedash = void 0;
+const MAX_PAYLOAD_SIZE = 4.5 * 1024 * 1024; // 4.5MB
+class Queuedash {
     constructor(options) {
         this.jobQueue = [];
         this.isFlushing = false;
@@ -9,29 +43,246 @@ class QueueDashClient {
         this.circuitBreakerFailures = 0;
         this.circuitBreakerOpen = false;
         this.syncingJobIds = new Set();
+        // Track attached resources for cleanup
+        this.managedQueueEvents = [];
+        this.attachedQueues = new Set();
         if (!options.apiKey) {
-            throw new Error("QueueDash: apiKey is required");
+            throw new Error("Queuedash: apiKey is required");
         }
         this.apiKey = options.apiKey;
-        // Hardcoded SaaS API URL - can be overridden for staging only
         this.apiUrl =
-            options.__dangerouslyOverrideApiUrl ||
+            options.baseUrl ||
                 (process.env.NODE_ENV === "production"
-                    ? "https://api.queuedash.com"
-                    : "http://localhost:4001");
-        this.batchSize = options.batchSize || 50;
-        this.flushInterval = options.flushInterval || 100; // ms
-        this.maxRetries = options.maxRetries || 5;
-        this.requestTimeout = options.requestTimeout || 30000; // 30s default
-        this.maxQueueSize = options.maxQueueSize || 10000; // Prevent memory issues
+                    ? "https://sync.queuedash.com"
+                    : "http://localhost:4002");
+        this.batchSize = options.batchSize ?? 50;
+        this.flushInterval = options.flushInterval ?? 100;
+        this.maxRetries = options.maxRetries ?? 5;
+        this.requestTimeout = options.requestTimeout ?? 30000;
+        this.maxQueueSize = options.maxQueueSize ?? 10000;
         this.onError = options.onError;
-        // Start auto-flush timer
         this.startFlushTimer();
-        // Graceful shutdown handlers
         this.setupShutdownHandlers();
     }
     /**
-     * Delete a job (soft delete in QueueDash)
+     * Attach a queue or worker for monitoring.
+     * Auto-detects the queue library (BullMQ, Bull, Bee-Queue, GroupMQ).
+     *
+     * @example
+     * ```ts
+     * const qd = new Queuedash({ apiKey: 'qd_...' });
+     *
+     * // BullMQ - we create QueueEvents internally
+     * qd.attach(myBullMQQueue);
+     *
+     * // Bull
+     * qd.attach(myBullQueue);
+     *
+     * // Bee-Queue
+     * qd.attach(myBeeQueue);
+     * ```
+     */
+    attach(resource, options) {
+        const type = this.detectType(resource);
+        switch (type) {
+            case "bullmq-queue":
+                this.attachBullMQ(resource, options?.events);
+                break;
+            case "bullmq-worker":
+                this.attachBullMQWorker(resource);
+                break;
+            case "bull":
+                this.attachBull(resource);
+                break;
+            case "bee":
+                this.attachBee(resource);
+                break;
+            default:
+                throw new Error("Queuedash: Unknown queue type. Supported: BullMQ Queue/Worker, Bull, Bee-Queue, GroupMQ");
+        }
+        return this;
+    }
+    detectType(resource) {
+        if (!resource || typeof resource !== "object") {
+            return "unknown";
+        }
+        // BullMQ Queue: has getJob method and opts.connection
+        if (typeof resource.getJob === "function" &&
+            resource.opts?.connection !== undefined) {
+            return "bullmq-queue";
+        }
+        // BullMQ Worker: has opts.connection but no getJob
+        if (resource.opts?.connection !== undefined &&
+            typeof resource.getJob !== "function" &&
+            resource.name) {
+            return "bullmq-worker";
+        }
+        // Bull: has clients array (redis clients)
+        if (Array.isArray(resource.clients)) {
+            return "bull";
+        }
+        // Bee-Queue: has settings object
+        if (resource.settings !== undefined && resource.name) {
+            return "bee";
+        }
+        return "unknown";
+    }
+    async attachBullMQ(queue, existingEvents) {
+        const queueName = queue.name;
+        if (this.attachedQueues.has(`bullmq:${queueName}`)) {
+            console.warn(`Queuedash: Queue "${queueName}" is already attached`);
+            return;
+        }
+        let events;
+        // Create QueueEvents if not provided
+        if (existingEvents) {
+            events = existingEvents;
+        }
+        else {
+            try {
+                // Dynamic import to avoid requiring bullmq as a direct dependency
+                const { QueueEvents } = await Promise.resolve().then(() => __importStar(require("bullmq")));
+                const queueEvents = new QueueEvents(queueName, {
+                    connection: queue.opts?.connection,
+                    prefix: queue.redisPrefix,
+                });
+                this.managedQueueEvents.push(queueEvents);
+                events = queueEvents;
+            }
+            catch (e) {
+                throw new Error("Queuedash: Failed to create QueueEvents. Make sure bullmq is installed.");
+            }
+        }
+        this.attachedQueues.add(`bullmq:${queueName}`);
+        const syncJobById = (jobId, status) => {
+            queue.getJob(jobId).then((job) => {
+                if (job) {
+                    this.syncJob(this.extractBullMQJobData(job, queueName, status));
+                }
+            });
+        };
+        events.on("waiting", ({ jobId }) => syncJobById(jobId, "waiting"));
+        events.on("progress", ({ jobId }) => syncJobById(jobId, "active"));
+        events.on("completed", ({ jobId }) => syncJobById(jobId, "completed"));
+        events.on("failed", ({ jobId }) => syncJobById(jobId, "failed"));
+        events.on("waiting-children", ({ jobId }) => syncJobById(jobId, "waiting-children"));
+        events.on("added", ({ jobId }) => syncJobById(jobId, "waiting"));
+        events.on("active", ({ jobId }) => syncJobById(jobId, "active"));
+        events.on("delayed", ({ jobId }) => syncJobById(jobId, "delayed"));
+        events.on("deduplicated", ({ jobId }) => syncJobById(jobId));
+        events.on("removed", ({ jobId }) => this.deleteJob(jobId, queueName));
+        events.on("stalled", ({ jobId }) => syncJobById(jobId, "waiting"));
+        events.on("retries-exhausted", ({ jobId }) => syncJobById(jobId, "failed"));
+    }
+    attachBullMQWorker(_worker) {
+        // TODO: Worker monitoring - track active jobs, stalled detection, etc.
+        console.warn("Queuedash: Worker monitoring coming soon");
+    }
+    attachBull(queue) {
+        const queueName = queue.name;
+        if (this.attachedQueues.has(`bull:${queueName}`)) {
+            console.warn(`Queuedash: Queue "${queueName}" is already attached`);
+            return;
+        }
+        this.attachedQueues.add(`bull:${queueName}`);
+        queue.on("global:completed", (job) => {
+            this.syncJob(this.extractBullJobData(job, queueName));
+        });
+        queue.on("global:failed", (job) => {
+            if (job) {
+                this.syncJob(this.extractBullJobData(job, queueName));
+            }
+        });
+        queue.on("active", (job) => {
+            this.syncJob(this.extractBullJobData(job, queueName));
+        });
+        queue.on("progress", (job) => {
+            this.syncJob(this.extractBullJobData(job, queueName));
+        });
+        queue.on("removed", (job) => {
+            this.syncJob(this.extractBullJobData(job, queueName));
+        });
+    }
+    attachBee(queue) {
+        const queueName = queue.name;
+        if (this.attachedQueues.has(`bee:${queueName}`)) {
+            console.warn(`Queuedash: Queue "${queueName}" is already attached`);
+            return;
+        }
+        this.attachedQueues.add(`bee:${queueName}`);
+        queue.on("succeeded", (job) => {
+            const jobData = this.extractBeeJobData(job, queueName);
+            jobData.finishedAt = new Date();
+            this.syncJob(jobData);
+        });
+        queue.on("failed", (job, err) => {
+            const jobData = this.extractBeeJobData(job, queueName);
+            jobData.failedReason = err.message;
+            jobData.stacktrace = err.stack ? [err.stack] : undefined;
+            jobData.finishedAt = new Date();
+            this.syncJob(jobData);
+        });
+        queue.on("retrying", (job) => {
+            const jobData = this.extractBeeJobData(job, queueName);
+            jobData.retriedAt = new Date();
+            this.syncJob(jobData);
+        });
+    }
+    extractBullMQJobData(job, queueName, status) {
+        return {
+            jobId: job.id,
+            name: job.name,
+            queueName,
+            data: job.data,
+            opts: job.opts,
+            addedAt: new Date(job.timestamp),
+            processedAt: job.processedOn ? new Date(job.processedOn) : null,
+            finishedAt: job.finishedOn ? new Date(job.finishedOn) : null,
+            failedReason: job.failedReason,
+            stacktrace: job.stacktrace,
+            priority: job.opts?.priority,
+            delay: job.opts?.delay,
+            timestamp: job.timestamp,
+            progress: typeof job.progress === "number" ? job.progress : null,
+            status,
+        };
+    }
+    extractBullJobData(job, queueName) {
+        return {
+            jobId: job.id,
+            name: job.name || "default",
+            queueName,
+            data: job.data,
+            opts: job.opts,
+            addedAt: new Date(job.timestamp),
+            processedAt: job.processedOn ? new Date(job.processedOn) : null,
+            finishedAt: job.finishedOn ? new Date(job.finishedOn) : null,
+            failedReason: job.failedReason,
+            stacktrace: job.stacktrace,
+            priority: job.opts?.priority,
+            delay: job.opts?.delay,
+            timestamp: job.timestamp,
+            progress: job.progress(),
+        };
+    }
+    extractBeeJobData(job, queueName) {
+        return {
+            jobId: String(job.id),
+            name: "default",
+            queueName,
+            data: job.data,
+            opts: job.options || {},
+            addedAt: job.options?.timestamp
+                ? new Date(job.options.timestamp)
+                : new Date(),
+            processedAt: null,
+            finishedAt: null,
+            priority: null,
+            delay: job.options?.delay,
+        };
+    }
+    /**
+     * Delete a job (soft delete in Queuedash)
      */
     async deleteJob(jobId, queueName) {
         try {
@@ -56,32 +307,27 @@ class QueueDashClient {
                 this.onError(new Error(`Failed to delete job ${jobId}: ${err.message}`));
             }
             else {
-                console.error(`QueueDash: Failed to delete job ${jobId}`, err);
+                console.error(`Queuedash: Failed to delete job ${jobId}`, err);
             }
         }
     }
     /**
-     * Queue a job for syncing
+     * Queue a job for syncing (internal use)
      */
     syncJob(job) {
-        // Check circuit breaker
         if (this.circuitBreakerOpen) {
-            this.logger("Circuit breaker open, dropping job");
             return;
         }
-        // Prevent duplicate syncs of same job
         const jobKey = `${job.queueName}:${job.jobId}`;
         if (this.syncingJobIds.has(jobKey)) {
             return;
         }
         this.syncingJobIds.add(jobKey);
-        // Prevent memory leak - drop oldest jobs if queue too large
         if (this.jobQueue.length >= this.maxQueueSize) {
             const dropped = this.jobQueue.shift();
-            this.logger(`Queue size limit reached (${this.maxQueueSize}), dropping oldest job: ${dropped?.jobId}`);
+            this.log(`Queue size limit reached (${this.maxQueueSize}), dropping oldest job: ${dropped?.jobId}`);
         }
         this.jobQueue.push(job);
-        // Flush if batch size reached
         if (this.jobQueue.length >= this.batchSize) {
             this.flush();
         }
@@ -99,30 +345,23 @@ class QueueDashClient {
         const jobsToSync = [...this.jobQueue];
         this.jobQueue = [];
         try {
-            // Add timeout to fetch
-            const controller = new AbortController();
-            const timeoutId = setTimeout(() => controller.abort(), this.requestTimeout);
-            const response = await fetch(`${this.apiUrl}/api/v1/jobs/sync`, {
-                method: "POST",
-                headers: {
-                    "Content-Type": "application/json",
-                    Authorization: `Bearer ${this.apiKey}`,
-                },
-                body: JSON.stringify({
-                    jobs: jobsToSync,
-                }),
-                signal: controller.signal,
-            });
-            clearTimeout(timeoutId);
-            if (!response.ok) {
-                const errorText = await response.text();
-                throw new Error(`HTTP ${response.status}: ${errorText}`);
-            }
-            const result = (await response.json());
-            if (!result.success && result.errors) {
-                console.warn("QueueDash: Some jobs failed to sync:", result.errors);
+            // Check if payload is too large and split into chunks if needed
+            const payload = JSON.stringify({ jobs: jobsToSync });
+            if (payload.length > MAX_PAYLOAD_SIZE) {
+                const chunks = this.splitIntoChunks(jobsToSync, MAX_PAYLOAD_SIZE);
+                for (const chunk of chunks) {
+                    await this.sendBatch(chunk);
+                    // Remove successfully sent jobs so they aren't re-queued on later chunk failure
+                    jobsToSync.splice(0, chunk.length);
+                    chunk.forEach((job) => {
+                        this.syncingJobIds.delete(`${job.queueName}:${job.jobId}`);
+                    });
+                }
+                this.retryCount = 0;
+                this.circuitBreakerFailures = 0;
+                return;
             }
-            // Success - reset counters and clear syncing jobs
+            await this.sendBatch(jobsToSync);
             this.retryCount = 0;
             this.circuitBreakerFailures = 0;
             jobsToSync.forEach((job) => {
@@ -133,17 +372,13 @@ class QueueDashClient {
             this.retryCount++;
             this.circuitBreakerFailures++;
             const err = error instanceof Error ? error : new Error(String(error));
-            // Circuit breaker - open after 10 consecutive failures
             if (this.circuitBreakerFailures >= 10) {
                 this.openCircuitBreaker();
             }
-            // Check if we should retry
             if (this.retryCount <= this.maxRetries) {
-                // Put jobs back in queue for retry with exponential backoff
                 this.jobQueue.unshift(...jobsToSync);
-                const backoffMs = Math.min(1000 * Math.pow(2, this.retryCount - 1), 30000); // Max 30s
-                this.logger(`Retry ${this.retryCount}/${this.maxRetries} after ${backoffMs}ms`, err);
-                // Schedule retry
+                const backoffMs = Math.min(1000 * Math.pow(2, this.retryCount - 1), 30000);
+                this.log(`Retry ${this.retryCount}/${this.maxRetries} after ${backoffMs}ms`, err);
                 setTimeout(() => {
                     if (this.jobQueue.length > 0) {
                         this.flush();
@@ -151,7 +386,6 @@ class QueueDashClient {
                 }, backoffMs);
             }
             else {
-                // Max retries reached - clear syncing jobs and alert
                 this.retryCount = 0;
                 jobsToSync.forEach((job) => {
                     this.syncingJobIds.delete(`${job.queueName}:${job.jobId}`);
@@ -161,7 +395,7 @@ class QueueDashClient {
                     this.onError(new Error(`${errorMsg}: ${err.message}`));
                 }
                 else {
-                    console.error(`QueueDash: ${errorMsg}`, err);
+                    console.error(`Queuedash: ${errorMsg}`, err);
                 }
             }
         }
@@ -169,32 +403,71 @@ class QueueDashClient {
             this.isFlushing = false;
         }
     }
+    splitIntoChunks(jobs, maxSize) {
+        const chunks = [];
+        let currentChunk = [];
+        let currentSize = 0;
+        for (const job of jobs) {
+            const jobSize = JSON.stringify(job).length;
+            if (currentSize + jobSize > maxSize && currentChunk.length > 0) {
+                chunks.push(currentChunk);
+                currentChunk = [];
+                currentSize = 0;
+            }
+            currentChunk.push(job);
+            currentSize += jobSize;
+        }
+        if (currentChunk.length > 0) {
+            chunks.push(currentChunk);
+        }
+        return chunks;
+    }
+    async sendBatch(jobs) {
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), this.requestTimeout);
+        const response = await fetch(`${this.apiUrl}/api/v1/jobs/sync`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                Authorization: `Bearer ${this.apiKey}`,
+            },
+            body: JSON.stringify({ jobs }),
+            signal: controller.signal,
+        });
+        clearTimeout(timeoutId);
+        if (!response.ok) {
+            const errorText = await response.text();
+            throw new Error(`HTTP ${response.status}: ${errorText}`);
+        }
+        const result = (await response.json());
+        if (!result.success && result.errors) {
+            console.warn("Queuedash: Some jobs failed to sync:", result.errors);
+        }
+    }
     openCircuitBreaker() {
         if (this.circuitBreakerOpen)
             return;
         this.circuitBreakerOpen = true;
-        console.error("QueueDash: Circuit breaker opened after 10 consecutive failures. Will retry in 60s.");
-        // Auto-reset after 60 seconds
+        console.error("Queuedash: Circuit breaker opened after 10 consecutive failures. Will retry in 60s.");
         this.circuitBreakerResetTimer = setTimeout(() => {
             this.circuitBreakerOpen = false;
             this.circuitBreakerFailures = 0;
-            console.log("QueueDash: Circuit breaker reset, resuming sync attempts");
+            console.log("Queuedash: Circuit breaker reset, resuming sync attempts");
         }, 60000);
         if (this.circuitBreakerResetTimer.unref) {
             this.circuitBreakerResetTimer.unref();
         }
     }
-    logger(message, error) {
+    log(message, error) {
         if (this.onError && error) {
-            // If onError is provided, use it for warnings too
-            console.warn(`QueueDash: ${message}`, error);
+            console.warn(`Queuedash: ${message}`, error);
         }
         else {
-            console.warn(`QueueDash: ${message}`);
+            console.warn(`Queuedash: ${message}`);
         }
     }
     /**
-     * Stop the client and flush remaining jobs
+     * Stop the client, close managed resources, and flush remaining jobs
      */
     async stop() {
         if (this.flushTimer) {
@@ -203,6 +476,15 @@ class QueueDashClient {
         if (this.circuitBreakerResetTimer) {
             clearTimeout(this.circuitBreakerResetTimer);
         }
+        // Close any QueueEvents we created
+        for (const events of this.managedQueueEvents) {
+            try {
+                await events.close();
+            }
+            catch {
+                // Ignore close errors
+            }
+        }
         await this.flush();
     }
     startFlushTimer() {
@@ -211,25 +493,23 @@ class QueueDashClient {
                 this.flush();
             }
         }, this.flushInterval);
-        // Don't keep process alive
         if (this.flushTimer.unref) {
             this.flushTimer.unref();
         }
     }
     setupShutdownHandlers() {
         const shutdown = async () => {
-            console.log("QueueDash: Graceful shutdown - flushing remaining jobs...");
+            console.log("Queuedash: Graceful shutdown - flushing remaining jobs...");
             await this.stop();
             process.exit(0);
         };
-        // Handle graceful shutdown
         process.once("SIGTERM", shutdown);
         process.once("SIGINT", shutdown);
         process.once("beforeExit", () => {
             if (this.jobQueue.length > 0) {
-                console.warn(`QueueDash: Process exiting with ${this.jobQueue.length} unsync'd jobs`);
+                console.warn(`Queuedash: Process exiting with ${this.jobQueue.length} unsync'd jobs`);
             }
         });
     }
 }
-exports.QueueDashClient = QueueDashClient;
+exports.Queuedash = Queuedash;

package/dist/client.spec.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=client.spec.d.ts.map

package/dist/client.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.spec.d.ts","sourceRoot":"","sources":["../src/client.spec.ts"],"names":[],"mappings":""}