@bitclaw/jobs 1.1.0

package/dist/queue.js

@@ -0,0 +1,522 @@
+// packages/jobs/src/queue.ts
+// JobQueue — SQLite-backed job queue with typed payloads and dependency support
+import { Database } from 'bun:sqlite';
+import { mkdirSync } from 'node:fs';
+import { dirname } from 'node:path';
+import { applyPragmas, initializeSchema } from './schema';
+import { nowISO } from './utils';
+import { JobWorker } from './worker';
+function toJob(row) {
+    return {
+        id: row.id,
+        type: row.type,
+        data: JSON.parse(row.data),
+        status: row.status,
+        priority: row.priority,
+        progress: row.progress,
+        maxRetries: row.max_retries,
+        retryCount: row.retry_count,
+        runAt: row.run_at,
+        createdAt: row.created_at,
+        updatedAt: row.updated_at,
+        startedAt: row.started_at,
+        completedAt: row.completed_at,
+        error: row.error,
+        batchId: row.batch_id,
+        requestLog: row.request_log,
+        responseLog: row.response_log
+    };
+}
+function toBatch(row) {
+    return {
+        id: row.id,
+        name: row.name,
+        totalJobs: row.total_jobs,
+        pendingJobs: row.pending_jobs,
+        failedJobs: row.failed_jobs,
+        failedJobIds: JSON.parse(row.failed_job_ids),
+        options: row.options ? JSON.parse(row.options) : null,
+        cancelledAt: row.cancelled_at,
+        createdAt: row.created_at,
+        finishedAt: row.finished_at
+    };
+}
+function toFailedJob(row) {
+    return {
+        id: row.id,
+        originalJobId: row.original_job_id,
+        type: row.type,
+        data: JSON.parse(row.data),
+        error: row.error,
+        retryCount: row.retry_count,
+        maxRetries: row.max_retries,
+        createdAt: row.created_at,
+        failedAt: row.failed_at,
+        requestLog: row.request_log,
+        responseLog: row.response_log
+    };
+}
+export class JobQueue {
+    db;
+    insertJobStmt;
+    insertDepStmt;
+    selectJobStmt;
+    selectPendingStmt;
+    markProcessingStmt;
+    markDoneStmt;
+    markFailedStmt;
+    updateProgressStmt;
+    selectStatsStmt;
+    countFailedStmt;
+    insertFailedJobStmt;
+    deleteJobStmt;
+    selectDependentsStmt;
+    countUnmetDepsStmt;
+    unblockJobStmt;
+    lastInsertRowIdStmt;
+    // Batch statements
+    insertBatchStmt;
+    selectBatchStmt;
+    decrementBatchPendingStmt;
+    incrementBatchFailedStmt;
+    finishBatchStmt;
+    cancelBatchStmt;
+    cancelBatchJobsStmt;
+    constructor(dbPath) {
+        mkdirSync(dirname(dbPath), { recursive: true });
+        this.db = new Database(dbPath, { create: true });
+        applyPragmas(this.db);
+        initializeSchema(this.db);
+        this.insertJobStmt = this.db.query(`
+      INSERT INTO jobs (type, data, status, priority, max_retries, run_at, batch_id)
+      VALUES ($type, $data, $status, $priority, $maxRetries, $runAt, $batchId)
+    `);
+        this.insertDepStmt = this.db.query(`
+      INSERT INTO job_dependencies (job_id, depends_on_id) VALUES ($jobId, $depsOnId)
+    `);
+        this.selectJobStmt = this.db.query('SELECT * FROM jobs WHERE id = $id');
+        this.selectPendingStmt = this.db.query(`
+      SELECT * FROM jobs
+      WHERE status = 'pending' AND type = $type AND run_at <= $now
+      ORDER BY priority DESC, created_at ASC
+      LIMIT 1
+    `);
+        this.markProcessingStmt = this.db.query(`
+      UPDATE jobs SET status = 'processing', started_at = $now, updated_at = $now
+      WHERE id = $id
+    `);
+        this.markDoneStmt = this.db.query(`
+      UPDATE jobs SET status = 'done', completed_at = $now, updated_at = $now, progress = 100
+      WHERE id = $id
+    `);
+        this.markFailedStmt = this.db.query(`
+      UPDATE jobs
+      SET status = 'pending',
+          retry_count = retry_count + 1,
+          error = $error,
+          updated_at = $now
+      WHERE id = $id
+    `);
+        this.updateProgressStmt = this.db.query(`
+      UPDATE jobs SET progress = $progress, updated_at = $now WHERE id = $id
+    `);
+        this.selectStatsStmt = this.db.query('SELECT status, COUNT(*) as count FROM jobs GROUP BY status');
+        this.countFailedStmt = this.db.query('SELECT COUNT(*) as count FROM failed_jobs');
+        this.insertFailedJobStmt = this.db.query(`
+      INSERT INTO failed_jobs (original_job_id, type, data, error, retry_count, max_retries, created_at, request_log, response_log)
+      VALUES ($originalJobId, $type, $data, $error, $retryCount, $maxRetries, $createdAt, $requestLog, $responseLog)
+    `);
+        this.deleteJobStmt = this.db.query('DELETE FROM jobs WHERE id = $id');
+        this.selectDependentsStmt = this.db.query(`
+      SELECT job_id FROM job_dependencies WHERE depends_on_id = $depsOnId
+    `);
+        this.countUnmetDepsStmt = this.db.query(`
+      SELECT COUNT(*) as count FROM job_dependencies jd
+      JOIN jobs j ON jd.depends_on_id = j.id
+      WHERE jd.job_id = $jobId AND j.status != 'done'
+    `);
+        this.unblockJobStmt = this.db.query(`
+      UPDATE jobs SET status = 'pending', updated_at = $now
+      WHERE id = $id AND status = 'blocked'
+    `);
+        this.lastInsertRowIdStmt = this.db.query('SELECT last_insert_rowid() as id');
+        // Batch statements
+        this.insertBatchStmt = this.db.query(`
+      INSERT INTO job_batches (id, name, options, created_at)
+      VALUES ($id, $name, $options, $createdAt)
+    `);
+        this.selectBatchStmt = this.db.query('SELECT * FROM job_batches WHERE id = $id');
+        this.decrementBatchPendingStmt = this.db.query(`
+      UPDATE job_batches SET pending_jobs = pending_jobs - 1
+      WHERE id = $id
+    `);
+        this.incrementBatchFailedStmt = this.db.query(`
+      UPDATE job_batches
+      SET failed_jobs = failed_jobs + 1,
+          failed_job_ids = json_insert(failed_job_ids, '$[#]', $jobId)
+      WHERE id = $id
+    `);
+        this.finishBatchStmt = this.db.query(`
+      UPDATE job_batches SET finished_at = $now WHERE id = $id
+    `);
+        this.cancelBatchStmt = this.db.query(`
+      UPDATE job_batches SET cancelled_at = $now WHERE id = $id
+    `);
+        this.cancelBatchJobsStmt = this.db.query(`
+      UPDATE jobs SET status = 'cancelled', updated_at = $now
+      WHERE batch_id = $batchId AND status IN ('pending', 'blocked')
+    `);
+    }
+    add(type, data, options) {
+        return this.insertJob(type, data, null, options);
+    }
+    getJob(id) {
+        const row = this.selectJobStmt.get({ $id: id });
+        return row ? toJob(row) : null;
+    }
+    getStats() {
+        const rows = this.selectStatsStmt.all();
+        const deadRow = this.countFailedStmt.get();
+        const stats = {
+            pending: 0,
+            blocked: 0,
+            processing: 0,
+            done: 0,
+            failed: 0,
+            cancelled: 0,
+            dead: deadRow.count
+        };
+        for (const row of rows) {
+            const key = row.status;
+            if (key in stats) {
+                stats[key] = row.count;
+            }
+        }
+        return stats;
+    }
+    getFailedJobs(options) {
+        const limit = options?.limit ?? 100;
+        const offset = options?.offset ?? 0;
+        let rows;
+        let total;
+        if (options?.type) {
+            rows = this.db
+                .query('SELECT * FROM failed_jobs WHERE type = $type ORDER BY failed_at DESC LIMIT $limit OFFSET $offset')
+                .all({
+                $type: options.type,
+                $limit: limit,
+                $offset: offset
+            });
+            total = this.db
+                .query('SELECT COUNT(*) as count FROM failed_jobs WHERE type = $type')
+                .get({ $type: options.type }).count;
+        }
+        else {
+            rows = this.db
+                .query('SELECT * FROM failed_jobs ORDER BY failed_at DESC LIMIT $limit OFFSET $offset')
+                .all({ $limit: limit, $offset: offset });
+            total = this.countFailedStmt.get().count;
+        }
+        return { items: rows.map(toFailedJob), total };
+    }
+    listJobs(options) {
+        const limit = options?.limit ?? 50;
+        const offset = options?.offset ?? 0;
+        const conditions = [];
+        const filterParams = {};
+        if (options?.status) {
+            conditions.push('status = $status');
+            filterParams.$status = options.status;
+        }
+        if (options?.type) {
+            conditions.push('type = $type');
+            filterParams.$type = options.type;
+        }
+        const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
+        const rows = this.db
+            .query(`SELECT * FROM jobs ${where} ORDER BY created_at DESC LIMIT $limit OFFSET $offset`)
+            .all({ ...filterParams, $limit: limit, $offset: offset });
+        const total = this.db
+            .query(`SELECT COUNT(*) as count FROM jobs ${where}`)
+            .get(filterParams).count;
+        return { items: rows.map(toJob), total };
+    }
+    cancelJob(id) {
+        const result = this.db
+            .query("UPDATE jobs SET status = 'cancelled', updated_at = $now WHERE id = $id AND status IN ('pending', 'blocked')")
+            .run({ $id: id, $now: nowISO() });
+        return result.changes > 0;
+    }
+    forceRetryJob(id) {
+        const result = this.db
+            .query("UPDATE jobs SET status = 'pending', retry_count = 0, error = NULL, run_at = $now, started_at = NULL, updated_at = $now WHERE id = $id AND status IN ('processing', 'cancelled')")
+            .run({ $id: id, $now: nowISO() });
+        return result.changes > 0;
+    }
+    setJobHttpLog(id, requestLog, responseLog) {
+        this.db
+            .query('UPDATE jobs SET request_log = $req, response_log = $res, updated_at = $now WHERE id = $id')
+            .run({ $id: id, $req: requestLog, $res: responseLog, $now: nowISO() });
+    }
+    getJobTypes() {
+        const rows = this.db
+            .query('SELECT DISTINCT type FROM jobs ORDER BY type')
+            .all();
+        return rows.map(r => r.type);
+    }
+    retryFailedJob(failedJobId) {
+        const row = this.db
+            .query('SELECT * FROM failed_jobs WHERE id = $id')
+            .get({ $id: failedJobId });
+        if (!row) {
+            throw new Error(`Failed job ${failedJobId} not found`);
+        }
+        const now = nowISO();
+        this.insertJobStmt.run({
+            $type: row.type,
+            $data: row.data,
+            $status: 'pending',
+            $priority: 0,
+            $maxRetries: row.max_retries,
+            $runAt: now,
+            $batchId: null
+        });
+        const newJobId = this.lastInsertRowIdStmt.get();
+        this.db
+            .query('DELETE FROM failed_jobs WHERE id = $id')
+            .run({ $id: failedJobId });
+        return newJobId.id;
+    }
+    purgeFailedJobs(olderThanMs) {
+        const cutoff = new Date(Date.now() - olderThanMs).toISOString();
+        const result = this.db
+            .query('DELETE FROM failed_jobs WHERE failed_at < $cutoff')
+            .run({ $cutoff: cutoff });
+        return result.changes;
+    }
+    purge(options) {
+        const cutoff = new Date(Date.now() - options.olderThanMs).toISOString();
+        const result = this.db
+            .query('DELETE FROM jobs WHERE status = $status AND updated_at < $cutoff')
+            .run({ $status: options.status, $cutoff: cutoff });
+        return result.changes;
+    }
+    pollAndClaim(type) {
+        const now = nowISO();
+        const claimTx = this.db.transaction(() => {
+            const row = this.selectPendingStmt.get({
+                $type: type,
+                $now: now
+            });
+            if (!row)
+                return null;
+            this.markProcessingStmt.run({ $id: row.id, $now: now });
+            return row;
+        });
+        const row = claimTx.immediate();
+        return row ? toJob(row) : null;
+    }
+    markJobDone(id) {
+        this.db.transaction(() => {
+            const now = nowISO();
+            const row = this.selectJobStmt.get({ $id: id });
+            this.markDoneStmt.run({ $id: id, $now: now });
+            this.unblockDependents(id);
+            if (row?.batch_id) {
+                this.handleBatchJobComplete(row.batch_id);
+            }
+        })();
+    }
+    markJobDead(id, error) {
+        this.db.transaction(() => {
+            const row = this.selectJobStmt.get({ $id: id });
+            if (!row)
+                return;
+            this.insertFailedJobStmt.run({
+                $originalJobId: row.id,
+                $type: row.type,
+                $data: row.data,
+                $error: error,
+                $retryCount: row.retry_count,
+                $maxRetries: row.max_retries,
+                $createdAt: row.created_at,
+                $requestLog: row.request_log,
+                $responseLog: row.response_log
+            });
+            this.deleteJobStmt.run({ $id: id });
+            if (row.batch_id) {
+                this.incrementBatchFailedStmt.run({
+                    $id: row.batch_id,
+                    $jobId: row.id
+                });
+                this.handleBatchJobComplete(row.batch_id);
+            }
+        })();
+    }
+    markJobFailed(id, error) {
+        this.db.transaction(() => {
+            const now = nowISO();
+            const row = this.selectJobStmt.get({ $id: id });
+            if (!row)
+                return;
+            if (row.retry_count + 1 >= row.max_retries) {
+                this.insertFailedJobStmt.run({
+                    $originalJobId: row.id,
+                    $type: row.type,
+                    $data: row.data,
+                    $error: error,
+                    $retryCount: row.retry_count + 1,
+                    $maxRetries: row.max_retries,
+                    $createdAt: row.created_at,
+                    $requestLog: row.request_log,
+                    $responseLog: row.response_log
+                });
+                this.deleteJobStmt.run({ $id: id });
+                // Job is permanently dead — decrement batch counter and track failure
+                if (row.batch_id) {
+                    this.incrementBatchFailedStmt.run({
+                        $id: row.batch_id,
+                        $jobId: row.id
+                    });
+                    this.handleBatchJobComplete(row.batch_id);
+                }
+            }
+            else {
+                this.markFailedStmt.run({ $id: id, $error: error, $now: now });
+            }
+        })();
+    }
+    updateProgress(id, progress) {
+        this.updateProgressStmt.run({
+            $id: id,
+            $progress: progress,
+            $now: nowISO()
+        });
+    }
+    // --- Batch API ---
+    createBatch(name, options) {
+        const id = crypto.randomUUID();
+        this.insertBatchStmt.run({
+            $id: id,
+            $name: name,
+            $options: options ? JSON.stringify(options) : null,
+            $createdAt: nowISO()
+        });
+        return id;
+    }
+    addToBatch(batchId, type, data, options) {
+        const jobId = this.insertJob(type, data, batchId, options);
+        this.db
+            .query('UPDATE job_batches SET total_jobs = total_jobs + 1, pending_jobs = pending_jobs + 1 WHERE id = $id')
+            .run({ $id: batchId });
+        return jobId;
+    }
+    getBatch(batchId) {
+        const row = this.selectBatchStmt.get({
+            $id: batchId
+        });
+        return row ? toBatch(row) : null;
+    }
+    cancelBatch(batchId) {
+        const now = nowISO();
+        this.cancelBatchStmt.run({ $id: batchId, $now: now });
+        this.cancelBatchJobsStmt.run({ $batchId: batchId, $now: now });
+    }
+    createWorker(options) {
+        return new JobWorker(this, options);
+    }
+    insertJob(type, data, batchId, options) {
+        const now = nowISO();
+        const runAt = options?.runAt ? options.runAt.toISOString() : now;
+        const hasDeps = options?.dependsOn && options.dependsOn.length > 0;
+        const status = hasDeps ? 'blocked' : 'pending';
+        this.insertJobStmt.run({
+            $type: type,
+            $data: JSON.stringify(data),
+            $status: status,
+            $priority: options?.priority ?? 0,
+            $maxRetries: options?.maxRetries ?? 3,
+            $runAt: runAt,
+            $batchId: batchId
+        });
+        const jobId = this.lastInsertRowIdStmt.get();
+        if (hasDeps) {
+            for (const depId of options.dependsOn) {
+                const dep = this.selectJobStmt.get({ $id: depId });
+                if (!dep) {
+                    throw new Error(`Dependency job ${depId} does not exist`);
+                }
+                this.insertDepStmt.run({
+                    $jobId: jobId.id,
+                    $depsOnId: depId
+                });
+            }
+        }
+        return jobId.id;
+    }
+    close() {
+        try {
+            if (this.db.filename !== ':memory:' && this.db.filename !== '') {
+                this.db.run('PRAGMA wal_checkpoint(PASSIVE)');
+            }
+        }
+        catch {
+            // ignore checkpoint errors on close
+        }
+        this.db.close();
+    }
+    unblockDependents(completedJobId) {
+        const now = nowISO();
+        const dependents = this.selectDependentsStmt.all({
+            $depsOnId: completedJobId
+        });
+        for (const dep of dependents) {
+            const unmetCount = this.countUnmetDepsStmt.get({
+                $jobId: dep.job_id
+            });
+            if (unmetCount.count === 0) {
+                this.unblockJobStmt.run({ $id: dep.job_id, $now: now });
+            }
+        }
+    }
+    handleBatchJobComplete(batchId) {
+        this.decrementBatchPendingStmt.run({ $id: batchId });
+        const batch = this.selectBatchStmt.get({
+            $id: batchId
+        });
+        if (!batch || batch.pending_jobs > 0)
+            return;
+        // Batch is complete
+        const now = nowISO();
+        this.finishBatchStmt.run({ $id: batchId, $now: now });
+        const options = batch.options
+            ? JSON.parse(batch.options)
+            : null;
+        if (!options)
+            return;
+        // Enqueue "then" callback job only if zero failures
+        if (batch.failed_jobs === 0 && options.thenType) {
+            this.insertJobStmt.run({
+                $type: options.thenType,
+                $data: JSON.stringify(options.thenData ?? {}),
+                $status: 'pending',
+                $priority: 0,
+                $maxRetries: 3,
+                $runAt: now,
+                $batchId: null
+            });
+        }
+        // Enqueue "finally" callback job regardless of failures
+        if (options.finallyType) {
+            this.insertJobStmt.run({
+                $type: options.finallyType,
+                $data: JSON.stringify(options.finallyData ?? {}),
+                $status: 'pending',
+                $priority: 0,
+                $maxRetries: 3,
+                $runAt: now,
+                $batchId: null
+            });
+        }
+    }
+}

package/dist/rate-limiter.d.ts

@@ -0,0 +1,11 @@
+export declare class SlidingWindowRateLimiter {
+    private timestamps;
+    private maxCount;
+    private windowMs;
+    constructor(maxCount: number, windowMs: number);
+    canProceed(): boolean;
+    record(): void;
+    reset(): void;
+    private prune;
+}
+//# sourceMappingURL=rate-limiter.d.ts.map

package/dist/rate-limiter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rate-limiter.d.ts","sourceRoot":"","sources":["../src/rate-limiter.ts"],"names":[],"mappings":"AAGA,qBAAa,wBAAwB;IACnC,OAAO,CAAC,UAAU,CAAgB;IAClC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,QAAQ,CAAS;gBAEb,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM;IAK9C,UAAU,IAAI,OAAO;IAKrB,MAAM,IAAI,IAAI;IAId,KAAK,IAAI,IAAI;IAIb,OAAO,CAAC,KAAK;CAMd"}

package/dist/rate-limiter.js

@@ -0,0 +1,27 @@
+// packages/jobs/src/rate-limiter.ts
+// In-memory sliding window rate limiter for per-type job throttling
+export class SlidingWindowRateLimiter {
+    timestamps = [];
+    maxCount;
+    windowMs;
+    constructor(maxCount, windowMs) {
+        this.maxCount = maxCount;
+        this.windowMs = windowMs;
+    }
+    canProceed() {
+        this.prune();
+        return this.timestamps.length < this.maxCount;
+    }
+    record() {
+        this.timestamps.push(Date.now());
+    }
+    reset() {
+        this.timestamps = [];
+    }
+    prune() {
+        const cutoff = Date.now() - this.windowMs;
+        while (this.timestamps.length > 0 && this.timestamps[0] < cutoff) {
+            this.timestamps.shift();
+        }
+    }
+}

package/dist/scheduler.d.ts

@@ -0,0 +1,26 @@
+import type { JobQueue } from './queue';
+import type { AddScheduleOptions, JobMap, Schedule } from './types';
+export declare class Scheduler<TMap extends JobMap = Record<string, unknown>> {
+    private readonly queue;
+    private timer;
+    private readonly upsertStmt;
+    private readonly selectByNameStmt;
+    private readonly selectAllStmt;
+    private readonly selectDueStmt;
+    private readonly updateLastRunStmt;
+    private readonly pauseStmt;
+    private readonly resumeStmt;
+    private readonly removeStmt;
+    constructor(queue: JobQueue<TMap>);
+    register(name: string, type: string, cron: string, options?: AddScheduleOptions): Schedule;
+    cleanup(registeredNames: string[]): number;
+    pauseSchedule(name: string): void;
+    resumeSchedule(name: string): void;
+    getSchedules(): Schedule[];
+    getSchedule(name: string): Schedule | null;
+    removeSchedule(name: string): boolean;
+    tick(): number;
+    start(intervalMs?: number): void;
+    stop(): void;
+}
+//# sourceMappingURL=scheduler.d.ts.map

package/dist/scheduler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scheduler.d.ts","sourceRoot":"","sources":["../src/scheduler.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACxC,OAAO,KAAK,EACV,kBAAkB,EAClB,MAAM,EACN,QAAQ,EAET,MAAM,SAAS,CAAC;AAqBjB,qBAAa,SAAS,CAAC,IAAI,SAAS,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAClE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiB;IACvC,OAAO,CAAC,KAAK,CAA+C;IAE5D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC;IAC5B,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC;IAClC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;IAC/B,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;IAC/B,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IACnC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;IAC3B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC;IAC5B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC;gBAEhB,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC;IAwCjC,QAAQ,CACN,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,kBAAkB,GAC3B,QAAQ;IAoBX,OAAO,CAAC,eAAe,EAAE,MAAM,EAAE,GAAG,MAAM;IAoB1C,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAIjC,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAIlC,YAAY,IAAI,QAAQ,EAAE;IAK1B,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI;IAO1C,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAKrC,IAAI,IAAI,MAAM;IA0Dd,KAAK,CAAC,UAAU,SAAS,GAAG,IAAI;IAKhC,IAAI,IAAI,IAAI;CAMb"}