@blamejs/core 0.4.19 → 0.4.21

package/CHANGELOG.md

@@ -8,6 +8,8 @@ upgrading across more than a few patches at a time.
 
 ## v0.4.x
 
+- **0.4.20** (2026-04-30) — b.queue + b.jobs: priority, rate-limit, progress
+- **0.4.19** (2026-04-30) — b.router: schema-validated routes + OpenAPI gen
 - **0.4.18** (2026-04-30) — cookieJar forensic-test strengthening (real crypto, replay, nonce)
 - **0.4.17** (2026-04-30) — b.httpClient.cookieJar (encrypted) + wiki catch-up sweep
 - **0.4.16** (2026-04-30) — b.httpClient: interceptors + progress events

package/lib/db.js

@@ -350,9 +350,23 @@ var FRAMEWORK_SCHEMA = [
       finishedAt:      "INTEGER",
       traceId:         "TEXT",
       classification:  "TEXT",
+      priority:        "INTEGER NOT NULL DEFAULT 0",
+      // Repeat-in-queue: cron-shaped recurring jobs re-enqueue themselves
+      // after each successful completion. NULL = one-shot (no repeat).
+      repeatCron:      "TEXT",
+      repeatTimezone:  "TEXT",
+      // Flows: parent-child job graphs with dependency edges.
+      // flowId groups jobs in the same flow; dependsOn is a JSON array
+      // of jobIds this row waits for; flowChildName is the human-readable
+      // label inside the flow used by dependsOn resolution.
+      flowId:          "TEXT",
+      flowChildName:   "TEXT",
+      dependsOn:       "TEXT",
     },
     indexes: [
       { name: "idx_jobs_lease",    columns: ["queueName", "status", "availableAt"] },
+      { name: "idx_jobs_priority", columns: ["queueName", "status", "priority", "availableAt"] },
+      { name: "idx_jobs_flow",     columns: ["flowId"] },
       "leaseExpiresAt",
       "finishedAt",
     ],

package/lib/framework-schema.js

@@ -476,10 +476,18 @@ function _jobsDDL(dialect) {
       "  lastError        TEXT," +
       "  finishedAt       " + t.INT + "," +
       "  traceId          TEXT," +
-      "  classification   TEXT" +
+      "  classification   TEXT," +
+      "  priority         " + t.INT + " NOT NULL DEFAULT 0," +
+      "  repeatCron       TEXT," +
+      "  repeatTimezone   TEXT," +
+      "  flowId           TEXT," +
+      "  flowChildName    TEXT," +
+      "  dependsOn        TEXT" +
       ")",
     indexes: [
       "CREATE INDEX IF NOT EXISTS idx_" + name + "_lease ON " + name + " (queueName, status, availableAt)",
+      "CREATE INDEX IF NOT EXISTS idx_" + name + "_priority ON " + name + " (queueName, status, priority, availableAt)",
+      "CREATE INDEX IF NOT EXISTS idx_" + name + "_flow ON " + name + " (flowId)",
       "CREATE INDEX IF NOT EXISTS idx_" + name + "_leaseExpiresAt ON " + name + " (leaseExpiresAt)",
       "CREATE INDEX IF NOT EXISTS idx_" + name + "_finishedAt ON " + name + " (finishedAt)",
     ],

package/lib/queue-local.js

@@ -50,15 +50,23 @@ var JOB_COLS = [
   "_id", "queueName", "payload", "status",
   "enqueuedAt", "availableAt", "leasedAt", "leaseExpiresAt",
   "attempts", "maxAttempts", "lastError", "finishedAt",
-  "traceId", "classification",
+  "traceId", "classification", "priority",
+  "repeatCron", "repeatTimezone",
+  "flowId", "flowChildName", "dependsOn",
 ];
 
+// Sentinel availableAt for flow children that haven't yet had their
+// dependencies satisfied — far future so the lease query never picks
+// them. Parent-completion sets a real availableAt when all deps complete.
+var FLOW_BLOCKED_AVAILABLE_AT = Number.MAX_SAFE_INTEGER;
+
 // Columns returned by lease() / used by RETURNING. Subset of JOB_COLS
 // — only what callers need; fewer bytes over the wire in cluster mode.
 var LEASE_RETURN_COLS = [
   "_id", "queueName", "payload",
   "attempts", "maxAttempts", "traceId", "classification",
   "enqueuedAt", "leaseExpiresAt",
+  "repeatCron", "repeatTimezone", "flowId", "flowChildName",
 ];
 
 function _quotedList(cols) {
@@ -84,6 +92,10 @@ function _shapeLeasedRow(raw) {
     classification: unsealed.classification,
     enqueuedAt:     Number(unsealed.enqueuedAt),
     leaseExpiresAt: Number(unsealed.leaseExpiresAt),
+    repeatCron:     unsealed.repeatCron     || null,
+    repeatTimezone: unsealed.repeatTimezone || null,
+    flowId:         unsealed.flowId         || null,
+    flowChildName:  unsealed.flowChildName  || null,
   };
 }
 
@@ -97,13 +109,27 @@ function create(_config) {
     var nowMs = Date.now();
     var availableAt = nowMs + (opts.delaySeconds ? opts.delaySeconds * 1000 : 0);
 
+    var priority = (typeof opts.priority === "number" && isFinite(opts.priority))
+      ? Math.floor(opts.priority) : 0;
+    var repeatCron     = opts.repeat && typeof opts.repeat.cron === "string"
+                            ? opts.repeat.cron : null;
+    var repeatTimezone = opts.repeat && typeof opts.repeat.timezone === "string"
+                            ? opts.repeat.timezone : null;
+    var flowId         = typeof opts.flowId === "string" ? opts.flowId : null;
+    var flowChildName  = typeof opts.flowChildName === "string" ? opts.flowChildName : null;
+    var dependsOn      = Array.isArray(opts.dependsOn) && opts.dependsOn.length > 0
+                            ? JSON.stringify(opts.dependsOn) : null;
+    // Flow children with deps wait at MAX_SAFE_INTEGER until parent
+    // completion bumps availableAt — keeps them out of the lease index.
+    var effectiveAvailableAt = (dependsOn ? FLOW_BLOCKED_AVAILABLE_AT : availableAt);
+
     var row = {
       _id:             generateToken(16),
       queueName:       queueName,
       payload:         payload === undefined ? null : JSON.stringify(payload),
       status:          "pending",
       enqueuedAt:      nowMs,
-      availableAt:     availableAt,
+      availableAt:     effectiveAvailableAt,
       leasedAt:        null,
       leaseExpiresAt:  null,
       attempts:        0,
@@ -112,6 +138,12 @@ function create(_config) {
       finishedAt:      null,
       traceId:         opts.traceId || null,
       classification:  opts.classification || null,
+      priority:        priority,
+      repeatCron:      repeatCron,
+      repeatTimezone:  repeatTimezone,
+      flowId:          flowId,
+      flowChildName:   flowChildName,
+      dependsOn:       dependsOn,
     };
     var sealed = cryptoField.sealRow("_blamejs_jobs", row);
     var values = JOB_COLS.map(function (c) { return c in sealed ? sealed[c] : null; });
@@ -148,7 +180,7 @@ function create(_config) {
       "WHERE _id IN (" +
       "  SELECT _id FROM _blamejs_jobs " +
       "  WHERE queueName = ? AND status = 'pending' AND availableAt <= ? " +
-      "  ORDER BY availableAt ASC, enqueuedAt ASC " +
+      "  ORDER BY priority DESC, availableAt ASC, enqueuedAt ASC " +
       "  LIMIT ?" +
       ") " +
       "RETURNING " + _quotedList(LEASE_RETURN_COLS);
@@ -184,14 +216,94 @@ function create(_config) {
 
   async function complete(jobId) {
     cluster.requireLeader();
+    var nowMs = Date.now();
+    // Read the row first so we can act on repeat / flow metadata after
+    // the status flip. Single SELECT + UPDATE pair under the same
+    // jobId — race-free under SQLite (single-writer); cluster-storage
+    // dispatches both calls to the same backend.
+    var rowRes = await clusterStorage.execute(
+      "SELECT _id, queueName, payload, repeatCron, repeatTimezone, " +
+      "       flowId, flowChildName, priority, classification, traceId " +
+      "FROM _blamejs_jobs WHERE _id = ?",
+      [jobId]
+    );
+    var row = (rowRes && rowRes.rows && rowRes.rows[0]) || null;
+
     await clusterStorage.execute(
       "UPDATE _blamejs_jobs SET status = 'done', finishedAt = ?, leaseExpiresAt = NULL " +
       "WHERE _id = ? AND status = 'inflight'",
-      [Date.now(), jobId]
+      [nowMs, jobId]
     );
+
+    // Repeat-in-queue: cron-recurring job re-enqueues itself for the
+    // next firing time. Failures (which take the fail() path) don't
+    // re-enqueue — operators investigate before the cron resumes.
+    if (row && row.repeatCron) {
+      try {
+        var unsealedRow = cryptoField.unsealRow("_blamejs_jobs", row);
+        var scheduler = require("./scheduler");
+        var cron = scheduler.parseCron(unsealedRow.repeatCron);
+        var nextMs = scheduler.nextCronFire(cron, new Date(nowMs), unsealedRow.repeatTimezone || null);
+        await enqueue(unsealedRow.queueName,
+          unsealedRow.payload ? safeJson.parse(unsealedRow.payload) : null,
+          {
+            availableAt:     nextMs,
+            delaySeconds:    Math.max(0, Math.floor((nextMs - nowMs) / 1000)),
+            repeat:          { cron: unsealedRow.repeatCron, timezone: unsealedRow.repeatTimezone },
+            priority:        Number(unsealedRow.priority) || 0,
+            classification:  unsealedRow.classification || null,
+            traceId:         unsealedRow.traceId || null,
+          });
+      } catch (_e) { /* repeat re-enqueue best-effort — cron resumes next tick if op fixes the issue */ }
+    }
+
+    // Flow propagation: walk siblings whose dependsOn includes this
+    // jobId (or this job's flowChildName) and bump availableAt to now
+    // if ALL their deps are now complete.
+    if (row && row.flowId) {
+      await _maybeReleaseFlowChildren(row.flowId, jobId, row.flowChildName, nowMs);
+    }
     return true;
   }
 
+  async function _maybeReleaseFlowChildren(flowId, completedJobId, completedChildName, nowMs) {
+    var siblingsRes = await clusterStorage.execute(
+      "SELECT _id, dependsOn, flowChildName, status, availableAt FROM _blamejs_jobs " +
+      "WHERE flowId = ? AND status = 'pending' AND availableAt > ?",
+      [flowId, nowMs]
+    );
+    var siblings = (siblingsRes && siblingsRes.rows) || [];
+    for (var i = 0; i < siblings.length; i++) {
+      var sib = siblings[i];
+      if (!sib.dependsOn) continue;
+      var deps;
+      try { deps = JSON.parse(sib.dependsOn); }
+      catch (_e) { continue; }
+      if (!Array.isArray(deps) || deps.length === 0) continue;
+      // Resolve which deps are satisfied. Each dep is either a jobId
+      // or a flowChildName; we accept both shapes against the flow.
+      var allDone = true;
+      for (var d = 0; d < deps.length; d++) {
+        var dep = deps[d];
+        // Quick path: just-completed job matches by id or child name.
+        if (dep === completedJobId || (completedChildName && dep === completedChildName)) continue;
+        // Otherwise SELECT to confirm done.
+        var depRes = await clusterStorage.execute(
+          "SELECT 1 FROM _blamejs_jobs WHERE flowId = ? AND status = 'done' AND " +
+          "  (_id = ? OR flowChildName = ?) LIMIT 1",
+          [flowId, dep, dep]
+        );
+        if (!depRes || !depRes.rows || depRes.rows.length === 0) { allDone = false; break; }
+      }
+      if (allDone) {
+        await clusterStorage.execute(
+          "UPDATE _blamejs_jobs SET availableAt = ? WHERE _id = ?",
+          [nowMs, sib._id]
+        );
+      }
+    }
+  }
+
   async function fail(jobId, errorMessage, opts) {
     cluster.requireLeader();
     opts = opts || {};

package/lib/queue.js

@@ -177,6 +177,42 @@ function consume(queueName, handler, opts) {
   var pollIntervalMs   = opts.pollIntervalMs   || 1000;
   var fastPollMs       = opts.fastPollMs       || 50;
 
+  // Rate-limit: { max, perSeconds } caps how many handler INVOCATIONS
+  // start within any rolling perSeconds window. Token-bucket-style
+  // accounting keeps it cheap (just a sliding deque of timestamps).
+  var rateLimit = null;
+  if (opts.rateLimit) {
+    if (!opts.rateLimit.max || !opts.rateLimit.perSeconds ||
+        typeof opts.rateLimit.max !== "number" ||
+        typeof opts.rateLimit.perSeconds !== "number") {
+      throw _err("BAD_RATE_LIMIT",
+        "consume({ rateLimit }): expected { max: number, perSeconds: number }, got " +
+        JSON.stringify(opts.rateLimit), true);
+    }
+    rateLimit = {
+      max:        opts.rateLimit.max,
+      windowMs:   opts.rateLimit.perSeconds * 1000,
+      timestamps: [],
+    };
+  }
+  function _rateLimitWaitMs() {
+    if (!rateLimit) return 0;
+    var now = Date.now();
+    var cutoff = now - rateLimit.windowMs;
+    while (rateLimit.timestamps.length > 0 && rateLimit.timestamps[0] <= cutoff) {
+      rateLimit.timestamps.shift();
+    }
+    if (rateLimit.timestamps.length < rateLimit.max) return 0;
+    return rateLimit.timestamps[0] + rateLimit.windowMs - now + 1;
+  }
+  function _rateLimitConsume() {
+    if (rateLimit) rateLimit.timestamps.push(Date.now());
+  }
+
+  // Progress audit-emit rate-limit — protect the audit chain from a
+  // chatty handler that calls progress() every loop iteration.
+  var PROGRESS_MIN_INTERVAL_MS = 250;
+
   // Each consumer has its own AbortController so cancel() unblocks any
   // in-flight poll-sleep immediately rather than waiting up to
   // pollIntervalMs (default 1s) for the next while-loop iteration.
@@ -209,6 +245,18 @@ function consume(queueName, handler, opts) {
         await _pollSleep(fastPollMs);
         continue;
       }
+      // If rate-limited and we'd exceed the budget, sleep until the
+      // next slot opens. We lease at most `max - currentTokens` jobs to
+      // stay under the cap.
+      if (rateLimit) {
+        var wait = _rateLimitWaitMs();
+        if (wait > 0) {
+          await _pollSleep(Math.min(wait, pollIntervalMs));
+          continue;
+        }
+        var remainingTokens = rateLimit.max - rateLimit.timestamps.length;
+        if (remainingTokens < slots) slots = Math.max(1, remainingTokens);
+      }
       var jobs;
       try { jobs = await b.lease(queueName, leaseDurationMs, slots); }
       catch {
@@ -227,9 +275,17 @@ function consume(queueName, handler, opts) {
           _emit("system.queue.consume.start", {
             metadata: { queue: queueName, backend: b.name, jobId: job.jobId, attempt: job.attempts, traceId: job.traceId },
           });
+          // Consume a rate-limit slot at handler-start so the budget
+          // tracks invocation rate, not lease rate (a single lease that
+          // splits work across many sub-units doesn't double-count).
+          _rateLimitConsume();
+
           // Handler context — second arg to handler. Carries
-          // ctx.extendLease(ms) for long-running handlers that need to
-          // bump their own lease before the sweeper reclaims the job.
+          // ctx.extendLease(ms) for long-running handlers and
+          // ctx.progress(0..100) for surfacing job progress to the
+          // audit chain (rate-limited so chatty handlers don't drown it).
+          var lastProgressEmitAt = 0;
+          var lastProgressValue = -1;
           var ctx = {
             extendLease: function (additionalMs) {
               if (typeof b.extendLease !== "function") {
@@ -246,6 +302,25 @@ function consume(queueName, handler, opts) {
                 return ok;
               });
             },
+            progress: function (pct) {
+              if (typeof pct !== "number" || !isFinite(pct)) return;
+              var clamped = Math.max(0, Math.min(100, Math.floor(pct)));
+              var now = Date.now();
+              // Always emit 0 and 100 (start/done markers); throttle the rest.
+              var isMarker = clamped === 0 || clamped === 100;
+              if (!isMarker && (now - lastProgressEmitAt) < PROGRESS_MIN_INTERVAL_MS) return;
+              if (clamped === lastProgressValue && !isMarker) return;
+              lastProgressEmitAt = now;
+              lastProgressValue = clamped;
+              observability.event("queue.progress", clamped, { queueName: queueName });
+              _emit("system.queue.progress", {
+                metadata: {
+                  queue: queueName, backend: b.name, jobId: job.jobId,
+                  attempt: job.attempts, traceId: job.traceId,
+                  percent: clamped,
+                },
+              });
+            },
           };
           observability.tap("queue.consume",
             { queueName: queueName, backend: b.name, jobId: job.jobId, attempt: job.attempts },
@@ -413,9 +488,142 @@ function _resetForTest() {
   audit.reset();
 }
 
+// enqueueFlow — atomic registration of a parent-child job graph.
+//
+//   await b.queue.enqueueFlow({
+//     queueName: "ingest",
+//     children: [
+//       { name: "fetch",      payload: { url } },
+//       { name: "transform",  payload: { ... }, dependsOn: ["fetch"] },
+//       { name: "publish",    payload: { ... }, dependsOn: ["transform"] },
+//     ],
+//   });
+//
+// Cycle detection runs at registration (Tier-A throw). Each child enters
+// the queue with availableAt = MAX_SAFE_INTEGER until parent completion
+// bumps it. Returns { flowId, jobs: [{ name, jobId }, ...] }.
+function enqueueFlow(spec) {
+  _requireInit();
+  if (!spec || typeof spec !== "object") {
+    return Promise.reject(_err("BAD_FLOW", "enqueueFlow requires an opts object", true));
+  }
+  if (typeof spec.queueName !== "string" || !spec.queueName) {
+    return Promise.reject(_err("BAD_FLOW", "enqueueFlow requires queueName", true));
+  }
+  if (!Array.isArray(spec.children) || spec.children.length === 0) {
+    return Promise.reject(_err("BAD_FLOW", "enqueueFlow requires children: [...]", true));
+  }
+  // Validate each child's shape.
+  var byName = {};
+  for (var i = 0; i < spec.children.length; i++) {
+    var c = spec.children[i];
+    if (!c || typeof c !== "object") {
+      return Promise.reject(_err("BAD_FLOW", "children[" + i + "] must be an object", true));
+    }
+    if (typeof c.name !== "string" || !c.name) {
+      return Promise.reject(_err("BAD_FLOW", "children[" + i + "].name must be a non-empty string", true));
+    }
+    if (byName[c.name]) {
+      return Promise.reject(_err("BAD_FLOW", "duplicate child name '" + c.name + "'", true));
+    }
+    byName[c.name] = c;
+    if (c.dependsOn !== undefined) {
+      if (!Array.isArray(c.dependsOn)) {
+        return Promise.reject(_err("BAD_FLOW",
+          "children[" + i + "].dependsOn must be an array of names", true));
+      }
+      for (var di = 0; di < c.dependsOn.length; di++) {
+        if (typeof c.dependsOn[di] !== "string") {
+          return Promise.reject(_err("BAD_FLOW",
+            "children[" + i + "].dependsOn[" + di + "] must be a string name", true));
+        }
+      }
+    }
+  }
+  // Cycle detection — depth-first traversal with visited set.
+  function _visit(name, stack) {
+    if (stack.indexOf(name) !== -1) {
+      throw _err("FLOW_CYCLE", "flow cycle detected: " +
+        stack.concat([name]).join(" → "), true);
+    }
+    var child = byName[name];
+    if (!child || !child.dependsOn) return;
+    var nextStack = stack.concat([name]);
+    for (var k = 0; k < child.dependsOn.length; k++) {
+      var dep = child.dependsOn[k];
+      if (!byName[dep]) {
+        throw _err("FLOW_UNKNOWN_DEP",
+          "child '" + name + "' dependsOn unknown name '" + dep + "'", true);
+      }
+      _visit(dep, nextStack);
+    }
+  }
+  try {
+    var names = Object.keys(byName);
+    for (var n = 0; n < names.length; n++) _visit(names[n], []);
+  } catch (e) {
+    return Promise.reject(e);
+  }
+
+  var nodeCrypto = require("node:crypto");
+  var flowId = "flow-" + nodeCrypto.randomBytes(8).toString("hex");
+
+  return observability.tap("queue.enqueueFlow",
+    { queueName: spec.queueName, flowId: flowId, childCount: spec.children.length },
+    async function () {
+      var jobs = [];
+      // Two-pass insert: first pass enqueues all children with their
+      // names attached so the second pass can write dependsOn jobIds
+      // resolved by name. Children with deps land at MAX_SAFE_INTEGER
+      // availableAt automatically (see queue-local enqueue logic).
+      var nameToJobId = {};
+      for (var p = 0; p < spec.children.length; p++) {
+        var ch = spec.children[p];
+        // Hold off setting dependsOn until we know all sibling jobIds.
+        var enqOpts = {
+          flowId:        flowId,
+          flowChildName: ch.name,
+          priority:      ch.priority || 0,
+          classification: ch.classification || null,
+          traceId:       ch.traceId || null,
+          maxAttempts:   ch.maxAttempts,
+          // dependsOn intentionally omitted on first pass — will be patched
+          // in via direct UPDATE after all jobIds are known. This means
+          // root children (no deps) are immediately leaseable; deps-bearing
+          // children get patched to MAX_SAFE_INTEGER via second pass.
+        };
+        var result = await enqueue(spec.queueName, ch.payload, enqOpts);
+        nameToJobId[ch.name] = result.jobId;
+        jobs.push({ name: ch.name, jobId: result.jobId, dependsOn: ch.dependsOn || [] });
+      }
+      // Second pass: write dependsOn (translated to jobIds) for children
+      // that need it, and parking-lot their availableAt to MAX_SAFE_INTEGER.
+      var clusterStorage = require("./cluster-storage");
+      for (var q = 0; q < jobs.length; q++) {
+        var j = jobs[q];
+        if (j.dependsOn.length === 0) continue;
+        var depIds = j.dependsOn.map(function (n2) { return nameToJobId[n2]; });
+        await clusterStorage.execute(
+          "UPDATE _blamejs_jobs SET dependsOn = ?, availableAt = ? WHERE _id = ?",
+          [JSON.stringify(depIds), Number.MAX_SAFE_INTEGER, j.jobId]
+        );
+      }
+      _emit("system.queue.flow.enqueue", {
+        metadata: {
+          queue:      spec.queueName,
+          flowId:     flowId,
+          childCount: spec.children.length,
+        },
+      });
+      return { flowId: flowId, jobs: jobs.map(function (j) { return { name: j.name, jobId: j.jobId }; }) };
+    }
+  );
+}
+
 module.exports = {
   init:               init,
   enqueue:            enqueue,
+  enqueueFlow:        enqueueFlow,
   consume:            consume,
   size:               size,
   purge:              purge,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blamejs/core",
-  "version": "0.4.19",
+  "version": "0.4.21",
   "description": "The Node framework that owns its stack.",
   "license": "Apache-2.0",
   "author": "blamejs contributors",