@pikku/queue-bullmq 0.12.4 → 0.12.6

package/CHANGELOG.md

@@ -1,3 +1,46 @@
+## 0.12.6
+
+### Patch Changes
+
+- 6794681: Publish the ecosystem surface as per-area sub-barrels under `@pikku/core/ecosystem/*`, and point generated code, the CLI, the inspector and the runtime adapters at them.
+
+  348 names that only generated code, the toolchain or a runtime adapter ever imports now have a second home on `@pikku/core/ecosystem/<area>` — one sub-barrel per area, matching how core already publishes its entrypoints, so no single barrel grows without bound and a consumer only pulls in the area it uses.
+
+  This step is additive: every name is still exported from the entrypoint it was published from before, so nothing downstream breaks. Removing them from the app-facing barrels is a later change, and needs a release carrying `./ecosystem/*` first.
+
+- Updated dependencies [7406bfe]
+- Updated dependencies [6794681]
+  - @pikku/core@0.12.84
+
+## 0.12.5
+
+### Patch Changes
+
+- e3d4454: Add job groups, so one shared queue can stay fair without splitting into one
+  queue per producer.
+
+  A job may now carry `group: { id, tier }`, and a worker may cap how many jobs
+  of any one group run at once via `groupConcurrency`. On pg-boss this maps to
+  `localGroupConcurrency`, which excludes at-capacity groups from the fetch query
+  itself, so a capped group costs nothing rather than being fetched and restored.
+  BullMQ declares it unsupported (groups are a BullMQ Pro feature) — being
+  push-based, it can simply use a queue per group at no polling cost.
+
+  Workflow services accept a `queueStrategy`. The default `'per-workflow'` is
+  unchanged: every workflow gets its own `wf-orchestrator-*` / `wf-step-*` queue,
+  which is also what lets serverless providers deploy one unit per workflow. The
+  new `'shared-groups'` routes every workflow through the shared
+  orchestrator/step-worker queues and isolates them by group instead, so a
+  monolith runs one set of pollers rather than one per workflow — on a
+  pull-based backend with dozens of workflows that is the difference between
+  hundreds of poll loops and twenty. It is for single-process runtimes only; a
+  per-unit serverless deploy still needs the per-workflow queues to route to its
+  units.
+
+- Updated dependencies [24252b8]
+- Updated dependencies [e3d4454]
+  - @pikku/core@0.12.69
+
 ## 0.12.4
 
 ### Patch Changes

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 - present Yasser Fadl and Pikku contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,29 @@
+# @pikku/queue-bullmq
+
+BullMQ queue, worker and scheduler services for Pikku. Backed by Redis, with
+job results, retries and priorities.
+
+## Install
+
+```bash
+npm install @pikku/queue-bullmq
+```
+
+## Usage
+
+```typescript
+import { BullQueueService, BullQueueWorkers } from '@pikku/queue-bullmq'
+
+const queue = new BullQueueService({ host: 'localhost', port: 6379 })
+
+const workers = new BullQueueWorkers({ host: 'localhost', port: 6379 })
+
+await workers.registerQueues()
+```
+
+Use `BullSchedulerService` for cron wirings. For a Postgres-backed alternative
+see `@pikku/queue-pg-boss`.
+
+## Docs
+
+https://pikku.dev/docs

package/dist/bull-queue-service.d.ts

@@ -1,6 +1,7 @@
 import type { ConnectionOptions, JobsOptions } from 'bullmq';
 import type Bull from 'bullmq';
-import type { QueueService, QueueJob, JobOptions } from '@pikku/core/queue';
+import type { QueueService, QueueJob } from '@pikku/core/queue';
+import type { JobOptions } from '@pikku/core/ecosystem/queue';
 export declare const mapPikkuJobToBull: (options?: JobOptions) => JobsOptions;
 /**
  * Bull/Redis queue service implementation

package/dist/bull-queue-worker.d.ts

@@ -1,5 +1,5 @@
 import type { ConnectionOptions, WorkerOptions } from 'bullmq';
-import type { QueueWorkers, PikkuWorkerConfig, QueueConfigMapping, ConfigValidationResult } from '@pikku/core/queue';
+import type { QueueWorkers, PikkuWorkerConfig, QueueConfigMapping, ConfigValidationResult } from '@pikku/core/ecosystem/queue';
 export declare const mapPikkuWorkerToBull: (workerConfig?: PikkuWorkerConfig) => Omit<WorkerOptions, "connection">;
 /**
  * Bull/Redis queue service implementation

package/dist/bull-queue-worker.js

@@ -1,6 +1,7 @@
 import { Worker } from 'bullmq';
-import { registerQueueWorkers, runQueueJob, QueueJobFailedError, QueueJobDiscardedError, } from '@pikku/core/queue';
-import { pikkuState } from '@pikku/core/internal';
+import { runQueueJob } from '@pikku/core/queue';
+import { registerQueueWorkers, QueueJobFailedError, QueueJobDiscardedError, } from '@pikku/core/ecosystem/queue';
+import { pikkuState } from '@pikku/core/ecosystem';
 import { mapBullJobToQueueJob } from './utils.js';
 export const mapPikkuWorkerToBull = (workerConfig) => {
     const workerOptions = {};
@@ -82,6 +83,10 @@ export class BullQueueWorkers {
         },
         // Configurations that are not supported
         unsupported: {
+            groupConcurrency: {
+                reason: 'Job groups are a BullMQ Pro feature, not available in BullMQ',
+                explanation: 'Give each group its own queue instead — BullMQ is push-based, so extra queues cost no polling',
+            },
             visibilityTimeout: {
                 reason: 'Bull does not use visibility timeout',
                 explanation: 'BullMQ locks jobs during processing instead of using visibility timeouts',

package/dist/bull-scheduler-service.d.ts

@@ -1,5 +1,6 @@
 import type { ConnectionOptions } from 'bullmq';
-import type { ScheduledTaskInfo, ScheduledTaskSummary, CoreUserSession } from '@pikku/core';
+import type { CoreUserSession } from '@pikku/core';
+import type { ScheduledTaskInfo, ScheduledTaskSummary } from '@pikku/core/ecosystem/services';
 import { SchedulerService } from '@pikku/core';
 export declare class BullSchedulerService extends SchedulerService {
     private redisConnectionOptions;

package/dist/bull-scheduler-service.js

@@ -1,7 +1,9 @@
 import { Queue, Worker } from 'bullmq';
-import { SchedulerService, parseDurationString } from '@pikku/core';
-import { pikkuState } from '@pikku/core/internal';
-import { runScheduledTask, getScheduledTasks } from '@pikku/core/scheduler';
+import { SchedulerService } from '@pikku/core';
+import { parseDurationString } from '@pikku/core/ecosystem/types';
+import { pikkuState } from '@pikku/core/ecosystem';
+import { runScheduledTask } from '@pikku/core/scheduler';
+import { getScheduledTasks } from '@pikku/core/ecosystem/scheduler';
 const RECURRING_QUEUE_NAME = 'pikku-recurring-scheduled-task';
 export class BullSchedulerService extends SchedulerService {
     redisConnectionOptions;

package/dist/index.d.ts

@@ -8,4 +8,5 @@ export { BullServiceFactory } from './bull-service-factory.js';
 export { BullQueueService } from './bull-queue-service.js';
 export { BullQueueWorkers } from './bull-queue-worker.js';
 export { BullSchedulerService } from './bull-scheduler-service.js';
-export type { QueueService, PikkuWorkerConfig, PikkuJobConfig, QueueJob, JobOptions, } from '@pikku/core/queue';
+export type { QueueService, QueueJob } from '@pikku/core/queue';
+export type { PikkuWorkerConfig, PikkuJobConfig, JobOptions, } from '@pikku/core/ecosystem/queue';

package/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@pikku/queue-bullmq",
-  "version": "0.12.4",
+  "version": "0.12.6",
+  "description": "BullMQ queue, worker and scheduler services for Pikku",
   "author": "yasser.fadl@gmail.com",
   "license": "MIT",
   "module": "dist/index.js",
@@ -15,7 +16,7 @@
     "prepublishOnly": "yarn build"
   },
   "peerDependencies": {
-    "@pikku/core": "^0.12.44"
+    "@pikku/core": "^0.12.84"
   },
   "dependencies": {
     "bullmq": "^5.70.1",

package/run-tests.sh

@@ -1,5 +1,56 @@
 #!/bin/bash
-cd "$(dirname "$0")"
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$script_dir"
+
+# Enable nullglob to handle cases where no files match the pattern
+shopt -s nullglob
+
+# Initialize variables for options
+watch_mode=false
+coverage_mode=false
+
+# Parse command-line options
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --watch)
+      watch_mode=true
+      shift
+      ;;
+    --coverage)
+      coverage_mode=true
+      shift
+      ;;
+    *)
+      echo "Unknown option: $1"
+      exit 1
+      ;;
+  esac
+done
+
+# Define the pattern to match your test files
+pattern="src/*.test.ts"
+
+# Expand the pattern into an array of files
 files=($(find src -type f -name "*.test.ts"))
-if [ ${#files[@]} -eq 0 ]; then echo "No test files found"; exit 0; fi
-node --import tsx --test "$@" "${files[@]}"
+
+# Check if any files matched the pattern
+if [ ${#files[@]} -eq 0 ]; then
+  echo "No test files found matching pattern: $pattern"
+  exit 0
+fi
+
+# Construct the node command
+node_cmd=(node --import tsx --test)
+
+# Append options based on flags
+if [ "$watch_mode" = true ]; then
+  node_cmd+=(--watch)
+fi
+
+if [ "$coverage_mode" = true ]; then
+  node_cmd+=(--test-coverage-include="src/**/*.{ts,js}" --test-coverage-exclude="**/dist/**" --experimental-test-coverage --test-reporter=lcov --test-reporter-destination=lcov.info)
+fi
+
+# Execute the node command with the expanded list of files
+"${node_cmd[@]}" "${files[@]}"

package/src/bull-queue-service.ts

@@ -1,7 +1,8 @@
 import type { ConnectionOptions, JobsOptions } from 'bullmq'
 import type Bull from 'bullmq'
 import { Queue, QueueEvents, RedisConnection } from 'bullmq'
-import type { QueueService, QueueJob, JobOptions } from '@pikku/core/queue'
+import type { QueueService, QueueJob } from '@pikku/core/queue'
+import type { JobOptions } from '@pikku/core/ecosystem/queue'
 import { mapBullJobToQueueJob } from './utils.js'
 
 export const mapPikkuJobToBull = (options?: JobOptions): JobsOptions => {

package/src/bull-queue-worker.ts

@@ -6,14 +6,14 @@ import type {
   PikkuWorkerConfig,
   QueueConfigMapping,
   ConfigValidationResult,
-} from '@pikku/core/queue'
+} from '@pikku/core/ecosystem/queue'
+import { runQueueJob } from '@pikku/core/queue'
 import {
   registerQueueWorkers,
-  runQueueJob,
   QueueJobFailedError,
   QueueJobDiscardedError,
-} from '@pikku/core/queue'
-import { pikkuState } from '@pikku/core/internal'
+} from '@pikku/core/ecosystem/queue'
+import { pikkuState } from '@pikku/core/ecosystem'
 import { mapBullJobToQueueJob } from './utils.js'
 
 export const mapPikkuWorkerToBull = (
@@ -111,6 +111,11 @@ export class BullQueueWorkers implements QueueWorkers {
 
     // Configurations that are not supported
     unsupported: {
+      groupConcurrency: {
+        reason: 'Job groups are a BullMQ Pro feature, not available in BullMQ',
+        explanation:
+          'Give each group its own queue instead — BullMQ is push-based, so extra queues cost no polling',
+      },
       visibilityTimeout: {
         reason: 'Bull does not use visibility timeout',
         explanation:

package/src/bull-scheduler-service.ts

@@ -1,13 +1,15 @@
 import type { ConnectionOptions } from 'bullmq'
 import { Queue, Worker } from 'bullmq'
+import type { CoreUserSession } from '@pikku/core'
 import type {
   ScheduledTaskInfo,
   ScheduledTaskSummary,
-  CoreUserSession,
-} from '@pikku/core'
-import { SchedulerService, parseDurationString } from '@pikku/core'
-import { pikkuState } from '@pikku/core/internal'
-import { runScheduledTask, getScheduledTasks } from '@pikku/core/scheduler'
+} from '@pikku/core/ecosystem/services'
+import { SchedulerService } from '@pikku/core'
+import { parseDurationString } from '@pikku/core/ecosystem/types'
+import { pikkuState } from '@pikku/core/ecosystem'
+import { runScheduledTask } from '@pikku/core/scheduler'
+import { getScheduledTasks } from '@pikku/core/ecosystem/scheduler'
 
 /**
  * Data stored in scheduled job

package/src/index.ts

@@ -11,10 +11,9 @@ export { BullQueueWorkers } from './bull-queue-worker.js'
 export { BullSchedulerService } from './bull-scheduler-service.js'
 
 // Re-export core queue types for convenience
+export type { QueueService, QueueJob } from '@pikku/core/queue'
 export type {
-  QueueService,
   PikkuWorkerConfig,
   PikkuJobConfig,
-  QueueJob,
   JobOptions,
-} from '@pikku/core/queue'
+} from '@pikku/core/ecosystem/queue'