@open-mercato/cli 0.6.6-develop.5558.1.748adcd5fc → 0.6.6-develop.5588.1.a8f6c51d1f

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@open-mercato/cli",
-  "version": "0.6.6-develop.5558.1.748adcd5fc",
+  "version": "0.6.6-develop.5588.1.a8f6c51d1f",
   "type": "module",
   "main": "./dist/index.js",
   "exports": {
@@ -59,8 +59,8 @@
     "@mikro-orm/decorators": "^7.1.4",
     "@mikro-orm/migrations": "^7.1.4",
     "@mikro-orm/postgresql": "^7.1.4",
-    "@open-mercato/queue": "0.6.6-develop.5558.1.748adcd5fc",
-    "@open-mercato/shared": "0.6.6-develop.5558.1.748adcd5fc",
+    "@open-mercato/queue": "0.6.6-develop.5588.1.a8f6c51d1f",
+    "@open-mercato/shared": "0.6.6-develop.5588.1.a8f6c51d1f",
     "cross-spawn": "^7.0.6",
     "pg": "8.21.0",
     "semver": "^7.8.4",
@@ -70,10 +70,10 @@
     "typescript": "^6.0.3"
   },
   "peerDependencies": {
-    "@open-mercato/shared": "0.6.6-develop.5558.1.748adcd5fc"
+    "@open-mercato/shared": "0.6.6-develop.5588.1.a8f6c51d1f"
   },
   "devDependencies": {
-    "@open-mercato/shared": "0.6.6-develop.5558.1.748adcd5fc",
+    "@open-mercato/shared": "0.6.6-develop.5588.1.a8f6c51d1f",
     "@types/jest": "^30.0.0",
     "jest": "^30.4.2",
     "ts-jest": "^29.4.11"

package/src/lib/__tests__/worker-connection-budget.test.ts

@@ -0,0 +1,115 @@
+import {
+  planWorkerConcurrency,
+  resolveWorkerConnectionBudget,
+} from '../worker-connection-budget'
+
+describe('resolveWorkerConnectionBudget', () => {
+  it('defaults to the pool max when no override is set', () => {
+    expect(resolveWorkerConnectionBudget({} as NodeJS.ProcessEnv, 20)).toBe(20)
+  })
+
+  it('honors a positive OM_WORKERS_DB_CONNECTION_BUDGET override', () => {
+    expect(
+      resolveWorkerConnectionBudget(
+        { OM_WORKERS_DB_CONNECTION_BUDGET: '8' } as unknown as NodeJS.ProcessEnv,
+        20,
+      ),
+    ).toBe(8)
+  })
+
+  it('falls back to the pool max for non-numeric or non-positive overrides', () => {
+    for (const value of ['0', '-3', 'abc', '']) {
+      expect(
+        resolveWorkerConnectionBudget(
+          { OM_WORKERS_DB_CONNECTION_BUDGET: value } as unknown as NodeJS.ProcessEnv,
+          16,
+        ),
+      ).toBe(16)
+    }
+  })
+
+  it('clamps an invalid pool max to at least 1', () => {
+    expect(resolveWorkerConnectionBudget({} as NodeJS.ProcessEnv, 0)).toBe(1)
+    expect(resolveWorkerConnectionBudget({} as NodeJS.ProcessEnv, Number.NaN)).toBe(1)
+  })
+})
+
+describe('planWorkerConcurrency', () => {
+  it('passes concurrency through untouched when it fits the budget', () => {
+    const plan = planWorkerConcurrency(
+      [
+        { queue: 'events', concurrency: 5 },
+        { queue: 'vector-indexing', concurrency: 2 },
+        { queue: 'fulltext-indexing', concurrency: 2 },
+      ],
+      20,
+    )
+    expect(plan.clamped).toBe(false)
+    expect(plan.totalEffective).toBe(9)
+    expect(plan.entries.map((entry) => entry.effective)).toEqual([5, 2, 2])
+  })
+
+  it('scales down to exactly the budget when over-subscribed', () => {
+    const plan = planWorkerConcurrency(
+      [
+        { queue: 'events', concurrency: 10 },
+        { queue: 'vector-indexing', concurrency: 10 },
+        { queue: 'fulltext-indexing', concurrency: 10 },
+      ],
+      12,
+    )
+    expect(plan.clamped).toBe(true)
+    expect(plan.belowQueueFloor).toBe(false)
+    expect(plan.totalEffective).toBe(12)
+    for (const entry of plan.entries) {
+      expect(entry.effective).toBeGreaterThanOrEqual(1)
+      expect(entry.effective).toBeLessThanOrEqual(entry.requested)
+    }
+  })
+
+  it('keeps a floor of 1 per queue and never exceeds the request', () => {
+    const plan = planWorkerConcurrency(
+      [
+        { queue: 'events', concurrency: 16 },
+        { queue: 'vector-indexing', concurrency: 2 },
+        { queue: 'fulltext-indexing', concurrency: 2 },
+      ],
+      6,
+    )
+    const byQueue = Object.fromEntries(plan.entries.map((entry) => [entry.queue, entry.effective]))
+    expect(byQueue['vector-indexing']).toBeGreaterThanOrEqual(1)
+    expect(byQueue['fulltext-indexing']).toBeGreaterThanOrEqual(1)
+    // The largest requester absorbs most of the remaining budget.
+    expect(byQueue['events']).toBeGreaterThan(byQueue['vector-indexing'])
+    expect(plan.totalEffective).toBe(6)
+  })
+
+  it('flags belowQueueFloor when the budget is smaller than the queue count', () => {
+    const plan = planWorkerConcurrency(
+      [
+        { queue: 'a', concurrency: 4 },
+        { queue: 'b', concurrency: 4 },
+        { queue: 'c', concurrency: 4 },
+      ],
+      2,
+    )
+    expect(plan.clamped).toBe(true)
+    expect(plan.belowQueueFloor).toBe(true)
+    // Floor of 1 wins even though it exceeds the budget.
+    expect(plan.entries.every((entry) => entry.effective === 1)).toBe(true)
+    expect(plan.totalEffective).toBe(3)
+  })
+
+  it('treats zero/negative requested concurrency as a floor of 1', () => {
+    const plan = planWorkerConcurrency(
+      [
+        { queue: 'events', concurrency: 0 },
+        { queue: 'vector-indexing', concurrency: -5 },
+      ],
+      20,
+    )
+    expect(plan.entries.map((entry) => entry.requested)).toEqual([1, 1])
+    expect(plan.totalEffective).toBe(2)
+    expect(plan.clamped).toBe(false)
+  })
+})

package/src/lib/worker-connection-budget.ts

@@ -0,0 +1,131 @@
+/**
+ * Bound the DB-connection demand of background queue workers.
+ *
+ * Since #2970/#3011 each worker job builds its own request container, so its own
+ * `EntityManager` checks out a dedicated pooled DB connection for the job's
+ * duration. The peak connection demand of `worker --all` is therefore the sum of
+ * every queue's concurrency. Nothing previously bounded that sum against the DB
+ * pool (`DB_POOL_MAX`) or the database's global `max_connections`, so a storm of
+ * slow/failing jobs could over-subscribe connections — thrashing on the worker's
+ * own acquire timeout and, across the worker + web processes, starving the
+ * request/onboarding path that shares the same database.
+ *
+ * These helpers derive a per-queue effective-concurrency plan whose total never
+ * exceeds a connection budget (defaulting to the resolved pool max), while
+ * guaranteeing every queue keeps at least one worker.
+ */
+
+export type WorkerQueueConcurrency = {
+  queue: string
+  concurrency: number
+}
+
+export type WorkerConcurrencyPlanEntry = {
+  queue: string
+  requested: number
+  effective: number
+}
+
+export type WorkerConcurrencyPlan = {
+  /** Connection budget the plan was fitted to. */
+  budget: number
+  /** Sum of requested concurrency across all queues. */
+  totalRequested: number
+  /** Sum of effective concurrency the plan grants. */
+  totalEffective: number
+  /** True when any queue's concurrency was reduced to fit the budget. */
+  clamped: boolean
+  /**
+   * True when the budget is smaller than the number of queues, so the per-queue
+   * floor of 1 forces the total above the budget. The caller should surface this
+   * as a misconfiguration (raise `DB_POOL_MAX` or the budget).
+   */
+  belowQueueFloor: boolean
+  entries: WorkerConcurrencyPlanEntry[]
+}
+
+function toPositiveInt(value: number | undefined, fallback: number): number {
+  if (value === undefined || !Number.isFinite(value)) return fallback
+  const floored = Math.floor(value)
+  return floored > 0 ? floored : fallback
+}
+
+/**
+ * Resolve the connection budget for background workers. Defaults to the resolved
+ * DB pool max so the worker never tries to use more connections than its own pool
+ * can hand out; override with `OM_WORKERS_DB_CONNECTION_BUDGET` (positive int).
+ */
+export function resolveWorkerConnectionBudget(
+  env: NodeJS.ProcessEnv,
+  poolMax: number,
+): number {
+  const safePoolMax = toPositiveInt(poolMax, 1)
+  const raw = env.OM_WORKERS_DB_CONNECTION_BUDGET
+  if (!raw) return safePoolMax
+  const parsed = Number.parseInt(raw, 10)
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : safePoolMax
+}
+
+/**
+ * Fit per-queue concurrency to a connection budget.
+ *
+ * - Every queue keeps a floor of 1 worker (no queue is starved).
+ * - No queue exceeds its requested concurrency.
+ * - When the sum exceeds the budget, leftover capacity is distributed greedily to
+ *   the queues furthest below their request, so the total matches the budget
+ *   exactly (when `budget >= queueCount`) and the allocation is deterministic.
+ */
+export function planWorkerConcurrency(
+  queues: WorkerQueueConcurrency[],
+  budget: number,
+): WorkerConcurrencyPlan {
+  const safeBudget = toPositiveInt(budget, 1)
+  const requested = queues.map((entry) => ({
+    queue: entry.queue,
+    requested: toPositiveInt(entry.concurrency, 1),
+  }))
+  const totalRequested = requested.reduce((sum, entry) => sum + entry.requested, 0)
+
+  if (totalRequested <= safeBudget) {
+    return {
+      budget: safeBudget,
+      totalRequested,
+      totalEffective: totalRequested,
+      clamped: false,
+      belowQueueFloor: false,
+      entries: requested.map((entry) => ({ ...entry, effective: entry.requested })),
+    }
+  }
+
+  // Floor every queue at 1, then water-fill the remaining budget to the queues
+  // with the largest unmet request first.
+  const effective = requested.map(() => 1)
+  let used = effective.length
+  while (used < safeBudget) {
+    let bestIndex = -1
+    let bestDeficit = 0
+    for (let index = 0; index < requested.length; index += 1) {
+      const deficit = requested[index].requested - effective[index]
+      if (deficit > bestDeficit) {
+        bestDeficit = deficit
+        bestIndex = index
+      }
+    }
+    if (bestIndex === -1) break
+    effective[bestIndex] += 1
+    used += 1
+  }
+
+  const totalEffective = effective.reduce((sum, value) => sum + value, 0)
+  return {
+    budget: safeBudget,
+    totalRequested,
+    totalEffective,
+    clamped: true,
+    belowQueueFloor: requested.length > safeBudget,
+    entries: requested.map((entry, index) => ({
+      ...entry,
+      effective: effective[index],
+    })),
+  }
+}

package/src/mercato.ts

@@ -16,6 +16,11 @@ import {
 } from './lib/auto-spawn-workers'
 import { startLazyWorkerSupervisor } from './lib/queue-worker-supervisor'
 import { createPerJobWorkerHandler } from './lib/worker-job-handler'
+import {
+  planWorkerConcurrency,
+  resolveWorkerConnectionBudget,
+  type WorkerConcurrencyPlan,
+} from './lib/worker-connection-budget'
 import {
   resolveAutoSpawnSchedulerMode,
   resolveLazySchedulerPollMs,
@@ -525,6 +530,46 @@ function formatQueueWorkerLabel(queueNames: string[]): string {
   return `Queue worker (${preview})`
 }
 
+/**
+ * Fit the requested per-queue worker concurrency to the worker process's DB
+ * connection budget and log the resolved plan. Since each job runs in its own
+ * request container (one pooled connection per in-flight job), the sum of worker
+ * concurrency is the worker's peak connection demand — it MUST stay within the
+ * pool so background jobs cannot starve the request/onboarding path that shares
+ * the same database.
+ */
+async function resolveWorkerBudgetPlan(
+  requestedByQueue: { queue: string; concurrency: number }[],
+): Promise<WorkerConcurrencyPlan> {
+  const { resolvePoolConfig } = await import('@open-mercato/shared/lib/db/mikro')
+  const poolMax = resolvePoolConfig(process.env).poolMax
+  const budget = resolveWorkerConnectionBudget(process.env, poolMax)
+  const plan = planWorkerConcurrency(requestedByQueue, budget)
+
+  console.log(
+    `[worker] DB connection budget: ${plan.budget} (pool max ${poolMax}); ` +
+      `requested Σconcurrency ${plan.totalRequested}, effective ${plan.totalEffective}`,
+  )
+  if (plan.clamped) {
+    const perQueue = plan.entries
+      .map((entry) => `${entry.queue}=${entry.effective}/${entry.requested}`)
+      .join(', ')
+    console.warn(
+      `[worker] Worker concurrency clamped to fit the DB connection budget (${plan.budget}): ${perQueue}. ` +
+        `Raise DB_POOL_MAX or set OM_WORKERS_DB_CONNECTION_BUDGET to change this. ` +
+        `Keep web_pool_max + worker_pool_max + overhead <= Postgres max_connections.`,
+    )
+  }
+  if (plan.belowQueueFloor) {
+    console.warn(
+      `[worker] DB connection budget (${plan.budget}) is smaller than the number of queues ` +
+        `(${requestedByQueue.length}); every queue runs at concurrency 1 and total demand ` +
+        `(${plan.totalEffective}) still exceeds the budget. Raise DB_POOL_MAX.`,
+    )
+  }
+  return plan
+}
+
 function lookupModuleCommand(
   allModules: Module[],
   moduleName: string,
@@ -1520,12 +1565,31 @@ export async function run(argv = process.argv) {
             }
 
             const { createRequestContainer } = await import('@open-mercato/shared/lib/di/container')
+
+            // Fit Σconcurrency to the worker's DB connection budget before
+            // starting any worker, so the per-job containers (one connection each)
+            // can never over-subscribe the pool the request path shares.
+            const requestedByQueue = discoveredQueues.map((queue) => {
+              const queueWorkers = allWorkers.filter((w) => w.queue === queue)
+              return {
+                queue,
+                concurrency: concurrencyOverride ?? Math.max(...queueWorkers.map((w) => w.concurrency), 1),
+              }
+            })
+            const budgetPlan = await resolveWorkerBudgetPlan(requestedByQueue)
+            const effectiveByQueue = new Map(
+              budgetPlan.entries.map((entry) => [entry.queue, entry.effective]),
+            )
+
             console.log(`[worker] Starting workers for all queues: ${discoveredQueues.join(', ')}`)
 
             // Start all queue workers in background mode
             const workerPromises = discoveredQueues.map(async (queue) => {
               const queueWorkers = allWorkers.filter((w) => w.queue === queue)
-              const concurrency = concurrencyOverride ?? Math.max(...queueWorkers.map((w) => w.concurrency), 1)
+              const concurrency =
+                effectiveByQueue.get(queue) ??
+                concurrencyOverride ??
+                Math.max(...queueWorkers.map((w) => w.concurrency), 1)
 
               console.log(`[worker] Starting "${queue}" with ${queueWorkers.length} handler(s), concurrency: ${concurrency}`)
 
@@ -1552,7 +1616,11 @@ export async function run(argv = process.argv) {
             if (queueWorkers.length > 0) {
               // Use discovered workers
               const { createRequestContainer } = await import('@open-mercato/shared/lib/di/container')
-              const concurrency = concurrencyOverride ?? Math.max(...queueWorkers.map((w) => w.concurrency), 1)
+              const requested = concurrencyOverride ?? Math.max(...queueWorkers.map((w) => w.concurrency), 1)
+              // Bound a single-queue run to the connection budget too, so it can
+              // never check out more pooled connections than the worker pool holds.
+              const budgetPlan = await resolveWorkerBudgetPlan([{ queue: queueName!, concurrency: requested }])
+              const concurrency = budgetPlan.entries[0]?.effective ?? requested
 
               console.log(`[worker] Found ${queueWorkers.length} worker(s) for queue "${queueName}"`)