@open-mercato/cli 0.6.6-develop.6389.1.135a466c47 → 0.6.6-develop.6390.1.210bbbc703

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@open-mercato/cli",
-  "version": "0.6.6-develop.6389.1.135a466c47",
+  "version": "0.6.6-develop.6390.1.210bbbc703",
   "license": "MIT",
   "type": "module",
   "main": "./dist/index.js",
@@ -60,8 +60,8 @@
     "@mikro-orm/decorators": "^7.1.4",
     "@mikro-orm/migrations": "^7.1.4",
     "@mikro-orm/postgresql": "^7.1.4",
-    "@open-mercato/queue": "0.6.6-develop.6389.1.135a466c47",
-    "@open-mercato/shared": "0.6.6-develop.6389.1.135a466c47",
+    "@open-mercato/queue": "0.6.6-develop.6390.1.210bbbc703",
+    "@open-mercato/shared": "0.6.6-develop.6390.1.210bbbc703",
     "cross-spawn": "^7.0.6",
     "pg": "8.22.0",
     "semver": "^7.8.5",
@@ -71,10 +71,10 @@
     "typescript": "^6.0.3"
   },
   "peerDependencies": {
-    "@open-mercato/shared": "0.6.6-develop.6389.1.135a466c47"
+    "@open-mercato/shared": "0.6.6-develop.6390.1.210bbbc703"
   },
   "devDependencies": {
-    "@open-mercato/shared": "0.6.6-develop.6389.1.135a466c47",
+    "@open-mercato/shared": "0.6.6-develop.6390.1.210bbbc703",
     "@types/jest": "^30.0.0",
     "jest": "^30.4.2",
     "ts-jest": "^29.4.11"

package/src/__tests__/mercato.test.ts

@@ -645,6 +645,7 @@ describe('server dev managed process exits', () => {
   const originalAutoSpawnScheduler = process.env.AUTO_SPAWN_SCHEDULER
   const originalAutoSpawnWorkers = process.env.AUTO_SPAWN_WORKERS
   const originalLazy = process.env.OM_AUTO_SPAWN_WORKERS_LAZY
+  const originalLazyMode = process.env.OM_AUTO_SPAWN_WORKERS_LAZY_MODE
   const originalLazyScheduler = process.env.OM_AUTO_SPAWN_SCHEDULER_LAZY
   const originalGenerateWatchMode = process.env.OM_DEV_GENERATE_WATCH_MODE
   const originalSingleDelivery = process.env.OM_EVENTS_SINGLE_DELIVERY
@@ -656,6 +657,7 @@ describe('server dev managed process exits', () => {
     process.env.AUTO_SPAWN_SCHEDULER = 'false'
     process.env.AUTO_SPAWN_WORKERS = 'true'
     delete process.env.OM_AUTO_SPAWN_WORKERS_LAZY
+    delete process.env.OM_AUTO_SPAWN_WORKERS_LAZY_MODE
     delete process.env.OM_AUTO_SPAWN_SCHEDULER_LAZY
     // These tests toggle AUTO_SPAWN_WORKERS to exercise scheduler/Next exits, not
     // the events single-delivery guard. Acknowledge an external events worker so
@@ -683,6 +685,7 @@ describe('server dev managed process exits', () => {
     if (originalGenerateWatchMode === undefined) delete process.env.OM_DEV_GENERATE_WATCH_MODE
     else process.env.OM_DEV_GENERATE_WATCH_MODE = originalGenerateWatchMode
     delete process.env.OM_AUTO_SPAWN_WORKERS_LAZY
+    delete process.env.OM_AUTO_SPAWN_WORKERS_LAZY_MODE
     delete process.env.OM_AUTO_SPAWN_SCHEDULER_LAZY
   })
 
@@ -691,6 +694,8 @@ describe('server dev managed process exits', () => {
     process.env.AUTO_SPAWN_WORKERS = originalAutoSpawnWorkers
     if (originalLazy === undefined) delete process.env.OM_AUTO_SPAWN_WORKERS_LAZY
     else process.env.OM_AUTO_SPAWN_WORKERS_LAZY = originalLazy
+    if (originalLazyMode === undefined) delete process.env.OM_AUTO_SPAWN_WORKERS_LAZY_MODE
+    else process.env.OM_AUTO_SPAWN_WORKERS_LAZY_MODE = originalLazyMode
     if (originalLazyScheduler === undefined) delete process.env.OM_AUTO_SPAWN_SCHEDULER_LAZY
     else process.env.OM_AUTO_SPAWN_SCHEDULER_LAZY = originalLazyScheduler
     if (originalSingleDelivery === undefined) delete process.env.OM_EVENTS_SINGLE_DELIVERY
@@ -793,6 +798,7 @@ describe('server dev managed process exits', () => {
     const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation()
     const consoleLogSpy = jest.spyOn(console, 'log').mockImplementation()
     process.env.OM_AUTO_SPAWN_WORKERS_LAZY = 'true'
+    process.env.OM_AUTO_SPAWN_WORKERS_LAZY_MODE = 'shared'
 
     jest.doMock('node:fs', () => {
       const actual = jest.requireActual('node:fs')
@@ -843,8 +849,15 @@ describe('server dev managed process exits', () => {
     expect(supervisorCall.workers.map((worker) => worker.queue)).toEqual(['events'])
     expect(supervisorCall.pollMs).toBe(1000)
     expect(supervisorCall.restartOnUnexpectedExit).toBe(true)
+    expect(supervisorCall.spawnMode).toBe('shared')
     expect(supervisorClose).toHaveBeenCalled()
 
+    const lazyLogLine = consoleLogSpy.mock.calls
+      .map((call) => call[0])
+      .find((line) => typeof line === 'string' && line.startsWith('[server] Lazy worker auto-spawn enabled'))
+    expect(lazyLogLine).toContain('shared worker mode')
+    expect(lazyLogLine).toContain('OM_AUTO_SPAWN_WORKERS_LAZY_MODE=per-queue')
+
     const { spawn } = await import('child_process')
     const allSpawnCalls = (spawn as jest.Mock).mock.calls.map((call) => call[1] as string[])
     const queueWorkerSpawn = allSpawnCalls.find((args) => args.slice(1).join(' ') === 'queue worker --all')

package/src/lib/__tests__/auto-spawn-workers.test.ts

@@ -1,5 +1,6 @@
 import {
   resolveAutoSpawnWorkersMode,
+  resolveLazySpawnMode,
   resolveLazyPollMs,
   resolveLazyRestart,
 } from '../auto-spawn-workers'
@@ -82,6 +83,24 @@ describe('resolveLazyPollMs', () => {
   })
 })
 
+describe('resolveLazySpawnMode', () => {
+  it('defaults to per-queue workers when unset', () => {
+    expect(resolveLazySpawnMode({})).toBe('per-queue')
+  })
+
+  it('uses a shared worker when explicitly requested', () => {
+    expect(resolveLazySpawnMode({ OM_AUTO_SPAWN_WORKERS_LAZY_MODE: 'shared' })).toBe('shared')
+  })
+
+  it('accepts the explicit per-queue mode', () => {
+    expect(resolveLazySpawnMode({ OM_AUTO_SPAWN_WORKERS_LAZY_MODE: 'per-queue' })).toBe('per-queue')
+  })
+
+  it('falls back to per-queue for invalid values', () => {
+    expect(resolveLazySpawnMode({ OM_AUTO_SPAWN_WORKERS_LAZY_MODE: 'maybe' })).toBe('per-queue')
+  })
+})
+
 describe('resolveLazyRestart', () => {
   it('defaults to true when unset', () => {
     expect(resolveLazyRestart({})).toBe(true)

package/src/lib/__tests__/queue-worker-supervisor.test.ts

@@ -128,6 +128,48 @@ describe('startLazyWorkerSupervisor', () => {
     await handle.close()
   })
 
+  it('can start one shared worker for all queues when any queue has ready jobs', async () => {
+    const child = createFakeChild()
+    const spawnFn = jest.fn(() => child) as unknown as jest.MockedFunction<LazySupervisorSpawnFn>
+    const probeFn = jest.fn(async (queueName, strategy) => {
+      if (queueName === 'events' || queueName === 'emails') return readyProbe(queueName, strategy)
+      return emptyProbe(queueName, strategy)
+    }) as jest.MockedFunction<LazySupervisorProbeFn>
+
+    const handle = startLazyWorkerSupervisor({
+      mercatoBin: '/tmp/mercato',
+      appDir: '/tmp/app',
+      runtimeEnv: { ...process.env },
+      workers: [makeWorker('events'), makeWorker('emails'), makeWorker('search')],
+      pollMs: 250,
+      restartOnUnexpectedExit: false,
+      strategy: 'local',
+      spawnMode: 'shared',
+      spawnFn,
+      probeFn,
+      logger: silentLogger,
+    })
+
+    await flushAsync(20)
+
+    expect(spawnFn).toHaveBeenCalledTimes(1)
+    const [command, args] = spawnFn.mock.calls[0]
+    expect(command).toBe('node')
+    expect(args).toEqual(['/tmp/mercato', 'queue', 'worker', '--all'])
+    expect(handle.startedQueues.has('events')).toBe(true)
+    expect(handle.startedQueues.has('emails')).toBe(true)
+    expect(handle.startedQueues.has('search')).toBe(true)
+    expect(handle.getActiveChild('events')).toBe(child)
+    expect(handle.getActiveChild('emails')).toBe(child)
+    expect(handle.getActiveChild('search')).toBe(child)
+
+    await flushAsync(20)
+    expect(spawnFn).toHaveBeenCalledTimes(1)
+
+    await handle.close()
+    expect((child as any).kill).toHaveBeenCalledWith('SIGTERM')
+  })
+
   it('does not spawn when probe reports an error', async () => {
     const spawnFn = jest.fn() as unknown as jest.MockedFunction<LazySupervisorSpawnFn>
     const probeFn = jest.fn(async (queueName, strategy) => ({

package/src/lib/auto-spawn-workers.ts

@@ -1,6 +1,7 @@
 import { parseBooleanToken } from '@open-mercato/shared/lib/boolean'
 
 export type AutoSpawnWorkersMode = 'off' | 'eager' | 'lazy'
+export type LazyWorkerSpawnMode = 'per-queue' | 'shared'
 
 export type AutoSpawnEnvSource = Pick<NodeJS.ProcessEnv, string> | Record<string, string | undefined>
 
@@ -33,6 +34,13 @@ export function resolveLazyPollMs(env: AutoSpawnEnvSource = process.env): number
   return Math.max(MIN_LAZY_POLL_MS, Math.floor(parsed))
 }
 
+export function resolveLazySpawnMode(env: AutoSpawnEnvSource = process.env): LazyWorkerSpawnMode {
+  const raw = env.OM_AUTO_SPAWN_WORKERS_LAZY_MODE?.trim().toLowerCase()
+  if (raw === 'shared') return 'shared'
+  if (raw === 'per-queue') return 'per-queue'
+  return 'per-queue'
+}
+
 export function resolveLazyRestart(env: AutoSpawnEnvSource = process.env): boolean {
   const parsed = parseBooleanToken(env.OM_AUTO_SPAWN_WORKERS_LAZY_RESTART)
   return parsed === null ? true : parsed

package/src/lib/queue-worker-supervisor.ts

@@ -30,6 +30,7 @@ import {
   type QueueStrategyType,
 } from '@open-mercato/queue'
 import type { ModuleWorker } from '@open-mercato/shared/modules/registry'
+import type { LazyWorkerSpawnMode } from './auto-spawn-workers'
 
 export type LazySupervisorSpawnFn = (
   command: string,
@@ -49,6 +50,7 @@ export type LazyWorkerSupervisorOptions = {
   workers: ModuleWorker[]
   pollMs: number
   restartOnUnexpectedExit: boolean
+  spawnMode?: LazyWorkerSpawnMode
   strategy?: QueueStrategyType
   /** Override for tests. Defaults to `child_process.spawn`. */
   spawnFn?: LazySupervisorSpawnFn
@@ -112,6 +114,7 @@ export function startLazyWorkerSupervisor(
 ): LazyWorkerSupervisorHandle {
   const logger = options.logger ?? console
   const strategy: QueueStrategyType = options.strategy ?? resolveStrategyFromEnv(options.runtimeEnv)
+  const spawnMode: LazyWorkerSpawnMode = options.spawnMode ?? 'per-queue'
   const probe: LazySupervisorProbeFn = options.probeFn
     ?? ((queueName) => getQueuePendingProbe(queueName, strategy))
   const groups = groupWorkersByQueue(options.workers)
@@ -119,6 +122,8 @@ export function startLazyWorkerSupervisor(
   const startedQueues = new Set<string>()
   const activeChildren = new Map<string, ChildProcess>()
   const startingQueues = new Set<string>()
+  let activeSharedChild: ChildProcess | undefined
+  let startingSharedWorker = false
   const probeErrorLastLoggedAt = new Map<string, number>()
 
   let stopping = false
@@ -143,6 +148,22 @@ export function startLazyWorkerSupervisor(
   const spawnFn: LazySupervisorSpawnFn =
     options.spawnFn ?? ((command, args, opts) => nodeSpawn(command, args as string[], opts))
 
+  const markSharedWorkerStarted = () => {
+    for (const group of groups) {
+      startedQueues.add(group.queueName)
+      activeChildren.set(group.queueName, activeSharedChild as ChildProcess)
+    }
+  }
+
+  const clearSharedWorker = () => {
+    for (const group of groups) {
+      if (activeChildren.get(group.queueName) === activeSharedChild) {
+        activeChildren.delete(group.queueName)
+      }
+    }
+    activeSharedChild = undefined
+  }
+
   function logProbeError(queueName: string, message: string): void {
     const now = Date.now()
     const lastAt = probeErrorLastLoggedAt.get(queueName) ?? 0
@@ -215,6 +236,81 @@ export function startLazyWorkerSupervisor(
     })
   }
 
+  function spawnSharedWorker(triggerQueueNames: string[]): void {
+    if (stopping) return
+    if (activeSharedChild) return
+    if (startingSharedWorker) return
+    startingSharedWorker = true
+
+    const triggerSummary = triggerQueueNames.length > 0
+      ? ` after pending job(s) on ${triggerQueueNames.join(', ')}`
+      : ''
+    logger.log(`[lazy-supervisor] Pending job detected${triggerSummary} — starting shared worker for all queues`)
+    let child: ChildProcess
+    try {
+      child = spawnFn(
+        'node',
+        [options.mercatoBin, 'queue', 'worker', '--all'],
+        {
+          stdio: 'inherit',
+          env: options.runtimeEnv,
+          cwd: options.appDir,
+        },
+      )
+    } catch (err) {
+      startingSharedWorker = false
+      const message = err instanceof Error ? err.message : String(err)
+      logger.error(`[lazy-supervisor] Failed to spawn shared worker: ${message}`)
+      return
+    }
+
+    activeSharedChild = child
+    markSharedWorkerStarted()
+    startingSharedWorker = false
+    for (const group of groups) {
+      if (options.onSpawn) {
+        try {
+          options.onSpawn(group.queueName, child)
+        } catch (err) {
+          const message = err instanceof Error ? err.message : String(err)
+          logger.warn(`[lazy-supervisor] onSpawn callback threw for "${group.queueName}": ${message}`)
+        }
+      }
+    }
+
+    child.on('exit', (code, signal) => {
+      clearSharedWorker()
+      for (const group of groups) {
+        if (options.onChildExit) {
+          try {
+            options.onChildExit(group.queueName, code, signal)
+          } catch (err) {
+            const message = err instanceof Error ? err.message : String(err)
+            logger.warn(`[lazy-supervisor] onChildExit callback threw for "${group.queueName}": ${message}`)
+          }
+        }
+      }
+      if (stopping) return
+      const expected = signal === 'SIGTERM' || signal === 'SIGINT'
+      if (expected) return
+      const reason = code !== null ? `code ${code}` : `signal ${signal ?? 'unknown'}`
+      logger.warn(`[lazy-supervisor] Shared worker exited (${reason}).`)
+      if (!options.restartOnUnexpectedExit) return
+      void (async () => {
+        const readyQueues: string[] = []
+        for (const group of groups) {
+          const result = await safeProbe(group.queueName)
+          if (!result || result.error) continue
+          if (result.ready > 0) readyQueues.push(group.queueName)
+        }
+        if (readyQueues.length > 0) {
+          logger.warn('[lazy-supervisor] Restarting shared worker because jobs remain pending.')
+          spawnSharedWorker(readyQueues)
+        }
+      })()
+    })
+  }
+
   async function safeProbe(queueName: string): Promise<QueuePendingProbeResult | null> {
     try {
       return await probe(queueName, strategy)
@@ -228,6 +324,7 @@ export function startLazyWorkerSupervisor(
   async function tickOnce(): Promise<void> {
     if (stopping) return
     if (groups.length === 0) return
+    if (spawnMode === 'shared' && (activeSharedChild || startingSharedWorker)) return
 
     const toCheck = groups.filter((g) => !activeChildren.has(g.queueName) && !startingQueues.has(g.queueName))
     if (toCheck.length === 0) return
@@ -241,6 +338,7 @@ export function startLazyWorkerSupervisor(
 
     if (stopping) return
 
+    const readySharedQueues: string[] = []
     for (const { group, result } of probes) {
       if (!result) continue
       if (result.error) {
@@ -250,9 +348,16 @@ export function startLazyWorkerSupervisor(
         continue
       }
       if (result.ready > 0) {
+        if (spawnMode === 'shared') {
+          readySharedQueues.push(group.queueName)
+          continue
+        }
         spawnQueueWorker(group.queueName)
       }
     }
+    if (spawnMode === 'shared' && readySharedQueues.length > 0) {
+      spawnSharedWorker(readySharedQueues)
+    }
   }
 
   function scheduleNext(): void {

package/src/mercato.ts

@@ -11,6 +11,7 @@ import { getRedisUrl, getRedisUrlOrThrow } from '@open-mercato/shared/lib/redis/
 import { resolveInitDerivedSecrets } from './lib/init-secrets'
 import {
   resolveAutoSpawnWorkersMode,
+  resolveLazySpawnMode,
   resolveLazyPollMs,
   resolveLazyRestart,
 } from './lib/auto-spawn-workers'
@@ -2119,7 +2120,11 @@ export async function run(argv = process.argv) {
                   if (discoveredWorkerQueues.length === 0) {
                     console.error('[server] AUTO_SPAWN_WORKERS is enabled, but no queues were discovered from CLI modules. Run `yarn generate` and verify `.mercato/generated/modules.cli.generated.ts` contains worker entries. Continuing without auto-spawned workers.')
                   } else if (autoSpawnWorkersMode === 'lazy') {
-                    console.log(`[server] Lazy worker auto-spawn enabled — workers will start on first job (${discoveredWorkerQueues.length} queue(s) watched).`)
+                    const lazySpawnMode = resolveLazySpawnMode(process.env)
+                    const lazyModeHint = lazySpawnMode === 'shared'
+                      ? 'shared worker mode — one `queue worker --all` process; set OM_AUTO_SPAWN_WORKERS_LAZY_MODE=per-queue to change'
+                      : 'per-queue worker mode — one process per queue; set OM_AUTO_SPAWN_WORKERS_LAZY_MODE=shared to change'
+                    console.log(`[server] Lazy worker auto-spawn enabled — workers will start on first job (${discoveredWorkerQueues.length} queue(s) watched, ${lazyModeHint}).`)
                     activeLazySupervisor = startLazyWorkerSupervisor({
                       mercatoBin,
                       appDir,
@@ -2127,6 +2132,7 @@ export async function run(argv = process.argv) {
                       workers: discoveredWorkers,
                       pollMs: resolveLazyPollMs(process.env),
                       restartOnUnexpectedExit: resolveLazyRestart(process.env),
+                      spawnMode: lazySpawnMode,
                     })
                   } else {
                     console.log('[server] Eager worker auto-spawn enabled - starting workers for all queues...')
@@ -2324,7 +2330,11 @@ export async function run(argv = process.argv) {
               if (discoveredWorkerQueues.length === 0) {
                 console.error('[server] AUTO_SPAWN_WORKERS is enabled, but no queues were discovered from CLI modules. Run `yarn generate` and verify `.mercato/generated/modules.cli.generated.ts` contains worker entries. Continuing without auto-spawned workers.')
               } else if (autoSpawnWorkersMode === 'lazy') {
-                console.log(`[server] Lazy worker auto-spawn enabled — workers will start on first job (${discoveredWorkerQueues.length} queue(s) watched).`)
+                const lazySpawnMode = resolveLazySpawnMode(process.env)
+                const lazyModeHint = lazySpawnMode === 'shared'
+                  ? 'shared worker mode — one `queue worker --all` process; set OM_AUTO_SPAWN_WORKERS_LAZY_MODE=per-queue to change'
+                  : 'per-queue worker mode — one process per queue; set OM_AUTO_SPAWN_WORKERS_LAZY_MODE=shared to change'
+                console.log(`[server] Lazy worker auto-spawn enabled — workers will start on first job (${discoveredWorkerQueues.length} queue(s) watched, ${lazyModeHint}).`)
                 activeLazySupervisor = startLazyWorkerSupervisor({
                   mercatoBin,
                   appDir,
@@ -2332,6 +2342,7 @@ export async function run(argv = process.argv) {
                   workers: discoveredWorkers,
                   pollMs: resolveLazyPollMs(process.env),
                   restartOnUnexpectedExit: resolveLazyRestart(process.env),
+                  spawnMode: lazySpawnMode,
                 })
               } else {
                 console.log('[server] Eager worker auto-spawn enabled - starting workers for all queues...')