@open-mercato/cli 0.6.6-develop.5586.1.c9ed1d68a8 → 0.6.6-develop.5594.1.30cd738303

package/.turbo/turbo-build.log

@@ -1,4 +1,4 @@
-[build:cli] found 89 entry points
+[build:cli] found 91 entry points
 Copied create-app/agentic/ → dist/agentic/
 Discovered 16 standalone guides → dist/agentic/guides/
 [build:cli] built successfully

package/dist/lib/events-single-delivery.js

@@ -0,0 +1,27 @@
+import { parseBooleanWithDefault } from "@open-mercato/shared/lib/boolean";
+const EVENTS_SINGLE_DELIVERY_ENV = "OM_EVENTS_SINGLE_DELIVERY";
+const EVENTS_EXTERNAL_WORKER_ENV = "OM_EVENTS_EXTERNAL_WORKER";
+function reconcileEventsSingleDelivery(env, autoSpawnWorkersMode) {
+  const requested = parseBooleanWithDefault(env[EVENTS_SINGLE_DELIVERY_ENV], true);
+  if (!requested) return { effective: false };
+  const externalWorker = parseBooleanWithDefault(env[EVENTS_EXTERNAL_WORKER_ENV], false);
+  const workersAvailable = autoSpawnWorkersMode !== "off" || externalWorker;
+  if (workersAvailable) return { effective: true };
+  return {
+    effective: false,
+    warning: `[events] ${EVENTS_SINGLE_DELIVERY_ENV} is on (default) but this process auto-spawns no events worker (AUTO_SPAWN_WORKERS=off) and ${EVENTS_EXTERNAL_WORKER_ENV} is not set. Persistent subscribers would be skipped inline with nothing to drain the queue, silently dropping notifications, queued emails, and indexing. Falling back to legacy inline dual-dispatch for safety. To keep single-delivery, run an events worker (\`mercato queue worker events\`) and set ${EVENTS_EXTERNAL_WORKER_ENV}=true, or enable AUTO_SPAWN_WORKERS.`
+  };
+}
+function applyEventsSingleDeliveryGuard(args) {
+  const result = reconcileEventsSingleDelivery(args.processEnv, args.autoSpawnWorkersMode);
+  if (result.warning) console.error(result.warning);
+  const value = result.effective ? "true" : "false";
+  args.processEnv[EVENTS_SINGLE_DELIVERY_ENV] = value;
+  args.runtimeEnv[EVENTS_SINGLE_DELIVERY_ENV] = value;
+  return result;
+}
+export {
+  applyEventsSingleDeliveryGuard,
+  reconcileEventsSingleDelivery
+};
+//# sourceMappingURL=events-single-delivery.js.map

package/dist/lib/events-single-delivery.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/lib/events-single-delivery.ts"],
+  "sourcesContent": ["import { parseBooleanWithDefault } from '@open-mercato/shared/lib/boolean'\nimport type { AutoSpawnWorkersMode } from './auto-spawn-workers'\n\n// Keep these env names in sync with the runtime source of truth,\n// `@open-mercato/events/single-delivery`. The CLI cannot import the events\n// package (no dependency edge), so the reconcile logic is mirrored here for the\n// server-bootstrap guard only; the bus and worker own the runtime behavior.\nconst EVENTS_SINGLE_DELIVERY_ENV = 'OM_EVENTS_SINGLE_DELIVERY'\nconst EVENTS_EXTERNAL_WORKER_ENV = 'OM_EVENTS_EXTERNAL_WORKER'\n\ntype EnvSource = NodeJS.ProcessEnv | Record<string, string | undefined>\n\nexport type SingleDeliveryReconciliation = {\n  effective: boolean\n  warning?: string\n}\n\n/**\n * Server-bootstrap guard for the default-on single-delivery dispatch.\n *\n * With single-delivery on, persistent subscribers are skipped inline and ONLY\n * the events worker dispatches them. A process that runs NO events worker\n * (auto-spawn off and no acknowledged external worker) would skip those\n * subscribers with nothing to drain the queue \u2014 silently dropping notifications,\n * queued emails, and indexing. This fails safe by disabling single-delivery for\n * such a process (back to inline dual-dispatch) and returning a loud warning.\n *\n * Transient worker downtime is NOT this guard's concern: the durable queue holds\n * the job until a worker returns. Only the \"no worker at all\" config is guarded.\n */\nexport function reconcileEventsSingleDelivery(\n  env: EnvSource,\n  autoSpawnWorkersMode: AutoSpawnWorkersMode,\n): SingleDeliveryReconciliation {\n  const requested = parseBooleanWithDefault(env[EVENTS_SINGLE_DELIVERY_ENV], true)\n  if (!requested) return { effective: false }\n  const externalWorker = parseBooleanWithDefault(env[EVENTS_EXTERNAL_WORKER_ENV], false)\n  const workersAvailable = autoSpawnWorkersMode !== 'off' || externalWorker\n  if (workersAvailable) return { effective: true }\n  return {\n    effective: false,\n    warning:\n      `[events] ${EVENTS_SINGLE_DELIVERY_ENV} is on (default) but this process auto-spawns no events worker ` +\n      `(AUTO_SPAWN_WORKERS=off) and ${EVENTS_EXTERNAL_WORKER_ENV} is not set. Persistent subscribers would be ` +\n      `skipped inline with nothing to drain the queue, silently dropping notifications, queued emails, and ` +\n      `indexing. Falling back to legacy inline dual-dispatch for safety. To keep single-delivery, run an events ` +\n      `worker (\\`mercato queue worker events\\`) and set ${EVENTS_EXTERNAL_WORKER_ENV}=true, or enable ` +\n      `AUTO_SPAWN_WORKERS.`,\n  }\n}\n\n/**\n * Applies {@link reconcileEventsSingleDelivery} and writes the effective value\n * into both the current process env (for the in-process app/bus) and the spawned\n * worker/child env (so a child worker reads the same value the bus does). Logs\n * the warning once when single-delivery is disabled for safety.\n */\nexport function applyEventsSingleDeliveryGuard(args: {\n  processEnv: NodeJS.ProcessEnv\n  runtimeEnv: NodeJS.ProcessEnv\n  autoSpawnWorkersMode: AutoSpawnWorkersMode\n}): SingleDeliveryReconciliation {\n  const result = reconcileEventsSingleDelivery(args.processEnv, args.autoSpawnWorkersMode)\n  if (result.warning) console.error(result.warning)\n  const value = result.effective ? 'true' : 'false'\n  args.processEnv[EVENTS_SINGLE_DELIVERY_ENV] = value\n  args.runtimeEnv[EVENTS_SINGLE_DELIVERY_ENV] = value\n  return result\n}\n"],
+  "mappings": "AAAA,SAAS,+BAA+B;AAOxC,MAAM,6BAA6B;AACnC,MAAM,6BAA6B;AAsB5B,SAAS,8BACd,KACA,sBAC8B;AAC9B,QAAM,YAAY,wBAAwB,IAAI,0BAA0B,GAAG,IAAI;AAC/E,MAAI,CAAC,UAAW,QAAO,EAAE,WAAW,MAAM;AAC1C,QAAM,iBAAiB,wBAAwB,IAAI,0BAA0B,GAAG,KAAK;AACrF,QAAM,mBAAmB,yBAAyB,SAAS;AAC3D,MAAI,iBAAkB,QAAO,EAAE,WAAW,KAAK;AAC/C,SAAO;AAAA,IACL,WAAW;AAAA,IACX,SACE,YAAY,0BAA0B,+FACN,0BAA0B,8SAGN,0BAA0B;AAAA,EAElF;AACF;AAQO,SAAS,+BAA+B,MAId;AAC/B,QAAM,SAAS,8BAA8B,KAAK,YAAY,KAAK,oBAAoB;AACvF,MAAI,OAAO,QAAS,SAAQ,MAAM,OAAO,OAAO;AAChD,QAAM,QAAQ,OAAO,YAAY,SAAS;AAC1C,OAAK,WAAW,0BAA0B,IAAI;AAC9C,OAAK,WAAW,0BAA0B,IAAI;AAC9C,SAAO;AACT;",
+  "names": []
+}

package/dist/lib/worker-connection-budget.js

@@ -0,0 +1,63 @@
+function toPositiveInt(value, fallback) {
+  if (value === void 0 || !Number.isFinite(value)) return fallback;
+  const floored = Math.floor(value);
+  return floored > 0 ? floored : fallback;
+}
+function resolveWorkerConnectionBudget(env, poolMax) {
+  const safePoolMax = toPositiveInt(poolMax, 1);
+  const raw = env.OM_WORKERS_DB_CONNECTION_BUDGET;
+  if (!raw) return safePoolMax;
+  const parsed = Number.parseInt(raw, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : safePoolMax;
+}
+function planWorkerConcurrency(queues, budget) {
+  const safeBudget = toPositiveInt(budget, 1);
+  const requested = queues.map((entry) => ({
+    queue: entry.queue,
+    requested: toPositiveInt(entry.concurrency, 1)
+  }));
+  const totalRequested = requested.reduce((sum, entry) => sum + entry.requested, 0);
+  if (totalRequested <= safeBudget) {
+    return {
+      budget: safeBudget,
+      totalRequested,
+      totalEffective: totalRequested,
+      clamped: false,
+      belowQueueFloor: false,
+      entries: requested.map((entry) => ({ ...entry, effective: entry.requested }))
+    };
+  }
+  const effective = requested.map(() => 1);
+  let used = effective.length;
+  while (used < safeBudget) {
+    let bestIndex = -1;
+    let bestDeficit = 0;
+    for (let index = 0; index < requested.length; index += 1) {
+      const deficit = requested[index].requested - effective[index];
+      if (deficit > bestDeficit) {
+        bestDeficit = deficit;
+        bestIndex = index;
+      }
+    }
+    if (bestIndex === -1) break;
+    effective[bestIndex] += 1;
+    used += 1;
+  }
+  const totalEffective = effective.reduce((sum, value) => sum + value, 0);
+  return {
+    budget: safeBudget,
+    totalRequested,
+    totalEffective,
+    clamped: true,
+    belowQueueFloor: requested.length > safeBudget,
+    entries: requested.map((entry, index) => ({
+      ...entry,
+      effective: effective[index]
+    }))
+  };
+}
+export {
+  planWorkerConcurrency,
+  resolveWorkerConnectionBudget
+};
+//# sourceMappingURL=worker-connection-budget.js.map

package/dist/lib/worker-connection-budget.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/lib/worker-connection-budget.ts"],
+  "sourcesContent": ["/**\n * Bound the DB-connection demand of background queue workers.\n *\n * Since #2970/#3011 each worker job builds its own request container, so its own\n * `EntityManager` checks out a dedicated pooled DB connection for the job's\n * duration. The peak connection demand of `worker --all` is therefore the sum of\n * every queue's concurrency. Nothing previously bounded that sum against the DB\n * pool (`DB_POOL_MAX`) or the database's global `max_connections`, so a storm of\n * slow/failing jobs could over-subscribe connections \u2014 thrashing on the worker's\n * own acquire timeout and, across the worker + web processes, starving the\n * request/onboarding path that shares the same database.\n *\n * These helpers derive a per-queue effective-concurrency plan whose total never\n * exceeds a connection budget (defaulting to the resolved pool max), while\n * guaranteeing every queue keeps at least one worker.\n */\n\nexport type WorkerQueueConcurrency = {\n  queue: string\n  concurrency: number\n}\n\nexport type WorkerConcurrencyPlanEntry = {\n  queue: string\n  requested: number\n  effective: number\n}\n\nexport type WorkerConcurrencyPlan = {\n  /** Connection budget the plan was fitted to. */\n  budget: number\n  /** Sum of requested concurrency across all queues. */\n  totalRequested: number\n  /** Sum of effective concurrency the plan grants. */\n  totalEffective: number\n  /** True when any queue's concurrency was reduced to fit the budget. */\n  clamped: boolean\n  /**\n   * True when the budget is smaller than the number of queues, so the per-queue\n   * floor of 1 forces the total above the budget. The caller should surface this\n   * as a misconfiguration (raise `DB_POOL_MAX` or the budget).\n   */\n  belowQueueFloor: boolean\n  entries: WorkerConcurrencyPlanEntry[]\n}\n\nfunction toPositiveInt(value: number | undefined, fallback: number): number {\n  if (value === undefined || !Number.isFinite(value)) return fallback\n  const floored = Math.floor(value)\n  return floored > 0 ? floored : fallback\n}\n\n/**\n * Resolve the connection budget for background workers. Defaults to the resolved\n * DB pool max so the worker never tries to use more connections than its own pool\n * can hand out; override with `OM_WORKERS_DB_CONNECTION_BUDGET` (positive int).\n */\nexport function resolveWorkerConnectionBudget(\n  env: NodeJS.ProcessEnv,\n  poolMax: number,\n): number {\n  const safePoolMax = toPositiveInt(poolMax, 1)\n  const raw = env.OM_WORKERS_DB_CONNECTION_BUDGET\n  if (!raw) return safePoolMax\n  const parsed = Number.parseInt(raw, 10)\n  return Number.isFinite(parsed) && parsed > 0 ? parsed : safePoolMax\n}\n\n/**\n * Fit per-queue concurrency to a connection budget.\n *\n * - Every queue keeps a floor of 1 worker (no queue is starved).\n * - No queue exceeds its requested concurrency.\n * - When the sum exceeds the budget, leftover capacity is distributed greedily to\n *   the queues furthest below their request, so the total matches the budget\n *   exactly (when `budget >= queueCount`) and the allocation is deterministic.\n */\nexport function planWorkerConcurrency(\n  queues: WorkerQueueConcurrency[],\n  budget: number,\n): WorkerConcurrencyPlan {\n  const safeBudget = toPositiveInt(budget, 1)\n  const requested = queues.map((entry) => ({\n    queue: entry.queue,\n    requested: toPositiveInt(entry.concurrency, 1),\n  }))\n  const totalRequested = requested.reduce((sum, entry) => sum + entry.requested, 0)\n\n  if (totalRequested <= safeBudget) {\n    return {\n      budget: safeBudget,\n      totalRequested,\n      totalEffective: totalRequested,\n      clamped: false,\n      belowQueueFloor: false,\n      entries: requested.map((entry) => ({ ...entry, effective: entry.requested })),\n    }\n  }\n\n  // Floor every queue at 1, then water-fill the remaining budget to the queues\n  // with the largest unmet request first.\n  const effective = requested.map(() => 1)\n  let used = effective.length\n  while (used < safeBudget) {\n    let bestIndex = -1\n    let bestDeficit = 0\n    for (let index = 0; index < requested.length; index += 1) {\n      const deficit = requested[index].requested - effective[index]\n      if (deficit > bestDeficit) {\n        bestDeficit = deficit\n        bestIndex = index\n      }\n    }\n    if (bestIndex === -1) break\n    effective[bestIndex] += 1\n    used += 1\n  }\n\n  const totalEffective = effective.reduce((sum, value) => sum + value, 0)\n  return {\n    budget: safeBudget,\n    totalRequested,\n    totalEffective,\n    clamped: true,\n    belowQueueFloor: requested.length > safeBudget,\n    entries: requested.map((entry, index) => ({\n      ...entry,\n      effective: effective[index],\n    })),\n  }\n}\n"],
+  "mappings": "AA8CA,SAAS,cAAc,OAA2B,UAA0B;AAC1E,MAAI,UAAU,UAAa,CAAC,OAAO,SAAS,KAAK,EAAG,QAAO;AAC3D,QAAM,UAAU,KAAK,MAAM,KAAK;AAChC,SAAO,UAAU,IAAI,UAAU;AACjC;AAOO,SAAS,8BACd,KACA,SACQ;AACR,QAAM,cAAc,cAAc,SAAS,CAAC;AAC5C,QAAM,MAAM,IAAI;AAChB,MAAI,CAAC,IAAK,QAAO;AACjB,QAAM,SAAS,OAAO,SAAS,KAAK,EAAE;AACtC,SAAO,OAAO,SAAS,MAAM,KAAK,SAAS,IAAI,SAAS;AAC1D;AAWO,SAAS,sBACd,QACA,QACuB;AACvB,QAAM,aAAa,cAAc,QAAQ,CAAC;AAC1C,QAAM,YAAY,OAAO,IAAI,CAAC,WAAW;AAAA,IACvC,OAAO,MAAM;AAAA,IACb,WAAW,cAAc,MAAM,aAAa,CAAC;AAAA,EAC/C,EAAE;AACF,QAAM,iBAAiB,UAAU,OAAO,CAAC,KAAK,UAAU,MAAM,MAAM,WAAW,CAAC;AAEhF,MAAI,kBAAkB,YAAY;AAChC,WAAO;AAAA,MACL,QAAQ;AAAA,MACR;AAAA,MACA,gBAAgB;AAAA,MAChB,SAAS;AAAA,MACT,iBAAiB;AAAA,MACjB,SAAS,UAAU,IAAI,CAAC,WAAW,EAAE,GAAG,OAAO,WAAW,MAAM,UAAU,EAAE;AAAA,IAC9E;AAAA,EACF;AAIA,QAAM,YAAY,UAAU,IAAI,MAAM,CAAC;AACvC,MAAI,OAAO,UAAU;AACrB,SAAO,OAAO,YAAY;AACxB,QAAI,YAAY;AAChB,QAAI,cAAc;AAClB,aAAS,QAAQ,GAAG,QAAQ,UAAU,QAAQ,SAAS,GAAG;AACxD,YAAM,UAAU,UAAU,KAAK,EAAE,YAAY,UAAU,KAAK;AAC5D,UAAI,UAAU,aAAa;AACzB,sBAAc;AACd,oBAAY;AAAA,MACd;AAAA,IACF;AACA,QAAI,cAAc,GAAI;AACtB,cAAU,SAAS,KAAK;AACxB,YAAQ;AAAA,EACV;AAEA,QAAM,iBAAiB,UAAU,OAAO,CAAC,KAAK,UAAU,MAAM,OAAO,CAAC;AACtE,SAAO;AAAA,IACL,QAAQ;AAAA,IACR;AAAA,IACA;AAAA,IACA,SAAS;AAAA,IACT,iBAAiB,UAAU,SAAS;AAAA,IACpC,SAAS,UAAU,IAAI,CAAC,OAAO,WAAW;AAAA,MACxC,GAAG;AAAA,MACH,WAAW,UAAU,KAAK;AAAA,IAC5B,EAAE;AAAA,EACJ;AACF;",
+  "names": []
+}

package/dist/mercato.js

@@ -10,7 +10,12 @@ import {
   resolveLazyRestart
 } from "./lib/auto-spawn-workers.js";
 import { startLazyWorkerSupervisor } from "./lib/queue-worker-supervisor.js";
+import { applyEventsSingleDeliveryGuard } from "./lib/events-single-delivery.js";
 import { createPerJobWorkerHandler } from "./lib/worker-job-handler.js";
+import {
+  planWorkerConcurrency,
+  resolveWorkerConnectionBudget
+} from "./lib/worker-connection-budget.js";
 import {
   resolveAutoSpawnSchedulerMode,
   resolveLazySchedulerPollMs,
@@ -379,6 +384,27 @@ function formatQueueWorkerLabel(queueNames) {
   const preview = sorted.length > 4 ? `${sorted.slice(0, 4).join(", ")}, +${sorted.length - 4} more` : sorted.join(", ");
   return `Queue worker (${preview})`;
 }
+async function resolveWorkerBudgetPlan(requestedByQueue) {
+  const { resolvePoolConfig } = await import("@open-mercato/shared/lib/db/mikro");
+  const poolMax = resolvePoolConfig(process.env).poolMax;
+  const budget = resolveWorkerConnectionBudget(process.env, poolMax);
+  const plan = planWorkerConcurrency(requestedByQueue, budget);
+  console.log(
+    `[worker] DB connection budget: ${plan.budget} (pool max ${poolMax}); requested \u03A3concurrency ${plan.totalRequested}, effective ${plan.totalEffective}`
+  );
+  if (plan.clamped) {
+    const perQueue = plan.entries.map((entry) => `${entry.queue}=${entry.effective}/${entry.requested}`).join(", ");
+    console.warn(
+      `[worker] Worker concurrency clamped to fit the DB connection budget (${plan.budget}): ${perQueue}. Raise DB_POOL_MAX or set OM_WORKERS_DB_CONNECTION_BUDGET to change this. Keep web_pool_max + worker_pool_max + overhead <= Postgres max_connections.`
+    );
+  }
+  if (plan.belowQueueFloor) {
+    console.warn(
+      `[worker] DB connection budget (${plan.budget}) is smaller than the number of queues (${requestedByQueue.length}); every queue runs at concurrency 1 and total demand (${plan.totalEffective}) still exceeds the budget. Raise DB_POOL_MAX.`
+    );
+  }
+  return plan;
+}
 function lookupModuleCommand(allModules, moduleName, commandName) {
   const mod = allModules.find((entry) => entry.id === moduleName);
   if (!mod) {
@@ -1226,10 +1252,21 @@ async function run(argv = process.argv) {
               return;
             }
             const { createRequestContainer } = await import("@open-mercato/shared/lib/di/container");
+            const requestedByQueue = discoveredQueues.map((queue) => {
+              const queueWorkers = allWorkers.filter((w) => w.queue === queue);
+              return {
+                queue,
+                concurrency: concurrencyOverride ?? Math.max(...queueWorkers.map((w) => w.concurrency), 1)
+              };
+            });
+            const budgetPlan = await resolveWorkerBudgetPlan(requestedByQueue);
+            const effectiveByQueue = new Map(
+              budgetPlan.entries.map((entry) => [entry.queue, entry.effective])
+            );
             console.log(`[worker] Starting workers for all queues: ${discoveredQueues.join(", ")}`);
             const workerPromises = discoveredQueues.map(async (queue) => {
               const queueWorkers = allWorkers.filter((w) => w.queue === queue);
-              const concurrency = concurrencyOverride ?? Math.max(...queueWorkers.map((w) => w.concurrency), 1);
+              const concurrency = effectiveByQueue.get(queue) ?? concurrencyOverride ?? Math.max(...queueWorkers.map((w) => w.concurrency), 1);
               console.log(`[worker] Starting "${queue}" with ${queueWorkers.length} handler(s), concurrency: ${concurrency}`);
               const queueRedisUrl = getRedisUrl("QUEUE");
               await runWorker({
@@ -1248,7 +1285,9 @@ async function run(argv = process.argv) {
             const queueWorkers = allWorkers.filter((w) => w.queue === queueName);
             if (queueWorkers.length > 0) {
               const { createRequestContainer } = await import("@open-mercato/shared/lib/di/container");
-              const concurrency = concurrencyOverride ?? Math.max(...queueWorkers.map((w) => w.concurrency), 1);
+              const requested = concurrencyOverride ?? Math.max(...queueWorkers.map((w) => w.concurrency), 1);
+              const budgetPlan = await resolveWorkerBudgetPlan([{ queue: queueName, concurrency: requested }]);
+              const concurrency = budgetPlan.entries[0]?.effective ?? requested;
               console.log(`[worker] Found ${queueWorkers.length} worker(s) for queue "${queueName}"`);
               const queueRedisUrl = getRedisUrl("QUEUE");
               await runWorker({
@@ -1639,6 +1678,7 @@ async function run(argv = process.argv) {
               envReloader.reload();
               const runtimeEnv = buildServerProcessEnvironment(process.env);
               const autoSpawnWorkersMode = resolveAutoSpawnWorkersMode(process.env);
+              applyEventsSingleDeliveryGuard({ processEnv: process.env, runtimeEnv, autoSpawnWorkersMode });
               const autoSpawnSchedulerMode = resolveAutoSpawnSchedulerMode(process.env);
               const queueStrategy = process.env.QUEUE_STRATEGY || "local";
               const schedulerCommand = lookupModuleCommand(getCliModules(), "scheduler", "start");
@@ -1773,6 +1813,7 @@ async function run(argv = process.argv) {
           const autoSpawnSchedulerMode = resolveAutoSpawnSchedulerMode(process.env);
           const queueStrategy = process.env.QUEUE_STRATEGY || "local";
           const runtimeEnv = buildServerProcessEnvironment(process.env);
+          applyEventsSingleDeliveryGuard({ processEnv: process.env, runtimeEnv, autoSpawnWorkersMode });
           assertSingleInstanceStrategies(runtimeEnv);
           const schedulerCommand = lookupModuleCommand(getCliModules(), "scheduler", "start");
           const serverStartLock = acquireServerStartLock(appDir, {