@pikku/core 0.12.66 → 0.12.69

package/src/wirings/queue/queue.types.ts

@@ -31,6 +31,36 @@ export interface PikkuWorkerConfig {
   maxStalledCount?: number
   /** Condition to start processor at instance creation */
   autorun?: boolean
+  /**
+   * Cap how many jobs of any one group ({@link JobOptions.group}) may run at
+   * once, so a single group can't occupy the whole worker. Lets one shared
+   * queue stay fair across producers instead of splitting it into one queue
+   * per producer — which multiplies polling cost on pull-based backends.
+   * Must not exceed {@link batchSize}.
+   */
+  groupConcurrency?: number | GroupConcurrencyConfig
+}
+
+/**
+ * Per-group concurrency limits, optionally varied by tier so slow groups can
+ * be allowed more (or fewer) slots than the default.
+ */
+export interface GroupConcurrencyConfig {
+  /** Limit applied to any group without a matching tier */
+  default: number
+  /** Per-tier overrides, keyed by {@link JobGroup.tier} */
+  tiers?: Record<string, number>
+}
+
+/**
+ * Fairness key for a job. Jobs sharing an `id` count against the same
+ * {@link PikkuWorkerConfig.groupConcurrency} limit.
+ */
+export interface JobGroup {
+  /** Group this job belongs to (e.g. a workflow name) */
+  id: string
+  /** Optional tier selecting a per-tier limit */
+  tier?: string
 }
 
 /**
@@ -113,6 +143,8 @@ export interface JobOptions {
   jobId?: string
   /** Pikku user ID to propagate to the queue worker for credential resolution */
   pikkuUserId?: string
+  /** Fairness key — counts against the worker's {@link PikkuWorkerConfig.groupConcurrency} */
+  group?: JobGroup
 }
 
 /**

package/src/wirings/scope/scope.test.ts

@@ -62,6 +62,31 @@ describe('flattenScopeDefinitions', () => {
       ['admin', 'admin:users', 'billing']
     )
   })
+
+  // An addon and its host app may both contribute the same root. Codegen writes
+  // these ids into an object literal, where a repeat is a TypeScript error.
+  test('a root declared twice is flattened once', () => {
+    const tree = { users: { scopes: { ban: {} } } }
+    const definitions: ScopeDefinitions = [
+      { name: 'admin', description: 'Administration', scopes: tree },
+      { name: 'admin', description: 'Administration', scopes: tree },
+    ]
+
+    assert.deepEqual(
+      flattenScopeDefinitions(definitions).map((s) => s.id),
+      ['admin', 'admin:users', 'admin:users:ban']
+    )
+  })
+
+  test('the first description of a repeated scope wins', () => {
+    assert.deepEqual(
+      flattenScopeDefinitions([
+        { name: 'admin', description: 'Administration' },
+        { name: 'admin', description: 'Something else' },
+      ]),
+      [{ id: 'admin', description: 'Administration' }]
+    )
+  })
 })
 
 describe('validateAndBuildScopeDefinitionsMeta', () => {

package/src/wirings/scope/validate-scope-definitions.ts

@@ -52,6 +52,14 @@ const flattenNodes = (
  * Flattens declared scope trees into the full list of grantable scope ids,
  * depth-first. Every node is emitted, including intermediate ones.
  *
+ * Ids are unique. A root may legitimately be declared more than once — an addon
+ * and its host app both contributing the same `admin` tree, say — and
+ * {@link validateAndBuildScopeDefinitionsMeta} already guarantees those
+ * declarations are identical, so the second one is redundant rather than
+ * conflicting. Collapsing it here keeps every consumer honest: codegen emits an
+ * object literal keyed by id (duplicates are a TypeScript error), and a
+ * ScopeService syncs one row per scope instead of re-writing the same one.
+ *
  * Used by codegen to build the `ScopeId` union, and by a ScopeService to sync
  * the declared set into its store.
  */
@@ -63,7 +71,14 @@ export const flattenScopeDefinitions = (
     out.push({ id: def.name, description: def.description })
     flattenNodes(def.scopes, def.name, out)
   }
-  return out
+  const seen = new Set<string>()
+  return out.filter((scope) => {
+    if (seen.has(scope.id)) {
+      return false
+    }
+    seen.add(scope.id)
+    return true
+  })
 }
 
 /**

package/src/wirings/workflow/index.ts

@@ -59,6 +59,7 @@ export type {
 // Re-export all types from workflow.types
 export type {
   WorkflowService,
+  WorkflowQueueOptions,
   WorkflowServiceConfig,
   WorkflowPlannedStep,
   WorkflowRunWire,

package/src/wirings/workflow/pikku-workflow-service.test.ts

@@ -29,6 +29,77 @@ describe('pikku-workflow-service worker registration', () => {
   })
 })
 
+describe('pikku-workflow-service per-workflow queue warning', () => {
+  // Losing the per-workflow orchestrator queues is silent — dispatch just falls
+  // back to the single shared queue and one slow step starves every workflow
+  // behind it. These assert the misconfiguration is announced at wiring time.
+  const setup = (workflowNames: string[], perWorkflowQueues: string[] = []) => {
+    resetPikkuState()
+    const warnings: string[] = []
+    pikkuState(null, 'package', 'singletonServices', {
+      logger: {
+        error: () => {},
+        info: () => {},
+        debug: () => {},
+        warn: (message: string) => warnings.push(message),
+      },
+    } as any)
+
+    const metaState = pikkuState(null, 'workflows', 'meta')
+    for (const name of workflowNames) {
+      metaState[name] = {
+        name,
+        pikkuFuncId: name,
+        source: 'dsl',
+        graphHash: `${name}-hash`,
+      } as any
+    }
+
+    const queueMeta = pikkuState(null, 'queue', 'meta')
+    for (const queueName of perWorkflowQueues) {
+      queueMeta[queueName] = {
+        pikkuFuncId: queueName,
+        name: queueName,
+      } as any
+    }
+
+    return warnings
+  }
+
+  test('warns when workflows exist but no per-workflow orchestrator queue is registered', () => {
+    const warnings = setup(['someWorkflow'])
+
+    new InMemoryWorkflowService().wireQueueWorkers()
+
+    assert.ok(
+      warnings.some((w) => /no per-workflow orchestrator queues/.test(w)),
+      `expected a per-workflow queue warning, got: ${JSON.stringify(warnings)}`
+    )
+  })
+
+  test('does not warn when per-workflow orchestrator queues are present', () => {
+    const warnings = setup(['someWorkflow'], ['wf-orchestrator-some-workflow'])
+
+    new InMemoryWorkflowService().wireQueueWorkers()
+
+    assert.ok(
+      !warnings.some((w) => /no per-workflow orchestrator queues/.test(w)),
+      `expected no warning, got: ${JSON.stringify(warnings)}`
+    )
+  })
+
+  test('does not warn when the app registers no workflows at all', () => {
+    const warnings = setup([])
+
+    new InMemoryWorkflowService().wireQueueWorkers()
+
+    assert.ok(
+      !warnings.some((w) => /no per-workflow orchestrator queues/.test(w)),
+      `expected no warning, got: ${JSON.stringify(warnings)}`
+    )
+  })
+})
+
 describe('pikku-workflow-service run-level inline', () => {
   test('workflow runs inline (and queues nothing) when no queue service is configured', async () => {
     const ws = new InMemoryWorkflowService()

package/src/wirings/workflow/pikku-workflow-service.ts

@@ -60,6 +60,7 @@ import type {
   WorkflowRunWire,
   WorkflowStatus,
   WorkflowVersionStatus,
+  WorkflowQueueOptions,
   WorkflowServiceConfig,
   WorkflowStepOptions,
   WorkflowExpectEventuallyOptions,
@@ -88,7 +89,12 @@ import {
   type RunTimeline,
   type ReconstructedRunState,
 } from './run-timeline.js'
-import type { JobOptions } from '../queue/queue.types.js'
+import type {
+  GroupConcurrencyConfig,
+  JobGroup,
+  JobOptions,
+  PikkuWorkerConfig,
+} from '../queue/queue.types.js'
 
 /**
  * Default number of retries for a workflow step when none is specified. The
@@ -257,11 +263,21 @@ export abstract class PikkuWorkflowService implements WorkflowService {
 
   protected mirror?: WorkflowRunMirror
 
+  protected readonly queueStrategy: 'per-workflow' | 'shared-groups'
+  protected readonly queueConcurrency: number
+  protected readonly queueGroupConcurrency: number | GroupConcurrencyConfig
+
   constructor(
-    options: { wireQueues?: boolean; mirror?: WorkflowRunMirror } = {}
+    options: {
+      wireQueues?: boolean
+      mirror?: WorkflowRunMirror
+    } & WorkflowQueueOptions = {}
   ) {
     const wireQueues = options.wireQueues ?? true
     this.mirror = options.mirror
+    this.queueStrategy = options.queueStrategy ?? 'per-workflow'
+    this.queueConcurrency = options.queueConcurrency ?? 20
+    this.queueGroupConcurrency = options.queueGroupConcurrency ?? 2
     if (wireQueues) {
       this.wireQueueWorkers()
     }
@@ -306,29 +322,44 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     const registerWorkflowFunc = (
       funcId: string,
       func: { func: unknown },
-      queueName: string
+      queueName: string,
+      config?: PikkuWorkerConfig
     ) => {
       if (functions.has(funcId)) return
       addFunction(funcId, func as never)
       if (!queueMeta[queueName]) {
         queueMeta[queueName] = { pikkuFuncId: funcId, name: queueName }
       }
-      wireQueueWorker({ name: queueName, func } as never)
+      wireQueueWorker({ name: queueName, func, config } as never)
       if (!functionsMeta[funcId]) {
         functionsMeta[funcId] = mkMeta(funcId)
       }
     }
 
+    // Under 'shared-groups' every workflow runs through these two queues and is
+    // kept from hogging them by the per-group cap, so the per-workflow queues
+    // below are left unconsumed — one set of pollers for the whole system
+    // instead of one per workflow.
+    const sharedGroups = this.queueStrategy === 'shared-groups'
+    const sharedQueueConfig: PikkuWorkerConfig | undefined = sharedGroups
+      ? {
+          batchSize: this.queueConcurrency,
+          groupConcurrency: this.queueGroupConcurrency,
+        }
+      : undefined
+
     // Register shared queue workers for monolith deployments
     registerWorkflowFunc(
       'pikkuWorkflowOrchestrator',
       { func: pikkuWorkflowOrchestratorFunc },
-      'pikku-workflow-orchestrator'
+      'pikku-workflow-orchestrator',
+      sharedQueueConfig
     )
     registerWorkflowFunc(
       'pikkuWorkflowStepWorker',
       { func: pikkuWorkflowWorkerFunc },
-      'pikku-workflow-step-worker'
+      'pikku-workflow-step-worker',
+      sharedQueueConfig
     )
 
     // Register per-workflow queue workers (root + addon packages)
@@ -351,18 +382,41 @@ export abstract class PikkuWorkflowService implements WorkflowService {
       }
     }
 
-    registerQueueWorkers(pikkuState(null, 'queue', 'meta'))
+    if (!sharedGroups) {
+      registerQueueWorkers(pikkuState(null, 'queue', 'meta'))
 
-    const addons = pikkuState(null, 'addons', 'packages')
-    if (addons) {
-      for (const [, addon] of addons) {
-        const addonQueueMeta = pikkuState(addon.package, 'queue', 'meta')
-        if (addonQueueMeta) {
-          registerQueueWorkers(addonQueueMeta)
+      const addons = pikkuState(null, 'addons', 'packages')
+      if (addons) {
+        for (const [, addon] of addons) {
+          const addonQueueMeta = pikkuState(addon.package, 'queue', 'meta')
+          if (addonQueueMeta) {
+            registerQueueWorkers(addonQueueMeta)
+          }
         }
       }
     }
 
+    // Workflows exist but no per-workflow orchestrator queue was registered:
+    // the generated queue meta never reached the runtime (most often the
+    // bootstrap doesn't import the queue-workers meta, so `queue.meta` is
+    // empty). Everything still "works" — dispatch silently falls back to the
+    // single shared orchestrator queue — but the isolation is gone: one slow
+    // workflow step head-of-line-blocks every other workflow behind it. That
+    // is invisible until a queue starves, so say so loudly at wiring time.
+    const workflowCount = Object.keys(
+      pikkuState(null, 'workflows', 'meta') ?? {}
+    ).length
+    const perWorkflowQueues = Object.keys(queueMeta).filter((name) =>
+      name.startsWith('wf-orchestrator-')
+    ).length
+    if (workflowCount > 0 && perWorkflowQueues === 0) {
+      this.logger?.warn?.(
+        `[pikku] ${workflowCount} workflow(s) registered but no per-workflow orchestrator queues were found in queue meta. ` +
+          `All workflows will share a single orchestrator queue, where one slow step blocks every other workflow behind it. ` +
+          `Check that the generated bootstrap imports the queue-workers meta (pikku-queue-workers-wirings-meta.gen.js).`
+      )
+    }
+
     if (!functions.has('pikkuWorkflowSleeper')) {
       addFunction('pikkuWorkflowSleeper', {
         func: pikkuWorkflowSleeperFunc,
@@ -927,7 +981,10 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     await queueService.add(
       this.getOrchestratorQueueName(workflowName),
       { runId },
-      this.resolveStepJobOptions()
+      {
+        ...this.resolveStepJobOptions(),
+        group: this.getJobGroup(workflowName),
+      }
     )
   }
 
@@ -970,7 +1027,12 @@ export abstract class PikkuWorkflowService implements WorkflowService {
       JSON.parse(
         JSON.stringify({ runId, stepName, rpcName, data, fromStepName })
       ),
-      this.resolveStepJobOptions(stepOptions)
+      {
+        ...this.resolveStepJobOptions(stepOptions),
+        // Group by step function, mirroring how per-step queues split them —
+        // one slow step function can't monopolise the shared step worker.
+        group: this.getJobGroup(rpcName),
+      }
     )
   }
 
@@ -1005,7 +1067,12 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     await queueService.add(
       this.getOrchestratorQueueName(workflowName),
       { runId },
-      retryDelay ? { delay: getDurationInMilliseconds(retryDelay) } : undefined
+      {
+        ...(retryDelay
+          ? { delay: getDurationInMilliseconds(retryDelay) }
+          : undefined),
+        group: this.getJobGroup(workflowName),
+      }
     )
   }
 
@@ -1054,7 +1121,10 @@ export abstract class PikkuWorkflowService implements WorkflowService {
         JSON.parse(
           JSON.stringify({ runId, stepName, rpcName, data, fromStepName })
         ),
-        this.resolveStepJobOptions(stepOptions)
+        {
+          ...this.resolveStepJobOptions(stepOptions),
+          group: this.getJobGroup(rpcName),
+        }
       )
     } catch (cause) {
       // The queue is down/unreachable — NOT a step failure. Surface it as a
@@ -2225,7 +2295,11 @@ export abstract class PikkuWorkflowService implements WorkflowService {
       await queueService.add(
         this.getOrchestratorQueueName(run.workflow),
         { runId },
-        { ...this.resolveStepJobOptions(), delay }
+        {
+          ...this.resolveStepJobOptions(),
+          delay,
+          group: this.getJobGroup(run.workflow),
+        }
       )
     } catch (error) {
       this.logger?.warn(
@@ -2655,7 +2729,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
    * queues — but it produces to them — so registrations would miss them.
    */
   protected getOrchestratorQueueName(workflowName?: string): string {
-    if (workflowName) {
+    if (workflowName && this.queueStrategy !== 'shared-groups') {
       const perWorkflow = `wf-orchestrator-${toKebab(workflowName)}`
       const meta = pikkuState(null, 'queue', 'meta')
       if (meta[perWorkflow]) {
@@ -2666,7 +2740,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
   }
 
   protected getStepWorkerQueueName(rpcName?: string): string {
-    if (rpcName) {
+    if (rpcName && this.queueStrategy !== 'shared-groups') {
       const perStep = `wf-step-${toKebab(rpcName)}`
       const meta = pikkuState(null, 'queue', 'meta')
       if (meta[perStep]) {
@@ -2675,4 +2749,20 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     }
     return this.getConfig().stepWorkerQueueName
   }
+
+  /**
+   * Fairness key for a job on a shared queue. Under `'per-workflow'` the queue
+   * name already isolates workflows, so no group is needed — returning one
+   * anyway would cap a workflow inside its own dedicated queue.
+   *
+   * The tier repeats the id so a workflow can be given its own limit purely
+   * from config, with no per-workflow wiring; an unmatched tier falls back to
+   * the default limit.
+   */
+  protected getJobGroup(id?: string): JobGroup | undefined {
+    if (!id || this.queueStrategy !== 'shared-groups') {
+      return undefined
+    }
+    return { id, tier: id }
+  }
 }

package/src/wirings/workflow/workflow.types.ts

@@ -1,5 +1,6 @@
 import type { SerializedError, CommonWireMeta } from '../../types/core.types.js'
 import type { CorePikkuFunctionConfig } from '../../function/functions.types.js'
+import type { GroupConcurrencyConfig } from '../queue/queue.types.js'
 
 // Re-export WorkflowService from services module
 export type { WorkflowService } from '../../services/workflow-service.js'
@@ -62,6 +63,44 @@ export interface WorkflowServiceConfig {
   sleeperRPCName: string
 }
 
+/**
+ * How a workflow service spreads its jobs across queues.
+ *
+ * Passed to the service constructor rather than read from `config.workflow`:
+ * the queues are wired during construction, before singleton services (and so
+ * before `config`) exist.
+ */
+export interface WorkflowQueueOptions {
+  /**
+   * - `'per-workflow'` (default) — each workflow gets its own
+   *   `wf-orchestrator-*` / `wf-step-*` queue. Complete isolation, and it's
+   *   what lets serverless providers deploy one unit per workflow. Costs one
+   *   set of pollers per queue, which adds up on pull-based backends.
+   * - `'shared-groups'` — every workflow shares the orchestrator/step-worker
+   *   queues and stays isolated via {@link queueGroupConcurrency}, so no
+   *   workflow can occupy more than its share. One set of pollers total.
+   *   Only for single-process (monolith) runtimes; a per-unit serverless
+   *   deploy needs the per-workflow queues to route to its units.
+   */
+  queueStrategy?: 'per-workflow' | 'shared-groups'
+  /**
+   * Total concurrent workflow jobs per node under `'shared-groups'`.
+   * Defaults to 20.
+   */
+  queueConcurrency?: number
+  /**
+   * How many jobs of one workflow may run at once under `'shared-groups'`.
+   * Defaults to 2. Tiers are keyed by workflow name, so a specific workflow
+   * can be given its own limit without any extra wiring.
+   *
+   * Note tiers can only lower a workflow's limit, not raise it above
+   * `default`: the backend's pre-fetch exclusion is applied per group using
+   * `default` alone. To give one workflow more room, raise `default` and
+   * restrict the others by tier.
+   */
+  queueGroupConcurrency?: number | GroupConcurrencyConfig
+}
+
 export interface WorkflowPlannedStep {
   /** Durable step key — matches the runtime step name stored in the DB */
   stepName: string