@pikku/core 0.12.71 → 0.12.74

package/src/wirings/queue/queue-identity.ts

@@ -0,0 +1,173 @@
+import {
+  decodeBase64UrlText,
+  encodeBase64UrlText,
+  signWithKeyMaterial,
+  verifyWithKeyMaterial,
+} from '../../crypto-utils.js'
+import type { Logger } from '../../services/logger.js'
+import type { SecretService } from '../../services/secret-service.js'
+
+export const QUEUE_IDENTITY_SECRET_NAME = 'PIKKU_QUEUE_IDENTITY_SECRET'
+
+/** Namespaces the queue signing key away from every other use of the same material. */
+export const QUEUE_IDENTITY_INFO = 'pikku:queue-identity'
+
+export const QUEUE_IDENTITY_CLAIM_VERSION = 'pq1'
+
+/**
+ * What the signature is bound to. `jobId` is only enforced when the producer
+ * knew it at signing time — most adapters mint their own id after `add`.
+ */
+export interface QueueIdentityBinding {
+  queueName: string
+  jobId?: string
+  data: unknown
+}
+
+interface QueueIdentityClaim {
+  u: string
+  q: string
+  j?: string
+}
+
+const stableStringify = (value: unknown): string => {
+  if (value === null || typeof value !== 'object') {
+    return JSON.stringify(value) ?? 'null'
+  }
+  if (Array.isArray(value)) {
+    return `[${value.map(stableStringify).join(',')}]`
+  }
+  const record = value as Record<string, unknown>
+  const keys = Object.keys(record).sort()
+  return `{${keys
+    .map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`)
+    .join(',')}}`
+}
+
+/**
+ * Round-trips through JSON first so the producer hashes exactly what the
+ * transport will deliver, then sorts keys so property order cannot change the
+ * digest.
+ */
+const canonicalizePayload = (data: unknown): string =>
+  stableStringify(JSON.parse(JSON.stringify(data ?? null)))
+
+const signingPayload = (encodedClaim: string, data: unknown): string =>
+  `${QUEUE_IDENTITY_CLAIM_VERSION}.${encodedClaim}.${canonicalizePayload(data)}`
+
+export const getQueueIdentitySecret = async (
+  secrets?: SecretService
+): Promise<string | undefined> => {
+  try {
+    return (await secrets?.getSecret(QUEUE_IDENTITY_SECRET_NAME)) || undefined
+  } catch {
+    return undefined
+  }
+}
+
+export const signQueueIdentity = async (
+  secret: string,
+  pikkuUserId: string,
+  binding: QueueIdentityBinding
+): Promise<string> => {
+  const claim: QueueIdentityClaim = {
+    u: pikkuUserId,
+    q: binding.queueName,
+    ...(binding.jobId ? { j: binding.jobId } : {}),
+  }
+  const encodedClaim = encodeBase64UrlText(JSON.stringify(claim))
+  const signature = await signWithKeyMaterial(
+    QUEUE_IDENTITY_SECRET_NAME,
+    secret,
+    QUEUE_IDENTITY_INFO,
+    signingPayload(encodedClaim, binding.data)
+  )
+  return `${QUEUE_IDENTITY_CLAIM_VERSION}.${encodedClaim}.${signature}`
+}
+
+/** The claimed user id, or undefined for anything that does not verify. */
+export const verifyQueueIdentity = async (
+  secret: string,
+  token: string,
+  binding: QueueIdentityBinding
+): Promise<string | undefined> => {
+  const parts = token.split('.')
+  if (parts.length !== 3) return undefined
+  const [version, encodedClaim, signature] = parts as [string, string, string]
+  if (version !== QUEUE_IDENTITY_CLAIM_VERSION) return undefined
+
+  const verified = await verifyWithKeyMaterial(
+    QUEUE_IDENTITY_SECRET_NAME,
+    secret,
+    QUEUE_IDENTITY_INFO,
+    signingPayload(encodedClaim, binding.data),
+    signature
+  )
+  if (!verified) return undefined
+
+  let claim: QueueIdentityClaim
+  try {
+    claim = JSON.parse(decodeBase64UrlText(encodedClaim)) as QueueIdentityClaim
+  } catch {
+    return undefined
+  }
+  if (typeof claim?.u !== 'string' || claim.u.length === 0) return undefined
+  if (claim.q !== binding.queueName) return undefined
+  if (claim.j !== undefined && claim.j !== binding.jobId) return undefined
+  return claim.u
+}
+
+let missingSecretLogged = false
+
+/** Test seam: the latch is process-wide so the warning is emitted once. */
+export const resetQueueIdentityLogLatch = () => {
+  missingSecretLogged = false
+}
+
+export const warnQueueIdentitySecretMissing = (logger: Logger) => {
+  if (missingSecretLogged) return
+  missingSecretLogged = true
+  logger.warn(
+    `No '${QUEUE_IDENTITY_SECRET_NAME}' secret is configured — queue job identities cannot be signed or verified, so pikkuUserId is dropped and queue workers run without one.`
+  )
+}
+
+export const resolveQueueJobIdentity = async ({
+  claimedIdentity,
+  binding,
+  secrets,
+  logger,
+}: {
+  claimedIdentity: string | undefined
+  binding: QueueIdentityBinding
+  secrets: SecretService | undefined
+  logger: Logger
+}): Promise<string | undefined> => {
+  if (!claimedIdentity) return undefined
+
+  const secret = await getQueueIdentitySecret(secrets)
+  if (!secret) {
+    warnQueueIdentitySecretMissing(logger)
+    return undefined
+  }
+
+  let pikkuUserId: string | undefined
+  try {
+    pikkuUserId = await verifyQueueIdentity(secret, claimedIdentity, binding)
+  } catch (error) {
+    logger.error(
+      `Could not verify the identity claim on job ${binding.jobId} in queue ${binding.queueName} — the job runs without a pikkuUserId.`,
+      error
+    )
+    return undefined
+  }
+
+  if (!pikkuUserId) {
+    logger.error(
+      `Rejected an unsigned or invalid identity claim on job ${binding.jobId} in queue ${binding.queueName} — the job runs without a pikkuUserId.`
+    )
+    return undefined
+  }
+
+  return pikkuUserId
+}

package/src/wirings/queue/queue-runner.ts

@@ -12,18 +12,14 @@ import {
   pikkuState,
 } from '../../pikku-state.js'
 import { addFunction, runPikkuFunc } from '../../function/function-runner.js'
-/**
- * Error class for queue processor not found
- */
+import { resolveQueueJobIdentity } from './queue-identity.js'
+
 class QueueWorkerNotFoundError extends PikkuError {
   constructor(name: string) {
     super(`Queue processor not found: ${name}`)
   }
 }
 
-/**
- * Error class for when a queue job is explicitly failed
- */
 export class QueueJobFailedError extends PikkuError {
   constructor(jobId: string, reason?: string) {
     super(`Queue job ${jobId} failed${reason ? `: ${reason}` : ''}`)
@@ -31,18 +27,13 @@ export class QueueJobFailedError extends PikkuError {
   }
 }
 
-/**
- * Error class for when a queue job is explicitly discarded
- */
 export class QueueJobDiscardedError extends PikkuError {
   constructor(jobId: string, reason?: string) {
     super(`Queue job ${jobId} discarded${reason ? `: ${reason}` : ''}`)
     this.name = 'QueueJobDiscardedError'
   }
 }
-/**
- * Add a queue processor to the system
- */
+
 export const wireQueueWorker = <
   InputData = any,
   OutputData = any,
@@ -54,7 +45,6 @@ export const wireQueueWorker = <
 >(
   queueWorker: CoreQueueWorker<PikkuFunctionConfig>
 ) => {
-  // Get processor metadata
   const meta = pikkuState(null, 'queue', 'meta')
   const processorMeta = meta[queueWorker.name]
   if (!processorMeta) {
@@ -64,7 +54,6 @@ export const wireQueueWorker = <
     return
   }
 
-  // Register the function with pikku
   addFunction(processorMeta.pikkuFuncId, {
     func: queueWorker.func.func,
     auth: queueWorker.func.auth,
@@ -73,21 +62,14 @@ export const wireQueueWorker = <
     tags: queueWorker.func.tags,
   })
 
-  // Store processor definition in state - runtime adapters will pick this up
   const registrations = pikkuState(null, 'queue', 'registrations')
   registrations.set(queueWorker.name, queueWorker)
 }
 
-/**
- * Get all registered queue processors
- */
 export function getQueueWorkers(): Map<string, CoreQueueWorker> {
   return pikkuState(null, 'queue', 'registrations')
 }
 
-/**
- * Stop and remove a queue processor
- */
 export async function removeQueueWorker(name: string): Promise<void> {
   const registrations = pikkuState(null, 'queue', 'registrations')
   const registration = registrations.get(name)
@@ -99,9 +81,6 @@ export async function removeQueueWorker(name: string): Promise<void> {
   registrations.delete(name)
 }
 
-/**
- * Process a single queue job - this function is called by queue consumers
- */
 export async function runQueueJob({
   job,
   updateProgress,
@@ -109,7 +88,6 @@ export async function runQueueJob({
 }: {
   job: QueueJob
   updateProgress?: (progress: number | string | object) => Promise<void>
-  /** Pre-resolved trace ID (e.g. from queue message metadata) */
   traceId?: string
 }): Promise<void> {
   const singletonServices = getSingletonServices()
@@ -127,23 +105,28 @@ export async function runQueueJob({
     )
   }
 
-  // Get the queue worker registration to access middleware
   const registrations = pikkuState(null, 'queue', 'registrations')
   const queueWorker = registrations.get(job.queueName)
   if (!queueWorker) {
     throw new Error(`Queue worker registration not found for: ${job.queueName}`)
   }
 
-  // Create the queue wire object
+  // knowledge: decisions/security/queue-job-identities-are-signed-at-enqueue.md
+  const pikkuUserId = await resolveQueueJobIdentity({
+    claimedIdentity: job.pikkuUserId,
+    binding: { queueName: job.queueName, jobId: job.id, data: job.data },
+    secrets: singletonServices.secrets,
+    logger,
+  })
+
   const queue: PikkuQueue = {
     queueName: job.queueName,
     jobId: job.id,
-    pikkuUserId: job.pikkuUserId,
+    pikkuUserId,
     updateProgress:
       updateProgress ||
       (async (progress: number | string | object) => {
         logger.info(`Job ${job.id} progress: ${progress}`)
-        // Default implementation - just log the progress
       }),
     fail: async (reason?: string) => {
       throw new QueueJobFailedError(job.id, reason)
@@ -161,7 +144,6 @@ export async function runQueueJob({
       queue,
     }
 
-    // Execute the pikku function with the job data
     const result = await runPikkuFunc(
       'queue',
       job.queueName,
@@ -194,7 +176,6 @@ export async function runQueueJob({
       logger.error('Processed error response:', errorResponse)
     }
 
-    // Re-throw the error so the queue service can handle retries/DLQ
     throw error
   }
 }

package/src/wirings/queue/queue.types.ts

@@ -2,94 +2,58 @@ import type { CommonWireMeta } from '../../types/core.types.js'
 import type { CorePikkuFunctionConfig } from '../../function/functions.types.js'
 import type { QueueConfigMapping } from './validate-worker-config.js'
 
-/**
- * Configuration for queue workers - how jobs are processed
- */
-/**
- * Configuration for queue workers - how jobs are processed
- */
 export interface PikkuWorkerConfig {
-  /** Optional worker name for identification and monitoring */
   name?: string
-  /** Number of messages to process in batch / in parallel */
+  /** Total worker concurrency. */
   batchSize?: number
-  /** Number of messages to prefetch for efficiency */
   prefetch?: number
-  /** Polling interval for pull-based queues in ms */
+  /** Milliseconds. */
   pollInterval?: number
-  /** Message visibility timeout in seconds */
+  /** Seconds. */
   visibilityTimeout?: number
-  /** Duration of job lock in milliseconds */
+  /** Milliseconds. */
   lockDuration?: number
-  /** Number of seconds to wait when queue is empty before polling again */
+  /** Seconds. */
   drainDelay?: number
-  /** Keep N completed jobs for inspection */
+  /** Number of completed jobs to retain, not an age. */
   removeOnComplete?: number
-  /** Keep N failed jobs for inspection */
+  /** Number of failed jobs to retain, not an age. */
   removeOnFail?: number
-  /** Maximum number of times a job can be recovered from stalled state */
   maxStalledCount?: number
-  /** Condition to start processor at instance creation */
   autorun?: boolean
-  /**
-   * Cap how many jobs of any one group ({@link JobOptions.group}) may run at
-   * once, so a single group can't occupy the whole worker. Lets one shared
-   * queue stay fair across producers instead of splitting it into one queue
-   * per producer — which multiplies polling cost on pull-based backends.
-   * Must not exceed {@link batchSize}.
-   */
+  /** Must not exceed {@link batchSize}. */
   groupConcurrency?: number | GroupConcurrencyConfig
 }
 
-/**
- * Per-group concurrency limits, optionally varied by tier so slow groups can
- * be allowed more (or fewer) slots than the default.
- */
 export interface GroupConcurrencyConfig {
-  /** Limit applied to any group without a matching tier */
+  /** Limit applied to any group without a matching tier. */
   default: number
-  /** Per-tier overrides, keyed by {@link JobGroup.tier} */
+  /** Keyed by {@link JobGroup.tier}. */
   tiers?: Record<string, number>
 }
 
-/**
- * Fairness key for a job. Jobs sharing an `id` count against the same
- * {@link PikkuWorkerConfig.groupConcurrency} limit.
- */
 export interface JobGroup {
-  /** Group this job belongs to (e.g. a workflow name) */
   id: string
-  /** Optional tier selecting a per-tier limit */
   tier?: string
 }
 
-/**
- * Configuration for individual jobs - how jobs behave
- */
 export interface PikkuJobConfig {
-  /** Maximum retry attempts for failed jobs */
   retryAttempts?: number
-  /** Initial retry delay in milliseconds */
+  /** Milliseconds. */
   retryDelay?: number
-  /** Retry backoff strategy */
   retryBackoff?: 'linear' | 'exponential' | 'fixed'
-  /** Queue for failed messages after max retries */
   deadLetterQueue?: string
-  /** How long to retain completed jobs in seconds */
+  /** Seconds. */
   messageRetention?: number
-  /** Job priority (higher numbers = higher priority) */
+  /** Higher numbers run first. */
   priority?: number
-  /** Enable FIFO ordering where supported */
   fifo?: boolean
-  /** Job timeout in milliseconds */
+  /** Milliseconds. */
   timeout?: number
-  /** Delay before job execution in milliseconds */
+  /** Milliseconds. */
   delay?: number
 }
 
-/**
- * Configuration validation result with warnings and fallbacks
- */
 export interface ConfigValidationResult {
   applied: Partial<PikkuWorkerConfig>
   ignored: Partial<PikkuWorkerConfig>
@@ -97,9 +61,6 @@ export interface ConfigValidationResult {
   fallbacks: { [key: string]: any }
 }
 
-/**
- * Queue job representation
- */
 export type QueueJobStatus =
   | 'waiting'
   | 'active'
@@ -126,13 +87,9 @@ export interface QueueJob<T = any, R = any> {
   result?: R
   waitForCompletion?: (ttl?: number) => Promise<R>
   metadata?: () => Promise<QueueJobMetadata> | QueueJobMetadata
-  /** Pikku user ID propagated from the job producer */
   pikkuUserId?: string
 }
 
-/**
- * Job options for queue operations
- */
 export interface JobOptions {
   priority?: number
   delay?: number
@@ -141,46 +98,30 @@ export interface JobOptions {
   removeOnComplete?: number
   removeOnFail?: number
   jobId?: string
-  /** Pikku user ID to propagate to the queue worker for credential resolution */
+  // knowledge: decisions/security/queue-jobs-carry-the-producers-pikku-user-id.md
   pikkuUserId?: string
-  /** Fairness key — counts against the worker's {@link PikkuWorkerConfig.groupConcurrency} */
+  /** Counts against the worker's {@link PikkuWorkerConfig.groupConcurrency}. */
   group?: JobGroup
 }
 
-/**
- * Queue provider interface for job publishing operations
- */
 export interface QueueService {
-  /** Whether this queue provider supports job results */
   readonly supportsResults: boolean
 
-  /** Add a job to the queue with type safety */
   add<T>(queueName: string, data: T, options?: JobOptions): Promise<string>
 
-  /** Get job status and result */
   getJob<T, R>(queueName: string, jobId: string): Promise<QueueJob<T, R> | null>
 }
 
-/**
- * Queue service interface that queue adapters implement
- */
 export interface QueueWorkers {
-  /** Service name identifier */
   name: string
 
-  /** Whether this queue service supports job results */
   supportsResults: boolean
 
-  /** Configuration mapping for validation */
   configMappings: QueueConfigMapping
 
-  /** Scan state and register all compatible processors */
   registerQueues(): Promise<Record<string, ConfigValidationResult[]>>
 }
 
-/**
- * Queue processor metadata
- */
 export type QueueWorkersMeta = Record<
   string,
   CommonWireMeta & {
@@ -189,9 +130,6 @@ export type QueueWorkersMeta = Record<
   }
 >
 
-/**
- * Core queue processor definition
- */
 export type CoreQueueWorker<
   PikkuFunctionConfig extends CorePikkuFunctionConfig<any, any, any> =
     CorePikkuFunctionConfig<any, any, any>,
@@ -204,21 +142,13 @@ export type CoreQueueWorker<
   middleware?: PikkuFunctionConfig['middleware']
 }
 
-/**
- * Represents a queue wire object for middleware
- * Provides information and actions for the current queue job execution
- */
 export interface PikkuQueue {
-  /** The name of the queue being processed */
   queueName: string
-  /** The current job ID */
   jobId: string
-  /** Pikku user ID propagated from the job producer for credential resolution */
   pikkuUserId?: string
-  /** Update job progress (0-100 or custom value) */
   updateProgress: (progress: number | string | object) => Promise<void>
-  /** Fail the current job with optional reason */
+  /** Never returns — throws QueueJobFailedError. */
   fail: (reason?: string) => Promise<void>
-  /** Discard/delete the job without retrying */
+  /** Never returns — throws QueueJobDiscardedError. */
   discard: (reason?: string) => Promise<void>
 }

package/src/wirings/queue/register-queue-helper.ts

@@ -4,23 +4,11 @@ import type { QueueConfigMapping } from './validate-worker-config.js'
 import { validateWorkerConfig } from './validate-worker-config.js'
 import type { Logger } from '../../services/logger.js'
 
-/**
- * Queue processor registration callback
- */
 export type QueueRegistrationCallback<T = any> = (
   queueName: string,
   processor: CoreQueueWorker
 ) => Promise<T>
 
-/**
- * Helper function to register queue processors with validation
- * This centralizes the common logic for looping over processors and validating configs
- *
- * @param configMappings - Configuration mapping for the queue implementation
- * @param registerCallback - Callback to register each individual queue processor
- * @param logger - Optional logger for info/error messages
- * @returns Record of validation results by queue name
- */
 export async function registerQueueWorkers<T = any>(
   configMappings: QueueConfigMapping,
   logger: Logger,
@@ -31,13 +19,11 @@ export async function registerQueueWorkers<T = any>(
 
   for (const [queueName, processor] of queueWorkers) {
     logger?.info(`Registering queue processor: ${queueName}`)
-    // Validate the processor configuration
     const validationResult = validateWorkerConfig(
       configMappings,
       processor.config
     )
 
-    // Store validation results
     configValidation[queueName] = configValidation[queueName] || []
     configValidation[queueName].push(validationResult)
 

package/src/wirings/queue/signed-queue-service.ts

@@ -0,0 +1,59 @@
+import type { Logger } from '../../services/logger.js'
+import type { SecretService } from '../../services/secret-service.js'
+import {
+  getQueueIdentitySecret,
+  signQueueIdentity,
+  warnQueueIdentitySecretMissing,
+} from './queue-identity.js'
+import type { JobOptions, QueueJob, QueueService } from './queue.types.js'
+
+/**
+ * Wraps any {@link QueueService} so `JobOptions.pikkuUserId` leaves the process
+ * as a signed claim that `runQueueJob` can verify.
+ */
+export class SignedQueueService implements QueueService {
+  constructor(
+    private readonly queueService: QueueService,
+    private readonly secrets: SecretService,
+    private readonly logger: Logger
+  ) {}
+
+  get supportsResults(): boolean {
+    return this.queueService.supportsResults
+  }
+
+  public async add<T>(
+    queueName: string,
+    data: T,
+    options?: JobOptions
+  ): Promise<string> {
+    const pikkuUserId = options?.pikkuUserId
+    if (!pikkuUserId) {
+      return this.queueService.add(queueName, data, options)
+    }
+
+    const secret = await getQueueIdentitySecret(this.secrets)
+    if (!secret) {
+      warnQueueIdentitySecretMissing(this.logger)
+      const { pikkuUserId: _unsigned, ...rest } = options as JobOptions
+      return this.queueService.add(queueName, data, rest)
+    }
+
+    const signed = await signQueueIdentity(secret, pikkuUserId, {
+      queueName,
+      jobId: undefined,
+      data,
+    })
+    return this.queueService.add(queueName, data, {
+      ...options,
+      pikkuUserId: signed,
+    })
+  }
+
+  public async getJob<T, R>(
+    queueName: string,
+    jobId: string
+  ): Promise<QueueJob<T, R> | null> {
+    return this.queueService.getJob<T, R>(queueName, jobId)
+  }
+}