nmtjs 0.15.0-beta.2 → 0.15.0-beta.21

package/src/runtime/jobs/runner.ts

@@ -0,0 +1,320 @@
+import type { Container, Logger, LoggingOptions } from '@nmtjs/core'
+import type { Job } from 'bullmq'
+import { anyAbortSignal } from '@nmtjs/common'
+import { Scope } from '@nmtjs/core'
+import { UnrecoverableError } from 'bullmq'
+
+import type { LifecycleHooks } from '../core/hooks.ts'
+import type { AnyJob } from './job.ts'
+import type { AnyJobStep } from './step.ts'
+import { LifecycleHook } from '../enums.ts'
+import { jobAbortSignal } from '../injectables.ts'
+
+export type JobRunnerOptions = { logging?: LoggingOptions }
+
+export interface StepResultEntry {
+  data: Record<string, unknown> | null
+  duration: number
+}
+
+export interface JobRunnerRunOptions {
+  signal: AbortSignal
+  result: Record<string, unknown>
+  stepResults: StepResultEntry[]
+  currentStepIndex: number
+  progress: Record<string, unknown>
+}
+
+export interface JobRunnerRunBeforeStepParams<
+  Options extends JobRunnerRunOptions,
+> {
+  job: AnyJob
+  step: AnyJobStep
+  stepIndex: number
+  result: Record<string, unknown>
+  stepResults: StepResultEntry[]
+  options: Options
+}
+
+export interface JobRunnerRunAfterStepParams<
+  Options extends JobRunnerRunOptions,
+> extends JobRunnerRunBeforeStepParams<Options> {
+  stepResult: StepResultEntry
+}
+
+export class JobRunner<
+  RunOptions extends JobRunnerRunOptions = JobRunnerRunOptions,
+> {
+  logger: Logger
+
+  constructor(
+    protected runtime: {
+      logger: Logger
+      container: Container
+      lifecycleHooks: LifecycleHooks
+    },
+  ) {
+    this.logger = runtime.logger.child({ $group: JobRunner.name })
+  }
+
+  get container() {
+    return this.runtime.container
+  }
+
+  async runJob<T extends AnyJob>(
+    job: T,
+    data: any,
+    options: Partial<RunOptions> = {},
+  ): Promise<T['_']['output']> {
+    const {
+      signal: runSignal,
+      result: runResult = {},
+      stepResults: runStepResults = [] as RunOptions['stepResults'],
+      progress: runProgress = {},
+      currentStepIndex = 0,
+      ...rest
+    } = options
+
+    const { input, output, steps } = job
+
+    const result: Record<string, unknown> = { ...runResult }
+    const decodedInput = input.decode(data)
+
+    // Initialize progress: decode from checkpoint or start fresh
+    const progress: Record<string, unknown> = job.progress
+      ? job.progress.decode(runProgress)
+      : { ...runProgress }
+
+    using stopListener = this.runtime.lifecycleHooks.once(
+      LifecycleHook.BeforeDispose,
+    )
+    const signal = anyAbortSignal(runSignal, stopListener.signal)
+    await using container = this.container.fork(Scope.Global)
+    await container.provide(jobAbortSignal, signal)
+
+    const jobDependencyContext = await container.createContext(job.dependencies)
+    const jobData = job.options.data
+      ? await job.options.data(jobDependencyContext, decodedInput, progress)
+      : undefined
+
+    const stepResults: StepResultEntry[] = Array.from({ length: steps.length })
+
+    // Restore previous step results and reconstruct accumulated result
+    for (let stepIndex = 0; stepIndex < runStepResults.length; stepIndex++) {
+      const entry = runStepResults[stepIndex]
+      stepResults[stepIndex] = entry
+      if (entry?.data) Object.assign(result, entry.data)
+    }
+
+    // @ts-expect-error
+    const runOptions: RunOptions = {
+      signal,
+      result,
+      stepResults,
+      currentStepIndex: currentStepIndex,
+      progress,
+      ...rest,
+    } satisfies JobRunnerRunOptions
+
+    for (
+      let stepIndex = currentStepIndex;
+      stepIndex < steps.length;
+      stepIndex++
+    ) {
+      const step = steps[stepIndex]
+      const resultSnapshot = Object.freeze(
+        Object.assign({}, decodedInput, result),
+      )
+
+      try {
+        if (signal.aborted) {
+          const { reason } = signal
+          if (reason instanceof UnrecoverableError) throw reason
+          throw new UnrecoverableError('Job cancelled')
+        }
+
+        const condition = job.conditions.get(stepIndex)
+        if (condition) {
+          const shouldRun = await condition({
+            context: jobDependencyContext,
+            data: jobData,
+            input: decodedInput,
+            result: resultSnapshot,
+            progress,
+          })
+          if (!shouldRun) {
+            stepResults[stepIndex] = { data: null, duration: 0 }
+            continue
+          }
+        }
+
+        const stepStartTime = Date.now()
+
+        await this.beforeStep({
+          job,
+          step,
+          stepIndex,
+          result,
+          options: runOptions,
+          stepResults,
+        })
+
+        const stepContext = await container.createContext(step.dependencies)
+        const stepInput = step.input.decode(resultSnapshot)
+
+        await job.beforeEachHandler?.({
+          context: jobDependencyContext,
+          data: jobData,
+          input: decodedInput,
+          result: resultSnapshot,
+          progress,
+          step,
+          stepIndex,
+        })
+
+        const handlerReturn = await step.handler(
+          stepContext,
+          stepInput,
+          jobData,
+        )
+
+        const produced = step.output.encode(handlerReturn ?? {})
+        const duration = Date.now() - stepStartTime
+
+        stepResults[stepIndex] = { data: produced, duration }
+        Object.assign(result, produced)
+
+        await job.afterEachHandler?.({
+          context: jobDependencyContext,
+          data: jobData,
+          input: decodedInput,
+          result,
+          progress,
+          step,
+          stepIndex,
+        })
+
+        await this.afterStep({
+          job,
+          step,
+          stepIndex,
+          result,
+          stepResult: stepResults[stepIndex],
+          stepResults,
+          options: runOptions,
+        })
+      } catch (error) {
+        const wrapped = new Error(`Error during step [${stepIndex}]`, {
+          cause: error,
+        })
+        this.logger.error(wrapped)
+
+        const allowRetry = await job.onErrorHandler?.({
+          context: jobDependencyContext,
+          data: jobData,
+          input: decodedInput,
+          result: result,
+          progress,
+          step,
+          stepIndex,
+          error,
+        })
+
+        if (allowRetry === false) {
+          throw new UnrecoverableError('Job failed (unrecoverable)')
+        }
+
+        throw wrapped
+      }
+    }
+
+    const finalPayload = await job.returnHandler!({
+      context: jobDependencyContext,
+      data: jobData,
+      input: decodedInput,
+      result,
+      progress,
+    })
+
+    return output.encode(finalPayload)
+  }
+
+  protected async beforeStep(
+    params: JobRunnerRunBeforeStepParams<RunOptions>,
+  ): Promise<void> {
+    this.logger.debug(
+      {
+        job: params.job.name,
+        step: params.step.label || params.stepIndex + 1,
+        stepIndex: params.stepIndex,
+      },
+      'Executing job step',
+    )
+  }
+
+  protected async afterStep(
+    params: JobRunnerRunAfterStepParams<RunOptions>,
+  ): Promise<void> {
+    this.logger.debug(
+      {
+        job: params.job.name,
+        step: params.step.label || params.stepIndex + 1,
+        stepIndex: params.stepIndex,
+      },
+      'Completed job step',
+    )
+  }
+}
+
+export class ApplicationWorkerJobRunner extends JobRunner<
+  JobRunnerRunOptions & { queueJob: Job }
+> {
+  constructor(
+    protected runtime: {
+      logger: Logger
+      container: Container
+      lifecycleHooks: LifecycleHooks
+    },
+  ) {
+    super(runtime)
+  }
+
+  protected async afterStep(
+    params: JobRunnerRunAfterStepParams<
+      JobRunnerRunOptions & { queueJob: Job }
+    >,
+  ): Promise<void> {
+    await super.afterStep(params)
+    const {
+      job,
+      step,
+      result,
+      stepResult,
+      stepResults,
+      stepIndex,
+      options: { queueJob, progress },
+    } = params
+    const nextStepIndex = stepIndex + 1
+    const totalSteps = job.steps.length
+    const percentage = Math.round((nextStepIndex / totalSteps) * 100)
+
+    // Encode progress before persisting if schema is defined
+    const encodedProgress = job.progress
+      ? job.progress.encode(progress)
+      : progress
+
+    await Promise.all([
+      queueJob.log(
+        `Step ${step.label || nextStepIndex} completed in ${(stepResult.duration / 1000).toFixed(3)}s`,
+      ),
+      queueJob.updateProgress({
+        stepIndex: nextStepIndex,
+        stepLabel: step.label,
+        result,
+        stepResults,
+        progress: encodedProgress,
+        percentage,
+      }),
+    ])
+  }
+}

package/src/runtime/jobs/step.ts

@@ -0,0 +1,66 @@
+import type { MaybePromise } from '@nmtjs/common'
+import type { Dependant, Dependencies, DependencyContext } from '@nmtjs/core'
+import type { AnyObjectLikeType, ObjectType } from '@nmtjs/type/object'
+import { tryCaptureStackTrace } from '@nmtjs/common'
+import { t } from '@nmtjs/type'
+
+import { kJobStepKey } from '../constants.ts'
+
+export type AnyJobStep = JobStep<any, any, any, any, any>
+
+export type JobStepHandler<
+  Deps extends Dependencies,
+  Input extends AnyObjectLikeType,
+  Output extends AnyObjectLikeType,
+  Return,
+  Data = any,
+> = (
+  context: DependencyContext<Deps>,
+  input: t.infer.decode.output<Input>,
+  data: Data,
+) => MaybePromise<null extends Return ? t.infer.encode.input<Output> : Return>
+
+export interface JobStep<
+  Input extends AnyObjectLikeType = AnyObjectLikeType,
+  Output extends AnyObjectLikeType = AnyObjectLikeType,
+  Deps extends Dependencies = Dependencies,
+  Return = unknown,
+  Data = any,
+> extends Dependant {
+  [kJobStepKey]: any
+  label?: string
+  input: Input
+  output: Output
+  dependencies: Deps
+  handler: JobStepHandler<Deps, Input, Output, Return, Data>
+}
+
+export function createStep<
+  Input extends AnyObjectLikeType,
+  Output extends AnyObjectLikeType = ObjectType<{}>,
+  Deps extends Dependencies = {},
+  Return = unknown,
+  Data = any,
+>(step: {
+  label?: string
+  input: Input
+  output?: Output
+  dependencies?: Deps
+  handler: JobStepHandler<Deps, Input, Output, Return, Data>
+}): JobStep<Input, Output, Deps, Return, Data> {
+  return Object.freeze({
+    [kJobStepKey]: true,
+    output: t.object({}) as unknown as Output,
+    dependencies: {} as Deps,
+    stack: tryCaptureStackTrace(),
+    ...step,
+  })
+}
+
+export function isJobStep(value: unknown): value is AnyJobStep {
+  return (
+    typeof value === 'object' &&
+    value !== null &&
+    (value as AnyJobStep)[kJobStepKey] === true
+  )
+}

package/src/runtime/jobs/ui.ts

@@ -0,0 +1,21 @@
+import { createServer } from 'node:http'
+
+import type { Queue } from 'bullmq'
+import { createBullBoard } from '@bull-board/api'
+import { BullMQAdapter } from '@bull-board/api/bullMQAdapter'
+import { H3Adapter } from '@bull-board/h3'
+import { createApp, toNodeListener } from 'h3'
+
+export function createJobsUI(queues: Queue[]) {
+  const app = createApp()
+  const serverAdapter = new H3Adapter()
+  createBullBoard({
+    queues: queues.map((q) => new BullMQAdapter(q, { readOnlyMode: true })),
+    serverAdapter,
+  })
+  const router = serverAdapter.registerHandlers()
+  app.use(router)
+  return createServer(toNodeListener(app))
+}
+
+export type JobsUI = ReturnType<typeof createJobsUI>

package/src/runtime/pubsub/manager.ts

@@ -0,0 +1,211 @@
+import assert from 'node:assert'
+import { createHash } from 'node:crypto'
+import { PassThrough, Readable } from 'node:stream'
+
+import type {
+  SubcriptionOptions,
+  TAnyEventContract,
+  TAnySubscriptionContract,
+} from '@nmtjs/contract'
+import type { Container, Logger } from '@nmtjs/core'
+import type { t } from '@nmtjs/type'
+import { isAbortError } from '@nmtjs/common'
+
+import { pubSubAdapter } from '../injectables.ts'
+
+export type PubSubAdapterEvent = { channel: string; payload: any }
+
+export interface PubSubAdapterType {
+  publish(channel: string, payload: any): Promise<boolean>
+  subscribe(
+    channel: string,
+    signal?: AbortSignal,
+  ): AsyncGenerator<PubSubAdapterEvent>
+  initialize(): Promise<void>
+  dispose(): Promise<void>
+}
+
+export type PubSubChannel = {
+  stream: Readable
+  subscription: TAnySubscriptionContract
+  event: TAnyEventContract
+}
+
+export type PubSubSubscribe = <
+  Contract extends TAnySubscriptionContract,
+  Events extends {
+    [K in keyof Contract['events']]?: true
+  },
+>(
+  subscription: Contract,
+  events: Events,
+  options: Contract['options'],
+  signal?: AbortSignal,
+) => Omit<Readable, typeof Symbol.asyncIterator> & {
+  [Symbol.asyncIterator]: () => AsyncIterator<
+    {} extends Events
+      ? {
+          [K in keyof Contract['events']]: {
+            event: K
+            data: t.infer.decode.output<Contract['events'][K]['payload']>
+          }
+        }[keyof Contract['events']]
+      : {
+          [K in keyof Events]: K extends keyof Contract['events']
+            ? {
+                event: K
+                data: t.infer.decode.output<Contract['events'][K]['payload']>
+              }
+            : never
+        }[keyof Events]
+  >
+}
+
+export type PubSubPublish = <
+  S extends TAnySubscriptionContract,
+  E extends S['events'][keyof S['events']],
+>(
+  event: E,
+  options: S['options'],
+  data: t.infer.decode.input<E['payload']>,
+) => Promise<boolean>
+
+export type PubSubManagerOptions = { logger: Logger; container: Container }
+
+export class PubSubManager {
+  readonly subscriptions = new Map<string, PubSubChannel>()
+
+  constructor(protected readonly options: PubSubManagerOptions) {}
+
+  protected get adapter() {
+    return this.options.container.get(pubSubAdapter)
+  }
+
+  subscribe: PubSubSubscribe = (subscription, events, options, signal) => {
+    assert(this.adapter, 'PubSub adapter is not configured')
+
+    const eventKeys =
+      Object.keys(events).length === 0
+        ? Object.keys(subscription.events)
+        : Object.keys(events)
+
+    const streams = Array(eventKeys.length)
+
+    for (const index in eventKeys) {
+      const event = subscription.events[eventKeys[index]]
+      const channel = getChannelName(event, options)
+      if (this.subscriptions.has(channel)) {
+        streams[index] = this.subscriptions.get(channel)!.stream
+      } else {
+        const iterable = this.adapter.subscribe(channel, signal)
+        const stream = this.createEventStream(iterable)
+        stream.on('close', () => this.subscriptions.delete(channel))
+        streams[index] = stream
+        this.subscriptions.set(channel, { subscription, event, stream })
+      }
+    }
+
+    return mergeEventStreams(streams, signal)
+  }
+
+  publish: PubSubPublish = async (event, options, data) => {
+    assert(this.adapter, 'PubSub adapter is not configured')
+
+    const channel = getChannelName(event, options)
+
+    try {
+      const payload = event.payload.encode(data)
+      return await this.adapter.publish(channel, payload)
+    } catch (error: any) {
+      this.options.logger.error(
+        `Failed to publish event "${event.name}" on channel "${channel}": ${error.message}`,
+      )
+      return Promise.reject(error)
+    }
+  }
+
+  private createEventStream(
+    iterable: AsyncGenerator<PubSubAdapterEvent>,
+  ): Readable {
+    const { subscriptions } = this
+    return new Readable({
+      objectMode: true,
+      read() {
+        iterable.next().then(
+          ({ value, done }) => {
+            if (done) {
+              this.push(null)
+            } else {
+              const subscription = subscriptions.get(value.channel)
+              if (subscription) {
+                const { event } = subscription
+                try {
+                  const data = event.payload.decode(value.payload)
+                  this.push({ event: event.name, data })
+                } catch (error: any) {
+                  this.destroy(error)
+                }
+              }
+            }
+          },
+          (error) => {
+            if (isAbortError(error)) {
+              this.push(null)
+            } else {
+              this.destroy(error)
+            }
+          },
+        )
+      },
+    })
+  }
+}
+
+function concat(...args: any) {
+  return args.filter(Boolean).join('/')
+}
+
+function getChannelName<T extends TAnyEventContract>(
+  contract: T,
+  options: T['options'],
+) {
+  const key = options ? serializerOptions(options) : ''
+  assert(contract.name, 'Event contract must have a name')
+  return concat(contract.name, key)
+}
+
+function serializerOptions(options: Exclude<SubcriptionOptions, null>): string {
+  const hash = createHash('sha1')
+  const serialized = Object.entries(options)
+    .sort((a, b) => a[0].localeCompare(b[0]))
+    .map(([key, value]) => `${key}=${value}`)
+    .join(';')
+  hash.update(serialized)
+  return hash.digest('base64url')
+}
+
+function mergeEventStreams(
+  streams: Readable[],
+  signal?: AbortSignal,
+): Readable {
+  const destination = new PassThrough({
+    signal,
+    objectMode: true,
+    readableObjectMode: true,
+    writableObjectMode: true,
+  })
+
+  let ended = 0
+
+  for (const source of streams) {
+    source.pipe(destination, { end: false })
+    source.once('end', () => {
+      ended++
+      if (ended === streams.length) {
+        destination.end()
+      }
+    })
+  }
+
+  return destination
+}