nmtjs 0.15.3 → 0.16.0-beta.10

package/src/runtime/runtime.ts

@@ -1,9 +1,11 @@
+import type { RuntimePlugin } from '@nmtjs/application'
 import type {
   AnyInjectable,
   Dependant,
   Logger,
   LoggingOptions,
 } from '@nmtjs/core'
+import { LifecycleHook, LifecycleHooks } from '@nmtjs/application'
 import {
   Container,
   createLogger,
@@ -11,10 +13,6 @@ import {
   Scope,
 } from '@nmtjs/core'
 
-import type { RuntimePlugin } from './plugin.ts'
-import { LifecycleHook } from './enums.ts'
-import { LifecycleHooks } from './hooks.ts'
-
 export type BaseRuntimeOptions = {
   logger?: LoggingOptions
   container?: Container

package/src/runtime/server/config.ts

@@ -1,13 +1,15 @@
+import type {
+  ApplicationConfig,
+  ApplicationTransport,
+} from '@nmtjs/application'
 import type { LoggingOptions } from '@nmtjs/core'
-import type { Transport } from '@nmtjs/gateway'
 import type { ApplicationOptions } from '@nmtjs/proxy'
 import type { Applications } from 'nmtjs/runtime/types'
 
-import type { ApplicationConfig } from '../application/config.ts'
-import type { JobWorkerPool, StoreType } from '../enums.ts'
+import type { StoreType } from '../enums.ts'
 import type { AnyJob } from '../jobs/job.ts'
-import type { PubSubAdapterType } from '../pubsub/manager.ts'
 import type { JobsSchedulerOptions } from '../scheduler/index.ts'
+import type { SubscriptionAdapterType } from '../subscription/manager.ts'
 import type { StoreTypeOptions } from '../types.ts'
 import { kServerConfig } from '../constants.ts'
 
@@ -30,7 +32,7 @@ export type ServerApplicationConfig<T = ApplicationConfig> =
   T extends ApplicationConfig<any, infer Transports>
     ? {
         threads: {
-          [K in keyof Transports]: Transports[K] extends Transport<
+          [K in keyof Transports]: Transports[K] extends ApplicationTransport<
             any,
             infer Options
           >
@@ -41,17 +43,13 @@ export type ServerApplicationConfig<T = ApplicationConfig> =
     : any
 
 export type ServerJobsConfig = {
-  pools: {
-    [JobWorkerPool.Io]: ServerPoolOptions
-    [JobWorkerPool.Compute]: ServerPoolOptions
-  }
+  pools: Record<string, ServerPoolOptions>
   jobs?: AnyJob[]
   /**
    * @deprecated Scheduler is currently being refactored and is not available.
    * Using this option will throw an error at startup.
    */
   scheduler?: JobsSchedulerOptions
-  ui?: { hostname?: string; port?: number }
 }
 
 export interface ServerConfigInit {
@@ -86,7 +84,7 @@ export interface ServerConfigInit {
   }
   jobs?: ServerJobsConfig
   commands?: {}
-  pubsub?: { adapter: PubSubAdapterType }
+  subscription?: { adapter: SubscriptionAdapterType }
   deploymentId?: string
   metrics?: {
     /**
@@ -133,8 +131,6 @@ export interface ServerConfig {
    * The applications will be accessible via `<hostname>:<port>/<application-name>/**`
    *
    * Requires adding `@nmtjs/proxy` package to your dependencies
-   * and [`cargo`](https://doc.rust-lang.org/cargo/getting-started/installation.html) alongside with
-   * [`rustc`](https://www.rust-lang.org/tools/install) to be available globally for building native modules.
    */
   proxy: ServerConfigInit['proxy']
   jobs?: {
@@ -145,10 +141,9 @@ export interface ServerConfig {
      * Using this option will throw an error at startup.
      */
     scheduler?: JobsSchedulerOptions
-    ui?: ServerJobsConfig['ui']
   }
   commands?: {}
-  pubsub: ServerConfigInit['pubsub']
+  subscription: ServerConfigInit['subscription']
   deploymentId: ServerConfigInit['deploymentId']
   metrics?: ServerConfigInit['metrics']
 }
@@ -161,7 +156,7 @@ export function defineServer(options: ServerConfigInit): ServerConfig {
     proxy,
     store,
     applications,
-    pubsub,
+    subscription,
     metrics,
   } = options
 
@@ -175,7 +170,6 @@ export function defineServer(options: ServerConfigInit): ServerConfig {
       pools: options.jobs.pools,
       jobs: map,
       scheduler: options.jobs.scheduler,
-      ui: options.jobs.ui,
     }
   }
 
@@ -187,7 +181,7 @@ export function defineServer(options: ServerConfigInit): ServerConfig {
     proxy,
     store,
     applications,
-    pubsub,
+    subscription,
     jobs,
     metrics,
   } as const)

package/src/runtime/server/jobs.ts

@@ -2,10 +2,9 @@ import { inspect } from 'node:util'
 
 import type { Logger } from '@nmtjs/core'
 import type { RedisClient } from 'bullmq'
-import { Queue, UnrecoverableError, Worker } from 'bullmq'
+import { UnrecoverableError, Worker } from 'bullmq'
 
 import type { AnyJob } from '../jobs/job.ts'
-import type { JobsUI } from '../jobs/ui.ts'
 import type { Store, WorkerJobTask } from '../types.ts'
 import type { ServerConfig } from './config.ts'
 import type { ErrorPolicy } from './error-policy.ts'
@@ -15,11 +14,12 @@ import type {
   WorkerPoolConfig,
   WorkerPoolFactory,
 } from './worker-pool.ts'
-import { JobWorkerPool, WorkerType } from '../enums.ts'
+import { WorkerType } from '../enums.ts'
 import { getJobQueueName } from '../jobs/manager.ts'
-import { createJobsUI } from '../jobs/ui.ts'
 import { JobRunnersPool } from './worker-pool.ts'
 
+type JobsPoolConfig = Exclude<ServerConfig['jobs'], undefined>['pools']
+
 function enrichBullMqErrorStack(error: unknown): Error {
   const normalized =
     error instanceof Error
@@ -33,6 +33,38 @@ function enrichBullMqErrorStack(error: unknown): Error {
   return normalized
 }
 
+function getJobsByPool(jobs: Iterable<AnyJob>): Map<string, AnyJob[]> {
+  const jobsByPool = new Map<string, AnyJob[]>()
+
+  for (const job of jobs) {
+    const poolJobs = jobsByPool.get(job.options.pool)
+    if (poolJobs) {
+      poolJobs.push(job)
+    } else {
+      jobsByPool.set(job.options.pool, [job])
+    }
+  }
+
+  return jobsByPool
+}
+
+function assertActiveJobPoolsConfigured(
+  jobsByPool: Map<string, AnyJob[]>,
+  pools: JobsPoolConfig,
+) {
+  const missingPoolMessages = [...jobsByPool]
+    .filter(([poolName]) => !pools[poolName])
+    .flatMap(([poolName, jobs]) =>
+      jobs.map((job) => `${job.name} -> ${poolName}`),
+    )
+
+  if (missingPoolMessages.length > 0) {
+    throw new Error(
+      `Invalid jobs pool configuration: missing pool config for jobs: ${missingPoolMessages.join(', ')}`,
+    )
+  }
+}
+
 /**
  * ApplicationServerJobs manages job worker pools and BullMQ queue workers.
  *
@@ -40,23 +72,21 @@ function enrichBullMqErrorStack(error: unknown): Error {
  * - Uses JobRunnersPool (extends WorkerPool) for worker management
  * - Uses ManagedWorker for restart logic and state tracking
  * - Integrates with ErrorPolicy for restart decisions
- * - Proper health tracking per job pool type
+ * - Proper health tracking per job pool
  */
 export class ApplicationServerJobs {
   /**
    * BullMQ workers - one per job (dedicated queues)
    */
   queueWorkers = new Set<Worker>()
-  ui?: JobsUI
-  protected uiQueues: Queue[] = []
 
   jobs: Map<string, AnyJob>
 
   /**
-   * Shared resource pools by pool type (Io, Compute).
-   * All jobs of a given pool type share the same pool for resource management.
+   * Shared resource pools by configured pool name.
+   * All jobs using the same pool name share the same pool for resource management.
    */
-  protected pools = new Map<JobWorkerPool, JobRunnersPool>()
+  protected pools = new Map<string, JobRunnersPool>()
 
   constructor(
     readonly params: {
@@ -90,44 +120,16 @@ export class ApplicationServerJobs {
       return
     }
 
-    if (jobsConfig.ui) {
-      const hostname = jobsConfig.ui.hostname ?? '127.0.0.1'
-      const port = jobsConfig.ui.port ?? 3000
-
-      this.uiQueues = [...this.jobs.values()].map(
-        (job) =>
-          new Queue(getJobQueueName(job), {
-            connection: store as unknown as RedisClient,
-          }),
-      )
-
-      this.ui = createJobsUI(this.uiQueues)
-
-      await new Promise<void>((resolve, reject) => {
-        if (!this.ui) return reject(new Error('Jobs UI server is missing'))
-        this.ui.once('error', reject)
-        this.ui.listen(port, hostname, resolve)
-      })
-
-      const address = this.ui.address()
-      const resolved =
-        address && typeof address !== 'string'
-          ? { hostname: address.address, port: address.port }
-          : { hostname, port }
-
-      logger.info({ ...resolved }, 'Jobs UI started')
-    }
-
-    // Step 1: Initialize shared resource pools (Io, Compute)
-    const poolTypes = Object.values(JobWorkerPool)
+    const jobsByPool = getJobsByPool(this.jobs.values())
+    assertActiveJobPoolsConfigured(jobsByPool, jobsConfig.pools)
 
-    for (const poolType of poolTypes) {
-      const poolConfig = jobsConfig.pools[poolType]
-      if (!poolConfig) continue
+    // Step 1: Initialize shared resource pools used by active jobs
+    for (const poolName of jobsByPool.keys()) {
+      const poolConfig = jobsConfig.pools[poolName]!
 
-      // Create a JobRunnersPool for this pool type
+      // Create a JobRunnersPool for this pool
       const config: WorkerPoolConfig = {
-        name: `job-pool-${poolType}`,
+        name: `job-pool-${poolName}`,
         workerType: WorkerType.Job,
         path: workerConfig.path,
         workerData: { ...workerConfig.workerData },
@@ -143,15 +145,15 @@ export class ApplicationServerJobs {
 
       // Add workers to the pool
       for (let i = 0; i < poolConfig.threads; i++) {
-        pool.add({ runtime: { type: 'jobs', jobWorkerPool: poolType } }, i)
+        pool.add({ runtime: { type: 'jobs', jobWorkerPool: poolName } }, i)
       }
 
       await pool.startAll()
-      this.pools.set(poolType, pool)
+      this.pools.set(poolName, pool)
 
       logger.info(
         {
-          pool: poolType,
+          pool: poolName,
           threads: poolConfig.threads,
           jobsPerThread: poolConfig.jobs,
         },
@@ -161,28 +163,18 @@ export class ApplicationServerJobs {
 
     // Step 2: Create a dedicated BullMQ Worker for each job
     // Calculate how many jobs use each pool for fair concurrency distribution
-    const jobsPerPool = new Map<JobWorkerPool, number>()
-    for (const job of this.jobs.values()) {
-      const count = jobsPerPool.get(job.options.pool) ?? 0
-      jobsPerPool.set(job.options.pool, count + 1)
-    }
-
     for (const job of this.jobs.values()) {
       const queueName = getJobQueueName(job)
-      const poolType = job.options.pool
-      const pool = this.pools.get(poolType)
+      const poolName = job.options.pool
+      const pool = this.pools.get(poolName)
 
       if (!pool) {
-        logger.warn(
-          { job: job.name, pool: poolType },
-          'No pool configured for job, skipping worker creation',
-        )
-        continue
+        throw new Error(`Job "${job.name}" pool "${poolName}" is not started`)
       }
 
-      const poolConfig = jobsConfig.pools[poolType]
+      const poolConfig = jobsConfig.pools[poolName]!
       const poolCapacity = poolConfig.threads * poolConfig.jobs
-      const jobCountInPool = jobsPerPool.get(poolType) ?? 1
+      const jobCountInPool = jobsByPool.get(poolName)?.length ?? 1
       // Use job-specific concurrency if provided, otherwise distribute pool capacity evenly
       const defaultConcurrency = Math.max(
         1,
@@ -224,7 +216,7 @@ export class ApplicationServerJobs {
 
       this.queueWorkers.add(queueWorker)
       logger.info(
-        { job: job.name, queue: queueName, pool: poolType, concurrency },
+        { job: job.name, queue: queueName, pool: poolName, concurrency },
         'Job queue worker started',
       )
     }
@@ -233,14 +225,6 @@ export class ApplicationServerJobs {
   async stop() {
     const { logger } = this.params
 
-    if (this.ui) {
-      await new Promise<void>((resolve) => {
-        this.ui?.close(() => resolve())
-      }).catch((error) => {
-        logger.warn({ error }, 'Failed to stop Jobs UI server')
-      })
-    }
-
     // Force-close BullMQ workers immediately — stops accepting new jobs
     // and doesn't wait for in-flight processor callbacks.
     // We must use force=true on the first call because BullMQ caches the
@@ -271,25 +255,12 @@ export class ApplicationServerJobs {
       }),
     )
     this.pools.clear()
-
-    // Close UI queues last (non-critical)
-    await Promise.all(
-      this.uiQueues.map(async (queue) => {
-        try {
-          await queue.close()
-        } catch (error) {
-          logger.warn({ error }, 'Failed to close Jobs UI queue')
-        }
-      }),
-    )
-    this.uiQueues = []
-    this.ui = undefined
   }
 
   /**
-   * Get a job pool by type.
+   * Get a job pool by name.
    */
-  getPool(poolType: JobWorkerPool): JobRunnersPool | undefined {
-    return this.pools.get(poolType)
+  getPool(poolName: string): JobRunnersPool | undefined {
+    return this.pools.get(poolName)
   }
 }

package/src/runtime/{pubsub → subscription}/manager.ts

@@ -11,27 +11,27 @@ import type { Container, Logger } from '@nmtjs/core'
 import type { t } from '@nmtjs/type'
 import { isAbortError } from '@nmtjs/common'
 
-import { pubSubAdapter } from '../injectables.ts'
+import { subscriptionAdapter } from '../injectables.ts'
 
-export type PubSubAdapterEvent = { channel: string; payload: any }
+export type SubscriptionAdapterEvent = { channel: string; payload: any }
 
-export interface PubSubAdapterType {
+export interface SubscriptionAdapterType {
   publish(channel: string, payload: any): Promise<boolean>
   subscribe(
     channel: string,
     signal?: AbortSignal,
-  ): AsyncGenerator<PubSubAdapterEvent>
+  ): AsyncGenerator<SubscriptionAdapterEvent>
   initialize(): Promise<void>
   dispose(): Promise<void>
 }
 
-export type PubSubChannel = {
+export type SubscriptionChannel = {
   stream: Readable
   subscription: TAnySubscriptionContract
   event: TAnyEventContract
 }
 
-export type PubSubSubscribe = <
+export type SubscribeFn = <
   Contract extends TAnySubscriptionContract,
   Events extends {
     [K in keyof Contract['events']]?: true
@@ -63,7 +63,7 @@ export type PubSubSubscribe = <
   }
 >
 
-export type PubSubPublish = <
+export type PublishFn = <
   S extends TAnySubscriptionContract,
   E extends S['events'][keyof S['events']],
 >(
@@ -72,23 +72,24 @@ export type PubSubPublish = <
   data: t.infer.decode.input<E['payload']>,
 ) => Promise<boolean>
 
-export type PubSubManagerOptions = { logger: Logger; container: Container }
+export type SubscriptionManagerOptions = {
+  logger: Logger
+  container: Container
+}
 
-export class PubSubManager {
-  readonly subscriptions = new Map<string, PubSubChannel>()
+export class SubscriptionManager {
+  readonly subscriptions = new Map<string, SubscriptionChannel>()
+  protected readonly logger: Logger
 
-  constructor(protected readonly options: PubSubManagerOptions) {}
+  constructor(protected readonly options: SubscriptionManagerOptions) {
+    this.logger = options.logger.child({ component: SubscriptionManager.name })
+  }
 
   protected get adapter() {
-    return this.options.container.resolve(pubSubAdapter)
+    return this.options.container.resolve(subscriptionAdapter)
   }
 
-  subscribe: PubSubSubscribe = async (
-    subscription,
-    events,
-    options,
-    signal,
-  ) => {
+  subscribe: SubscribeFn = async (subscription, events, options, signal) => {
     const adapter = await this.adapter
 
     const eventKeys =
@@ -96,6 +97,11 @@ export class PubSubManager {
         ? Object.keys(subscription.events)
         : Object.keys(events)
 
+    this.logger.debug(
+      { subscription: subscription.name, eventCount: eventKeys.length },
+      'Opening subscription',
+    )
+
     const streams = Array(eventKeys.length)
 
     for (const index in eventKeys) {
@@ -103,44 +109,115 @@ export class PubSubManager {
       const channel = getChannelName(event, options)
       if (this.subscriptions.has(channel)) {
         streams[index] = this.subscriptions.get(channel)!.stream
+        this.logger.trace(
+          {
+            channel,
+            event: event.name,
+            activeChannels: this.subscriptions.size,
+          },
+          'Reusing pubsub channel stream',
+        )
       } else {
         const iterable = adapter.subscribe(channel, signal)
         const stream = this.createEventStream(iterable)
-        stream.on('close', () => this.subscriptions.delete(channel))
+        stream.on('close', () => {
+          this.subscriptions.delete(channel)
+          this.logger.debug(
+            {
+              channel,
+              event: event.name,
+              activeChannels: this.subscriptions.size,
+            },
+            'Pubsub channel stream closed',
+          )
+        })
+        stream.on('error', (error) => {
+          this.logger.warn(
+            { channel, event: event.name, error },
+            'Pubsub channel stream failed',
+          )
+        })
         streams[index] = stream
         this.subscriptions.set(channel, { subscription, event, stream })
+        this.logger.debug(
+          {
+            channel,
+            event: event.name,
+            activeChannels: this.subscriptions.size,
+          },
+          'Creating channel stream',
+        )
       }
     }
 
-    return mergeEventStreams(streams, signal)
+    const mergedStream = mergeEventStreams(streams, signal)
+
+    mergedStream.once('close', () => {
+      this.logger.debug(
+        { subscription: subscription.name, eventCount: eventKeys.length },
+        'Pubsub subscription stream closed',
+      )
+    })
+    mergedStream.once('error', (error) => {
+      this.logger.warn(
+        {
+          subscription: subscription.name,
+          eventCount: eventKeys.length,
+          error,
+        },
+        'Pubsub subscription stream failed',
+      )
+    })
+
+    return mergedStream
   }
 
-  publish: PubSubPublish = async (event, options, data) => {
+  publish: PublishFn = async (event, options, data) => {
     const adapter = await this.adapter
 
     const channel = getChannelName(event, options)
 
+    this.logger.trace({ channel, event: event.name }, 'Publishing pubsub event')
+
     try {
       const payload = event.payload.encode(data)
-      return await adapter.publish(channel, payload)
+      const published = await adapter.publish(channel, payload)
+
+      if (published) {
+        this.logger.trace(
+          { channel, event: event.name },
+          'Published pubsub event',
+        )
+      } else {
+        this.logger.warn(
+          { channel, event: event.name },
+          'Pubsub adapter reported publish failure',
+        )
+      }
+
+      return published
     } catch (error: any) {
-      this.options.logger.error(
-        `Failed to publish event "${event.name}" on channel "${channel}": ${error.message}`,
+      this.logger.error(
+        { channel, event: event.name, error },
+        'Failed to publish pubsub event',
       )
-      return Promise.reject(error)
+      throw error
     }
   }
 
   private createEventStream(
-    iterable: AsyncGenerator<PubSubAdapterEvent>,
+    iterable: AsyncGenerator<SubscriptionAdapterEvent>,
   ): Readable {
     const { subscriptions } = this
+    const logger = this.logger
+
     return new Readable({
       objectMode: true,
       read() {
         iterable.next().then(
           ({ value, done }) => {
             if (done) {
+              logger.trace('Pubsub adapter stream ended')
               this.push(null)
             } else {
               const subscription = subscriptions.get(value.channel)
@@ -148,17 +225,32 @@ export class PubSubManager {
                 const { event } = subscription
                 try {
                   const data = event.payload.decode(value.payload)
+                  logger.trace(
+                    { channel: value.channel, event: event.name },
+                    'Received event',
+                  )
                   this.push({ event: event.name, data })
                 } catch (error: any) {
+                  logger.warn(
+                    { channel: value.channel, event: event.name, error },
+                    'Failed to decode pubsub event payload',
+                  )
                   this.destroy(error)
                 }
+              } else {
+                logger.trace(
+                  { channel: value.channel },
+                  'Dropped pubsub event for inactive channel',
+                )
               }
             }
           },
           (error) => {
             if (isAbortError(error)) {
+              logger.trace('Pubsub adapter stream aborted')
               this.push(null)
             } else {
+              logger.warn({ error }, 'Pubsub adapter stream failed')
               this.destroy(error)
             }
           },