nmtjs 0.16.1 → 0.17.0-beta.2

package/src/runtime/subscription/redis.ts

@@ -1,159 +0,0 @@
-import EventEmitter, { on } from 'node:events'
-
-import type { RuntimePlugin } from '@nmtjs/application'
-import type { Logger } from '@nmtjs/core'
-import { isAbortError } from '@nmtjs/common'
-import {
-  CoreInjectables,
-  createFactoryInjectable,
-  provision,
-} from '@nmtjs/core'
-
-import type { Store } from '../types.ts'
-import type {
-  SubscriptionAdapterEvent,
-  SubscriptionAdapterType,
-} from './manager.ts'
-import { storeConfig, subscriptionAdapter } from '../injectables.ts'
-import { createStoreClient } from '../store/index.ts'
-
-export class RedisSubscriptionAdapter implements SubscriptionAdapterType {
-  protected readonly events = new EventEmitter()
-  protected readonly listeners = new Map<string, number>()
-  protected readonly logger?: Logger
-  protected subscriberClient?: Store
-
-  constructor(
-    protected readonly client: Store,
-    logger?: Logger,
-  ) {
-    this.logger = logger?.child({ component: RedisSubscriptionAdapter.name })
-  }
-
-  async initialize() {
-    this.logger?.debug('Initializing Redis adapter')
-
-    // Create a dedicated subscriber client (Redis requires separate clients for pub/sub)
-    this.subscriberClient = this.client.duplicate()
-
-    // Set up message handler
-    this.subscriberClient.on('message', (channel: string, message: string) => {
-      try {
-        const parsed = JSON.parse(message)
-        this.logger?.trace({ channel }, 'Received Redis message')
-        this.events.emit(channel, parsed)
-      } catch (error) {
-        this.logger?.warn({ channel, error }, 'Failed to parse Redis message')
-      }
-    })
-
-    this.logger?.debug('Redis adapter initialized')
-  }
-
-  async dispose() {
-    this.logger?.debug('Disposing Redis adapter')
-
-    if (this.subscriberClient) {
-      await this.subscriberClient.quit()
-      this.subscriberClient = undefined
-    }
-
-    this.logger?.debug('Redis adapter disposed')
-  }
-
-  async publish(channel: string, payload: any): Promise<boolean> {
-    this.logger?.trace({ channel }, 'Publishing Redis message')
-
-    try {
-      await this.client.publish(channel, JSON.stringify(payload))
-      this.logger?.trace({ channel }, 'Published Redis message')
-      return true
-    } catch (error) {
-      this.logger?.warn({ channel, error }, 'Failed to publish Redis message')
-      return false
-    }
-  }
-
-  async *subscribe(
-    channel: string,
-    signal?: AbortSignal,
-  ): AsyncGenerator<SubscriptionAdapterEvent> {
-    if (!this.subscriberClient) {
-      throw new Error('RedisSubscriptionAdapter not initialized')
-    }
-
-    this.logger?.debug({ channel }, 'Opening Redis channel listener')
-
-    if (!this.listeners.has(channel)) {
-      await this.subscriberClient.subscribe(channel)
-      this.listeners.set(channel, 1)
-      this.logger?.debug({ channel, listeners: 1 }, 'Subscribed Redis channel')
-    } else {
-      const listeners = this.listeners.get(channel)! + 1
-      this.listeners.set(channel, listeners)
-      this.logger?.trace(
-        { channel, listeners },
-        'Reusing Redis channel listener',
-      )
-    }
-
-    try {
-      signal?.throwIfAborted()
-      for await (const [payload] of on(this.events, channel, { signal })) {
-        this.logger?.trace({ channel }, 'Delivering Redis message')
-        yield { channel, payload }
-      }
-    } catch (error: any) {
-      if (isAbortError(error)) {
-        this.logger?.trace({ channel }, 'Redis channel listener aborted')
-        throw error
-      }
-
-      this.logger?.warn({ channel, error }, 'Redis channel listener failed')
-    } finally {
-      const count = this.listeners.get(channel)
-      if (count !== undefined) {
-        if (count > 1) {
-          const listeners = count - 1
-          this.listeners.set(channel, listeners)
-          this.logger?.trace(
-            { channel, listeners },
-            'Detached Redis channel listener',
-          )
-        } else {
-          await this.subscriberClient?.unsubscribe(channel)
-          this.listeners.delete(channel)
-          this.logger?.debug(
-            { channel, listeners: 0 },
-            'Unsubscribed Redis channel',
-          )
-        }
-      }
-    }
-  }
-}
-
-export const RedisSubscriptionAdapterPlugin = (): RuntimePlugin => {
-  const adapterFactory = createFactoryInjectable(
-    {
-      dependencies: { config: storeConfig, logger: CoreInjectables.logger },
-      factory: async ({ config, logger }) => {
-        const connection = await createStoreClient(config)
-        const adapter = new RedisSubscriptionAdapter(connection, logger)
-        await adapter.initialize()
-        return { adapter, connection }
-      },
-      pick: ({ adapter }) => adapter,
-      dispose: async ({ adapter, connection }) => {
-        await adapter.dispose()
-        await connection.quit()
-      },
-    },
-    'RedisSubscriptionAdapter',
-  )
-
-  return {
-    name: 'redis-subscription-adapter',
-    injections: [provision(subscriptionAdapter, adapterFactory)],
-  }
-}

package/src/runtime/types.ts

@@ -1,84 +0,0 @@
-import type { ApplicationHostDefinition } from '@nmtjs/application'
-import type { ProxyableTransportType } from '@nmtjs/gateway'
-import type { Redis, RedisOptions } from 'ioredis'
-import type { Redis as Valkey, RedisOptions as ValkeyOptions } from 'iovalkey'
-
-import type { StoreType } from './enums.ts'
-
-export type WorkerThreadErrorOrigin = 'bootstrap' | 'start' | 'runtime'
-
-export type ThreadErrorMessage = {
-  message: string
-  name?: string
-  stack?: string
-  origin: WorkerThreadErrorOrigin
-  fatal: boolean
-}
-
-export type WorkerThreadError = Error & {
-  origin?: WorkerThreadErrorOrigin
-  fatal?: boolean
-}
-
-export type ServerPortMessageTypes = {
-  stop: undefined
-  task: { id: string; task: WorkerJobTask }
-}
-
-export type ThreadPortMessageTypes = {
-  ready: { hosts?: { type: ProxyableTransportType; url: string }[] }
-  error: ThreadErrorMessage
-  task: { id: string; task: JobTaskResult }
-}
-
-export type ServerPortMessage = {
-  [K in keyof ServerPortMessageTypes]: {
-    type: K
-    data: ServerPortMessageTypes[K]
-  }
-}[keyof ServerPortMessageTypes]
-
-export type ThreadPortMessage = {
-  [K in keyof ThreadPortMessageTypes]: {
-    type: K
-    data: ThreadPortMessageTypes[K]
-  }
-}[keyof ThreadPortMessageTypes]
-
-export interface WorkerTask {
-  type?: string
-  payload?: any
-}
-
-export type WorkerJobTask = { jobId: string; jobName: string; data: any }
-
-export type JobTaskResult = {
-  [K in keyof JobTaskResultTypes]: { type: K } & JobTaskResultTypes[K]
-}[keyof JobTaskResultTypes]
-
-export type JobTaskResultTypes = {
-  success: { result?: unknown }
-  error: { error: any }
-  unrecoverable_error: { error: any }
-  job_not_found: {}
-  queue_job_not_found: {}
-}
-
-export type ApplicationDefinitionType =
-  | { type: 'neemata'; definition: ApplicationHostDefinition }
-  | { type: 'custom'; definition: any }
-
-export interface Applications
-  extends Record<string, ApplicationDefinitionType> {}
-
-export type StoreTypes = {
-  [StoreType.Redis]: Redis
-  [StoreType.Valkey]: Valkey
-}
-export type StoreTypeOptions = {
-  [StoreType.Redis]: RedisOptions
-  [StoreType.Valkey]: ValkeyOptions
-}
-
-export type Store = StoreTypes[StoreType]
-export type StoreOptions = StoreTypeOptions[StoreType]

package/src/runtime/workers/application.ts

@@ -1,133 +0,0 @@
-import type {
-  ApplicationHost,
-  ApplicationHostDefinition,
-  ApplicationHostOptions,
-  ApplicationResolvedProcedure,
-  ApplicationTransport,
-  NeemataApplication,
-  TransportOptionsOf,
-} from '@nmtjs/application'
-import type { Gateway, GatewayOptions } from '@nmtjs/gateway'
-import { createApplicationHost, LifecycleHook } from '@nmtjs/application'
-import { JsonFormat } from '@nmtjs/json-format/server'
-import { MsgpackFormat } from '@nmtjs/msgpack-format/server'
-import { ProtocolFormats } from '@nmtjs/protocol/server'
-
-import type { ServerApplicationConfig, ServerConfig } from '../server/config.ts'
-import { WorkerType } from '../enums.ts'
-import { BaseWorkerRuntime } from './base.ts'
-
-export interface ApplicationWorkerRuntimeOptions {
-  name: string
-  path: string
-  transports: ServerApplicationConfig['threads'][number]
-}
-
-export class ApplicationWorkerRuntime extends BaseWorkerRuntime {
-  host!: ApplicationHost
-
-  constructor(
-    readonly config: ServerConfig,
-    readonly runtimeOptions: ApplicationWorkerRuntimeOptions,
-    protected hostDefinition: ApplicationHostDefinition,
-  ) {
-    super(
-      config,
-      {
-        logger: config.logger,
-        name: `Worker ${runtimeOptions.name}`,
-        plugins: [],
-      },
-      WorkerType.Application,
-    )
-  }
-
-  get application(): NeemataApplication {
-    return this.host.application
-  }
-
-  get api() {
-    return this.application.api
-  }
-
-  get applicationHooks() {
-    return this.application.applicationHooks
-  }
-
-  get gateway(): Gateway<ApplicationResolvedProcedure> {
-    return this.host.gateway
-  }
-
-  get transports(): GatewayOptions<ApplicationResolvedProcedure>['transports'] {
-    return this.host.transports
-  }
-
-  async start() {
-    await this.initialize()
-    this.host = createApplicationHost(this.hostDefinition.application, {
-      name: this.runtimeOptions.name,
-      logger: this.logger,
-      container: this.container,
-      formats: new ProtocolFormats([new JsonFormat(), new MsgpackFormat()]),
-      ...this.resolveHostOptions(),
-    })
-    return this.host.start()
-  }
-
-  async stop() {
-    await this.host.stop()
-    await this.dispose()
-    await this.lifecycleHooks.callHook(LifecycleHook.Stop)
-  }
-
-  async reload(hostDefinition: ApplicationHostDefinition): Promise<void> {
-    this.hostDefinition = hostDefinition
-    await this.host.reloadApplication(hostDefinition.application)
-  }
-
-  protected *_dependents() {}
-
-  protected resolveHostOptions(): Pick<
-    ApplicationHostOptions,
-    'transports' | 'gateway' | 'identity'
-  > {
-    const config = this.config.applications[
-      this.runtimeOptions.name
-    ] as ServerApplicationConfig
-
-    if (!config) {
-      throw new Error(
-        `Missing server application config: ${this.runtimeOptions.name}`,
-      )
-    }
-
-    return {
-      transports: createHostTransportConfig(
-        this.hostDefinition.transports,
-        this.runtimeOptions.transports,
-      ),
-      gateway: this.hostDefinition.gateway,
-      identity: this.hostDefinition.identity,
-    }
-  }
-}
-
-function createHostTransportConfig<
-  Transports extends Record<string, ApplicationTransport>,
->(
-  transports: Transports,
-  options: { [K in keyof Transports]: TransportOptionsOf<Transports[K]> },
-) {
-  const config = {} as {
-    [K in keyof Transports]: {
-      transport: Transports[K]
-      options: TransportOptionsOf<Transports[K]>
-    }
-  }
-
-  for (const key in transports) {
-    config[key] = { transport: transports[key], options: options[key] }
-  }
-
-  return config
-}

package/src/runtime/workers/base.ts

@@ -1,65 +0,0 @@
-import type { Provision } from '@nmtjs/core'
-import { CoreInjectables, provision } from '@nmtjs/core'
-
-import type { WorkerType } from '../enums.ts'
-import type { BaseRuntimeOptions } from '../runtime.ts'
-import type { ServerConfig } from '../server/config.ts'
-import * as injectables from '../injectables.ts'
-import { JobManager } from '../jobs/manager.ts'
-import { BaseRuntime } from '../runtime.ts'
-
-export abstract class BaseWorkerRuntime extends BaseRuntime {
-  jobManager?: JobManager
-
-  constructor(
-    readonly config: ServerConfig,
-    options: BaseRuntimeOptions,
-    readonly workerType: WorkerType,
-  ) {
-    super(options)
-
-    if (this.config.store) {
-      this.jobManager = new JobManager(
-        this.config.store,
-        this.config.jobs ? Array.from(this.config.jobs.jobs.values()) : [],
-      )
-    }
-  }
-
-  async initialize(): Promise<void> {
-    const injections: Provision[] = [
-      provision(CoreInjectables.logger, this.logger),
-      provision(injectables.workerType, this.workerType),
-    ]
-
-    if (this.config.store) {
-      injections.push(provision(injectables.storeConfig, this.config.store))
-    }
-
-    if (this.config.subscription) {
-      injections.push(
-        provision(
-          injectables.subscriptionAdapter,
-          this.config.subscription.adapter,
-        ),
-      )
-    }
-
-    if (this.jobManager) {
-      injections.push(
-        provision(injectables.jobManager, this.jobManager.publicInstance),
-      )
-    }
-
-    this.container.provide(injections)
-    await super.initialize()
-  }
-
-  protected async _initialize(): Promise<void> {
-    await this.jobManager?.initialize()
-  }
-
-  protected async _dispose(): Promise<void> {
-    await this.jobManager?.terminate()
-  }
-}

package/src/runtime/workers/job.ts

@@ -1,142 +0,0 @@
-import type { MessagePort } from 'node:worker_threads'
-
-import { LifecycleHook } from '@nmtjs/application'
-import { UnrecoverableError } from 'bullmq'
-
-import type { JobProgressCheckpoint } from '../jobs/types.ts'
-import type { ServerConfig } from '../server/config.ts'
-import type { ServerPortMessage, ThreadPortMessage } from '../types.ts'
-import { WorkerType } from '../enums.ts'
-import { jobWorkerPool } from '../injectables.ts'
-import { ApplicationWorkerJobRunner } from '../jobs/runner.ts'
-import { BaseWorkerRuntime } from './base.ts'
-
-export interface JobWorkerRuntimeOptions {
-  poolName: string
-  port: MessagePort
-}
-
-export class JobWorkerRuntime extends BaseWorkerRuntime {
-  jobRunner!: ApplicationWorkerJobRunner
-
-  constructor(
-    readonly config: ServerConfig,
-    readonly runtimeOptions: JobWorkerRuntimeOptions,
-  ) {
-    super(
-      config,
-      { logger: config.logger, name: `Job Worker ${runtimeOptions.poolName}` },
-      WorkerType.Job,
-    )
-  }
-
-  async start() {
-    await this.initialize()
-    await this.lifecycleHooks.callHook(LifecycleHook.Start)
-  }
-
-  async stop() {
-    await this.lifecycleHooks.callHook(LifecycleHook.Stop)
-    await this.dispose()
-  }
-
-  async initialize(): Promise<void> {
-    // Validate all jobs are complete before accepting work
-    if (this.config.jobs) {
-      for (const job of this.config.jobs.jobs.values()) {
-        if (!job.returnHandler) {
-          throw new Error(
-            `Job "${job.name}" is incomplete. Jobs must call .return() to be finalized before use.`,
-          )
-        }
-      }
-    }
-
-    this.container.provide(jobWorkerPool, this.runtimeOptions.poolName)
-    await super.initialize()
-  }
-
-  protected async _initialize(): Promise<void> {
-    await super._initialize()
-
-    this.jobRunner = new ApplicationWorkerJobRunner({
-      logger: this.logger,
-      container: this.container,
-      lifecycleHooks: this.lifecycleHooks,
-    })
-
-    this.runtimeOptions.port.on('message', async (msg: ServerPortMessage) => {
-      if (msg.type === 'task') {
-        const { id, task } = msg.data
-        try {
-          const job = this.config.jobs?.jobs.get(task.jobName)
-          if (!job) {
-            this.runtimeOptions.port.postMessage({
-              type: 'task',
-              data: { id, task: { type: 'job_not_found' } },
-            } satisfies ThreadPortMessage)
-            return
-          }
-
-          using cancellationSignal = this.jobManager!.cancellationSignal(
-            job,
-            task.jobId,
-          )
-          const queue = this.jobManager!.getQueue(job).queue
-          const bullJob = await queue.getJob(task.jobId)
-          if (!bullJob) {
-            throw new UnrecoverableError(
-              `Job ${task.jobId} not found in queue (may have been removed)`,
-            )
-          }
-
-          // Load checkpoint from BullMQ progress for resume support
-          const checkpoint = bullJob.progress as
-            | JobProgressCheckpoint
-            | undefined
-
-          const result = await this.jobRunner.runJob(job, task.data, {
-            signal: cancellationSignal,
-            queueJob: bullJob,
-            result: checkpoint?.result,
-            stepResults: checkpoint?.stepResults,
-            currentStepIndex: checkpoint?.stepIndex ?? 0,
-            progress: checkpoint?.progress,
-          })
-          this.runtimeOptions.port.postMessage({
-            type: 'task',
-            data: { id, task: { type: 'success', result } },
-          })
-        } catch (error) {
-          if (error instanceof UnrecoverableError) {
-            this.runtimeOptions.port.postMessage({
-              type: 'task',
-              data: { id, task: { type: 'unrecoverable_error', error } },
-            } satisfies ThreadPortMessage)
-          } else {
-            this.runtimeOptions.port.postMessage({
-              type: 'task',
-              data: { id, task: { type: 'error', error } },
-            } satisfies ThreadPortMessage)
-          }
-        }
-      }
-    })
-  }
-
-  protected async _dispose(): Promise<void> {
-    this.runtimeOptions.port.removeAllListeners('message')
-    await super._dispose()
-  }
-
-  protected *_dependents() {
-    if (this.config.jobs) {
-      for (const job of this.config.jobs.jobs.values()) {
-        yield job
-        // we explicitly DO NOT WANT to yield steps here, so per-job container
-        // resolves them independently each time — creates more isolation
-        // yield* job.jobSteps
-      }
-    }
-  }
-}

package/src/vite/builder.ts

@@ -1,94 +0,0 @@
-import { isBuiltin } from 'node:module'
-import { resolve } from 'node:path'
-
-import type { UserConfig } from 'vite'
-import { mergeConfig, build as viteBuild } from 'vite'
-
-import type { NeemataConfig } from '../config.ts'
-import type { ViteConfigOptions } from './config.ts'
-import { createConfig } from './config.ts'
-import { plugins } from './plugins.ts'
-
-export async function createBuilder(
-  configOptions: ViteConfigOptions,
-  neemataConfig: NeemataConfig,
-) {
-  const config = createConfig(configOptions)
-  async function build(): Promise<any> {
-    const packageJson = await import(resolve('./package.json'), {
-      with: { type: 'json' },
-    }).then((mod) => mod.default)
-
-    return await viteBuild(
-      mergeConfig(
-        {
-          appType: 'custom',
-          clearScreen: false,
-          resolve: { alias: config.alias, noExternal: true },
-          plugins: [...plugins],
-          build: {
-            lib: { entry: config.entries, formats: ['es'] },
-            ssr: true,
-            ssrEmitAssets: true,
-            target: 'node24',
-            sourcemap: true,
-            outDir: resolve('./dist'),
-            minify: true,
-            emptyOutDir: true,
-            rolldownOptions: {
-              platform: 'node',
-              external: (id) => {
-                if (neemataConfig.externalDependencies === 'all') return true
-                if (
-                  isBuiltin(id) ||
-                  id.includes('nmtjs/src/vite/servers') ||
-                  id.includes('nmtjs/src/vite/runners')
-                )
-                  return true
-
-                if (neemataConfig.externalDependencies === 'prod') {
-                  const prodDeps = Object.keys(packageJson.dependencies ?? {})
-                  if (prodDeps.includes(id)) return true
-                }
-
-                if (Array.isArray(neemataConfig.externalDependencies)) {
-                  for (const dep of neemataConfig.externalDependencies) {
-                    if (typeof dep === 'string' && dep === id) return true
-                    if (dep instanceof RegExp && dep.test(id)) return true
-                  }
-                }
-
-                return false
-              },
-              transform: {
-                define: {
-                  __VITE_CONFIG__: '""',
-                  __APPLICATIONS_CONFIG__: JSON.stringify(
-                    JSON.stringify(
-                      Object.fromEntries(
-                        Object.entries(configOptions.applicationImports).map(
-                          ([appName, { type }]) => [
-                            appName,
-                            { type, specifier: `./application.${appName}.js` },
-                          ],
-                        ),
-                      ),
-                    ),
-                  ),
-                },
-              },
-              output: {
-                entryFileNames: '[name].js',
-                chunkFileNames: '[name]-[hash].js',
-                assetFileNames: '[name][extname]',
-              },
-            },
-            chunkSizeWarningLimit: 10_000,
-          },
-        } satisfies UserConfig,
-        neemataConfig.vite,
-      ),
-    )
-  }
-  return { build }
-}