@kubb/core 5.0.0-beta.15 → 5.0.0-beta.16

package/src/createKubb.ts

@@ -2,10 +2,10 @@ import { resolve } from 'node:path'
 import { version as nodeVersion } from 'node:process'
 import type { PossiblePromise } from '@internals/utils'
 import { AsyncEventEmitter, BuildError, exists, formatMs, getElapsedMs, URLPath } from '@internals/utils'
-import type { FileNode, InputNode, OperationNode, SchemaNode } from '@kubb/ast'
+import type { FileNode, InputNode, InputStreamNode, OperationNode, SchemaNode } from '@kubb/ast'
 import { collectUsedSchemaNames, transform, walk } from '@kubb/ast'
 import { version as KubbVersion } from '../package.json'
-import { DEFAULT_BANNER, DEFAULT_EXTENSION, DEFAULT_STUDIO_URL } from './constants.ts'
+import { DEFAULT_BANNER, DEFAULT_EXTENSION, DEFAULT_STUDIO_URL, STREAM_FLUSH_EVERY, STREAM_SCHEMA_THRESHOLD } from './constants.ts'
 import type { Adapter, AdapterSource } from './createAdapter.ts'
 import type { RendererFactory } from './createRenderer.ts'
 import { createStorage, type Storage } from './createStorage.ts'
@@ -674,7 +674,7 @@ export type CLIOptions = {
 
 /**
  * All accepted forms of a Kubb configuration.
- * Accepts `Config`/`Config[]`/promise or a factory (optionally receiving `TCliOptions`).
+ * Accepts `Config`/`Config[]`/promise or a factory (optionally receiving `TCliOptions`.
  */
 export type PossibleConfig<TCliOptions = undefined> =
   | PossiblePromise<Config | Config[]>
@@ -929,16 +929,45 @@ async function setup(userConfig: UserConfig, options: SetupOptions = {}): Promis
     })
 
     driver.adapter = config.adapter
-    driver.inputNode = await config.adapter.parse(source)
 
-    await hooks.emit('kubb:debug', {
-      date: new Date(),
-      logs: [
-        `✓ Adapter '${config.adapter.name}' resolved InputNode`,
-        `  • Schemas: ${driver.inputNode.schemas.length}`,
-        `  • Operations: ${driver.inputNode.operations.length}`,
-      ],
-    })
+    if (config.adapter.count && config.adapter.stream) {
+      const { schemas: schemaCount, operations: operationCount } = await config.adapter.count(source)
+
+      if (schemaCount > STREAM_SCHEMA_THRESHOLD) {
+        driver.inputStreamNode = await config.adapter.stream(source)
+
+        await hooks.emit('kubb:debug', {
+          date: new Date(),
+          logs: [
+            `✓ Adapter '${config.adapter.name}' streaming InputStreamNode`,
+            `  • Schemas: ${schemaCount} (threshold: ${STREAM_SCHEMA_THRESHOLD})`,
+            `  • Operations: ${operationCount}`,
+          ],
+        })
+      } else {
+        driver.inputNode = await config.adapter.parse(source)
+
+        await hooks.emit('kubb:debug', {
+          date: new Date(),
+          logs: [
+            `✓ Adapter '${config.adapter.name}' resolved InputNode`,
+            `  • Schemas: ${driver.inputNode.schemas.length}`,
+            `  • Operations: ${driver.inputNode.operations.length}`,
+          ],
+        })
+      }
+    } else {
+      driver.inputNode = await config.adapter.parse(source)
+
+      await hooks.emit('kubb:debug', {
+        date: new Date(),
+        logs: [
+          `✓ Adapter '${config.adapter.name}' resolved InputNode`,
+          `  • Schemas: ${driver.inputNode.schemas.length}`,
+          `  • Operations: ${driver.inputNode.operations.length}`,
+        ],
+      })
+    }
   }
 
   return {
@@ -965,6 +994,156 @@ async function setup(userConfig: UserConfig, options: SetupOptions = {}): Promis
  * schemas that fall outside that set. This ensures that component schemas referenced
  * exclusively by excluded operations are not generated.
  */
+type PluginStreamEntry = {
+  plugin: NormalizedPlugin
+  context: GeneratorContext
+  hrStart: ReturnType<typeof process.hrtime>
+}
+
+/**
+ * Single-pass fan-out for streaming mode.
+ *
+ * Iterates `inputStreamNode.schemas` and `.operations` exactly once, distributing each
+ * node to every plugin in parallel. This replaces the N-pass-per-plugin pattern (where
+ * each plugin got its own `for await` iterator) with a single parse pass fanned to all
+ * plugins — eliminating the N×parse-time overhead for multi-plugin builds.
+ *
+ * Each plugin still gets independent `plugin:start` / `plugin:end` events and its own
+ * timing, but the schema and operation nodes are parsed only once total.
+ */
+async function runPluginStreamHooks(
+  inputStreamNode: InputStreamNode,
+  entries: PluginStreamEntry[],
+  driver: PluginDriver,
+  hooks: AsyncEventEmitter<KubbHooks>,
+  config: Config,
+  pluginTimings: Map<string, number>,
+  failedPlugins: Set<{ plugin: Plugin; error: Error }>,
+  flushPendingFiles: () => Promise<void>,
+): Promise<void> {
+  type PluginState = {
+    plugin: NormalizedPlugin
+    generatorContext: GeneratorContext
+    generators: Generator[]
+    hrStart: ReturnType<typeof process.hrtime>
+    failed: boolean
+    error: Error | undefined
+  }
+
+  function resolveRendererFor(gen: Generator, state: PluginState): RendererFactory | undefined {
+    return gen.renderer === null ? undefined : (gen.renderer ?? state.plugin.renderer ?? state.generatorContext.config.renderer)
+  }
+
+  const states: PluginState[] = entries.map(({ plugin, context, hrStart }) => ({
+    plugin,
+    generatorContext: { ...context, resolver: driver.getResolver(plugin.name) },
+    generators: plugin.generators ?? [],
+    hrStart,
+    failed: false,
+    error: undefined,
+  }))
+
+  let schemasProcessed = 0
+  for await (const node of inputStreamNode.schemas) {
+    for (const state of states) {
+      if (state.failed) continue
+      try {
+        const { plugin, generatorContext, generators } = state
+        const { exclude, include, override } = plugin.options
+        const transformedNode = plugin.transformer ? transform(node, plugin.transformer) : node
+        const options = generatorContext.resolver.resolveOptions(transformedNode, { options: plugin.options, exclude, include, override })
+        if (options === null) continue
+
+        const ctx = { ...generatorContext, options }
+        for (const gen of generators) {
+          if (!gen.schema) continue
+          const result = await gen.schema(transformedNode, ctx)
+          await applyHookResult(result, driver, resolveRendererFor(gen, state))
+        }
+        await driver.hooks.emit('kubb:generate:schema', transformedNode, ctx)
+      } catch (caughtError) {
+        state.failed = true
+        state.error = caughtError as Error
+      }
+    }
+    schemasProcessed++
+    if (schemasProcessed % STREAM_FLUSH_EVERY === 0) {
+      await flushPendingFiles()
+    }
+  }
+
+  const collectedOperations: OperationNode[] = []
+  for await (const node of inputStreamNode.operations) {
+    collectedOperations.push(node)
+    for (const state of states) {
+      if (state.failed) continue
+      try {
+        const { plugin, generatorContext, generators } = state
+        const { exclude, include, override } = plugin.options
+        const transformedNode = plugin.transformer ? transform(node, plugin.transformer) : node
+        const options = generatorContext.resolver.resolveOptions(transformedNode, { options: plugin.options, exclude, include, override })
+        if (options === null) continue
+
+        const ctx = { ...generatorContext, options }
+        for (const gen of generators) {
+          if (!gen.operation) continue
+          const result = await gen.operation(transformedNode, ctx)
+          await applyHookResult(result, driver, resolveRendererFor(gen, state))
+        }
+        await driver.hooks.emit('kubb:generate:operation', transformedNode, ctx)
+      } catch (caughtError) {
+        state.failed = true
+        state.error = caughtError as Error
+      }
+    }
+  }
+
+  // After stream: gen.operations for each plugin, then emit plugin:end
+  for (const state of states) {
+    if (!state.failed) {
+      try {
+        const { plugin, generatorContext, generators } = state
+        const ctx = { ...generatorContext, options: plugin.options }
+        for (const gen of generators) {
+          if (!gen.operations) continue
+          const result = await gen.operations(collectedOperations, ctx)
+          await applyHookResult(result, driver, resolveRendererFor(gen, state))
+        }
+        await driver.hooks.emit('kubb:generate:operations', collectedOperations, ctx)
+      } catch (caughtError) {
+        state.failed = true
+        state.error = caughtError as Error
+      }
+    }
+
+    const duration = getElapsedMs(state.hrStart)
+    pluginTimings.set(state.plugin.name, duration)
+
+    await hooks.emit('kubb:plugin:end', {
+      plugin: state.plugin,
+      duration,
+      success: !state.failed,
+      ...(state.failed && state.error ? { error: state.error } : {}),
+      config,
+      get files() {
+        return driver.fileManager.files
+      },
+      upsertFile: (...files) => driver.fileManager.upsert(...files),
+    })
+
+    if (state.failed && state.error) {
+      failedPlugins.add({ plugin: state.plugin, error: state.error })
+    }
+
+    await hooks.emit('kubb:debug', {
+      date: new Date(),
+      logs: [state.failed ? '✗ Plugin start failed' : `✓ Plugin started successfully (${formatMs(duration)})`],
+    })
+  }
+
+  await flushPendingFiles()
+}
+
 async function runPluginAstHooks(plugin: NormalizedPlugin, context: GeneratorContext): Promise<void> {
   const { adapter, inputNode, resolver, driver } = context
   const { exclude, include, override } = plugin.options
@@ -985,6 +1164,7 @@ async function runPluginAstHooks(plugin: NormalizedPlugin, context: GeneratorCon
     resolver: driver.getResolver(plugin.name),
   }
 
+  // ── BATCH PATH ────────────────────────────────────────────────────────────
   // When `include` has operation-based filters (tag, operationId, path, method, contentType)
   // but no schema-level filters (schemaName), pre-compute the set of top-level schema names
   // that are transitively referenced by the included operations. Schemas outside that set are
@@ -995,11 +1175,11 @@ async function runPluginAstHooks(plugin: NormalizedPlugin, context: GeneratorCon
 
   let allowedSchemaNames: Set<string> | undefined
   if (hasOperationBasedIncludes && !hasSchemaNameIncludes) {
-    const includedOps = inputNode.operations.filter((op) => resolver.resolveOptions(op, { options: plugin.options, exclude, include, override }) !== null)
-    allowedSchemaNames = collectUsedSchemaNames(includedOps, inputNode.schemas)
+    const includedOps = inputNode!.operations.filter((op) => resolver.resolveOptions(op, { options: plugin.options, exclude, include, override }) !== null)
+    allowedSchemaNames = collectUsedSchemaNames(includedOps, inputNode!.schemas)
   }
 
-  await walk(inputNode, {
+  await walk(inputNode!, {
     depth: 'shallow',
     async schema(node) {
       const transformedNode = plugin.transformer ? transform(node, plugin.transformer) : node
@@ -1081,34 +1261,8 @@ async function safeBuild(setupResult: SetupResult): Promise<BuildOutput> {
   }
   const fileProcessor = new FileProcessor()
 
-  fileProcessor.events.on('start', async (processingFiles) => {
-    await hooks.emit('kubb:files:processing:start', { files: processingFiles })
-  })
-
-  fileProcessor.events.on('update', async ({ file, source, processed, total, percentage }) => {
-    await hooks.emit('kubb:file:processing:update', {
-      file,
-      source,
-      processed,
-      total,
-      percentage,
-      config,
-    })
-    if (source) {
-      await storage.setItem(file.path, source)
-    }
-  })
-
-  fileProcessor.events.on('end', async (processed) => {
-    await hooks.emit('kubb:files:processing:end', { files: processed })
-    await hooks.emit('kubb:debug', {
-      date: new Date(),
-      logs: [`✓ File write process completed for ${processed.length} files`],
-    })
-  })
-
-  async function flushPendingFiles(): Promise<void> {
-    const files = driver.fileManager.files.filter((f) => !writtenPaths.has(f.path))
+  async function flushPendingFiles(snapshot?: ReadonlySet<string>): Promise<void> {
+    const files = driver.fileManager.files.filter((f) => !writtenPaths.has(f.path) && (!snapshot || !snapshot.has(f.path)))
     if (files.length === 0) {
       return
     }
@@ -1118,25 +1272,33 @@ async function safeBuild(setupResult: SetupResult): Promise<BuildOutput> {
       logs: [`Writing ${files.length} files...`],
     })
 
-    await fileProcessor.run(files, {
-      parsers: parsersMap,
-      mode: 'parallel',
-      extension: config.output.extension,
-    })
+    await hooks.emit('kubb:files:processing:start', { files })
 
-    for (const file of files) {
+    const stream = fileProcessor.stream(files, { parsers: parsersMap, extension: config.output.extension })
+
+    for await (const { file, source, processed, total, percentage } of stream) {
+      await hooks.emit('kubb:file:processing:update', { file, source, processed, total, percentage, config })
+      if (source) {
+        await storage.setItem(file.path, source)
+      }
       writtenPaths.add(file.path)
     }
+
+    await hooks.emit('kubb:files:processing:end', { files })
+    await hooks.emit('kubb:debug', {
+      date: new Date(),
+      logs: [`✓ File write process completed for ${files.length} files`],
+    })
   }
 
   try {
     await driver.emitSetupHooks()
 
-    if (driver.adapter && driver.inputNode) {
+    if (driver.adapter && (driver.inputNode || driver.inputStreamNode)) {
       await hooks.emit('kubb:build:start', {
         config,
         adapter: driver.adapter,
-        inputNode: driver.inputNode,
+        inputNode: driver.inputNode ?? { kind: 'Input' as const, schemas: [], operations: [], meta: driver.inputStreamNode?.meta },
         getPlugin: driver.getPlugin.bind(driver),
         get files() {
           return driver.fileManager.files
@@ -1145,70 +1307,117 @@ async function safeBuild(setupResult: SetupResult): Promise<BuildOutput> {
       })
     }
 
-    for (const plugin of driver.plugins.values()) {
-      const context = driver.getContext(plugin)
-      const hrStart = process.hrtime()
+    const inputStreamNode = driver.inputStreamNode
+    if (inputStreamNode) {
+      // ── STREAMING: fan-out single-pass ────────────────────────────────────
+      // Emit plugin:start for all plugins up front, collect generator-plugins
+      // for the fan-out pass, then handle non-generator plugins immediately.
+      const streamPluginEntries: PluginStreamEntry[] = []
 
-      try {
-        const timestamp = new Date()
+      for (const plugin of driver.plugins.values()) {
+        const context = driver.getContext(plugin)
+        const hrStart = process.hrtime()
 
         await hooks.emit('kubb:plugin:start', { plugin })
         await hooks.emit('kubb:debug', {
-          date: timestamp,
+          date: new Date(),
           logs: ['Starting plugin...', `  • Plugin Name: ${plugin.name}`],
         })
 
         if (plugin.generators?.length || driver.hasRegisteredGenerators(plugin.name)) {
-          await runPluginAstHooks(plugin, context)
+          streamPluginEntries.push({ plugin, context, hrStart })
+        } else {
+          // No generators: plugin ran via setup hooks; finish it now.
+          const duration = getElapsedMs(hrStart)
+          pluginTimings.set(plugin.name, duration)
+          await hooks.emit('kubb:plugin:end', {
+            plugin,
+            duration,
+            success: true,
+            config,
+            get files() {
+              return driver.fileManager.files
+            },
+            upsertFile: (...files) => driver.fileManager.upsert(...files),
+          })
+          await hooks.emit('kubb:debug', {
+            date: new Date(),
+            logs: [`✓ Plugin started successfully (${formatMs(duration)})`],
+          })
         }
+      }
 
-        const duration = getElapsedMs(hrStart)
-        pluginTimings.set(plugin.name, duration)
-
-        await hooks.emit('kubb:plugin:end', {
-          plugin,
-          duration,
-          success: true,
-          config,
-          get files() {
-            return driver.fileManager.files
-          },
-          upsertFile: (...files) => driver.fileManager.upsert(...files),
-        })
-
-        await hooks.emit('kubb:debug', {
-          date: new Date(),
-          logs: [`✓ Plugin started successfully (${formatMs(duration)})`],
-        })
-      } catch (caughtError) {
-        const error = caughtError as Error
-        const errorTimestamp = new Date()
-        const duration = getElapsedMs(hrStart)
-
-        await hooks.emit('kubb:plugin:end', {
-          plugin,
-          duration,
-          success: false,
-          error,
-          config,
-          get files() {
-            return driver.fileManager.files
-          },
-          upsertFile: (...files) => driver.fileManager.upsert(...files),
-        })
-
-        await hooks.emit('kubb:debug', {
-          date: errorTimestamp,
-          logs: [
-            '✗ Plugin start failed',
-            `  • Plugin Name: ${plugin.name}`,
-            `  • Error: ${error.constructor.name} - ${error.message}`,
-            '  • Stack Trace:',
-            error.stack || 'No stack trace available',
-          ],
-        })
-
-        failedPlugins.add({ plugin, error })
+      if (streamPluginEntries.length > 0) {
+        await runPluginStreamHooks(inputStreamNode, streamPluginEntries, driver, hooks, config, pluginTimings, failedPlugins, flushPendingFiles)
+      }
+    } else {
+      // ── BATCH: existing per-plugin sequential loop ────────────────────────
+      for (const plugin of driver.plugins.values()) {
+        const context = driver.getContext(plugin)
+        const hrStart = process.hrtime()
+
+        try {
+          const timestamp = new Date()
+
+          await hooks.emit('kubb:plugin:start', { plugin })
+          await hooks.emit('kubb:debug', {
+            date: timestamp,
+            logs: ['Starting plugin...', `  • Plugin Name: ${plugin.name}`],
+          })
+
+          if (plugin.generators?.length || driver.hasRegisteredGenerators(plugin.name)) {
+            await runPluginAstHooks(plugin, context)
+          }
+
+          const duration = getElapsedMs(hrStart)
+          pluginTimings.set(plugin.name, duration)
+
+          await hooks.emit('kubb:plugin:end', {
+            plugin,
+            duration,
+            success: true,
+            config,
+            get files() {
+              return driver.fileManager.files
+            },
+            upsertFile: (...files) => driver.fileManager.upsert(...files),
+          })
+
+          await hooks.emit('kubb:debug', {
+            date: new Date(),
+            logs: [`✓ Plugin started successfully (${formatMs(duration)})`],
+          })
+        } catch (caughtError) {
+          const error = caughtError as Error
+          const errorTimestamp = new Date()
+          const duration = getElapsedMs(hrStart)
+
+          await hooks.emit('kubb:plugin:end', {
+            plugin,
+            duration,
+            success: false,
+            error,
+            config,
+            get files() {
+              return driver.fileManager.files
+            },
+            upsertFile: (...files) => driver.fileManager.upsert(...files),
+          })
+
+          await hooks.emit('kubb:debug', {
+            date: errorTimestamp,
+            logs: [
+              '✗ Plugin start failed',
+              `  • Plugin Name: ${plugin.name}`,
+              `  • Error: ${error.constructor.name} - ${error.message}`,
+              '  • Stack Trace:',
+              error.stack || 'No stack trace available',
+            ],
+          })
+
+          failedPlugins.add({ plugin, error })
+        }
+        await flushPendingFiles()
       }
     }
 

package/src/defineGenerator.ts

@@ -160,7 +160,10 @@ export type Generator<TOptions extends PluginFactoryOptions = PluginFactoryOptio
    * `ctx` carries the plugin context with `adapter` and `inputNode` guaranteed present,
    * plus `ctx.options` with the plugin-level options for the batch call.
    */
-  operations?: (nodes: Array<OperationNode>, ctx: GeneratorContext<TOptions>) => PossiblePromise<TElement | Array<FileNode> | void>
+  operations?: (
+    nodes: Array<OperationNode> | AsyncIterable<OperationNode>,
+    ctx: GeneratorContext<TOptions>,
+  ) => PossiblePromise<TElement | Array<FileNode> | void>
 }
 
 /**

package/src/types.ts

@@ -32,7 +32,7 @@ export type {
 } from './createKubb.ts'
 export type { Renderer, RendererFactory } from './createRenderer.ts'
 export type { Storage } from './createStorage.ts'
-export type { FileProcessorEvents } from './FileProcessor.ts'
+export type { FileProcessorEvents, ParsedFile } from './FileProcessor.ts'
 export type { Generator, GeneratorContext } from './defineGenerator.ts'
 export type { Logger, LoggerContext, LoggerOptions, UserLogger } from './defineLogger.ts'
 export type { Middleware } from './defineMiddleware.ts'