@kubb/core 5.0.0-beta.6 → 5.0.0-beta.60

package/package.json

@@ -1,20 +1,13 @@
 {
   "name": "@kubb/core",
-  "version": "5.0.0-beta.6",
-  "description": "Core functionality for Kubb's plugin-based code generation system, providing the foundation for transforming OpenAPI specifications.",
+  "version": "5.0.0-beta.60",
+  "description": "Core engine for Kubb's plugin-based code generation system. Provides the plugin driver, file manager, defineConfig, and build orchestration used by every Kubb plugin.",
   "keywords": [
-    "ast",
     "code-generator",
     "codegen",
-    "core-library",
-    "file-system",
     "kubb",
-    "oas",
     "openapi",
-    "plugin-framework",
     "plugin-system",
-    "plugins",
-    "swagger",
     "typescript"
   ],
   "license": "MIT",
@@ -63,39 +56,26 @@
     "registry": "https://registry.npmjs.org/"
   },
   "dependencies": {
-    "fflate": "^0.8.2",
-    "tinyexec": "^1.1.2",
-    "@kubb/ast": "5.0.0-beta.6"
+    "@kubb/ast": "5.0.0-beta.60"
   },
   "devDependencies": {
-    "p-limit": "^7.3.0",
     "@internals/utils": "0.0.0",
-    "@kubb/renderer-jsx": "5.0.0-beta.6"
+    "@kubb/renderer-jsx": "5.0.0-beta.60"
   },
   "peerDependencies": {
-    "@kubb/renderer-jsx": "5.0.0-beta.6"
+    "@kubb/renderer-jsx": "5.0.0-beta.60"
   },
-  "size-limit": [
-    {
-      "path": "./dist/*.js",
-      "limit": "510 KiB",
-      "gzip": true
-    }
-  ],
   "engines": {
     "node": ">=22"
   },
-  "inlinedDependencies": {
-    "p-limit": "7.3.0",
-    "yocto-queue": "1.2.2"
-  },
   "scripts": {
-    "build": "tsdown && size-limit",
-    "clean": "npx rimraf ./dist",
+    "build": "tsdown",
+    "clean": "node -e \"require('node:fs').rmSync('./dist', {recursive:true,force:true})\"",
     "lint": "oxlint .",
     "lint:fix": "oxlint --fix .",
     "release": "pnpm publish --no-git-check",
     "release:canary": "bash ../../.github/canary.sh && node ../../scripts/build.js canary && pnpm publish --no-git-check",
+    "release:stage": "pnpm stage publish --no-git-check",
     "start": "tsdown --watch",
     "test": "vitest --passWithNoTests",
     "typecheck": "tsc -p ./tsconfig.json --noEmit --emitDeclarationOnly false"

package/src/FileManager.ts

@@ -1,80 +1,103 @@
+import { AsyncEventEmitter } from '@internals/utils'
 import type { FileNode } from '@kubb/ast'
-import { createFile } from '@kubb/ast'
+import * as factory from '@kubb/ast/factory'
+
+/**
+ * Hooks fired by a `FileManager`.
+ *
+ * - `upsert` fires once per resolved file added through `add` or `upsert`.
+ */
+export type FileManagerHooks = {
+  upsert: [file: FileNode]
+}
 
 function mergeFile<TMeta extends object = object>(a: FileNode<TMeta>, b: FileNode<TMeta>): FileNode<TMeta> {
   return {
     ...a,
-    // Incoming file (b) takes precedence for banner/footer so that barrel files,
-    // which never carry a banner, can clear banners set by plugin-generated files
-    // at the same path.
+    // Incoming file (b) takes precedence for banner/footer so a barrel file (whose
+    // banner/footer the barrel plugin resolves last) wins over a plugin-generated
+    // file at the same path.
     banner: b.banner,
     footer: b.footer,
-    sources: [...(a.sources || []), ...(b.sources || [])],
-    imports: [...(a.imports || []), ...(b.imports || [])],
-    exports: [...(a.exports || []), ...(b.exports || [])],
+    sources: a.sources.length ? (b.sources.length ? [...a.sources, ...b.sources] : a.sources) : b.sources,
+    imports: a.imports.length ? (b.imports.length ? [...a.imports, ...b.imports] : a.imports) : b.imports,
+    exports: a.exports.length ? (b.exports.length ? [...a.exports, ...b.exports] : a.exports) : b.exports,
   }
 }
 
-/**
- * Collapses a list of files so that duplicates sharing the same `path` are merged
- * in arrival order. Keeps the original order of first occurrence.
- */
-function mergeFilesByPath(files: ReadonlyArray<FileNode>): Map<string, FileNode> {
-  const merged = new Map<string, FileNode>()
-  for (const file of files) {
-    const existing = merged.get(file.path)
-    merged.set(file.path, existing ? mergeFile(existing, file) : file)
-  }
-  return merged
+function isIndexPath(path: string): boolean {
+  return path.endsWith('/index.ts') || path === 'index.ts'
+}
+
+// Sort order: shortest path first. Within a length bucket, index.ts barrels last.
+function compareFiles(a: FileNode, b: FileNode): number {
+  const lenDiff = a.path.length - b.path.length
+  if (lenDiff !== 0) return lenDiff
+  const aIsIndex = isIndexPath(a.path)
+  const bIsIndex = isIndexPath(b.path)
+  if (aIsIndex && !bIsIndex) return 1
+  if (!aIsIndex && bIsIndex) return -1
+  return 0
 }
 
 /**
- * In-memory file store for generated files.
- *
- * Files with the same `path` are merged — sources, imports, and exports are concatenated.
- * The `files` getter returns all stored files sorted by path length (shortest first).
+ * In-memory file store for generated files. Files sharing a `path` are merged
+ * (sources/imports/exports concatenated). The `files` getter is sorted by
+ * path length (barrel `index.ts` last within a bucket).
  *
  * @example
  * ```ts
- * import { FileManager } from '@kubb/core'
- *
  * const manager = new FileManager()
  * manager.upsert(myFile)
- * console.log(manager.files) // all stored files
+ * manager.files // sorted view
  * ```
  */
 export class FileManager {
-  readonly #cache = new Map<string, FileNode>()
-  #filesCache: Array<FileNode> | null = null
-
   /**
-   * Adds one or more files. Incoming files with the same path are merged
-   * (sources/imports/exports concatenated), but existing cache entries are
-   * replaced — use {@link upsert} when you want to merge into the cache too.
+   * Subscribe to file-store changes. Listeners on `upsert` see each resolved file as it lands
+   * through `add` or `upsert`.
    */
+  readonly hooks = new AsyncEventEmitter<FileManagerHooks>()
+  readonly #cache = new Map<string, FileNode>()
+  // Cached sorted view. Null means stale and rebuilt lazily on next `files` read.
+  // Nulled (not mutated) on every write so callers holding a prior reference
+  // keep their snapshot, `dispose()` must not silently empty an array the
+  // consumer already holds.
+  #sorted: Array<FileNode> | null = null
+
   add(...files: Array<FileNode>): Array<FileNode> {
     return this.#store(files, false)
   }
 
-  /**
-   * Adds or merges one or more files.
-   * If a file with the same path already exists in the cache, its
-   * sources/imports/exports are merged into the incoming file.
-   */
   upsert(...files: Array<FileNode>): Array<FileNode> {
     return this.#store(files, true)
   }
 
   #store(files: ReadonlyArray<FileNode>, mergeExisting: boolean): Array<FileNode> {
-    const resolvedFiles: Array<FileNode> = []
-    for (const file of mergeFilesByPath(files).values()) {
-      const existing = mergeExisting ? this.#cache.get(file.path) : undefined
-      const resolvedFile = createFile(existing ? mergeFile(existing, file) : file)
-      this.#cache.set(resolvedFile.path, resolvedFile)
-      resolvedFiles.push(resolvedFile)
+    const batch = files.length > 1 ? this.#dedupe(files) : files
+    const resolved: Array<FileNode> = []
+
+    for (const file of batch) {
+      const existing = this.#cache.get(file.path)
+      const merged = existing && mergeExisting ? factory.createFile(mergeFile(existing, file)) : factory.createFile(file)
+      this.#cache.set(merged.path, merged)
+      resolved.push(merged)
+      this.hooks.emit('upsert', merged)
+    }
+
+    if (resolved.length > 0) this.#sorted = null
+    return resolved
+  }
+
+  // Merges same-path entries within a batch so the cache update loop stays
+  // uniform. Only called for multi-file batches.
+  #dedupe(files: ReadonlyArray<FileNode>): Array<FileNode> {
+    const seen = new Map<string, FileNode>()
+    for (const file of files) {
+      const prev = seen.get(file.path)
+      seen.set(file.path, prev ? mergeFile(prev, file) : file)
     }
-    this.#filesCache = null
-    return resolvedFiles
+    return [...seen.values()]
   }
 
   getByPath(path: string): FileNode | null {
@@ -82,34 +105,33 @@ export class FileManager {
   }
 
   deleteByPath(path: string): void {
-    this.#cache.delete(path)
-    this.#filesCache = null
+    if (!this.#cache.delete(path)) return
+    this.#sorted = null
   }
 
   clear(): void {
     this.#cache.clear()
-    this.#filesCache = null
+    this.#sorted = null
   }
 
   /**
-   * All stored files, sorted by path length (shorter paths first).
+   * Releases all stored files and clears every `hooks` listener. Called by the core after
+   * `kubb:build:end`.
    */
-  get files(): Array<FileNode> {
-    if (this.#filesCache) {
-      return this.#filesCache
-    }
+  dispose(): void {
+    this.clear()
+    this.hooks.removeAll()
+  }
+
+  [Symbol.dispose](): void {
+    this.dispose()
+  }
 
-    this.#filesCache = [...this.#cache.values()].sort((a, b) => {
-      const lenDiff = a.path.length - b.path.length
-      if (lenDiff !== 0) return lenDiff
-      // Within the same length bucket, index.ts barrel files go last so other
-      // files are always processed before their barrel file.
-      const aIsIndex = a.path.endsWith('/index.ts') || a.path === 'index.ts'
-      const bIsIndex = b.path.endsWith('/index.ts') || b.path === 'index.ts'
-      if (aIsIndex && !bIsIndex) return 1
-      if (!aIsIndex && bIsIndex) return -1
-      return 0
-    })
-    return this.#filesCache
+  /**
+   * All stored files in stable sort order (shortest path first, barrel files
+   * last within a length bucket). Returns a cached view, do not mutate.
+   */
+  get files(): Array<FileNode> {
+    return (this.#sorted ??= [...this.#cache.values()].sort(compareFiles))
   }
 }

package/src/FileProcessor.ts

@@ -1,42 +1,102 @@
+import { AsyncEventEmitter } from '@internals/utils'
 import type { CodeNode, FileNode } from '@kubb/ast'
-import { extractStringsFromNodes } from '@kubb/ast'
-import pLimit from 'p-limit'
-import { PARALLEL_CONCURRENCY_LIMIT } from './constants.ts'
+import { extractStringsFromNodes } from '@kubb/ast/utils'
+import { STREAM_FLUSH_EVERY } from './constants.ts'
+import type { Storage } from './createStorage.ts'
 import type { Parser } from './defineParser.ts'
 
-type ParseOptions = {
-  parsers?: Map<FileNode['extname'], Parser>
-  extension?: Record<FileNode['extname'], FileNode['extname'] | ''>
+/**
+ * Hooks fired by a `FileProcessor`.
+ *
+ * - `start` opens a batch, from `run` or a queue flush.
+ * - `update` fires once per file as it is converted.
+ * - `end` closes a batch.
+ * - `enqueue` fires for every `enqueue` call.
+ * - `drain` fires when `drain()` empties the queue with no in-flight batch left.
+ */
+export type FileProcessorHooks = {
+  start: [files: Array<FileNode>]
+  update: [params: { file: FileNode; source?: string; processed: number; total: number; percentage: number }]
+  end: [files: Array<FileNode>]
+  enqueue: [file: FileNode]
+  drain: []
 }
 
-type RunOptions = ParseOptions & {
+/**
+ * Per-file progress record yielded by `stream` and surfaced through the `update` event.
+ */
+export type ParsedFile = {
+  file: FileNode
+  source: string
+  processed: number
+  total: number
+  percentage: number
+}
+
+type FileProcessorOptions = {
+  /**
+   * Storage destination for queued writes.
+   */
+  storage: Storage
   /**
-   * @default 'sequential'
+   * Parsers indexed by file extension.
    */
-  mode?: 'sequential' | 'parallel'
-  onStart?: (files: Array<FileNode>) => Promise<void> | void
-  onEnd?: (files: Array<FileNode>) => Promise<void> | void
-  onUpdate?: (params: { file: FileNode; source?: string; processed: number; total: number; percentage: number }) => Promise<void> | void
+  parsers?: Map<FileNode['extname'], Parser>
+  /**
+   * Output extname per source extname, applied during conversion.
+   */
+  extension?: Record<FileNode['extname'], FileNode['extname'] | ''>
 }
 
 function joinSources(file: FileNode): string {
-  return file.sources
-    .map((item) => extractStringsFromNodes(item.nodes as Array<CodeNode>))
-    .filter(Boolean)
-    .join('\n\n')
+  const sources = file.sources
+  if (sources.length === 0) return ''
+  const parts: Array<string> = []
+  for (const source of sources) {
+    const text = extractStringsFromNodes(source.nodes as Array<CodeNode>)
+    if (text) parts.push(text)
+  }
+  return parts.join('\n\n')
 }
 
 /**
- * Converts a single file to a string using the registered parsers.
- * Falls back to joining source values when no matching parser is found.
+ * Turns `FileNode`s into source strings and writes them to storage.
+ *
+ * Two modes share the same instance. Stateless mode (`parse`, `stream`, `run`) just runs the
+ * conversion. Queue mode (`enqueue`, `flush`, `drain`) buffers files deduped by path and
+ * writes each batch through storage with up to `STREAM_FLUSH_EVERY` requests in flight.
  *
- * @internal
+ * `flush` does not wait for its batch to finish, so dispatch can overlap with IO. The next
+ * `flush` or `drain` picks the in-flight batch up. `drain` blocks until everything has been
+ * written and is meant for the end of a build.
+ *
+ * To surface build-level hook signals (`kubb:files:processing:*` and friends) subscribe to
+ * `hooks` and re-emit on the kubb bus.
  */
 export class FileProcessor {
-  readonly #limit = pLimit(PARALLEL_CONCURRENCY_LIMIT)
+  readonly hooks = new AsyncEventEmitter<FileProcessorHooks>()
+  readonly #parsers: Map<FileNode['extname'], Parser> | null
+  readonly #storage: Storage
+  readonly #extension: Record<FileNode['extname'], FileNode['extname'] | ''> | null
+  readonly #pending = new Map<string, FileNode>()
+  #runningFlush: Promise<void> | null = null
+
+  constructor(options: FileProcessorOptions) {
+    this.#parsers = options.parsers ?? null
+    this.#storage = options.storage
+    this.#extension = options.extension ?? null
+  }
+
+  /**
+   * Files waiting in the queue.
+   */
+  get size(): number {
+    return this.#pending.size
+  }
 
-  async parse(file: FileNode, { parsers, extension }: ParseOptions = {}): Promise<string> {
-    const parseExtName = extension?.[file.extname] || undefined
+  parse(file: FileNode): string {
+    const parsers = this.#parsers
+    const parseExtName = this.#extension?.[file.extname] || undefined
 
     if (!parsers || !file.extname) {
       return joinSources(file)
@@ -51,36 +111,102 @@ export class FileProcessor {
     return parser.parse(file, { extname: parseExtName })
   }
 
-  async run(files: Array<FileNode>, { parsers, mode = 'sequential', extension, onStart, onEnd, onUpdate }: RunOptions = {}): Promise<Array<FileNode>> {
-    await onStart?.(files)
-
+  *stream(files: ReadonlyArray<FileNode>): Generator<ParsedFile> {
     const total = files.length
+    if (total === 0) return
+
     let processed = 0
+    for (const file of files) {
+      const source = this.parse(file)
+      processed++
 
-    const processOne = async (file: FileNode) => {
-      const source = await this.parse(file, { extension, parsers })
-      const currentProcessed = ++processed
-      const percentage = (currentProcessed / total) * 100
-
-      await onUpdate?.({
-        file,
-        source,
-        processed: currentProcessed,
-        percentage,
-        total,
-      })
+      yield { file, source, processed, total, percentage: (processed / total) * 100 }
     }
+  }
 
-    if (mode === 'sequential') {
-      for (const file of files) {
-        await processOne(file)
-      }
-    } else {
-      await Promise.all(files.map((file) => this.#limit(() => processOne(file))))
+  async run(files: Array<FileNode>): Promise<Array<FileNode>> {
+    await this.hooks.emit('start', files)
+
+    for (const { file, source, processed, total, percentage } of this.stream(files)) {
+      await this.hooks.emit('update', { file, source, processed, percentage, total })
     }
 
-    await onEnd?.(files)
+    await this.hooks.emit('end', files)
 
     return files
   }
+
+  /**
+   * Adds a file to the next flush. A later `enqueue` for the same path replaces the previous
+   * entry, matching `FileManager.upsert`. Fires the `enqueue` event.
+   */
+  enqueue(file: FileNode): void {
+    this.#pending.set(file.path, file)
+    this.hooks.emit('enqueue', file)
+  }
+
+  /**
+   * Starts processing the queued files. Waits for any previous flush to finish (so two
+   * batches never run together) and then returns without waiting for the new one. The next
+   * `flush` or `drain` picks up the in-flight task.
+   */
+  async flush(): Promise<void> {
+    if (this.#runningFlush) await this.#runningFlush
+    if (this.#pending.size === 0) return
+
+    const batch = [...this.#pending.values()]
+    this.#pending.clear()
+
+    this.#runningFlush = this.#processAndWrite(batch).finally(() => {
+      this.#runningFlush = null
+    })
+  }
+
+  /**
+   * Waits for the in-flight flush and writes any files still queued. Fires the `drain` event
+   * when both are done.
+   */
+  async drain(): Promise<void> {
+    if (this.#runningFlush) await this.#runningFlush
+
+    if (this.#pending.size > 0) {
+      const batch = [...this.#pending.values()]
+      this.#pending.clear()
+      await this.#processAndWrite(batch)
+    }
+
+    await this.hooks.emit('drain')
+  }
+
+  async #processAndWrite(files: Array<FileNode>): Promise<void> {
+    const storage = this.#storage
+
+    await this.hooks.emit('start', files)
+
+    // Single pass: each file's write starts right after its `update` fires, so IO overlaps
+    // parsing and the batch never holds every rendered source in memory at once.
+    const queue: Array<Promise<void>> = []
+    for (const item of this.stream(files)) {
+      await this.hooks.emit('update', item)
+      if (item.source) {
+        queue.push(storage.setItem(item.file.path, item.source))
+        if (queue.length >= STREAM_FLUSH_EVERY) await Promise.all(queue.splice(0))
+      }
+    }
+    await Promise.all(queue)
+
+    await this.hooks.emit('end', files)
+  }
+
+  /**
+   * Clears every listener and the pending queue.
+   */
+  dispose(): void {
+    this.hooks.removeAll()
+    this.#pending.clear()
+  }
+
+  [Symbol.dispose](): void {
+    this.dispose()
+  }
 }