@kubb/core 5.0.0-beta.40 → 5.0.0-beta.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -13
- package/dist/{diagnostics-DhfW8YpT.d.ts → diagnostics-DZGgDzSv.d.ts} +183 -33
- package/dist/index.cjs +270 -34
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +16 -2
- package/dist/index.js +271 -37
- package/dist/index.js.map +1 -1
- package/dist/{memoryStorage-DTv1Kub1.js → memoryStorage-BOnaknb7.js} +170 -23
- package/dist/memoryStorage-BOnaknb7.js.map +1 -0
- package/dist/{memoryStorage-Dkxnid2K.cjs → memoryStorage-Dldu8sRT.cjs} +169 -22
- package/dist/memoryStorage-Dldu8sRT.cjs.map +1 -0
- package/dist/mocks.cjs +1 -1
- package/dist/mocks.d.ts +1 -1
- package/dist/mocks.js +1 -1
- package/package.json +4 -4
- package/src/Fingerprint.ts +98 -0
- package/src/KubbDriver.ts +102 -33
- package/src/Manifest.ts +85 -0
- package/src/Telemetry.ts +13 -1
- package/src/caches/fsCache.ts +103 -0
- package/src/createCache.ts +74 -0
- package/src/createKubb.ts +57 -4
- package/src/defineLogger.ts +10 -28
- package/src/defineParser.ts +1 -1
- package/src/index.ts +2 -0
- package/src/storages/fsStorage.ts +15 -23
- package/src/types.ts +3 -0
- package/dist/memoryStorage-DTv1Kub1.js.map +0 -1
- package/dist/memoryStorage-Dkxnid2K.cjs.map +0 -1
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
2
|
+
import { readFile } from 'node:fs/promises'
|
|
3
|
+
import { relative } from 'node:path'
|
|
4
|
+
import { URLPath } from '@internals/utils'
|
|
5
|
+
import type { AdapterSource } from './createAdapter.ts'
|
|
6
|
+
import type { Config } from './createKubb.ts'
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Computes the cache key for an incremental build. All methods are static, so call them as
|
|
10
|
+
* `Fingerprint.compute(...)` and `Fingerprint.stringify(...)`. The key holds no absolute
|
|
11
|
+
* paths or modification times, so it never depends on where the project lives on disk.
|
|
12
|
+
*/
|
|
13
|
+
export class Fingerprint {
|
|
14
|
+
/**
|
|
15
|
+
* Bumped when the snapshot format or fingerprint inputs change in an incompatible way, so stale
|
|
16
|
+
* cache entries from older Kubb builds are never reused.
|
|
17
|
+
*/
|
|
18
|
+
static version = 1
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Deterministically serializes a value to JSON: object keys are sorted recursively and
|
|
22
|
+
* `undefined` values and functions are dropped. Two structurally equal configs produce the same
|
|
23
|
+
* string regardless of key order, which keeps the fingerprint stable across machines.
|
|
24
|
+
*/
|
|
25
|
+
static stringify(value: unknown): string {
|
|
26
|
+
return JSON.stringify(Fingerprint.#normalize(value))
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Computes a cache key from everything that affects the generated output: the spec content, the
|
|
31
|
+
* output-shaping config, each plugin's name and options, the middleware names, the running
|
|
32
|
+
* `@kubb/core` version, and the cache format version. Returns `null` when the input can't be
|
|
33
|
+
* fingerprinted (remote URL or no adapter source), which disables caching for that build.
|
|
34
|
+
*/
|
|
35
|
+
static async compute({ config, adapterSource, version }: { config: Config; adapterSource: AdapterSource | null; version: string }): Promise<string | null> {
|
|
36
|
+
if (!adapterSource) {
|
|
37
|
+
return null
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const spec = await Fingerprint.#readSpec(adapterSource, config.root)
|
|
41
|
+
if (spec === null) {
|
|
42
|
+
return null
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const input = {
|
|
46
|
+
cacheVersion: Fingerprint.version,
|
|
47
|
+
version,
|
|
48
|
+
spec,
|
|
49
|
+
name: config.name,
|
|
50
|
+
output: config.output,
|
|
51
|
+
adapter: config.adapter?.name,
|
|
52
|
+
parsers: config.parsers.map((parser) => parser.name),
|
|
53
|
+
plugins: config.plugins.map((plugin) => ({ name: plugin.name, options: plugin.options })),
|
|
54
|
+
middleware: (config.middleware ?? []).map((middleware) => middleware.name),
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
return createHash('sha256').update(Fingerprint.stringify(input)).digest('hex')
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
static #normalize(value: unknown): unknown {
|
|
61
|
+
if (value === null || typeof value !== 'object') {
|
|
62
|
+
return typeof value === 'function' ? undefined : value
|
|
63
|
+
}
|
|
64
|
+
if (Array.isArray(value)) {
|
|
65
|
+
return value.map((item) => Fingerprint.#normalize(item))
|
|
66
|
+
}
|
|
67
|
+
const source = value as Record<string, unknown>
|
|
68
|
+
const result: Record<string, unknown> = {}
|
|
69
|
+
for (const key of Object.keys(source).sort()) {
|
|
70
|
+
const normalized = Fingerprint.#normalize(source[key])
|
|
71
|
+
if (normalized !== undefined) {
|
|
72
|
+
result[key] = normalized
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return result
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Reads the spec content that feeds the fingerprint. Returns `null` for a remote URL source
|
|
80
|
+
* (hashing remote content would mean fetching it on every run) or when a file can't be read, so a
|
|
81
|
+
* missing or virtual spec disables caching instead of failing the build.
|
|
82
|
+
*/
|
|
83
|
+
static async #readSpec(source: AdapterSource, root: string): Promise<unknown> {
|
|
84
|
+
if (source.type === 'data') {
|
|
85
|
+
return { kind: 'data', data: typeof source.data === 'string' ? source.data : Fingerprint.stringify(source.data) }
|
|
86
|
+
}
|
|
87
|
+
const paths = source.type === 'paths' ? source.paths : [source.path]
|
|
88
|
+
if (paths.some((path) => new URLPath(path).isURL)) {
|
|
89
|
+
return null
|
|
90
|
+
}
|
|
91
|
+
try {
|
|
92
|
+
const contents = await Promise.all(paths.map(async (path) => ({ path: relative(root, path), content: await readFile(path, 'utf8') })))
|
|
93
|
+
return { kind: 'path', contents }
|
|
94
|
+
} catch {
|
|
95
|
+
return null
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
package/src/KubbDriver.ts
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
|
-
import { resolve } from 'node:path'
|
|
1
|
+
import { basename, join, relative, resolve } from 'node:path'
|
|
2
2
|
import { arrayToAsyncIterable, type AsyncEventEmitter, forBatches, getElapsedMs, isPromise, memoize, URLPath } from '@internals/utils'
|
|
3
3
|
import { collectUsedSchemaNames, createFile, createStreamInput } from '@kubb/ast'
|
|
4
4
|
import type { FileNode, InputMeta, InputStreamNode, OperationNode, SchemaNode } from '@kubb/ast'
|
|
5
|
+
import { version as coreVersion } from '../package.json'
|
|
5
6
|
import { OPERATION_FILTER_TYPES, SCHEMA_PARALLEL } from './constants.ts'
|
|
7
|
+
import { Fingerprint } from './Fingerprint.ts'
|
|
6
8
|
import { type Diagnostic, Diagnostics, type ProblemDiagnostic } from './diagnostics.ts'
|
|
9
|
+
import type { Cache } from './createCache.ts'
|
|
7
10
|
import type { RendererFactory } from './createRenderer.ts'
|
|
8
11
|
import type { Storage } from './createStorage.ts'
|
|
9
12
|
import type { Generator } from './defineGenerator.ts'
|
|
@@ -105,9 +108,11 @@ export class KubbDriver {
|
|
|
105
108
|
async setup() {
|
|
106
109
|
const normalized: Array<NormalizedPlugin> = this.config.plugins.map((rawPlugin) => this.#normalizePlugin(rawPlugin as Plugin))
|
|
107
110
|
|
|
111
|
+
const dependenciesByName = new Map(normalized.map((plugin) => [plugin.name, new Set(plugin.dependencies ?? [])]))
|
|
112
|
+
|
|
108
113
|
normalized.sort((a, b) => {
|
|
109
|
-
if (b.
|
|
110
|
-
if (a.
|
|
114
|
+
if (dependenciesByName.get(b.name)?.has(a.name)) return -1
|
|
115
|
+
if (dependenciesByName.get(a.name)?.has(b.name)) return 1
|
|
111
116
|
|
|
112
117
|
return enforceOrder(a.enforce) - enforceOrder(b.enforce)
|
|
113
118
|
})
|
|
@@ -352,8 +357,12 @@ export class KubbDriver {
|
|
|
352
357
|
await hooks.emit('kubb:files:processing:start', { files })
|
|
353
358
|
})
|
|
354
359
|
const updateBuffer: Array<{ file: FileNode; source?: string; processed: number; total: number; percentage: number }> = []
|
|
360
|
+
// Final rendered source per output path, captured for the cache snapshot on a miss. Barrel
|
|
361
|
+
// files flow through here too, after the second `drain()`.
|
|
362
|
+
const snapshotSources = new Map<string, string>()
|
|
355
363
|
processor.hooks.on('update', (item) => {
|
|
356
364
|
updateBuffer.push(item)
|
|
365
|
+
if (item.source !== undefined) snapshotSources.set(item.file.path, item.source)
|
|
357
366
|
})
|
|
358
367
|
processor.hooks.on('end', async (files) => {
|
|
359
368
|
await hooks.emit('kubb:files:processing:update', {
|
|
@@ -373,6 +382,16 @@ export class KubbDriver {
|
|
|
373
382
|
(diagnostic) => diagnostics.push(diagnostic),
|
|
374
383
|
async () => {
|
|
375
384
|
try {
|
|
385
|
+
const cache = config.cache
|
|
386
|
+
const outputRoot = resolve(config.root, config.output.path)
|
|
387
|
+
const cacheKey = cache ? await Fingerprint.compute({ config, adapterSource: this.#adapterSource, version: coreVersion }) : null
|
|
388
|
+
|
|
389
|
+
// On a cache hit, restore the snapshot and skip everything below. Skipping the work is the
|
|
390
|
+
// whole point, so the only event emitted is `kubb:build:end`, which reporters key off.
|
|
391
|
+
if (cache && cacheKey && (await this.#restoreSnapshot({ cache, cacheKey, outputRoot, storage }))) {
|
|
392
|
+
return { diagnostics: Diagnostics.dedupe(diagnostics) }
|
|
393
|
+
}
|
|
394
|
+
|
|
376
395
|
// Parse the adapter source into the streaming `InputNode`.
|
|
377
396
|
await this.#parseInput()
|
|
378
397
|
// Emit `kubb:plugin:setup` so plugins can register transformers via `setTransformer`.
|
|
@@ -431,7 +450,11 @@ export class KubbDriver {
|
|
|
431
450
|
// Plugins-end listeners (barrel middleware etc.) may have queued more files.
|
|
432
451
|
await processor.drain()
|
|
433
452
|
|
|
434
|
-
await hooks.emit('kubb:build:end', { files: this.fileManager.files, config, outputDir:
|
|
453
|
+
await hooks.emit('kubb:build:end', { files: this.fileManager.files, config, outputDir: outputRoot })
|
|
454
|
+
|
|
455
|
+
if (cache && cacheKey && !Diagnostics.hasError(diagnostics)) {
|
|
456
|
+
await this.#persistSnapshot({ cache, cacheKey, outputRoot, sources: snapshotSources })
|
|
457
|
+
}
|
|
435
458
|
|
|
436
459
|
return { diagnostics: Diagnostics.dedupe(diagnostics) }
|
|
437
460
|
} catch (caughtError) {
|
|
@@ -444,6 +467,45 @@ export class KubbDriver {
|
|
|
444
467
|
)
|
|
445
468
|
}
|
|
446
469
|
|
|
470
|
+
/**
|
|
471
|
+
* Writes a restored snapshot straight to storage and emits `kubb:build:end`. Returns `true` on a
|
|
472
|
+
* hit (the build is done), `false` on a miss so the caller falls through to a full build.
|
|
473
|
+
*/
|
|
474
|
+
async #restoreSnapshot({ cache, cacheKey, outputRoot, storage }: { cache: Cache; cacheKey: string; outputRoot: string; storage: Storage }): Promise<boolean> {
|
|
475
|
+
const snapshot = await cache.restore({ key: cacheKey })
|
|
476
|
+
if (!snapshot) return false
|
|
477
|
+
|
|
478
|
+
for (const [relativePath, source] of Object.entries(snapshot.files)) {
|
|
479
|
+
const absolutePath = join(outputRoot, relativePath)
|
|
480
|
+
this.fileManager.upsert(createFile({ path: absolutePath, baseName: basename(relativePath) as `${string}.${string}` }))
|
|
481
|
+
await storage.setItem(absolutePath, source)
|
|
482
|
+
}
|
|
483
|
+
await this.hooks.emit('kubb:build:end', { files: this.fileManager.files, config: this.config, outputDir: outputRoot })
|
|
484
|
+
return true
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
/**
|
|
488
|
+
* Stores this run's rendered output, keyed by the input fingerprint, so the next unchanged build
|
|
489
|
+
* restores it instead of regenerating. `sources` is keyed by absolute path and relativized here.
|
|
490
|
+
*/
|
|
491
|
+
async #persistSnapshot({
|
|
492
|
+
cache,
|
|
493
|
+
cacheKey,
|
|
494
|
+
outputRoot,
|
|
495
|
+
sources,
|
|
496
|
+
}: {
|
|
497
|
+
cache: Cache
|
|
498
|
+
cacheKey: string
|
|
499
|
+
outputRoot: string
|
|
500
|
+
sources: ReadonlyMap<string, string>
|
|
501
|
+
}): Promise<void> {
|
|
502
|
+
const files: Record<string, string> = {}
|
|
503
|
+
for (const [absolutePath, source] of sources) {
|
|
504
|
+
files[relative(outputRoot, absolutePath)] = source
|
|
505
|
+
}
|
|
506
|
+
await cache.persist({ key: cacheKey, snapshot: { files } })
|
|
507
|
+
}
|
|
508
|
+
|
|
447
509
|
// Returns a fresh object with a lazy `files` getter and a bound `upsertFile`.
|
|
448
510
|
// Caller must use `Object.assign(extra, this.#filesPayload())`, not object spread.
|
|
449
511
|
// Spread would eagerly invoke the getter and freeze a stale snapshot into the payload.
|
|
@@ -473,6 +535,11 @@ export class KubbDriver {
|
|
|
473
535
|
* A failing plugin contributes an error diagnostic so the rest of the build continues.
|
|
474
536
|
* Every plugin also contributes a `timing` diagnostic.
|
|
475
537
|
*
|
|
538
|
+
* Plugins run sequentially so `kubb:plugin:end` fires as each plugin completes, instead
|
|
539
|
+
* of all at once after every plugin has marched through the parallel batches together.
|
|
540
|
+
* That ordering is what drives the CLI's `Plugins N/M` counter; without it the bar would
|
|
541
|
+
* sit at the initial value until the very end of the run.
|
|
542
|
+
*
|
|
476
543
|
* When `entries` is empty or `this.inputNode` is `null`, every entry still gets a
|
|
477
544
|
* `kubb:plugin:end` so middleware listeners (the barrel writer and friends) complete.
|
|
478
545
|
*/
|
|
@@ -526,22 +593,25 @@ export class KubbDriver {
|
|
|
526
593
|
|
|
527
594
|
const emitsSchemaHook = this.hooks.listenerCount('kubb:generate:schema') > 0
|
|
528
595
|
const emitsOperationHook = this.hooks.listenerCount('kubb:generate:operation') > 0
|
|
596
|
+
const emitsOperationsHook = this.hooks.listenerCount('kubb:generate:operations') > 0
|
|
597
|
+
|
|
598
|
+
// Buffer the streaming adapter's nodes once. Each plugin reads the same buffer
|
|
599
|
+
// instead of re-parsing the document per pass, and the pruning pre-scan below
|
|
600
|
+
// shares it too (previously it iterated its own copies).
|
|
601
|
+
const schemasBuffer: Array<SchemaNode> = await Array.fromAsync(schemas)
|
|
602
|
+
const operationsBuffer: Array<OperationNode> = await Array.fromAsync(operations)
|
|
529
603
|
|
|
530
604
|
// Pre-scan: plugins with operation-based includes (but no schemaName include) need
|
|
531
605
|
// the reachable schema set. This requires the full schema graph in memory at once,
|
|
532
|
-
// since transitive reachability can't be derived from a single node.
|
|
533
|
-
// released as soon as the pre-scan returns, so the main passes get fresh iterators.
|
|
606
|
+
// since transitive reachability can't be derived from a single node.
|
|
534
607
|
const pruningStates = states.filter(({ plugin }) => {
|
|
535
608
|
const { include } = plugin.options
|
|
536
609
|
return (include?.some(({ type }) => OPERATION_FILTER_TYPES.has(type)) ?? false) && !(include?.some(({ type }) => type === 'schemaName') ?? false)
|
|
537
610
|
})
|
|
538
611
|
|
|
539
612
|
if (pruningStates.length > 0) {
|
|
540
|
-
const allSchemas: Array<SchemaNode> = []
|
|
541
|
-
for await (const schema of schemas) allSchemas.push(schema)
|
|
542
|
-
|
|
543
613
|
const includedOpsByState = new Map<PluginState, Array<OperationNode>>(pruningStates.map((state) => [state, []]))
|
|
544
|
-
for
|
|
614
|
+
for (const operation of operationsBuffer) {
|
|
545
615
|
for (const state of pruningStates) {
|
|
546
616
|
const { exclude, include, override } = state.plugin.options
|
|
547
617
|
const options = state.generatorContext.resolver.resolveOptions(operation, { options: state.plugin.options, exclude, include, override })
|
|
@@ -550,7 +620,7 @@ export class KubbDriver {
|
|
|
550
620
|
}
|
|
551
621
|
|
|
552
622
|
for (const state of pruningStates) {
|
|
553
|
-
state.allowedSchemaNames = collectUsedSchemaNames(includedOpsByState.get(state) ?? [],
|
|
623
|
+
state.allowedSchemaNames = collectUsedSchemaNames(includedOpsByState.get(state) ?? [], schemasBuffer)
|
|
554
624
|
includedOpsByState.delete(state)
|
|
555
625
|
}
|
|
556
626
|
}
|
|
@@ -628,30 +698,29 @@ export class KubbDriver {
|
|
|
628
698
|
emit: emitsOperationHook ? (node: OperationNode, ctx: GeneratorContext) => this.hooks.emit('kubb:generate:operation', node, ctx) : null,
|
|
629
699
|
} as const
|
|
630
700
|
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
701
|
+
for (const state of states) {
|
|
702
|
+
// Skip building the aggregated operations array when this plugin doesn't consume it.
|
|
703
|
+
// Saves an N-sized allocation when the plugin only defines per-node `gen.operation`.
|
|
704
|
+
const needsCollectedOperations = emitsOperationsHook || state.generators.some((gen) => !!gen.operations)
|
|
705
|
+
const collectedOperations: Array<OperationNode> | undefined = needsCollectedOperations ? [] : undefined
|
|
706
|
+
|
|
707
|
+
// Run schemas before operations: the two passes share `flushPending` and the
|
|
708
|
+
// FileProcessor's event emitter, so running them concurrently would interleave
|
|
709
|
+
// `kubb:files:processing:start|end` events and race on the shared dirty list.
|
|
710
|
+
await forBatches(schemasBuffer, (nodes) => Promise.all(nodes.map((node) => dispatchNode(state, node, schemaDispatch))), {
|
|
711
|
+
concurrency: SCHEMA_PARALLEL,
|
|
712
|
+
flush: flushPending,
|
|
713
|
+
})
|
|
644
714
|
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
715
|
+
await forBatches(
|
|
716
|
+
operationsBuffer,
|
|
717
|
+
(nodes) => {
|
|
718
|
+
if (needsCollectedOperations) collectedOperations?.push(...nodes)
|
|
719
|
+
return Promise.all(nodes.map((node) => dispatchNode(state, node, operationDispatch)))
|
|
720
|
+
},
|
|
721
|
+
{ concurrency: SCHEMA_PARALLEL, flush: flushPending },
|
|
722
|
+
)
|
|
653
723
|
|
|
654
|
-
for (const state of states) {
|
|
655
724
|
if (!state.failed && needsCollectedOperations) {
|
|
656
725
|
try {
|
|
657
726
|
const { plugin, generatorContext, generators } = state
|
package/src/Manifest.ts
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { join } from 'node:path'
|
|
2
|
+
import { read } from '@internals/utils'
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Bookkeeping for one cached build: the relative paths it covers and timestamps used by the pruner.
|
|
6
|
+
*/
|
|
7
|
+
export type ManifestEntry = {
|
|
8
|
+
files: Array<string>
|
|
9
|
+
createdAt: number
|
|
10
|
+
lastAccess: number
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* The on-disk manifest: a version marker plus an entry per cached build, keyed by fingerprint.
|
|
15
|
+
*/
|
|
16
|
+
export type ManifestData = {
|
|
17
|
+
version: number
|
|
18
|
+
entries: Record<string, ManifestEntry>
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Reads and prunes the local cache manifest. All methods are static, so call them as
|
|
23
|
+
* `Manifest.read(dir)` and `Manifest.prune(data, ...)`. A damaged manifest reads as empty so the
|
|
24
|
+
* cache degrades to misses instead of throwing. Writing goes through `write` from `@internals/utils`.
|
|
25
|
+
*/
|
|
26
|
+
export class Manifest {
|
|
27
|
+
/**
|
|
28
|
+
* On-disk layout version for the manifest itself. Bumped when the manifest shape changes; a
|
|
29
|
+
* mismatch makes the whole local cache read as empty.
|
|
30
|
+
*/
|
|
31
|
+
static version = 1
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Reads the manifest at `dir/manifest.json`. A missing, corrupt, or version-mismatched file reads
|
|
35
|
+
* as an empty manifest.
|
|
36
|
+
*/
|
|
37
|
+
static async read(dir: string): Promise<ManifestData> {
|
|
38
|
+
try {
|
|
39
|
+
const parsed = JSON.parse(await read(join(dir, 'manifest.json'))) as ManifestData
|
|
40
|
+
if (parsed.version !== Manifest.version || typeof parsed.entries !== 'object') {
|
|
41
|
+
return Manifest.#empty()
|
|
42
|
+
}
|
|
43
|
+
return parsed
|
|
44
|
+
} catch {
|
|
45
|
+
return Manifest.#empty()
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Selects the keys to evict so the cache stays within `ttlDays` and `maxEntries`. Returns the
|
|
51
|
+
* surviving manifest plus the evicted keys (the caller deletes their blobs). Pure, does no IO.
|
|
52
|
+
*/
|
|
53
|
+
static prune(
|
|
54
|
+
manifest: ManifestData,
|
|
55
|
+
{ maxEntries, ttlDays, now }: { maxEntries: number; ttlDays: number; now: number },
|
|
56
|
+
): {
|
|
57
|
+
manifest: ManifestData
|
|
58
|
+
removed: Array<string>
|
|
59
|
+
} {
|
|
60
|
+
const ttlMs = ttlDays * 24 * 60 * 60 * 1000
|
|
61
|
+
const removed: Array<string> = []
|
|
62
|
+
const kept: Array<[string, ManifestEntry]> = []
|
|
63
|
+
|
|
64
|
+
for (const [key, entry] of Object.entries(manifest.entries)) {
|
|
65
|
+
if (now - entry.lastAccess > ttlMs) {
|
|
66
|
+
removed.push(key)
|
|
67
|
+
} else {
|
|
68
|
+
kept.push([key, entry])
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
if (kept.length > maxEntries) {
|
|
73
|
+
kept.sort((a, b) => b[1].lastAccess - a[1].lastAccess)
|
|
74
|
+
for (const [key] of kept.splice(maxEntries)) {
|
|
75
|
+
removed.push(key)
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
return { manifest: { version: Manifest.version, entries: Object.fromEntries(kept) }, removed }
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
static #empty(): ManifestData {
|
|
83
|
+
return { version: Manifest.version, entries: {} }
|
|
84
|
+
}
|
|
85
|
+
}
|
package/src/Telemetry.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { randomBytes } from 'node:crypto'
|
|
2
2
|
import os from 'node:os'
|
|
3
3
|
import process from 'node:process'
|
|
4
|
-
import { executeIfOnline, isCIEnvironment } from '@internals/utils'
|
|
4
|
+
import { executeIfOnline, getRuntimeName, getRuntimeVersion, isCIEnvironment, type RuntimeName } from '@internals/utils'
|
|
5
5
|
import { OTLP_ENDPOINT } from './constants.ts'
|
|
6
6
|
|
|
7
7
|
// OpenTelemetry OTLP JSON types
|
|
@@ -114,6 +114,14 @@ export type TelemetryEvent = {
|
|
|
114
114
|
command: string
|
|
115
115
|
kubbVersion: string
|
|
116
116
|
nodeVersion: string
|
|
117
|
+
/**
|
|
118
|
+
* Name of the JavaScript runtime that executed the run, `'bun'`, `'deno'`, or `'node'`.
|
|
119
|
+
*/
|
|
120
|
+
runtime: RuntimeName
|
|
121
|
+
/**
|
|
122
|
+
* Major version of the active runtime, e.g. `'1'` under Bun or `'22'` under Node.
|
|
123
|
+
*/
|
|
124
|
+
runtimeVersion: string
|
|
117
125
|
platform: string
|
|
118
126
|
ci: boolean
|
|
119
127
|
plugins: Array<TelemetryPlugin>
|
|
@@ -158,6 +166,8 @@ export class Telemetry {
|
|
|
158
166
|
command: options.command,
|
|
159
167
|
kubbVersion: options.kubbVersion,
|
|
160
168
|
nodeVersion: process.versions.node.split('.')[0] as string,
|
|
169
|
+
runtime: getRuntimeName(),
|
|
170
|
+
runtimeVersion: getRuntimeVersion().split('.')[0] as string,
|
|
161
171
|
platform: os.platform(),
|
|
162
172
|
ci: isCIEnvironment(),
|
|
163
173
|
plugins: options.plugins ?? [],
|
|
@@ -181,6 +191,8 @@ export class Telemetry {
|
|
|
181
191
|
{ key: 'kubb.command', value: { stringValue: event.command } },
|
|
182
192
|
{ key: 'kubb.version', value: { stringValue: event.kubbVersion } },
|
|
183
193
|
{ key: 'kubb.node_version', value: { stringValue: event.nodeVersion } },
|
|
194
|
+
{ key: 'kubb.runtime', value: { stringValue: event.runtime } },
|
|
195
|
+
{ key: 'kubb.runtime_version', value: { stringValue: event.runtimeVersion } },
|
|
184
196
|
{ key: 'kubb.platform', value: { stringValue: event.platform } },
|
|
185
197
|
{ key: 'kubb.ci', value: { boolValue: event.ci } },
|
|
186
198
|
{ key: 'kubb.files_created', value: { intValue: event.filesCreated } },
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
2
|
+
import { join, resolve } from 'node:path'
|
|
3
|
+
import { clean, read, write } from '@internals/utils'
|
|
4
|
+
import { createCache } from '../createCache.ts'
|
|
5
|
+
import { Manifest } from '../Manifest.ts'
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Options for {@link fsCache}.
|
|
9
|
+
*/
|
|
10
|
+
export type FsCacheOptions = {
|
|
11
|
+
/**
|
|
12
|
+
* Directory that holds the cache. Resolved against `process.cwd()` when relative.
|
|
13
|
+
*
|
|
14
|
+
* @default 'node_modules/.cache/kubb'
|
|
15
|
+
*/
|
|
16
|
+
dir?: string
|
|
17
|
+
/**
|
|
18
|
+
* Maximum number of build snapshots to keep. The least-recently-used entries are
|
|
19
|
+
* evicted once the cache grows past it.
|
|
20
|
+
*
|
|
21
|
+
* @default 50
|
|
22
|
+
*/
|
|
23
|
+
maxEntries?: number
|
|
24
|
+
/**
|
|
25
|
+
* Days a snapshot may go untouched before it is evicted.
|
|
26
|
+
*
|
|
27
|
+
* @default 7
|
|
28
|
+
*/
|
|
29
|
+
ttlDays?: number
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
type IndexFile = Array<{ path: string; blob: string }>
|
|
33
|
+
|
|
34
|
+
function blobName(relativePath: string): string {
|
|
35
|
+
return `${createHash('sha256').update(relativePath).digest('hex')}.blob`
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Local filesystem cache. Stores each build snapshot as content blobs plus an index,
|
|
40
|
+
* tracked by a manifest under `node_modules/.cache/kubb/` (the Nx and Vitest
|
|
41
|
+
* convention). Least-recently-used and expired entries are pruned on every persist.
|
|
42
|
+
*
|
|
43
|
+
* @example
|
|
44
|
+
* ```ts
|
|
45
|
+
* import { fsCache } from '@kubb/core'
|
|
46
|
+
*
|
|
47
|
+
* export default defineConfig({
|
|
48
|
+
* cache: fsCache(),
|
|
49
|
+
* })
|
|
50
|
+
* ```
|
|
51
|
+
*/
|
|
52
|
+
export const fsCache = createCache((options: FsCacheOptions = {}) => {
|
|
53
|
+
const dir = resolve(options.dir ?? join('node_modules', '.cache', 'kubb'))
|
|
54
|
+
const maxEntries = options.maxEntries ?? 50
|
|
55
|
+
const ttlDays = options.ttlDays ?? 7
|
|
56
|
+
const blobsDir = join(dir, 'blobs')
|
|
57
|
+
const manifestPath = join(dir, 'manifest.json')
|
|
58
|
+
|
|
59
|
+
return {
|
|
60
|
+
name: 'fs',
|
|
61
|
+
async restore({ key }) {
|
|
62
|
+
const manifest = await Manifest.read(dir)
|
|
63
|
+
const entry = manifest.entries[key]
|
|
64
|
+
if (!entry) {
|
|
65
|
+
return null
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
try {
|
|
69
|
+
const index = JSON.parse(await read(join(blobsDir, key, 'index.json'))) as IndexFile
|
|
70
|
+
const files: Record<string, string> = {}
|
|
71
|
+
for (const { path, blob } of index) {
|
|
72
|
+
files[path] = await read(join(blobsDir, key, blob))
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
entry.lastAccess = Date.now()
|
|
76
|
+
await write(manifestPath, JSON.stringify(manifest)).catch(() => {})
|
|
77
|
+
|
|
78
|
+
return { files }
|
|
79
|
+
} catch {
|
|
80
|
+
return null
|
|
81
|
+
}
|
|
82
|
+
},
|
|
83
|
+
async persist({ key, snapshot }) {
|
|
84
|
+
const entryDir = join(blobsDir, key)
|
|
85
|
+
|
|
86
|
+
const index: IndexFile = []
|
|
87
|
+
for (const [path, source] of Object.entries(snapshot.files)) {
|
|
88
|
+
const blob = blobName(path)
|
|
89
|
+
await write(join(entryDir, blob), source)
|
|
90
|
+
index.push({ path, blob })
|
|
91
|
+
}
|
|
92
|
+
await write(join(entryDir, 'index.json'), JSON.stringify(index))
|
|
93
|
+
|
|
94
|
+
const manifest = await Manifest.read(dir)
|
|
95
|
+
const now = Date.now()
|
|
96
|
+
manifest.entries[key] = { files: index.map((item) => item.path), createdAt: now, lastAccess: now }
|
|
97
|
+
|
|
98
|
+
const pruned = Manifest.prune(manifest, { maxEntries, ttlDays, now })
|
|
99
|
+
await Promise.all(pruned.removed.map((removedKey) => clean(join(blobsDir, removedKey))))
|
|
100
|
+
await write(manifestPath, JSON.stringify(pruned.manifest))
|
|
101
|
+
},
|
|
102
|
+
}
|
|
103
|
+
})
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A snapshot of a completed build: the final rendered source of every generated
|
|
3
|
+
* file, keyed by its path relative to the output root. Restoring a snapshot writes
|
|
4
|
+
* those sources straight to storage, skipping generation entirely.
|
|
5
|
+
*
|
|
6
|
+
* Paths are relative, not absolute, so the snapshot never depends on where the
|
|
7
|
+
* project lives on disk.
|
|
8
|
+
*/
|
|
9
|
+
export type CachedSnapshot = {
|
|
10
|
+
/**
|
|
11
|
+
* Final source per output file, keyed by the path relative to
|
|
12
|
+
* `resolve(config.root, config.output.path)`.
|
|
13
|
+
*/
|
|
14
|
+
files: Record<string, string>
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Backend that stores build snapshots for incremental ("hot") rebuilds. When the
|
|
19
|
+
* input fingerprint matches a stored key, Kubb restores the snapshot instead of
|
|
20
|
+
* regenerating. Kubb ships with `fsCache` (local disk). Implement this interface to
|
|
21
|
+
* back the cache with any other store.
|
|
22
|
+
*
|
|
23
|
+
* @see {@link createCache} to build a custom backend.
|
|
24
|
+
*/
|
|
25
|
+
export type Cache = {
|
|
26
|
+
/**
|
|
27
|
+
* Identifier used in logs and diagnostics (`'fs'`, `'memory'`).
|
|
28
|
+
*/
|
|
29
|
+
readonly name: string
|
|
30
|
+
/**
|
|
31
|
+
* Returns the snapshot stored under `key`, or `null` on a miss. A backend never
|
|
32
|
+
* throws on a miss or a transient failure. It returns `null` so the build falls
|
|
33
|
+
* through to regeneration.
|
|
34
|
+
*/
|
|
35
|
+
restore(params: { key: string }): Promise<CachedSnapshot | null>
|
|
36
|
+
/**
|
|
37
|
+
* Stores `snapshot` under `key`. Only called after a successful build with no
|
|
38
|
+
* error diagnostics.
|
|
39
|
+
*/
|
|
40
|
+
persist(params: { key: string; snapshot: CachedSnapshot }): Promise<void>
|
|
41
|
+
/**
|
|
42
|
+
* Optional teardown called after the build. Use to flush buffers or close
|
|
43
|
+
* connections.
|
|
44
|
+
*/
|
|
45
|
+
dispose?(): Promise<void>
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Defines a custom cache backend. The builder receives user options and returns a
|
|
50
|
+
* {@link Cache}. Reach for this when the filesystem backend doesn't fit, for
|
|
51
|
+
* example to store snapshots in Redis or a database.
|
|
52
|
+
*
|
|
53
|
+
* @example In-memory cache (the built-in implementation)
|
|
54
|
+
* ```ts
|
|
55
|
+
* import { createCache } from '@kubb/core'
|
|
56
|
+
*
|
|
57
|
+
* export const memoryCache = createCache(() => {
|
|
58
|
+
* const store = new Map<string, CachedSnapshot>()
|
|
59
|
+
*
|
|
60
|
+
* return {
|
|
61
|
+
* name: 'memory',
|
|
62
|
+
* async restore({ key }) {
|
|
63
|
+
* return store.get(key) ?? null
|
|
64
|
+
* },
|
|
65
|
+
* async persist({ key, snapshot }) {
|
|
66
|
+
* store.set(key, snapshot)
|
|
67
|
+
* },
|
|
68
|
+
* }
|
|
69
|
+
* })
|
|
70
|
+
* ```
|
|
71
|
+
*/
|
|
72
|
+
export function createCache<TOptions = Record<string, never>>(build: (options: TOptions) => Cache): (options?: TOptions) => Cache {
|
|
73
|
+
return (options) => build(options ?? ({} as TOptions))
|
|
74
|
+
}
|