@scalar/json-magic 0.1.0 → 0.3.1

package/src/bundle/bundle.ts

@@ -77,7 +77,7 @@ export type ResolveResult = { ok: true; data: unknown } | { ok: false }
  * // No matching plugin returns { ok: false }
  * await resolveContents('#/components/schemas/User', [urlPlugin, filePlugin])
  */
-async function resolveContents(value: string, plugins: Plugin[]): Promise<ResolveResult> {
+async function resolveContents(value: string, plugins: LoaderPlugin[]): Promise<ResolveResult> {
   const plugin = plugins.find((p) => p.validate(value))
 
   if (plugin) {
@@ -206,6 +206,12 @@ function resolveReferencePath(base: string, relativePath: string) {
   if (isRemoteUrl(base)) {
     const url = new URL(base)
 
+    // If the url stars with a / we want it to resolve from the origin so we replace the pathname
+    if (relativePath.startsWith('/')) {
+      url.pathname = relativePath
+      return url.toString()
+    }
+
     const mergedPath = path.join(path.dirname(url.pathname), relativePath)
     return new URL(mergedPath, base).toString()
   }
@@ -360,22 +366,68 @@ const resolveAndCopyReferences = (
 }
 
 /**
- * Represents a plugin that handles resolving references from external sources.
- * Plugins are responsible for fetching and processing data from different sources
- * like URLs or the filesystem. Each plugin must implement validation to determine
- * if it can handle a specific reference, and an execution function to perform
- * the actual resolution.
+ * A loader plugin for resolving external references during bundling.
+ * Loader plugins are responsible for handling specific types of external references,
+ * such as files, URLs, or custom protocols. Each loader plugin must provide:
+ *
+ * - A `validate` function to determine if the plugin can handle a given reference string.
+ * - An `exec` function to asynchronously fetch and resolve the referenced data,
+ *   returning a Promise that resolves to a `ResolveResult`.
  *
- * @property validate - Determines if this plugin can handle the given reference
- * @property exec - Fetches and processes the reference, returning the resolved data
+ * Loader plugins enable extensible support for different reference sources in the bundler.
+ *
+ * @property type - The plugin type, always 'loader' for loader plugins.
+ * @property validate - Function to check if the plugin can handle a given reference value.
+ * @property exec - Function to fetch and resolve the reference, returning the resolved data.
  */
-export type Plugin = {
-  // Determines if this plugin can handle the given reference value
+export type LoaderPlugin = {
+  type: 'loader'
+  // Returns true if this plugin can handle the given reference value
   validate: (value: string) => boolean
-  // Fetches and processes the reference, returning the resolved data
+  // Asynchronously fetches and resolves the reference, returning the resolved data
   exec: (value: string) => Promise<ResolveResult>
 }
 
+/**
+ * Context information for a node during traversal or processing.
+ *
+ * Note: The `path` parameter represents the path to the current node being processed.
+ * If you are performing a partial bundle (i.e., providing a custom root), this path will be relative
+ * to the root you provide, not the absolute root of the original document. You may need to prefix
+ * it with your own base path if you want to construct a full path from the absolute document root.
+ *
+ * - `path`: The JSON pointer path (as an array of strings) from the document root to the current node.
+ * - `resolutionCache`: A cache for storing promises of resolved references.
+ */
+type NodeProcessContext = {
+  path: readonly string[]
+  resolutionCache: Map<string, Promise<Readonly<ResolveResult>>>
+  parentNode: UnknownObject | null
+  rootNode: UnknownObject
+  loaders: LoaderPlugin[]
+}
+
+/**
+ * A plugin type for lifecycle hooks, allowing custom logic to be injected into the bundler's process.
+ * This type extends the Config['hooks'] interface and is identified by type: 'lifecycle'.
+ */
+export type LifecyclePlugin = { type: 'lifecycle' } & Config['hooks']
+
+/**
+ * Represents a plugin used by the bundler for extensibility.
+ *
+ * Plugins can be either:
+ * - Loader plugins: Responsible for resolving and loading external references (e.g., from files, URLs, or custom sources).
+ * - Lifecycle plugins: Provide lifecycle hooks to customize or extend the bundling process.
+ *
+ * Loader plugins must implement:
+ *   - `validate`: Checks if the plugin can handle a given reference value.
+ *   - `exec`: Asynchronously resolves and returns the referenced data.
+ *
+ * Lifecycle plugins extend the bundler's lifecycle hooks for custom logic.
+ */
+export type Plugin = LoaderPlugin | LifecyclePlugin
+
 /**
  * Configuration options for the bundler.
  * Controls how external references are resolved and processed during bundling.
@@ -402,6 +454,13 @@ type Config = {
    */
   depth?: number
 
+  /**
+   * Optional origin path for the bundler.
+   * Used to resolve relative paths in references, especially when the input is a string URL or file path.
+   * If not provided, the bundler will use the input value as the origin.
+   */
+  origin?: string
+
   /**
    * Optional cache to store promises of resolved references.
    * Helps avoid duplicate fetches/reads of the same resource by storing
@@ -441,12 +500,31 @@ type Config = {
    * Allows tracking the progress and status of reference resolution.
    */
   hooks?: Partial<{
-    /** Called when starting to resolve a reference */
-    onResolveStart: (node: Record<string, unknown> & Record<'$ref', unknown>) => void
-    /** Called when a reference resolution fails */
-    onResolveError: (node: Record<string, unknown> & Record<'$ref', unknown>) => void
-    /** Called when a reference is successfully resolved */
-    onResolveSuccess: (node: Record<string, unknown> & Record<'$ref', unknown>) => void
+    /**
+     * Optional hook called when the bundler starts resolving a $ref.
+     * Useful for tracking or logging the beginning of a reference resolution.
+     */
+    onResolveStart: (node: UnknownObject & Record<'$ref', unknown>) => void
+    /**
+     * Optional hook called when the bundler fails to resolve a $ref.
+     * Can be used for error handling, logging, or custom error reporting.
+     */
+    onResolveError: (node: UnknownObject & Record<'$ref', unknown>) => void
+    /**
+     * Optional hook called when the bundler successfully resolves a $ref.
+     * Useful for tracking successful resolutions or custom post-processing.
+     */
+    onResolveSuccess: (node: UnknownObject & Record<'$ref', unknown>) => void
+    /**
+     * Optional hook invoked before processing a node.
+     * Can be used for preprocessing, mutation, or custom logic before the node is handled by the bundler.
+     */
+    onBeforeNodeProcess: (node: UnknownObject, context: NodeProcessContext) => void | Promise<void>
+    /**
+     * Optional hook invoked after processing a node.
+     * Useful for postprocessing, cleanup, or custom logic after the node has been handled by the bundler.
+     */
+    onAfterNodeProcess: (node: UnknownObject, context: NodeProcessContext) => void | Promise<void>
   }>
 }
 
@@ -541,6 +619,9 @@ export async function bundle(input: UnknownObject | string, config: Config) {
   // to avoid duplicate fetches/reads of the same resource
   const cache = config.cache ?? new Map<string, Promise<ResolveResult>>()
 
+  const loaderPlugins = config.plugins.filter((it) => it.type === 'loader')
+  const lifecyclePlugin = config.plugins.filter((it) => it.type === 'lifecycle')
+
   /**
    * Resolves the input value by either returning it directly if it's not a string,
    * or attempting to resolve it using the provided plugins if it is a string.
@@ -550,7 +631,7 @@ export async function bundle(input: UnknownObject | string, config: Config) {
     if (typeof input !== 'string') {
       return input
     }
-    const result = await resolveContents(input, config.plugins)
+    const result = await resolveContents(input, loaderPlugins)
 
     if (result.ok && typeof result.data === 'object') {
       return result.data
@@ -583,6 +664,10 @@ export async function bundle(input: UnknownObject | string, config: Config) {
   // For string inputs that are URLs or file paths, uses the input as the origin.
   // For non-string inputs or other string types, returns an empty string.
   const defaultOrigin = () => {
+    if (config.origin) {
+      return config.origin
+    }
+
     if (typeof input !== 'string') {
       return ''
     }
@@ -603,7 +688,14 @@ export async function bundle(input: UnknownObject | string, config: Config) {
     documentRoot[extensions.externalDocumentsMappings],
   )
 
-  const bundler = async (root: unknown, origin: string = defaultOrigin(), isChunkParent = false, depth = 0) => {
+  const bundler = async (
+    root: unknown,
+    origin: string = defaultOrigin(),
+    isChunkParent = false,
+    depth = 0,
+    path: readonly string[] = [],
+    parent: UnknownObject = null,
+  ) => {
     // If a maximum depth is set in the config, stop bundling when the current depth reaches or exceeds it
     if (config.depth !== undefined && depth > config.depth) {
       return
@@ -621,22 +713,39 @@ export async function bundle(input: UnknownObject | string, config: Config) {
     // Mark this node as processed before continuing
     processedNodes.add(root)
 
+    // Invoke the onBeforeNodeProcess hook for the current node before any further processing
+    await config.hooks?.onBeforeNodeProcess?.(root as UnknownObject, {
+      path,
+      resolutionCache: cache,
+      parentNode: parent,
+      rootNode: documentRoot as UnknownObject,
+      loaders: loaderPlugins,
+    })
+    // Invoke onBeforeNodeProcess hooks from all registered lifecycle plugins
+    for (const plugin of lifecyclePlugin) {
+      await plugin.onBeforeNodeProcess?.(root as UnknownObject, {
+        path,
+        resolutionCache: cache,
+        parentNode: parent,
+        rootNode: documentRoot as UnknownObject,
+        loaders: loaderPlugins,
+      })
+    }
+
     if (typeof root === 'object' && '$ref' in root && typeof root['$ref'] === 'string') {
       const ref = root['$ref']
       const isChunk = '$global' in root && typeof root['$global'] === 'boolean' && root['$global']
 
       if (isLocalRef(ref)) {
         if (isPartialBundling) {
+          const segments = getSegmentsFromPath(ref.substring(1))
+          const parent = segments.length > 0 ? getNestedValue(documentRoot, segments.slice(0, -1)) : undefined
+
           // When doing partial bundling, we need to recursively bundle all dependencies
           // referenced by this local reference to ensure the partial bundle is complete.
           // This includes not just the direct reference but also all its dependencies,
           // creating a complete and self-contained partial bundle.
-          await bundler(
-            getNestedValue(documentRoot, getSegmentsFromPath(ref.substring(1))),
-            origin,
-            isChunkParent,
-            depth + 1,
-          )
+          await bundler(getNestedValue(documentRoot, segments), origin, isChunkParent, depth + 1, segments, parent)
         }
         return
       }
@@ -655,10 +764,11 @@ export async function bundle(input: UnknownObject | string, config: Config) {
       const seen = cache.has(resolvedPath)
 
       if (!seen) {
-        cache.set(resolvedPath, resolveContents(resolvedPath, config.plugins))
+        cache.set(resolvedPath, resolveContents(resolvedPath, loaderPlugins))
       }
 
       config?.hooks?.onResolveStart?.(root)
+      lifecyclePlugin.forEach((it) => it.onResolveStart?.(root))
 
       // Resolve the remote document
       const result = await cache.get(resolvedPath)
@@ -686,7 +796,11 @@ export async function bundle(input: UnknownObject | string, config: Config) {
           // to handle any nested references it may contain. We pass the resolvedPath as the new origin
           // to ensure any relative references within this content are resolved correctly relative to
           // their new location in the bundled document.
-          await bundler(result.data, isChunk ? origin : resolvedPath, isChunk, depth + 1)
+          await bundler(result.data, isChunk ? origin : resolvedPath, isChunk, depth + 1, [
+            extensions.externalDocuments,
+            compressedPath,
+            documentRoot[extensions.externalDocumentsMappings],
+          ])
 
           // Store the mapping between hashed keys and original URLs in x-ext-urls
           // This allows tracking which external URLs were bundled and their corresponding locations
@@ -721,28 +835,55 @@ export async function bundle(input: UnknownObject | string, config: Config) {
         // This is necessary because we need to maintain the correct path context
         // for the embedded document while preserving its internal structure
         root.$ref = prefixInternalRef(`#${path}`, [extensions.externalDocuments, compressedPath])
+
         config?.hooks?.onResolveSuccess?.(root)
+        lifecyclePlugin.forEach((it) => it.onResolveSuccess?.(root))
+
         return
       }
 
       config?.hooks?.onResolveError?.(root)
+      lifecyclePlugin.forEach((it) => it.onResolveError?.(root))
+
       return console.warn(
         `Failed to resolve external reference "${resolvedPath}". The reference may be invalid, inaccessible, or missing a loader for this type of reference.`,
       )
     }
 
-    // Recursively process all child objects to handle nested references
-    // This ensures we catch and resolve any $refs that exist deeper in the object tree
-    // We skip EXTERNAL_KEY to avoid processing already bundled content
+    // Recursively traverse all child properties of the current object to resolve nested $ref references.
+    // This step ensures that any $refs located deeper within the object hierarchy are discovered and processed.
+    // We explicitly skip the extension keys (x-ext and x-ext-urls) to avoid reprocessing already bundled or mapped content.
     await Promise.all(
       Object.entries(root).map(async ([key, value]) => {
-        if (key === extensions.externalDocuments) {
+        if (key === extensions.externalDocuments || key === extensions.externalDocumentsMappings) {
           return
         }
 
-        await bundler(value, origin, isChunkParent, depth + 1)
+        await bundler(value, origin, isChunkParent, depth + 1, [...path, key], root as UnknownObject)
       }),
     )
+
+    // Invoke the optional onAfterNodeProcess hook from the config, if provided.
+    // This allows for custom post-processing logic after a node has been handled by the bundler.
+    await config.hooks?.onAfterNodeProcess?.(root as UnknownObject, {
+      path,
+      resolutionCache: cache,
+      parentNode: parent,
+      rootNode: documentRoot as UnknownObject,
+      loaders: loaderPlugins,
+    })
+
+    // Iterate through all lifecycle plugins and invoke their onAfterNodeProcess hooks, if defined.
+    // This enables plugins to perform additional post-processing or cleanup after the node is processed.
+    for (const plugin of lifecyclePlugin) {
+      await plugin.onAfterNodeProcess?.(root as UnknownObject, {
+        path,
+        resolutionCache: cache,
+        parentNode: parent,
+        rootNode: documentRoot as UnknownObject,
+        loaders: loaderPlugins,
+      })
+    }
   }
 
   await bundler(rawSpecification)

package/src/bundle/index.ts

@@ -1,2 +1,3 @@
 // biome-ignore lint/performance/noBarrelFile: <explanation>
-export { bundle, type Plugin, type ResolveResult } from './bundle'
+export { bundle } from './bundle'
+export type { Plugin, LoaderPlugin, LifecyclePlugin, ResolveResult } from './bundle'

package/src/bundle/plugins/fetch-urls/index.ts

@@ -1,6 +1,6 @@
 import { normalize } from '@/utils/normalize'
 import { createLimiter } from '@/bundle/create-limiter'
-import type { Plugin, ResolveResult } from '@/bundle'
+import type { LoaderPlugin, ResolveResult } from '@/bundle'
 import { isRemoteUrl } from '@/bundle/bundle'
 
 type FetchConfig = Partial<{
@@ -83,11 +83,12 @@ export async function fetchUrl(
  *   const result = await urlPlugin.exec('https://example.com/schema.json')
  * }
  */
-export function fetchUrls(config?: FetchConfig & Partial<{ limit: number | null }>): Plugin {
+export function fetchUrls(config?: FetchConfig & Partial<{ limit: number | null }>): LoaderPlugin {
   // If there is a limit specified we limit the number of concurrent calls
   const limiter = config?.limit ? createLimiter(config.limit) : <T>(fn: () => Promise<T>) => fn()
 
   return {
+    type: 'loader',
     validate: isRemoteUrl,
     exec: (value) => fetchUrl(value, limiter, config),
   }

package/src/bundle/plugins/parse-json/index.ts

@@ -1,5 +1,5 @@
 import { isJsonObject } from '@/utils/is-json-object'
-import type { Plugin, ResolveResult } from '@/bundle'
+import type { LoaderPlugin, ResolveResult } from '@/bundle'
 
 /**
  * Creates a plugin that parses JSON strings into JavaScript objects.
@@ -11,8 +11,9 @@ import type { Plugin, ResolveResult } from '@/bundle'
  * // result = { name: 'John', age: 30 }
  * ```
  */
-export function parseJson(): Plugin {
+export function parseJson(): LoaderPlugin {
   return {
+    type: 'loader',
     validate: isJsonObject,
     exec: async (value): Promise<ResolveResult> => {
       try {

package/src/bundle/plugins/parse-yaml/index.ts

@@ -1,4 +1,4 @@
-import type { Plugin, ResolveResult } from '@/bundle/bundle'
+import type { LoaderPlugin, ResolveResult } from '@/bundle'
 import { isYaml } from '@/utils/is-yaml'
 import YAML from 'yaml'
 
@@ -12,8 +12,9 @@ import YAML from 'yaml'
  * // result = { name: 'John', age: 30 }
  * ```
  */
-export function parseYaml(): Plugin {
+export function parseYaml(): LoaderPlugin {
   return {
+    type: 'loader',
     validate: isYaml,
     exec: async (value): Promise<ResolveResult> => {
       try {

package/src/bundle/plugins/read-files/index.ts

@@ -1,5 +1,6 @@
 import { normalize } from '@/utils/normalize'
-import { isFilePath, type Plugin, type ResolveResult } from '@/bundle/bundle'
+import type { LoaderPlugin, ResolveResult } from '@/bundle'
+import { isFilePath } from '@/bundle/bundle'
 
 /**
  * Reads and normalizes data from a local file
@@ -47,8 +48,9 @@ export async function readFile(path: string): Promise<ResolveResult> {
  *   const result = await filePlugin.exec('./local-schema.json')
  * }
  */
-export function readFiles(): Plugin {
+export function readFiles(): LoaderPlugin {
   return {
+    type: 'loader',
     validate: isFilePath,
     exec: readFile,
   }

package/src/dereference/dereference.test.ts

@@ -37,18 +37,22 @@ describe('dereference', () => {
             },
           },
           profile: {
-            'x-original-ref': '#/users',
-            name: 'John Doe',
-            age: 30,
-            address: {
-              city: 'New York',
-              street: '5th Avenue',
+            '$ref': '#/users',
+            '$ref-value': {
+              name: 'John Doe',
+              age: 30,
+              address: {
+                city: 'New York',
+                street: '5th Avenue',
+              },
             },
           },
           address: {
-            'x-original-ref': '#/users/address',
-            city: 'New York',
-            street: '5th Avenue',
+            '$ref': '#/users/address',
+            '$ref-value': {
+              city: 'New York',
+              street: '5th Avenue',
+            },
           },
         },
       })
@@ -97,18 +101,22 @@ describe('dereference', () => {
         success: true,
         data: {
           profile: {
-            'x-original-ref': '#/x-ext/5bd1cdd',
-            name: 'Jane Doe',
-            age: 25,
-            address: {
-              city: 'Los Angeles',
-              street: 'Sunset Boulevard',
+            '$ref': '#/x-ext/5bd1cdd',
+            '$ref-value': {
+              name: 'Jane Doe',
+              age: 25,
+              address: {
+                city: 'Los Angeles',
+                street: 'Sunset Boulevard',
+              },
             },
           },
           address: {
-            'x-original-ref': '#/x-ext/5bd1cdd/address',
-            city: 'Los Angeles',
-            street: 'Sunset Boulevard',
+            '$ref': '#/x-ext/5bd1cdd/address',
+            '$ref-value': {
+              city: 'Los Angeles',
+              street: 'Sunset Boulevard',
+            },
           },
           'x-ext': {
             '5bd1cdd': userProfile,

package/src/diff/apply.ts

@@ -36,9 +36,12 @@ export class InvalidChangesDetectedError extends Error {
  * const updated = apply(original, changes)
  * // Result: original document with content added to the 200 response
  */
-export const apply = (document: Record<string, unknown>, diff: Difference[]): Record<string, unknown> => {
+export const apply = <T extends Record<string, unknown>>(
+  document: Record<string, unknown>,
+  diff: Difference<T>[],
+): T => {
   // Traverse the object and apply the change
-  const applyChange = (current: any, path: string[], d: Difference, depth = 0) => {
+  const applyChange = (current: any, path: string[], d: Difference<T>, depth = 0) => {
     if (path[depth] === undefined) {
       throw new InvalidChangesDetectedError(
         `Process aborted. Path ${path.join('.')} at depth ${depth} is undefined, check diff object`,
@@ -74,5 +77,7 @@ export const apply = (document: Record<string, unknown>, diff: Difference[]): Re
     applyChange(document, d.path, d)
   }
 
-  return document
+  // It is safe to cast here because this function mutates the input document
+  // to match the target type T as described by the diff changeset.
+  return document as T
 }

package/src/diff/diff.ts

@@ -12,7 +12,7 @@ type ChangeType = 'add' | 'update' | 'delete'
  * @property changes - The new value for the property (for add/update) or the old value (for delete)
  * @property type - The type of change that occurred
  */
-export type Difference = { path: string[]; changes: any; type: ChangeType }
+export type Difference<_T> = { path: string[]; changes: any; type: ChangeType }
 
 /**
  * Get the difference between two objects.
@@ -49,8 +49,8 @@ export type Difference = { path: string[]; changes: any; type: ChangeType }
  * //   { path: ['user', 'settings', 'theme'], changes: 'dark', type: 'update' }
  * // ]
  */
-export const diff = (doc1: Record<string, unknown>, doc2: Record<string, unknown>): Difference[] => {
-  const diff: Difference[] = []
+export const diff = <T extends Record<string, unknown>>(doc1: Record<string, unknown>, doc2: T) => {
+  const diff: Difference<T>[] = []
 
   const bfs = (el1: unknown, el2: unknown, prefix = []) => {
     // If the types are different, we know that the property has been added, deleted or updated

package/src/diff/merge.ts

@@ -39,7 +39,7 @@ import { isArrayEqual, isKeyCollisions, mergeObjects } from '@/diff/utils'
  * //   ]
  * // }
  */
-export const merge = (diff1: Difference[], diff2: Difference[]) => {
+export const merge = <T>(diff1: Difference<T>[], diff2: Difference<T>[]) => {
   // Here we need to use a trie to optimize searching for a prefix
   // With the naive approach time complexity of the algorithm would be
   //                         O(n * m)
@@ -49,7 +49,7 @@ export const merge = (diff1: Difference[], diff2: Difference[]) => {
   // Assuming that the maximum depth of the nested objects would be constant lets say 0 <= D <= 100
   // we try to optimize for that using the tire data structure.
   // So the new time complexity would be O(n * D) where D is the maximum depth of the nested object
-  const trie = new Trie<{ index: number; changes: Difference }>()
+  const trie = new Trie<{ index: number; changes: Difference<T> }>()
 
   // Create the trie
   for (const [index, diff] of diff1.entries()) {
@@ -62,10 +62,10 @@ export const merge = (diff1: Difference[], diff2: Difference[]) => {
   // Keep related conflicts together for easy A, B pick conflict resolution
   // map key is going to be conflicting index of first diff list where the diff will be
   // a delete operation or an add/update operation with a one to many conflicts
-  const conflictsMap1 = new Map<number, [Difference[], Difference[]]>()
+  const conflictsMap1 = new Map<number, [Difference<T>[], Difference<T>[]]>()
   // map key will be the index from the second diff list where the diff will be
   // a delete operation with one to many conflicts
-  const conflictsMap2 = new Map<number, [Difference[], Difference[]]>()
+  const conflictsMap2 = new Map<number, [Difference<T>[], Difference<T>[]]>()
 
   for (const [index, diff] of diff2.entries()) {
     trie.findMatch(diff.path, (value) => {
@@ -123,11 +123,11 @@ export const merge = (diff1: Difference[], diff2: Difference[]) => {
     })
   }
 
-  const conflicts: [Difference[], Difference[]][] = [...conflictsMap1.values(), ...conflictsMap2.values()]
+  const conflicts: [Difference<T>[], Difference<T>[]][] = [...conflictsMap1.values(), ...conflictsMap2.values()]
 
   // Filter all changes that should be skipped because of conflicts
   // or auto conflict resolution
-  const diffs: Difference[] = [
+  const diffs: Difference<T>[] = [
     ...diff1.filter((_, index) => !skipDiff1.has(index)),
     ...diff2.filter((_, index) => !skipDiff2.has(index)),
   ]