@redocly/openapi-core 1.0.0-beta.100

package/src/ref-utils.ts

@@ -0,0 +1,80 @@
+import { Source } from './resolve';
+import { OasRef } from './typings/openapi';
+
+export function joinPointer(base: string, key: string | number) {
+  if (base === '') base = '#/';
+  return base[base.length - 1] === '/' ? base + key : base + '/' + key;
+}
+
+export function isRef(node: any): node is OasRef {
+  return node && typeof node.$ref === 'string';
+}
+
+export class Location {
+  constructor(public source: Source, public pointer: string) {}
+
+  child(components: (string | number)[] | string | number) {
+    return new Location(
+      this.source,
+      joinPointer(
+        this.pointer,
+        (Array.isArray(components) ? components : [components]).map(escapePointer).join('/'),
+      ),
+    );
+  }
+
+  key() {
+    return { ...this, reportOnKey: true };
+  }
+
+  get absolutePointer() {
+    return this.source.absoluteRef + (this.pointer === '#/' ? '' : this.pointer);
+  }
+}
+
+export function unescapePointer(fragment: string): string {
+  return decodeURIComponent(fragment.replace(/~1/g, '/').replace(/~0/g, '~'));
+}
+
+export function escapePointer<T extends string | number>(fragment: T): T {
+  if (typeof fragment === 'number') return fragment;
+  return (fragment as string).replace(/~/g, '~0').replace(/\//g, '~1') as T;
+}
+
+export function parseRef(ref: string): { uri: string | null; pointer: string[] } {
+  const [uri, pointer] = ref.split('#/');
+  return {
+    uri: uri || null,
+    pointer: pointer ? pointer.split('/').map(unescapePointer).filter(Boolean) : [],
+  };
+}
+
+export function parsePointer(pointer: string) {
+  return pointer.substr(2).split('/').map(unescapePointer);
+}
+
+export function pointerBaseName(pointer: string) {
+  const parts = pointer.split('/');
+  return parts[parts.length - 1];
+}
+
+export function refBaseName(ref: string) {
+  const parts = ref.split(/[\/\\]/); // split by '\' and '/'
+  return parts[parts.length - 1].replace(/\.[^.]+$/, ''); // replace extension with empty string
+}
+
+export function isAbsoluteUrl(ref: string) {
+  return ref.startsWith('http://') || ref.startsWith('https://');
+}
+
+export function isMappingRef(mapping: string) {
+  // TODO: proper detection of mapping refs
+  return (
+    mapping.startsWith('#') ||
+    mapping.startsWith('https://') ||
+    mapping.startsWith('http://') ||
+    mapping.startsWith('./') ||
+    mapping.startsWith('../') ||
+    mapping.indexOf('/') > -1
+  );
+}

package/src/resolve.ts

@@ -0,0 +1,378 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { OasRef } from './typings/openapi';
+import { isRef, joinPointer, escapePointer, parseRef, isAbsoluteUrl } from './ref-utils';
+import type { YAMLNode, LoadOptions } from 'yaml-ast-parser';
+import { NormalizedNodeType, isNamedType } from './types';
+import { readFileFromUrl, parseYaml } from './utils';
+import { ResolveConfig } from './config/types';
+
+export type CollectedRefs = Map<string /* absoluteFilePath */, Document>;
+
+export class Source {
+  constructor(public absoluteRef: string, public body: string, public mimeType?: string) {}
+
+  private _ast: YAMLNode | undefined;
+  private _lines: string[] | undefined;
+
+  // pass safeLoad as argument to separate it from browser bundle
+  getAst(safeLoad: (input: string, options?: LoadOptions | undefined) => YAMLNode) {
+    if (this._ast === undefined) {
+      this._ast = safeLoad(this.body, { filename: this.absoluteRef }) ?? undefined;
+
+      // fix ast representation of file with newlines only
+      if (
+        this._ast &&
+        this._ast.kind === 0 && // KIND.scalar = 0
+        this._ast.value === '' &&
+        this._ast.startPosition !== 1
+      ) {
+        this._ast.startPosition = 1;
+        this._ast.endPosition = 1;
+      }
+    }
+    return this._ast;
+  }
+
+  getLines() {
+    if (this._lines === undefined) {
+      this._lines = this.body.split(/\r\n|[\n\r]/g);
+    }
+    return this._lines;
+  }
+}
+
+export class ResolveError extends Error {
+  constructor(public originalError: Error) {
+    super(originalError.message);
+    // Set the prototype explicitly.
+    Object.setPrototypeOf(this, ResolveError.prototype);
+  }
+}
+
+const jsYamlErrorLineColRegexp = /\((\d+):(\d+)\)$/;
+
+export class YamlParseError extends Error {
+  col: number;
+  line: number;
+
+  constructor(public originalError: Error, public source: Source) {
+    super(originalError.message.split('\n')[0]);
+    // Set the prototype explicitly.
+    Object.setPrototypeOf(this, YamlParseError.prototype);
+
+    const [, line, col] = this.message.match(jsYamlErrorLineColRegexp) || [];
+    this.line = parseInt(line, 10);
+    this.col = parseInt(col, 10);
+  }
+}
+
+export type Document = {
+  source: Source;
+  parsed: any;
+};
+
+export function makeRefId(absoluteRef: string, pointer: string) {
+  return absoluteRef + '::' + pointer;
+}
+
+export function makeDocumentFromString(sourceString: string, absoluteRef: string) {
+  const source = new Source(absoluteRef, sourceString);
+  try {
+    return {
+      source,
+      parsed: parseYaml(sourceString, { filename: absoluteRef }),
+    };
+  } catch (e) {
+    throw new YamlParseError(e, source);
+  }
+}
+
+export class BaseResolver {
+  cache: Map<string, Promise<Document | ResolveError>> = new Map();
+
+  constructor(private config: ResolveConfig = { http: { headers: [] } }) {}
+
+  getFiles() {
+    return new Set(Array.from(this.cache.keys()));
+  }
+
+  resolveExternalRef(base: string | null, ref: string): string {
+    if (isAbsoluteUrl(ref)) {
+      return ref;
+    }
+
+    if (base && isAbsoluteUrl(base)) {
+      return new URL(ref, base).href;
+    }
+
+    return path.resolve(base ? path.dirname(base) : process.cwd(), ref);
+  }
+
+  async loadExternalRef(absoluteRef: string): Promise<Source> {
+    try {
+      if (isAbsoluteUrl(absoluteRef)) {
+        const { body, mimeType } = await readFileFromUrl(absoluteRef, this.config.http);
+        return new Source(absoluteRef, body, mimeType);
+      } else {
+        return new Source(absoluteRef, await fs.promises.readFile(absoluteRef, 'utf-8'));
+      }
+    } catch (error) {
+      throw new ResolveError(error);
+    }
+  }
+
+  parseDocument(source: Source, isRoot: boolean = false): Document {
+    const ext = source.absoluteRef.substr(source.absoluteRef.lastIndexOf('.'));
+    if (
+      !['.json', '.json', '.yml', '.yaml'].includes(ext) &&
+      !source.mimeType?.match(/(json|yaml|openapi)/) &&
+      !isRoot // always parse root
+    ) {
+      return { source, parsed: source.body };
+    }
+
+    try {
+      return {
+        source,
+        parsed: parseYaml(source.body, { filename: source.absoluteRef }),
+      };
+    } catch (e) {
+      throw new YamlParseError(e, source);
+    }
+  }
+
+  async resolveDocument(
+    base: string | null,
+    ref: string,
+    isRoot: boolean = false
+  ): Promise<Document | ResolveError | YamlParseError> {
+    const absoluteRef = this.resolveExternalRef(base, ref);
+    const cachedDocument = this.cache.get(absoluteRef);
+    if (cachedDocument) {
+      return cachedDocument;
+    }
+
+    const doc = this.loadExternalRef(absoluteRef).then((source) => {
+      return this.parseDocument(source, isRoot);
+    });
+
+    this.cache.set(absoluteRef, doc);
+
+    return doc;
+  }
+}
+
+export type ResolvedRef =
+  | {
+      resolved: false;
+      isRemote: boolean;
+      nodePointer?: string;
+      document?: Document;
+      source?: Source;
+      error?: ResolveError | YamlParseError;
+      node?: any;
+    }
+  | {
+      resolved: true;
+      node: any;
+      document: Document;
+      nodePointer: string;
+      isRemote: boolean;
+      error?: undefined;
+    };
+
+export type ResolvedRefMap = Map<string, ResolvedRef>;
+
+type RefFrame = {
+  prev: RefFrame | null;
+  node: any;
+};
+
+function pushRef(head: RefFrame, node: any): RefFrame {
+  return {
+    prev: head,
+    node,
+  };
+}
+
+function hasRef(head: RefFrame | null, node: any): boolean {
+  while (head) {
+    if (head.node === node) {
+      return true;
+    }
+    head = head.prev;
+  }
+  return false;
+}
+
+const unknownType = { name: 'unknown', properties: {} };
+const resolvableScalarType = { name: 'scalar', properties: {} };
+
+export async function resolveDocument(opts: {
+  rootDocument: Document;
+  externalRefResolver: BaseResolver;
+  rootType: NormalizedNodeType;
+}): Promise<ResolvedRefMap> {
+  const { rootDocument, externalRefResolver, rootType } = opts;
+  const resolvedRefMap: ResolvedRefMap = new Map();
+  const seedNodes = new Set<string>(); // format "${type}::${absoluteRef}${pointer}"
+
+  const resolvePromises: Array<Promise<void>> = [];
+  resolveRefsInParallel(rootDocument.parsed, rootDocument, '#/', rootType);
+
+  let resolved;
+  do {
+    resolved = await Promise.all(resolvePromises);
+  } while (resolvePromises.length !== resolved.length);
+
+  return resolvedRefMap;
+
+  function resolveRefsInParallel(
+    rootNode: any,
+    rootNodeDocument: Document,
+    rootNodePointer: string,
+    type: any,
+  ) {
+    const rootNodeDocAbsoluteRef = rootNodeDocument.source.absoluteRef;
+
+    walk(rootNode, type, rootNodeDocAbsoluteRef + rootNodePointer);
+
+    function walk(node: any, type: NormalizedNodeType, nodeAbsoluteRef: string) {
+      if (typeof node !== 'object' || node === null) {
+        return;
+      }
+
+      const nodeId = `${type.name}::${nodeAbsoluteRef}`;
+      if (seedNodes.has(nodeId)) {
+        return;
+      }
+
+      seedNodes.add(nodeId);
+
+      if (Array.isArray(node)) {
+        const itemsType = type.items;
+        // we continue resolving unknown types, but stop early on known scalars
+        if (type !== unknownType && itemsType === undefined) {
+          return;
+        }
+        for (let i = 0; i < node.length; i++) {
+          walk(node[i], itemsType || unknownType, joinPointer(nodeAbsoluteRef, i));
+        }
+        return;
+      }
+
+      for (const propName of Object.keys(node)) {
+        let propValue = node[propName];
+        let propType = type.properties[propName];
+        if (propType === undefined) propType = type.additionalProperties;
+        if (typeof propType === 'function') propType = propType(propValue, propName);
+        if (propType === undefined) propType = unknownType;
+
+        if (!isNamedType(propType) && propType?.directResolveAs) {
+          propType = propType.directResolveAs;
+          propValue = { $ref: propValue };
+        }
+
+        if (propType && propType.name === undefined && propType.resolvable !== false) {
+          propType = resolvableScalarType;
+        }
+
+        if (!isNamedType(propType) || typeof propValue !== 'object') {
+          continue;
+        }
+
+        walk(propValue, propType, joinPointer(nodeAbsoluteRef, escapePointer(propName)));
+      }
+
+      if (isRef(node)) {
+        const promise = followRef(rootNodeDocument, node, {
+          prev: null,
+          node,
+        }).then((resolvedRef) => {
+          if (resolvedRef.resolved) {
+            resolveRefsInParallel(
+              resolvedRef.node,
+              resolvedRef.document,
+              resolvedRef.nodePointer!,
+              type,
+            );
+          }
+        });
+        resolvePromises.push(promise);
+      }
+    }
+
+    async function followRef(
+      document: Document,
+      ref: OasRef,
+      refStack: RefFrame,
+    ): Promise<ResolvedRef> {
+      if (hasRef(refStack.prev, ref)) {
+        throw new Error('Self-referencing circular pointer');
+      }
+      const { uri, pointer } = parseRef(ref.$ref);
+      const isRemote = uri !== null;
+      let targetDoc: Document;
+      try {
+        targetDoc = isRemote
+          ? ((await externalRefResolver.resolveDocument(document.source.absoluteRef, uri!)) as Document)
+          : document;
+      } catch (error) {
+        const resolvedRef = {
+          resolved: false as false,
+          isRemote,
+          document: undefined,
+          error: error,
+        };
+        const refId = makeRefId(document.source.absoluteRef, ref.$ref);
+        resolvedRefMap.set(refId, resolvedRef);
+        return resolvedRef;
+      }
+
+      let resolvedRef: ResolvedRef = {
+        resolved: true as true,
+        document: targetDoc,
+        isRemote,
+        node: document.parsed,
+        nodePointer: '#/',
+      };
+
+      let target = targetDoc.parsed as any;
+
+      const segments = pointer;
+      for (let segment of segments) {
+        if (typeof target !== 'object') {
+          target = undefined;
+          break;
+        } else if (target[segment] !== undefined) {
+          target = target[segment];
+          resolvedRef.nodePointer = joinPointer(resolvedRef.nodePointer!, escapePointer(segment));
+        } else if (isRef(target)) {
+          resolvedRef = await followRef(targetDoc, target, pushRef(refStack, target));
+          targetDoc = resolvedRef.document || targetDoc;
+
+          if (typeof resolvedRef.node !== 'object') {
+            target = undefined;
+            break;
+          }
+
+          target = resolvedRef.node[segment];
+          resolvedRef.nodePointer = joinPointer(resolvedRef.nodePointer!, escapePointer(segment));
+        } else {
+          target = undefined;
+          break;
+        }
+      }
+
+      resolvedRef.node = target;
+      resolvedRef.document = targetDoc;
+      const refId = makeRefId(document.source.absoluteRef, ref.$ref);
+      if (resolvedRef.document && isRef(target)) {
+        resolvedRef = await followRef(resolvedRef.document, target, pushRef(refStack, target));
+      }
+      resolvedRefMap.set(refId, resolvedRef);
+      return { ...resolvedRef };
+    }
+  }
+}

package/src/rules/__tests__/fixtures/invalid-yaml.yaml

@@ -0,0 +1 @@
+test: '

package/src/rules/__tests__/fixtures/ref.yaml

@@ -0,0 +1 @@
+test: 1

package/src/rules/__tests__/no-unresolved-refs.test.ts

@@ -0,0 +1,168 @@
+import path = require('path');
+import { outdent } from 'outdent';
+import { lintDocument } from '../../lint';
+import { BaseResolver } from '../../resolve';
+import { parseYamlToDocument, replaceSourceWithRef, makeConfig } from '../../../__tests__/utils';
+
+describe('oas3 boolean-parameter-prefixes', () => {
+  it('should report on unresolved $ref', async () => {
+    const document = parseYamlToDocument(
+      outdent`
+        openapi: 3.0.0
+        paths:
+          '/test':
+            put:
+              requestBody:
+                $ref: 'invalid.yaml'
+      `,
+      path.join(__dirname, 'foobar.yaml'),
+    );
+
+    const results = await lintDocument({
+      externalRefResolver: new BaseResolver(),
+      document,
+      config: await makeConfig({
+        'no-unresolved-refs': 'error',
+      }),
+    });
+
+    expect(replaceSourceWithRef(results, __dirname)).toMatchInlineSnapshot(`
+      Array [
+        Object {
+          "location": Array [
+            Object {
+              "pointer": "#/paths/~1test/put/requestBody",
+              "reportOnKey": false,
+              "source": "foobar.yaml",
+            },
+          ],
+          "message": "Can't resolve $ref: ENOENT: no such file or directory, open 'invalid.yaml'",
+          "ruleId": "no-unresolved-refs",
+          "severity": "error",
+          "suggest": Array [],
+        },
+      ]
+    `);
+  });
+
+  it('should report on unresolved $ref yaml error', async () => {
+    const document = parseYamlToDocument(
+      outdent`
+        openapi: 3.0.0
+        paths:
+          '/test':
+            put:
+              requestBody:
+                $ref: 'fixtures/invalid-yaml.yaml'
+      `,
+      path.join(__dirname, 'foobar.yaml'),
+    );
+
+    const results = await lintDocument({
+      externalRefResolver: new BaseResolver(),
+      document,
+      config: await makeConfig({
+        'no-unresolved-refs': 'error',
+      }),
+    });
+
+    expect(replaceSourceWithRef(results, __dirname)).toMatchInlineSnapshot(`
+      Array [
+        Object {
+          "location": Array [
+            Object {
+              "pointer": undefined,
+              "reportOnKey": false,
+              "source": "fixtures/invalid-yaml.yaml",
+              "start": Object {
+                "col": 1,
+                "line": 2,
+              },
+            },
+          ],
+          "message": "Failed to parse: unexpected end of the stream within a single quoted scalar in \\"fixtures/invalid-yaml.yaml\\" (2:1)",
+          "ruleId": "no-unresolved-refs",
+          "severity": "error",
+          "suggest": Array [],
+        },
+        Object {
+          "location": Array [
+            Object {
+              "pointer": "#/paths/~1test/put/requestBody",
+              "reportOnKey": false,
+              "source": "foobar.yaml",
+            },
+          ],
+          "message": "Can't resolve $ref: unexpected end of the stream within a single quoted scalar in \\"fixtures/invalid-yaml.yaml\\" (2:1)",
+          "ruleId": "no-unresolved-refs",
+          "severity": "error",
+          "suggest": Array [],
+        },
+      ]
+    `);
+  });
+
+  it('should report on unresolved $ref yaml error', async () => {
+    const document = parseYamlToDocument(
+      outdent`
+        openapi: 3.0.0
+        paths:
+          '/test':
+            put:
+              requestBody:
+                $ref: 'fixtures/ref.yaml'
+      `,
+      path.join(__dirname, 'foobar.yaml'),
+    );
+
+    const results = await lintDocument({
+      externalRefResolver: new BaseResolver(),
+      document,
+      config: await makeConfig({
+        'no-unresolved-refs': 'error',
+      }),
+    });
+
+    expect(replaceSourceWithRef(results, __dirname)).toMatchInlineSnapshot(`Array []`);
+  });
+
+  it('should report on unresolved localr ref', async () => {
+    const document = parseYamlToDocument(
+      outdent`
+        openapi: 3.0.0
+        paths:
+          '/test':
+            put:
+              requestBody:
+                $ref: '#/components/requestBodies/a'
+      `,
+      path.join(__dirname, 'foobar.yaml'),
+    );
+
+    const results = await lintDocument({
+      externalRefResolver: new BaseResolver(),
+      document,
+      config: await makeConfig({
+        'no-unresolved-refs': 'error',
+      }),
+    });
+
+    expect(replaceSourceWithRef(results, __dirname)).toMatchInlineSnapshot(`
+      Array [
+        Object {
+          "location": Array [
+            Object {
+              "pointer": "#/paths/~1test/put/requestBody",
+              "reportOnKey": false,
+              "source": "foobar.yaml",
+            },
+          ],
+          "message": "Can't resolve $ref",
+          "ruleId": "no-unresolved-refs",
+          "severity": "error",
+          "suggest": Array [],
+        },
+      ]
+    `);
+  });
+});