@redocly/openapi-core 1.0.0-beta.100

package/src/benchmark/benches/recommended-oas3.bench.ts

@@ -0,0 +1,22 @@
+import { readFileSync } from 'fs';
+import { join as pathJoin, resolve as pathResolve } from 'path';
+import { lintDocument } from '../../lint';
+import { LintConfig, defaultPlugin, resolvePreset } from '../../config';
+import { BaseResolver } from '../../resolve';
+import { parseYamlToDocument } from '../utils';
+
+export const name = 'Validate with recommended rules';
+export const count = 10;
+const rebillyDefinitionRef = pathResolve(pathJoin(__dirname, 'rebilly.yaml'));
+const rebillyDocument = parseYamlToDocument(
+  readFileSync(rebillyDefinitionRef, 'utf-8'),
+  rebillyDefinitionRef,
+);
+
+export function measureAsync() {
+  return lintDocument({
+    externalRefResolver: new BaseResolver(),
+    document: rebillyDocument,
+    config: new LintConfig(resolvePreset('recommended', [defaultPlugin])),
+  });
+}

package/src/benchmark/benches/resolve-with-no-external.bench.ts

@@ -0,0 +1,23 @@
+import * as path from 'path';
+import { readFileSync } from 'fs';
+import { resolveDocument, BaseResolver } from '../../resolve';
+import { parseYamlToDocument } from '../utils';
+import { Oas3Types } from '../../types/oas3';
+import { normalizeTypes } from '../../types';
+
+export const name = 'Resolve with no external refs';
+export const count = 10;
+const rebillyDefinitionRef = path.resolve(path.join(__dirname, 'rebilly.yaml'));
+const rebillyDocument = parseYamlToDocument(
+  readFileSync(rebillyDefinitionRef, 'utf-8'),
+  rebillyDefinitionRef,
+);
+const externalRefResolver = new BaseResolver();
+
+export function measureAsync() {
+  return resolveDocument({
+    rootDocument: rebillyDocument,
+    externalRefResolver,
+    rootType: normalizeTypes(Oas3Types).DefinitionRoot,
+  });
+}

package/src/benchmark/benchmark.js

@@ -0,0 +1,307 @@
+'use strict';
+const os = require('os');
+const fs = require('fs');
+const path = require('path');
+const assert = require('assert');
+const { execSync } = require('child_process');
+const { red, green, yellow, cyan, grey } = require('./colors');
+const { sampleModule } = require('./fork');
+const NS_PER_SEC = 1e9;
+const LOCAL = 'local';
+
+// The maximum time in seconds a benchmark is allowed to run before finishing.
+const maxTime = 10;
+// The minimum sample size required to perform statistical analysis.
+const minSamples = 5;
+
+function LOCAL_DIR(...paths) {
+  return path.join(__dirname, '..', ...paths);
+}
+
+// Build a benchmark-friendly environment for the given revision
+// and returns path to its 'lib' directory.
+function prepareRevision(revision) {
+  console.log(`🍳  Preparing ${revision}...`);
+
+  if (revision === LOCAL) {
+    return tscBuild(path.join(LOCAL_DIR(), '../../..'));
+  }
+
+  // Returns the complete git hash for a given git revision reference.
+  const hash = exec(`git rev-parse "${revision}"`);
+  const dir = path.join(os.tmpdir(), 'redocly-cli-benchmark', hash);
+  if (fs.existsSync(dir)) {
+    fs.rmdirSync(dir, { recursive: true});
+  }
+  fs.mkdirSync(dir, { recursive: true });
+
+  exec(`git archive "${hash}" | tar -xC "${dir}"`);
+  exec('npm ci', { cwd: dir });
+  const to = path.join(dir, 'packages/core/src/benchmark/benches');
+  exec(`rm -rf ${to}`);
+  exec(`cp -R ${LOCAL_DIR('benchmark/benches')} ${to}`);
+  return tscBuild(dir);
+}
+
+function tscBuild(dir) {
+  const oldCwd = process.cwd();
+  process.chdir(dir);
+  execSync('npm run compile', { stdio: 'inherit' });
+  exec(
+    `cp ${path.join(dir, 'packages/core/src/benchmark/benches/*.yaml')} ${path.join(dir, 'packages/core/lib/benchmark/benches/')}`,
+  );
+  process.chdir(oldCwd);
+  return path.join(dir, 'packages/core/lib/benchmark/benches');
+}
+
+async function collectSamples(modulePath) {
+  const samples = [];
+
+  // If time permits, increase sample size to reduce the margin of error.
+  const start = Date.now();
+  while (samples.length < minSamples || (Date.now() - start) / 1e3 < maxTime) {
+    const { clocked, memUsed } = await sampleModule(modulePath);
+    assert(clocked > 0);
+    assert(memUsed > 0);
+    samples.push({ clocked, memUsed });
+  }
+  return samples;
+}
+
+// T-Distribution two-tailed critical values for 95% confidence.
+// See http://www.itl.nist.gov/div898/handbook/eda/section3/eda3672.htm.
+const tTable = /* prettier-ignore */ {
+  '1':  12.706, '2':  4.303, '3':  3.182, '4':  2.776, '5':  2.571, '6':  2.447,
+  '7':  2.365,  '8':  2.306, '9':  2.262, '10': 2.228, '11': 2.201, '12': 2.179,
+  '13': 2.16,   '14': 2.145, '15': 2.131, '16': 2.12,  '17': 2.11,  '18': 2.101,
+  '19': 2.093,  '20': 2.086, '21': 2.08,  '22': 2.074, '23': 2.069, '24': 2.064,
+  '25': 2.06,   '26': 2.056, '27': 2.052, '28': 2.048, '29': 2.045, '30': 2.042,
+  infinity: 1.96,
+};
+
+// Computes stats on benchmark results.
+function computeStats(samples) {
+  assert(samples.length > 1);
+
+  // Compute the sample mean (estimate of the population mean).
+  let mean = 0;
+  let meanMemUsed = 0;
+  for (const { clocked, memUsed } of samples) {
+    mean += clocked;
+    meanMemUsed += memUsed;
+  }
+  mean /= samples.length;
+  meanMemUsed /= samples.length;
+
+  // Compute the sample variance (estimate of the population variance).
+  let variance = 0;
+  for (const { clocked } of samples) {
+    variance += (clocked - mean) ** 2;
+  }
+  variance /= samples.length - 1;
+
+  // Compute the sample standard deviation (estimate of the population standard deviation).
+  const sd = Math.sqrt(variance);
+
+  // Compute the standard error of the mean (a.k.a. the standard deviation of the sampling distribution of the sample mean).
+  const sem = sd / Math.sqrt(samples.length);
+
+  // Compute the degrees of freedom.
+  const df = samples.length - 1;
+
+  // Compute the critical value.
+  const critical = tTable[df] || tTable.infinity;
+
+  // Compute the margin of error.
+  const moe = sem * critical;
+
+  // The relative margin of error (expressed as a percentage of the mean).
+  const rme = (moe / mean) * 100 || 0;
+
+  return {
+    memPerOp: Math.floor(meanMemUsed),
+    ops: NS_PER_SEC / mean,
+    deviation: rme,
+    numSamples: samples.length,
+  };
+}
+
+function beautifyBenchmark(results) {
+  const nameMaxLen = maxBy(results, ({ name }) => name.length);
+  const opsTop = maxBy(results, ({ ops }) => ops);
+  const opsMaxLen = maxBy(results, ({ ops }) => beautifyNumber(ops).length);
+  const memPerOpMaxLen = maxBy(results, ({ memPerOp }) => beautifyBytes(memPerOp).length);
+
+  for (const result of results) {
+    printBench(result);
+  }
+
+  function printBench(bench) {
+    const { name, memPerOp, ops, deviation, numSamples } = bench;
+    console.log(
+      '  ' +
+        nameStr() +
+        grey(' x ') +
+        opsStr() +
+        ' ops/sec ' +
+        grey('\xb1') +
+        deviationStr() +
+        cyan('%') +
+        grey(' x ') +
+        memPerOpStr() +
+        '/op' +
+        grey(' (' + numSamples + ' runs sampled)'),
+    );
+
+    function nameStr() {
+      const nameFmt = name.padEnd(nameMaxLen);
+      return ops === opsTop ? green(nameFmt) : nameFmt;
+    }
+
+    function opsStr() {
+      const percent = ops / opsTop;
+      const colorFn = percent > 0.95 ? green : percent > 0.8 ? yellow : red;
+      return colorFn(beautifyNumber(ops).padStart(opsMaxLen));
+    }
+
+    function deviationStr() {
+      const colorFn = deviation > 5 ? red : deviation > 2 ? yellow : green;
+      return colorFn(deviation.toFixed(2));
+    }
+
+    function memPerOpStr() {
+      return beautifyBytes(memPerOp).padStart(memPerOpMaxLen);
+    }
+  }
+}
+
+function beautifyBytes(bytes) {
+  const sizes = ['Bytes', 'KB', 'MB', 'GB'];
+  const i = Math.floor(Math.log2(bytes) / 10);
+  return beautifyNumber(bytes / 2 ** (i * 10)) + ' ' + sizes[i];
+}
+
+function beautifyNumber(num) {
+  return Number(num.toFixed(num > 100 ? 0 : 2)).toLocaleString();
+}
+
+function maxBy(array, fn) {
+  return Math.max(...array.map(fn));
+}
+
+// Prepare all revisions and run benchmarks matching a pattern against them.
+async function prepareAndRunBenchmarks(benchmarkPatterns, revisions) {
+  const environments = revisions.map((revision) => ({
+    revision,
+    distPath: prepareRevision(revision),
+  }));
+
+  console.log(`🚀  Starting benchmarks...`);
+
+  for (const benchmark of matchBenchmarks(benchmarkPatterns)) {
+    const results = [];
+    for (let i = 0; i < environments.length; ++i) {
+      const environment = environments[i];
+      const modulePath = path.join(environment.distPath, benchmark);
+
+      if (i === 0) {
+        const { name } = await sampleModule(modulePath);
+        console.log('⏱️   ' + name);
+      }
+
+      try {
+        const samples = await collectSamples(modulePath);
+
+        results.push({
+          name: environment.revision,
+          samples,
+          ...computeStats(samples),
+        });
+        process.stdout.write('  ' + cyan(i + 1) + ' tests completed.\u000D');
+      } catch (error) {
+        console.log('  ' + environment.revision + ': ' + red(String(error)));
+      }
+    }
+    console.log('\n');
+
+    beautifyBenchmark(results);
+    console.log('');
+  }
+}
+
+function findFiles(cwd, pattern) {
+  const out = exec(`find . -path '${pattern}'`, { cwd });
+  return out.split('\n').filter(Boolean);
+}
+
+function removeTrailingNewLine(str) {
+  if (str == null) {
+    return str;
+  }
+  return str.split('\n').slice(0, -1).join('\n');
+}
+
+function exec(command, options) {
+  const output = execSync(command, {
+    maxBuffer: 10 * 1024 * 1024, // 10MB
+    encoding: 'utf-8',
+    ...options,
+  });
+  return removeTrailingNewLine(output);
+}
+
+// Find all benchmark tests to be run.
+function matchBenchmarks(patterns) {
+  let benchmarks = findFiles(LOCAL_DIR('../lib/benchmark/benches'), '*.bench.js');
+  if (patterns.length > 0) {
+    benchmarks = benchmarks.filter((benchmark) =>
+      patterns.some((pattern) => path.join('../lib/benchmark/benches', benchmark).includes(pattern)),
+    );
+  }
+
+  if (benchmarks.length === 0) {
+    console.warn('No benchmarks matching: ' + patterns.map(bold).join(''));
+  }
+
+  return benchmarks;
+}
+
+function getArguments(argv) {
+  const revsIdx = argv.indexOf('--revs');
+  const revsArgs = revsIdx === -1 ? [] : argv.slice(revsIdx + 1);
+  const benchmarkPatterns = revsIdx === -1 ? argv : argv.slice(0, revsIdx);
+  let assumeArgs;
+  let revisions;
+  switch (revsArgs.length) {
+    case 0:
+      assumeArgs = [...benchmarkPatterns, '--revs', 'local', 'HEAD'];
+      revisions = [LOCAL, 'HEAD'];
+      break;
+    case 1:
+      if (revsArgs[0] === LOCAL) {
+        assumeArgs = [...benchmarkPatterns, '--revs', revsArgs[0]];
+        revisions = [revsArgs[0]];
+      } else {
+        assumeArgs = [...benchmarkPatterns, '--revs', 'local', revsArgs[0]];
+        revisions = [LOCAL, revsArgs[0]];
+      }
+      break;
+    default:
+      revisions = revsArgs;
+      break;
+  }
+  if (assumeArgs) {
+    console.warn('Assuming you meant: ' + bold('benchmark ' + assumeArgs.join(' ')));
+  }
+  return { benchmarkPatterns, revisions };
+}
+
+function bold(str) {
+  return '\u001b[1m' + str + '\u001b[0m';
+}
+
+// Get the revisions and make things happen!
+if (require.main === module) {
+  const { benchmarkPatterns, revisions } = getArguments(process.argv.slice(2));
+  prepareAndRunBenchmarks(benchmarkPatterns, revisions);
+}

package/src/benchmark/colors.js

@@ -0,0 +1,29 @@
+'use strict';
+
+function red(str) {
+  return '\u001b[31m' + str + '\u001b[0m';
+}
+
+function green(str) {
+  return '\u001b[32m' + str + '\u001b[0m';
+}
+
+function yellow(str) {
+  return '\u001b[33m' + str + '\u001b[0m';
+}
+
+function cyan(str) {
+  return '\u001b[36m' + str + '\u001b[0m';
+}
+
+function grey(str) {
+  return '\u001b[90m' + str + '\u001b[0m';
+}
+
+module.exports = {
+  red,
+  green,
+  yellow,
+  cyan,
+  grey,
+};

package/src/benchmark/fork.js

@@ -0,0 +1,83 @@
+'use strict';
+
+const assert = require('assert');
+const cp = require('child_process');
+
+// Clocks the time taken to execute a test per cycle (secs).
+function clock(count, fn) {
+  const start = process.hrtime.bigint();
+  for (let i = 0; i < count; ++i) {
+    fn();
+  }
+  return Number(process.hrtime.bigint() - start);
+}
+
+async function clockAsync(count, fn) {
+  const start = process.hrtime.bigint();
+  for (let i = 0; i < count; ++i) {
+    await fn();
+  }
+  return Number(process.hrtime.bigint() - start);
+}
+
+if (require.main === module) {
+  const modulePath = process.env.BENCHMARK_MODULE_PATH;
+  assert(typeof modulePath === 'string');
+  assert(process.send);
+  const module = require(modulePath);
+
+  if (module.measureAsync) {
+    async function run() {
+      await clockAsync(7, module.measureAsync); // warm up
+      global.gc();
+      process.nextTick(async () => {
+        const memBaseline = process.memoryUsage().heapUsed;
+        const clocked = await clockAsync(module.count, module.measureAsync);
+        process.send({
+          name: module.name,
+          clocked: clocked / module.count,
+          memUsed: (process.memoryUsage().heapUsed - memBaseline) / module.count,
+        });
+      });
+    }
+    run();
+  } else {
+    clock(7, module.measure); // warm up
+    global.gc();
+    process.nextTick(() => {
+      const memBaseline = process.memoryUsage().heapUsed;
+      const clocked = clock(module.count, module.measure);
+      process.send({
+        name: module.name,
+        clocked: clocked / module.count,
+        memUsed: (process.memoryUsage().heapUsed - memBaseline) / module.count,
+      });
+    });
+  }
+}
+
+function sampleModule(modulePath) {
+  return new Promise((resolve, reject) => {
+    const env = {
+      NODE_ENV: 'production',
+      BENCHMARK_MODULE_PATH: modulePath,
+    };
+    const child = cp.fork(__filename, { env });
+    let message;
+    let error;
+
+    child.on('message', (msg) => (message = msg));
+    child.on('error', (e) => (error = e));
+    child.on('close', () => {
+      if (message) {
+        return resolve(message);
+      }
+      reject(error || new Error('Forked process closed without error'));
+    });
+  }).then((result) => {
+    global.gc();
+    return result;
+  });
+}
+
+module.exports = { sampleModule };

package/src/benchmark/utils.ts

@@ -0,0 +1,36 @@
+import { parseYaml } from '../js-yaml';
+import { Document, Source } from '../resolve';
+import { Oas3RuleSet } from '../oas-types';
+import { LintConfig, mergeExtends, resolvePlugins } from '../config';
+
+import type { RuleConfig, Plugin, ResolvedLintConfig } from '../config/types';
+
+export function parseYamlToDocument(body: string, absoluteRef: string = ''): Document {
+  return {
+    source: new Source(absoluteRef, body),
+    parsed: parseYaml(body, { filename: absoluteRef }),
+  };
+}
+
+export function makeConfigForRuleset(rules: Oas3RuleSet, plugin?: Partial<Plugin>) {
+  const rulesConf: Record<string, RuleConfig> = {};
+  const ruleId = 'test';
+  Object.keys(rules).forEach((name) => {
+    rulesConf[`${ruleId}/${name}`] = 'error';
+  });
+  const extendConfigs = [
+    resolvePlugins([
+      {
+        ...plugin,
+        id: ruleId,
+        rules: { oas3: rules },
+      },
+    ]) as ResolvedLintConfig,
+  ];
+  if (rules) {
+    extendConfigs.push({ rules });
+  }
+  const lint = mergeExtends(extendConfigs);
+
+  return new LintConfig(lint);
+}