@blamejs/core 0.18.50 → 0.18.53

package/lib/guard-yaml.js

@@ -68,6 +68,7 @@
  */
 
 var codepointClass = require("./codepoint-class");
+var yamlLex = require("./yaml-lex");
 var lazyRequire = require("./lazy-require");
 var gateContract = require("./gate-contract");
 var C = require("./constants");
@@ -95,11 +96,11 @@ var SAFE_CORE_TAGS = Object.freeze([
   "!!binary", "!!timestamp", "!!merge",
 ]);
 
-// Characters that may precede an anchor declaration (`&name`) or an alias
-// reference (`*name`) — the start of the document, whitespace, or one of the
-// structural characters a name can follow.
-var ANCHOR_LEAD_CHARS = ":-";
-var ALIAS_LEAD_CHARS  = ":-[{,";
+// There is no table of characters an anchor or alias is allowed to follow any
+// more. Two of them existed, one per sigil, and each was a guess at YAML's
+// grammar written as a character class: they admitted a sigil in the middle of
+// a plain scalar and excluded the compact flow form `{"a":&anchor v}` that YAML
+// permits. Position is decided once, by the shared lexer's mask.
 
 // The YAML 1.1 boolean-shaped tokens that make an unquoted scalar change type.
 // `true` and `false` are valid YAML 1.2 booleans and are not flagged; these
@@ -119,14 +120,17 @@ function _isSpace(cc) {
 
 // Every `&name` / `*name` in the document, given the sigil and the characters
 // that may precede it. Returns the names, in order.
-function _collectSigilNames(text, sigil, leadChars) {
+// `text` is the MASK, in which a sigil survives only where it opens a node, so
+// there is no test on the preceding character here at all. That judgement used
+// to be a hand-written list of characters a sigil was allowed to follow, kept
+// separately for anchors and for aliases, and it was wrong in both directions:
+// it admitted a `&` in the middle of a plain scalar, and it had no way to allow
+// `{"a":&anchor v}`, where YAML's JSON compatibility lets a quoted key take its
+// colon with no space after it. One scanner decides position now.
+function _collectSigilNames(text, sigil) {
   var out = [];
   for (var i = 0; i < text.length; i += 1) {
     if (text.charAt(i) !== sigil) continue;
-    if (i > 0) {
-      var lead = text.charCodeAt(i - 1);
-      if (!_isSpace(lead) && leadChars.indexOf(text.charAt(i - 1)) === -1) continue;
-    }
     if (!_isNameStart(text.charCodeAt(i + 1))) continue;
     var end = i + 2;
     while (end < text.length && _isNameChar(text.charCodeAt(end))) end += 1;
@@ -325,9 +329,11 @@ function _scanTags(text) {
   var tags = [];
   for (var i = 0; i < text.length; i += 1) {
     if (text.charAt(i) !== "!") continue;
-    var atStart = i === 0 ||
-                  codepointClass.inRanges(text.charCodeAt(i - 1), codepointClass.WHITESPACE_RANGES);
-    if (!atStart) continue;
+    // No "preceded by whitespace" test. That WAS the original defect, and once
+    // the mask decides position it is not merely redundant but harmful: in
+    // `{"a":!!python/object x}` the character before the tag is the colon of a
+    // JSON-style key, so the test skipped it and a deserialization tag reached
+    // both screens unreported. Position is the mask's question now.
     var nameAt = text.charAt(i + 1) === "!" ? i + 2 : i + 1;
     if (!codepointClass.isAsciiLetter(text.charCodeAt(nameAt))) continue;
     var end = nameAt + 1;
@@ -354,8 +360,22 @@ function _detectIssues(input, opts) {
   if (pre.done) return pre.issues;
   var issues = pre.issues;
 
+  // Sigil scans run against the MASK, not the source. A `!`, `&` or `*` means
+  // what it looks like only at a node start, and the previous rule — "after
+  // whitespace" — cannot tell a node start from the middle of a scalar. It
+  // reported a tag for the bang in a comment, in a quoted string, in a block
+  // scalar's shell script, and in ordinary prose: `x: 1 # note !bang` was
+  // refused. The mask is index-aligned with the source and the same length, so
+  // every location and line number reported below is still the source's.
+  //
+  // Only the three sigil scans use it. The value-shaped detectors further down
+  // (the Norway problem, leading zeros, merge keys) are asking about scalar
+  // CONTENT, which is exactly what the mask removes, so they keep reading the
+  // source.
+  var masked = yamlLex.maskNonStructural(input);
+
   // 1. Tag-injection scan.
-  var tagHits = _scanTags(input);
+  var tagHits = _scanTags(masked);
   for (var ti = 0; ti < tagHits.length; ti += 1) {
     var t = tagHits[ti];
     if (t.kind === "dangerous") {
@@ -392,8 +412,8 @@ function _detectIssues(input, opts) {
   }
 
   // 2. Anchor / alias recursion scan.
-  var anchors = _collectSigilNames(input, "&", ANCHOR_LEAD_CHARS);
-  var aliases = _collectSigilNames(input, "*", ALIAS_LEAD_CHARS);
+  var anchors = _collectSigilNames(masked, "&");
+  var aliases = _collectSigilNames(masked, "*");
   if (anchors.length > opts.maxAnchors) {
     issues.push({
       kind: "anchor-cap", severity: "high",
@@ -531,6 +551,24 @@ function _mappingEntryAt(line) {
   return null;
 }
 
+// The column the line's content starts in.
+function _indentOfLine(line) {
+  var i = 0;
+  while (i < line.length && _isSpace(line.charCodeAt(i))) i += 1;
+  return i;
+}
+
+// The indent of a sequence dash opening this line, or -1 when the line does not
+// open a sequence item. A dash counts only when it stands alone as a token: `-`
+// at end of line, or followed by a space. `-quux` and `-1` are scalars.
+function _sequenceDashIndent(line) {
+  var i = 0;
+  while (i < line.length && _isSpace(line.charCodeAt(i))) i += 1;
+  if (i >= line.length || line.charAt(i) !== "-") return -1;
+  if (i + 1 < line.length && !_isSpace(line.charCodeAt(i + 1))) return -1;
+  return i;
+}
+
 // Is the line blank, or a comment?
 function _isCommentLine(line) {
   var i = 0;
@@ -544,19 +582,117 @@ function _detectDuplicateKeysYaml(text) {
   var dups = Object.create(null);
   var lines = _splitLines(text);
   var indentScopes = Object.create(null);
+  // The sequence items currently open, outermost first, each with the column
+  // its first key sat in. Every key of an item at its top level is filed under
+  // one scope whatever column it is written in, because it is one mapping.
+  //
+  // A STACK, because sequences nest. Holding the innermost item in a pair of
+  // variables let a nested sequence overwrite its parent's scope and never give
+  // it back, so a key repeated in the OUTER item after the nested one closed
+  // was filed somewhere else and went unreported:
+  //
+  //     - a: 1
+  //       inner:
+  //         - x: 1
+  //       a: 2      <- a duplicate of the first `a`, and it was missed
+  //
+  // The same shape as the defect this detector reports, one level up: a single
+  // slot standing in for something there can be several of.
+  var itemStack = [];
   for (var i = 0; i < lines.length; i += 1) {
     var line = lines[i];
     if (line.length === 0 || _isCommentLine(line)) continue;
+    // A sequence item OPENS A NEW MAPPING, so it ends the previous item's and
+    // everything nested inside it. That is decided from the DASH, before
+    // anything else, because a line may be a sequence item and carry no mapping
+    // entry of its own:
+    //
+    //     steps:
+    //       -
+    //         p: 1
+    //       -
+    //         p: 2
+    //
+    // A dash alone has no `key: value` on it, so a reset that waited for one
+    // never ran and the second item's `p` was still read as a duplicate of the
+    // first's. The boundary is the dash; whether the item writes its first key
+    // beside it or underneath it is a matter of layout.
+    var dashAt = _sequenceDashIndent(line);
+    // An item is over once the document comes back out to its dash's column or
+    // further left. That is true of the next item's own dash as much as of a
+    // key belonging to the enclosing mapping, so it is measured here, before
+    // this line is classified at all.
+    var lineIndent = dashAt >= 0 ? dashAt : _indentOfLine(line);
+    while (itemStack.length &&
+           itemStack[itemStack.length - 1].dash >= lineIndent) itemStack.pop();
+    if (dashAt >= 0) {
+      Object.keys(indentScopes).forEach(function (k) {
+        if (Number(k) > dashAt) delete indentScopes[k];
+      });
+      // keyIndent is set by this item's first key, wherever it is written.
+      itemStack.push({ dash: dashAt, keyIndent: -1 });
+    }
     var entry = _mappingEntryAt(line);
     if (!entry) continue;
     var indent = entry.indent;
     var key = entry.key.trim();
-    if (key.charAt(0) === "-" || key.charAt(0) === "[" || key.charAt(0) === "{") continue;
-    if (!indentScopes[indent]) indentScopes[indent] = Object.create(null);
-    if (indentScopes[indent][key]) dups[key] = true;
-    else indentScopes[indent][key] = true;
+    if (key.charAt(0) === "[" || key.charAt(0) === "{") continue;
+    // The key written INLINE with the dash belongs to the item's mapping, and
+    // sits at the indent AFTER the dash and its space — so that is the scope it
+    // is registered in, alongside the keys written underneath it. It used to be
+    // skipped entirely, which meant repeating it went unreported.
+    var dash = key.charAt(0) === "-" &&
+               (key.length === 1 || _isSpace(key.charCodeAt(1)));
+    var scopeAt = indent;
+    if (dash) {
+      var after = 1;
+      while (after < key.length && _isSpace(key.charCodeAt(after))) after += 1;
+      key = key.slice(after).trim();
+      // `- ` alone, or `- - x`: no inline key of this item's own to register.
+      if (!key || key.charAt(0) === "-" || key.charAt(0) === "[" ||
+          key.charAt(0) === "{") continue;
+      scopeAt = indent + after;
+    }
+    // The item's mapping is ONE mapping however its keys are laid out, so the
+    // scope it is tracked under must not depend on spacing. `-   a: 1` puts its
+    // inline key at column 4 while the key written under it sits at column 2,
+    // and keying on the raw column filed them separately — so a repeat across
+    // those two lines went unreported, which is exactly the smuggling shape
+    // this detector exists for (one parser reads two keys, another reads one).
+    //
+    // The item's top level is every key deeper than the dash and no deeper than
+    // the first key it saw. Anything past that is genuinely nested and keeps
+    // its own column, so `- a: 1` / `  b:` / `    a: 2` is still not a
+    // duplicate.
+    // Half a column past the dash: a number, so the pruning comparisons below
+    // and at the dash keep working unchanged, and one that sorts between the
+    // dash and anything nested inside the item.
+    var item = itemStack.length ? itemStack[itemStack.length - 1] : null;
+    if (item && scopeAt > item.dash) {
+      // The item's top level is every key no deeper than the SHALLOWEST key it
+      // has shown, and the bound moves down as shallower ones appear. Two
+      // failures pinned this from opposite sides:
+      //
+      //   -   a: 1        the inline key sits at column 4 because of the extra
+      //     b:            spacing, but the item's mapping is written at 2. If
+      //       a: 2        4 is taken as the bound, the NESTED `a` at 4 counts
+      //                   as top level and reads as a duplicate.
+      //
+      //   - a:            here the inline key IS the bound, at 2. If it sets
+      //       x: 1        nothing, the first nested key at 4 becomes the bound
+      //       a: 2        and the nested `a` reads as a duplicate instead.
+      //
+      // So the inline key establishes the bound and a later, shallower key
+      // lowers it. Extra spacing after the indicator is presentation; the
+      // shallowest key is the structure.
+      if (item.keyIndent === -1 || scopeAt < item.keyIndent) item.keyIndent = scopeAt;
+      if (scopeAt <= item.keyIndent) scopeAt = item.dash + 0.5;
+    }
+    if (!indentScopes[scopeAt]) indentScopes[scopeAt] = Object.create(null);
+    if (indentScopes[scopeAt][key]) dups[key] = true;
+    else indentScopes[scopeAt][key] = true;
     Object.keys(indentScopes).forEach(function (k) {
-      if (Number(k) > indent) delete indentScopes[k];
+      if (Number(k) > scopeAt) delete indentScopes[k];
     });
   }
   return Object.keys(dups);

package/lib/json-schema.js

@@ -141,10 +141,52 @@ function _unescapePointerToken(t) { return t.replace(/~1/g, "/").replace(/~0/g,
 
 // --- registry: indexes every subschema by canonical URI + anchors ---
 
-function _Registry() { this.schemas = {}; this.dynamicAnchors = {}; this.baseByNode = new Map(); }
+// A ceiling on how many subschema POSITIONS one registry will index, across
+// every document added to it.
+//
+// This is the total-work half of the bound, and it is a different guarantee
+// from a depth ceiling: depth limits how far one path runs, and says nothing
+// about how many paths there are. The walk below cannot use identity tracking
+// instead, because it indexes each node by `base#pointer` and a shared object
+// legitimately has one entry per pointer that reaches it — skipping the second
+// visit would drop a `$ref` target rather than save work. So what gets bounded
+// is the number of entries, which is the thing that actually grows.
+//
+// A schema graph that reaches one object through a branching position —
+// `anyOf: [shared, shared]`, which is what building a schema out of reused
+// JavaScript objects produces, and which needs no `$ref` — has 2^depth pointers
+// over a handful of objects. A cyclic one has no end at all. Neither is refused
+// by a depth cap.
+//
+// Far above any authored schema: the JSON Schema meta-schema is on the order of
+// a hundred positions, and an OpenAPI document of a thousand endpoints is tens
+// of thousands. A schema that reaches this is not one someone wrote out.
+var MAX_REGISTRY_NODES = 100000;                            // subschema positions, not bytes
+
+// How deep a schema may nest before it is refused. This is the OTHER half of
+// the bound, and it is not the node ceiling in disguise: a chain ten thousand
+// levels deep is ten thousand positions, far under that ceiling, and still runs
+// the JavaScript stack out. What comes back then is `RangeError: Maximum call
+// stack size exceeded` — the engine's error, which says nothing about the
+// schema and is not catchable as a JsonSchemaError.
+//
+// It THROWS rather than stopping the walk. A walk that silently returns at the
+// ceiling leaves a node looking examined when its descendants were skipped,
+// which is how a pattern below one escapes screening entirely. Refusing is the
+// only reading that cannot be mistaken for completion.
+//
+// Well above anything authored — the JSON Schema meta-schema nests about a
+// dozen levels and an OpenAPI document a few dozen — and well below where the
+// stack gives out.
+var MAX_SCHEMA_NESTING = 1000;                              // nesting levels, not bytes
+
+function _Registry() {
+  this.schemas = {}; this.dynamicAnchors = {}; this.baseByNode = new Map();
+  this.nodesIndexed = 0;
+}
 
 _Registry.prototype.add = function (schema, baseUri) {
-  this._walk(schema, baseUri || "", "");
+  this._walk(schema, baseUri || "", "", null, 0);
   // A document retrieved from URI X is addressable by X even when its own
   // $id is a different (canonical) URI — register the retrieval URI too.
   if (baseUri && (_isObject(schema) || typeof schema === "boolean")) {
@@ -156,10 +198,43 @@ _Registry.prototype.add = function (schema, baseUri) {
 // Walk a schema document, registering $id base changes, $anchor and
 // $dynamicAnchor names, and indexing every subschema by its base URI +
 // JSON-pointer fragment.
-_Registry.prototype._walk = function (node, baseUri, pointer) {
+_Registry.prototype._walk = function (node, baseUri, pointer, path, depth) {
   if (!_isObject(node) && typeof node !== "boolean") return;
+  if (depth > MAX_SCHEMA_NESTING) {
+    throw new JsonSchemaError("json-schema/schema-too-deep",
+      "jsonSchema: schema nests deeper than " + MAX_SCHEMA_NESTING +
+      " levels — deeper than the walk can index without exhausting the stack");
+  }
+  this.nodesIndexed += 1;
+  if (this.nodesIndexed > MAX_REGISTRY_NODES) {
+    throw new JsonSchemaError("json-schema/schema-too-large",
+      "jsonSchema: schema indexes more than " + MAX_REGISTRY_NODES +
+      " subschema positions — a shared or cyclic schema graph reaches one " +
+      "object through many pointers; give the reused subschema an $id and " +
+      "reference it with $ref instead of embedding the same object twice");
+  }
   if (typeof node === "boolean") { this.schemas[baseUri + "#" + pointer] = node; return; }
 
+  // A node already on the ACTIVE PATH is a cycle, and a cycle with one
+  // reference per level never reaches the ceiling above: it does not branch, so
+  // it recurses straight down and exhausts the JavaScript stack while the count
+  // is still in the thousands, surfacing as a RangeError from the engine rather
+  // than the refusal this validator documents.
+  //
+  // The test is the path and not a depth cap, because deep is not the same as
+  // cyclic: a schema nesting four hundred levels with no cycle and no `$ref`
+  // compiles today, and a depth cap would refuse it — under a `ref-loop` code
+  // naming a reference chain it does not have. The path set answers the
+  // question actually being asked, and leaves depth alone.
+  path = path || new Set();
+  var isOwnAncestor = path.has(node);
+  if (isOwnAncestor) {
+    throw new JsonSchemaError("json-schema/ref-loop",
+      "jsonSchema: schema contains a cycle — a subschema is its own ancestor, " +
+      "so indexing it has no end; break the cycle with $id + $ref");
+  }
+  path.add(node);
+
   var thisBase = baseUri;
   if (typeof node.$id === "string") {
     thisBase = _resolveUri(node.$id, baseUri);
@@ -191,7 +266,7 @@ _Registry.prototype._walk = function (node, baseUri, pointer) {
   // Recurse. Keywords whose values are schemas vs maps-of-schemas vs
   // arrays-of-schemas are walked with the right shape.
   var self = this;
-  function child(key, sub, ptr) { self._walk(sub, thisBase, ptr); }
+  function child(key, sub, ptr) { self._walk(sub, thisBase, ptr, path, depth + 1); }
   SCHEMA_KEYWORDS.forEach(function (k) {
     if (node[k] !== undefined) child(k, node[k], pointer + "/" + k);
   });
@@ -207,6 +282,12 @@ _Registry.prototype._walk = function (node, baseUri, pointer) {
       node[k].forEach(function (sub, idx) { child(k, sub, pointer + "/" + k + "/" + idx); });
     }
   });
+
+  // Off the active path on the way out. A node reached again through a SIBLING
+  // branch is shared rather than cyclic, and indexing it under its second
+  // pointer is the correct thing to do — leaving it in the set would call that
+  // a cycle and refuse a schema that merely reuses a subschema.
+  path.delete(node);
 };
 
 _Registry.prototype.resolve = function (uri) {
@@ -255,7 +336,13 @@ var SCHEMA_KEYWORDS = ["additionalProperties", "propertyNames", "items",
   "unevaluatedProperties"];
 var SCHEMA_MAP_KEYWORDS = ["$defs", "definitions", "properties",
   "patternProperties", "dependentSchemas"];
-var SCHEMA_ARRAY_KEYWORDS = ["allOf", "anyOf", "oneOf", "prefixItems"];
+// `items` appears in BOTH lists on purpose. Draft 2020-12 makes it a single
+// schema, and the legacy tuple form makes it an array of them; each branch
+// checks the type it wants, so exactly one of them walks any given `items`.
+// Listed only as a single schema, an array-valued one was dropped at the
+// `_isObject` test at the top of the walk and never indexed at all — so the
+// subschemas inside a tuple were unreachable by `$ref`, and unbudgeted.
+var SCHEMA_ARRAY_KEYWORDS = ["allOf", "anyOf", "oneOf", "prefixItems", "items"];
 
 module.exports = _buildModule();
 
@@ -315,25 +402,55 @@ var _SUBSCHEMA_MAP_KEYS = ["$defs", "definitions", "dependentSchemas",
 
 // Walk the subschema positions for the two keywords that carry a regular
 // expression, and hand each to the same screen the compile site uses.
-// Depth-bounded by the ceiling the validator applies to `$ref` chains, so a
-// deeply nested schema cannot spend the walk instead of the match.
-function _screenPatterns(node, depth) {
-  depth = depth || 0;
-  if (depth > MAX_REF_DEPTH || !_isObject(node)) return;
+//
+// Every object is visited ONCE, tracked by identity. The depth ceiling below is
+// not a substitute for that and never was: it limits how far a single path runs
+// and says nothing about how many paths there are. A schema graph reaching one
+// object through a branching position — `anyOf: [shared, shared]`, which is what
+// building a schema in JavaScript out of reused subschema objects produces, and
+// which needs no `$ref` — was walked once per path. Twenty-one objects cost
+// 2.6 seconds, doubling per added level, against a cap that allows 256; a cyclic
+// schema never returned. This screen exists so a pattern cannot make the
+// validator hang, and SECURITY.md promises a screen costs the length of its
+// input and never a function of its shape, so a walk that is itself a function
+// of shape contradicts the thing it was added to guarantee.
+//
+// Identity tracking bounds the total work by the number of distinct objects,
+// which is the length of the schema — so the promise holds without a separate
+// node budget.
+//
+// The nesting ceiling here THROWS, and that distinction is the whole point. A
+// ceiling that silently stopped walking would leave a node looking examined
+// when its descendants were skipped, so a later and shallower path would find
+// it already marked and skip it too — and a catastrophic pattern below it would
+// never be screened at all, which is the opposite of what this walk is for.
+// Refusing cannot be mistaken for completion.
+function _screenPatterns(node, seen, depth) {
+  if (!_isObject(node)) return;
+  if (depth > MAX_SCHEMA_NESTING) {
+    throw new JsonSchemaError("json-schema/schema-too-deep",
+      "jsonSchema: schema nests deeper than " + MAX_SCHEMA_NESTING +
+      " levels — deeper than the pattern screen can walk without exhausting " +
+      "the stack");
+  }
+  seen = seen || new WeakSet();
+  if (seen.has(node)) return;
+  seen.add(node);
 
   if (typeof node.pattern === "string") _compileRegex(node.pattern);
   if (_isObject(node.patternProperties)) {
     Object.keys(node.patternProperties).forEach(function (p) { _compileRegex(p); });
   }
 
-  _SUBSCHEMA_KEYS.forEach(function (k) { _screenPatterns(node[k], depth + 1); });
+  var next = (depth || 0) + 1;
+  _SUBSCHEMA_KEYS.forEach(function (k) { _screenPatterns(node[k], seen, next); });
   _SUBSCHEMA_LIST_KEYS.forEach(function (k) {
     if (!Array.isArray(node[k])) return;
-    node[k].forEach(function (sub) { _screenPatterns(sub, depth + 1); });
+    node[k].forEach(function (sub) { _screenPatterns(sub, seen, next); });
   });
   _SUBSCHEMA_MAP_KEYS.forEach(function (k) {
     if (!_isObject(node[k])) return;
-    Object.keys(node[k]).forEach(function (name) { _screenPatterns(node[k][name], depth + 1); });
+    Object.keys(node[k]).forEach(function (name) { _screenPatterns(node[k][name], seen, next); });
   });
 }