@contrast/assess 1.46.2 → 1.47.0

package/lib/dataflow/propagation/install/string/replace.js

@@ -18,11 +18,10 @@
 const {
   DataflowTag: { UNTRUSTED },
   primordials: {
-    StringPrototypeMatch,
     ArrayPrototypeJoin,
-    ArrayPrototypeSlice,
-    StringPrototypeSubstring,
-    StringPrototypeReplace
+    StringPrototypeMatchAll,
+    StringPrototypeReplace,
+    StringPrototypeReplaceAll,
   }
 } = require('@contrast/common');
 const {
@@ -43,98 +42,157 @@ module.exports = function(core) {
   } = core;
 
   function parseArgs(args) {
-    // [match, p1, p2, ..., matchIdx, str, ?groups]
-    const match = args[0];
-    const len = args.length;
-    const lastEl = args[len - 1];
-    const hasNamedGroup = typeof lastEl === 'object';
-    const captureGroups = ArrayPrototypeSlice.call(args, 1, hasNamedGroup ? -3 : -2);
-    const matchIdx = args[hasNamedGroup ? len - 3 : len - 2];
-    const str = hasNamedGroup ? args[len - 2] : lastEl;
-    const namedGroups = hasNamedGroup ? lastEl : null;
-    return { match, captureGroups, matchIdx, str, namedGroups };
-  }
-
-  // these functions specifically use the patched versions of Substring so that
-  // the tags are propagated.
-  function replaceSpecialCharacters(replacement, { captureGroups, match, str, namedGroups }, replacementType) {
-    let ret = replacement;
-    [
-      {
-        regex: /\$\$/g,
-        replace: '$'
-      },
-      {
-        regex: /\$&/g,
-        replace: match
-      },
-      {
-        regex: /\$`/g,
-        replace: str.substring(0, str.indexOf(match))
-      },
-      {
-        regex: /\$'/g,
-        replace: str.substring(str.indexOf(match) + match.length, str.length)
-      }
-    ].forEach(({ regex, replace }) => {
-      if (ret && StringPrototypeMatch.call(ret, regex)) {
-        // If the match string is tracked, we can actually use the patched replace
-        // to keep track of its tag ranges
-        if (tracker.getData(replace)) {
-          ret = ret.replace(regex, replace);
-        } else {
-          ret = StringPrototypeReplace.call(ret, regex, replace);
-        }
-      }
-    });
-
-    const numberedGroupMatches = replacementType !== 'function' && StringPrototypeMatch.call(replacement, /\$[1-9][0-9]|\$[1-9]/g);
-    if (numberedGroupMatches) {
-      numberedGroupMatches.forEach((numberedGroup) => {
-        const group = Number(StringPrototypeSubstring.call(numberedGroup, 1));
-        ret = StringPrototypeReplace.call(ret, numberedGroup, captureGroups[group - 1] || '');
-      });
+    //
+    // convert replace args (match, p1, p2, ..., matchIdx, str, ?groups) to
+    // re.exec(string) format [match, g1, g2, ..., index: N, input: '', groups?]
+    //
+    const r = [];
+    let ix = -1;
+    if (typeof args.at(-1) === 'object') {
+      r.groups = args.at(-1);
+      ix = -2;
     }
+    r.input = args.at(ix);
+    ix -= 1;
+    r.index = args.at(ix);
 
-    if (namedGroups) {
-      for (const name in namedGroups) {
-        ret = StringPrototypeReplace.call(ret, `$${name}`, namedGroups[name]);
-      }
+    // ix is negative from the end, so add it
+    ix = args.length + ix;
+    for (let i = 0; i < ix; i++) {
+      r.push(args[i]);
     }
 
-    return ret;
+    return r;
   }
 
+  const RE_REPS = /\$(\$|&|`|'|[1-9][0-9]?|<[a-zA-Z0-9_]+>)/g;
+  const STR_REPS = /\$(\$|&|`|')/g;
+
   function getReplacementInfo(data, replacerArgs, parsedArgs) {
-    let replacement = data._replacementType === 'function' ?
-      data._replacement.call(global, ...replacerArgs) :
-      data._replacement;
+    // patternIsRE is two different flags, both booleans.
+    // - set true if the first argument to String.prototype.replace is an RE.
+    // - after the $N, $<name> checks, it flags that a substitution was made.
+    const patternIsRE = data.args[0] instanceof RegExp;
+    let replacement;
+    if (typeof data._replacement === 'function') {
+      // no special replacements apply to the string returned by a replacement
+      // function.
+      replacement = data._replacement.call(global, ...replacerArgs);
+    } else {
+      // if it's not a function then the valid special replacements depend on
+      // whether the pattern is a regex or a string. first find substitution
+      // patterns present in the replacement string. Don't find patterns that
+      // aren't valid, e.g., $0, $<name> when the pattern is a string.
+      replacement = String(data._replacement);
+
+      const matches = [...StringPrototypeMatchAll.call(replacement, patternIsRE ? RE_REPS : STR_REPS)];
+      let trackedReplacementsDone = 0;
 
-    replacement = replaceSpecialCharacters(String(replacement), parsedArgs, data._replacementType);
+      for (const m of matches) {
+        let substitution;
+        let substitutionDone = false;
+        // format of m: ['$`', '`', index: 0, input: string, groups: undefined|{}]
+        // if the pattern is a regex, then $1 to $99 and $<name> are valid
+        if (patternIsRE) {
+          // my guess is $1 to $99 are most likely, after that, who knows? so
+          // we check named groups next, because 1) they seem more useful than
+          // the other $ patterns and 2) they are in the same patternIsRE test.
+          //
+          // in any case, the following will be false if m[1][0] is not a number.
+          if (m[1][0] >= 1 && m[1][0] <= 9) {
+            //const group = Number(m[1]);
+            // a group might not be present, e.g., (pattern)?(a) or (a)|(b).
+            if (parsedArgs[m[1]] === undefined && m[1] in parsedArgs) {
+              // need to remove $m[1] from replacement pattern. N.B. this could
+              // mess up if the replacement text contains text that matches a
+              // subsequent replacement (either RE or string).
+              replacement = StringPrototypeReplaceAll.call(replacement, m[0], '');
+              continue;
+            }
+            substitution = parsedArgs[m[1]];
+            substitutionDone = true;
+          } else if (m[1][0] === '<') {
+            // named group
+            const groupName = m[1].substring(1, m[1].length - 1);
+            if (parsedArgs.groups[groupName] === undefined && groupName in parsedArgs.groups) {
+              // remove $<groupName> from the replacement pattern. N.B. this
+              // also could mess up if the replacement text containts text that
+              // matches a subsequent replacement.
+              replacement = StringPrototypeReplaceAll.call(replacement, m[0], '');
+              continue;
+            }
+            substitution = parsedArgs.groups[groupName];
+            substitutionDone = true;
+          }
+        }
+
+        // the following are valid whether the pattern is a regex or a string.
+        //
+        // any idea what order $&'` should be in from a most-common to least-
+        // common perspective? nfi.
+        if (!substitutionDone) {
+          if (m[1] === '$') {
+            // this could actually be tracked but in order to do this "right"
+            // we have to be able to distinguish whether it is the first or
+            // the second $ that is tracked. so punt, and just ignore it, as
+            // originally implemented.
+            substitution = '$';
+          } else if (m[1] === '&') {
+            // replace these '$&' in parsedArgs[0] (replacerArgs[0], i.e., the
+            // match). if tracked, handle it. do so by index, so we don't have to
+            // call replace again? e.g., string[m.index..m.index+2]
+            substitution = parsedArgs[0];
+          } else if (m[1] === '`') {
+            substitution = parsedArgs.input.substring(0, parsedArgs.index);
+          } else if (m[1] === "'") {
+            substitution = parsedArgs.input.substring(parsedArgs.index + parsedArgs[0].length);
+          } //  else {
+          //   throw new Error('how can it have matched RE and gotten here?');
+          // }
+          // i'm not sure what the proper handling of this is. if either char
+          // of the $$&`' sequence is tracked, then something the user input
+          // is manipulating the output, even if it is just duplicating what
+          // might be tracked or untracked input. does that count?
+        }
+        // if the substitution is tracked just use heavy-weight replacer.
+        if (trackedReplacementsDone || tracker.isTracked(substitution)) {
+          replacement = replacement.replace(m[0], substitution);
+          trackedReplacementsDone += 1;
+        } else {
+          replacement = StringPrototypeReplace.call(replacement, m[0], substitution);
+        }
+      }
+
+    }
+
+    // coerce the replacement to a string, e.g., null => 'null'
+    replacement = String(replacement);
 
     data._replacementInfo = tracker.getData(replacement);
     if (data._replacementInfo) {
       data._history.add(data._replacementInfo);
     }
+
     return { replacement, replacementInfo: data._replacementInfo };
   }
 
   function getReplacer(data) {
     return function replacer(...args) {
       const parsedArgs = parseArgs(args);
-      const { match, matchIdx, str } = parsedArgs;
+      const match = parsedArgs[0];
+      const { index, input } = parsedArgs;
 
       const { _accumOffset, _accumTags } = data;
       const { replacement, replacementInfo } = getReplacementInfo(data, args, parsedArgs);
 
-      const preTags = createSubsetTags(_accumTags, 0, _accumOffset + matchIdx);
-      const postTags = createSubsetTags(_accumTags, _accumOffset + matchIdx + match.length, str.length - matchIdx - match.length);
+      const preTags = createSubsetTags(_accumTags, 0, _accumOffset + index);
+      const postTags = createSubsetTags(_accumTags, _accumOffset + index + match.length, input.length - index - match.length);
       data._accumOffset += (replacement.length - match.length);
       if (preTags || postTags || replacementInfo) {
         data._accumTags = createAppendTags(
-          createAppendTags(preTags, replacementInfo?.tags, _accumOffset + matchIdx),
+          createAppendTags(preTags, replacementInfo?.tags, _accumOffset + index),
           postTags,
-          data._accumOffset + matchIdx + match.length
+          data._accumOffset + index + match.length
         );
       } else {
         data._accumTags = {};

package/lib/dataflow/propagation/install/string/replace.node-test.mjs

@@ -0,0 +1,705 @@
+import { describe, it, beforeEach, afterEach, after } from 'node:test';
+import assert from 'node:assert';
+import { inspect } from 'util';
+import { initAssessFixture } from '@contrast/test/fixtures/index.js';
+
+describe('assess dataflow propagation string replace', function () {
+  let core, simulateRequestScope, trackString, tracker;
+  const allPatcher = new Map();
+
+  beforeEach(function () {
+    ({
+      core,
+      trackString,
+      simulateRequestScope
+    } = initAssessFixture());
+
+    // print test name
+    //console.log(`beforeEach TestContext(${this.name})`);
+
+    tracker = core.assess.dataflow.tracker;
+
+    core.assess.dataflow.propagation.stringInstrumentation.substring.install();
+    core.assess.dataflow.propagation.stringInstrumentation.replace.install();
+  });
+
+  afterEach(function () {
+    core.assess.dataflow.propagation.stringInstrumentation.replace.uninstall();
+  });
+
+  // eslint-disable-next-line mocha/no-sibling-hooks
+  afterEach(function() {
+    core.Perf.fromAllToMap('patcher', allPatcher);
+  });
+
+  after(function() {
+    const stats = core.Perf.getStats(allPatcher);
+    for (const [key, { n, totalMicros, mean }] of stats.entries()) {
+      console.log(key, n, totalMicros, 'nsec', mean, 'mean');
+    }
+  });
+
+  ['string', 'function'].forEach((replacerType) => {
+    function getReplacement(replacerType, replacement) {
+      return replacerType === 'function' ? () => replacement : replacement;
+    }
+    describe(`string-arg custom ${replacerType} replacer`, function () {
+      it('untracked string replace with untracked string', function () {
+        simulateRequestScope(() => {
+          const val = '?bcd';
+          const ret = val.replace('?', getReplacement(replacerType, 'a'));
+          assert.strictEqual(ret, 'abcd');
+          assert.strictEqual(tracker.getData(ret), null);
+        });
+      });
+
+      [
+        [],
+        [undefined],
+        [null]
+      ].forEach((rest) => {
+        it(`full replacement with nullish value: ${inspect(rest[0])}`, function () {
+          simulateRequestScope(() => {
+            const val = 'abcd';
+            const extern = trackString(val);
+            const ret = extern.replace(val, getReplacement(replacerType, rest[0]));
+            assert.strictEqual(ret, String(rest[0]));
+            assert.strictEqual(tracker.getData(ret), null);
+          });
+        });
+      });
+
+      it('empty match string', function () {
+        simulateRequestScope(() => {
+          const extern = trackString('bcde', {
+            tags: {
+              UNTRUSTED: [0, 3]
+            }
+          });
+          const ret = extern.replace('', getReplacement(replacerType, 'a'));
+          assert.strictEqual(ret, 'abcde');
+          const tags = tracker.getData(ret).tags;
+          const expectedTags = { UNTRUSTED: [1, 4] };
+          assertTagsEqual(tags, expectedTags);
+        });
+      });
+
+      it('empty replacer string', function () {
+        simulateRequestScope(() => {
+          const extern = trackString('abcd', {
+            tags: {
+              UNTRUSTED: [0, 3]
+            }
+          });
+          const ret = extern.replace('', getReplacement(replacerType, ''));
+          assert.strictEqual(ret, 'abcd');
+          const tags = tracker.getData(ret).tags;
+          const expectedTags = { UNTRUSTED: [0, 3] };
+          assertTagsEqual(tags, expectedTags);
+        });
+      });
+
+      it('untracked string replaced with tracked', function () {
+        simulateRequestScope(() => {
+          const val = '?bcd';
+          const replacement = trackString('a', {
+            tags: {
+              UNTRUSTED: [0, 0]
+            }
+          });
+          const ret = val.replace('?', getReplacement(replacerType, replacement));
+          assert.strictEqual(ret, 'abcd');
+          const tags = tracker.getData(ret).tags;
+          const expectedTags = { UNTRUSTED: [0, 0] };
+          assertTagsEqual(tags, expectedTags);
+        });
+      });
+
+      it('does not track return with all tracked values removed', function () {
+        simulateRequestScope(() => {
+          const extern = trackString('_TT_', {
+            tags: {
+              UNTRUSTED: [1, 2]
+            }
+          });
+          const ret = extern.replace('TT', getReplacement(replacerType, '__'));
+          assert.strictEqual(ret, '____');
+          assert.strictEqual(tracker.getData(ret), null);
+        });
+      });
+
+      it('tracks result highspan', function () {
+        simulateRequestScope(() => {
+          const extern = trackString('_TT_', {
+            tags: {
+              UNTRUSTED: [1, 2],
+              history: [{ mock: 'sourceEvent' }]
+            }
+          });
+          const ret = extern.replace('_T', getReplacement(replacerType, '__'));
+          assert.strictEqual(ret, '__T_');
+          const tags = tracker.getData(ret).tags;
+          const expectedTags = { UNTRUSTED: [2, 2] };
+          assertTagsEqual(tags, expectedTags);
+        });
+      });
+
+      it('tracks result lospan', function () {
+        simulateRequestScope(() => {
+          const extern = trackString('_TT_', {
+            tags: {
+              UNTRUSTED: [1, 2],
+              history: [{ mock: 'sourceEvent' }]
+            }
+          });
+          const ret = extern.replace('T_', getReplacement(replacerType, '__'));
+          assert.strictEqual(ret, '_T__');
+          const tags = tracker.getData(ret).tags;
+          const expectedTags = { UNTRUSTED: [1, 1] };
+          assertTagsEqual(tags, expectedTags);
+        });
+      });
+
+      it('tracks result highspan tracked replacement', function () {
+        simulateRequestScope(() => {
+          const extern = trackString('_TT_', {
+            tags: {
+              UNTRUSTED: [1, 2]
+            }
+          });
+          const replacement = trackString('TT', {
+            tags: {
+              UNTRUSTED: [0, 1]
+            }
+          });
+          const ret = extern.replace('T_', getReplacement(replacerType, replacement));
+          assert.strictEqual(ret, '_TTT');
+          const tags = tracker.getData(ret).tags;
+          const expectedTags = { UNTRUSTED: [1, 3] };
+          assertTagsEqual(tags, expectedTags);
+
+        });
+      });
+
+      it('keeps track of multiple tag ranges', function () {
+        simulateRequestScope(() => {
+          const extern = trackString('a?cd', {
+            tags: {
+              UNTRUSTED: [0, 3],
+              foo: [1, 2]
+            }
+          });
+          const replacement = trackString('b', {
+            tags: {
+              foo: [0, 0],
+              bar: [0, 0]
+            }
+          });
+          const ret = extern.replace('?', getReplacement(replacerType, replacement));
+          assert.strictEqual(ret, 'abcd');
+          const tags = tracker.getData(ret).tags;
+          assertTagsEqual(tags, {
+            UNTRUSTED: [0, 0, 2, 3],
+            foo: [1, 2],
+            bar: [1, 1]
+          });
+        });
+      });
+    });
+
+    describe(`regEx custom ${replacerType} replacer`, function () {
+      it('keeps track of tagged strings not replaced', function () {
+        simulateRequestScope(() => {
+          const extern = trackString('abcd');
+          const ret = extern.replace(/e/g, getReplacement(replacerType, 'e'));
+          assert.strictEqual(ret, 'abcd');
+          const tags = tracker.getData(ret).tags;
+          assertTagsEqual(tags, {
+            UNTRUSTED: [0, 3]
+          });
+        });
+      });
+
+      it('keeps track of tagged strings replaced by untracked strings', function () {
+        simulateRequestScope(() => {
+          const extern = trackString('a?a?');
+          const ret = extern.replace(/\?/g, getReplacement(replacerType, 'b'));
+          assert.strictEqual(ret, 'abab');
+          const tags = tracker.getData(ret).tags;
+          const expectedTags = {
+            UNTRUSTED: [0, 0, 2, 2]
+          };
+          assertTagsEqual(tags, expectedTags);
+        });
+      });
+
+      it('keeps track of tagged strings replaced by larger untracked strings', function () {
+        simulateRequestScope(() => {
+          const extern = trackString('a?a?');
+          const ret = extern.replace(/\?/g, getReplacement(replacerType, 'bbb'));
+          assert.strictEqual(ret, 'abbbabbb');
+          const tags = tracker.getData(ret).tags;
+          const expectedTags = {
+            UNTRUSTED: [0, 0, 4, 4]
+          };
+          assertTagsEqual(tags, expectedTags);
+        });
+      });
+
+      it('keeps track of tagged strings replaced by smaller untracked strings', function () {
+        simulateRequestScope(() => {
+          const extern = trackString('a???a???');
+          const ret = extern.replace(/\?\?\?/g, getReplacement(replacerType, 'b'));
+          assert.strictEqual(ret, 'abab');
+          const tags = tracker.getData(ret).tags;
+          const expectedTags = {
+            UNTRUSTED: [0, 0, 2, 2]
+          };
+          assertTagsEqual(tags, expectedTags);
+        });
+      });
+
+      it('keeps track of tagged strings replaced by tracked strings', function () {
+        simulateRequestScope(() => {
+          const extern = trackString('a?a?', {
+            tags: {
+              UNTRUSTED: [0, 3]
+            }
+          });
+          const replacement = trackString('b', {
+            tags: {
+              UNTRUSTED: [0, 0]
+            }
+          });
+          const ret = extern.replace(/\?/g, getReplacement(replacerType, replacement));
+          assert.strictEqual(ret, 'abab');
+          const tags = tracker.getData(ret).tags;
+          const expectedTags = {
+            UNTRUSTED: [0, 3]
+          };
+          assertTagsEqual(tags, expectedTags);
+        });
+      });
+
+      it('keeps track of tagged strings replaced by larger tracked strings', function () {
+        simulateRequestScope(() => {
+          const extern = trackString('a?a?', {
+            tags: {
+              UNTRUSTED: [0, 3]
+            }
+          });
+          const replacement = trackString('bbb', {
+            tags: {
+              UNTRUSTED: [0, 2]
+            }
+          });
+          const ret = extern.replace(/\?/g, getReplacement(replacerType, replacement));
+          assert.strictEqual(ret, 'abbbabbb');
+          const tags = tracker.getData(ret).tags;
+          const expectedTags = {
+            UNTRUSTED: [0, 7]
+          };
+          assertTagsEqual(tags, expectedTags);
+        });
+      });
+
+      it('keeps track of tagged strings replaced by smaller tracked strings', function () {
+        simulateRequestScope(() => {
+          const extern = trackString('a???a???', {
+            tags: {
+              UNTRUSTED: [0, 7]
+            }
+          });
+          const replacement = trackString('b', {
+            tags: {
+              UNTRUSTED: [0, 0]
+            }
+          });
+          const ret = extern.replace(/\?\?\?/g, getReplacement(replacerType, replacement));
+          assert.strictEqual(ret, 'abab');
+          const tags = tracker.getData(ret).tags;
+          const expectedTags = {
+            UNTRUSTED: [0, 3]
+          };
+          assertTagsEqual(tags, expectedTags);
+        });
+      });
+
+      it('keeps track of multiple tag ranges', function () {
+        simulateRequestScope(() => {
+          const extern = trackString('a??a??a', {
+            tags: {
+              UNTRUSTED: [0, 6],
+              foo: [0, 2],
+              bar: [4, 6]
+            }
+          });
+          const replacement = trackString('b', {
+            tags: {
+              foo: [0, 0],
+              bar: [0, 0]
+            }
+          });
+          const ret = extern.replace(/\?/g, replacement);
+          assert.strictEqual(ret, 'abbabba');
+          const tags = tracker.getData(ret).tags;
+          const expectedTags = {
+            UNTRUSTED: [0, 0, 3, 3, 6, 6],
+            foo: [0, 2, 4, 5],
+            bar: [1, 2, 4, 6],
+          };
+          assertTagsEqual(tags, expectedTags);
+        });
+      });
+    });
+  });
+
+  describe('complex/edge cases', function () {
+    it('replacer function returns a non-string', function () {
+      simulateRequestScope(() => {
+        const extern = trackString('123?', {
+          tags: {
+            UNTRUSTED: [0, 3]
+          }
+        });
+        const ret = extern.replace('?', (match, offset) => offset + 1);
+        assert.strictEqual(ret, '1234');
+        const tags = tracker.getData(ret).tags;
+        assertTagsEqual(tags, {
+          UNTRUSTED: [0, 2]
+        });
+      });
+    });
+
+    it('replacer operates on capture groups', function () {
+      simulateRequestScope(() => {
+        const extern = trackString('AaaBbbCcc', {
+          tags: {
+            UNTRUSTED: [0, 8],
+            groupOne: [0, 2],
+            groupTwo: [3, 5],
+            groupThree: [6, 8]
+          }
+        });
+        const ret = extern.replace(/(A)|(B)|(C)/g, (match => match.toLowerCase()));
+        assert.strictEqual(ret, 'aaabbbccc');
+        const tags = tracker.getData(ret).tags;
+        assertTagsEqual(tags, {
+          UNTRUSTED: [1, 2, 4, 5, 7, 8],
+          groupOne: [1, 2],
+          groupTwo: [4, 5],
+          groupThree: [7, 8]
+        });
+      });
+    });
+
+    it('handles edge case where we have a function that sets the replacement to be a special value',
+      function () {
+        simulateRequestScope(() => {
+          let counter = 0;
+          const extern = trackString('SELECT ? FROM ?', {
+            tags: {
+              UNTRUSTED: [0, 14],
+              groupOne: [9, 12],
+            }
+          });
+          const ret = extern.replace(/(\?)/g, () => `$${counter++}`);
+          assert.strictEqual(ret, 'SELECT $0 FROM $1');
+          const tags = tracker.getData(ret).tags;
+          assertTagsEqual(tags, {
+            UNTRUSTED: [0, 6, 9, 14],
+            groupOne: [10, 13],
+          });
+        });
+      });
+
+    it('replacer operates on named capture groups', function () {
+      simulateRequestScope(() => {
+        const extern = trackString('AaaBbbCcc', {
+          tags: {
+            UNTRUSTED: [0, 8],
+            groupOne: [0, 2],
+            groupTwo: [3, 5],
+            groupThree: [6, 8]
+          }
+        });
+        const ret = extern.replace(/(?<g1>A)|(?<g2>B)|(?<g3>C)/g, (match => match.toLowerCase()));
+        assert.strictEqual(ret, 'aaabbbccc');
+        const tags = tracker.getData(ret).tags;
+        assertTagsEqual(tags, {
+          UNTRUSTED: [1, 2, 4, 5, 7, 8],
+          groupOne: [1, 2],
+          groupTwo: [4, 5],
+          groupThree: [7, 8]
+        });
+      });
+    });
+
+    it('keeps track of multiple overlapping tag ranges over multiple calls', function () {
+      simulateRequestScope(() => {
+        let ret;
+        const extern = trackString('a??cd?g?', {
+          tags: {
+            UNTRUSTED: [0, 7],
+            foo: [0, 2],
+            bar: [1, 3],
+            fizz: [4, 7],
+            buzz: [1, 6]
+          }
+        });
+        const replB = trackString('b', {
+          tags: {
+            foo: [0, 0],
+            bar: [0, 0]
+          }
+        });
+        const replEf = trackString('ef', {
+          tags: {
+            fizz: [0, 0],
+            buzz: [1, 1]
+          }
+        });
+        const replH = trackString('h', {
+          tags: {
+            foobar: [0, 0]
+          }
+        });
+        ret = extern.replace('??', replB);
+        ret = ret.replace('?', () => replEf);
+        ret = ret.replace(/\?/g, replH);
+        assert.strictEqual(ret, 'abcdefgh');
+        const tags = tracker.getData(ret).tags;
+        assertTagsEqual(tags, {
+          UNTRUSTED: [0, 0, 2, 3, 6, 6],
+          foo: [0, 1],
+          bar: [1, 2],
+          fizz: [3, 4, 6, 6],
+          buzz: [2, 3, 5, 6],
+          foobar: [7, 7]
+        });
+      });
+    });
+
+    it('keeps track of replacement history', function () {
+      simulateRequestScope(() => {
+        const extern = trackString('aa??');
+        const replacementOne = trackString('bb');
+        const replacementTwo = trackString('cc');
+        const replacements = ['', '', replacementOne, replacementTwo];
+        const ret = extern.replace(/\?/g, (match, offset) => replacements[offset]);
+        assert.strictEqual(ret, 'aabbcc');
+        const history = tracker.getData(ret).history;
+        assert.strictEqual(history.length, 3);
+        const expected = [
+          { value: 'aa??', tags: { UNTRUSTED: [0, 3] } },
+          { value: 'bb', tags: { UNTRUSTED: [0, 1] } },
+          { value: 'cc', tags: { UNTRUSTED: [0, 1] } }
+        ];
+        for (const {value, tags: expectedTags} of expected) {
+          const {tags} = history.find(({value: v}) => v === value);
+          assertTagsEqual(tags, expectedTags);
+        }
+      });
+    });
+  });
+
+  describe('special characters', function () {
+
+    it('replaces and handles $$', function () {
+      simulateRequestScope(() => {
+        const extern = trackString('foo?foo');
+        const ret = extern.replace('?', '$$ $$');
+        assert.strictEqual(ret, 'foo$ $foo');
+        const tags = tracker.getData(ret).tags;
+        const expectedTags = { UNTRUSTED: [0, 2, 6, 8] };
+        assertTagsEqual(tags, expectedTags);
+      });
+    });
+
+    it('BUG: does not replace $$ from a function replacer', function() {
+      // BUG
+      // $$ should no be interpreted when returned from a function
+      //
+      // NEED TEST when '$$' replacement is tracked?
+      simulateRequestScope(() => {
+        const extern = trackString('foo?foo');
+        const ret = extern.replace('?', () => '$$');
+        assert.strictEqual(ret, 'foo$$foo');
+        const tags = tracker.getData(ret).tags;
+        const expectedTags = { UNTRUSTED: [0, 2, 5, 7] };
+        assertTagsEqual(tags, expectedTags);
+      });
+    });
+
+    it('replaces and handles untracked $&', function () {
+      simulateRequestScope(() => {
+        const extern = trackString('foo bar');
+        const ret = extern.replace('bar', '"$&"-[$&]');
+        assert.strictEqual(ret, 'foo "bar"-[bar]');
+        const tags = tracker.getData(ret).tags;
+        const expectedTags = { UNTRUSTED: [0, 3] };
+        assertTagsEqual(tags, expectedTags);
+      });
+    });
+
+    it('replaces and handles tracked $&', function () {
+      simulateRequestScope(() => {
+        const extern = trackString('foo bar');
+        const replacement = trackString('bar');
+        const ret = extern.replace(replacement, '[$&]-[$&]');
+        assert.strictEqual(ret, 'foo [bar]-[bar]');
+        const tags = tracker.getData(ret).tags;
+        const expectedTags = { UNTRUSTED: [0, 3, 5, 7, 11, 13] };
+        assertTagsEqual(tags, expectedTags);
+      });
+    });
+
+    it('replaces and handles $`', function () {
+      simulateRequestScope(() => {
+        const extern = trackString('foobar');
+        const ret = extern.replace('bar', '$`$`');
+        assert.strictEqual(ret, 'foofoofoo');
+        const tags = tracker.getData(ret).tags;
+        const expectedTags = { UNTRUSTED: [0, 8] };
+        assertTagsEqual(tags, expectedTags);
+      });
+    });
+
+    it('replaces and handles $\'', function () {
+      simulateRequestScope(() => {
+        const extern = trackString('foobar');
+        const ret = extern.replace('foo', '$\'$\'');
+        assert.strictEqual(ret, 'barbarbar');
+        const tags = tracker.getData(ret).tags;
+        const expectedTags = { UNTRUSTED: [0, 8] };
+        assertTagsEqual(tags, expectedTags);
+      });
+    });
+
+    it('replaces and handles $N', function () {
+      simulateRequestScope(() => {
+        const extern = trackString('aa bbcc');
+        const ret = extern.replace(/(aa) (bb)/g, '$2 $1');
+        assert.strictEqual(ret, 'bb aacc');
+        const { tags } = tracker.getData(ret);
+        // BUG? the entire input string is tracked. replace() reorganizes it,
+        // but shouldn't the entire resultant string be tracked? maybe just
+        // handle the simplest case where the entire input string is tracked
+        // because if only subsets of the string are tracked it will be quite
+        // expensive to propagate the tag ranges correctly.
+        const expectedTags = { UNTRUSTED: [5, 6] };
+        assertTagsEqual(tags, expectedTags);
+      });
+    });
+
+    it('BUG: replaces and handles $name', function () {
+      // BUG string needs to be $<name> not $name
+      simulateRequestScope(() => {
+        const extern = trackString('aabbcc');
+        const ret = extern.replace(/(?<g1>aa)(?<g2>bb)/g, '$<g2>$<g1>');
+        assert.strictEqual(ret, 'bbaacc');
+        const tags = tracker.getData(ret).tags;
+        const expectedTags = { UNTRUSTED: [4, 5] };
+        assertTagsEqual(tags, expectedTags);
+      });
+    });
+
+    it('maintains tracking with $$', function() {
+      simulateRequestScope(() => {
+        const extern = trackString('foobarfoo');
+        const ret = extern.replace('bar', '$$');
+        assert.strictEqual(ret, 'foo$foo');
+        const tags = tracker.getData(ret).tags;
+        const expectedTags = { UNTRUSTED: [0, 2, 4, 6] };
+        assertTagsEqual(tags, expectedTags);
+      });
+    });
+
+    it('BUG: maintains tracking when $$ is tracked', function() {
+      // BUG this should be [0, 6] but isn't because we don't consider whether
+      // the $$ is tracked or not.
+      simulateRequestScope(() => {
+        const dollardollar = trackString('$$');
+        const extern = trackString('foobarfoo');
+        const ret = extern.replace('bar', dollardollar);
+        assert.strictEqual(ret, 'foo$foo');
+        const tags = tracker.getData(ret).tags;
+        const expectedTags = { UNTRUSTED: [0, 2, 4, 6] };
+        assertTagsEqual(tags, expectedTags);
+      });
+    });
+
+    it('replaces and handles strings containing multiple special characters', function () {
+      simulateRequestScope(() => {
+        const extern = trackString('aabbcc');
+        const ret = extern.replace('bb', '($&)-($\')-($`)');
+        assert.strictEqual(ret, 'aa(bb)-(cc)-(aa)cc');
+        const tags = tracker.getData(ret).tags;
+        const expectedTags = { UNTRUSTED: [0, 1, 8, 9, 13, 14, 16, 17] };
+        assertTagsEqual(tags, expectedTags);
+      });
+    });
+
+    it('replaces and handles strings containing multiple special characters that replace multiple strings', function () {
+      simulateRequestScope(() => {
+        const extern = trackString('aa?bb?cc?');
+        const ret = extern.replace(/(\?)/g, '$$$1');
+        assert.strictEqual(ret, 'aa$?bb$?cc$?');
+        const tags = tracker.getData(ret).tags;
+        const expectedTags = { UNTRUSTED: [0, 1, 4, 5, 8, 9 ]};
+        assertTagsEqual(tags, expectedTags);
+      });
+    });
+
+    it('replaces and handles multiple special characters and tracked strings', function () {
+      simulateRequestScope(() => {
+        const extern = trackString('foobar');
+        const replacement = trackString('bar');
+        const ret = extern.replace(replacement, '$$$&$$$&');
+        assert.strictEqual(ret, 'foo$bar$bar');
+        const tags = tracker.getData(ret).tags;
+        const expectedTags = {UNTRUSTED: [0, 2, 4, 6, 8, 10]};
+        assertTagsEqual(tags, expectedTags);
+      });
+    });
+
+    // example is from shell-quote
+    it('replaces and handles $\\d replacements when capture group exists or doesn\'t', function() {
+      const s = 'http://example.com?a=1&a=2';
+      const expected = s.replace(/([A-Za-z]:)?([#!"$&'()*,:;<=>?@[\\\]^`{|}])/g, '$1\\$2');
+      simulateRequestScope(() => {
+        const result = s.replace(/([A-Za-z]:)?([#!"$&'()*,:;<=>?@[\\\]^`{|}])/g, '$1\\$2');
+        // http\://example.com\?a\=1\&a\=2
+        assert.strictEqual(result, expected);
+      });
+    });
+
+    it('handles disjunction (unmatched but present group)', function() {
+      const s = 'http://example.com?a=1&a=2';
+      const expected = s.replace(/(http)|(https)/g, '$2');
+      simulateRequestScope(() => {
+        const result = s.replace(/(http)|(https)/g, '$2');
+        // http\://example.com\?a\=1\&a\=2
+        assert.strictEqual(result, expected);
+      });
+    });
+
+    it('replaces and handles $<name> replacements when capture group exists or doesn\'t', function() {
+      const s = 'http://example.com?a=1&a=2';
+      const expected = s.replace(/(?<protocol>[A-Za-z]:)?(?<special>[#!"$&'()*,:;<=>?@[\\\]^`{|}])/g, '$<protocol>\\$<special>');
+      simulateRequestScope(() => {
+        const result = s.replace(/(?<protocol>[A-Za-z]:)?(?<special>[#!"$&'()*,:;<=>?@[\\\]^`{|}])/g, '$<protocol>\\$<special>');
+        // http\://example.com\?a\=1\&a\=2
+        assert.strictEqual(result, expected);
+      });
+    });
+
+  });
+});
+
+function assertTagsEqual(tags, expectedTags) {
+  assert.deepStrictEqual(Object.keys(tags), Object.keys(expectedTags), 'tag mismatch');
+  for (const tag of Object.keys(tags)) {
+    assert.deepStrictEqual(tags[tag], expectedTags[tag], "tag range mismatch");
+  }
+}

package/lib/dataflow/tracker.js

@@ -37,6 +37,10 @@ module.exports = function tracker(core) {
     return objMap.get(value) || null;
   }
 
+  function isTracked(value) {
+    return distringuish.isExternal(value);
+  }
+
   function track(value, metadata) {
     let ret = Object.create(null);
 
@@ -148,5 +152,6 @@ module.exports = function tracker(core) {
     untrack,
     getData,
     getInfo: getData,
+    isTracked,
   };
 };

package/lib/get-source-context.js

@@ -61,7 +61,7 @@ module.exports = function(core) {
       return null;
     }
 
-    if (ctx.propagationEventsCount > config.assess.max_propagation_events) return null;
+    if (ctx.propagationEventsCount >= config.assess.max_propagation_events) return null;
 
     return ctx;
   };
@@ -105,9 +105,8 @@ module.exports = function(core) {
       return null;
     }
 
-    if (ctx.sourceEventsCount > config.assess.max_context_source_events) return null;
+    if (ctx.sourceEventsCount >= config.assess.max_context_source_events) return null;
 
     return ctx;
   };
-
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@contrast/assess",
-  "version": "1.46.2",
+  "version": "1.47.0",
   "description": "Contrast service providing framework-agnostic Assess support",
   "license": "SEE LICENSE IN LICENSE",
   "author": "Contrast Security <nodejs@contrastsecurity.com> (https://www.contrastsecurity.com)",
@@ -28,7 +28,7 @@
     "@contrast/instrumentation": "1.24.2",
     "@contrast/logger": "1.18.2",
     "@contrast/patcher": "1.17.2",
-    "@contrast/rewriter": "1.21.3",
+    "@contrast/rewriter": "1.21.4",
     "@contrast/route-coverage": "1.35.2",
     "@contrast/scopes": "1.15.2",
     "semver": "^7.6.0"