@blamejs/blamejs-shop 0.5.20 → 0.5.22

package/lib/push-notifications.js

@@ -69,6 +69,7 @@
  */
 
 var b = require("./vendor/blamejs");
+var textGuard = require("./text-guard");
 var C = b.constants;
 
 // ---- constants ----------------------------------------------------------
@@ -114,10 +115,6 @@ var RETRY_BACKOFF_MS = [
 
 var SLUG_RE         = /^[a-z](?:[a-z0-9-]*[a-z0-9])?$/;
 var LOCALE_RE       = /^[A-Za-z]{2,8}(?:[-_][A-Za-z0-9]{2,8}){0,3}$/;
-var CONTROL_BYTE_RE = /[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/;
-var ZERO_WIDTH_RE   = new RegExp(
-  "[\\u200B-\\u200F\\u202A-\\u202E\\u2060-\\u2064\\u2066-\\u2069\\uFEFF\\u061C]"
-);
 
 // ---- validators ---------------------------------------------------------
 
@@ -162,12 +159,7 @@ function _credentials(s) {
   if (s.length > MAX_CREDENTIALS_LEN) {
     throw new TypeError("pushNotifications: credentials must be <= " + MAX_CREDENTIALS_LEN + " characters");
   }
-  if (CONTROL_BYTE_RE.test(s)) {
-    throw new TypeError("pushNotifications: credentials contains control bytes");
-  }
-  if (ZERO_WIDTH_RE.test(s)) {
-    throw new TypeError("pushNotifications: credentials contains zero-width / bidi-override codepoints");
-  }
+  textGuard.freeText(s, "pushNotifications: credentials", { zeroWidth: "reject" });
   return s;
 }
 
@@ -178,12 +170,7 @@ function _deviceToken(s) {
   if (s.length > MAX_TOKEN_LEN) {
     throw new TypeError("pushNotifications: device_token must be <= " + MAX_TOKEN_LEN + " characters");
   }
-  if (CONTROL_BYTE_RE.test(s)) {
-    throw new TypeError("pushNotifications: device_token contains control bytes");
-  }
-  if (ZERO_WIDTH_RE.test(s)) {
-    throw new TypeError("pushNotifications: device_token contains zero-width / bidi-override codepoints");
-  }
+  textGuard.freeText(s, "pushNotifications: device_token", { zeroWidth: "reject" });
   return s;
 }
 
@@ -208,12 +195,7 @@ function _title(s) {
   if (s.length > MAX_TITLE_LEN) {
     throw new TypeError("pushNotifications: title must be <= " + MAX_TITLE_LEN + " characters");
   }
-  if (CONTROL_BYTE_RE.test(s)) {
-    throw new TypeError("pushNotifications: title contains control bytes");
-  }
-  if (ZERO_WIDTH_RE.test(s)) {
-    throw new TypeError("pushNotifications: title contains zero-width / bidi-override codepoints");
-  }
+  textGuard.freeText(s, "pushNotifications: title", { zeroWidth: "reject", bidiMarks: "allow" });
   return s;
 }
 
@@ -224,12 +206,7 @@ function _body(s) {
   if (s.length > MAX_BODY_LEN) {
     throw new TypeError("pushNotifications: body must be <= " + MAX_BODY_LEN + " characters");
   }
-  if (CONTROL_BYTE_RE.test(s)) {
-    throw new TypeError("pushNotifications: body contains control bytes");
-  }
-  if (ZERO_WIDTH_RE.test(s)) {
-    throw new TypeError("pushNotifications: body contains zero-width / bidi-override codepoints");
-  }
+  textGuard.freeText(s, "pushNotifications: body", { zeroWidth: "reject", bidiMarks: "allow" });
   return s;
 }
 
@@ -267,7 +244,7 @@ function _reason(s, label) {
   if (s.length > MAX_REASON_LEN) {
     throw new TypeError("pushNotifications: " + label + " must be <= " + MAX_REASON_LEN + " characters");
   }
-  if (CONTROL_BYTE_RE.test(s) || ZERO_WIDTH_RE.test(s)) {
+  if (textGuard.hasCodepointThreat(s, { zeroWidth: "reject" })) {
     throw new TypeError("pushNotifications: " + label + " contains control / zero-width bytes");
   }
   return s;
@@ -280,9 +257,7 @@ function _providerMessageId(s) {
   if (s.length > MAX_PROV_MSG_ID_LEN) {
     throw new TypeError("pushNotifications: provider_message_id must be <= " + MAX_PROV_MSG_ID_LEN + " characters");
   }
-  if (CONTROL_BYTE_RE.test(s) || ZERO_WIDTH_RE.test(s)) {
-    throw new TypeError("pushNotifications: provider_message_id contains control / zero-width bytes");
-  }
+  textGuard.freeText(s, "pushNotifications: provider_message_id", { zeroWidth: "reject" });
   return s;
 }
 

package/lib/pwa-manifest.js

@@ -168,12 +168,9 @@ var ICON_SIZES_RE = /^(?:any|\d+x\d+)(?:\s+(?:any|\d+x\d+))*$/;
 // spec recommends (image/png / image/svg+xml / image/webp).
 var ICON_TYPE_RE = /^[a-z]+\/[a-z0-9.+-]+$/;
 
-var CONTROL_BYTE_RE = /[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/;
-var ZERO_WIDTH_RE = new RegExp(
-  "[\\u200B-\\u200F\\u202A-\\u202E\\u2060-\\u2064\\u2066-\\u2069\\uFEFF\\u061C]"
-);
 
 var b = require("./vendor/blamejs");
+var textGuard = require("./text-guard");
 
 // ---- monotonic clock ---------------------------------------------------
 //
@@ -202,10 +199,10 @@ function _string(s, label, max) {
   if (s.length > max) {
     throw new TypeError("pwaManifest: " + label + " must be <= " + max + " characters");
   }
-  if (CONTROL_BYTE_RE.test(s)) {
+  if (textGuard.hasCodepointThreat(s)) {
     throw new TypeError("pwaManifest: " + label + " contains control bytes");
   }
-  if (ZERO_WIDTH_RE.test(s)) {
+  if (textGuard.hasCodepointThreat(s, { zeroWidth: "reject" })) {
     throw new TypeError("pwaManifest: " + label + " contains zero-width / direction-override characters");
   }
   return s;
@@ -270,7 +267,7 @@ function _url(s, label) {
   if (s.length > MAX_URL_LEN) {
     throw new TypeError("pwaManifest: " + label + " must be <= " + MAX_URL_LEN + " characters");
   }
-  if (CONTROL_BYTE_RE.test(s) || ZERO_WIDTH_RE.test(s)) {
+  if (textGuard.hasCodepointThreat(s, { zeroWidth: "reject" })) {
     throw new TypeError("pwaManifest: " + label + " contains control / zero-width bytes");
   }
   if (s.charCodeAt(0) === 47 /* "/" */) {
@@ -416,10 +413,8 @@ function _validateRuntimeRules(rules) {
         MAX_URL_PATTERN_LEN + " characters"
       );
     }
-    if (CONTROL_BYTE_RE.test(r.url_pattern) || ZERO_WIDTH_RE.test(r.url_pattern)) {
-      throw new TypeError(
-        "pwaManifest: runtime_rules[" + i + "].url_pattern contains control / zero-width bytes"
-      );
+    if (textGuard.hasCodepointThreat(r.url_pattern, { zeroWidth: "reject" })) {
+      throw new TypeError("pwaManifest: runtime_rules[" + i + "].url_pattern contains control / zero-width bytes");
     }
     _enum(r.strategy, ALLOWED_STRATEGIES, "runtime_rules[" + i + "].strategy");
     if (seen[r.url_pattern]) {

package/lib/quotes.js

@@ -145,11 +145,6 @@ var VIEW_TOKEN_RE        = /^[A-Za-z0-9_-]{43}$/;
 // are allowed inside multi-line free-text fields (message,
 // operator_notes, line.notes); single-line fields (terms, reject /
 // cancel reason, accepted_by) refuse them too.
-var CONTROL_BYTE_RE             = /[\x00-\x1f\x7f]/;
-var CONTROL_BYTE_MULTILINE_RE   = /[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/;
-var ZERO_WIDTH_RE = new RegExp(
-  "[\\u200B-\\u200F\\u202A-\\u202E\\u2060-\\u2064\\u2066-\\u2069\\uFEFF\\u061C]"
-);
 
 var QUOTE_STATUSES = Object.freeze([
   "requested", "responded", "accepted", "rejected",
@@ -160,6 +155,7 @@ var TERMINAL_STATUSES = Object.freeze([
 ]);
 
 var b = require("./vendor/blamejs");
+var textGuard = require("./text-guard");
 
 // The quote lifecycle, modelled on b.fsm — the same composition the order
 // primitive uses (lib/order.js#_getOrderFsm). The FSM is the single source of
@@ -316,10 +312,10 @@ function _optShortText(s, label, max) {
   if (s.length > max) {
     throw new TypeError("quotes: " + label + " must be <= " + max + " characters");
   }
-  if (CONTROL_BYTE_RE.test(s)) {
+  if (textGuard.hasCodepointThreat(s, { singleLine: "reject" })) {
     throw new TypeError("quotes: " + label + " contains control bytes");
   }
-  if (ZERO_WIDTH_RE.test(s)) {
+  if (textGuard.hasCodepointThreat(s, { zeroWidth: "reject" })) {
     throw new TypeError("quotes: " + label + " contains zero-width / direction-override bytes");
   }
   return s;
@@ -336,10 +332,10 @@ function _optLongText(s, label, max) {
   if (s.length > max) {
     throw new TypeError("quotes: " + label + " must be <= " + max + " characters");
   }
-  if (CONTROL_BYTE_MULTILINE_RE.test(s)) {
+  if (textGuard.hasCodepointThreat(s)) {
     throw new TypeError("quotes: " + label + " contains control bytes");
   }
-  if (ZERO_WIDTH_RE.test(s)) {
+  if (textGuard.hasCodepointThreat(s, { zeroWidth: "reject" })) {
     throw new TypeError("quotes: " + label + " contains zero-width / direction-override bytes");
   }
   return s;

package/lib/reviews.js

@@ -50,9 +50,9 @@ var REVIEW_ORDER_KEY   = ["created_at:desc", "id:desc"];
 var CUSTOMER_ORDER_KEY = ["created_at:desc", "id:desc"];
 
 var ALLOWED_STATUSES   = ["pending", "published", "rejected"];
-var CONTROL_BYTE_RE    = /[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/;
 
 var b = require("./vendor/blamejs");
+var textGuard = require("./text-guard");
 
 // ---- validators ---------------------------------------------------------
 
@@ -75,9 +75,7 @@ function _title(s) {
   if (s.length > MAX_TITLE_LEN) {
     throw new TypeError("reviews: title must be <= " + MAX_TITLE_LEN + " characters");
   }
-  if (CONTROL_BYTE_RE.test(s)) {
-    throw new TypeError("reviews: title contains control bytes");
-  }
+  textGuard.freeText(s, "reviews: title", { bidiMarks: "allow" });
   return s;
 }
 
@@ -89,9 +87,7 @@ function _body(s) {
   if (s.length > MAX_BODY_LEN) {
     throw new TypeError("reviews: body must be <= " + MAX_BODY_LEN + " characters");
   }
-  if (CONTROL_BYTE_RE.test(s)) {
-    throw new TypeError("reviews: body contains control bytes");
-  }
+  textGuard.freeText(s, "reviews: body", { bidiMarks: "allow" });
   return s;
 }
 
@@ -288,9 +284,7 @@ function create(opts) {
         if (reason.length > MAX_REJECT_REASON_LEN) {
           throw new TypeError("reviews.reject: reason must be <= " + MAX_REJECT_REASON_LEN + " characters");
         }
-        if (CONTROL_BYTE_RE.test(reason)) {
-          throw new TypeError("reviews.reject: reason contains control bytes");
-        }
+        textGuard.freeText(reason, "reviews.reject: reason", { bidiMarks: "allow" });
       }
       var ts = _now();
       var existing = (await query("SELECT id, status FROM reviews WHERE id = ?1", [id])).rows[0];

package/lib/robots-config.js

@@ -96,9 +96,9 @@ var MAX_CRAWL_DELAY     = 86400;     // 24 h — anything larger is a typo
 var MAX_PRIORITY        = 1000000;
 var MAX_RULE_ID_LEN     = 80;
 
-var CONTROL_BYTE_RE = /[\x00-\x1f\x7f]/;
 
 var b = require("./vendor/blamejs");
+var textGuard = require("./text-guard");
 
 // ---- predefined templates ---------------------------------------------
 //
@@ -183,9 +183,7 @@ function _userAgent(s) {
       "robotsConfig: user_agent must be a string 1.." + MAX_USER_AGENT_LEN + " chars"
     );
   }
-  if (CONTROL_BYTE_RE.test(s)) {
-    throw new TypeError("robotsConfig: user_agent must not contain control bytes");
-  }
+  textGuard.freeText(s, "robotsConfig: user_agent", { singleLine: "reject" });
   // robots.txt is a colon-delimited key/value format — a colon in the
   // user-agent value would shift the parser onto a different key.
   if (s.indexOf(":") !== -1) {
@@ -200,7 +198,7 @@ function _pathEntry(s, label) {
       "robotsConfig: " + label + " entry must be a string 1.." + MAX_PATH_LEN + " chars"
     );
   }
-  if (CONTROL_BYTE_RE.test(s)) {
+  if (textGuard.hasCodepointThreat(s, { singleLine: "reject" })) {
     throw new TypeError("robotsConfig: " + label + " entry must not contain control bytes");
   }
   // robots.txt path entries are /-rooted absolute paths. Wildcards
@@ -258,9 +256,7 @@ function _ruleId(s) {
       "robotsConfig: rule_id must be a string 1.." + MAX_RULE_ID_LEN + " chars"
     );
   }
-  if (CONTROL_BYTE_RE.test(s)) {
-    throw new TypeError("robotsConfig: rule_id must not contain control bytes");
-  }
+  textGuard.freeText(s, "robotsConfig: rule_id", { singleLine: "reject" });
   return s;
 }
 
@@ -270,9 +266,7 @@ function _sitemapUrl(u) {
       "robotsConfig: sitemap url must be a string 1.." + MAX_SITEMAP_URL_LEN + " chars"
     );
   }
-  if (CONTROL_BYTE_RE.test(u)) {
-    throw new TypeError("robotsConfig: sitemap url must not contain control bytes");
-  }
+  textGuard.freeText(u, "robotsConfig: sitemap url", { singleLine: "reject" });
   // robots.txt only carries the URL itself, so the path-rooted form
   // promo-banners accepts isn't meaningful here — every crawler treats
   // the Sitemap: value as an absolute URL. https-only protects against
@@ -293,9 +287,7 @@ function _hostDirective(s) {
       "robotsConfig: host must be a string 1.." + MAX_HOST_LEN + " chars"
     );
   }
-  if (CONTROL_BYTE_RE.test(s)) {
-    throw new TypeError("robotsConfig: host must not contain control bytes");
-  }
+  textGuard.freeText(s, "robotsConfig: host", { singleLine: "reject" });
   // The host directive is a bare hostname (optionally with :port) —
   // no scheme, no path. Refuse anything that looks like a URL so an
   // operator can't accidentally paste the full origin URL in.

package/lib/sales-tax-filings.js

@@ -104,6 +104,7 @@
  */
 
 var b = require("./vendor/blamejs");
+var textGuard = require("./text-guard");
 var C = b.constants;
 
 // ---- constants ----------------------------------------------------------
@@ -122,7 +123,6 @@ var DEFAULT_LIMIT          = 50;
 var MAX_LIMIT              = 500;
 var MAX_DAYS_AHEAD         = 365;
 
-var CONTROL_BYTE_RE        = /[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/;
 
 // FSM transition map. The keys are (from_status, action) → to_status.
 // Defined as a table so a misuse surfaces with the actual current
@@ -211,7 +211,7 @@ function _shortText(s, label, max) {
     throw new TypeError("salesTaxFilings: " + label +
                         " must be a non-empty string <= " + max + " chars");
   }
-  if (CONTROL_BYTE_RE.test(s)) {
+  if (textGuard.hasCodepointThreat(s)) {
     throw new TypeError("salesTaxFilings: " + label + " must not contain control bytes");
   }
   return s;

package/lib/search-facets.js

@@ -106,7 +106,6 @@ var ALLOWED_KINDS  = ["categorical", "numeric_range", "boolean"];
 var ALLOWED_COLS   = ["field", "kind", "buckets_json", "display_limit", "active"];
 
 // Refuse C0 control bytes + DEL on every operator-supplied string.
-var CONTROL_BYTE_RE = /[\x00-\x1f\x7f]/;
 
 // Operator-chosen identifier shape — lowercase alnum + hyphen +
 // underscore, must start with a letter. Same shape the rest of the
@@ -118,6 +117,7 @@ var KEY_RE   = /^[a-z][a-z0-9_-]*$/;
 var FIELD_RE = /^[a-z][a-z0-9_-]*$/;
 
 var b = require("./vendor/blamejs");
+var textGuard = require("./text-guard");
 
 // ---- validators ---------------------------------------------------------
 
@@ -175,7 +175,7 @@ function _label(s, fnLabel) {
   if (s.length > MAX_LABEL_LEN) {
     throw new TypeError(fnLabel + ": bucket label must be <= " + MAX_LABEL_LEN + " characters");
   }
-  if (CONTROL_BYTE_RE.test(s)) {
+  if (textGuard.hasCodepointThreat(s, { singleLine: "reject" })) {
     throw new TypeError(fnLabel + ": bucket label must not contain control bytes");
   }
   return s;
@@ -291,7 +291,7 @@ function _appliedFilters(input, fnLabel) {
       if (s.length > MAX_VALUE_LEN) {
         throw new TypeError(fnLabel + ": applied_filters['" + k + "'][" + j + "] must be <= " + MAX_VALUE_LEN + " characters");
       }
-      if (CONTROL_BYTE_RE.test(s)) {
+      if (textGuard.hasCodepointThreat(s, { singleLine: "reject" })) {
         throw new TypeError(fnLabel + ": applied_filters['" + k + "'][" + j + "] must not contain control bytes");
       }
       values.push(s);
@@ -318,7 +318,7 @@ function _value(raw, fnLabel) {
   if (s.length > MAX_VALUE_LEN) {
     throw new TypeError(fnLabel + ": value must be <= " + MAX_VALUE_LEN + " characters");
   }
-  if (CONTROL_BYTE_RE.test(s)) {
+  if (textGuard.hasCodepointThreat(s, { singleLine: "reject" })) {
     throw new TypeError(fnLabel + ": value must not contain control bytes");
   }
   return s;
@@ -331,7 +331,7 @@ function _sessionId(s, fnLabel) {
   if (s.length > 256) {
     throw new TypeError(fnLabel + ": session_id must be <= 256 characters");
   }
-  if (CONTROL_BYTE_RE.test(s)) {
+  if (textGuard.hasCodepointThreat(s, { singleLine: "reject" })) {
     throw new TypeError(fnLabel + ": session_id must not contain control bytes");
   }
   return s;

package/lib/search-ranking.js

@@ -121,12 +121,9 @@ var SIGNAL_RE             = /^[a-z][a-z0-9_-]*$/;
 // which covers SKU notation across every common upstream system
 // without opening the door to control bytes or whitespace.
 var PRODUCT_ID_RE         = /^[A-Za-z0-9._:/-]+$/;
-var CONTROL_BYTE_RE       = /[\x00-\x1f\x7f]/;
-var ZERO_WIDTH_RE         = new RegExp(
-  "[\\u200B-\\u200F\\u202A-\\u202E\\u2060-\\u2064\\u2066-\\u2069\\uFEFF\\u061C]"
-);
 
 var b = require("./vendor/blamejs");
+var textGuard = require("./text-guard");
 
 // ---- monotonic clock ---------------------------------------------------
 //
@@ -175,7 +172,7 @@ function _name(s, fnLabel) {
   if (s.length > MAX_NAME_LEN) {
     throw new TypeError(fnLabel + ": name must be <= " + MAX_NAME_LEN + " characters");
   }
-  if (CONTROL_BYTE_RE.test(s) || ZERO_WIDTH_RE.test(s)) {
+  if (textGuard.hasCodepointThreat(s, { singleLine: "reject", zeroWidth: "reject" })) {
     throw new TypeError(fnLabel + ": name contains control / zero-width bytes");
   }
   return s;
@@ -226,7 +223,7 @@ function _normalizeQuery(s, fnLabel) {
   if (s.length > MAX_QUERY_LEN) {
     throw new TypeError(fnLabel + ": query must be <= " + MAX_QUERY_LEN + " characters");
   }
-  if (CONTROL_BYTE_RE.test(s) || ZERO_WIDTH_RE.test(s)) {
+  if (textGuard.hasCodepointThreat(s, { singleLine: "reject", zeroWidth: "reject" })) {
     throw new TypeError(fnLabel + ": query contains control / zero-width bytes");
   }
   var normalized = s.toLowerCase().replace(/\s+/g, " ").trim();
@@ -270,7 +267,7 @@ function _sessionIdRaw(s, fnLabel) {
   if (s.length > MAX_SESSION_ID_LEN) {
     throw new TypeError(fnLabel + ": session_id must be <= " + MAX_SESSION_ID_LEN + " characters");
   }
-  if (CONTROL_BYTE_RE.test(s) || ZERO_WIDTH_RE.test(s)) {
+  if (textGuard.hasCodepointThreat(s, { singleLine: "reject", zeroWidth: "reject" })) {
     throw new TypeError(fnLabel + ": session_id contains control / zero-width bytes");
   }
   return s;

package/lib/search-synonyms.js

@@ -102,18 +102,6 @@ var MAX_QUERY_LEN            = 500;
 var ALLOWED_KINDS         = ["bidirectional", "directional"];
 var ALLOWED_GROUP_COLS    = ["kind", "terms_json"];
 
-// Refuse C0 control bytes + DEL. Tokens are single-line search terms;
-// any newline / tab in a synonym entry would either be malformed
-// operator input or an attempt to smuggle separator-confusion payload
-// into the search log.
-var CONTROL_BYTE_RE = /[\x00-\x1f\x7f]/;
-// Zero-width + direction-override family — same defense as
-// `supportTickets`. Spelled with \u escapes so the file stays
-// ESLint no-irregular-whitespace clean.
-var ZERO_WIDTH_RE   = new RegExp(
-  "[\\u200B-\\u200F\\u202A-\\u202E\\u2060-\\u2064\\u2066-\\u2069\\uFEFF\\u061C]"
-);
-
 // Slug shape — operator-chosen identifier on synonym groups. Same
 // shape the rest of the shop uses: lowercase alnum + hyphen, must
 // start with a letter, 1..MAX_SLUG_LEN.
@@ -126,6 +114,7 @@ var TOKEN_SPLIT_RE = /[^a-z0-9-]+/g;
 
 // Framework handle (the vendored blamejs); index.js re-exports this as .framework.
 var b = require("./vendor/blamejs");
+var textGuard = require("./text-guard");
 
 // ---- validators ---------------------------------------------------------
 
@@ -166,10 +155,10 @@ function _term(s, label, fnLabel) {
   if (trimmed.length > MAX_TERM_LEN) {
     throw new TypeError(fnLabel + ": " + label + " must be <= " + MAX_TERM_LEN + " characters");
   }
-  if (CONTROL_BYTE_RE.test(trimmed)) {
+  if (textGuard.hasCodepointThreat(trimmed, { singleLine: "reject" })) {
     throw new TypeError(fnLabel + ": " + label + " contains control bytes");
   }
-  if (ZERO_WIDTH_RE.test(trimmed)) {
+  if (textGuard.hasCodepointThreat(trimmed, { zeroWidth: "reject" })) {
     throw new TypeError(fnLabel + ": " + label + " contains zero-width / direction-override characters");
   }
   return trimmed;
@@ -247,13 +236,10 @@ function _queryString(s, fnLabel) {
 // give attackers a denial-of-search vector against legitimate
 // shoppers who happened to paste a stray BOM.
 function _sanitiseQuery(s) {
-  return s
-    .replace(/[\x00-\x1f\x7f]/g, " ")
-    .replace(
-      // Zero-width + direction-override family — see ZERO_WIDTH_RE.
-      new RegExp("[\\u200B-\\u200F\\u202A-\\u202E\\u2060-\\u2064\\u2066-\\u2069\\uFEFF\\u061C]", "g"),
-      ""
-    );
+  // Controls become a SPACE, not nothing: deleting them outright would weld
+  // the words on either side into one token. The invisible characters do get
+  // deleted — they sit inside a word rather than between two.
+  return textGuard.scrubInvisible(s.replace(/[\x00-\x1f\x7f]/g, " "));
 }
 
 function _tokenise(s) {