json-fs-query 2.0.0 → 2.1.0

package/README.md

@@ -101,6 +101,19 @@ await query(`read exercises where name = ${quote(name)}`)
 
 `quote(value, '"')` wraps in double quotes instead.
 
+### Numbers
+
+A number may be negative, have a fraction and an exponent: `12`, `-3`, `1.5`, `2.5e-4`. They work the same in
+`values`, `set`, `where`, arrays and defaults:
+
+```sql
+read workouts where reps > -1
+update workouts set weight = 102.5 where _id = 4
+```
+
+A whole number past 9007199254740991 (`Number.MAX_SAFE_INTEGER`), however it is written (`1e20`,
+`9007199254740993.0`), is an error rather than being stored rounded: quote it to keep it as text. A fraction is stored as the nearest double, as in JSON.
+
 ### Arrays
 
 Arrays are JSON-style, and their strings can use either quote. They can be nested, and are used by `in` and
@@ -135,6 +148,24 @@ read workouts where reps > 8 and where date = '2024-06-02'
 `or where` is not supported yet, and is an error (in 1.x it was silently read as `and where`). And/or with grouping
 and precedence is planned.
 
+### Null and missing fields
+
+A field can be missing (left out) or hold `null`, and both are their own cases, as in SQL: they match no
+comparison (`=`, `!=`, `<`, `between`, `in`, `like`...), and a `null` never equals anything, not even another
+`null`. These test for them instead:
+
+| Condition | Matches when the field |
+|---|---|
+| `where note is null` | is there and holds `null` |
+| `where note is not null` | is there and holds a value |
+| `where note exists` | is there (`null` counts) |
+| `where note does not exist` | is missing |
+
+`where note = null` is an error that points to `is null`, since it could never match. Sorting puts missing first,
+then `null`, then the values (`asc`; `desc` is the reverse).
+
+In 2.0 and earlier, a `null` acted as `0` in comparisons and sorting (`where n < 5` matched it), and `= null` matched it.
+
 ### Creating a record: `create`
 
 The query form of `create()`. Inside the brackets: the fields, then the keys, separated by commas. Everything is
@@ -162,7 +193,7 @@ A field is `<name> <type>`, then any of these, in any order:
 | `optional` | may be left out (see "Fields" above) |
 | `null` / `not null` | may hold `null` / may not (the default) |
 | `auto` | a UUID is generated on insert; the type must be `string` |
-| `default <value>` | used when the field is left out: a string, a whole number, an array, `null`, `true` or `false` (so an `object` field can only default to `null` here; `create()` takes any default) |
+| `default <value>` | used when the field is left out: a string, a number, an array, `null`, `true` or `false` (so an `object` field can only default to `null` here; `create()` takes any default) |
 
 The types are `string`, `int`, `boolean`, `object` and `array`; any other is an error.
 

package/browser/jsonFsQuery.js

@@ -487,6 +487,10 @@ const typeCheck = (type = 'string', value = undefined, nullable = false) => {
         if (Number.isNaN(coerced) || !Number.isInteger(coerced)) {
           throw new Error(`Value "${value}" is not a valid int`);
         }
+        // Past Number.MAX_SAFE_INTEGER an int can't be held exactly: '9007199254740993' would be stored as ...992
+        if (!Number.isSafeInteger(coerced)) {
+          throw new Error(`Value "${value}" is too large for an int (the largest is ${Number.MAX_SAFE_INTEGER})`);
+        }
         return coerced;
       }
     case 'boolean':
@@ -1207,6 +1211,14 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.sortBy = void 0;
+// Where a value sorts before any other, ascending: missing first, then null, then the values (MySQL's null placement,
+// with missing below null). Descending reverses it.
+const presence = (data, property) => {
+  if (!Object.prototype.hasOwnProperty.call(data, property)) {
+    return 0;
+  }
+  return data[property] === null ? 1 : 2;
+};
 const sortBy = (dataSet = [], sortClauses = []) => {
   if (!sortClauses.length) {
     return dataSet;
@@ -1215,8 +1227,13 @@ const sortBy = (dataSet = [], sortClauses = []) => {
     for (const sortClause of sortClauses) {
       const property = sortClause.property;
       const direction = sortClause.direction;
+      const rankA = presence(a, property);
+      const rankB = presence(b, property);
+      if (rankA !== rankB) {
+        return direction === 'asc' ? rankA - rankB : rankB - rankA;
+      }
       // Ordering compares the raw stored values, not by declared type (that's only for equality, see where), so
-      // numeric ids in a string field still sort numerically. Loose so '3' and 3 still tie.
+      // numeric ids in a string field still sort numerically. Loose so '3' and 3 still tie, as do two nulls or two missing.
       // eslint-disable-next-line eqeqeq
       if (a[property] == b[property]) {
         continue;
@@ -1498,6 +1515,12 @@ const likeMatch = (dataValue, pattern) => {
 /**
  * Equality (=, !=, <>, in) compares by the field's declared type when given one (see sameValue), so
  * '3' and 3 match in a string or int field. Ordering (>, <, between, like) compares the raw values.
+ *
+ * Null and missing are their own cases, as in SQL: a field that is null or missing matches no comparison (not even
+ * `!=`), and a null never equals anything, not even another null. Only these test for them:
+ * - `is null`: the field is there and holds null
+ * - `is not null`: the field is there and holds a value (so a missing field doesn't match either)
+ * - `exists` / `does not exist`: the field is there (null counts) / is missing
  * @param data
  * @param property
  * @param comparator
@@ -1505,7 +1528,18 @@ const likeMatch = (dataValue, pattern) => {
  * @param type
  */
 const makeCondition = (data, property, comparator, value, type = undefined) => {
-  if (!Object.prototype.hasOwnProperty.call(data, property)) {
+  const exists = Object.prototype.hasOwnProperty.call(data, property);
+  switch (comparator) {
+    case 'exists':
+      return exists;
+    case 'does not exist':
+      return !exists;
+    case 'is null':
+      return exists && data[property] === null;
+    case 'is not null':
+      return exists && data[property] !== null;
+  }
+  if (!exists || data[property] === null || value === null) {
     return false;
   }
   const dataValue = data[property];
@@ -1526,7 +1560,8 @@ const makeCondition = (data, property, comparator, value, type = undefined) => {
     case 'in':
       return Array.isArray(value) && (type ? value.some(item => (0, _compareAs.sameValue)(type, dataValue, item)) : value.includes(dataValue));
     case 'between':
-      return Array.isArray(value) && value.length === 2 && dataValue >= value[0] && dataValue <= value[1];
+      // A null bound would act as 0 in >= / <=: it bounds nothing, so nothing matches (the parser refuses it)
+      return Array.isArray(value) && value.length === 2 && !value.includes(null) && dataValue >= value[0] && dataValue <= value[1];
     case 'like':
       return likeMatch(dataValue, value);
   }
@@ -2131,18 +2166,20 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.findConditions = void 0;
+require("core-js/modules/es.array.includes.js");
 var _removeFromQuery = require("./removeFromQuery");
 var _parseLiteralValue = require("./parseLiteralValue");
 var _maskStrings = require("./maskStrings");
 // `and`/`or`/`where` count only as whole words: in `on t.x = u.color where ...` the `or` is part of the name `color`
-const conditionMatch = new RegExp(`(^|[^a-z0-9._-])(?:(and|or)\\s+)?(where)\\s+([a-z0-9_-]+\\.?[a-z0-9_-]*)\\s+(=|!=|>|<|>=|<=|in|between|like)\\s+(${_maskStrings.placeholderSource}|([0-9]+)|${_maskStrings.arraySource}|(null)|(true)|(false))`, 'ig');
+const conditionMatch = new RegExp(`(^|[^a-z0-9._-])(?:(and|or)\\s+)?(where)\\s+([a-z0-9_-]+\\.?[a-z0-9_-]*)\\s+(?:(is\\s+not\\s+null|is\\s+null|exists|does\\s+not\\s+exist)(?![a-z0-9_.-])|(=|!=|>|<|>=|<=|in|between|like)\\s+(${_maskStrings.placeholderSource}|(${_parseLiteralValue.numberSource})|${_maskStrings.arraySource}|(null)|(true)|(false)))`, 'ig');
 /**
- * Extract condition clauses from the query: `where <field> <comparator> <value>`, each further one joined with
- * `and where`. `or where` is refused until or is built (TASKS: the where-clauses epic): it used to be read the same as
+ * Extract condition clauses from the query: `where <field> <comparator> <value>`, or `where <field> is null`,
+ * `is not null`, `exists` or `does not exist` (no value), each further one joined with `and where`. A comparison
+ * with the literal null (`= null`) is refused: it would never match, as null equals nothing (see where). `or where` is refused until or is built (TASKS: the where-clauses epic): it used to be read the same as
  * `and where`, which silently returned the wrong rows.
  * @param parsed
  * @param query
- * @throws {Error} For `or where`.
+ * @throws {Error} For `or where`, and for a comparison with null.
  */
 const findConditions = (parsed, query) => {
   const conditionsFound = query.matchAll(conditionMatch);
@@ -2151,18 +2188,33 @@ const findConditions = (parsed, query) => {
     if (typeof condition[2] !== 'undefined' && condition[2].toLowerCase() === 'or') {
       throw new Error('`or where` is not supported yet: use `and where`, or run one query per alternative');
     }
-    parsed['conditions'].push({
-      property: condition[4],
-      comparator: condition[5],
-      value: (0, _parseLiteralValue.parseLiteralValue)(condition, 7)
-    });
+    if (typeof condition[5] !== 'undefined') {
+      parsed['conditions'].push({
+        property: condition[4],
+        comparator: condition[5].toLowerCase().replace(/\s+/g, ' '),
+        value: null
+      });
+    } else {
+      const value = (0, _parseLiteralValue.parseLiteralValue)(condition, 8);
+      if (value === null) {
+        throw new Error(`\`${condition[4]} ${condition[6]} null\` never matches (null equals nothing, as in SQL): use \`${condition[4]} is null\` or \`${condition[4]} is not null\``);
+      }
+      if (condition[6].toLowerCase() === 'between' && Array.isArray(value) && value.includes(null)) {
+        throw new Error(`\`${condition[4]} between\` can't have a null bound: it would never match`);
+      }
+      parsed['conditions'].push({
+        property: condition[4],
+        comparator: condition[6],
+        value
+      });
+    }
     // Leave out the character before the condition (group 1): it isn't part of it, and an earlier removal may have trimmed it
     query = (0, _removeFromQuery.removeFromQuery)(condition[0].slice(condition[1].length), query);
   }
   return query;
 };
 exports.findConditions = findConditions;
-},{"./maskStrings":58,"./parseLiteralValue":60,"./removeFromQuery":63}],47:[function(require,module,exports){
+},{"./maskStrings":58,"./parseLiteralValue":60,"./removeFromQuery":63,"core-js/modules/es.array.includes.js":218}],47:[function(require,module,exports){
 "use strict";
 
 Object.defineProperty(exports, "__esModule", {
@@ -2195,9 +2247,9 @@ exports.findInsertValues = void 0;
 var _removeFromQuery = require("./removeFromQuery");
 var _parseLiteralValue = require("./parseLiteralValue");
 var _maskStrings = require("./maskStrings");
-const literal = `(?:${_maskStrings.placeholderSource.replace('(', '(?:')}|[0-9]+|${_maskStrings.arraySource.replace('(', '(?:')}|null|true|false)`;
+const literal = `(?:${_maskStrings.placeholderSource.replace('(', '(?:')}|${_parseLiteralValue.numberSource}|${_maskStrings.arraySource.replace('(', '(?:')}|null|true|false)`;
 const valuesClauseMatch = new RegExp(`(values)\\s+([a-z0-9_-]+\\s*=\\s*${literal}(?:\\s*,\\s*[a-z0-9_-]+\\s*=\\s*${literal})*)`, 'i');
-const assignmentMatch = new RegExp(`([a-z0-9_-]+)\\s*=\\s*(${_maskStrings.placeholderSource}|([0-9]+)|${_maskStrings.arraySource}|(null)|(true)|(false))`, 'ig');
+const assignmentMatch = new RegExp(`([a-z0-9_-]+)\\s*=\\s*(${_maskStrings.placeholderSource}|(${_parseLiteralValue.numberSource})|${_maskStrings.arraySource}|(null)|(true)|(false))`, 'ig');
 /**
  * Extract an insert's "values field = value, field2 = value2" clause from the query, the same way
  * findUpdateValues() does for "set". insertEntity() accepts an array of rows for bulk inserts, but
@@ -2257,7 +2309,9 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.findJoinEntity = void 0;
 var _removeFromQuery = require("./removeFromQuery");
-const joinEntityMatch = /\.([a-z0-9_-]+)/i;
+// Only straight after the entity (findCommand leaves `.joined ...`): a `.` later on is a number's (`1.5`) or a
+// condition's `entity.field`, not the join
+const joinEntityMatch = /^\.([a-z0-9_-]+)/i;
 /**
  * Extract the main join entity from the query.
  * @param parsed
@@ -2423,9 +2477,9 @@ exports.findUpdateValues = void 0;
 var _removeFromQuery = require("./removeFromQuery");
 var _parseLiteralValue = require("./parseLiteralValue");
 var _maskStrings = require("./maskStrings");
-const literal = `(?:${_maskStrings.placeholderSource.replace('(', '(?:')}|[0-9]+|${_maskStrings.arraySource.replace('(', '(?:')}|null|true|false)`;
+const literal = `(?:${_maskStrings.placeholderSource.replace('(', '(?:')}|${_parseLiteralValue.numberSource}|${_maskStrings.arraySource.replace('(', '(?:')}|null|true|false)`;
 const setClauseMatch = new RegExp(`(set)\\s+([a-z0-9_-]+\\s*=\\s*${literal}(?:\\s*,\\s*[a-z0-9_-]+\\s*=\\s*${literal})*)`, 'i');
-const assignmentMatch = new RegExp(`([a-z0-9_-]+)\\s*=\\s*(${_maskStrings.placeholderSource}|([0-9]+)|${_maskStrings.arraySource}|(null)|(true)|(false))`, 'ig');
+const assignmentMatch = new RegExp(`([a-z0-9_-]+)\\s*=\\s*(${_maskStrings.placeholderSource}|(${_parseLiteralValue.numberSource})|${_maskStrings.arraySource}|(null)|(true)|(false))`, 'ig');
 /**
  * Extract an update's "set field = value, field2 = value2" clause from the query. Uses a two-pass
  * match, unlike the other parsers here: the set clause's own boundary is found first (so it can
@@ -2539,11 +2593,15 @@ const unmaskStrings = (text, quote = 'json') => text.replace(placeholderMatch, (
 exports.unmaskStrings = unmaskStrings;
 const arrayJson = text => {
   try {
-    // A number too big for a double reads as Infinity, which JSON would write back as null: refuse it instead
+    // A number too big for a double reads as Infinity, which JSON would write back as null, and a whole number past
+    // Number.MAX_SAFE_INTEGER may already be rounded: refuse both, as parseNumber does outside arrays
     return JSON.stringify(JSON.parse(unmaskStrings(text), (key, value) => {
       if (typeof value === 'number' && !Number.isFinite(value)) {
         throw new Error(`${value} is too large`);
       }
+      if (Number.isInteger(value) && !Number.isSafeInteger(value)) {
+        throw new Error(`${value} is too large to store exactly (the largest whole number is ${Number.MAX_SAFE_INTEGER}): quote it to keep it as text`);
+      }
       return value;
     }));
   } catch (e) {
@@ -2613,11 +2671,13 @@ require("core-js/modules/esnext.set.symmetric-difference.js");
 require("core-js/modules/esnext.set.union.js");
 var _typeCheck = require("../../introspect/policies/typeCheck");
 var _maskStrings = require("./maskStrings");
+var _parseLiteralValue = require("./parseLiteralValue");
 var _startsQuery = require("./startsQuery");
 const createMatch = exports.createMatch = /^\s*create(?![a-z0-9_-])/i;
 const headMatch = /^\s*create\s+(if\s+not\s+exists\s+)?([a-z0-9_-]+)\s*\(/i;
 const nameSource = '[a-z0-9_-]+';
 const nameMatch = new RegExp(`^${nameSource}$`, 'i');
+const numberMatch = new RegExp(`^${_parseLiteralValue.numberSource}$`, 'i');
 const fieldMatch = new RegExp(`^(${nameSource})\\s+(${nameSource})((?:\\s[\\s\\S]*)?)$`, 'i');
 const primaryMatch = /^primary\s+key\s*\(([^()]*)\)$/i;
 const uniqueMatch = /^unique\s*\(([^()]*)\)$/i;
@@ -2708,8 +2768,8 @@ const parseDefault = token => {
   if (array) {
     return JSON.parse((0, _maskStrings.fromPlaceholder)(array[1]));
   }
-  if (/^[0-9]+$/.test(token)) {
-    return Number(token);
+  if (numberMatch.test(token)) {
+    return (0, _parseLiteralValue.parseNumber)(token);
   }
   switch (token.toLowerCase()) {
     case 'null':
@@ -2913,14 +2973,39 @@ const parseCreate = query => {
   };
 };
 exports.parseCreate = parseCreate;
-},{"../../introspect/policies/typeCheck":13,"./maskStrings":58,"./startsQuery":67,"core-js/modules/es.array.includes.js":218,"core-js/modules/esnext.iterator.constructor.js":232,"core-js/modules/esnext.iterator.filter.js":235,"core-js/modules/esnext.iterator.find.js":236,"core-js/modules/esnext.iterator.map.js":238,"core-js/modules/esnext.iterator.some.js":240,"core-js/modules/esnext.set.add-all.js":254,"core-js/modules/esnext.set.delete-all.js":255,"core-js/modules/esnext.set.difference.js":256,"core-js/modules/esnext.set.every.js":257,"core-js/modules/esnext.set.filter.js":258,"core-js/modules/esnext.set.find.js":259,"core-js/modules/esnext.set.intersection.js":260,"core-js/modules/esnext.set.is-disjoint-from.js":261,"core-js/modules/esnext.set.is-subset-of.js":262,"core-js/modules/esnext.set.is-superset-of.js":263,"core-js/modules/esnext.set.join.js":264,"core-js/modules/esnext.set.map.js":265,"core-js/modules/esnext.set.reduce.js":266,"core-js/modules/esnext.set.some.js":267,"core-js/modules/esnext.set.symmetric-difference.js":268,"core-js/modules/esnext.set.union.js":269}],60:[function(require,module,exports){
+},{"../../introspect/policies/typeCheck":13,"./maskStrings":58,"./parseLiteralValue":60,"./startsQuery":67,"core-js/modules/es.array.includes.js":218,"core-js/modules/esnext.iterator.constructor.js":232,"core-js/modules/esnext.iterator.filter.js":235,"core-js/modules/esnext.iterator.find.js":236,"core-js/modules/esnext.iterator.map.js":238,"core-js/modules/esnext.iterator.some.js":240,"core-js/modules/esnext.set.add-all.js":254,"core-js/modules/esnext.set.delete-all.js":255,"core-js/modules/esnext.set.difference.js":256,"core-js/modules/esnext.set.every.js":257,"core-js/modules/esnext.set.filter.js":258,"core-js/modules/esnext.set.find.js":259,"core-js/modules/esnext.set.intersection.js":260,"core-js/modules/esnext.set.is-disjoint-from.js":261,"core-js/modules/esnext.set.is-subset-of.js":262,"core-js/modules/esnext.set.is-superset-of.js":263,"core-js/modules/esnext.set.join.js":264,"core-js/modules/esnext.set.map.js":265,"core-js/modules/esnext.set.reduce.js":266,"core-js/modules/esnext.set.some.js":267,"core-js/modules/esnext.set.symmetric-difference.js":268,"core-js/modules/esnext.set.union.js":269}],60:[function(require,module,exports){
 "use strict";
 
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
-exports.parseLiteralValue = void 0;
+exports.parseNumber = exports.parseLiteralValue = exports.numberSource = void 0;
 var _maskStrings = require("./maskStrings");
+/**
+ * Regex source matching one number literal: an optional `-`, digits, an optional fraction and an optional exponent
+ * (`12`, `-3`, `1.5`, `1e3`, `2.5E-4`), with no group of its own. Anything left after it (`5abc`, `1.5.2`) is an
+ * unrecognized query value, as for any other leftover text.
+ */
+const numberSource = exports.numberSource = '-?[0-9]+(?:\\.[0-9]+)?(?:e[+-]?[0-9]+)?';
+/**
+ * Read one number literal's text (see numberSource) as a number, refusing any it can't hold exactly as written: a
+ * whole number past Number.MAX_SAFE_INTEGER, however it is written (`9007199254740993`, `9007199254740993.0`, `1e20`),
+ * may be stored rounded (9007199254740993 as 9007199254740992), and a number too big for a double reads as Infinity,
+ * which JSON writes back as null. Fractions are kept as the nearest double, as JSON does (an exact decimal type is
+ * planned).
+ * @param text
+ * @throws {Error} For an integer past Number.MAX_SAFE_INTEGER, or a number too big for a double.
+ */
+const parseNumber = text => {
+  const value = Number(text);
+  if (!Number.isFinite(value)) {
+    throw new Error(`${text} is too large`);
+  }
+  if (Number.isInteger(value) && !Number.isSafeInteger(value)) {
+    throw new Error(`${text} is too large to store exactly (the largest whole number is ${Number.MAX_SAFE_INTEGER}): quote it to keep it as text`);
+  }
+  return value;
+};
 /**
  * Several parsers (findConditions, findInsertValues, findUpdateValues) capture a query-string
  * literal into one of six consecutive alternation groups depending on its shape - quoted string
@@ -2931,12 +3016,13 @@ var _maskStrings = require("./maskStrings");
  * @param match
  * @param baseIndex
  */
+exports.parseNumber = parseNumber;
 const parseLiteralValue = (match, baseIndex) => {
   if (typeof match[baseIndex] !== 'undefined') {
     return (0, _maskStrings.fromPlaceholder)(match[baseIndex]);
   }
   if (typeof match[baseIndex + 1] !== 'undefined') {
-    return Number(match[baseIndex + 1]);
+    return parseNumber(match[baseIndex + 1]);
   }
   if (typeof match[baseIndex + 2] !== 'undefined') {
     // An array is a placeholder from maskArrays, holding its JSON