@jarenjs/json 0.9.2 → 0.34.2

package/src/query/normalize.js

@@ -18,9 +18,11 @@
 // allocating in the same frame). Free names become external parameters,
 // collected in order of first appearance.
 
-import { parseJSONPath, JSONPathSyntaxError } from '../path.js';
+import { parseJSONPath, JSONPathSyntaxError, RE_JSONPATH_VARIABLE_HEAD } from '../path.js';
 import { isSingularSegments } from '../segments.js';
+import { encodeJSONPointerSegment } from '../pointer.js';
 import { JsonQueryCompileError } from './errors.js';
+import { deepFreeze, isJsonObject } from '@jarenjs/core/object';
 // The operator registry: `name -> { params, result, compile }`. Only
 // referenced inside functions (never at module evaluation time), so the
 // import cycle normalize.js <-> operators.js is initialization-safe.
@@ -28,6 +30,17 @@ import { OPERATORS } from './operators.js';
 
 //#region cardinality
 
+/**
+ * The twelve node kinds of the normalized form, sorted — the published
+ * AST contract (QUERY-FORMAT.md Appendix C). An exhaustiveness gate
+ * asserts a corpus exercising every construct produces exactly this
+ * set, so a new kind cannot ship undocumented.
+ */
+export const NODE_KINDS = Object.freeze([
+  'array', 'call', 'flwor', 'let', 'literal', 'map',
+  'object', 'op', 'path', 'quant', 'raw', 'var',
+]);
+
 /** Statically empty (the node always evaluates to the empty sequence). */
 export const CARD_ZERO = 0;
 /** Always exactly one item; compile.js skips all sequence checks. */
@@ -74,17 +87,38 @@ export function sumCard(a, b) {
 const hasOwn = Object.hasOwn;
 
 const VAR_NAME_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
-const VAR_HEAD_RE = /^\$([A-Za-z_][A-Za-z0-9_]*)/;
+// The '$name' head of a variable-rooted path string; shared with the
+// json-path-segments format tester so both split at the same point.
+const VAR_HEAD_RE = RE_JSONPATH_VARIABLE_HEAD;
 
 // FLWOR clause keys (QUERY-FORMAT.md section 6.1) and quantifier keys
 // (section 7). Clauses apply in the fixed semantic order of section 6.1
 // regardless of JSON key order (D7).
-const FLWOR_KEYS = new Set(['$for', '$let', '$as', '$where', '$groupby', '$orderby', '$count', '$return']);
+const FLWOR_KEYS = new Set(['$fold', '$for', '$let', '$as', '$where', '$groupby', '$orderby', '$count', '$return']);
 const QUANTIFIER_KEYS = new Set(['$some', '$every', '$satisfies']);
 
 // Keys of the explicit $orderby key-spec form (section 6.6). Contextual:
 // they are not operators and stay outside the KNOWN_KEYS vocabulary.
-const ORDERBY_SPEC_KEYS = new Set(['$key', '$dir', '$empty']);
+const ORDERBY_SPEC_KEYS = new Set(['$key', '$dir', '$empty', '$collation']);
+
+// Keys of the extended $for binding form (sections 6.2, 6.10): the
+// source, a positional variable, the outer-join switch, and the window
+// specification. Also contextual - not operators.
+const FOR_BINDING_KEYS = new Set(['$in', '$at', '$allowing-empty', '$window', '$size', '$step']);
+
+// Whether a binding source uses the extended object form rather than
+// being an ordinary expression. Any of its keys marks it, so a malformed
+// combination is diagnosed as a bad binding instead of silently
+// normalizing as a map constructor.
+function isExtendedBinding(source) {
+  if (!isJsonObject(source))
+    return false;
+  for (const key of FOR_BINDING_KEYS) {
+    if (hasOwn(source, key))
+      return true;
+  }
+  return false;
+}
 
 // Escape hatches (QUERY-FORMAT.md section 3.5): structural forms with
 // dedicated normalizer cases; every other operator lives in the registry.
@@ -106,19 +140,32 @@ function isVocabularyKey(key, ctx) {
 
 //#region helpers
 
-function isPlainObject(v) {
-  return typeof v === 'object' && v !== null && !Array.isArray(v);
-}
-
-// RFC 6901 reference token escaping for docPath pointers
-function escapeToken(token) {
-  if (token.indexOf('~') < 0 && token.indexOf('/') < 0)
-    return token;
-  return token.replace(/~/g, '~0').replace(/\//g, '~1');
+function fail(code, message, docPath, options) {
+  throw new JsonQueryCompileError(code, message, docPath, options);
 }
 
-function fail(code, message, docPath) {
-  throw new JsonQueryCompileError(code, message, docPath);
+/**
+ * Project a safe diagnostic string from whatever a host hook threw:
+ * no `.message` read on a raw value, no user coercion, no
+ * proxy-observable reflection — the compiler must never fail while
+ * describing a host failure.
+ * @param {unknown} e
+ * @returns {string}
+ */
+export function hostFailureText(e) {
+  try {
+    if (e instanceof Error) {
+      const message = e.message;
+      if (typeof message === 'string') return message;
+      return 'host error (message unavailable)';
+    }
+  }
+  catch { /* hostile classification or accessor */ }
+  if (e === null) return 'null';
+  const t = typeof e;
+  if (t === 'string') return e.length > 80 ? e.slice(0, 80) + '…' : e;
+  if (t === 'number' || t === 'boolean' || t === 'bigint' || t === 'undefined') return String(e);
+  return t === 'symbol' ? 'a symbol' : t === 'function' ? 'a function' : 'an object';
 }
 
 /**
@@ -144,22 +191,13 @@ export function deepFreezeCopy(value) {
   return Object.freeze(out);
 }
 
-function deepFreeze(value) {
-  if (typeof value !== 'object' || value === null)
-    return value;
-  const keys = Object.keys(value);
-  for (let i = 0; i < keys.length; i++)
-    deepFreeze(value[keys[i]]);
-  return Object.freeze(value);
-}
-
 //#endregion
 
 //#region strings (Rule 2)
 
-function parsePathString(source, original, docPath) {
+function parsePathString(source, original, docPath, ctx) {
   try {
-    return parseJSONPath(source);
+    return parseJSONPath(source, ctx.pathOptions);
   }
   catch (e) {
     /* c8 ignore next 2 -- parseJSONPath only throws syntax errors */
@@ -186,14 +224,23 @@ function makePathNode(name, rootSlot, external, rootCard, segments, docPath) {
 
 // resolve a variable reference: walk the lexical scope chain; a free name
 // is an external parameter, allocated a slot on first appearance (spec
-// section 9: use is the declaration)
-function resolveVariable(name, scope, ctx) {
+// section 9: use is the declaration). Under a closed-world compilation
+// (options.externals), only declared names may stay free - every other
+// free name is JQ0005 at its own reference site.
+function resolveVariable(name, scope, ctx, docPath) {
   for (let sc = scope; sc !== null; sc = sc.parent) {
     if (sc.name === name)
       return { slot: sc.slot, card: sc.card, external: false };
   }
   let slot = ctx.externals.get(name);
   if (slot === undefined) {
+    if (ctx.declaredExternals !== null && !ctx.declaredExternals.has(name)) {
+      const declared = [...ctx.declaredExternals];
+      fail('JQ0005', `'$${name}' is neither bound by an enclosing phrase nor a declared external`
+        + (declared.length === 0
+          ? ' (this query was compiled closed-world, declaring no externals)'
+          : ` (declared externals: ${declared.map((n) => "'" + n + "'").join(', ')})`), docPath);
+    }
     slot = ctx.nextSlot++;
     ctx.externals.set(name, slot);
   }
@@ -210,7 +257,7 @@ function normalizeString(s, docPath, scope, ctx) {
   if (c1 === 0x24) // '$$' escape: drop exactly one leading '$'
     return makeLiteral(s.slice(1), docPath);
   if (c1 === 0x2E || c1 === 0x5B) { // '$.' | '$[' | '$..' - absolute path
-    const ast = parsePathString(s, s, docPath);
+    const ast = parsePathString(s, s, docPath, ctx);
     return makePathNode('$', 0, false, CARD_ONE, ast.segments, docPath);
   }
   const m = VAR_HEAD_RE.exec(s);
@@ -218,12 +265,12 @@ function normalizeString(s, docPath, scope, ctx) {
     return fail('JQ0004', `'${s}' is not a valid path or escape`, docPath);
   const name = m[1];
   const rest = s.slice(m[0].length);
-  const ref = resolveVariable(name, scope, ctx);
+  const ref = resolveVariable(name, scope, ctx, docPath);
   if (rest === '') // bare '$name': whole-variable reference
     return Object.freeze({ kind: 'var', card: ref.card, docPath, slot: ref.slot, external: ref.external, name });
   // variable-rooted path: the grammar is RFC 9535 with the root identifier
   // replaced by the variable reference - parse with a substituted '$'
-  const ast = parsePathString('$' + rest, s, docPath);
+  const ast = parsePathString('$' + rest, s, docPath, ctx);
   return makePathNode(name, ref.slot, ref.external, ref.card, ast.segments, docPath);
 }
 
@@ -244,7 +291,7 @@ function normalizeObject(obj, docPath, scope, ctx) {
       const name = keys[i];
       entries[i] = Object.freeze({
         name,
-        expr: normalizeExpr(obj[name], docPath + '/' + escapeToken(name), scope, ctx),
+        expr: normalizeExpr(obj[name], docPath + '/' + encodeJSONPointerSegment(name), scope, ctx),
       });
     }
     return Object.freeze({ kind: 'object', card: CARD_ONE, docPath, entries: Object.freeze(entries) });
@@ -265,7 +312,8 @@ function normalizePhrase(obj, keys, docPath, scope, ctx) {
       break;
     }
   }
-  if (allFlwor && keys.length >= 2 && hasOwn(obj, '$return') && (hasOwn(obj, '$for') || hasOwn(obj, '$let'))) {
+  if (allFlwor && keys.length >= 2 && hasOwn(obj, '$return')
+    && (hasOwn(obj, '$for') || hasOwn(obj, '$let') || hasOwn(obj, '$fold'))) {
     if (keys.length === 2 && hasOwn(obj, '$let'))
       return normalizeLetPhrase(obj, docPath, scope, ctx);
     return normalizeFlworPhrase(obj, docPath, scope, ctx);
@@ -318,11 +366,57 @@ function levenshtein(a, b) {
 // suggestion candidates, built lazily (see the isVocabularyKey note)
 let VOCABULARY_NAMES = null;
 
-// JQ0002 for an unknown $-key, with a "did you mean" suggestion when a
-// vocabulary key is within Levenshtein distance 2 (compile-time only).
-// With host extensions the candidate list is built per call (this is the
-// error path); the lazy global stays for the core-only case.
+// SEMANTIC aliases — the names a writer reaches for from other query
+// languages (SQL, XPath, JSONata, JS array methods) or a plausible
+// synonym, which Levenshtein cannot reach because they are lexically
+// far from the real operator. This is the failure mode an LLM lands in
+// most: it guesses `$first`/`$filter`/`$map` and, without a pointer,
+// abandons the language. Each entry is the CLOSEST real spelling; a
+// value of null means "no operator does this — here is how instead".
+const OPERATOR_ALIASES = {
+  // sequence access
+  $first: "$head", $last: "$head of $reverse", $nth: "$get", $at: "$get",
+  $take: "$subsequence", $skip: "$subsequence", $slice: "$subsequence",
+  $drop: "$subsequence", $limit: "$subsequence",
+  // filtering / mapping / folding: these are FLWOR or a JSONPath filter,
+  // not operators ($where, $map, $fold, $orderby ARE real keys and never
+  // reach here — only the guesses that miss them are listed)
+  $filter: "a JSONPath filter like $[?(@.x > 1)] or $where in a $for",
+  $select: "a $for phrase with $return", $flatmap: "a $for phrase",
+  $reduce: "$fold", $foldl: "$fold", $aggregate: "$fold",
+  $group: "$groupby", "$group-by": "$groupby",
+  $sortby: "$sort (sorts scalars; order objects via a $for over a sorted key)",
+  "$sort-by": "$sort (scalars only)", $order: "$sort",
+  // aggregates / arithmetic
+  $size: "$count", $len: "$length", $abs: null, $round: null, $floor: "$idiv",
+  $ceil: null, $sqrt: null, $pow: null, $modulo: "$mod", $remainder: "$mod",
+  $subtract: "$sub", $multiply: "$mul", $divide: "$div", $minus: "$sub",
+  $times: "$mul", $negate: "$neg", $product: "$mul", $total: "$sum",
+  // strings / collections
+  $join: "$string-join", $split: null, $includes: "$contains",
+  $indexof: "$index-of", $find: "$index-of", $keys: "$entries",
+  $values: "$entries then $get", $has: "$exists", $tostring: "$string",
+  $tonumber: "$number", $len_str: "$string-length", $trim: "$normalize-space",
+  $lowercase: "$lower", $uppercase: "$upper", $startswith: "$starts-with",
+  $endswith: "$ends-with", $unique: "$distinct", $flatten: "a $for phrase",
+  // conditionals ($coalesce is a real operator; the guesses are here)
+  $case: "$if", $cond: "$if", $switch: "$if", $ternary: "$if",
+  $ifnull: "$default", $ifempty: "$default", $nvl: "$default",
+};
+
+// JQ0002 for an unknown $-key, with a "did you mean" suggestion. A
+// curated SEMANTIC alias wins first (the cross-language guesses
+// Levenshtein misses); otherwise the nearest vocabulary key within
+// Levenshtein distance 2. Compile-time only; with host extensions the
+// candidate list is built per call.
 function failUnknownOperator(key, docPath, ctx) {
+  if (hasOwn(OPERATOR_ALIASES, key)) {
+    const target = OPERATOR_ALIASES[key];
+    const hint = target === null
+      ? ' (no operator does this in jaren-query)'
+      : ` (use ${target.startsWith('$') && !target.includes(' ') ? `'${target}'` : target})`;
+    return fail('JQ0002', `unknown operator '${key}'${hint}`, docPath);
+  }
   if (VOCABULARY_NAMES === null) {
     VOCABULARY_NAMES = [
       ...FLWOR_KEYS, ...QUANTIFIER_KEYS, ...ESCAPE_KEYS, ...Object.keys(OPERATORS),
@@ -378,15 +472,22 @@ function normalizeElements(arg, docPath, scope, ctx) {
 // No hook installed is JQ0008; a hook rejection (invalid schema) is
 // JQ0009, both at the owning operator's/clause's docPath.
 function compileSchemaLiteral(value, schemaPath, opPath, ctx) {
-  if (ctx.compileTypeTest === null)
+  if (ctx.compileTypeTest === null) {
+    // Analysis mode (QUERY-FORMAT.md Appendix C.1): the schema literal
+    // normalizes without a compiled predicate, so a consumer can analyse
+    // a document it could not execute. Strictly opt-in — compilation
+    // never sets ctx.analysis, so its behaviour is untouched.
+    if (ctx.analysis)
+      return { schema: deepFreezeCopy(value), test: null };
     fail('JQ0008', 'schema operators require a type-test compiler (options.compileTypeTest)', opPath);
+  }
   const schema = deepFreezeCopy(value);
   let test;
   try {
     test = ctx.compileTypeTest(schema, schemaPath);
   }
   catch (e) {
-    fail('JQ0009', `invalid schema literal: ${e.message}`, opPath);
+    fail('JQ0009', `invalid schema literal: ${hostFailureText(e)}`, opPath, { cause: e });
   }
   if (typeof test !== 'function')
     fail('JQ0009', 'the type-test compiler did not return a predicate function', opPath);
@@ -404,7 +505,7 @@ function makeRaw(value, docPath) {
 // One argument position of a registry operator, per its declared kind:
 // 'expr' normalizes an ordinary expression; 'raw' captures the value
 // verbatim, unevaluated; 'schema' is 'raw' plus a compiled type-test
-// predicate (the type-system work order's schema arguments); 'name'
+// predicate, for schema-literal arguments (`compileSchemaLiteral`); 'name'
 // captures a validated variable name string. 'raw', 'schema' and 'name'
 // produce inert `raw` nodes - compile-time data, never compiled.
 function normalizeArg(kind, value, argPath, scope, ctx, opPath) {
@@ -450,9 +551,15 @@ function argCards(args) {
 // `result` - individual operators never re-check structure.
 function normalizeOperatorCall(key, entry, arg, docPath, opPath, scope, ctx) {
   const args = normalizeParams(key, entry.params, arg, opPath, scope, ctx);
-  return Object.freeze({
+  /** @type {any} */
+  const node = {
     kind: 'op', card: entry.result(argCards(args)), docPath, name: key, args: Object.freeze(args),
-  });
+  };
+  // $range materializes its whole result: the resource guard becomes
+  // configurable through the compilation's limits (compileOp hands the
+  // node through to the entry's compile)
+  if (key === '$range' && ctx.limits !== null) node.limits = ctx.limits;
+  return Object.freeze(node);
 }
 
 // Helpers handed to an extension entry's `normalize` override; see
@@ -485,10 +592,30 @@ function normalizeExtensionCall(key, entry, arg, docPath, opPath, scope, ctx) {
 
 function normalizeOperator(key, arg, docPath, scope, ctx) {
   const opPath = docPath + '/' + key;
+  ctx.usedOps.add(key); // dependency reporting (query.explain)
   switch (key) {
     case '$const': // quote: verbatim single item, nothing inside evaluated
       return Object.freeze({ kind: 'literal', card: CARD_ONE, docPath, value: deepFreezeCopy(arg) });
 
+    case '$call': { // a registered trusted host function (options.functions)
+      if (!Array.isArray(arg) || arg.length < 1 || typeof arg[0] !== 'string')
+        return fail('JQ0010', "'$call' requires ['name', ...argument expressions]", opPath);
+      const name = arg[0];
+      const fn = ctx.functions !== null && hasOwn(ctx.functions, name)
+        ? ctx.functions[name]
+        : undefined;
+      if (fn === undefined)
+        return fail('JQ0010', `'$call' names no registered function '${name}'`, opPath);
+      ctx.usedFunctions.add(name);
+      const callArgs = new Array(arg.length - 1);
+      for (let i = 1; i < arg.length; i++)
+        callArgs[i - 1] = normalizeExpr(arg[i], opPath + '/' + i, scope, ctx);
+      return Object.freeze({
+        kind: 'call', card: CARD_OPT, docPath, name, fn,
+        args: Object.freeze(callArgs),
+      });
+    }
+
     case '$map': { // general map constructor (section 3.5.2)
       const list = requireExprArray(key, arg, 0, Infinity, opPath);
       const pairs = new Array(list.length);
@@ -538,7 +665,7 @@ function bindPhraseName(name, phraseNames, bindPath) {
 }
 
 function requireBindingObject(clause, bindObj, clausePath) {
-  if (!isPlainObject(bindObj))
+  if (!isJsonObject(bindObj))
     fail('JQ0003', `'${clause}' takes an object of variable bindings`, clausePath);
   const names = Object.keys(bindObj);
   if (names.length === 0)
@@ -556,11 +683,11 @@ function normalizeLetBindings(letObj, letPath, scope, ctx, phraseNames, bindings
   let sc = scope;
   for (let i = 0; i < names.length; i++) {
     const name = names[i];
-    const bindPath = letPath + '/' + escapeToken(name);
+    const bindPath = letPath + '/' + encodeJSONPointerSegment(name);
     bindPhraseName(name, phraseNames, bindPath);
     const source = letObj[name];
-    if (isPlainObject(source) && (hasOwn(source, '$in') || hasOwn(source, '$at')))
-      return fail('JQ0003', "the extended '$in'/'$at' binding form is not available in '$let'", bindPath);
+    if (isExtendedBinding(source))
+      return fail('JQ0003', "the extended binding form is not available in '$let'", bindPath);
     const expr = normalizeExpr(source, bindPath, sc, ctx);
     const slot = ctx.nextSlot++;
     sc = { name, slot, card: expr.card, parent: sc };
@@ -581,6 +708,34 @@ function normalizeLetPhrase(obj, docPath, scope, ctx) {
   });
 }
 
+// The window specification of an extended $for binding (section 6.10).
+// `$size`/`$step` are integer literals, not expressions: a window width
+// that varied per tuple could not be compiled into a specialized loop,
+// and no use has asked for one.
+//
+// A **tumbling** window partitions the item stream - every item lands in
+// exactly one window - so its final window is kept even when short,
+// because dropping it would silently lose data. A **sliding** window is
+// a moving view of fixed width, so a short window is not one of them and
+// the tail is not emitted. That is the whole rule.
+function normalizeWindowSpec(source, bindPath) {
+  const kind = source.$window;
+  if (kind !== 'tumbling' && kind !== 'sliding')
+    return fail('JQ0003', "'$window' must be 'tumbling' or 'sliding'", bindPath + '/$window');
+  if (!hasOwn(source, '$size'))
+    return fail('JQ0003', "a '$window' binding requires '$size'", bindPath);
+  const size = source.$size;
+  if (!Number.isInteger(size) || size < 1)
+    return fail('JQ0003', "'$size' must be a positive integer", bindPath + '/$size');
+  let step = kind === 'tumbling' ? size : 1;
+  if (hasOwn(source, '$step')) {
+    step = source.$step;
+    if (!Number.isInteger(step) || step < 1)
+      return fail('JQ0003', "'$step' must be a positive integer", bindPath + '/$step');
+  }
+  return Object.freeze({ sliding: kind === 'sliding', size, step });
+}
+
 // $for bindings (section 6.2): each name iterates its source, one item
 // per tuple (card ONE), with D4 array unpacking at runtime. The extended
 // {"$in": expr, "$at": name} form additionally binds a 0-based position
@@ -591,23 +746,45 @@ function normalizeForBindings(forObj, forPath, scope, ctx, phraseNames, bindings
   let sc = scope;
   for (let i = 0; i < names.length; i++) {
     const name = names[i];
-    const bindPath = forPath + '/' + escapeToken(name);
+    const bindPath = forPath + '/' + encodeJSONPointerSegment(name);
     bindPhraseName(name, phraseNames, bindPath);
     let source = forObj[name];
     let sourcePath = bindPath;
     let atName = null;
-    if (isPlainObject(source) && (hasOwn(source, '$in') || hasOwn(source, '$at'))) {
-      if (Object.keys(source).length !== 2 || !hasOwn(source, '$in') || !hasOwn(source, '$at'))
-        return fail('JQ0003', "the extended binding form takes exactly the keys '$in' and '$at'", bindPath);
-      atName = source.$at;
-      if (typeof atName !== 'string' || !VAR_NAME_RE.test(atName))
-        return fail('JQ0003', "'$at' takes a variable name string", bindPath + '/$at');
+    let allowingEmpty = false;
+    let window = null;
+    if (isExtendedBinding(source)) {
+      const bindKeys = Object.keys(source);
+      for (let k = 0; k < bindKeys.length; k++) {
+        if (!FOR_BINDING_KEYS.has(bindKeys[k]))
+          return fail('JQ0003', `'${bindKeys[k]}' is not a valid key of an extended '$for' binding`, bindPath);
+      }
+      if (!hasOwn(source, '$in'))
+        return fail('JQ0003', "an extended '$for' binding requires '$in'", bindPath);
+      if (hasOwn(source, '$at')) {
+        atName = source.$at;
+        if (typeof atName !== 'string' || !VAR_NAME_RE.test(atName))
+          return fail('JQ0003', "'$at' takes a variable name string", bindPath + '/$at');
+      }
+      if (hasOwn(source, '$allowing-empty')) {
+        if (typeof source['$allowing-empty'] !== 'boolean')
+          return fail('JQ0003', "'$allowing-empty' takes a boolean", bindPath + '/$allowing-empty');
+        allowingEmpty = source['$allowing-empty'];
+      }
+      if (hasOwn(source, '$window'))
+        window = normalizeWindowSpec(source, bindPath);
+      else if (hasOwn(source, '$size') || hasOwn(source, '$step'))
+        return fail('JQ0003', "'$size'/'$step' require '$window'", bindPath);
       source = source.$in;
       sourcePath = bindPath + '/$in';
     }
     const expr = normalizeExpr(source, sourcePath, sc, ctx);
     const slot = ctx.nextSlot++;
-    sc = { name, slot, card: CARD_ONE, parent: sc };
+    // a window variable holds the window's items, and an $allowing-empty
+    // binding may hold the empty sequence: neither is the plain
+    // exactly-one-item binding the compiler specializes for
+    const bindCard = window !== null ? CARD_MANY : (allowingEmpty ? CARD_OPT : CARD_ONE);
+    sc = { name, slot, card: bindCard, parent: sc };
     tupleSlots.push({ name, slot });
     let atSlot = -1;
     if (atName !== null) {
@@ -616,19 +793,25 @@ function normalizeForBindings(forObj, forPath, scope, ctx, phraseNames, bindings
       sc = { name: atName, slot: atSlot, card: CARD_ONE, parent: sc };
       tupleSlots.push({ name: atName, slot: atSlot });
     }
-    bindings.push(Object.freeze({ name, slot, expr, atSlot }));
+    bindings.push(Object.freeze({ name, slot, expr, atSlot, allowingEmpty, window }));
   }
   return sc;
 }
 
 // One $orderby key spec (section 6.6): an expression (ascending,
-// empty-least) or the explicit {"$key", "$dir"?, "$empty"?} form.
+// empty-least) or the explicit {"$key", "$dir"?, "$empty"?,
+// "$collation"?} form. A $collation names a registered pure compare
+// function (options.collations) applied to STRING keys; number keys
+// keep numeric order.
 function normalizeOrderbySpec(spec, specPath, scope, ctx) {
   let key = spec;
   let keyPath = specPath;
   let desc = false;
   let emptyGreatest = false;
-  if (isPlainObject(spec) && (hasOwn(spec, '$key') || hasOwn(spec, '$dir') || hasOwn(spec, '$empty'))) {
+  let collation = null;
+  let collationName = null;
+  if (isJsonObject(spec)
+    && (hasOwn(spec, '$key') || hasOwn(spec, '$dir') || hasOwn(spec, '$empty') || hasOwn(spec, '$collation'))) {
     const specKeys = Object.keys(spec);
     for (let i = 0; i < specKeys.length; i++) {
       if (!ORDERBY_SPEC_KEYS.has(specKeys[i]))
@@ -646,21 +829,37 @@ function normalizeOrderbySpec(spec, specPath, scope, ctx) {
         return fail('JQ0003', "'$empty' must be 'least' or 'greatest'", specPath + '/$empty');
       emptyGreatest = spec.$empty === 'greatest';
     }
+    if (hasOwn(spec, '$collation')) {
+      if (typeof spec.$collation !== 'string')
+        return fail('JQ0003', "'$collation' must be a registered collation name", specPath + '/$collation');
+      collationName = spec.$collation;
+      collation = ctx.collations !== null && hasOwn(ctx.collations, collationName)
+        ? ctx.collations[collationName]
+        : undefined;
+      if (collation === undefined)
+        return fail('JQ0010', `'$collation' names no registered collation '${collationName}'`, specPath + '/$collation');
+      ctx.usedCollations.add(collationName);
+    }
     key = spec.$key;
     keyPath = specPath + '/$key';
   }
   return Object.freeze({
     key: normalizeExpr(key, keyPath, scope, ctx),
-    desc, emptyGreatest, docPath: specPath,
+    desc, emptyGreatest, collation, collationName, docPath: specPath,
   });
 }
 
-// Collect the frame slots an expression subtree reads (variable
-// references and path roots). Barrier liveness: $groupby/$orderby
-// materialize only the phrase binding slots that later clauses read.
-// Over-approximation is safe; slots written by nested phrases before
-// being read merely widen a snapshot harmlessly.
-function collectReadSlots(node, out) {
+// Barrier liveness: $groupby/$orderby materialize only the phrase
+// binding slots that later clauses read. Slots written by nested phrases
+// before being read merely widen a snapshot harmlessly. The hash-join
+// planner (compile.js) reuses this to prove a probe side uncorrelated.
+/**
+ * Collect the frame slots an expression subtree reads (variable
+ * references and path roots) into `out`. Over-approximation is safe.
+ * @param {object} node - a frozen AST node
+ * @param {Set<number>} out - accumulator of slot indexes
+ */
+export function collectReadSlots(node, out) {
   switch (node.kind) {
     case 'literal':
       return;
@@ -687,6 +886,7 @@ function collectReadSlots(node, out) {
     case 'raw': // compile-time data of a registry operator, never evaluated
       return;
     case 'op':
+    case 'call':
       for (let i = 0; i < node.args.length; i++)
         collectReadSlots(node.args[i], out);
       return;
@@ -701,6 +901,8 @@ function collectReadSlots(node, out) {
       collectReadSlots(node.satisfies, out);
       return;
     default: { // 'flwor'
+      if (node.fold !== null)
+        collectReadSlots(node.fold.expr, out);
       for (let i = 0; i < node.forBindings.length; i++)
         collectReadSlots(node.forBindings[i].expr, out);
       for (let i = 0; i < node.letBindings.length; i++)
@@ -739,8 +941,34 @@ function normalizeFlworPhrase(obj, docPath, scope, ctx) {
   const letBindings = [];
   let sc = scope;
 
+  // $fold (section 6.9): the accumulator clause. Its initial value is
+  // evaluated ONCE, in the enclosing scope, before the tuple stream
+  // starts; $return then yields the accumulator's next value per
+  // surviving tuple, and the phrase's value is the final accumulator
+  // instead of the collected $return sequence. This is what gives the
+  // language its fold without giving JSON a way to spell a function
+  // value: the accumulator is a binding, not a lambda parameter.
+  let fold = null;
+  if (hasOwn(obj, '$fold')) {
+    const foldPath = docPath + '/$fold';
+    const names = requireBindingObject('$fold', obj.$fold, foldPath);
+    if (names.length !== 1)
+      return fail('JQ0003', "'$fold' takes exactly one accumulator binding", foldPath);
+    const name = names[0];
+    const bindPath = foldPath + '/' + encodeJSONPointerSegment(name);
+    bindPhraseName(name, phraseNames, bindPath);
+    // the initial value cannot see this phrase's own bindings
+    const expr = normalizeExpr(obj.$fold[name], bindPath, scope, ctx);
+    fold = { name, slot: ctx.nextSlot++, expr, docPath: bindPath };
+  }
+
   if (hasOwn(obj, '$for'))
     sc = normalizeForBindings(obj.$for, docPath + '/$for', sc, ctx, phraseNames, forBindings, tupleSlots);
+  // the accumulator binds after $for - a $for source is iterated once and
+  // must not depend on a value that changes per tuple - and before $let,
+  // so every later clause sees the accumulation so far
+  if (fold !== null)
+    sc = { name: fold.name, slot: fold.slot, card: CARD_MANY, parent: sc };
   if (hasOwn(obj, '$let')) {
     const before = letBindings.length;
     sc = normalizeLetBindings(obj.$let, docPath + '/$let', sc, ctx, phraseNames, letBindings);
@@ -757,7 +985,7 @@ function normalizeFlworPhrase(obj, docPath, scope, ctx) {
   if (hasOwn(obj, '$as')) {
     const asPath = docPath + '/$as';
     const asObj = obj.$as;
-    if (!isPlainObject(asObj))
+    if (!isJsonObject(asObj))
       return fail('JQ0003', "'$as' takes an object of variable-name to schema members", asPath);
     const names = Object.keys(asObj);
     if (names.length === 0)
@@ -765,7 +993,7 @@ function normalizeFlworPhrase(obj, docPath, scope, ctx) {
     const checks = new Array(names.length);
     for (let i = 0; i < names.length; i++) {
       const name = names[i];
-      const checkPath = asPath + '/' + escapeToken(name);
+      const checkPath = asPath + '/' + encodeJSONPointerSegment(name);
       let slot = -1;
       for (let j = 0; j < tupleSlots.length; j++) {
         if (tupleSlots[j].name === name) {
@@ -799,7 +1027,7 @@ function normalizeFlworPhrase(obj, docPath, scope, ctx) {
     const keys = new Array(names.length);
     for (let i = 0; i < names.length; i++) {
       const name = names[i];
-      const bindPath = groupPath + '/' + escapeToken(name);
+      const bindPath = groupPath + '/' + encodeJSONPointerSegment(name);
       bindPhraseName(name, phraseNames, bindPath);
       // key expressions evaluate per tuple, in the pre-group scope
       const expr = normalizeExpr(obj.$groupby[name], bindPath, sc, ctx);
@@ -807,6 +1035,10 @@ function normalizeFlworPhrase(obj, docPath, scope, ctx) {
     }
     // post-group scope: same slots, rebound cards
     sc = scope;
+    // the accumulator is not a tuple variable - $groupby does not rebind
+    // it, it keeps accumulating, now once per group
+    if (fold !== null)
+      sc = { name: fold.name, slot: fold.slot, card: CARD_MANY, parent: sc };
     for (let i = 0; i < tupleSlots.length; i++)
       sc = { name: tupleSlots[i].name, slot: tupleSlots[i].slot, card: CARD_MANY, parent: sc };
     for (let i = 0; i < keys.length; i++)
@@ -888,19 +1120,25 @@ function normalizeFlworPhrase(obj, docPath, scope, ctx) {
   // phrase cardinality: MANY unless provably otherwise - a $let-only
   // phrase yields exactly one tuple ($where may still drop it)
   let card;
-  if (forBindings.length !== 0 || groupby !== null)
+  if (fold !== null)
+    // the phrase IS the accumulator: either it never updated (the initial
+    // value) or it holds some $return result
+    card = joinCard(fold.expr.card, ret.card);
+  else if (forBindings.length !== 0 || groupby !== null)
     card = CARD_MANY;
   else
     card = where !== null ? joinCard(ret.card, CARD_ZERO) : ret.card;
 
   return Object.freeze({
     kind: 'flwor', card, docPath,
+    fold: fold === null ? null : Object.freeze(fold),
     forBindings: Object.freeze(forBindings),
     letBindings: Object.freeze(letBindings),
     asChecks, where,
     groupby: groupby === null ? null : Object.freeze(groupby),
     orderby: orderby === null ? null : Object.freeze(orderby),
     count, ret,
+    limits: ctx.limits,
   });
 }
 
@@ -918,11 +1156,11 @@ function normalizeQuantifierPhrase(obj, docPath, scope, ctx) {
   let sc = scope;
   for (let i = 0; i < names.length; i++) {
     const name = names[i];
-    const bindPath = clausePath + '/' + escapeToken(name);
+    const bindPath = clausePath + '/' + encodeJSONPointerSegment(name);
     bindPhraseName(name, phraseNames, bindPath);
     const source = bindObj[name];
-    if (isPlainObject(source) && (hasOwn(source, '$in') || hasOwn(source, '$at')))
-      return fail('JQ0003', "the extended '$in'/'$at' binding form is not available in quantifiers", bindPath);
+    if (isExtendedBinding(source))
+      return fail('JQ0003', 'the extended binding form is not available in quantifiers', bindPath);
     const expr = normalizeExpr(source, bindPath, sc, ctx);
     const slot = ctx.nextSlot++;
     sc = { name, slot, card: CARD_ONE, parent: sc };
@@ -945,22 +1183,160 @@ function normalizeQuantifierPhrase(obj, docPath, scope, ctx) {
 // vocabulary - the closed format is unchanged, extensions are host
 // machinery. Violations are host programming errors (TypeError), not
 // JQ0xxx document errors.
+// True when `name` is already part of the closed query vocabulary — a
+// FLWOR/quantifier/escape key, a reserved member, or a core operator.
+// A host extension (options.extensions) or a registry pack (the JSLT
+// operator registry) must not shadow any of these. Exported so the
+// registry builder can reject a colliding pack at `.use()` time with the
+// same rule the compiler enforces at normalize time.
+export function isReservedQueryName(name) {
+  return FLWOR_KEYS.has(name) || QUANTIFIER_KEYS.has(name) || ESCAPE_KEYS.has(name)
+    || ORDERBY_SPEC_KEYS.has(name) || name === '$in' || name === '$at'
+    || name === '$query' || name === '$expr' || hasOwn(OPERATORS, name);
+}
+
 function validateExtensions(extensions) {
-  if (!isPlainObject(extensions))
+  if (!isJsonObject(extensions))
     throw new TypeError('options.extensions must be a plain object of operator entries');
   const names = Object.keys(extensions);
   for (let i = 0; i < names.length; i++) {
     const name = names[i];
     if (name.charCodeAt(0) !== 0x24) // '$'
       throw new TypeError(`extension operator '${name}' must start with '$'`);
-    if (FLWOR_KEYS.has(name) || QUANTIFIER_KEYS.has(name) || ESCAPE_KEYS.has(name)
-      || ORDERBY_SPEC_KEYS.has(name) || name === '$in' || name === '$at'
-      || name === '$query' || name === '$expr' || hasOwn(OPERATORS, name))
+    if (isReservedQueryName(name))
       throw new TypeError(`extension operator '${name}' collides with the core vocabulary`);
   }
   return extensions;
 }
 
+// Validate a named registry of trusted pure host functions
+// (options.functions / options.collations): a plain object of
+// `name -> function`. Violations are host programming errors
+// (TypeError), like options.extensions.
+function validateNamedFunctions(value, what) {
+  if (!isJsonObject(value))
+    throw new TypeError(`options.${what} must be a plain object of named functions`);
+  for (const name in value) {
+    // the schema twins require non-empty names (minLength 1); the
+    // registry enforces the same, so document and runtime agree
+    if (name === '')
+      throw new TypeError(`options.${what} must not register an empty-string name`);
+    if (typeof value[name] !== 'function')
+      throw new TypeError(`options.${what}['${name}'] must be a function`);
+  }
+  return value;
+}
+
+// The known limits (section 8.12). Only limits the engine actually
+// enforces are accepted - an accepted-but-unenforced limit would be a
+// silent false guarantee.
+const LIMIT_NAMES = new Set(['sequenceItems', 'resultItems', 'steps', 'depth']);
+
+function validateLimits(value) {
+  if (!isJsonObject(value))
+    throw new TypeError('options.limits must be a plain object');
+  for (const name in value) {
+    if (!LIMIT_NAMES.has(name))
+      throw new TypeError(`options.limits.${name} is not a known limit`);
+    const v = value[name];
+    if (!Number.isInteger(v) || v < 1)
+      throw new TypeError(`options.limits.${name} must be a positive integer`);
+  }
+  return Object.freeze({
+    sequenceItems: value.sequenceItems ?? null,
+    resultItems: value.resultItems ?? null,
+    steps: value.steps ?? null,
+    depth: value.depth ?? null,
+  });
+}
+
+// Validate `options.externals`: the closed-world declaration list. An
+// array of variable names (without the '$' sigil); `[]` declares none,
+// so every free variable is JQ0005. Absent means the open world, where
+// use is the declaration (section 9).
+function validateDeclaredExternals(value) {
+  if (!Array.isArray(value))
+    throw new TypeError('options.externals must be an array of variable names');
+  const set = new Set();
+  for (let i = 0; i < value.length; i++) {
+    const name = value[i];
+    if (typeof name !== 'string' || !VAR_NAME_RE.test(name))
+      throw new TypeError(`options.externals[${i}] must be a variable name string (no '$' sigil)`);
+    set.add(name);
+  }
+  return set;
+}
+
+// Maximum expression nesting of a normalized AST. The language has no
+// recursion - no user-defined functions, no self-reference - so the
+// compiled closure tree's evaluation depth is exactly this static depth,
+// which is why `limits.depth` is a compile-time check (JQ0011) rather
+// than a runtime counter: exact, and free at evaluation time. Returns
+// the deepest node found alongside its depth, for the error position.
+function measureDepth(node, depth, worst) {
+  if (depth > worst.depth) {
+    worst.depth = depth;
+    worst.docPath = node.docPath;
+  }
+  const next = depth + 1;
+  switch (node.kind) {
+    case 'literal':
+    case 'var':
+    case 'path':
+    case 'raw':
+      return;
+    case 'object':
+      for (let i = 0; i < node.entries.length; i++)
+        measureDepth(node.entries[i].expr, next, worst);
+      return;
+    case 'map':
+      for (let i = 0; i < node.pairs.length; i++) {
+        measureDepth(node.pairs[i].key, next, worst);
+        measureDepth(node.pairs[i].value, next, worst);
+      }
+      return;
+    case 'array':
+      for (let i = 0; i < node.elements.length; i++)
+        measureDepth(node.elements[i], next, worst);
+      return;
+    case 'op':
+    case 'call':
+      for (let i = 0; i < node.args.length; i++)
+        measureDepth(node.args[i], next, worst);
+      return;
+    case 'let':
+      for (let i = 0; i < node.bindings.length; i++)
+        measureDepth(node.bindings[i].expr, next, worst);
+      measureDepth(node.ret, next, worst);
+      return;
+    case 'quant':
+      for (let i = 0; i < node.bindings.length; i++)
+        measureDepth(node.bindings[i].expr, next, worst);
+      measureDepth(node.satisfies, next, worst);
+      return;
+    default: { // 'flwor'
+      for (let i = 0; i < node.forBindings.length; i++)
+        measureDepth(node.forBindings[i].expr, next, worst);
+      for (let i = 0; i < node.letBindings.length; i++)
+        measureDepth(node.letBindings[i].expr, next, worst);
+      if (node.fold !== null)
+        measureDepth(node.fold.expr, next, worst);
+      if (node.where !== null)
+        measureDepth(node.where, next, worst);
+      if (node.groupby !== null) {
+        for (let i = 0; i < node.groupby.keys.length; i++)
+          measureDepth(node.groupby.keys[i].expr, next, worst);
+      }
+      if (node.orderby !== null) {
+        for (let i = 0; i < node.orderby.specs.length; i++)
+          measureDepth(node.orderby.specs[i].key, next, worst);
+      }
+      measureDepth(node.ret, next, worst);
+      return;
+    }
+  }
+}
+
 function normalizeExpr(value, docPath, scope, ctx) {
   switch (typeof value) {
     case 'string':
@@ -1014,11 +1390,31 @@ export function normalizeQuery(doc, options = {}) {
     ? options.compileTypeTest
     : null;
   const extensions = options.extensions == null ? null : validateExtensions(options.extensions);
-  const ctx = { nextSlot: 1, externals: new Map(), compileTypeTest, extensions };
+  const functions = options.functions == null ? null : validateNamedFunctions(options.functions, 'functions');
+  const collations = options.collations == null ? null : validateNamedFunctions(options.collations, 'collations');
+  const limits = options.limits == null ? null : validateLimits(options.limits);
+  const declaredExternals = options.externals == null
+    ? null
+    : validateDeclaredExternals(options.externals);
+  // JSONPath function extensions are a separate registry from
+  // options.functions ($call's host functions): they extend the RFC 9535
+  // grammar inside path strings, not the query vocabulary. Built once
+  // per compile so parsePathString hands the parser the same object.
+  const pathOptions = options.pathFunctions == null
+    ? undefined
+    : { pathFunctions: options.pathFunctions };
+  const ctx = {
+    nextSlot: 1, externals: new Map(), compileTypeTest, extensions,
+    functions, collations, limits, pathOptions, declaredExternals,
+    // package-internal: set only by analyzeQuery (Appendix C.1); the
+    // compile entry point never passes it
+    analysis: options.analysis === true,
+    usedOps: new Set(), usedFunctions: new Set(), usedCollations: new Set(),
+  };
   let expr = doc;
   let rootPath = '';
   // the version envelope is only recognized at the top level (section 4)
-  if (isPlainObject(doc) && (hasOwn(doc, '$query') || hasOwn(doc, '$expr'))) {
+  if (isJsonObject(doc) && (hasOwn(doc, '$query') || hasOwn(doc, '$expr'))) {
     const keys = Object.keys(doc);
     let allDollar = true;
     for (let i = 0; i < keys.length; i++) {
@@ -1035,11 +1431,26 @@ export function normalizeQuery(doc, options = {}) {
     }
   }
   const root = normalizeExpr(expr, rootPath, null, ctx);
+  if (limits !== null && limits.depth !== null) {
+    const worst = { depth: 0, docPath: rootPath };
+    measureDepth(root, 1, worst);
+    if (worst.depth > limits.depth)
+      fail('JQ0011', `the query nests ${worst.depth} expressions deep, more than limits.depth (${limits.depth})`, worst.docPath);
+  }
+  // limits.steps instruments every node evaluation; the counter lives in
+  // its own frame slot, so nothing is threaded through the closures
+  const stepSlot = limits !== null && limits.steps !== null ? ctx.nextSlot++ : -1;
   const externals = new Array(ctx.externals.size);
   let i = 0;
   for (const [name, slot] of ctx.externals)
     externals[i++] = Object.freeze({ name, slot });
-  return { root, frameSize: ctx.nextSlot, externals: Object.freeze(externals) };
+  return {
+    root, frameSize: ctx.nextSlot, externals: Object.freeze(externals),
+    limits, stepSlot,
+    usedOps: ctx.usedOps,
+    usedFunctions: ctx.usedFunctions,
+    usedCollations: ctx.usedCollations,
+  };
 }
 
 //#endregion