@marktiderman/genesis-forms 0.4.0 → 0.4.2

package/dist/testing.js

@@ -32,6 +32,7 @@ function tokenize(source) {
         i++;
         while (isDigit(source[i] ?? "")) i++;
       }
+      if (isIdStart(source[i] ?? "")) throw new FormulaError("Write a number in digits only, like 1000000.", at);
       const v = Number(source.slice(at, i));
       if (!Number.isFinite(v)) throw new FormulaError("This number is too large.", at);
       tokens.push({ t: "num", v, at });
@@ -230,6 +231,27 @@ function numbersIn(values, out = []) {
   }
   return out;
 }
+var listTotals = /* @__PURE__ */ new WeakMap();
+var ownLists = /* @__PURE__ */ new WeakSet();
+function ownFormulaList(list) {
+  Object.freeze(list);
+  ownLists.add(list);
+  return list;
+}
+function totalsOf(list) {
+  const own2 = ownLists.has(list);
+  let t = own2 ? listTotals.get(list) : void 0;
+  if (t) return t;
+  const nums = numbersIn(list);
+  t = { sum: 0, count: nums.length, min: Infinity, max: -Infinity };
+  for (const n of nums) {
+    t.sum += n;
+    if (n < t.min) t.min = n;
+    if (n > t.max) t.max = n;
+  }
+  if (own2) listTotals.set(list, t);
+  return t;
+}
 function compare(op, a, b) {
   const blank2 = (v) => v === null || Array.isArray(v) || typeof v === "string" && v.trim() === "";
   if (blank2(a) || blank2(b)) return op === "!=";
@@ -254,7 +276,7 @@ function evaluateFormula(ast, read) {
       case "text":
         return node.value;
       case "ref":
-        return read(node.column !== void 0 ? { key: node.key, column: node.column } : { key: node.key });
+        return read(node);
       case "unary": {
         const n = formulaNumber(scalar(run(node.arg)));
         return n === null ? null : -n;
@@ -262,12 +284,31 @@ function evaluateFormula(ast, read) {
       case "if":
         return truthy(run(node.test)) ? run(node.then) : run(node.else);
       case "call": {
-        const nums = numbersIn(node.args.map(run));
-        const total = nums.reduce((a, b) => a + b, 0);
+        let total = 0;
+        let count = 0;
+        let min = Infinity;
+        let max = -Infinity;
+        for (const arg of node.args) {
+          const v = run(arg);
+          if (Array.isArray(v)) {
+            const t = totalsOf(v);
+            total += t.sum;
+            count += t.count;
+            if (t.min < min) min = t.min;
+            if (t.max > max) max = t.max;
+            continue;
+          }
+          const n = formulaNumber(v);
+          if (n === null) continue;
+          total += n;
+          count++;
+          if (n < min) min = n;
+          if (n > max) max = n;
+        }
         if (node.fn === "sum") return finite(total);
-        if (!nums.length) return null;
-        if (node.fn === "avg") return finite(total / nums.length);
-        return nums.reduce((a, b) => node.fn === "min" ? Math.min(a, b) : Math.max(a, b));
+        if (!count) return null;
+        if (node.fn === "avg") return finite(total / count);
+        return node.fn === "min" ? min : max;
       }
       case "binary": {
         const a = run(node.left);
@@ -294,6 +335,7 @@ function evaluateFormula(ast, read) {
 
 // src/calculated.ts
 var CALCULATED_TYPE = "calculated";
+var MAX_FORM_FORMULA_LENGTH = 2e4;
 var MAX_CALCULATED_DECIMALS = 10;
 function calculatedProps(field) {
   const p = field.props ?? {};
@@ -310,8 +352,7 @@ var round = (n, decimals) => {
   const r = Number(n.toFixed(decimals));
   return Object.is(r, -0) ? 0 : r;
 };
-function calculatedValue(result, field) {
-  const { format, decimals } = calculatedProps(field);
+function valueAs(result, { format, decimals }) {
   if (result === null || Array.isArray(result)) return void 0;
   if (format === "number" || format === "percent") {
     const n = formulaNumber(result);
@@ -332,24 +373,48 @@ function formatCalculated(value, field) {
   return typeof value === "string" ? value : "";
 }
 var MAX_FORMULA_ITEMS = 1e4;
+var MAX_SETTLE_ITEMS = 5e4;
+var extract = (value, column) => column !== void 0 ? value.flatMap((row) => {
+  if (!row || typeof row !== "object" || Array.isArray(row)) return [];
+  const cell = row[column];
+  return typeof cell === "string" || typeof cell === "number" ? [cell] : [];
+}) : value.flatMap((v) => typeof v === "string" || typeof v === "number" || typeof v === "boolean" ? [v] : []);
 function formulaInput(value, column) {
   if (value === void 0 || value === null) return null;
   if (Array.isArray(value) && value.length > MAX_FORMULA_ITEMS) return null;
-  if (column !== void 0) {
-    if (!Array.isArray(value)) return null;
-    return value.flatMap((row) => {
-      if (!row || typeof row !== "object" || Array.isArray(row)) return [];
-      const cell = row[column];
-      return typeof cell === "string" || typeof cell === "number" ? [cell] : [];
-    });
-  }
-  if (Array.isArray(value)) {
-    return value.flatMap((v) => typeof v === "string" || typeof v === "number" || typeof v === "boolean" ? [v] : []);
-  }
+  if (Array.isArray(value)) return extract(value, void 0);
   return typeof value === "object" ? null : value;
 }
+function refNodes(node, out = []) {
+  if (node.kind === "ref") out.push(node);
+  else if (node.kind === "unary") refNodes(node.arg, out);
+  else if (node.kind === "binary") refNodes(node.left, out), refNodes(node.right, out);
+  else if (node.kind === "if") refNodes(node.test, out), refNodes(node.then, out), refNodes(node.else, out);
+  else if (node.kind === "call") for (const a of node.args) refNodes(a, out);
+  return out;
+}
+function formulaSize(node) {
+  if (node.kind === "unary") return 1 + formulaSize(node.arg);
+  if (node.kind === "binary") return 1 + formulaSize(node.left) + formulaSize(node.right);
+  if (node.kind === "if") return 1 + formulaSize(node.test) + formulaSize(node.then) + formulaSize(node.else);
+  if (node.kind === "call") return node.args.reduce((n, a) => n + formulaSize(a), 1);
+  return 1;
+}
+var ReplayWorkExceeded = class extends Error {
+  constructor(limit) {
+    super(`Replaying the response would take more than ${limit} steps of work.`);
+    this.limit = limit;
+  }
+  limit;
+};
+function spend(budget, amount) {
+  if (!budget) return;
+  budget.used += amount;
+  if (budget.used > budget.max) throw new ReplayWorkExceeded(budget.max);
+}
 var UNREAD = /* @__PURE__ */ Symbol("unread");
-function calculator(definition) {
+var OVER = /* @__PURE__ */ Symbol("over");
+function calculator(definition, work) {
   const compiled = [];
   const idByKey = /* @__PURE__ */ new Map();
   for (const block of definition.blocks) {
@@ -358,35 +423,106 @@ function calculator(definition) {
   }
   for (const block of definition.blocks) {
     if (block.kind !== "field" || !isCalculated(block)) continue;
-    const parsed = parseFormula(calculatedProps(block).formula);
-    const reads = parsed.ok ? parsed.refs.map((r) => idByKey.get(r.key)) : [];
-    compiled.push({ field: block, ast: parsed.ok ? parsed.ast : null, reads, last: null });
+    const props = calculatedProps(block);
+    const parsed = parseFormula(props.formula);
+    const reads = [];
+    const slots = /* @__PURE__ */ new Map();
+    const byKey = /* @__PURE__ */ new Map();
+    for (const ref of parsed.ok ? refNodes(parsed.ast) : []) {
+      const id = idByKey.get(ref.key);
+      const key = `${id ?? ""}\0${ref.column ?? ""}`;
+      let slot = byKey.get(key);
+      if (slot === void 0) {
+        slot = reads.push({ id, column: ref.column, key }) - 1;
+        byKey.set(key, slot);
+      }
+      slots.set(ref, slot);
+    }
+    compiled.push({ field: block, props, ast: parsed.ok ? parsed.ast : null, size: parsed.ok ? formulaSize(parsed.ast) : 0, reads, slots, last: null });
   }
+  const extracted = /* @__PURE__ */ new WeakMap();
+  const cachedList = (value, column) => {
+    let byColumn = extracted.get(value);
+    if (!byColumn) extracted.set(value, byColumn = /* @__PURE__ */ new Map());
+    const key = column ?? "";
+    let list = byColumn.get(key);
+    if (!list) {
+      byColumn.set(key, list = ownFormulaList(extract(value, column)));
+    }
+    return list;
+  };
+  const readAnswer = (value, column) => {
+    if (Array.isArray(value)) return cachedList(value, column);
+    return column !== void 0 ? null : formulaInput(value);
+  };
+  const forget = (value) => {
+    if (!value || typeof value !== "object") return;
+    if (Array.isArray(value)) extracted.delete(value);
+    for (const entry of compiled) if (entry.last?.inputs.includes(value)) entry.last = null;
+  };
   const withCalculated = (values, path) => {
     if (!compiled.length) return values;
     const visible = new Set(path);
     let next = values;
-    const input = (id) => id === void 0 || !visible.has(id) || !Object.prototype.hasOwnProperty.call(next, id) ? UNREAD : next[id];
+    let copied = false;
+    let budget = MAX_SETTLE_ITEMS;
+    const within = /* @__PURE__ */ new Map();
+    const input = ({ id }) => {
+      if (id === void 0 || !visible.has(id) || !Object.prototype.hasOwnProperty.call(next, id)) return UNREAD;
+      const value = next[id];
+      if (!Array.isArray(value)) return value;
+      let ok = within.get(id);
+      if (ok === void 0) {
+        ok = value.length <= MAX_FORMULA_ITEMS && value.length <= budget;
+        if (ok) budget -= value.length;
+        within.set(id, ok);
+      }
+      return ok ? value : OVER;
+    };
     for (const entry of compiled) {
       const { field, ast } = entry;
-      const inputs = entry.reads.map(input);
+      const { reads } = entry;
+      spend(work, 1);
+      const inputs = new Array(reads.length);
+      let over = false;
+      let same2 = entry.last !== null;
+      for (let i = 0; i < reads.length; i++) {
+        const v = input(reads[i]);
+        inputs[i] = v;
+        if (v === OVER) over = true;
+        if (same2 && entry.last.inputs[i] !== v) same2 = false;
+      }
       let value;
-      if (entry.last && entry.last.inputs.length === inputs.length && entry.last.inputs.every((v, i) => v === inputs[i])) {
+      if (same2) {
         value = entry.last.value;
       } else {
+        spend(work, entry.size);
+        const resolved = new Array(inputs.length);
         const read = (ref) => {
-          const raw = input(idByKey.get(ref.key));
-          return raw === UNREAD ? null : formulaInput(raw, ref.column);
+          const slot = entry.slots.get(ref);
+          if (slot === void 0) return null;
+          let v = resolved[slot];
+          if (v === void 0) {
+            const raw = inputs[slot];
+            v = resolved[slot] = raw === UNREAD ? null : readAnswer(raw, reads[slot].column);
+          }
+          return v;
         };
-        value = ast ? calculatedValue(evaluateFormula(ast, read), field) : void 0;
+        value = ast && !over ? valueAs(evaluateFormula(ast, read), entry.props) : void 0;
         entry.last = { inputs, value };
       }
       const current = Object.prototype.hasOwnProperty.call(next, field.id) ? next[field.id] : void 0;
-      if (current !== value) next = { ...next, [field.id]: value };
+      if (current !== value) {
+        if (!copied) {
+          next = { ...next };
+          copied = true;
+        }
+        next[field.id] = value;
+      }
     }
     return next;
   };
-  return { active: compiled.length > 0, withCalculated };
+  return { active: compiled.length > 0, ids: new Set(compiled.map((c) => c.field.id)), withCalculated, forget };
 }
 var numberOps = ["eq", "neq", "lt", "lte", "gt", "gte", "in", "answered", "empty"];
 var calculatedFieldType = {
@@ -1043,8 +1179,8 @@ function rulesAllow(def, blockId, scope) {
 }
 function walkPath(def, scope) {
   const list = steps(def);
-  const index = new Map(list.map((b, i2) => [b.id, i2]));
-  const position = new Map(def.blocks.map((b, i2) => [b.id, i2]));
+  let index;
+  let position;
   const jumps = (def.logic ?? []).filter(
     (r) => r.effect === "jump"
   );
@@ -1068,6 +1204,7 @@ function walkPath(def, scope) {
         ending = jump.to.ending;
         break;
       }
+      index ??= new Map(list.map((b, at) => [b.id, at]));
       const target = index.get(jump.to);
       if (target !== void 0 && target > i) {
         i = nextVisible(target);
@@ -1077,6 +1214,7 @@ function walkPath(def, scope) {
     i = nextVisible(i + 1);
   }
   const scopeAt = (blockId, inclusive = false) => {
+    position ??= new Map(def.blocks.map((b, i2) => [b.id, i2]));
     const at = position.get(blockId) ?? -1;
     const values = /* @__PURE__ */ Object.create(null);
     for (const id of path) {
@@ -1139,10 +1277,19 @@ function validateField(field, value, options = {}) {
 }
 
 // src/engine.ts
+function readsAny(node, ids) {
+  if (Array.isArray(node)) return node.some((n) => readsAny(n, ids));
+  if (!node || typeof node !== "object") return false;
+  for (const [k, v] of Object.entries(node)) {
+    if (k === "field" && typeof v === "string" && ids.has(v)) return true;
+    if (typeof v === "object" && readsAny(v, ids)) return true;
+  }
+  return false;
+}
 function createForm(definition, options = {}) {
   return build(definition, options).engine;
 }
-function build(definition, options, replaying = false) {
+function build(definition, options, replaying = false, budget) {
   const fields = fieldMap(definition);
   const byKey = new Map([...fields.values()].map((f) => [f.key, f]));
   const blocks = new Map(definition.blocks.map((b) => [b.id, b]));
@@ -1160,15 +1307,24 @@ function build(definition, options, replaying = false) {
     if (!declared.has(name)) initialHidden[name] = value;
   }
   Object.assign(initialHidden, coerceHidden(definition, options.hidden).values);
-  const calc = calculator(definition);
+  const calc = calculator(definition, budget);
+  const pathOf = (values, hidden) => {
+    spend(budget, definition.blocks.length);
+    return computePath(definition, scopeOf(values, hidden));
+  };
+  const logicReadsCalculated = calc.active && readsAny(definition.logic, calc.ids);
   const settle = (values, hidden) => {
-    let { path } = computePath(definition, scopeOf(values, hidden));
+    let { path } = pathOf(values, hidden);
     if (!calc.active) return { values, path };
+    if (!logicReadsCalculated) return { values: calc.withCalculated(values, path), path };
     for (let round2 = 0; round2 < 4; round2++) {
       const next = calc.withCalculated(values, path);
       if (next === values) break;
       values = next;
-      path = computePath(definition, scopeOf(values, hidden)).path;
+      const again = pathOf(values, hidden).path;
+      const same2 = again.length === path.length && again.every((id, i) => id === path[i]);
+      path = again;
+      if (same2) break;
     }
     return { values, path };
   };
@@ -1210,14 +1366,30 @@ function build(definition, options, replaying = false) {
     return errors;
   };
   let unsettled = false;
-  const settleNow = () => {
-    if (!unsettled) return;
-    unsettled = false;
-    const { values, path } = settle(state.values, state.hidden);
-    state = { ...state, values, path };
+  let formulasStale = false;
+  const settleNow = (needValues = true) => {
+    if (unsettled && calc.active && !logicReadsCalculated && !needValues) {
+      unsettled = false;
+      formulasStale = true;
+      state = { ...state, path: pathOf(state.values, state.hidden).path };
+      return;
+    }
+    if (unsettled) {
+      unsettled = false;
+      formulasStale = false;
+      const { values, path } = settle(state.values, state.hidden);
+      state = { ...state, values, path };
+    } else if (formulasStale && needValues) {
+      formulasStale = false;
+      state = { ...state, values: calc.withCalculated(state.values, state.path) };
+    }
   };
   const answer = (id, value, event) => {
-    if (state.status === "completed" || own(state.values, id) === value || isCalculated(fields.get(id))) return;
+    if (state.status === "completed" || isCalculated(fields.get(id))) return;
+    if (own(state.values, id) === value) {
+      if (value === null || typeof value !== "object") return;
+      calc.forget(value);
+    }
     const errors = { ...state.errors };
     delete errors[id];
     if (batch) {
@@ -1283,7 +1455,7 @@ function build(definition, options, replaying = false) {
         emit({ ...state, errors, current: firstError, events });
         return false;
       }
-      const { ending } = computePath(definition, scopeOf(state.values, state.hidden));
+      const { ending } = pathOf(state.values, state.hidden);
       emit({ ...state, errors: {}, status: "completed", ending, events });
       return true;
     },
@@ -1315,10 +1487,11 @@ function build(definition, options, replaying = false) {
         answer(event.field, event.value, logged);
       }
     } else if (event.type === "next") {
-      settleNow();
+      settleNow(false);
+      if (formulasStale && isCalculated(fields.get(state.current ?? ""))) settleNow(true);
       engine.next();
     } else if (event.type === "back") {
-      settleNow();
+      settleNow(false);
       engine.back();
     } else if (event.type === "submit") {
       settleNow();
@@ -1333,9 +1506,18 @@ function build(definition, options, replaying = false) {
   return { engine, apply, finish };
 }
 function replay(definition, events, options = {}) {
-  const { engine, apply, finish } = build(definition, { ...options, initialValues: void 0, clock: () => void 0 }, true);
-  for (const event of events) apply(event);
-  finish();
+  return replayWithin(definition, events, options);
+}
+function replayWithin(definition, events, options, maxWork) {
+  const budget = maxWork === void 0 ? void 0 : { used: 0, max: maxWork };
+  const { engine, apply, finish } = build(definition, { ...options, initialValues: void 0, clock: () => void 0 }, true, budget);
+  try {
+    for (const event of events) apply(event);
+    finish();
+  } finally {
+    if (budget) budget.used;
+  }
+  if (budget) budget.max = Infinity;
   return engine;
 }
 
@@ -1780,9 +1962,15 @@ function parseForm(input) {
       issue(`${path}.kind`, "Unknown block kind.");
     }
   });
+  let formulaChars = 0;
   input.blocks.forEach((block, i) => {
     if (!isObject2(block) || block.kind !== "field" || block.type !== CALCULATED_TYPE) return;
     const path = `blocks[${i}].props.formula`;
+    const source = isObject2(block.props) && typeof block.props.formula === "string" ? block.props.formula : "";
+    formulaChars += source.length;
+    if (formulaChars > MAX_FORM_FORMULA_LENGTH && formulaChars - source.length <= MAX_FORM_FORMULA_LENGTH) {
+      issue(path, `The form's formulas together are over ${MAX_FORM_FORMULA_LENGTH} characters; shorten some, or work part out once in an earlier calculated field.`);
+    }
     if (block.required === true) {
       warnings.push({ path: `blocks[${i}].required`, message: "A calculated field is worked out, not answered, so it is never required." });
     }
@@ -2012,6 +2200,41 @@ function parseForm(input) {
 
 // src/server.ts
 var MAX_EVENTS = 1e4;
+var MAX_SUBMISSION_VALUES = 2e5;
+var MAX_REPLAY_WORK = 1e6;
+var RAW_EVENTS_PER_EVENT = 10;
+function countValues(value, limit) {
+  let count = 0;
+  const stack = [value];
+  while (stack.length && count <= limit) {
+    const v = stack.pop();
+    if (v === void 0) continue;
+    count++;
+    if (Array.isArray(v)) for (const item of v) stack.push(item);
+    else if (v !== null && typeof v === "object") for (const k in v) stack.push(v[k]);
+  }
+  return count;
+}
+var answerTo = (e) => isPlainObject(e) && e.type === "answer" && typeof e.field === "string" ? e.field : void 0;
+function leanEvents(events) {
+  const out = [];
+  for (let i = 0; i < events.length; i++) {
+    const field = answerTo(events[i]);
+    if (field !== void 0 && answerTo(events[i + 1]) === field) continue;
+    out.push(events[i]);
+  }
+  return out;
+}
+function submissionValues(input, limit) {
+  let count = countValues(input.hidden, limit);
+  if (Array.isArray(input.events)) {
+    for (const e of input.events) {
+      if (count > limit) break;
+      if (answerTo(e) !== void 0) count += countValues(e.value, limit - count);
+    }
+  } else count += countValues(input.answers, limit - count);
+  return count;
+}
 var MALFORMED = { ok: false, errors: { "": "Malformed submission." } };
 var parsedDefinitions = /* @__PURE__ */ new WeakMap();
 function checkDefinition(definition) {
@@ -2041,7 +2264,24 @@ function wellFormed(input) {
 function validateSubmission(definition, submission, options = {}) {
   const invalid = checkDefinition(definition);
   if (invalid) return { ok: false, errors: { "": invalid } };
-  const input = wellFormed(submission);
+  if (!isPlainObject(submission)) return MALFORMED;
+  const maxEvents = options.maxEvents ?? MAX_EVENTS;
+  let sent = submission;
+  if (Array.isArray(submission.events)) {
+    const maxRaw = maxEvents * RAW_EVENTS_PER_EVENT;
+    if (submission.events.length > maxRaw) {
+      return { ok: false, errors: { "": `Too many events: ${submission.events.length} sent; the limit is ${maxRaw} before compaction.` } };
+    }
+    sent = { ...submission, events: leanEvents(submission.events) };
+  }
+  const maxValues = options.maxValues ?? MAX_SUBMISSION_VALUES;
+  if (submissionValues(sent, maxValues) > maxValues) {
+    return {
+      ok: false,
+      errors: { "": `Too much data: the answers hold more than ${maxValues} values (items, rows and cells); the limit is ${maxValues}.` }
+    };
+  }
+  const input = wellFormed(sent);
   if (!input) return MALFORMED;
   const ignored = /* @__PURE__ */ new Set();
   const client = coerceHidden(definition, input.hidden, { sources: ["url", "host"] });
@@ -2060,12 +2300,20 @@ function validateSubmission(definition, submission, options = {}) {
   let engine;
   if (input.events) {
     const events = compactEvents(input.events);
-    const max = options.maxEvents ?? MAX_EVENTS;
-    if (events.length > max) {
-      return { ok: false, errors: { "": `Too many events: ${events.length} after compaction; the limit is ${max}.` } };
+    if (events.length > maxEvents) {
+      return { ok: false, errors: { "": `Too many events: ${events.length} after compaction; the limit is ${maxEvents}.` } };
     }
     const kept = events.filter((e) => e.type !== "answer" || accept(e.field) !== false);
-    engine = replay(definition, kept, { hidden: engineHidden, validators: options.validators });
+    const maxWork = options.maxReplayWork ?? MAX_REPLAY_WORK;
+    try {
+      engine = replayWithin(definition, kept, { hidden: engineHidden, validators: options.validators }, maxWork);
+    } catch (e) {
+      if (!(e instanceof ReplayWorkExceeded)) throw e;
+      return {
+        ok: false,
+        errors: { "": `Too much work: replaying these events (the path and calculated fields) would take more than ${maxWork} steps; the limit is ${maxWork}.` }
+      };
+    }
   } else {
     const answers2 = {};
     for (const id of Object.keys(input.answers ?? {})) {