@suss/datalog 0.31.0 → 0.31.2

package/dist/index.js

@@ -27,6 +27,7 @@ function deriveOnDemand(rules, complete) {
     }
   }
   const derives = (relation) => rulesByHead.has(relation);
+  const labels = labelColumns(rules, derives);
   const asked = /* @__PURE__ */ new Map();
   const pending = [];
   const want = (adorned) => {
@@ -48,7 +49,7 @@ function deriveOnDemand(rules, complete) {
   while (pending.length > 0) {
     const adorned = pending.pop();
     for (const written of rulesByHead.get(adorned.relation) ?? []) {
-      const r = boundFirst(written, adorned.adornment);
+      const r = boundFirst(written, adorned.adornment, labels);
       const adornments = bodyAdornments(r, adorned.adornment, derives);
       r.body.forEach((literal, at) => {
         const adornment = adornments[at];
@@ -91,7 +92,7 @@ function deriveOnDemand(rules, complete) {
       });
     }
     for (const written of rulesByHead.get(adorned.relation) ?? []) {
-      const r = boundFirst(written, adorned.adornment);
+      const r = boundFirst(written, adorned.adornment, labels);
       const guard = anyBound(adorned.adornment) ? [
         positiveLiteral(
           demandOn(adorned.relation, adorned.adornment),
@@ -132,7 +133,30 @@ function deriveOnDemand(rules, complete) {
   return { rules: rewritten, demandDriven: [...demandDriven], demands };
 }
 var sameLiteral = (premises, conclusion) => premises.length === 1 && JSON.stringify(premises[0]) === JSON.stringify(conclusion);
-function boundFirst(r, headAdornment) {
+function labelColumns(rules, derives) {
+  const labels = /* @__PURE__ */ new Map();
+  const note = ({
+    relation,
+    terms
+  }) => {
+    if (!derives(relation)) {
+      return;
+    }
+    terms.forEach((term, column) => {
+      if (term.type === "constant") {
+        const columns = labels.get(relation) ?? /* @__PURE__ */ new Set();
+        columns.add(column);
+        labels.set(relation, columns);
+      }
+    });
+  };
+  for (const r of rules) {
+    note(r.head);
+    r.body.forEach(note);
+  }
+  return labels;
+}
+function boundFirst(r, headAdornment, labels) {
   const bound = /* @__PURE__ */ new Set();
   r.head.terms.forEach((term, column) => {
     if (headAdornment[column] && term.type === "variable") {
@@ -140,7 +164,7 @@ function boundFirst(r, headAdornment) {
     }
   });
   const boundColumns = (literal) => literal.terms.filter(
-    (term) => term.type === "constant" || bound.has(term.name)
+    (term, column) => !labels.get(literal.relation)?.has(column) && (term.type === "constant" || bound.has(term.name))
   ).length;
   const remaining = [...r.body];
   const body = [];
@@ -185,21 +209,43 @@ function bodyAdornments(r, headAdornment, derives) {
 var open = [];
 var isProfiling = () => open.length > 0;
 var ruleKey = (head, body) => `${head} <- ${body.join(", ")}`;
-function addCost(into, key, head, body, ms, derived) {
+function addCost(into, key, head, body, ms, derived, examined) {
   const existing = into.get(key);
   if (existing === void 0) {
-    into.set(key, { head, body, ms, derived, attempts: 1 });
+    into.set(key, { head, body, ms, derived, examined, attempts: 1 });
     return;
   }
   existing.ms += ms;
   existing.derived += derived;
+  existing.examined += examined;
   existing.attempts += 1;
 }
-function chargeRule(ruleSet, head, body, ms, derived) {
+function chargeRule(ruleSet, head, body, ms, derived, examined) {
   const key = ruleKey(head, body);
   for (const collector of open) {
-    addCost(collector.rules, key, head, body, ms, derived);
-    addCost(setIn(collector, ruleSet).rules, key, head, body, ms, derived);
+    addCost(collector.rules, key, head, body, ms, derived, examined);
+    addCost(
+      setIn(collector, ruleSet).rules,
+      key,
+      head,
+      body,
+      ms,
+      derived,
+      examined
+    );
+  }
+}
+function chargeAbandoned(question, examined) {
+  for (const collector of open) {
+    collector.abandoned.push({ question, examined });
+  }
+}
+function chargeQuestion(outcome) {
+  for (const collector of open) {
+    collector.questions.asked += 1;
+    if (outcome !== void 0) {
+      collector.questions[outcome] += 1;
+    }
   }
 }
 function chargeEvaluation(ruleSet) {
@@ -208,6 +254,13 @@ function chargeEvaluation(ruleSet) {
     setIn(collector, ruleSet).evaluations += 1;
   }
 }
+function chargeEvaluationRows(examined) {
+  for (const collector of open) {
+    if (examined > collector.largestEvaluation) {
+      collector.largestEvaluation = examined;
+    }
+  }
+}
 function chargeRound(ruleSet) {
   for (const collector of open) {
     collector.rounds += 1;
@@ -267,7 +320,10 @@ function openScope() {
     rules: /* @__PURE__ */ new Map(),
     relations: /* @__PURE__ */ new Map(),
     derivedRelations: /* @__PURE__ */ new Set(),
-    ruleSets: /* @__PURE__ */ new Map()
+    ruleSets: /* @__PURE__ */ new Map(),
+    abandoned: [],
+    largestEvaluation: 0,
+    questions: { asked: 0, abandoned: 0, skipped: 0 }
   };
   open.push(mine);
   return mine;
@@ -285,6 +341,13 @@ var totalMs = (rules) => {
   }
   return ms;
 };
+var totalExamined = (rules) => {
+  let rows = 0;
+  for (const cost of rules) {
+    rows += cost.examined;
+  }
+  return rows;
+};
 var byCost = (a, b) => b.ms - a.ms;
 function summarise(mine) {
   return {
@@ -292,6 +355,10 @@ function summarise(mine) {
     datalogMs: totalMs(mine.rules.values()),
     rounds: mine.rounds,
     evaluations: mine.evaluations,
+    examined: totalExamined(mine.rules.values()),
+    largestEvaluation: mine.largestEvaluation,
+    abandoned: [...mine.abandoned],
+    questions: { ...mine.questions },
     relations: [...mine.relations].map(([relation, tuples]) => ({
       relation,
       tuples,
@@ -312,14 +379,26 @@ function ruleLines(rules, datalogMs) {
   return rules.slice(0, 20).map((r) => {
     const ms = `${r.ms.toFixed(0)}ms`.padStart(8);
     const derived = String(r.derived).padStart(7);
-    return `    ${ms} ${share(r.ms, datalogMs)} ${derived} tuples  ${r.attempts} attempts  ${ruleKey(r.head, r.body)}`;
+    const examined = String(r.examined).padStart(9);
+    return `    ${ms} ${share(r.ms, datalogMs)} ${derived} tuples ${examined} rows read  ${r.attempts} attempts  ${ruleKey(r.head, r.body)}`;
   });
 }
 function formatProfile(profile) {
   const lines = [];
   lines.push(
-    `datalog: ${profile.datalogMs.toFixed(0)}ms (${share(profile.datalogMs, profile.wallMs).trim()} of ${profile.wallMs.toFixed(0)}ms wall), ${profile.evaluations} evaluations, ${profile.rounds} rounds`
+    `datalog: ${profile.datalogMs.toFixed(0)}ms (${share(profile.datalogMs, profile.wallMs).trim()} of ${profile.wallMs.toFixed(0)}ms wall), ${profile.evaluations} evaluations, ${profile.rounds} rounds, ${profile.examined} rows read (${profile.largestEvaluation} in the biggest evaluation)`
   );
+  const { asked, abandoned, skipped } = profile.questions;
+  if (asked > 0) {
+    lines.push(
+      `  ${asked} questions under a site: ${abandoned} given up part way, ${skipped} never put once the run's rows were spent`
+    );
+  }
+  for (const given of profile.abandoned) {
+    lines.push(
+      `  gave up on ${given.question} after reading ${given.examined} rows`
+    );
+  }
   const derivedTuples = profile.relations.filter((r) => r.derived).reduce((sum, r) => sum + r.tuples, 0);
   lines.push("  tuples by relation (d = derived by a rule):");
   for (const { relation, tuples, derived } of profile.relations.slice(0, 20)) {
@@ -733,93 +812,109 @@ function headTuple(head, bindings) {
     return bound;
   });
 }
-var variablesOf = (literal) => literal.terms.filter((term) => term.type === "variable").map((term) => term.name);
-var plans = /* @__PURE__ */ new WeakMap();
-function bodyPlan(r, deltaIndex) {
-  let byDelta = plans.get(r);
-  if (byDelta === void 0) {
-    byDelta = /* @__PURE__ */ new Map();
-    plans.set(r, byDelta);
-  }
-  const known = byDelta.get(deltaIndex);
-  if (known !== void 0) {
-    return known;
-  }
-  const positives = r.body.map((literal, index) => literal.negated ? -1 : index).filter((index) => index !== -1);
-  const deltaAt = positives[deltaIndex] ?? -1;
-  const computed = { deltaAt, order: planBodyOrder(r, deltaAt) };
-  byDelta.set(deltaIndex, computed);
-  return computed;
+var positiveLiterals = /* @__PURE__ */ new WeakMap();
+function deltaLiteral(r, deltaIndex) {
+  let positives = positiveLiterals.get(r);
+  if (positives === void 0) {
+    positives = r.body.map((literal, index) => literal.negated ? -1 : index).filter((index) => index !== -1);
+    positiveLiterals.set(r, positives);
+  }
+  return positives[deltaIndex] ?? -1;
 }
-function planBodyOrder(r, deltaAt) {
-  const natural = r.body.map((_, i) => i);
-  if (deltaAt <= 0) {
-    return natural;
-  }
-  const taken = r.body.map(() => false);
-  const bound = /* @__PURE__ */ new Set();
-  const order = [];
-  const take = (index) => {
-    taken[index] = true;
-    order.push(index);
-    for (const name of variablesOf(r.body[index])) {
-      bound.add(name);
-    }
-  };
-  const firstWhere = (want) => natural.find((index) => !taken[index] && want(index)) ?? -1;
-  const nextLiteral = () => {
-    const ready = firstWhere(
-      (i) => r.body[i].negated && variablesOf(r.body[i]).every((name) => bound.has(name))
-    );
-    if (ready !== -1) {
-      return ready;
-    }
-    const joined = firstWhere(
-      (i) => !r.body[i].negated && variablesOf(r.body[i]).some((name) => bound.has(name))
-    );
-    if (joined !== -1) {
-      return joined;
-    }
-    const disconnected = firstWhere((i) => !r.body[i].negated);
-    if (disconnected !== -1) {
-      return disconnected;
+var allBound = (literal, bindings) => literal.terms.every(
+  (term) => term.type === "constant" || boundValue(bindings, term.name) !== void 0
+);
+var rowBudget = (limit) => ({
+  examined: 0,
+  limit
+});
+var OUT_OF_BUDGET = /* @__PURE__ */ Symbol("datalog row budget exhausted");
+var BudgetExhausted = class extends Error {
+  constructor(ruleSet, examined) {
+    super(`gave up after reading ${examined} rows without settling`);
+    this.ruleSet = ruleSet;
+    this.examined = examined;
+    this.name = "BudgetExhausted";
+  }
+  ruleSet;
+  examined;
+};
+function evaluateRule(db, deltas, r, deltaIndex, budget) {
+  const results = [];
+  const body = r.body;
+  const deltaAt = deltaLiteral(r, deltaIndex);
+  const whole = 2 ** body.length - 1;
+  const walk2 = (index, source, taken, bindings) => {
+    const literal = body[index];
+    for (const tuple of source) {
+      budget.examined++;
+      if (budget.examined > budget.limit) {
+        throw OUT_OF_BUDGET;
+      }
+      const next = unify(literal, tuple, bindings);
+      if (next !== NO_MATCH) {
+        step(taken | 1 << index, next);
+      }
     }
-    return firstWhere(() => true);
   };
-  take(deltaAt);
-  while (order.length < r.body.length) {
-    take(nextLiteral());
-  }
-  return order;
-}
-function evaluateRule(db, deltas, r, deltaIndex) {
-  const results = [];
-  const { deltaAt, order } = bodyPlan(r, deltaIndex);
-  const step = (orderIndex, bindings) => {
-    if (orderIndex === order.length) {
+  const step = (taken, bindings) => {
+    if (taken === whole) {
       results.push(headTuple(r.head, bindings));
       return;
     }
-    const literalIndex = order[orderIndex];
-    const literal = r.body[literalIndex];
-    if (literal.negated) {
-      if (!db.has(literal.relation, groundNegated(literal, bindings))) {
-        step(orderIndex + 1, bindings);
+    let pick = -1;
+    let narrowest = null;
+    for (let index = 0; index < body.length; index++) {
+      if (taken & 1 << index) {
+        continue;
+      }
+      const literal = body[index];
+      if (literal.negated) {
+        if (allBound(literal, bindings)) {
+          if (!db.has(literal.relation, groundNegated(literal, bindings))) {
+            step(taken | 1 << index, bindings);
+          }
+          return;
+        }
+        continue;
+      }
+      const source = narrowedSource(db, literal, bindings);
+      if (source === null) {
+        continue;
+      }
+      if (source.length === 0) {
+        return;
+      }
+      if (narrowest === null || source.length < narrowest.length) {
+        pick = index;
+        narrowest = source;
       }
+    }
+    if (narrowest !== null) {
+      walk2(pick, narrowest, taken, bindings);
       return;
     }
-    const source = literalIndex === deltaAt ? deltas.get(literal.relation) ?? [] : boundSource(db, literal, bindings);
-    for (const tuple of source) {
-      const next = unify(literal, tuple, bindings);
-      if (next !== NO_MATCH) {
-        step(orderIndex + 1, next);
+    for (let index = 0; index < body.length; index++) {
+      if (!(taken & 1 << index) && !body[index].negated) {
+        walk2(index, db.facts(body[index].relation), taken, bindings);
+        return;
+      }
+    }
+    for (let index = 0; index < body.length; index++) {
+      if (!(taken & 1 << index)) {
+        groundNegated(body[index], bindings);
       }
     }
   };
-  step(0, null);
+  if (deltaAt === -1) {
+    step(0, null);
+  } else {
+    walk2(deltaAt, deltas.get(body[deltaAt].relation) ?? [], 0, null);
+  }
   return results;
 }
-function boundSource(db, literal, bindings) {
+var boundSource = (db, literal, bindings) => narrowedSource(db, literal, bindings) ?? db.facts(literal.relation);
+function narrowedSource(db, literal, bindings) {
   let narrowest = null;
   for (let column = 0; column < literal.terms.length; column++) {
     const term = literal.terms[column];
@@ -835,10 +930,10 @@ function boundSource(db, literal, bindings) {
       narrowest = bucket;
     }
   }
-  return narrowest ?? db.facts(literal.relation);
+  return narrowest;
 }
 var EMPTY_TAGS = [];
-function evaluateRuleTagged(db, deltas, r, deltaIndex, algebra) {
+function evaluateRuleTagged(db, deltas, r, deltaIndex, algebra, budget) {
   const results = [];
   const bodyTags = [];
   const bodyMatches = [];
@@ -876,6 +971,7 @@ function evaluateRuleTagged(db, deltas, r, deltaIndex, algebra) {
     }
     const source = positiveIndex === deltaIndex ? deltas.get(literal.relation) ?? [] : boundSource(db, literal, bindings);
     for (const tuple of source) {
+      budget.examined++;
       const next = unify(literal, tuple, bindings);
       if (next !== NO_MATCH) {
         if (readsTags) {
@@ -959,15 +1055,18 @@ function currentMarks(db) {
   }
   return marks;
 }
-function evaluate(db, rules, algebra) {
+function evaluate(db, rules, algebra, budget) {
   if (algebra !== void 0 && isDemandRewritten(rules)) {
     throw new Error(
       "cannot evaluate demand-rewritten rules with a tag algebra: tags would follow the demand-transformed rules rather than the ones the caller wrote"
     );
   }
+  if (algebra !== void 0 && budget !== void 0) {
+    throw new Error("cannot evaluate with both a tag algebra and a row budget");
+  }
   deriving.set(db, (deriving.get(db) ?? 0) + 1);
   try {
-    return runRules(db, rules, algebra);
+    return runRules(db, rules, algebra, budget);
   } finally {
     const depth = (deriving.get(db) ?? 1) - 1;
     if (depth === 0) {
@@ -1017,13 +1116,19 @@ function storedMergeFor(algebra) {
     return stored === void 0 && merged === algebra.asserted ? void 0 : merged;
   };
 }
-function runRules(db, rules, algebra) {
+function runRules(db, rules, algebra, given) {
   const {
     signature,
     name: ruleSetName,
     derivedRelations,
     strata
   } = shapeOf(rules);
+  const budget = given ?? {
+    examined: 0,
+    limit: Number.POSITIVE_INFINITY
+  };
+  const startedAt = budget.examined;
+  const addedHere = given === void 0 ? null : [];
   chargeEvaluation(ruleSetName);
   const states = statesFor(db);
   const state = states.get(signature) ?? {
@@ -1045,10 +1150,11 @@ function runRules(db, rules, algebra) {
       derived.set(relation, ledger);
     }
     ledger.set(keyOf2(tuple), tuple);
+    addedHere?.push([relation, tuple]);
   };
   const mergeStored = algebra === void 0 ? void 0 : storedMergeFor(algebra);
   const marks = canResume(rules, state) ? state.marks : void 0;
-  for (const stratum of strata) {
+  const runStratum = (stratum) => {
     let delta = /* @__PURE__ */ new Map();
     const derivedHere = new Set(stratum.map((r) => r.head.relation));
     const record = (relation, tuple) => {
@@ -1074,13 +1180,14 @@ function runRules(db, rules, algebra) {
           seed,
           r,
           deltaIndex,
-          algebra
+          algebra,
+          budget
         )) {
           recordTagged(r.head.relation, found.tuple, found.tag);
         }
         return;
       }
-      for (const tuple of evaluateRule(db, seed, r, deltaIndex)) {
+      for (const tuple of evaluateRule(db, seed, r, deltaIndex, budget)) {
         record(r.head.relation, tuple);
       }
     };
@@ -1089,15 +1196,17 @@ function runRules(db, rules, algebra) {
         deriveInto(r, seed, deltaIndex);
         return;
       }
-      const startedAt = performance.now();
+      const startedAt2 = performance.now();
       const before = db.size(r.head.relation);
+      const readBefore = budget.examined;
       deriveInto(r, seed, deltaIndex);
       chargeRule(
         ruleSetName,
         r.head.relation,
         r.body.map(literalName),
-        performance.now() - startedAt,
-        db.size(r.head.relation) - before
+        performance.now() - startedAt2,
+        db.size(r.head.relation) - before,
+        budget.examined - readBefore
       );
     };
     const applyDelta = (seed, derivedOnly) => {
@@ -1142,7 +1251,24 @@ function runRules(db, rules, algebra) {
       applyDelta(lastDelta, true);
       chargeRound(ruleSetName);
     }
+  };
+  const readHere = () => budget.examined - startedAt;
+  try {
+    for (const stratum of strata) {
+      runStratum(stratum);
+    }
+  } catch (error) {
+    if (error !== OUT_OF_BUDGET) {
+      throw error;
+    }
+    for (const [relation, tuples] of byRelation(addedHere ?? [])) {
+      db.retract(relation, tuples);
+    }
+    state.marks = null;
+    chargeEvaluationRows(readHere());
+    throw new BudgetExhausted(ruleSetName, readHere());
   }
+  chargeEvaluationRows(readHere());
   state.marks = currentMarks(db);
   if (isProfiling()) {
     chargeRelationSizes(
@@ -1152,6 +1278,13 @@ function runRules(db, rules, algebra) {
   }
   return db;
 }
+function byRelation(added) {
+  const buckets = /* @__PURE__ */ new Map();
+  for (const [relation, tuple] of added) {
+    addTo(buckets, relation, tuple);
+  }
+  return buckets;
+}
 function addTo(buckets, relation, tuple) {
   const bucket = buckets.get(relation);
   if (bucket === void 0) {
@@ -1161,8 +1294,11 @@ function addTo(buckets, relation, tuple) {
   }
 }
 export {
+  BudgetExhausted,
   Database,
   Witness,
+  chargeAbandoned,
+  chargeQuestion,
   clearRelations,
   confidence,
   confidenceWith,
@@ -1175,6 +1311,7 @@ export {
   profileEvaluation,
   profileEvaluationAsync,
   proofOf,
+  rowBudget,
   rule,
   ruleLabel,
   stratify,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@suss/datalog",
-  "version": "0.31.0",
+  "version": "0.31.2",
   "type": "module",
   "description": "A small semi-naive Datalog evaluator with stratified negation, the rules engine suss runs its whole-program analyses on.",
   "keywords": [