@bablr/bablr-vm 0.13.2 → 0.14.0

package/README.md

@@ -1,3 +1,33 @@
-# @bablr/vm
+# @bablr/bablr-vm
 
-The BABLR VM!
+[![come chat on Discord](https://img.shields.io/discord/1151914613089251388)](https://discord.gg/NfMNyYN6cX)
+
+Welcome to the home of the BABLR VM! The VM is at the core of the BABLR language definition ecosystem: its purpose is to provide a powerful, well-specified, universally-available execution environment for parsers.
+
+By doing this BABLR makes it possible to create tools which integrate more deeply: it can create consensus around basic definitions that must be shared between well-integrated tools.
+
+## Usage
+
+Unless you are a power user, you should be using either [the BABLR API](https://github.com/bablr-lang/bablr) or [the BABLR CLI](https://github.com/bablr-lang/bablr-cli). If you are a power user (or otherwise have unique reuquirements) looking at how those packages are implemented will be a good place to start.
+
+## Features
+
+Parsers defined on the BABLR VM
+
+- Execute in a streaming, LR fashion
+  - Hold output as necessary for expression building
+- Do just-in-time tokenization
+- Support backtracking to resolve ambiguity
+- Support arbitrary-size textual lookahead
+- Offer high-level tools for defining trivia and precedence
+- Produce results as a CSTML stream or an agAST tree
+- Are backwards compatible with new VMs
+- Are freely extensible
+- Allow incremental reparsing (for responsivene editing)
+- Can be easily debugged
+- Get a comment attachment engine for free
+- Support parsing templates (programs with holes in them)
+
+Parsers defined on the BABLR VM do not have error recovery, and this is on purpose. In general error recovery is not science, it's guesswork necessary to keep the semantic model of the program from continually blinking into and out of existence as the user types syntax, passing through invalid syntactic states.
+
+BABLR instead aims to make the most obvious solution the most pleasant: help users edit syntax without passing through invalid states

package/lib/index.js

@@ -1,4 +1,3 @@
-export { evaluate } from './evaluate.js';
+export { createBablrStrategy } from './strategy.js';
 
 export { Source } from './source.js';
-export { Context } from './context.js';

package/lib/source.js

@@ -1,5 +1,5 @@
 import { WeakStackFrame } from '@bablr/weak-stack';
-import { maybeWait, getStreamIterator } from '@bablr/agast-helpers/stream';
+import { maybeWait, getStreamIterator, emptyStreamIterator } from '@bablr/agast-helpers/stream';
 import { facades, actuals } from './facades.js';
 
 // Queue item instances are shared between all forks.
@@ -197,7 +197,7 @@ export const Source = class BABLRSource extends WeakStackFrame {
   }
 
   get value() {
-    return this.fork.value;
+    return this.holding ? null : this.fork.value;
   }
 
   get done() {
@@ -247,7 +247,7 @@ export const Source = class BABLRSource extends WeakStackFrame {
   }
 
   [Symbol.for('@@streamIterator')]() {
-    return this.fork.clone()[Symbol.for('@@streamIterator')]();
+    return this.holding ? emptyStreamIterator : this.fork.clone()[Symbol.for('@@streamIterator')]();
   }
 
   formatIndex() {

package/lib/spans.js

@@ -5,12 +5,12 @@ export function updateSpans(ctx, s, node, phase) {
       const { span: innerSpan, balanced, balancedSpan, balancer, openSpan } = attributes || {};
 
       if (!flags.intrinsic && (balancer || balanced)) {
-        throw new Error('not implemented');
+        throw new Error('balanced tokens must be instrinsic');
       }
 
       if (flags.intrinsic) {
         if (s.path && balanced) {
-          s.spans = s.spans.push({
+            s.spans = s.spans.push({
             type: 'Lexical',
             name: balancedSpan || s.span.name,
             path: s.path,
@@ -47,7 +47,7 @@ export function updateSpans(ctx, s, node, phase) {
       }
 
       if (balanced) {
-        s.balanced = s.balanced.push(ctx.agast.nodeForTag(s.result));
+        s.balanced = s.balanced.push(ctx.nodeForTag(s.result));
       }
 
       if (innerSpan) {

package/lib/state.js

@@ -9,8 +9,8 @@ export const StateFacade = class BABLRStateFacade {
     facades.set(state, this);
   }
 
-  static from(context, source) {
-    return State.from(actuals.get(context), actuals.get(source));
+  static from(source) {
+    return State.from(actuals.get(source));
   }
 
   get span() {
@@ -25,10 +25,6 @@ export const StateFacade = class BABLRStateFacade {
     return actuals.get(this).holding;
   }
 
-  get context() {
-    return facades.get(actuals.get(this).context);
-  }
-
   get path() {
     return actuals.get(this).path;
   }
@@ -45,34 +41,29 @@ export const StateFacade = class BABLRStateFacade {
     return actuals.get(this).depth;
   }
 
-  get ctx() {
-    return this.context;
+  get status() {
+    return actuals.get(this).status;
   }
 };
 
 export const State = class BABLRState extends WeakStackFrame {
-  constructor(
-    context,
-    source,
-    agast,
-    balanced = emptyStack,
-    spans = emptyStack.push({ name: 'Bare' }),
-  ) {
+  constructor(source, agast, balanced = emptyStack, spans = emptyStack.push({ name: 'Bare' })) {
     super();
 
-    if (!context || !source || !agast) throw new Error('invalid args to State');
+    if (!source || !agast) throw new Error('invalid args to State');
 
-    this.context = context;
     this.source = source;
     this.agast = agast;
     this.balanced = balanced;
     this.spans = spans;
 
+    this.status = 'active';
+
     new StateFacade(this);
   }
 
-  static from(context, source, agast) {
-    return State.create(context, source, agast);
+  static from(source, agast) {
+    return State.create(source, agast);
   }
 
   get guardedSource() {
@@ -82,10 +73,6 @@ export const State = class BABLRState extends WeakStackFrame {
     return guard ? guardWithPattern(guard, source) : source;
   }
 
-  get ctx() {
-    return this.context;
-  }
-
   get span() {
     return this.spans.value;
   }
@@ -118,11 +105,12 @@ export const State = class BABLRState extends WeakStackFrame {
     let { span, spans, source } = this;
     let { guard } = span;
 
-    if (pattern?.flags?.intrinsic) {
-      if (pattern.type === 'OpenNodeTag') {
-        // TODO differntiate better between self-closing tags and matchers
-        pattern = pattern.value;
-      }
+    if (pattern?.intrinsicValue) {
+      // if (pattern.type === 'OpenNodeTag') {
+
+      //   // TODO differntiate better between self-closing tags and matchers
+      //   pattern = pattern.value;
+      // }
 
       ({ guard } = span);
 

package/lib/{evaluate.js → strategy.js}

@@ -1,33 +1,24 @@
 import { Coroutine } from '@bablr/coroutine';
 import { buildCall, buildNull, buildReference, reifyExpression } from '@bablr/agast-vm-helpers';
 import { StreamGenerator } from '@bablr/agast-helpers/stream';
-import { resolveLanguage } from '@bablr/helpers/grammar';
 import { buildTokens } from './utils/token.js';
 import { formatType } from './utils/format.js';
 import { facades } from './facades.js';
 import { State } from './state.js';
 import { updateSpans } from './spans.js';
 
-const nodeTopType = Symbol.for('@bablr/node');
-
-export const evaluate = (ctx, rootLanguage, rootSource, strategy) => {
-  return (agastCtx, agastState) => {
-    if (ctx.agast !== agastCtx) throw new Error();
-
-    if (rootLanguage !== ctx.languages.get(rootLanguage.canonicalURL)) {
-      throw new Error();
-    }
-
-    return new StreamGenerator(__evaluate(ctx, rootLanguage, rootSource, agastState, strategy));
+export const createBablrStrategy = (rootSource, strategy) => {
+  return (ctx, agastState) => {
+    return new StreamGenerator(__strategy(ctx, rootSource, agastState, strategy));
   };
 };
 
 const resolvedLanguages = new WeakMap();
 
-const __evaluate = function* bablrStrategy(ctx, rootLanguage, rootSource, agastState, strategy) {
-  let s = State.from(ctx, rootSource, agastState);
+const __strategy = function* bablrStrategy(ctx, rootSource, agastState, strategy) {
+  let s = State.from(rootSource, agastState);
 
-  let co = new Coroutine(strategy(facades.get(s), facades.get(ctx)));
+  let co = new Coroutine(strategy(facades.get(s), ctx));
 
   co.advance();
 
@@ -69,60 +60,31 @@ const __evaluate = function* bablrStrategy(ctx, rootLanguage, rootSource, agastS
           }
 
           case 'OpenNodeTag': {
-            const { flags, type, language: tagLanguage, intrinsicValue } = terminal.value;
-
-            const reference = s.result;
-
-            if (
-              s.path.depth > 1 &&
-              !flags.trivia &&
-              !flags.escape &&
-              reference?.type !== 'Reference' &&
-              reference?.type !== 'OpenFragmentTag'
-            ) {
-              throw new Error('Invalid location for OpenNodeTag');
-            }
-
-            const oldNode = s.node;
+            const { type, intrinsicValue } = terminal.value;
 
             const openTag = yield sourceInstr;
 
-            const resolvedLanguage =
-              oldNode.depth > 1 ? resolvedLanguages.get(oldNode) : rootLanguage;
-            const nextResolvedLanguage = resolveLanguage(resolvedLanguage, tagLanguage);
-
-            if (!nextResolvedLanguage) {
-              throw new Error(`Resolve failed { language: ${tagLanguage} }`);
-            }
-
-            const grammar = ctx.grammars.get(nextResolvedLanguage);
-            const isNode = grammar.covers.get(nodeTopType).has(type);
-
-            let intrinsicResult = intrinsicValue && s.guardedMatch(terminal.value);
+            if (type) {
+              let intrinsicResult = intrinsicValue && s.guardedMatch(terminal.value);
 
-            if (intrinsicResult instanceof Promise) {
-              intrinsicResult = yield intrinsicResult;
-            }
+              if (intrinsicResult instanceof Promise) {
+                intrinsicResult = yield intrinsicResult;
+              }
 
-            updateSpans(ctx, s, intrinsicValue ? ctx.agast.nodeForTag(openTag) : s.node, 'open');
+              updateSpans(ctx, s, intrinsicValue ? ctx.nodeForTag(openTag) : s.node, 'open');
 
-            if (intrinsicValue) {
-              if (!intrinsicResult) {
-                throw new Error('advance failed to match an intrinsic node');
-              }
+              if (intrinsicValue) {
+                if (!intrinsicResult) {
+                  throw new Error('advance failed to match an intrinsic node');
+                }
 
-              const sourceStep = s.source.advance(intrinsicResult.length);
+                const sourceStep = s.source.advance(intrinsicResult.length);
 
-              if (sourceStep instanceof Promise) {
-                yield sourceStep;
-              }
+                if (sourceStep instanceof Promise) {
+                  yield sourceStep;
+                }
 
-              updateSpans(ctx, s, ctx.agast.nodeForTag(openTag), 'close');
-            } else {
-              if (isNode) {
-                resolvedLanguages.set(s.node, nextResolvedLanguage);
-              } else {
-                resolvedLanguages.set(s.node, resolvedLanguage);
+                updateSpans(ctx, s, ctx.nodeForTag(openTag), 'close');
               }
             }
 
@@ -130,15 +92,6 @@ const __evaluate = function* bablrStrategy(ctx, rootLanguage, rootSource, agastS
             break;
           }
 
-          case 'OpenFragmentTag': {
-            const openTag = yield sourceInstr;
-
-            resolvedLanguages.set(s.node, rootLanguage);
-
-            returnValue = openTag;
-            break;
-          }
-
           case 'CloseNodeTag': {
             const { node } = s;
 
@@ -146,16 +99,7 @@ const __evaluate = function* bablrStrategy(ctx, rootLanguage, rootSource, agastS
 
             if (s.path) {
               updateSpans(ctx, s, node, 'close');
-            }
-
-            returnValue = endTag;
-            break;
-          }
-
-          case 'CloseFragmentTag': {
-            returnValue = yield sourceInstr;
-
-            if (!s.path) {
+            } else {
               if (!s.source.done) {
                 throw new Error('Parser failed to consume input');
               }
@@ -164,6 +108,8 @@ const __evaluate = function* bablrStrategy(ctx, rootLanguage, rootSource, agastS
                 throw new Error('Parser did not match all balanced nodes');
               }
             }
+
+            returnValue = endTag;
             break;
           }
 
@@ -192,10 +138,14 @@ const __evaluate = function* bablrStrategy(ctx, rootLanguage, rootSource, agastS
 
           case 'Gap': {
             if (s.source.value == null && !s.source.done) {
-              const sourceStep = s.source.advance(1);
+              if (s.source.holding) {
+                s.source.unshift();
+              } else {
+                const sourceStep = s.source.advance(1);
 
-              if (sourceStep instanceof Promise) {
-                yield sourceStep;
+                if (sourceStep instanceof Promise) {
+                  yield sourceStep;
+                }
               }
 
               returnValue = yield sourceInstr;
@@ -205,6 +155,13 @@ const __evaluate = function* bablrStrategy(ctx, rootLanguage, rootSource, agastS
             break;
           }
 
+          case 'Shift': {
+            s.source.shift();
+
+            returnValue = yield sourceInstr;
+            break;
+          }
+
           default: {
             returnValue = yield sourceInstr;
             break;
@@ -223,33 +180,17 @@ const __evaluate = function* bablrStrategy(ctx, rootLanguage, rootSource, agastS
           result = yield result;
         }
 
-        returnValue = result && ctx.agast.buildRange(buildTokens(result));
-        break;
-      }
-
-      case 'shift': {
-        s.source.shift();
-
-        yield sourceInstr;
-
-        break;
-      }
-
-      case 'unshift': {
-        s.source.unshift();
-
-        yield sourceInstr;
-
+        returnValue = result && ctx.buildRange(buildTokens(result));
         break;
       }
 
       case 'branch': {
         const baseState = s;
-        let { context, source, agast, balanced, spans, node } = baseState;
+        let { source, agast, balanced, spans, node } = baseState;
 
         agast = yield sourceInstr;
 
-        s = s.push(context, source.branch(), agast, balanced, spans);
+        s = s.push(source.branch(), agast, balanced, spans);
 
         resolvedLanguages.set(s.node, resolvedLanguages.get(node));
 
@@ -260,6 +201,8 @@ const __evaluate = function* bablrStrategy(ctx, rootLanguage, rootSource, agastS
       case 'accept': {
         const accepted = s;
 
+        s.status = 'accepted';
+
         const agastState = yield sourceInstr;
 
         s = s.parent;
@@ -281,27 +224,27 @@ const __evaluate = function* bablrStrategy(ctx, rootLanguage, rootSource, agastS
       case 'reject': {
         const rejectedState = s;
 
+        s.status = 'rejected';
+
         yield sourceInstr;
 
         s = s.parent;
 
-        if (rejectedState.path.depth > s.path.depth) {
+        if (s.path.depth && rejectedState.path.depth > s.path.depth) {
+          const didShift = rejectedState.node.at(s.node.depth) === s.node;
           const lowPath = rejectedState.path.at(
-            Math.min(s.path.depth + 1, rejectedState.path.depth),
+            Math.min(s.path.depth + (didShift ? 0 : 1), rejectedState.path.depth),
           );
+          const lowNode = s.node;
 
-          if (s.path.depth) {
-            const { name, isArray } = lowPath.reference?.value || {};
+          const { name, isArray } = lowPath.reference?.value || {};
 
-            if (s.node.depth === s.path.depth && !s.node.resolver.counters.has(name)) {
-              if (!lowPath.openTag?.value.flags.trivia && !lowPath.openTag?.value.flags.escape) {
-                yield buildCall('advance', buildReference(name, isArray));
-              }
-
-              if (s.result.type === 'Reference') {
-                yield buildCall('advance', buildNull());
-              }
+          if (lowPath.depth === lowNode.depth + 1 && !lowNode.resolver.counters.has(name)) {
+            if (!lowNode.openTag?.value.flags.trivia && !lowNode.openTag?.value.flags.escape) {
+              yield buildCall('advance', buildReference(name, isArray));
             }
+
+            yield buildCall('advance', buildNull());
           }
         }
 
@@ -313,6 +256,7 @@ const __evaluate = function* bablrStrategy(ctx, rootLanguage, rootSource, agastS
         break;
       }
 
+      case 'write':
       case 'bindAttribute': {
         returnValue = yield sourceInstr;
         break;
@@ -323,11 +267,6 @@ const __evaluate = function* bablrStrategy(ctx, rootLanguage, rootSource, agastS
         break;
       }
 
-      case 'getContext': {
-        returnValue = facades.get(ctx);
-        break;
-      }
-
       default: {
         throw new Error(`Unexpected call of {type: ${formatType(verb)}}`);
       }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@bablr/bablr-vm",
   "description": "A VM for parsing using BABLR languages",
-  "version": "0.13.2",
+  "version": "0.14.0",
   "author": "Conrad Buck<conartist6@gmail.com>",
   "type": "module",
   "files": [
@@ -12,11 +12,11 @@
   },
   "sideEffects": false,
   "dependencies": {
-    "@bablr/agast-helpers": "0.1.1",
-    "@bablr/agast-vm-helpers": "0.1.1",
+    "@bablr/agast-helpers": "0.1.6",
+    "@bablr/agast-vm-helpers": "0.1.5",
     "@bablr/coroutine": "0.1.0",
-    "@bablr/helpers": "0.15.1",
-    "@bablr/regex-vm": "0.5.0",
+    "@bablr/helpers": "0.16.0",
+    "@bablr/regex-vm": "0.5.1",
     "@bablr/weak-stack": "0.1.0",
     "@iter-tools/imm-stack": "1.1.0",
     "iter-tools-es": "^7.5.3"

package/lib/context.js

@@ -1,76 +0,0 @@
-import { facades, actuals } from './facades.js';
-
-export const ContextFacade = class BABLRContextFacade {
-  get languages() {
-    return actuals.get(this).languages;
-  }
-
-  get grammars() {
-    return actuals.get(this).grammars;
-  }
-
-  get productionEnhancer() {
-    return actuals.get(this).productionEnhancer;
-  }
-
-  get agast() {
-    return actuals.get(this).agast;
-  }
-
-  getPreviousTerminal(token) {
-    return actuals.get(this).agast.getPreviousTerminal(token);
-  }
-
-  ownTerminalsFor(range) {
-    return actuals.get(this).agast.ownTerminalsFor(range);
-  }
-
-  allTerminalsFor(range) {
-    return actuals.get(this).agast.allTerminalsFor(range);
-  }
-
-  getCooked(range) {
-    return actuals.get(this).agast.getCooked(range);
-  }
-
-  reifyExpression(value) {
-    return actuals.get(this).agast.reifyExpression(value);
-  }
-
-  getProperty(node, name) {
-    return actuals.get(this).agast.getProperty(node, name);
-  }
-
-  sourceTextFor(range) {
-    return actuals.get(this).agast.sourceTextFor(range);
-  }
-
-  nodeForTag(tag) {
-    return actuals.get(this).agast.nodeForTag(tag);
-  }
-
-  unbox(value) {
-    return actuals.get(this).agast.unbox(value);
-  }
-};
-
-export const Context = class BABLRContext {
-  static from(agastContext, languages, productionEnhancer) {
-    return new Context(agastContext, languages, productionEnhancer);
-  }
-
-  constructor(agastContext, languages, productionEnhancer) {
-    this.agast = agastContext;
-    this.languages = languages;
-    this.productionEnhancer = productionEnhancer;
-
-    this.grammars = new WeakMap();
-    this.facade = new ContextFacade();
-
-    for (const { 1: language } of this.languages) {
-      this.grammars.set(language, new language.grammar());
-    }
-
-    facades.set(this, this.facade);
-  }
-};