@bablr/bablr-vm 0.22.1 → 0.23.1

package/lib/evaluate.js

@@ -1,16 +1,9 @@
+/* global WeakSet */
 import { Coroutine } from '@bablr/coroutine';
-import {
-  buildAttributeDefinition,
-  buildBindingTag,
-  buildChild,
-  buildCloseNodeTag,
-  buildFacadeProperty,
-  buildProperty,
-} from '@bablr/agast-helpers/builders';
-import { getStreamIterator, printType, StreamIterable } from '@bablr/agast-helpers/stream';
+import { buildAttributeDefinition, buildCloseNodeTag } from '@bablr/agast-helpers/builders';
+import { getStreamIterator, printType, StreamIterable, wait } from '@bablr/agast-helpers/stream';
 import { facades } from './facades.js';
 import { State } from './state.js';
-import { updateSpans } from './spans.js';
 import {
   OpenNodeTag,
   CloseNodeTag,
@@ -18,34 +11,22 @@ import {
   BindingTag,
   LiteralTag,
   ReferenceTag,
-  NullTag,
   ShiftTag,
-  InitializerTag,
   AttributeDefinition,
-  Property,
 } from '@bablr/agast-helpers/symbols';
-import * as sym from '@bablr/agast-vm-helpers/symbols';
-import { agast, PathFacade, TagPathFacade as TagPath } from '@bablr/agast-vm';
-import { states } from './node.js';
-import {
-  buildOpenNodeTag,
-  buildReferenceTag,
-  buildStubNode,
-  getFlagsWithGap,
-  getRoot,
-  treeFromStreamSync,
-} from '@bablr/agast-helpers/tree';
-import { getEmbeddedObject, getEmbeddedTag } from '@bablr/agast-vm-helpers/deembed';
+import { getRoot, treeFromStreamSync } from '@bablr/agast-helpers/tree';
+import * as Tags from '@bablr/agast-helpers/tags';
+import { getEmbeddedTag } from '@bablr/agast-vm-helpers/deembed';
 import { Match } from './match.js';
-import { effectsFor, reifyExpression } from '@bablr/agast-vm-helpers';
-import { getProduction } from '@bablr/helpers/grammar';
-import { buildEmbeddedNode, buildWriteEffect } from '@bablr/agast-vm-helpers/builders';
+import { buildWriteEffect } from '@bablr/agast-vm-helpers/builders';
 import { has } from '@bablr/agast-helpers/object';
-import { referencesAreEqual } from '@bablr/agast-helpers/path';
-import { isPlainObject } from '@bablr/helpers/object';
+import { reifyBablrOptions } from '@bablr/agast-vm-helpers';
+import { buildNode, getTags } from '@bablr/agast-helpers/path';
+
+const validNodesByLanguage = new Map();
 
-const defineAttribute = (m, path, value) => {
-  m.agast.vm.next(buildAttributeDefinition(path, value));
+export const isKnownValid = (language, node) => {
+  return !!validNodesByLanguage.get(language)?.has(node);
 };
 
 const getSourceLength = (tags) => {
@@ -67,16 +48,22 @@ export const bablr = (ctx, rootSource, strategy, options = {}) => {
 function* __bablr(ctx, rootSource, strategy, options) {
   let s = null;
   let m = null;
-  let language = null;
   let finishedMatch = null;
+  let outerOptions = options;
 
-  let co = new Coroutine(getStreamIterator(strategy(facades.get(ctx))));
+  let getState = () => s.getPublic();
+
+  for (const language of ctx.languages.values()) {
+    validNodesByLanguage.set(language, validNodesByLanguage.get(language) || new WeakSet());
+  }
+
+  let co = new Coroutine(getStreamIterator(strategy(facades.get(ctx), getState)));
 
   co.advance();
 
   for (;;) {
     if (co.current instanceof Promise) {
-      co.current = yield co.current;
+      co.current = yield wait(co.current);
     }
 
     if (co.done) {
@@ -88,11 +75,11 @@ function* __bablr(ctx, rootSource, strategy, options) {
 
       s.source.release();
 
-      if (exchange && exchange.sources > 0) {
+      if (exchange?.sources > 0) {
         throw new Error('Source did not close all forks: ' + exchange.sources);
       }
 
-      return co.value && states.get(co.value).node;
+      return co.value;
     }
 
     const instr = co.value;
@@ -113,15 +100,15 @@ function* __bablr(ctx, rootSource, strategy, options) {
       case 'match': {
         let { arguments: { 0: pattern } = [] } = instr;
 
-        let result = s.guardedMatch(pattern, pattern.attributes);
+        let result = s.guardedMatch(pattern);
 
         if (result instanceof Promise) {
-          result = yield result;
+          result = yield wait(result);
         }
 
         let node = result && treeFromStreamSync(result);
 
-        returnValue = node && buildEmbeddedNode(node);
+        returnValue = node;
         break;
       }
 
@@ -130,80 +117,44 @@ function* __bablr(ctx, rootSource, strategy, options) {
 
         const tag = getEmbeddedTag(embeddedTags);
 
-        if (tag.type !== ReferenceTag) {
-          s.referenceTagPath = null;
-        }
-
-        if (
-          !m.isNode &&
-          m.coveredBoundary !== m &&
-          !((s.holding && tag.type === GapTag) || tag.type === OpenNodeTag)
-        ) {
-          throw new Error('cannot advance inside a cover');
-        }
-
         switch (tag.type) {
           case ReferenceTag: {
-            if (s.node.type && tag.value.type === '.') throw new Error();
-            if (s.held && s.node.tags.at(2)?.type === InitializerTag) {
-              if (!referencesAreEqual(s.node.tags.at(1).value, tag.value) || s.node.tags.size > 3) {
-                throw new Error();
-              }
-            }
+            // if (m.didShift) throw new Error();
 
             m.advance(tag, s);
 
-            s.referenceTagPath = TagPath.from(s.path, -1, 0);
-            if (s.referenceTagPath.tag.type === BindingTag) throw new Error();
             break;
           }
 
           case ShiftTag: {
-            let { index, height } = tag.value;
-            let refTag = s.node.tags.at(-1, 0);
-            let lastShiftTag = refTag;
-
-            if (refTag.type === ShiftTag) {
-              refTag = s.node.tags.at(-refTag.value.index - 1, 0);
-            }
+            let lastProp = Tags.getAt(-1, m.rootNode.value.tags);
 
-            if (refTag.type !== ReferenceTag) throw new Error();
+            m.advance(tag, s);
 
-            s.depths.shift = height;
-            s.depths.nodeShift = index;
+            let shiftProp = Tags.getAt(-1, m.rootNode.value.tags);
 
-            if (index !== (lastShiftTag.type === ShiftTag ? lastShiftTag.value.index + 1 : 1)) {
-              throw new Error();
-            }
+            let {
+              shift: { height, index },
+            } = shiftProp.value;
 
-            s.source.shift();
-            s.held = buildFacadeProperty(
-              refTag.value,
-              s.resultPath.tag.value,
-              s.resultPath.nextSibling.tag.value.node,
-            );
+            s.held = lastProp.value.node;
 
-            m.advance(tag, s);
-            s.referenceTagPath = TagPath.from(s.path, 0);
+            s.depths.shift = height;
+            s.depths.nodeShift = index;
             break;
           }
 
           case BindingTag: {
             m.advance(tag, s);
-
-            let propertyChild = buildChild(
-              sym.Property,
-              buildProperty(s.held.reference, tag.value, s.held.node.node),
-            );
-
-            m.agastFragment.vm.next(propertyChild);
-
-            s.held = null;
             break;
           }
 
           case OpenNodeTag: {
-            let { literalValue, selfClosing } = tag.value;
+            let { type, literalValue, selfClosing } = tag.value;
+
+            if (literalValue && s.held) {
+              // throw new Error('cannot advance literal while holding');
+            }
 
             s.depths.path++;
             s.depths.nodeShift = 0;
@@ -212,33 +163,25 @@ function* __bablr(ctx, rootSource, strategy, options) {
               s.depths.shift = 0;
             }
 
-            s.agast = m.agast;
-
             m.advance(tag, s);
-            const refPath = m.referencePath;
+
+            if (!type || !s.node) {
+              s.node = m.node;
+            }
 
             if (selfClosing) {
               if (literalValue) {
-                let result;
-                if (
-                  s.resultPath.tag.type === OpenNodeTag &&
-                  s.resultPath.tag.value.attributes.balancer &&
-                  s.balanced.value?.attributes.balanced === literalValue
-                ) {
-                  result = s.match(literalValue, tag.value.attributes);
-                } else {
-                  result = s.guardedMatch(literalValue, tag.value.attributes);
-                }
+                let result = s.guardedMatch(literalValue);
 
                 if (result instanceof Promise) {
-                  result = yield result;
+                  result = yield wait(result);
                 }
 
                 if (result) {
                   let sourceStep = s.source.advance(getSourceLength(result));
 
                   if (sourceStep instanceof Promise) {
-                    sourceStep = yield sourceStep;
+                    sourceStep = yield wait(sourceStep);
                   }
                 } else {
                   throw new Error('Failed to advance literal');
@@ -249,134 +192,64 @@ function* __bablr(ctx, rootSource, strategy, options) {
               s.depths.nodeShift = 0;
               s.depths.shift = 0;
 
-              s.agast = m.agastFragment;
-              s.held = Object.freeze({
-                reference: refPath.tag.value,
-                binding: null,
-                node: s.resultPath.node,
-              });
-            }
-
-            if (s.depths.path === 0) {
-              m.setRangePreviousIndex(0);
-            }
-
-            if (tag.value.type) {
-              updateSpans(m, s.resultPath.node, 'open');
-            }
-            if (literalValue) {
-              updateSpans(m, s.resultPath.node, 'close');
+              // s.held = literalValue
+              //   ? new NodeFacade(s.resultPath.tag.value.property.node)
+              //   : s.resultPath.node;
             }
 
             break;
           }
 
           case CloseNodeTag: {
-            const { node } = s;
+            let { node, language } = m;
 
             s.depths.path--;
             s.depths.nodeShift = 0;
             s.depths.shift = 0;
 
-            if (m.rangeInitial?.tag.type === ShiftTag) {
-              s.depths.nodeShift = m.rangeInitial.tag.value.index;
-              s.depths.shift = m.rangeInitial.tag.value.height;
+            if (m.shiftMatch) {
+              let { shift } = Tags.getAt(-1, m.rootNode.value.children).value;
+              s.depths.nodeShift = shift.index;
+              s.depths.shift = shift.height;
             }
 
-            if (node.tags.at(0).value.type) {
-              const refPath = m.referencePath;
-
-              m.advance(tag, s);
-              let gaplessNode = null;
-
-              if (refPath?.tag.value.type === '@') {
-                let { vm: escapeVm, state: escapeState } = agast();
-
-                let tagPath = PathFacade.from(node).tagPathAt(0);
-
-                while (tagPath) {
-                  if (tagPath.tag.type === GapTag) {
-                    throw new Error();
-                  }
-                  if (tagPath.tag.type === OpenNodeTag) {
-                    let { flags, type, attributes, literalValue } = tagPath.tag.value;
-
-                    escapeVm.next(
-                      buildOpenNodeTag(
-                        getFlagsWithGap(flags, false),
-                        type,
-                        attributes,
-                        literalValue,
-                      ),
-                    );
-                  } else if (tagPath.tag.type === ReferenceTag && tagPath.tag.value.flags.hasGap) {
-                    let { flags, name, type, isArray, index } = tagPath.tag.value;
-
-                    flags = getFlagsWithGap(flags, false);
-
-                    escapeVm.next(buildReferenceTag(type, name, isArray, flags, index));
-                  } else {
-                    escapeVm.next(tagPath.tag);
-                  }
-
-                  if (tagPath.tag.type === CloseNodeTag && !tagPath.nextUnshifted) {
-                    gaplessNode = escapeState.resultPath.node;
-                  }
+            m.advance(tag, s);
 
-                  tagPath = tagPath.nextUnshifted;
-                }
+            if (!m.parent) {
+              if (!s.source.done) {
+                throw new Error('Parser failed to consume input');
               }
 
-              s.agast = m.agastFragment;
-
-              s.held = Object.freeze({
-                reference: refPath?.tag.value,
-                binding: null,
-                node: gaplessNode || m.node,
-              });
-
-              updateSpans(m, s.resultPath.node, 'close');
-
-              if (!m.parent) {
-                if (!s.source.done) {
-                  throw new Error('Parser failed to consume input');
-                }
-
-                if (s.balanced.size) {
-                  throw new Error('Parser did not match all balanced nodes');
-                }
+              if (s.balanced.size) {
+                throw new Error('Parser did not match all balanced nodes');
               }
-            } else {
-              m.advance(tag, s);
-              s.agast = m.agastFragment;
             }
 
+            if (!node.value.type) {
+              s.node = m.node;
+            }
+
+            validNodesByLanguage.get(language).add(node);
+
             break;
           }
 
           case LiteralTag: {
             const { value: pattern } = tag;
 
-            let result;
-            if (
-              s.resultPath.tag.type === OpenNodeTag &&
-              s.resultPath.tag.value.attributes.balancer &&
-              s.balanced.value?.attributes.balanced === pattern
-            ) {
-              result = s.match(pattern);
-            } else {
-              result = s.guardedMatch(pattern);
-            }
+            // if (s.held) throw new Error('cannot advance literal while holding');
+
+            let result = s.guardedMatch(pattern);
 
             if (result instanceof Promise) {
-              result = yield result;
+              result = yield wait(result);
             }
 
             if (result) {
               let sourceStep = s.source.advance(getSourceLength(result));
 
               if (sourceStep instanceof Promise) {
-                sourceStep = yield sourceStep;
+                sourceStep = yield wait(sourceStep);
               }
 
               m.advance(tag, s);
@@ -387,79 +260,48 @@ function* __bablr(ctx, rootSource, strategy, options) {
           }
 
           case GapTag: {
-            if (s.source.value == null && (!s.source.done || s.source.holding)) {
-              if (s.source.holding) {
-                s.source.unshift();
-              } else {
+            if (s.source.value == null && !s.done) {
+              if (!s.held) {
                 const sourceStep = s.source.advance(1);
 
                 if (sourceStep instanceof Promise) {
-                  yield sourceStep;
+                  yield wait(sourceStep);
                 }
               }
+            }
 
-              if (s.held) {
-                if (!s.agast.state.held) throw new Error();
-                s.agast.vm.next(
-                  buildChild(
-                    Property,
-                    buildProperty(m.referencePath.tag.value, null, buildStubNode(tag)),
-                  ),
-                );
-
-                s.resultPath = TagPath.from(s.resultPath.path, -1);
-
-                s.held = null;
-                break;
-              }
-
-              if (s.expressions.size) {
-                let expression = s.expressions.value;
-                let node = getRoot(isPlainObject(expression) ? expression : expression.node);
-
-                let mergedBinding = buildBindingTag(m.mergedLanguagePath);
-
-                s.agast.vm.next(mergedBinding);
-                s.agast.vm.next(
-                  buildChild(
-                    Property,
-                    buildProperty(s.resultPath.tag.value, mergedBinding.value, node),
-                  ),
-                );
+            if (s.held) {
+              if (!m.agast.getState().held) throw new Error();
+              m.advance(buildNode(tag));
 
-                s.expressions = s.expressions.pop();
-                break;
-              }
+              s.held = null;
+              break;
+            }
 
-              if (s.node.tags.size) {
-                m.advance(tag, s);
-              } else {
-                m.add(m.node);
+            if (s.expressions.size) {
+              let expression = s.expressions.value;
+              let node = getRoot(expression);
 
-                m.advance(tag, s);
-              }
+              m.advance(node);
 
-              s.referenceTagPath = null;
+              s.expressions = s.expressions.pop();
+              break;
+            }
 
-              s.agast = m.agastFragment;
+            if (Tags.getSize(getTags(s.node))) {
+              m.advance(tag, s);
             } else {
-              throw new Error('Failed to advance gap');
-            }
-            break;
-          }
+              m.add(m.node);
 
-          case InitializerTag: {
-            if (s.held && s.node.tags.at(2)?.type === InitializerTag) {
-              throw new Error();
+              m.advance(tag, s);
             }
-            m.advance(tag, s);
             break;
           }
 
           case AttributeDefinition: {
             const { path, value } = tag.value;
 
-            if (!has(m.node.attributes, path)) {
+            if (!has(m.node.value.attributes, path)) {
               throw new Error('undefined attributes must be declared');
             }
 
@@ -467,7 +309,8 @@ function* __bablr(ctx, rootSource, strategy, options) {
 
             if (value && typeof value === 'object') throw new Error('unimplemented');
 
-            defineAttribute(m, path, value);
+            m.advance(buildAttributeDefinition(path, value));
+
             break;
           }
 
@@ -476,156 +319,81 @@ function* __bablr(ctx, rootSource, strategy, options) {
         }
 
         if (s.depth === 0 && (s.node || s.resultPath?.node)) {
-          yield* m.emit(options);
+          yield* m.emit(outerOptions);
         }
 
         returnValue = tag;
         break;
       }
 
-      case 'openSpan': {
-        let { arguments: { 0: name } = [] } = instr;
-        s.spans = s.spans.push({ guard: null, name, path: s.path, type: 'Instruction' });
+      case 'startSpan': {
+        let { arguments: { 0: name, 1: guard = null } = [] } = instr;
+        s.spans = s.spans.push({ type: 'Instruction', name, guard });
         break;
       }
 
-      case 'closeSpan': {
+      case 'endSpan': {
         if (s.spans.value.type !== 'Instruction') throw new Error();
         s.spans = s.spans.pop();
         break;
       }
 
-      case 'branch': {
-        s = s.branch();
-
-        // m.node = s.agast.node;
-
-        returnValue = facades.get(s);
-        break;
-      }
+      case 'startFrame': {
+        let { arguments: { 0: verb, 1: { value: matcher } = {}, 2: options = '    ' } = [] } =
+          instr;
 
-      case 'accept': {
-        s = s.accept(finishedMatch.fragmentNode);
+        m = Match.startFrame(ctx, s, m, null, verb, matcher, reifyBablrOptions(options));
+        s = m.state;
 
-        if (s.depth === 0 && (s.node || s.resultPath?.node)) {
-          yield* m.emit(options);
-        }
+        yield* m.emit(outerOptions);
 
-        returnValue = facades.get(s);
+        returnValue = facades.get(m);
         break;
       }
 
-      case 'reject': {
-        let finishedState = s;
-        const { arguments: { 0: embeddedOptions } = [] } = instr;
-        let options = getEmbeddedObject(embeddedOptions);
+      case 'endFrame': {
+        finishedMatch = m;
 
-        s = finishedState.reject(
-          finishedMatch,
-          // finishedMatch.effects.success === 'none',
-          options,
-        );
+        finishedMatch.advance(buildCloseNodeTag());
 
-        s = finishedState.parent;
+        m = m.endFrame(outerOptions);
+        s = m ? m.state : s;
 
-        if (s.depth === 0 && (s.node || s.resultPath?.node)) {
-          yield* m.emit(options);
+        if (m) {
+          m.running = null;
+          if (finishedMatch.shiftMatch) {
+            finishedMatch.shiftMatch.running = null;
+          }
+          yield* m.emit(outerOptions);
         }
 
-        returnValue = facades.get(s);
+        returnValue = m && facades.get(m);
         break;
       }
 
-      case 'startFrame': {
-        let {
-          arguments: { 0: verb, 1: { value: matcher } = {}, 2: { value: options = {} } = {} } = [],
-        } = instr;
-
-        let effects = effectsFor(verb.description);
-        let didShift = verb.description.startsWith('shift');
+      case 'shiftFrame': {
+        let { arguments: { 0: verb, 1: { value: matcher } = {}, 2: options = '    ' } = [] } =
+          instr;
 
         let parentMatch = m;
 
-        if (!language) throw new Error('not initialized');
-
-        let matcher_ = reifyExpression(matcher);
-
-        if (didShift && !parentMatch) throw new Error();
-
-        m = parentMatch
-          ? parentMatch.startFrame(s, matcher_, effects, didShift ? finishedMatch : null, options)
-          : Match.from(ctx, language, s, matcher_, null, options);
+        if (finishedMatch.type !== '_') throw new Error();
 
-        finishedMatch = null;
-
-        if (m.type !== sym.fragment && !getProduction(m.grammar, m.type))
-          throw new Error(`Production {type: ${printType(m.type)}} does not exist`);
-
-        if (m.flags.token && !m.isNode) {
-          throw new Error('tokens must be nodes');
-        }
-
-        if (parentMatch && parentMatch.cover && !parentMatch.isNode) {
-          if (matcher_.refMatcher) {
-            let m = matcher_.refMatcher;
-            if (!['.', '#'].includes(m.type) || m.flags.expression || m.flags.hasGap || m.isArray) {
-              throw new Error('no references inside covers');
-            }
-          }
-        }
-
-        if (didShift) {
-          if (s.node.tags.at(-1, 0)?.type !== ShiftTag) {
-            throw new Error('advance shift tag before starting new held frame');
-          }
-        }
-
-        let broken = !parentMatch || effects.success === 'none' || s.agast.state.node.type === null;
-
-        if (effects.success !== 'none') {
-          m.agastFragment = s.agast;
-        } else {
-          s.agast = m.agastFragment = m.agast;
-          s.referenceTagPath = null;
-        }
-
-        ({ language } = m);
-
-        let offset = [ReferenceTag, ShiftTag].includes(s.resultPath?.tag.type) ? -1 : 0;
-
-        if (!broken) {
-          let previousIndex = [CloseNodeTag, NullTag, GapTag].includes(s.resultPath.tag.type)
-            ? m.fragmentNode.tags.size - 1
-            : s.resultPath.tagsIndex;
-
-          m.setRangePreviousIndex(previousIndex + offset);
-        } else if (parentMatch) {
-          m.setRangePreviousIndex(m.fragmentNode.tags.size - 1 + offset);
-        }
-
-        returnValue = facades.get(m);
-
-        m.range;
-        break;
-      }
-
-      case 'endFrame': {
-        finishedMatch = m;
-
-        m = m.endFrame();
+        m = Match.startFrame(
+          ctx,
+          s,
+          parentMatch,
+          finishedMatch,
+          verb,
+          matcher,
+          reifyBablrOptions(options),
+        );
+        s = m ? m.state : s;
 
-        if (finishedMatch.effects.success !== 'none') {
-          s.agast = finishedMatch.agastFragment;
-        } else {
-          finishedMatch.agastFragment.vm.next(buildCloseNodeTag());
-          s.agast = m.agast;
-          s.referenceTagPath = m.referencePath;
-        }
+        finishedMatch.parent.running = m;
+        finishedMatch.running = m;
 
-        if (!m) {
-          returnValue = m;
-          break;
-        }
+        yield* finishedMatch.emit(outerOptions);
 
         returnValue = m && facades.get(m);
         break;
@@ -636,8 +404,10 @@ function* __bablr(ctx, rootSource, strategy, options) {
 
         m = m.throw_();
 
-        if (m && finishedMatch.s === s) {
-          s.agast = m.agast;
+        if (m) {
+          yield* m.emit(outerOptions);
+
+          s = m.state;
         }
 
         returnValue = m && facades.get(m);
@@ -652,20 +422,19 @@ function* __bablr(ctx, rootSource, strategy, options) {
       case 'init': {
         let { arguments: { 0: canonicalURL } = [] } = instr;
 
-        if (language !== null) throw new Error();
+        if (s !== null) throw new Error();
 
-        s = State.from(rootSource, ctx, canonicalURL, options.expressions);
+        // TODO get rid of ctx.languages
+        s = State.from(rootSource, ctx, ctx.languages.get(canonicalURL), options.expressions);
 
         s.source.advance();
 
         const sourceStep = s.source.head.step;
 
         if (sourceStep instanceof Promise) {
-          yield sourceStep;
+          yield wait(sourceStep);
         }
 
-        language = ctx.languages.get(canonicalURL);
-
         returnValue = facades.get(s);
         break;
       }