@bablr/agast-vm 0.2.0 → 0.4.0

package/lib/context.js

@@ -1,22 +1,15 @@
 import { reifyExpressionShallow, reifyExpression } from '@bablr/agast-vm-helpers';
-import { getCooked, sourceTextFor } from '@bablr/agast-helpers/stream';
+import {
+  getCooked as getCookedFromTree,
+  sourceTextFor as sourceTextForTree,
+} from '@bablr/agast-helpers/tree';
+import {
+  getCooked as getCookedFromStream,
+  sourceTextFor as sourceTextForStream,
+} from '@bablr/agast-helpers/stream';
 import { facades, actuals } from './facades.js';
 
-function* ownTerminalsFor(range, nextTerminals, tagNodes) {
-  if (!range) return;
-
-  const { 0: start, 1: end } = range;
-
-  const pastEnd = nextTerminals.get(end);
-
-  for (let term = start; term && term !== pastEnd; term = nextTerminals.get(term)) {
-    if (!(term === range[0] || term === range[1]) && term.type === 'OpenNodeTag') {
-      term = nextTerminals.get(tagNodes.get(term).endTag);
-    }
-
-    yield term;
-  }
-}
+const { isArray } = Array;
 
 function* allTerminalsFor(range, nextTerminals) {
   if (!range) return;
@@ -38,10 +31,6 @@ export const ContextFacade = class AgastContextFacade {
     return actuals.get(this).nextTerminals.get(token);
   }
 
-  ownTerminalsFor(range) {
-    return actuals.get(this).ownTerminalsFor(range);
-  }
-
   allTerminalsFor(range) {
     return actuals.get(this).allTerminalsFor(range);
   }
@@ -54,10 +43,6 @@ export const ContextFacade = class AgastContextFacade {
     return actuals.get(this).reifyExpression(range);
   }
 
-  getProperty(node, name) {
-    return actuals.get(this).getProperty(node, name);
-  }
-
   sourceTextFor(range) {
     return actuals.get(this).sourceTextFor(range);
   }
@@ -66,10 +51,6 @@ export const ContextFacade = class AgastContextFacade {
     return actuals.get(this).buildRange(terminals);
   }
 
-  nodeForTag(tag) {
-    return actuals.get(this).nodeForTag(tag);
-  }
-
   unbox(value) {
     return actuals.get(this).unbox(value);
   }
@@ -83,23 +64,12 @@ export const Context = class AgastContext {
   constructor() {
     this.prevTerminals = new WeakMap();
     this.nextTerminals = new WeakMap();
-    this.tagNodes = new WeakMap();
     this.unboxedValues = new WeakMap();
     this.facade = new ContextFacade();
 
     facades.set(this, this.facade);
   }
 
-  getProperty(result, name) {
-    let startTag = result[0] || result;
-
-    if (startTag.type === 'Reference') {
-      startTag = this.nextTerminals.get(startTag);
-    }
-
-    return this.tagNodes.get(startTag).properties.get(name);
-  }
-
   isEmpty(range) {
     const { path, parent } = this;
 
@@ -118,23 +88,21 @@ export const Context = class AgastContext {
   }
 
   allTerminalsReverseFor(range) {
-    return allTerminalsFor(range, this.prevTerminals);
+    return allTerminalsFor([...range].reverse(), this.prevTerminals);
   }
 
-  ownTerminalsFor(range) {
-    return ownTerminalsFor(range, this.nextTerminals, this.tagNodes);
-  }
-
-  ownTerminalsReverseFor(range) {
-    return ownTerminalsFor(range, this.prevTerminals, this.tagNodes);
+  getPreviousTerminal(token) {
+    return this.prevTerminals.get(token);
   }
 
-  nodeForTag(tag) {
-    return this.tagNodes.get(tag);
+  getNextTerminal(token) {
+    return this.nextTerminals.get(token);
   }
 
-  sourceTextFor(range) {
-    return sourceTextFor(this.allTerminalsFor(range));
+  sourceTextFor(nodeOrRange) {
+    return isArray(nodeOrRange)
+      ? sourceTextForStream(this.allTerminalsFor(nodeOrRange))
+      : sourceTextForTree(nodeOrRange);
   }
 
   buildRange(terminals) {
@@ -170,7 +138,9 @@ export const Context = class AgastContext {
     return reifyExpression(value);
   }
 
-  getCooked(range) {
-    return getCooked(this.ownTerminalsFor(range));
+  getCooked(nodeOrRange) {
+    return isArray(nodeOrRange)
+      ? getCookedFromStream(this.allTerminalsFor(nodeOrRange))
+      : getCookedFromTree(nodeOrRange);
   }
 };

package/lib/evaluate.js

@@ -9,11 +9,11 @@ import {
   buildDoctypeTag,
   buildNodeOpenTag,
   buildNodeCloseTag,
-} from '@bablr/agast-helpers/builders';
+} from '@bablr/agast-vm-helpers/internal-builders';
+import { deembedExpression } from '@bablr/agast-vm-helpers/deembed';
 import { StreamIterable, getStreamIterator } from '@bablr/agast-helpers/stream';
 import { printExpression } from '@bablr/agast-helpers/print';
-import { reifyExpression } from '@bablr/agast-vm-helpers';
-import { Path, Node } from './path.js';
+import { getRange, getOpenTag } from '@bablr/agast-helpers/tree';
 import { State } from './state.js';
 import { facades } from './facades.js';
 
@@ -21,7 +21,7 @@ export const evaluate = (ctx, strategy, options) =>
   new StreamIterable(__evaluate(ctx, strategy, options));
 
 const __evaluate = function* agast(ctx, strategy, options = {}) {
-  let s = State.from(ctx);
+  let s = State.from(ctx, options.expressions);
 
   const co = new Coroutine(getStreamIterator(strategy(facades.get(ctx), facades.get(s))));
 
@@ -34,8 +34,8 @@ const __evaluate = function* agast(ctx, strategy, options = {}) {
 
     if (co.done) break;
 
-    const sourceInstr = co.value;
-    const instr = reifyExpression(sourceInstr);
+    const instr = co.value;
+
     let returnValue = undefined;
 
     const { verb, arguments: args = [] } = instr;
@@ -67,7 +67,9 @@ const __evaluate = function* agast(ctx, strategy, options = {}) {
       }
 
       case 'advance': {
-        const { 0: terminal, 1: options } = args;
+        const { 0: embeddedTerminal, 1: options } = args;
+
+        const terminal = embeddedTerminal.value;
 
         if (
           s.held &&
@@ -83,24 +85,16 @@ const __evaluate = function* agast(ctx, strategy, options = {}) {
         switch (terminal?.type || 'Null') {
           case 'DoctypeTag': {
             const { attributes } = terminal.value;
-            const doctypeTag = buildDoctypeTag(attributes);
-            const rootPath = Path.from(ctx);
 
             if (s.path) {
               throw new Error();
             }
 
-            s.path = rootPath;
-
-            yield* s.emit(doctypeTag);
-
-            returnValue = doctypeTag;
+            returnValue = s.advance(buildDoctypeTag(attributes));
             break;
           }
 
           case 'Literal': {
-            const literal = buildLiteral(terminal.value);
-
             if (!s.node.flags.token) {
               throw new Error('literals must occur inside tokens');
             }
@@ -109,36 +103,22 @@ const __evaluate = function* agast(ctx, strategy, options = {}) {
               throw new Error('Cannot consume input while hold register is full');
             }
 
-            yield* s.emit(literal);
-
-            returnValue = literal;
+            returnValue = s.advance(buildLiteral(terminal.value));
             break;
           }
 
           case 'Reference': {
             const { name, isArray } = terminal.value;
 
-            const tag = buildReference(name, isArray);
-
             if (s.result.type === 'Reference') {
               throw new Error('A reference must have a non-reference value');
             }
 
-            if (!s.path.depth) {
-              throw new Error();
-            }
-
-            if (s.node.flags.token) {
-              throw new Error();
+            if (s.node?.flags.token) {
+              throw new Error('A token node cannot contain a reference');
             }
 
-            s.node.resolver.consume(tag);
-
-            s.path = s.path.push(ctx, tag);
-
-            yield* s.emit(tag);
-
-            returnValue = tag;
+            returnValue = s.advance(buildReference(name, isArray));
             break;
           }
 
@@ -147,36 +127,7 @@ const __evaluate = function* agast(ctx, strategy, options = {}) {
 
             if (reference?.type !== 'Reference') throw new Error();
 
-            s.path = s.path.parent;
-
-            const gapTag = buildGap();
-
-            s.held = null;
-
-            yield* s.emit(gapTag);
-
-            returnValue = gapTag;
-            break;
-          }
-
-          case 'Shift': {
-            const { tagNodes, prevTerminals } = ctx;
-            const tag = buildShift();
-
-            const finishedNode = tagNodes.get(s.result);
-            const finishedPath = finishedNode.path;
-
-            s.held = { node: finishedNode, path: finishedPath };
-
-            if (!finishedNode.openTag.value.flags.expression) {
-              throw new Error();
-            }
-
-            s.path = finishedPath;
-
-            yield* s.emit(tag);
-
-            returnValue = tag;
+            returnValue = s.advance(buildGap());
             break;
           }
 
@@ -185,127 +136,39 @@ const __evaluate = function* agast(ctx, strategy, options = {}) {
 
             if (reference?.type !== 'Reference') throw new Error();
 
-            s.path = s.path.parent;
+            returnValue = s.advance(buildNull());
+            break;
+          }
 
-            const null_ = buildNull();
+          case 'Shift': {
+            const finishedNode = s.nodeForTag(s.result);
 
-            yield* s.emit(null_);
+            if (!getOpenTag(finishedNode).value.flags.expression) {
+              throw new Error();
+            }
 
-            returnValue = null_;
+            returnValue = s.advance(buildShift());
             break;
           }
 
           case 'OpenNodeTag': {
-            const { flags, language, type, intrinsicValue, attributes } = terminal.value;
-            const { unboundAttributes } = options || {};
-            const reference = s.result;
-            const openTag = buildNodeOpenTag(flags, language, type, intrinsicValue, attributes);
-
-            if (!type) {
-              s.node = Node.from(s.path, openTag);
-              ctx.tagNodes.set(openTag, s.node);
-            } else {
-              if (!flags.trivia && !flags.escape && s.node.type) {
-                if (
-                  reference.type !== 'Reference' &&
-                  reference.type !== 'Shift' &&
-                  reference.type !== 'OpenFragmentTag'
-                ) {
-                  throw new Error('Invalid location for OpenNodeTag');
-                }
-              }
-
-              const { flags: openFlags } = openTag.value;
-
-              if (!(openFlags.trivia || openFlags.escape) && !s.path.depth) {
-                const tag = buildReference('root', false);
-                s.path = s.path.push(ctx, tag);
-                s.node.resolver.consume(tag);
-              }
-
-              const newNode = s.node.push(s.path, openTag);
-
-              newNode.unboundAttributes = new Set(unboundAttributes);
-
-              ctx.tagNodes.set(openTag, newNode);
-
-              if (!intrinsicValue) {
-                s.node = newNode;
-              } else {
-                s.path = s.path.parent;
-
-                if (s.path.depth > 1) {
-                  const { properties } = s.node;
-                  const { name: refName, isArray } = reference.value;
-
-                  if (!isArray) {
-                    properties.set(refName, [openTag, openTag]);
-                  } else {
-                    if (!properties.has(refName)) {
-                      properties.set(refName, []);
-                    }
-                    properties.get(refName).push([openTag, openTag]);
-                  }
-                }
-              }
-            }
+            const { flags, language, type, attributes } = terminal.value;
 
-            yield* s.emit(openTag);
+            if (language && !language.startsWith('https://')) {
+              throw new Error('Expected an absolute-language tag');
+            }
 
-            returnValue = openTag;
+            returnValue = s.advance(
+              buildNodeOpenTag(flags, language, type, attributes),
+              deembedExpression(options),
+            );
             break;
           }
 
           case 'CloseNodeTag': {
             const { type, language } = terminal.value;
-            const { openTag } = s.node;
-            const { flags, type: openType } = openTag.value;
-
-            const closeTag = buildNodeCloseTag(type, language);
-
-            if (openType) {
-              if (s.node.unboundAttributes?.size)
-                throw new Error('Grammar failed to bind all attributes');
-
-              if (!type) throw new Error(`CloseNodeTag must have type`);
-
-              if (s.path.depth > 1 && type !== openType)
-                throw new Error(
-                  `Grammar close {type: ${type}} did not match open {type: ${openType}}`,
-                );
 
-              if (!flags.escape && !flags.trivia) {
-                const { name: refName, isArray } = s.path.reference.value;
-
-                // is this right?
-                if (s.path.depth > 2) {
-                  const { properties } = s.node.parent;
-
-                  if (!isArray) {
-                    properties.set(refName, [openTag, closeTag]);
-                  } else {
-                    if (!properties.has(refName)) {
-                      properties.set(refName, []);
-                    }
-                    properties.get(refName).push([openTag, closeTag]);
-                  }
-                }
-              }
-
-              ctx.tagNodes.set(closeTag, s.node);
-
-              s.node.closeTag = closeTag;
-
-              s.node = s.node.parent;
-
-              if (!(flags.trivia || flags.escape)) {
-                s.path = s.path.parent;
-              }
-            }
-
-            yield* s.emit(closeTag, flags.expression);
-
-            returnValue = closeTag;
+            returnValue = s.advance(buildNodeCloseTag(type, language));
             break;
           }
 
@@ -313,19 +176,21 @@ const __evaluate = function* agast(ctx, strategy, options = {}) {
             throw new Error();
         }
 
+        yield* s.emit();
+
         break;
       }
 
       case 'bindAttribute': {
         const { 0: key, 1: value } = args;
 
-        const { unboundAttributes } = s.node;
+        const { unboundAttributes } = s;
 
         if (!unboundAttributes || !unboundAttributes.has(key)) {
           throw new Error('No unbound attribute to bind');
         }
 
-        if (!s.node.openTag.value.type) {
+        if (!s.node.type) {
           throw new Error('Cannot bind attribute to fragment');
         }
 
@@ -342,37 +207,43 @@ const __evaluate = function* agast(ctx, strategy, options = {}) {
 
         unboundAttributes.delete(key);
 
-        const { openTag } = s.node;
+        const openTag = s.node.children[0];
 
         if (value != null) {
-          const { flags, language, type, intrinsicValue } = openTag.value;
+          const { flags, language, type } = openTag.value;
           const attributes = { ...openTag.value.attributes, [key]: value };
-          const newStartTag = buildNodeOpenTag(flags, language, type, intrinsicValue, attributes);
+          const newOpenTag = buildNodeOpenTag(flags, language, type, attributes);
 
-          let startNext = ctx.nextTerminals.get(openTag);
+          let openNext = ctx.nextTerminals.get(openTag);
           let startPrev = ctx.prevTerminals.get(openTag);
 
-          ctx.prevTerminals.set(newStartTag, startPrev);
-          ctx.nextTerminals.set(startPrev, newStartTag);
+          ctx.prevTerminals.set(newOpenTag, startPrev);
+          ctx.nextTerminals.set(startPrev, newOpenTag);
+
+          if (s.node !== s.tagNodes.get(openTag)) throw new Error();
+          if (s.path !== s.tagPaths.get(openTag)) throw new Error();
 
-          ctx.tagNodes.set(newStartTag, s.node);
+          s.node.attributes = attributes;
 
-          if (startNext) {
-            ctx.nextTerminals.set(newStartTag, startNext);
-            ctx.prevTerminals.set(startNext, newStartTag);
+          s.tagNodes.set(newOpenTag, s.node);
+          s.tagPaths.set(newOpenTag, s.path);
+
+          if (openNext) {
+            ctx.nextTerminals.set(newOpenTag, openNext);
+            ctx.prevTerminals.set(openNext, newOpenTag);
           } else {
             // could this terminal be stored anywhere else?
-            s.result = newStartTag;
+            s.result = newOpenTag;
           }
 
-          s.node.openTag = newStartTag;
+          s.node.children[0] = newOpenTag;
         }
 
         if (!unboundAttributes.size) {
           yield* s.emit();
         }
 
-        returnValue = s.node.openTag;
+        returnValue = getRange(s.node);
         break;
       }
 
@@ -401,8 +272,6 @@ const __evaluate = function* agast(ctx, strategy, options = {}) {
     co.advance(returnValue);
   }
 
-  s.path = s.path.parent;
-
   if (s.depth > 0) {
     throw new Error('Did not unwind state stack');
   }
@@ -410,4 +279,6 @@ const __evaluate = function* agast(ctx, strategy, options = {}) {
   if (s.path?.depth > 0) {
     throw new Error('Did not unwind path stack');
   }
+
+  return s.nodeForTag(s.result);
 };

package/lib/node.js

@@ -0,0 +1,36 @@
+import { getRange, getOpenTag, getCloseTag } from '@bablr/agast-helpers/tree';
+import { facades, actuals } from './facades.js';
+
+export const NodeFacade = class AgastNodeFacade {
+  constructor(path) {
+    facades.set(path, this);
+  }
+
+  get language() {
+    return actuals.get(this).language;
+  }
+
+  get type() {
+    return actuals.get(this).type;
+  }
+
+  get range() {
+    return getRange(actuals.get(this));
+  }
+
+  get openTag() {
+    return getOpenTag(actuals.get(this));
+  }
+
+  get closeTag() {
+    return getCloseTag(actuals.get(this));
+  }
+
+  get flags() {
+    return actuals.get(this).flags;
+  }
+
+  get attributes() {
+    return actuals.get(this).attributes;
+  }
+};

package/lib/path.js

@@ -1,58 +1,7 @@
 import { WeakStackFrame } from '@bablr/weak-stack';
-import { Resolver } from '@bablr/agast-helpers/tree';
 import { skipToDepth, buildSkips } from './utils/skip.js';
 import { facades, actuals } from './facades.js';
 
-export const NodeFacade = class AgastNodeFacade {
-  constructor(path) {
-    facades.set(path, this);
-  }
-
-  get language() {
-    return actuals.get(this).language;
-  }
-
-  get type() {
-    return actuals.get(this).type;
-  }
-
-  get path() {
-    return actuals.get(this).path;
-  }
-
-  get parent() {
-    return facades.get(actuals.get(this).parent);
-  }
-
-  get range() {
-    return actuals.get(this).range;
-  }
-
-  get openTag() {
-    return actuals.get(this).openTag;
-  }
-
-  get closeTag() {
-    return actuals.get(this).closeTag;
-  }
-
-  get depth() {
-    return actuals.get(this).depth;
-  }
-
-  get flags() {
-    return actuals.get(this).flags;
-  }
-
-  get attributes() {
-    return actuals.get(this).attributes;
-  }
-
-  at(depth) {
-    return facades.get(actuals.get(this).at(depth));
-  }
-};
-
 export const PathFacade = class AgastPathFacade {
   constructor(path) {
     facades.set(path, this);
@@ -83,85 +32,6 @@ export const PathFacade = class AgastPathFacade {
   }
 };
 
-export const Node = class AgastNode extends WeakStackFrame {
-  static from(path, openTag) {
-    return AgastNode.create(path, openTag);
-  }
-
-  constructor(
-    path,
-    openTag,
-    closeTag = null,
-    properties = new Map(),
-    resolver = new Resolver(),
-    unboundAttributes = null,
-  ) {
-    super();
-
-    this.path = path;
-    this.openTag = openTag;
-    this.closeTag = closeTag;
-    this.properties = properties;
-    this.resolver = resolver;
-    this.unboundAttributes = unboundAttributes;
-
-    buildSkips(this);
-  }
-
-  get language() {
-    return this.openTag.value?.language;
-  }
-
-  get type() {
-    return this.openTag.value?.type || null;
-  }
-
-  get flags() {
-    return this.openTag.value?.flags || {};
-  }
-
-  get attributes() {
-    return this.openTag.value?.attributes || {};
-  }
-
-  at(depth) {
-    return skipToDepth(depth, this);
-  }
-
-  *parents(includeSelf = false) {
-    if (includeSelf) yield this;
-    let parent = this;
-    while ((parent = parent.parent)) {
-      yield parent;
-    }
-  }
-
-  branch() {
-    const { path, openTag, closeTag, properties, resolver, unboundAttributes } = this;
-
-    return this.replace(
-      path,
-      openTag,
-      closeTag,
-      new Map(properties), // there is probably a better way
-      resolver.branch(),
-      new Set(unboundAttributes),
-    );
-  }
-
-  accept(node) {
-    this.path = node.path;
-    this.openTag = node.openTag;
-    this.closeTag = node.closeTag;
-    this.properties = node.properties;
-    this.unboundAttributes = node.unboundAttributes;
-
-    this.resolver.accept(node.resolver);
-
-    return this;
-  }
-};
-
 export const Path = class AgastPath extends WeakStackFrame {
   static from(context, tag) {
     return Path.create(context, tag);
@@ -170,7 +40,7 @@ export const Path = class AgastPath extends WeakStackFrame {
   constructor(context, reference) {
     super();
 
-    if (reference && reference.type !== 'Reference') {
+    if (reference && reference.type !== 'Reference' && reference.type !== 'DoctypeTag') {
       throw new Error('Invalid reference for path');
     }
 
@@ -182,6 +52,14 @@ export const Path = class AgastPath extends WeakStackFrame {
     new PathFacade(this);
   }
 
+  get name() {
+    return this.reference?.value.name || '[anonymous]';
+  }
+
+  get isArray() {
+    return this.reference?.value.isArray || false;
+  }
+
   at(depth) {
     return skipToDepth(depth, this);
   }

package/lib/state.js

@@ -1,7 +1,58 @@
+import emptyStack from '@iter-tools/imm-stack';
 import { WeakStackFrame } from '@bablr/weak-stack';
-import { startsDocument } from '@bablr/agast-helpers/stream';
+import {
+  Resolver,
+  add,
+  createNode,
+  getOpenTag,
+  getCloseTag,
+  branchNode,
+  acceptNode,
+  finalizeNode,
+  getRoot,
+} from '@bablr/agast-helpers/tree';
+import {
+  buildBeginningOfStreamToken,
+  buildEmbeddedNode,
+  nodeFlags,
+} from '@bablr/agast-vm-helpers/internal-builders';
+import * as sym from '@bablr/agast-helpers/symbols';
 import { facades, actuals } from './facades.js';
-import { buildBeginningOfStreamToken } from '@bablr/agast-helpers/builders';
+import { Path } from './path.js';
+
+const { hasOwn } = Object;
+
+const arrayLast = (arr) => arr[arr.length - 1];
+
+const createNodeWithState = (startTag, options = {}) => {
+  const { unboundAttributes } = options;
+  const node = createNode(startTag);
+  nodeStates.set(node, {
+    resolver: new Resolver(node),
+    unboundAttributes: new Set(unboundAttributes || []),
+  });
+  return node;
+};
+
+const symbolTypeFor = (type) => {
+  // prettier-ignore
+  switch (type) {
+    case 'Null': return sym.null;
+    case 'Gap': return sym.gap;
+    default: throw new Error();
+  }
+};
+
+const buildStubNode = (tag) => {
+  return {
+    flags: nodeFlags,
+    language: null,
+    type: symbolTypeFor(tag.type),
+    children: [tag],
+    properties: {},
+    attributes: {},
+  };
+};
 
 export const StateFacade = class AgastStateFacade {
   constructor(state) {
@@ -28,6 +79,10 @@ export const StateFacade = class AgastStateFacade {
     return actuals.get(this).node;
   }
 
+  get parentNode() {
+    return actuals.get(this).parentNode;
+  }
+
   get holding() {
     return actuals.get(this).holding;
   }
@@ -39,41 +94,96 @@ export const StateFacade = class AgastStateFacade {
   get ctx() {
     return this.context;
   }
+
+  nodeForPath(path) {
+    return actuals.get(this).nodeForPath(path);
+  }
+
+  pathForTag(tag) {
+    return actuals.get(this).pathForTag(tag);
+  }
+
+  nodeForTag(tag) {
+    return actuals.get(this).nodeForTag(tag);
+  }
 };
 
+export const nodeStates = new WeakMap();
+
 export const State = class AgastState extends WeakStackFrame {
   constructor(
     context,
+    expressions = emptyStack,
     path = null,
     node = null,
     result = buildBeginningOfStreamToken(),
     emitted = null,
     held = null,
+    internalContext = {
+      pathNodes: new WeakMap(),
+      tagPaths: new WeakMap(),
+      tagNodes: new WeakMap(),
+    },
   ) {
     super();
 
     if (!context) throw new Error('invalid args to tagState');
 
     this.context = context;
+    this.expressions = expressions;
     this.path = path;
     this.node = node;
     this.result = result;
     this.emitted = emitted;
     this.held = held;
+    this.internalContext = internalContext;
 
     new StateFacade(this);
   }
 
-  static from(context) {
-    return State.create(context, null);
+  static from(context, expressions = []) {
+    return State.create(context, emptyStack.push(...[...expressions].reverse()));
+  }
+
+  get pathNodes() {
+    return this.internalContext.pathNodes;
+  }
+
+  get tagPaths() {
+    return this.internalContext.tagPaths;
+  }
+
+  get tagNodes() {
+    return this.internalContext.tagNodes;
+  }
+
+  get unboundAttributes() {
+    return nodeStates.get(this.node).unboundAttributes;
+  }
+
+  get resolver() {
+    return nodeStates.get(this.node).resolver;
   }
 
   get holding() {
     return !!this.held;
   }
 
-  *emit(terminal, suppressEmit) {
-    const { prevTerminals, nextTerminals, tagNodes } = this.context;
+  nodeForPath(path) {
+    return this.pathNodes.get(path);
+  }
+
+  pathForTag(ref) {
+    return this.tagPaths.get(ref);
+  }
+
+  nodeForTag(tag) {
+    return this.tagNodes.get(tag);
+  }
+
+  advance(terminal, options = {}) {
+    const ctx = this.context;
+    const { prevTerminals, nextTerminals } = ctx;
 
     if (terminal) {
       if (prevTerminals.has(terminal)) {
@@ -90,24 +200,203 @@ export const State = class AgastState extends WeakStackFrame {
       prevTerminals.set(terminal, this.result);
       nextTerminals.set(this.result, terminal);
 
-      this.result = terminal;
+      switch (terminal.type) {
+        case 'DoctypeTag': {
+          this.path = Path.from(ctx, terminal);
+
+          this.tagPaths.set(terminal, this.path);
+          break;
+        }
+
+        case 'OpenNodeTag': {
+          const openTag = terminal;
+          const { type, flags } = terminal.value;
+          this.node = createNodeWithState(terminal, options);
+
+          const reference = this.result;
+
+          this.node.children.push(terminal);
+
+          if (!type) {
+            this.node.attributes = this.result.value.attributes;
+          } else {
+            if (!flags.trivia && !flags.escape) {
+              if (
+                reference.type !== 'Reference' &&
+                reference.type !== 'Shift' &&
+                reference.type !== 'OpenNodeTag' &&
+                !reference.value.type
+              ) {
+                throw new Error('Invalid location for OpenNodeTag');
+              }
+            } else {
+              this.path = this.path.push(ctx, null);
+            }
+          }
+
+          this.pathNodes.set(this.node, this.path);
+          this.pathNodes.set(this.path, this.node);
+
+          this.tagNodes.set(openTag, this.node);
+          this.tagPaths.set(openTag, this.path);
+          break;
+        }
+
+        case 'CloseNodeTag': {
+          const openTag = this.node.children[0];
+          const { flags, type: openType } = openTag.value;
+          const closeTag = terminal;
+          const { type } = closeTag.value;
+
+          this.node.children.push(terminal);
+
+          if (openType) {
+            if (this.node.unboundAttributes?.size)
+              throw new Error('Grammar failed to bind all attributes');
+
+            if (!type) throw new Error(`CloseNodeTag must have type`);
+
+            if (type !== openType)
+              throw new Error(
+                `Grammar close {type: ${type}} did not match open {type: ${openType}}`,
+              );
 
-      if (!this.emitted) {
-        if (!startsDocument(terminal)) throw new Error();
-        this.emitted = terminal;
-        yield terminal;
+            if (!flags.escape && !flags.trivia) {
+              const { name: refName, isArray } = this.path.reference.value;
+
+              const { properties } = this.parentNode;
+
+              if (!isArray) {
+                properties[refName] = this.node;
+              } else {
+                if (!hasOwn(properties, refName)) {
+                  properties[refName] = [];
+                }
+                properties[refName].push(this.node);
+              }
+            } else {
+              this.parentNode.children.push(buildEmbeddedNode(this.node));
+            }
+          }
+
+          this.tagNodes.set(closeTag, this.node);
+          this.tagPaths.set(closeTag, this.path);
+
+          finalizeNode(this.node);
+
+          this.node = this.parentNode;
+          this.path = this.path.parent;
+          break;
+        }
+
+        case 'Reference': {
+          if (this.path.depth) {
+            nodeStates.get(this.node).resolver.consume(terminal);
+          }
+
+          this.node.children.push(terminal);
+
+          this.path = this.path.push(ctx, terminal);
+
+          this.tagPaths.set(terminal, this.path);
+          break;
+        }
+
+        case 'Gap': {
+          this.tagPaths.set(terminal, this.path);
+
+          let target;
+          let ref = arrayLast(this.node.children);
+
+          if (ref.type !== 'Reference') throw new Error();
+
+          if (this.held) {
+            target = this.held.node;
+
+            this.held = null;
+          } else if (this.expressions.size) {
+            const expression = this.expressions.value;
+            target = getRoot(expression);
+            this.expressions = this.expressions.pop();
+          } else {
+            target = buildStubNode(terminal);
+          }
+
+          this.pathNodes.set(this.pathForTag(ref), target);
+          add(this.node, ref, target);
+
+          this.path = this.path.parent;
+          break;
+        }
+
+        case 'Null': {
+          this.tagPaths.set(terminal, this.path);
+
+          const { properties } = this.node;
+          const { isArray, name } = this.result.value;
+
+          const newNode = buildStubNode(terminal);
+
+          if (!hasOwn(properties, name)) {
+            // TODO is this behavior right
+            properties[name] = isArray ? [] : newNode;
+          }
+
+          this.pathNodes.set(this.path, newNode);
+
+          this.path = this.path.parent;
+          break;
+        }
+
+        case 'Shift': {
+          const finishedNode = this.nodeForTag(this.result);
+          const ref = ctx.getPreviousTerminal(getOpenTag(finishedNode));
+          const finishedPath = this.pathForTag(ref);
+          const { properties } = this.node;
+
+          this.pathNodes.set(finishedPath, null);
+
+          this.held = { node: finishedNode, path: finishedPath };
+
+          let node = properties[ref.value.name];
+
+          if (ref.value.isArray) {
+            node = arrayLast(node);
+            properties[ref.value.name].pop();
+          } else {
+            properties[ref.value.name] = null;
+          }
+
+          this.path = finishedPath;
+          break;
+        }
+
+        case 'Literal': {
+          this.node.children.push(terminal);
+          break;
+        }
+
+        default:
+          throw new Error();
       }
     }
 
-    if (!this.depth && !suppressEmit) {
-      let emittable = nextTerminals.get(this.emitted);
+    this.result = terminal;
+
+    return terminal;
+  }
+
+  *emit() {
+    const { nextTerminals } = this.context;
+    if (!this.depth) {
+      let emittable = this.emitted ? nextTerminals.get(this.emitted) : this.result;
 
       while (
         emittable &&
         !(
           emittable.type === 'OpenNodeTag' &&
           emittable.value.type &&
-          tagNodes.get(emittable).unboundAttributes?.size
+          nodeStates.get(this.nodeForTag(emittable)).unboundAttributes?.size
         )
       ) {
         yield emittable;
@@ -115,8 +404,6 @@ export const State = class AgastState extends WeakStackFrame {
         emittable = nextTerminals.get(this.emitted);
       }
     }
-
-    return terminal;
   }
 
   get ctx() {
@@ -131,10 +418,28 @@ export const State = class AgastState extends WeakStackFrame {
     return !!this.parent;
   }
 
+  get parentNode() {
+    return this.pathNodes.get(this.path.parent);
+  }
+
   branch() {
-    const { context, path, node, result, emitted, held } = this;
+    const { context, expressions, path, node, result, emitted, held, internalContext } = this;
+    const { pathNodes } = internalContext;
+
+    const newNode = node && branchNode(node);
+
+    const nodeState = nodeStates.get(node);
+
+    pathNodes.set(path, newNode);
 
-    return this.push(context, path, node.branch(), result, emitted, held);
+    nodeStates.set(newNode, { ...nodeState, resolver: nodeState.resolver.branch() });
+
+    const nodeOpen = getOpenTag(node);
+    const nodeClose = getCloseTag(node);
+    if (nodeOpen) this.tagNodes.set(nodeOpen, newNode);
+    if (nodeClose) this.tagNodes.set(nodeClose, newNode);
+
+    return this.push(context, expressions, path, newNode, result, emitted, held, internalContext);
   }
 
   accept() {
@@ -144,10 +449,17 @@ export const State = class AgastState extends WeakStackFrame {
       return null;
     }
 
-    parent.node.accept(this.node);
+    if (this.node && parent.node) {
+      acceptNode(parent.node, this.node);
+      const nodeState = nodeStates.get(this.node);
+      Object.assign(nodeStates.get(parent.node), nodeState);
+    } else {
+      parent.node = this.node;
+    }
 
     // emitted isn't used here and probably doesn't need to be part of state
 
+    parent.expressions = this.expressions;
     parent.result = this.result;
     parent.held = this.held;
     parent.path = this.path;
@@ -156,12 +468,17 @@ export const State = class AgastState extends WeakStackFrame {
   }
 
   reject() {
-    const { parent, context } = this;
+    const { parent, context, pathNodes, tagNodes } = this;
 
     if (!parent) throw new Error('rejected root state');
 
     context.nextTerminals.delete(parent.result);
 
+    pathNodes.set(parent.path, parent.node);
+
+    if (getOpenTag(parent.node)) tagNodes.set(getOpenTag(parent.node), parent.node);
+    if (getCloseTag(parent.node)) tagNodes.set(getCloseTag(parent.node), parent.node);
+
     return parent;
   }
 };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@bablr/agast-vm",
   "description": "A VM providing DOM-like guarantees about agAST trees",
-  "version": "0.2.0",
+  "version": "0.4.0",
   "author": "Conrad Buck<conartist6@gmail.com>",
   "type": "module",
   "files": [
@@ -11,24 +11,20 @@
     ".": "./lib/index.js"
   },
   "sideEffects": false,
-  "scripts": {
-    "test": "node ./test/runner.js"
-  },
   "dependencies": {
-    "@bablr/agast-helpers": "0.1.6",
-    "@bablr/agast-vm-helpers": "0.1.5",
+    "@bablr/agast-helpers": "0.3.0",
+    "@bablr/agast-vm-helpers": "0.3.0",
     "@bablr/coroutine": "0.1.0",
-    "@bablr/weak-stack": "0.1.0"
+    "@bablr/weak-stack": "0.1.0",
+    "@iter-tools/imm-stack": "1.1.0"
   },
   "devDependencies": {
     "@bablr/agast-vm-strategy-passthrough": "github:bablr-lang/agast-vm-strategy-passthrough#2bd3a0c7311037af92c5b81941c79161499f6c9e",
     "@bablr/eslint-config-base": "github:bablr-lang/eslint-config-base#49f5952efed27f94ee9b94340eb1563c440bf64e",
-    "@bablr/strategy_enhancer-debug-log": "0.1.1",
     "enhanced-resolve": "^5.12.0",
     "eslint": "^8.32.0",
     "eslint-import-resolver-enhanced-resolve": "^1.0.5",
     "eslint-plugin-import": "^2.27.5",
-    "expect": "29.7.0",
     "iter-tools-es": "^7.3.1",
     "prettier": "^2.6.2"
   },