@bablr/bablr-vm 0.13.0

package/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2022 Conrad Buck
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,3 @@
+# @bablr/vm
+
+The BABLR VM!

package/lib/context.js

@@ -0,0 +1,76 @@
+import { facades, actuals } from './facades.js';
+
+export const ContextFacade = class BABLRContextFacade {
+  get languages() {
+    return actuals.get(this).languages;
+  }
+
+  get grammars() {
+    return actuals.get(this).grammars;
+  }
+
+  get productionEnhancer() {
+    return actuals.get(this).productionEnhancer;
+  }
+
+  get agast() {
+    return actuals.get(this).agast;
+  }
+
+  getPreviousTerminal(token) {
+    return actuals.get(this).agast.getPreviousTerminal(token);
+  }
+
+  ownTerminalsFor(range) {
+    return actuals.get(this).agast.ownTerminalsFor(range);
+  }
+
+  allTerminalsFor(range) {
+    return actuals.get(this).agast.allTerminalsFor(range);
+  }
+
+  getCooked(range) {
+    return actuals.get(this).agast.getCooked(range);
+  }
+
+  reifyExpression(value) {
+    return actuals.get(this).agast.reifyExpression(value);
+  }
+
+  getProperty(node, name) {
+    return actuals.get(this).agast.getProperty(node, name);
+  }
+
+  sourceTextFor(range) {
+    return actuals.get(this).agast.sourceTextFor(range);
+  }
+
+  nodeForTag(tag) {
+    return actuals.get(this).agast.nodeForTag(tag);
+  }
+
+  unbox(value) {
+    return actuals.get(this).agast.unbox(value);
+  }
+};
+
+export const Context = class BABLRContext {
+  static from(agastContext, languages, productionEnhancer) {
+    return new Context(agastContext, languages, productionEnhancer);
+  }
+
+  constructor(agastContext, languages, productionEnhancer) {
+    this.agast = agastContext;
+    this.languages = languages;
+    this.productionEnhancer = productionEnhancer;
+
+    this.grammars = new WeakMap();
+    this.facade = new ContextFacade();
+
+    for (const { 1: language } of this.languages) {
+      this.grammars.set(language, new language.grammar());
+    }
+
+    facades.set(this, this.facade);
+  }
+};

package/lib/evaluate.js

@@ -0,0 +1,414 @@
+import { Coroutine } from '@bablr/coroutine';
+import { reifyExpression } from '@bablr/agast-vm-helpers';
+import { StreamGenerator } from '@bablr/agast-helpers/stream';
+import { resolveLanguage } from '@bablr/helpers/grammar';
+import { buildTokens } from './utils/token.js';
+import { formatType } from './utils/format.js';
+import { facades } from './facades.js';
+import { State } from './state.js';
+
+const nodeTopType = Symbol.for('@bablr/node');
+
+export const evaluate = (ctx, rootLanguage, rootSource, strategy) => {
+  return (agastCtx, agastState) => {
+    if (ctx.agast !== agastCtx) throw new Error();
+
+    if (rootLanguage !== ctx.languages.get(rootLanguage.canonicalURL)) {
+      throw new Error();
+    }
+
+    return new StreamGenerator(__evaluate(ctx, rootLanguage, rootSource, agastState, strategy));
+  };
+};
+
+const resolvedLanguages = new WeakMap();
+
+function updateSpans(ctx, s, node, phase) {
+  switch (phase) {
+    case 'open': {
+      const { attributes, flags } = node;
+      const { span: innerSpan, balanced, balancedSpan, balancer, openSpan } = attributes || {};
+
+      if (!flags.intrinsic && (balancer || balanced)) {
+        throw new Error('not implemented');
+      }
+
+      if (flags.intrinsic) {
+        if (s.path) {
+          if (balancedSpan) {
+            if (!balanced) throw new Error();
+
+            s.spans = s.spans.push({
+              type: 'Lexical',
+              name: balancedSpan,
+              path: s.path,
+              guard: balanced,
+            });
+          }
+
+          if (innerSpan) {
+            throw new Error();
+          }
+        }
+      }
+
+      if (openSpan) {
+        s.spans = s.spans.push({
+          type: 'Explicit',
+          name: openSpan,
+          path: s.path,
+          guard: null,
+        });
+      }
+
+      if (balancer) {
+        const balancedNode = s.balanced.value;
+
+        if (!s.balanced.size) throw new Error();
+
+        s.balanced = s.balanced.pop();
+
+        if (balancer && balancedNode.openTag.value.attributes.balancedSpan) {
+          s.spans = s.spans.pop();
+        }
+      }
+
+      if (balanced) {
+        s.balanced = s.balanced.push(ctx.agast.nodeForTag(s.result));
+      }
+
+      if (innerSpan) {
+        s.spans = s.spans.push({
+          type: 'Inner',
+          name: innerSpan,
+          path: s.path,
+          guard: null,
+        });
+      }
+
+      break;
+    }
+
+    case 'close': {
+      const { openTag, flags } = node;
+      const { attributes } = openTag.value;
+      const { balancedSpan, span: innerSpan, closeSpan, balanced } = attributes || {};
+
+      if (balancedSpan && !flags.intrinsic) {
+        if (!balanced) throw new Error();
+
+        s.spans = s.spans.push({
+          type: 'Lexical',
+          name: balancedSpan,
+          path: s.path,
+          guard: balanced,
+        });
+      }
+
+      if (closeSpan) {
+        if (s.spans.value.type !== 'Explicit') throw new Error();
+        s.spans = s.spans.pop();
+      }
+
+      if (innerSpan) {
+        s.spans = s.spans.pop();
+      }
+      break;
+    }
+    default:
+      throw new Error();
+  }
+}
+
+const __evaluate = function* bablrStrategy(ctx, rootLanguage, rootSource, agastState, strategy) {
+  let s = State.from(ctx, rootSource, agastState);
+
+  let co = new Coroutine(strategy(facades.get(s), facades.get(ctx)));
+
+  co.advance();
+
+  {
+    s.source.advance();
+
+    const sourceStep = s.source.fork.head.step;
+
+    if (sourceStep instanceof Promise) {
+      yield sourceStep;
+    }
+  }
+
+  for (;;) {
+    if (co.current instanceof Promise) {
+      co.current = yield co.current;
+    }
+
+    if (co.done) break;
+
+    const sourceInstr = co.value;
+    const instr = reifyExpression(sourceInstr);
+    let returnValue = undefined;
+
+    const { verb } = instr;
+
+    switch (verb) {
+      case 'advance': {
+        const {
+          arguments: { 0: terminal = [] },
+        } = instr;
+
+        switch (terminal?.type || 'Null') {
+          case 'DoctypeTag': {
+            const doctypeTag = yield sourceInstr;
+
+            returnValue = doctypeTag;
+            break;
+          }
+
+          case 'OpenNodeTag': {
+            const { flags, type, language: tagLanguage, intrinsicValue } = terminal.value;
+
+            const reference = s.result;
+
+            if (
+              s.path.depth > 1 &&
+              !flags.trivia &&
+              !flags.escape &&
+              reference?.type !== 'Reference' &&
+              reference?.type !== 'OpenFragmentTag'
+            ) {
+              throw new Error('Invalid location for OpenNodeTag');
+            }
+
+            const oldNode = s.node;
+
+            const openTag = yield sourceInstr;
+
+            const resolvedLanguage =
+              oldNode.depth > 1 ? resolvedLanguages.get(oldNode) : rootLanguage;
+            const nextResolvedLanguage = resolveLanguage(resolvedLanguage, tagLanguage);
+
+            if (!nextResolvedLanguage) {
+              throw new Error(`Resolve failed { language: ${tagLanguage} }`);
+            }
+
+            const grammar = ctx.grammars.get(nextResolvedLanguage);
+            const isNode = grammar.covers.get(nodeTopType).has(type);
+
+            let intrinsicResult = intrinsicValue && s.guardedMatch(terminal.value);
+
+            if (intrinsicResult instanceof Promise) {
+              intrinsicResult = yield intrinsicResult;
+            }
+
+            updateSpans(ctx, s, intrinsicValue ? ctx.agast.nodeForTag(openTag) : s.node, 'open');
+
+            if (intrinsicValue) {
+              if (!intrinsicResult) {
+                throw new Error('advance failed to match an intrinsic node');
+              }
+
+              const sourceStep = s.source.advance(intrinsicResult.length);
+
+              if (sourceStep instanceof Promise) {
+                yield sourceStep;
+              }
+
+              updateSpans(ctx, s, ctx.agast.nodeForTag(openTag), 'close');
+            } else {
+              if (isNode) {
+                resolvedLanguages.set(s.node, nextResolvedLanguage);
+              } else {
+                resolvedLanguages.set(s.node, resolvedLanguage);
+              }
+            }
+
+            returnValue = openTag;
+            break;
+          }
+
+          case 'OpenFragmentTag': {
+            const openTag = yield sourceInstr;
+
+            resolvedLanguages.set(s.node, rootLanguage);
+
+            returnValue = openTag;
+            break;
+          }
+
+          case 'CloseNodeTag': {
+            const { node } = s;
+
+            const endTag = yield sourceInstr;
+
+            if (s.path) {
+              updateSpans(ctx, s, node, 'close');
+            }
+
+            returnValue = endTag;
+            break;
+          }
+
+          case 'CloseFragmentTag': {
+            returnValue = yield sourceInstr;
+
+            if (!s.path) {
+              if (!s.source.done) {
+                throw new Error('Parser failed to consume input');
+              }
+
+              if (s.balanced.size) {
+                throw new Error('Parser did not match all balanced nodes');
+              }
+            }
+            break;
+          }
+
+          case 'Literal': {
+            const { value: pattern } = terminal;
+
+            let result = s.guardedMatch(pattern);
+
+            if (result instanceof Promise) {
+              result = yield result;
+            }
+
+            if (result) {
+              let sourceStep = s.source.advance(result.length);
+
+              if (sourceStep instanceof Promise) {
+                sourceStep = yield sourceStep;
+              }
+
+              returnValue = yield sourceInstr;
+            } else {
+              throw new Error('Failed to advance literal');
+            }
+            break;
+          }
+
+          case 'Gap': {
+            if (s.source.value == null && !s.source.done) {
+              const sourceStep = s.source.advance(1);
+
+              if (sourceStep instanceof Promise) {
+                yield sourceStep;
+              }
+
+              returnValue = yield sourceInstr;
+            } else {
+              throw new Error('Failed to advance gap');
+            }
+            break;
+          }
+
+          default: {
+            returnValue = yield sourceInstr;
+            break;
+          }
+        }
+
+        break;
+      }
+
+      case 'match': {
+        let { arguments: { 0: pattern } = [] } = instr;
+
+        let result = s.guardedMatch(pattern);
+
+        if (result instanceof Promise) {
+          result = yield result;
+        }
+
+        returnValue = result && ctx.agast.buildRange(buildTokens(result));
+        break;
+      }
+
+      case 'shift': {
+        s.source.shift();
+
+        yield sourceInstr;
+
+        break;
+      }
+
+      case 'unshift': {
+        s.source.unshift();
+
+        yield sourceInstr;
+
+        break;
+      }
+
+      case 'branch': {
+        const baseState = s;
+        let { context, source, agast, balanced, spans, node } = baseState;
+
+        agast = yield sourceInstr;
+
+        s = s.push(context, source.branch(), agast, balanced, spans);
+
+        resolvedLanguages.set(s.node, resolvedLanguages.get(node));
+
+        returnValue = facades.get(s);
+        break;
+      }
+
+      case 'accept': {
+        const accepted = s;
+
+        const agastState = yield sourceInstr;
+
+        s = s.parent;
+
+        if (!s) {
+          throw new Error('accepted the root state');
+        }
+
+        s.spans = accepted.spans;
+        s.balanced = accepted.balanced;
+
+        s.source.accept(accepted.source);
+        s.agast = agastState;
+
+        returnValue = facades.get(s);
+        break;
+      }
+
+      case 'reject': {
+        const rejectedState = s;
+
+        yield sourceInstr;
+
+        s = s.parent;
+
+        if (!s) throw new Error('rejected root state');
+
+        rejectedState.source.reject();
+
+        returnValue = facades.get(s);
+        break;
+      }
+
+      case 'bindAttribute': {
+        returnValue = yield sourceInstr;
+        break;
+      }
+
+      case 'getState': {
+        returnValue = facades.get(s);
+        break;
+      }
+
+      case 'getContext': {
+        returnValue = facades.get(ctx);
+        break;
+      }
+
+      default: {
+        throw new Error(`Unexpected call of {type: ${formatType(verb)}}`);
+      }
+    }
+
+    co.advance(returnValue);
+  }
+};

package/lib/facades.js

@@ -0,0 +1,3 @@
+import { buildFacadeLayer } from '@bablr/agast-vm-helpers/facades';
+
+export const { facades, actuals } = buildFacadeLayer();

package/lib/index.js

@@ -0,0 +1,4 @@
+export { evaluate } from './evaluate.js';
+
+export { Source } from './source.js';
+export { Context } from './context.js';

package/lib/source.js

@@ -0,0 +1,256 @@
+import { WeakStackFrame } from '@bablr/weak-stack';
+import { maybeWait, getStreamIterator } from '@bablr/agast-helpers/stream';
+import { facades, actuals } from './facades.js';
+
+// Queue item instances are shared between all forks.
+class QueueItem {
+  constructor(step) {
+    this.step = step;
+    this.next = null;
+  }
+}
+
+class Exchange {
+  constructor(iterator) {
+    if (!iterator) throw new Error();
+
+    this.iterator = iterator;
+    this.tail = new QueueItem(null);
+    this.head = this.tail;
+    this.forks = 0;
+  }
+
+  static from(iterable) {
+    return new Exchange(getStreamIterator(iterable));
+  }
+
+  get isWaiting() {
+    return this.head.step instanceof Promise;
+  }
+
+  allocateFork(fork) {
+    const { head = this.tail, exchange = this, current } = fork || {};
+    ++this.forks;
+    return new Fork(head, exchange, current);
+  }
+
+  advance() {
+    this.tail = this.tail.next;
+  }
+
+  fetch() {
+    let step = this.iterator.next();
+
+    if (step instanceof Promise) {
+      step = step.then((step) => {
+        newItem.step = step;
+        return step;
+      });
+    }
+
+    let newItem = new QueueItem(step);
+
+    this.head.next = newItem;
+    this.head = this.head.next;
+
+    return newItem;
+  }
+
+  releaseFork(fork) {
+    --this.forks;
+    if (this.forks === 0) {
+      this.iterator.return?.();
+    }
+    return { value: undefined, done: true };
+  }
+}
+
+class ForkIterator {
+  constructor(fork) {
+    facades.set(fork.clone(), this);
+  }
+
+  next() {
+    const fork = actuals.get(this);
+    if (!fork.done) {
+      const { head } = fork;
+      fork.advance();
+      return maybeWait(fork.head.step, () => head.step);
+    } else {
+      return { value: undefined, done: true };
+    }
+  }
+
+  return() {
+    actuals.get(this).return();
+    return { value: undefined, done: true };
+  }
+
+  [Symbol.for('@@streamIterator')]() {
+    return this;
+  }
+}
+
+class Fork {
+  constructor(head, exchange, done = false) {
+    this.head = head; // QueueItem
+    this.exchange = exchange;
+    this._done = done;
+  }
+
+  get done() {
+    return this._done || this.head.step?.done;
+  }
+
+  get value() {
+    return this.done ? { value: undefined, done: true } : this.head.step?.value;
+  }
+
+  advance() {
+    const { exchange } = this;
+
+    if (this.done) {
+      throw new Error('cannot advance a fork that is done');
+    } else {
+      let { head } = this;
+
+      let nextItem = this.head.next;
+
+      if (!head.step?.done) {
+        if (!nextItem) {
+          nextItem = exchange.fetch();
+        }
+
+        this.head = nextItem;
+      }
+
+      return nextItem;
+    }
+  }
+
+  return() {
+    const { done, exchange } = this;
+
+    if (!done) exchange.releaseFork(this);
+
+    const step = { value: undefined, done: true };
+
+    this._current = step;
+
+    return step;
+  }
+
+  clone() {
+    const { exchange } = this;
+
+    return exchange.allocateFork(this);
+  }
+
+  [Symbol.for('@@streamIterator')]() {
+    return new ForkIterator(this);
+  }
+}
+
+export const SourceFacade = class BABLRSourceFacade {
+  static from(iterable) {
+    return facades.get(Source.from(iterable));
+  }
+
+  constructor(source) {
+    facades.set(source, this);
+  }
+
+  [Symbol.for('@@streamIterator')]() {
+    return actuals.get(this)[Symbol.for('@@streamIterator')]();
+  }
+
+  get done() {
+    return actuals.get(this).done;
+  }
+
+  get index() {
+    return actuals.get(this).index;
+  }
+
+  get atGap() {
+    return actuals.get(this).atGap;
+  }
+};
+
+export const Source = class BABLRSource extends WeakStackFrame {
+  static from(iterable) {
+    const exchange = Exchange.from(iterable);
+    return Source.create(exchange.allocateFork(), exchange);
+  }
+
+  constructor(fork, exchange, index = -1, holding = false) {
+    super();
+
+    if (!fork || !exchange) throw new Error();
+
+    this.fork = fork;
+    this.exchange = exchange;
+    this.index = index;
+    this.holding = holding;
+
+    new SourceFacade(this);
+  }
+
+  get value() {
+    return this.fork.value;
+  }
+
+  get done() {
+    return this.fork.done;
+  }
+
+  get atGap() {
+    return this.holding || (!this.done && this.value == null);
+  }
+
+  advance(n = 1) {
+    return new Array(n).fill(null).reduce((acc) => {
+      return maybeWait(acc, () => {
+        this.fork.advance();
+        this.index++;
+        return this.fork.step;
+      });
+    }, this.fork.step);
+  }
+
+  shift() {
+    this.holding = true;
+  }
+
+  unshift() {
+    this.holding = false;
+  }
+
+  branch() {
+    const { fork, exchange, index, holding } = this;
+    return this.push(exchange.allocateFork(fork), exchange, index, holding);
+  }
+
+  release() {
+    this.fork.return();
+  }
+
+  accept(source) {
+    this.release();
+    this.fork = source.fork;
+    this.index = source.index;
+    this.holding = source.holding;
+  }
+
+  reject() {
+    this.release();
+  }
+
+  [Symbol.for('@@streamIterator')]() {
+    return this.fork.clone()[Symbol.for('@@streamIterator')]();
+  }
+
+  formatIndex() {
+    return `source[${this.source.index}]`;
+  }
+};

package/lib/state.js

@@ -0,0 +1,140 @@
+import emptyStack from '@iter-tools/imm-stack';
+import { WeakStackFrame } from '@bablr/weak-stack';
+import { getCooked } from '@bablr/agast-helpers/stream';
+import { match, guardWithPattern } from './utils/pattern.js';
+import { facades, actuals } from './facades.js';
+
+export const StateFacade = class BABLRStateFacade {
+  constructor(state) {
+    facades.set(state, this);
+  }
+
+  static from(context, source) {
+    return State.from(actuals.get(context), actuals.get(source));
+  }
+
+  get span() {
+    return actuals.get(this).span.name;
+  }
+
+  get result() {
+    return actuals.get(this).result;
+  }
+
+  get holding() {
+    return actuals.get(this).holding;
+  }
+
+  get context() {
+    return facades.get(actuals.get(this).context);
+  }
+
+  get path() {
+    return actuals.get(this).path;
+  }
+
+  get node() {
+    return actuals.get(this).node;
+  }
+
+  get source() {
+    return facades.get(actuals.get(this).source);
+  }
+
+  get depth() {
+    return actuals.get(this).depth;
+  }
+
+  get ctx() {
+    return this.context;
+  }
+};
+
+export const State = class BABLRState extends WeakStackFrame {
+  constructor(
+    context,
+    source,
+    agast,
+    balanced = emptyStack,
+    spans = emptyStack.push({ name: 'Bare' }),
+  ) {
+    super();
+
+    if (!context || !source || !agast) throw new Error('invalid args to State');
+
+    this.context = context;
+    this.source = source;
+    this.agast = agast;
+    this.balanced = balanced;
+    this.spans = spans;
+
+    new StateFacade(this);
+  }
+
+  static from(context, source, agast) {
+    return State.create(context, source, agast);
+  }
+
+  get guardedSource() {
+    const { source, span } = this;
+    const { guard } = span;
+
+    return guard ? guardWithPattern(guard, source) : source;
+  }
+
+  get ctx() {
+    return this.context;
+  }
+
+  get span() {
+    return this.spans.value;
+  }
+
+  get path() {
+    return this.agast.path;
+  }
+
+  get node() {
+    return this.agast.node;
+  }
+
+  get holding() {
+    return this.agast.holding;
+  }
+
+  get result() {
+    return this.agast.result;
+  }
+
+  get isGap() {
+    return this.tag.type === 'NodeGapTag';
+  }
+
+  get speculative() {
+    return !!this.parent;
+  }
+
+  guardedMatch(pattern) {
+    let { span, spans, source } = this;
+    let { guard } = span;
+
+    if (pattern?.flags?.intrinsic) {
+      if (pattern.type === 'OpenNodeTag') {
+        // TODO differntiate better between self-closing tags and matchers
+        pattern = pattern.value;
+      }
+
+      ({ guard } = span);
+
+      if (span.type === 'Lexical' && pattern.attributes.balancer) {
+        // also check that the open node starts a lexical span?
+        span = spans.prev.value;
+        ({ guard } = span);
+      }
+
+      pattern = pattern.intrinsicValue || getCooked(pattern.children);
+    }
+
+    return match(pattern, guard ? guardWithPattern(guard, source) : source);
+  }
+};

package/lib/symbols.js

@@ -0,0 +1 @@
+export const node = Symbol.for('@bablr/node');

package/lib/utils/array.js

@@ -0,0 +1,13 @@
+export const isArray = Array;
+
+export const notEmpty = (arr) => arr != null && arr.length > 0;
+
+export const nullOr = (arr) => (arr.length === 0 ? null : arr);
+
+export function* arraySlice(arr, start, end) {
+  const increment = end > start ? 1 : -1;
+
+  for (let i = start; i < end; i += increment) {
+    yield arr[i];
+  }
+}

package/lib/utils/format.js

@@ -0,0 +1,9 @@
+export const formatType = (type) => {
+  return typeof type === 'symbol' ? `${type.description.replace(/^@bablr\//y, '')}` : `'${type}'`;
+};
+
+export const formatGraveString = (str) => {
+  return `\`${str
+    .replace(/`/g, '\\`')
+    .replace(/[\r\n\u{00}\u{08}\u{0B}\u{0C}\u{0E}-\u{1F}]/gu, '')}\``;
+};

package/lib/utils/object.js

@@ -0,0 +1,97 @@
+export const { hasOwn, freeze, isFrozen, seal, isSealed, getOwnPropertySymbols } = Object;
+export const { isArray } = Array;
+
+const intFrom = (str) => {
+  const value = parseInt(str, 10);
+  return isNaN(value) ? null : value;
+};
+
+export const has = (obj, property) => {
+  let value = obj;
+  for (const part of property.split('.')) {
+    if (!hasOwn(value, part)) return false;
+    value = value[part];
+  }
+  return true;
+};
+
+export const get = (obj, property) => {
+  let value = obj;
+  for (const part of property.split('.')) {
+    value = value[part];
+  }
+  return value;
+};
+
+export const set = (obj, property, value) => {
+  const parts = property.split('.');
+  let obj_ = obj;
+
+  let lastKey;
+  for (let i = 0; i < parts.length; i++) {
+    const intKey = intFrom(parts[i]);
+    const key = intKey !== null ? intKey : parts[i];
+    let value = obj_[key];
+
+    if (parts.length - 1 === i) {
+      lastKey = key;
+    } else if (value !== undefined) {
+      obj_ = value;
+    } else if (intFrom(parts[i + 1]) !== null) {
+      obj_ = value = obj_[key] = [];
+    } else {
+      throw new Error(`Unable to set {property: '${property}'} in obj`);
+    }
+  }
+
+  obj_[lastKey] = value;
+};
+
+export function objectKeys(obj) {
+  return {
+    *[Symbol.iterator]() {
+      for (let key in obj) if (hasOwn(obj, key)) yield key;
+      yield* getOwnPropertySymbols(obj);
+    },
+  };
+}
+
+export function objectValues(obj) {
+  return {
+    *[Symbol.iterator]() {
+      for (let key in obj) if (hasOwn(obj, key)) yield obj[key];
+      yield* getOwnPropertySymbols(obj).map((sym) => obj[sym]);
+    },
+  };
+}
+
+export function objectEntries(obj) {
+  return {
+    *[Symbol.iterator]() {
+      for (let key in obj) if (hasOwn(obj, key)) yield [key, obj[key]];
+      yield* getOwnPropertySymbols(obj).map((sym) => [sym, obj[sym]]);
+    },
+  };
+}
+
+export const isObject = (obj) => obj !== null && typeof obj === 'object';
+export const isFunction = (obj) => typeof obj === 'function';
+export const isSymbol = (obj) => typeof obj === 'symbol';
+export const isString = (obj) => typeof obj === 'string';
+export const isType = (obj) => isSymbol(obj) || isString(obj);
+export const isRegex = (obj) => obj instanceof RegExp;
+export const isPattern = (obj) => isString(obj) || isRegex(obj);
+
+export const memoize = (fn) => {
+  const cache = new WeakMap();
+  return (obj) => {
+    let result;
+    if (cache.has(obj)) {
+      result = cache.get(obj);
+    } else {
+      result = fn(obj);
+      cache.set(obj, result);
+    }
+    return result;
+  };
+};

package/lib/utils/pattern.js

@@ -0,0 +1,77 @@
+import isString from 'iter-tools-es/methods/is-string';
+import { generateMatches } from '@bablr/regex-vm';
+import { getStreamIterator, maybeWait } from '@bablr/agast-helpers/stream';
+import * as t from '@bablr/agast-helpers/shorthand';
+import * as l from '@bablr/agast-vm-helpers/languages';
+
+export const assertValidRegex = (expr) => {
+  const { flags } = expr;
+
+  if (!expr.language === 'Spamex' && expr.type === 'Regex') {
+    throw new Error();
+  }
+
+  // TODO validate the rest of it
+};
+
+const buildStringRegex = (str) => {
+  return t.node(l.Regex, 'Pattern', [t.ref`open`, t.ref`alternatives[]`, t.ref`close`], {
+    open: t.s_node(l.Regex, 'Punctuator', '/'),
+    alternatives: [
+      t.node(l.Regex, 'Alternative', [t.ref`elements[]`], {
+        elements: [...str].map((chr) => t.s_node(l.Regex, 'Character', chr)),
+      }),
+    ],
+    close: t.s_node(l.Regex, 'Punctuator', '/'),
+  });
+};
+
+export const match = (pattern, source) => {
+  const pattern_ = isString(pattern) ? buildStringRegex(pattern) : pattern;
+
+  if (pattern_.type !== 'Pattern') throw new Error();
+
+  assertValidRegex(pattern_);
+
+  const iter = getStreamIterator(generateMatches(pattern_, source));
+
+  const step = iter.next();
+
+  return maybeWait(step, (step) => (step.done ? null : step.value[0]));
+};
+
+class GuardedIterator {
+  constructor(pattern, source) {
+    this.pattern = pattern;
+    this.fork = source.fork.clone();
+    this.done = false;
+  }
+
+  next() {
+    const { pattern, fork } = this;
+
+    const guardMatch = match(pattern, fork.clone());
+
+    return maybeWait(guardMatch, (guardMatch) => {
+      if (guardMatch || this.done) {
+        this.done = true;
+        return { value: undefined, done: true };
+      } else {
+        const { value } = fork;
+        return maybeWait(fork.advance(), (_) => ({ value, done: false }));
+      }
+    });
+  }
+
+  return() {
+    this.fork.return();
+  }
+
+  [Symbol.for('@@streamIterator')]() {
+    return this;
+  }
+}
+
+export const guardWithPattern = (pattern, source) => {
+  return new GuardedIterator(pattern, source.branch());
+};

package/lib/utils/token.js

@@ -0,0 +1,30 @@
+import { buildLiteral, buildGap } from '@bablr/agast-helpers/builders';
+
+export const isNewlineToken = (token) => /^\r|\r\n|\n$/.test(token.value);
+
+export function* ownChildrenFor(range) {
+  throw new Error('unimplemented');
+}
+
+export function* allChildrenFor(range) {
+  throw new Error('unimplemented');
+}
+
+export function* buildTokens(array) {
+  let str = '';
+  for (const chr of array) {
+    if (chr == null) {
+      if (str) {
+        yield buildLiteral(str);
+        str = '';
+      }
+      yield buildGap();
+    } else {
+      str += chr;
+    }
+  }
+
+  if (str) {
+    yield buildLiteral(str);
+  }
+}

package/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "@bablr/bablr-vm",
+  "description": "A VM for parsing using BABLR languages",
+  "version": "0.13.0",
+  "author": "Conrad Buck<conartist6@gmail.com>",
+  "type": "module",
+  "files": [
+    "lib"
+  ],
+  "exports": {
+    ".": "./lib/index.js"
+  },
+  "sideEffects": false,
+  "dependencies": {
+    "@bablr/agast-helpers": "0.1.0",
+    "@bablr/agast-vm-helpers": "0.1.0",
+    "@bablr/coroutine": "0.1.0",
+    "@bablr/helpers": "0.15.0",
+    "@bablr/regex-vm": "0.5.0",
+    "@bablr/weak-stack": "0.1.0",
+    "@iter-tools/imm-stack": "1.1.0",
+    "iter-tools-es": "^7.5.3"
+  },
+  "devDependencies": {
+    "@bablr/eslint-config-base": "github:bablr-lang/eslint-config-base#d834ccc52795d6c3b96ecc6c419960fceed221a6",
+    "enhanced-resolve": "^5.12.0",
+    "eslint": "^8.32.0",
+    "eslint-import-resolver-enhanced-resolve": "^1.0.5",
+    "eslint-plugin-import": "^2.27.5",
+    "iter-tools-es": "^7.3.1",
+    "prettier": "^2.6.2"
+  },
+  "repository": "github:bablr-lang/bablr-vm",
+  "homepage": "https://github.com/bablr-lang/bablr-vm",
+  "license": "MIT"
+}