@bablr/agast-helpers 0.2.0 → 0.3.1

package/lib/builders.js

@@ -1,5 +1,60 @@
+import * as sym from './symbols.js';
+
 const { freeze } = Object;
 
+const isObject = (val) => val !== null && typeof value !== 'object';
+
+export const buildEmbeddedExpression = (expr) => {
+  if (!isObject(expr)) return expr;
+  return freeze({ type: 'EmbeddedExpression', value: expr });
+};
+
+export const buildEmbeddedTag = (tag) => {
+  return freeze({ type: 'EmbeddedTag', value: tag });
+};
+
+export const buildEffect = (value) => {
+  return freeze({ type: 'Effect', value });
+};
+
+export const buildWriteEffect = (text, options = {}) => {
+  return buildEffect(
+    buildEmbeddedExpression(
+      freeze({
+        verb: 'write',
+        value: buildEmbeddedExpression(
+          freeze({ text, options: buildEmbeddedExpression(freeze(options)) }),
+        ),
+      }),
+    ),
+  );
+};
+
+export const buildAnsiPushEffect = (spans = '') => {
+  return buildEffect(
+    buildEmbeddedExpression(
+      freeze({
+        verb: 'ansi-push',
+        value: buildEmbeddedExpression(
+          freeze({ spans: spans === '' ? freeze([]) : freeze(spans.split(' ')) }),
+        ),
+      }),
+    ),
+  );
+};
+
+export const buildAnsiPopEffect = () => {
+  return buildEffect(buildEmbeddedExpression(freeze({ verb: 'ansi-pop', value: undefined })));
+};
+
+export const buildTokenGroup = (tokens) => {
+  return freeze({ type: 'TokenGroup', value: tokens });
+};
+
+export const buildCall = (verb, ...args) => {
+  return { verb, arguments: args };
+};
+
 export const buildBeginningOfStreamToken = () => {
   return freeze({ type: Symbol.for('@bablr/beginning-of-stream'), value: undefined });
 };
@@ -20,8 +75,8 @@ export const buildShift = () => {
   return freeze({ type: 'Shift', value: undefined });
 };
 
-export const buildEmbedded = (node) => {
-  return freeze({ type: 'Embedded', value: node });
+export const buildEmbeddedNode = (node) => {
+  return freeze({ type: 'EmbeddedNode', value: node });
 };
 
 export const buildDoctypeTag = (attributes) => {
@@ -31,13 +86,7 @@ export const buildDoctypeTag = (attributes) => {
   });
 };
 
-export const buildNodeOpenTag = (
-  flags = {},
-  language = null,
-  type = null,
-  intrinsicValue = null,
-  attributes = {},
-) => {
+export const buildNodeOpenTag = (flags = {}, language = null, type = null, attributes = {}) => {
   let { token, trivia, escape, expression, intrinsic } = flags;
 
   token = !!token;
@@ -52,7 +101,6 @@ export const buildNodeOpenTag = (
       flags: freeze({ token, trivia, escape, intrinsic, expression }),
       language,
       type,
-      intrinsicValue,
       attributes,
     }),
   });
@@ -223,20 +271,13 @@ export const buildTriviaNode = (language, type, children = [], properties = {},
     attributes: freeze(attributes),
   });
 
-export const buildEffect = (value) => {
-  return freeze({ type: 'Effect', value });
-};
-
-export const buildWriteEffect = (text, options = {}) => {
-  return buildEffect(freeze({ verb: 'write', value: freeze({ text, options }) }));
-};
-
-export const buildAnsiPushEffect = (spans = '') => {
-  return buildEffect(
-    freeze({ verb: 'ansi-push', value: { spans: spans === '' ? [] : spans.split(' ') } }),
-  );
-};
-
-export const buildAnsiPopEffect = () => {
-  return buildEffect(freeze({ verb: 'ansi-pop', value: undefined }));
+export const buildNullNode = () => {
+  return freeze({
+    flags: nodeFlags,
+    language: null,
+    type: sym.null,
+    children: freeze([buildNull()]),
+    properties: freeze({}),
+    attributes: freeze({}),
+  });
 };

package/lib/print.js

@@ -8,6 +8,22 @@ const isFunction = (val) => typeof val === 'function';
 const when = (condition, value) =>
   condition ? (isFunction(value) ? value() : value) : { *[Symbol.iterator]() {} };
 
+export const printCall = (call) => {
+  const { verb, arguments: args } = call;
+  return `${verb}${printTuple(args)}`;
+};
+
+export const printArray = (arr) => `[${arr.map((v) => printExpression(v)).join(', ')}]`;
+
+export const printTuple = (tup) => `(${tup.map((v) => printExpression(v)).join(', ')})`;
+
+export const printObject = (obj) => {
+  const entries = Object.entries(obj);
+  return entries.length
+    ? `{ ${entries.map(([k, v]) => `${k}: ${printExpression(v)}`).join(', ')} }`
+    : '{}';
+};
+
 export const printExpression = (expr) => {
   if (isString(expr)) {
     return printString(expr);
@@ -23,14 +39,33 @@ export const printExpression = (expr) => {
       throw new Error();
     }
   } else if (isArray(expr)) {
-    return `[${expr.map((v) => printExpression(v)).join(', ')}]`;
+    return printArray(expr);
   } else if (typeof expr === 'object') {
-    return `{${Object.entries(expr).map(([k, v]) => `${k}: ${printExpression(v)}`)}}`;
+    return printEmbedded(expr);
   } else {
     throw new Error();
   }
 };
 
+export const printEmbedded = (value) => {
+  switch (value.type) {
+    case 'EmbeddedTag':
+      return printTerminal(value.value);
+
+    case 'EmbeddedExpression': {
+      return printObject(value.value);
+    }
+
+    case 'EmbeddedNode': {
+      throw new Error('not implemented');
+      break;
+    }
+
+    default:
+      throw new Error();
+  }
+};
+
 export const printAttributes = (attributes) => {
   return Object.entries(attributes)
     .map(([k, v]) => (v === true ? k : v === false ? `!${k}` : `${k}=${printExpression(v)}`))
@@ -46,9 +81,10 @@ export const printLanguage = (language) => {
 };
 
 export const printTagPath = (language, type) => {
-  return [...when(language?.length, () => [printLanguage(language)]), ...when(type, [type])].join(
-    ':',
-  );
+  return [
+    ...when(type && language?.length, () => [printLanguage(language)]),
+    ...when(type, [type]),
+  ].join(':');
 };
 
 const escapeReplacer = (esc) => {
@@ -121,7 +157,8 @@ export const printDoctypeTag = (terminal) => {
 
   let { doctype, version, attributes } = terminal.value;
 
-  attributes = attributes ? ` ${printAttributes(attributes)}` : '';
+  attributes =
+    attributes && Object.values(attributes).length ? ` ${printAttributes(attributes)}` : '';
 
   return `<!${version}:${doctype}${attributes}>`;
 };
@@ -147,17 +184,27 @@ export const printFlags = (flags) => {
 export const printOpenNodeTag = (terminal) => {
   if (terminal?.type !== 'OpenNodeTag') throw new Error();
 
-  const { flags, language: tagLanguage, type, intrinsicValue, attributes } = terminal.value;
+  const { flags, language: tagLanguage, type, attributes } = terminal.value;
+
+  const printedAttributes = attributes && printAttributes(attributes);
+  const attributesFrag = printedAttributes ? ` ${printedAttributes}` : '';
+
+  return `<${printFlags(flags)}${printTagPath(tagLanguage, type)}${attributesFrag}>`;
+};
+
+export const printSelfClosingNodeTag = (terminal, intrinsicValue) => {
+  if (terminal?.type !== 'OpenNodeTag') throw new Error();
+
+  const { flags, language: tagLanguage, type, attributes } = terminal.value;
 
   const printedAttributes = attributes && printAttributes(attributes);
   const attributesFrag = printedAttributes ? ` ${printedAttributes}` : '';
   const intrinsicFrag = intrinsicValue ? ` ${printString(intrinsicValue)}` : '';
-  const selfCloser = intrinsicValue ? ' /' : '';
 
   return `<${printFlags(flags)}${printTagPath(
     tagLanguage,
     type,
-  )}${intrinsicFrag}${attributesFrag}${selfCloser}>`;
+  )}${intrinsicFrag}${attributesFrag} />`;
 };
 
 export const printCloseNodeTag = (terminal) => {

package/lib/shorthand.js

@@ -1,7 +1,7 @@
 import {
   buildReference,
   buildGap,
-  buildEmbedded,
+  buildEmbeddedNode,
   buildNodeOpenTag,
   buildNodeCloseTag,
   buildLiteral,
@@ -12,6 +12,7 @@ import {
   buildSyntacticEscapeNode,
   buildSyntacticTriviaNode,
   buildTriviaNode,
+  buildNullNode,
 } from './builders.js';
 
 export * from './builders.js';
@@ -43,7 +44,7 @@ export const ref = (path) => {
 export const lit = (str) => buildLiteral(stripArray(str));
 
 export const gap = buildGap;
-export const embedded = buildEmbedded;
+export const embedded = buildEmbeddedNode;
 export const nodeOpen = buildNodeOpenTag;
 export const nodeClose = buildNodeCloseTag;
 export const node = buildNode;
@@ -53,3 +54,4 @@ export const e_node = buildEscapeNode;
 export const s_e_node = buildSyntacticEscapeNode;
 export const s_t_node = buildSyntacticTriviaNode;
 export const t_node = buildTriviaNode;
+export const null_node = buildNullNode;

package/lib/stream.js

@@ -1,8 +1,14 @@
 import { Coroutine } from '@bablr/coroutine';
-import { printTerminal } from './print.js';
-import { buildWriteEffect } from './builders.js';
+import emptyStack from '@iter-tools/imm-stack';
+import { printSelfClosingNodeTag, printTerminal } from './print.js';
+import { buildWriteEffect, buildTokenGroup } from './builders.js';
 export * from './print.js';
 
+const getEmbeddedExpression = (obj) => {
+  if (obj.type !== 'EmbeddedExpression') throw new Error();
+  return obj.value;
+};
+
 export const getStreamIterator = (obj) => {
   return obj[Symbol.for('@@streamIterator')]?.() || obj[Symbol.iterator]?.();
 };
@@ -83,6 +89,12 @@ export class AsyncGenerator {
   }
 }
 
+export const isIntrinsicToken = (terminal) => {
+  return (
+    terminal.type === 'OpenNodeTag' && terminal.value.flags.intrinsic && terminal.value.flags.token
+  );
+};
+
 export class StreamGenerator {
   constructor(embeddedGenerator) {
     this.generator = embeddedGenerator;
@@ -137,45 +149,6 @@ export const maybeWait = (maybePromise, callback) => {
   }
 };
 
-function* __generateCSTMLStrategy(terminals) {
-  if (!terminals) {
-    yield buildWriteEffect('<//>');
-    return;
-  }
-
-  let prevTerminal = null;
-
-  const co = new Coroutine(getStreamIterator(terminals));
-
-  for (;;) {
-    co.advance();
-
-    if (co.current instanceof Promise) {
-      co.current = yield co.current;
-    }
-    if (co.done) break;
-
-    const terminal = co.value;
-
-    if (terminal.type === 'Reference' && prevTerminal.type === 'Null') {
-      yield buildWriteEffect(' ');
-    }
-
-    if (terminal.type !== 'Effect') {
-      buildWriteEffect(printTerminal(terminal));
-
-      prevTerminal = terminal;
-    } else {
-      yield terminal;
-    }
-  }
-
-  yield buildWriteEffect('\n');
-}
-
-export const generateCSTMLStrategy = (terminals) =>
-  new StreamIterable(__generateCSTMLStrategy(terminals));
-
 function* __generateStandardOutput(terminals) {
   const co = new Coroutine(getStreamIterator(terminals));
 
@@ -190,9 +163,12 @@ function* __generateStandardOutput(terminals) {
     const terminal = co.value;
 
     if (terminal.type === 'Effect') {
-      const effect = terminal.value;
-      if (effect.verb === 'write' && (effect.value.stream == null || effect.value.stream === 1)) {
-        yield* effect.value.text;
+      const effect = getEmbeddedExpression(terminal.value);
+      if (effect.verb === 'write') {
+        const writeEffect = getEmbeddedExpression(effect.value);
+        if (writeEffect.stream == null || writeEffect.stream === 1) {
+          yield* writeEffect.text;
+        }
       }
     }
   }
@@ -217,14 +193,15 @@ function* __generateAllOutput(terminals) {
     const terminal = co.value;
 
     if (terminal.type === 'Effect') {
-      const effect = terminal.value;
+      const effect = getEmbeddedExpression(terminal.value);
       if (effect.verb === 'write') {
+        const writeEffect = getEmbeddedExpression(effect.value);
         const prevStream = currentStream;
-        currentStream = effect.value.options.stream || 1;
-        if (prevStream && prevStream !== currentStream && !effect.value.text.startsWith('\n')) {
+        currentStream = getEmbeddedExpression(writeEffect.options).stream || 1;
+        if (prevStream && prevStream !== currentStream && !writeEffect.text.startsWith('\n')) {
           yield* '\n';
         }
-        yield* effect.value.text;
+        yield* writeEffect.text;
       }
     }
   }
@@ -278,17 +255,142 @@ export const stringFromStream = (stream) => {
   return str;
 };
 
-function* __generatePrettyCSTMLStrategy(terminals, options) {
-  const { indent = '  ', emitEffects = false, inline: inlineOption = true } = options;
+function* __generateCSTMLStrategy(terminals, options) {
+  let { emitEffects = false, inline: inlineOption = true } = options;
+
+  if (emitEffects) {
+    throw new Error('You must use generatePrettyCSTML with emitEffects');
+  }
 
   if (!terminals) {
     yield buildWriteEffect('<//>');
     return;
   }
 
+  let prevTerminal = null;
+
+  const co = new Coroutine(getStreamIterator(prettyGroupTokens(terminals)));
+
+  for (;;) {
+    co.advance();
+
+    if (co.current instanceof Promise) {
+      co.current = yield co.current;
+    }
+    if (co.done) break;
+
+    const terminal = co.value;
+
+    if (terminal.type === 'Reference' && prevTerminal.type === 'Null') {
+      yield buildWriteEffect(' ');
+    }
+
+    if (terminal.type === 'Effect') {
+      const effect = terminal.value;
+      if (emitEffects && effect.verb === 'write') {
+        yield buildWriteEffect(effect.value.text, effect.value.options);
+      }
+      continue;
+    }
+
+    if (terminal.type === 'TokenGroup') {
+      const intrinsicValue = getCooked(terminal.value.slice(1, -1));
+      yield buildWriteEffect(printSelfClosingNodeTag(terminal.value[0], intrinsicValue));
+    } else {
+      yield buildWriteEffect(printTerminal(terminal));
+    }
+
+    prevTerminal = terminal;
+  }
+
+  yield buildWriteEffect('\n');
+}
+
+export const prettyGroupTokens = (terminals) => new StreamIterable(__prettyGroupTokens(terminals));
+
+function* __prettyGroupTokens(terminals) {
+  let states = emptyStack.push({ holding: [], broken: false, open: null });
+  let state = states.value;
+
   const co = new Coroutine(getStreamIterator(terminals));
+
+  for (;;) {
+    co.advance();
+
+    if (co.done) break;
+
+    if (co.current instanceof Promise) {
+      co.current = yield co.current;
+    }
+
+    const terminal = co.value;
+    const isOpenClose =
+      terminal.type === 'CloseNodeTag' || (terminal.type === 'OpenNodeTag' && terminal.value.type);
+
+    if (
+      (terminal.type === 'Effect' && terminal.value.verb === 'write') ||
+      ['Reference', 'DoctypeTag', 'Gap', 'Null'].includes(terminal.type) ||
+      (terminal.type === 'OpenNodeTag' && !terminal.value.type) ||
+      (state.open &&
+        !isIntrinsicToken(state.open) &&
+        (terminal.type === 'Literal' ||
+          (terminal.type === 'OpenNodeTag' && terminal.value.flags.escape)))
+    ) {
+      state.broken = true;
+
+      if (state.holding.length) {
+        yield* state.holding;
+        state.holding = [];
+      }
+
+      if (!isOpenClose && terminal.type !== 'Effect') {
+        yield terminal;
+      }
+    } else if (!isOpenClose && terminal.type !== 'Effect') {
+      state.holding.push(terminal);
+    }
+
+    if (terminal.type === 'Effect') {
+      yield terminal;
+    }
+
+    if (terminal.type === 'CloseNodeTag') {
+      if (!state.broken && (isIntrinsicToken(state.open) || state.holding.length === 1)) {
+        state.holding.push(terminal);
+        yield buildTokenGroup(state.holding);
+      } else {
+        if (state.holding.length) {
+          yield* state.holding;
+        }
+        yield terminal;
+      }
+
+      states = states.pop();
+      state = states.value;
+    }
+
+    if (terminal.type === 'OpenNodeTag' && terminal.value.type) {
+      states = states.push({ holding: [terminal], broken: false, open: terminal });
+      state = states.value;
+    }
+  }
+}
+
+export const generateCSTMLStrategy = (terminals, options = {}) =>
+  new StreamIterable(__generateCSTMLStrategy(terminals, options));
+
+function* __generatePrettyCSTMLStrategy(terminals, options) {
+  let { indent = '  ', emitEffects = false, inline: inlineOption = true } = options;
+
+  if (!terminals) {
+    yield buildWriteEffect('<//>');
+    return;
+  }
+
+  const co = new Coroutine(getStreamIterator(prettyGroupTokens(terminals)));
   let indentLevel = 0;
   let first = true;
+  let inline = false;
 
   for (;;) {
     co.advance();
@@ -302,22 +404,24 @@ function* __generatePrettyCSTMLStrategy(terminals, options) {
     const terminal = co.value;
 
     if (terminal.type === 'Effect') {
-      const effect = terminal.value;
+      const effect = getEmbeddedExpression(terminal.value);
       if (emitEffects && effect.verb === 'write') {
-        yield buildWriteEffect((first ? '' : '\n') + effect.value.text, effect.value.options);
+        const writeEffect = getEmbeddedExpression(effect.value);
+        yield buildWriteEffect(
+          (first ? '' : '\n') + writeEffect.text,
+          getEmbeddedExpression(writeEffect.options),
+        );
 
+        inline = false;
         first = false;
       }
       continue;
     }
 
-    const inline =
+    inline =
       inlineOption &&
-      (terminal.type === 'Null' ||
-        terminal.type === 'Gap' ||
-        (terminal.type === 'OpenNodeTag' &&
-          terminal.value.intrinsicValue &&
-          terminal.value.flags.intrinsic));
+      inline &&
+      (terminal.type === 'Null' || terminal.type === 'Gap' || terminal.type === 'TokenGroup');
 
     if (!first && !inline) {
       yield buildWriteEffect('\n');
@@ -336,12 +440,19 @@ function* __generatePrettyCSTMLStrategy(terminals, options) {
     } else {
       yield buildWriteEffect(' ');
     }
-    yield buildWriteEffect(printTerminal(terminal));
 
-    if (
-      terminal.type === 'OpenNodeTag' &&
-      (!terminal.value.intrinsicValue || !terminal.value.type)
-    ) {
+    if (terminal.type === 'TokenGroup') {
+      const intrinsicValue = getCooked(terminal.value.slice(1, -1));
+      yield buildWriteEffect(printSelfClosingNodeTag(terminal.value[0], intrinsicValue));
+    } else {
+      yield buildWriteEffect(printTerminal(terminal));
+    }
+
+    if (terminal.type === 'Reference') {
+      inline = true;
+    }
+
+    if (terminal.type === 'OpenNodeTag') {
       indentLevel++;
     }
 
@@ -408,8 +519,6 @@ export const printSource = (terminals) => {
   for (const terminal of terminals) {
     if (terminal.type === 'Literal') {
       printed += terminal.value;
-    } else if (terminal.type === 'OpenNodeTag' && terminal.value.intrinsicValue) {
-      printed += terminal.value.intrinsicValue;
     }
   }
 
@@ -420,21 +529,8 @@ export function* generateSourceTextFor(terminals) {
   for (const terminal of terminals) {
     if (terminal.type === 'Literal') {
       yield* terminal.value;
-    } else if (terminal.type === 'OpenNodeTag' && terminal.value.intrinsicValue) {
-      yield* terminal.value.intrinsicValue;
     }
   }
 }
 
 export const sourceTextFor = printSource;
-
-export const startsDocument = (terminal) => {
-  const { type, value } = terminal;
-  if ((type === 'OpenNodeTag' && !value.type) || type === 'DoctypeTag') {
-    return true;
-  } else if (type === 'OpenNodeTag') {
-    const { flags } = value;
-
-    return flags.trivia || flags.escape;
-  }
-};

package/lib/symbols.js

@@ -0,0 +1,9 @@
+export const node = Symbol.for('@bablr/node');
+
+export const gap = Symbol.for('@bablr/gap');
+
+export const fragment = Symbol.for('@bablr/fragment');
+
+export const null_ = Symbol.for('@bablr/null');
+
+export { null_ as null };

package/lib/template.js

@@ -28,10 +28,10 @@ export const interpolateArrayChildren = (value, ref, sep) => {
 };
 
 const validateTerminal = (term) => {
-  if (!term || (term.type !== 'Literal' && term.type !== 'Embedded')) {
+  if (!term || (term.type !== 'Literal' && term.type !== 'EmbeddedNode')) {
     throw new Error('Invalid terminal');
   }
-  if (term.type === 'Embedded' && !term.value.flags.escape) {
+  if (term.type === 'EmbeddedNode' && !term.value.flags.escape) {
     throw new Error();
   }
 };

package/lib/tree.js

@@ -1,19 +1,13 @@
 import emptyStack from '@iter-tools/imm-stack';
 import { Coroutine } from '@bablr/coroutine';
-import {
-  buildNodeCloseTag,
-  buildNodeOpenTag,
-  buildNull,
-  buildEmbedded,
-  nodeFlags,
-  buildDoctypeTag,
-} from './builders.js';
+import { buildNull, nodeFlags, buildDoctypeTag, buildEmbeddedNode } from './builders.js';
 import {
   printPrettyCSTML as printPrettyCSTMLFromStream,
   printCSTML as printCSTMLFromStream,
   getStreamIterator,
   StreamIterable,
 } from './stream.js';
+import * as sym from './symbols.js';
 export * from './builders.js';
 export * from './print.js';
 
@@ -21,6 +15,8 @@ const arrayLast = (arr) => arr[arr.length - 1];
 
 const isString = (str) => typeof str === 'string';
 
+const { isArray } = Array;
+
 const buildFrame = (node) => {
   if (!node) throw new Error();
   return { node, childrenIdx: -1, resolver: new Resolver(node) };
@@ -28,13 +24,33 @@ const buildFrame = (node) => {
 
 const { hasOwn, freeze } = Object;
 
-const get = (node, path) => {
+export const get = (node, path) => {
+  const { type, properties } = node;
   const { 1: name, 2: index } = /^([^\.]+)(?:\.(\d+))?/.exec(path) || [];
 
+  if (!hasOwn(properties, name)) {
+    throw new Error(`Cannot find {name: ${name}} on node of {type: ${type}}`);
+  }
+
   if (index != null) {
-    return node.properties[name]?.[parseInt(index, 10)];
+    return properties[name]?.[parseInt(index, 10)];
   } else {
-    return node.properties[name];
+    return properties[name];
+  }
+};
+
+export const add = (node, ref, value) => {
+  const { name, isArray } = ref.value;
+
+  if (isArray) {
+    if (!hasOwn(node.properties, name)) {
+      node.properties[name] = [];
+    }
+    const array = node.properties[name];
+
+    array.push(value);
+  } else {
+    node.properties[name] = value;
   }
 };
 
@@ -64,6 +80,7 @@ function* __treeFromStream(tokens) {
 
       rootNode = freeze({
         flags: nodeFlags,
+        language: attributes['bablr-language'],
         type: null,
         children: [],
         properties: {},
@@ -79,6 +96,13 @@ function* __treeFromStream(tokens) {
 
     switch (token.type) {
       case 'Null': {
+        const node = nodes.value;
+        const ref = arrayLast(node.children);
+
+        if (ref.type !== 'Reference') throw new Error();
+
+        add(node, ref, null);
+
         break;
       }
 
@@ -90,19 +114,12 @@ function* __treeFromStream(tokens) {
 
       case 'Gap': {
         if (held) {
-          const { children, properties } = nodes.value;
-          const ref = arrayLast(children);
+          const node = nodes.value;
+          const ref = arrayLast(node.children);
 
           if (ref.type !== 'Reference') throw new Error();
 
-          if (ref.value.isArray) {
-            if (!properties[ref.value.name]) {
-              properties[ref.value.name] = [];
-            }
-            properties[ref.value.name].push(held);
-          } else {
-            properties[ref.value.name] = held;
-          }
+          add(node, ref, held);
 
           held = null;
         }
@@ -127,12 +144,7 @@ function* __treeFromStream(tokens) {
       }
 
       case 'OpenNodeTag': {
-        const buildStringTerminals = (str) => {
-          // do better
-          return [{ type: 'Literal', value: str }];
-        };
-
-        const { flags, language, type, intrinsicValue, attributes } = token.value;
+        const { flags, language, type, attributes } = token.value;
         const node = nodes.value;
 
         if (!type) {
@@ -143,7 +155,7 @@ function* __treeFromStream(tokens) {
           flags,
           language,
           type,
-          children: intrinsicValue && flags.intrinsic ? buildStringTerminals(intrinsicValue) : [],
+          children: [],
           properties: {},
           attributes: freeze(attributes),
         });
@@ -153,24 +165,9 @@ function* __treeFromStream(tokens) {
             throw new Error('Nodes must follow references');
           }
 
-          const { name, isArray } = arrayLast(node.children).value;
-
-          if (isArray) {
-            if (!hasOwn(node.properties, name)) {
-              node.properties[name] = [];
-            }
-            const array = node.properties[name];
+          const ref = arrayLast(node.children);
 
-            array.push(newNode);
-          } else {
-            node.properties[name] = newNode;
-          }
-        }
-
-        if (intrinsicValue && flags.intrinsic) {
-          freeze(newNode.children);
-          freeze(newNode.properties);
-          break;
+          add(node, ref, newNode);
         }
 
         nodes = nodes.push(newNode);
@@ -184,7 +181,7 @@ function* __treeFromStream(tokens) {
         if (flags.escape || flags.trivia) {
           const parentChildren = nodes.prev.value.children;
 
-          parentChildren.push(buildEmbedded(completedNode));
+          parentChildren.push(buildEmbeddedNode(completedNode));
         }
 
         freeze(completedNode.properties);
@@ -235,75 +232,66 @@ export const evaluateReturnAsync = async (generator) => {
   return co.value;
 };
 
-export function* streamFromTree(rootNode) {
+export const streamFromTree = (rootNode) => __streamFromTree(rootNode);
+
+function* __streamFromTree(rootNode) {
   if (!rootNode || rootNode.type === 'Gap') {
     return rootNode;
   }
 
   yield buildDoctypeTag(rootNode.attributes);
-  yield buildNodeOpenTag();
 
   let stack = emptyStack.push(buildFrame(rootNode));
 
   stack: while (stack.size) {
     const frame = stack.value;
     const { node, resolver } = frame;
-    const { flags, language, type, children, attributes } = node;
+    const { children } = node;
 
-    const intrinsicValue = flags.token && flags.intrinsic ? children[0]?.value : null;
+    while (++frame.childrenIdx < children.length) {
+      const terminal = children[frame.childrenIdx];
 
-    const children_ = intrinsicValue ? [intrinsicValue] : node.children;
+      switch (terminal.type) {
+        case 'EmbeddedNode': {
+          stack = stack.push(buildFrame(terminal.value));
 
-    if (frame.childrenIdx === -1 && stack.size > 1) {
-      yield buildNodeOpenTag(flags, language, type, intrinsicValue, attributes);
-    }
+          break;
+        }
 
-    if (!intrinsicValue) {
-      while (++frame.childrenIdx < children_.length) {
-        const terminal = children_[frame.childrenIdx];
+        case 'Reference': {
+          const resolved = resolver.consume(terminal).get(terminal);
 
-        switch (terminal.type) {
-          case 'Literal':
-          case 'Gap':
-          case 'Null': {
-            yield terminal;
-            break;
-          }
+          yield terminal;
 
-          case 'Embedded': {
-            stack = stack.push(buildFrame(terminal.value));
-            continue stack;
-          }
+          if (terminal.value.isArray && !resolved) {
+            // TODO evaluate if this is still smart
+            yield buildNull();
+          } else {
+            if (!resolved) throw new Error();
 
-          case 'Reference': {
-            if (stack.size > 1) {
-              yield terminal;
-            }
-
-            const resolved = resolver.consume(terminal).get(terminal);
-            if (resolved) {
-              stack = stack.push(buildFrame(resolved));
-              continue stack;
-            } else {
-              yield buildNull();
-              break;
-            }
+            stack = stack.push(buildFrame(resolved));
           }
 
-          default: {
-            throw new Error();
-          }
+          break;
+        }
+
+        case 'Null': {
+          yield terminal;
+
+          break;
         }
+
+        default:
+          yield terminal;
       }
 
-      if (stack.size > 1) {
-        yield buildNodeCloseTag(type, language);
+      if (terminal.type === 'EmbeddedNode' || terminal.type === 'Reference') {
+        continue stack;
       }
     }
 
     stack = stack.pop();
   }
-  yield buildNodeCloseTag();
 }
 
 export const getCooked = (cookable) => {
@@ -321,7 +309,7 @@ export const getCooked = (cookable) => {
         throw new Error('cookable nodes must not contain other nodes');
       }
 
-      case 'Embedded': {
+      case 'EmbeddedNode': {
         const { flags, attributes } = terminal.value;
 
         if (!(flags.trivia || (flags.escape && attributes.cooked))) {
@@ -358,26 +346,28 @@ export const printCSTML = (rootNode) => {
   return printCSTMLFromStream(streamFromTree(rootNode));
 };
 
-export const printPrettyCSTML = (rootNode, indent = '  ') => {
-  return printPrettyCSTMLFromStream(streamFromTree(rootNode), indent);
+export const printPrettyCSTML = (rootNode, options = {}) => {
+  return printPrettyCSTMLFromStream(streamFromTree(rootNode), options);
 };
 
 export const printSource = (node) => {
   const resolver = new Resolver(node);
   let printed = '';
 
+  if (!node) return '';
+
   if (node instanceof Promise) {
     printed += '$Promise';
   } else {
     for (const child of node.children) {
       if (child.type === 'Literal') {
         printed += child.value;
-      } else if (child.type === 'Embedded') {
+      } else if (child.type === 'EmbeddedNode') {
         printed += printSource(child.value);
       } else if (child.type === 'Reference') {
         const node_ = resolver.consume(child).get(child);
 
-        if (node_ || !child.value.isArray) {
+        if (node_) {
           printed += printSource(node_);
         }
       }
@@ -389,6 +379,88 @@ export const printSource = (node) => {
 
 export const sourceTextFor = printSource;
 
+export const getOpenTag = (node) => {
+  const tag = node.children[0];
+  if (tag && tag.type !== 'OpenNodeTag') throw new Error();
+  return tag;
+};
+
+export const getCloseTag = (node) => {
+  const { children } = node;
+  const tag = children[children.length - 1];
+  if (tag.type !== 'CloseNodeTag') return null;
+  return tag;
+};
+
+export const getRange = (node) => {
+  const { children } = node;
+  return children.length ? [children[0], children[children.length - 1]] : null;
+};
+
+export const createNode = (openTag) => {
+  const { flags, language, type, attributes } = openTag.value;
+  return { flags, language, type, children: [], properties: {}, attributes };
+};
+
+export const finalizeNode = (node) => {
+  freeze(node);
+  freeze(node.children);
+  freeze(node.properties);
+  freeze(node.attributes);
+  return node;
+};
+
+export const notNull = (node) => {
+  return node && node.type !== sym.null;
+};
+
+export const isNull = (node) => {
+  return !node || node.type === sym.null;
+};
+
+export const branchProperties = (properties) => {
+  const copy = { ...properties };
+
+  for (const { 0: key, 1: value } of Object.entries(copy)) {
+    if (isArray(value)) {
+      copy[key] = [...value];
+    }
+  }
+
+  return copy;
+};
+
+export const branchNode = (node) => {
+  const { flags, language, type, children, properties, attributes } = node;
+  return {
+    flags,
+    language,
+    type,
+    children: [...children],
+    properties: branchProperties(properties),
+    attributes: { ...attributes },
+  };
+};
+
+export const acceptNode = (node, accepted) => {
+  const { children, properties, attributes } = accepted;
+  node.children = children;
+  node.properties = properties;
+  node.attributes = attributes;
+  return node;
+};
+
+export const getRoot = (fragmentNode) => {
+  if (!fragmentNode) return null;
+
+  for (const terminal of fragmentNode.children) {
+    if (terminal.type === 'Reference') {
+      if (terminal.value.isArray) throw new Error();
+      return fragmentNode.properties[terminal.value.name];
+    }
+  }
+};
+
 export class Resolver {
   constructor(node, counters = new Map()) {
     this.node = node;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@bablr/agast-helpers",
   "description": "Helper functions for working with agAST trees",
-  "version": "0.2.0",
+  "version": "0.3.1",
   "author": "Conrad Buck<conartist6@gmail.com>",
   "type": "module",
   "files": [
@@ -13,6 +13,7 @@
     "./print": "./lib/print.js",
     "./shorthand": "./lib/shorthand.js",
     "./stream": "./lib/stream.js",
+    "./symbols": "./lib/symbols.js",
     "./template": "./lib/template.js",
     "./tree": "./lib/tree.js"
   },