@bablr/agast-helpers 0.10.9 → 0.11.0

package/lib/stream.js

@@ -1,7 +1,15 @@
 import { Coroutine } from '@bablr/coroutine';
 import emptyStack from '@iter-tools/imm-stack';
-import { getStreamIterator, StreamIterable, StreamGenerator, wait } from '@bablr/stream-iterator';
+import {
+  getStreamIterator,
+  StreamIterable,
+  StreamGenerator,
+  wait,
+  createDeferred,
+  continue_,
+} from '@bablr/stream-iterator';
 import * as Tags from './tags.js';
+import * as BList from './b-list.js';
 import { printTag } from './print.js';
 import {
   buildChild,
@@ -10,6 +18,8 @@ import {
   buildOpenNodeTag,
   buildProperty,
   buildReference,
+  parseTag,
+  parseTagType,
 } from './builders.js';
 import {
   OpenNodeTag,
@@ -22,78 +32,60 @@ import {
   DoctypeTag,
   ShiftTag,
   AttributeDefinition,
+  StreamTag,
 } from './symbols.js';
-import { buildNode, isGapNode, Path } from './path.js';
+import { buildNode, getRoot, isGapNode, isNode, Path } from './path.js';
 import { streamFromTree } from './tree.js';
+import { freeze, isObject } from './object.js';
+import { maybeWait } from './iterable.js';
+import { arrayValues } from '@bablr/record';
+import { buildParser } from './parse.js';
 
-export { getStreamIterator, StreamIterable, StreamGenerator, wait };
+export { getStreamIterator, StreamIterable, StreamGenerator, wait, continue_ };
 
 export * from './print.js';
 
-export const maybeWait = (maybePromise, callback) => {
-  if (maybePromise instanceof Promise) {
-    return maybePromise.then(callback);
-  } else {
-    return callback(maybePromise);
-  }
-};
-
-function* __flatMap(fn, tags) {
-  const co = new Coroutine(getStreamIterator(tags));
+function* __evaluateReturn(iterable) {
+  let iter = getStreamIterator(iterable);
+  let step;
 
   for (;;) {
-    co.advance();
-
-    if (co.current instanceof Promise) {
-      co.current = yield wait(co.current);
+    step = iter.next();
+    while (step === null || step instanceof Promise) {
+      if (step === null) yield continue_(), (step = iter.next());
+      if (step instanceof Promise) step = yield wait(step);
     }
-    if (co.done) break;
-
-    let { value } = co;
-
-    yield* fn(value);
+    if (step.done) break;
   }
 
-  return true;
+  return step.value;
 }
 
-export const flatMap = (fn, tags) => new StreamIterable(__flatMap(fn, tags));
-
-export const evaluateReturnSync = (iterable) => {
-  const co = new Coroutine(iterable[Symbol.iterator]());
-  while (!co.done) co.advance();
-  return co.value;
-};
+export const evaluateReturn = (iterable) => {
+  let iter = new StreamGenerator(__evaluateReturn(iterable));
 
-export const evaluateReturnAsync = async (iterable) => {
-  const co = new Coroutine(getStreamIterator(iterable));
-  while (true) {
-    if (co.current instanceof Promise) {
-      co.current = await co.current;
-    }
-    if (co.done) break;
-    co.advance();
-  }
-  return co.value;
+  return maybeWait(iter.next(), (step) => step.value);
 };
 
 function* __treeFromStream(tags, options) {
   let path = null;
-  const co = new Coroutine(getStreamIterator(tags));
-  const expressionsCo = new Coroutine(getStreamIterator(options.expressions || []));
+  let iter = getStreamIterator(tags);
+  let expressionsIter = getStreamIterator(options.expressions || []);
+  let step;
+  let expressionsStep;
 
   for (;;) {
-    co.advance();
-
-    if (co.current instanceof Promise) {
-      co.current = yield wait(co.current);
+    step = iter.next();
+    while (step === null || step instanceof Promise) {
+      if (step === null) yield continue_(), (step = iter.next());
+      if (step instanceof Promise) step = yield wait(step);
     }
+    if (step.done) break;
 
-    if (co.done) break;
+    let tag = step.value;
+    let tagType = parseTagType(tag);
 
-    let tag = co.value;
-
-    if (tag.type === 'Effect' || tag.type === DoctypeTag) {
+    if (tagType === 'Effect' || tagType === DoctypeTag) {
       continue;
     }
 
@@ -107,14 +99,18 @@ function* __treeFromStream(tags, options) {
       continue;
     }
 
-    if (tag.type === GapTag && !path.held) {
-      if (!expressionsCo.current || !expressionsCo.done) {
-        expressionsCo.advance();
+    if (tagType === GapTag && !path.held) {
+      if (!expressionsStep || !expressionsStep.done) {
+        expressionsStep = expressionsIter.next();
+        while (expressionsStep === null || expressionsStep instanceof Promise) {
+          if (expressionsStep === null) yield continue_(), (expressionsStep = iter.next());
+          if (expressionsStep instanceof Promise) expressionsStep = yield wait(expressionsStep);
+        }
       }
 
       if (path.node.value.flags.token) {
-        if (expressionsCo.done || expressionsCo.value != null) {
-          let node = expressionsCo.value;
+        if (expressionsStep.done || expressionsStep.value != null) {
+          let node = expressionsStep.value;
           if (isGapNode(node)) {
             throw new Error('not implemented');
           } else {
@@ -122,11 +118,11 @@ function* __treeFromStream(tags, options) {
           }
         }
       } else {
-        let node = expressionsCo.done
+        let node = expressionsStep.done
           ? buildNode(buildGapTag())
-          : path == null || expressionsCo.value == null
+          : path == null || expressionsStep.value == null
           ? buildNode(buildNullTag())
-          : expressionsCo.value;
+          : expressionsStep.value;
 
         path = path.advance(node);
       }
@@ -142,33 +138,30 @@ function* __treeFromStream(tags, options) {
   return path?.node;
 }
 
-export const treeFromStream = (tags, options = {}) => __treeFromStream(tags, options);
-
-export const treeFromStreamSync = (tokens, options = {}) => {
-  return evaluateReturnSync(treeFromStream(tokens, options));
-};
-
-export const treeFromStreamAsync = async (tokens, options = {}) => {
-  return evaluateReturnAsync(treeFromStream(tokens, options));
+export const treeFromStream = (tags, options = freeze({})) => {
+  let iter = new StreamGenerator(__treeFromStream(tags, options));
+  return maybeWait(iter.next(), (step) => step.value);
 };
 
 function* __isEmpty(tags) {
-  const co = new Coroutine(getStreamIterator(tags));
+  let iter = getStreamIterator(tags);
+  let step;
 
   for (;;) {
-    co.advance();
-
-    if (co.current instanceof Promise) {
-      co.current = yield wait(co.current);
+    step = iter.next();
+    while (step === null || step instanceof Promise) {
+      if (step === null) yield continue_(), (step = iter.next());
+      if (step instanceof Promise) step = yield wait(step);
     }
-    if (co.done) break;
+    if (step.done) break;
 
     let depth = 0;
     let ref = null;
 
-    const tag = co.value;
+    const tag = step.value;
+    let tagType = parseTagType(tag);
 
-    switch (tag.type) {
+    switch (tagType) {
       case ReferenceTag:
         ref = tag;
         break;
@@ -176,7 +169,7 @@ function* __isEmpty(tags) {
       case OpenNodeTag:
         ++depth;
 
-        if (tag.value.literalValue) return false;
+        if (parseTag(tag).value.literalValue) return false;
 
         if (depth === 0 && ref.type === '@') {
           return false;
@@ -204,73 +197,76 @@ export const printCSTML = (tags) => {
   return stringFromStream(generateCSTML(tags));
 };
 
+export function* streamFromString(input) {
+  let p = buildParser(input);
+  let { str } = p;
+  let chr = str[p.idx];
+
+  while (' \t\r\n'.includes(chr)) chr = str[++p.idx];
+
+  while (p.idx < str.length) {
+    yield printTag(parseTag(p));
+    chr = str[p.idx];
+
+    while (' \t\r\n'.includes(chr)) chr = str[++p.idx];
+  }
+}
+
 function* __emptyStreamIterator() {}
 
 export const emptyStreamIterator = () => new StreamIterable(__emptyStreamIterator());
 
-export const asyncStringFromStream = async (stream) => {
-  const co = new Coroutine(getStreamIterator(stream));
+function* __stringFromStream(stream) {
+  let iter = getStreamIterator(stream);
+  let step;
   let str = '';
 
   for (;;) {
-    co.advance();
-
-    if (co.current instanceof Promise) {
-      co.current = await co.current;
+    step = iter.next();
+    while (step === null || step instanceof Promise) {
+      if (step === null) yield continue_(), (step = iter.next());
+      if (step instanceof Promise) step = yield wait(step);
     }
+    if (step.done) break;
 
-    if (co.done) break;
-
-    const chr = co.value;
+    let chr = step.value;
 
     str += chr;
   }
 
   return str;
-};
+}
 
 export const stringFromStream = (stream) => {
-  const co = new Coroutine(stream[Symbol.iterator]());
-  let str = '';
-
-  for (;;) {
-    co.advance();
-
-    if (co.done) break;
-
-    const chr = co.value;
-
-    str += chr;
-  }
-
-  return str;
+  return evaluateReturn(__stringFromStream(stream));
 };
 
 function* __generateCSTML(tags, options) {
   if (!tags) {
-    yield* '<//>';
+    yield* 'null';
     return;
   }
 
   let prevTag = null;
-
-  const co = new Coroutine(getStreamIterator(prettyGroupTags(tags)));
+  let iter = getStreamIterator(prettyGroupTags(tags));
+  let step;
 
   for (;;) {
-    co.advance();
-
-    if (co.current instanceof Promise) {
-      co.current = yield wait(co.current);
+    step = iter.next();
+    while (step === null || step instanceof Promise) {
+      if (step === null) yield continue_(), (step = iter.next());
+      if (step instanceof Promise) step = yield wait(step);
     }
-    if (co.done) break;
+    if (step.done) break;
 
-    const tag = co.value;
+    const tag = step.value;
+    let tagType = parseTagType(tag);
 
-    if (tag.type === ReferenceTag && prevTag.type === NullTag) {
+    if (tagType === ReferenceTag && prevTag.type === NullTag) {
       yield* ' ';
     }
 
-    if (tag.type === 'Effect') {
+    if (tagType === 'Effect') {
       continue;
     }
 
@@ -280,11 +276,11 @@ function* __generateCSTML(tags, options) {
   }
 }
 
-export const generateCSTML = (tags, options = {}) =>
+export const generateCSTML = (tags, options = freeze({})) =>
   new StreamIterable(__generateCSTML(tags, options));
 
 const isToken = (tag) => {
-  return tag.value.flags.token;
+  return parseTag(tag).value.flags.token;
 };
 
 export const prettyGroupTags = (tags) => new StreamIterable(__prettyGroupTags(tags));
@@ -292,19 +288,19 @@ export const prettyGroupTags = (tags) => new StreamIterable(__prettyGroupTags(ta
 function* __prettyGroupTags(tags) {
   let states = emptyStack.push({ holding: [], broken: false, open: null });
   let state = states.value;
-
-  const co = new Coroutine(getStreamIterator(tags));
+  let iter = getStreamIterator(tags);
+  let step;
 
   for (;;) {
-    co.advance();
+    step = iter.next();
 
-    if (co.current instanceof Promise) {
-      co.current = yield wait(co.current);
+    while (step === null || step instanceof Promise) {
+      if (step === null) yield continue_(), (step = iter.next());
+      if (step instanceof Promise) step = yield wait(step);
     }
+    if (step.done) break;
 
-    if (co.done) break;
-
-    const tag = co.value;
+    const tag = parseTag(step.value);
     const isOpenClose = tag.type === CloseNodeTag || tag.type === OpenNodeTag;
 
     if (
@@ -367,6 +363,99 @@ function* __prettyGroupTags(tags) {
   }
 }
 
+function* __transformStream(tags, stream, strategy) {
+  let iter = getStreamIterator(tags);
+  let step;
+  let str = '';
+  let stream_ = 1;
+  let transforming = stream === stream_;
+  let alreadyAdvanced = false;
+
+  function* __streamTags() {
+    for (;;) {
+      while (step === null || step instanceof Promise) {
+        if (step === null) yield continue_(), (step = iter.next());
+        if (step instanceof Promise) step = yield wait(step);
+      }
+      if (step.done) break;
+
+      let tag = parseTag(step.value);
+
+      if (tag.type === StreamTag) {
+        if (tag.value.stream !== stream) {
+          transforming = false;
+          yield continue_();
+        }
+
+        stream_ = tag.value.stream;
+      } else {
+        yield step.value;
+      }
+
+      step = iter.next();
+    }
+  }
+
+  let strategyIter = getStreamIterator(strategy(new StreamGenerator(__streamTags())));
+  let strategyStep;
+
+  for (;;) {
+    if (!alreadyAdvanced) {
+      step = iter.next();
+    }
+    alreadyAdvanced = false;
+    while (step === null || step instanceof Promise) {
+      if (step === null) yield continue_(), (step = iter.next());
+      if (step instanceof Promise) step = yield wait(step);
+    }
+    if (step.done) break;
+
+    let tag = parseTag(step.value);
+
+    if (transforming) {
+      strategyStep = strategyIter.next();
+
+      while (strategyStep !== null) {
+        if (strategyStep instanceof Promise) strategyStep = yield wait(strategyStep);
+        if (strategyStep) {
+          if (strategyStep.done) break;
+          yield strategyStep.value;
+
+          strategyStep = strategyIter.next();
+        }
+      }
+      if (step.done) break;
+      tag = parseTag(step.value);
+      if (tag.type === StreamTag) {
+        stream_ = tag.value.stream;
+      } else {
+        throw new Error();
+      }
+      yield step.value;
+      transforming = stream_ === stream;
+    } else {
+      if (tag.type === StreamTag) {
+        stream_ = tag.value.stream;
+
+        yield step.value;
+
+        if (stream_ === stream) {
+          transforming = true;
+          alreadyAdvanced = true;
+        } else {
+          transforming = false;
+        }
+      } else {
+        yield step.value;
+      }
+    }
+  }
+}
+
+export const transformStream = (tags, stream, strategy) => {
+  return new StreamIterable(__transformStream(tags, stream, strategy));
+};
+
 let pushFrame = (parent = null) => {
   let heldProperties = parent ? [...parent.heldProperties] : [];
   if (parent) {
@@ -380,7 +469,7 @@ let pushFrame = (parent = null) => {
     open: null,
     coverDepth: 0,
     heldProperties,
-    expanded: false,
+    expanded: !parent,
     shifting: parent?.shifting || false,
   };
 };
@@ -406,7 +495,7 @@ function* emitHeld(frame, nextTag) {
     yield buildChild(ReferenceTag, reference);
     let isAnonymous = node.value.flags.token && !node.value.name;
     if (!isAnonymous) {
-      for (let binding of bindings) {
+      for (let binding of arrayValues(bindings)) {
         yield buildChild(BindingTag, binding);
       }
     }
@@ -424,16 +513,18 @@ export const hoistTrivia = (tags) => {
 function* __hoistTriviaOuter(co) {
   co.advance();
 
-  if (co.current instanceof Promise) {
-    co.current = yield wait(co.current);
+  while (co.current === null || co.current instanceof Promise) {
+    if (co.current === null) yield continue_(), co.advance();
+    if (co.current instanceof Promise) co.current = yield wait(co.current);
   }
 
-  let firstTag = co.value;
+  let firstTag = parseTag(co.value);
 
   co.advance();
 
-  if (co.current instanceof Promise) {
-    co.current = yield wait(co.current);
+  while (co.current === null || co.current instanceof Promise) {
+    if (co.current === null) yield continue_(), co.advance();
+    if (co.current instanceof Promise) co.current = yield wait(co.current);
   }
 
   yield* __hoistTrivia(co, firstTag);
@@ -443,25 +534,25 @@ function* __hoistTrivia(co, firstTag) {
   let frame = null;
   let lastFrame = null;
   let alreadyAdvanced = true;
-  let tag;
+  let tag = firstTag;
+
   let nextTag;
 
-  tag = firstTag;
-  nextTag = co.value;
+  nextTag = parseTag(co.value);
 
   for (;;) {
     if (co.done || !nextTag) break;
-
     if (!alreadyAdvanced) {
       co.advance();
 
-      if (co.current instanceof Promise) {
-        co.current = yield wait(co.current);
+      while (co.current === null || co.current instanceof Promise) {
+        if (co.current === null) yield continue_(), co.advance();
+        if (co.current instanceof Promise) co.current = yield wait(co.current);
       }
     }
     alreadyAdvanced = false;
 
-    nextTag = co.value;
+    nextTag = co.done ? nextTag : parseTag(co.value);
 
     if (tag.type === OpenNodeTag) {
       let isCover = tag.value.type === Symbol.for('_');
@@ -471,21 +562,53 @@ function* __hoistTrivia(co, firstTag) {
       if (frame?.ref.type === '#' && !frame.ref.name) {
         let { heldProperties } = frame;
         if (heldProperties.length) {
-          let property = heldProperties[heldProperties.length - 1];
+          let heldProperty = heldProperties[heldProperties.length - 1];
 
-          if (property.node) {
+          if (heldProperty.node) {
             throw new Error();
           }
 
-          let node = yield* treeFromStream(new StreamIterable(__hoistTrivia(co, tag)));
+          let node = treeFromStream(new StreamIterable(__hoistTrivia(co, tag)));
+
+          if (node instanceof Promise) {
+            node = yield wait(node);
+          }
 
-          frame.heldProperties[heldProperties.length - 1] = buildProperty([
-            property.tags[0],
-            property.tags[1] || [],
-            node,
-          ]);
+          if (node.value.type === Symbol.for('_')) {
+            let leading = true;
+            for (let property of Tags.traverse(node.value.children)) {
+              if (property.value.reference.type === '_') {
+                leading = false;
+                frame.heldProperties[heldProperties.length - 1] = buildProperty(
+                  BList.fromValues(
+                    [
+                      heldProperty.tags[1][0],
+                      heldProperty.tags[1][1] || Tags.fromValues([]),
+                      getRoot(node),
+                    ],
+                    1,
+                  ),
+                );
+              } else {
+                if (leading) {
+                  let top = frame.heldProperties.pop();
+                  frame.heldProperties.push(property);
+                  frame.heldProperties.push(top);
+                } else {
+                  frame.heldProperties.push(property);
+                }
+              }
+            }
+          } else {
+            frame.heldProperties[heldProperties.length - 1] = buildProperty(
+              BList.fromValues(
+                [heldProperty.tags[1][0], heldProperty.tags[1][1] || Tags.fromValues([]), node],
+                1,
+              ),
+            );
+          }
 
-          nextTag = co.value;
+          nextTag = parseTag(co.value);
 
           lastFrame = frame;
 
@@ -493,7 +616,7 @@ function* __hoistTrivia(co, firstTag) {
 
           if (
             !frame.ref.flags.expression &&
-            (frame.expanded || frame.parent?.open.value.type !== Symbol.for('_'))
+            (frame.expanded || frame.parent?.open?.value.type !== Symbol.for('_'))
           ) {
             yield* emitHeld(frame, nextTag);
           }
@@ -501,7 +624,7 @@ function* __hoistTrivia(co, firstTag) {
           done = true;
         }
       } else if (frame && !isCover) {
-        if (!frame.shifting && frame.ref && frame.ref.type !== '_') {
+        if (!frame.shifting && ((frame.ref && frame.ref.type !== '_') || frame.expanded)) {
           yield buildChild(ReferenceTag, frame.ref);
           let isAnonymous = tag.value.flags.token && !tag.value.name;
           if (!isAnonymous) {
@@ -549,7 +672,7 @@ function* __hoistTrivia(co, firstTag) {
             yield tag;
           }
         } else {
-          if (!isCover) {
+          if (!isCover || !parentFrame) {
             yield tag;
           }
         }
@@ -560,13 +683,7 @@ function* __hoistTrivia(co, firstTag) {
       frame = pushFrame(parentFrame);
 
       if (tag.value.type === '#' && !tag.value.name) {
-        if (parentFrame.depth === 0 && parentFrame.coverDepth && !parentFrame.expanded) {
-          yield parentFrame.open;
-
-          parentFrame.expanded = true;
-        }
-
-        frame.heldProperties.push(buildProperty([tag]));
+        frame.heldProperties.push(buildProperty(BList.fromValues([printTag(tag)])));
       } else {
         if (!parentFrame.shifting) {
           yield* emitHeld(frame, nextTag);
@@ -577,8 +694,9 @@ function* __hoistTrivia(co, firstTag) {
       frame.ref = tag.value.type === '_' ? parentFrame.ref : tag.value;
       frame.bindings = parentFrame && tag.value.type === '_' ? [...parentFrame.bindings] : [];
       frame.shifting = parentFrame && tag.value.type === '_' ? parentFrame.shifting : false;
+      frame.expanded = frame.depth === 1 && frame.coverDepth === 1;
 
-      if (tag.value.type === '_' && parentFrame.expanded) {
+      if (tag.value.type === '_' && parentFrame.expanded && parentFrame.depth) {
         yield tag;
       }
     } else if (tag.type === ShiftTag) {
@@ -588,7 +706,7 @@ function* __hoistTrivia(co, firstTag) {
 
       frame.coverDepth = doneFrame?.coverDepth || 0;
       frame.ref = lastFrame.ref;
-      frame.bindings = doneFrame && tag.value.type === '_' ? [...doneFrame.bindings] : [];
+      frame.bindings = doneFrame && tag.value?.type === '_' ? [...doneFrame.bindings] : [];
       frame.shifting = true;
       // frame.heldProperties = [...doneFrame.heldProperties];
 
@@ -599,10 +717,15 @@ function* __hoistTrivia(co, firstTag) {
         let property = heldProperties[heldProperties.length - 1];
         let { tags, node } = property;
         if (!node) {
-          heldProperties[heldProperties.length - 1] = buildProperty([
-            tags[0],
-            [...(tags[1] || []), tag],
-          ]);
+          heldProperties[heldProperties.length - 1] = buildProperty(
+            BList.fromValues(
+              [
+                tags[1][0] || Tags.fromValues([]),
+                Tags.push(tag, tags[1][1] || Tags.fromValues([])),
+              ],
+              1,
+            ),
+          );
           frame.bindings.push(tag.value);
         }
       }
@@ -611,11 +734,13 @@ function* __hoistTrivia(co, firstTag) {
         frame.ref = buildReference();
       }
 
-      frame.bindings.push(tag.value);
+      if (!heldProperties.length) {
+        frame.bindings.push(tag.value);
+      }
     } else if (tag.type === CloseNodeTag) {
       lastFrame = frame;
 
-      if (!lastFrame.open.value.type || !lastFrame.parent) {
+      if (!lastFrame.open?.value.type || !lastFrame.parent) {
         yield* emitHeld(lastFrame, nextTag);
       }
 
@@ -663,36 +788,30 @@ function* __generatePrettyCSTML(tags, options) {
   let { indent = '  ', inline: inlineOption = true } = options;
 
   if (!tags) {
-    yield* '<//>';
+    yield* 'null';
     return;
   }
 
-  const co = new Coroutine(getStreamIterator(prettyGroupTags(hoistTrivia(tags))));
-  // const co = new Coroutine(getStreamIterator(prettyGroupTags(tags)));
+  let iter = getStreamIterator(prettyGroupTags(hoistTrivia(tags)));
+  let step;
   let indentLevel = 0;
-  let first = true;
   let inline = false;
   let ref = null;
 
   for (;;) {
-    co.advance();
-
-    if (co.current instanceof Promise) {
-      co.current = yield wait(co.current);
+    step = iter.next();
+    while (step === null || step instanceof Promise) {
+      if (step === null) yield continue_(), (step = iter.next());
+      if (step instanceof Promise) step = yield wait(step);
     }
+    if (step.done) break;
 
-    if (co.done) break;
-
-    const tag = co.value;
+    const tag = parseTag(step.value);
 
     if (tag.type === 'Effect') {
       continue;
     }
 
-    if (tag.type === BindingTag && !tag.value.segments?.length) {
-      continue;
-    }
-
     inline =
       inlineOption &&
       inline &&
@@ -702,7 +821,7 @@ function* __generatePrettyCSTML(tags, options) {
         tag.type === BindingTag ||
         (tag.type === OpenNodeTag && tag.value.selfClosing));
 
-    if (!first && !inline) {
+    if (!inline) {
       yield* '\n';
     }
 
@@ -728,8 +847,6 @@ function* __generatePrettyCSTML(tags, options) {
     if (tag.type === OpenNodeTag) {
       indentLevel += tag.value.selfClosing ? 0 : 1;
     }
-
-    first = false;
   }
 
   if (indentLevel !== 0) {
@@ -739,11 +856,11 @@ function* __generatePrettyCSTML(tags, options) {
   yield* '\n';
 }
 
-export const generatePrettyCSTML = (tags, options = {}) => {
+export const generatePrettyCSTML = (tags, options = freeze({})) => {
   return new StreamIterable(__generatePrettyCSTML(tags, options));
 };
 
-export const printPrettyCSTML = (tags, options = {}) => {
+export const printPrettyCSTML = (tags, options = freeze({})) => {
   return stringFromStream(generatePrettyCSTML(tags, options));
 };
 
@@ -756,11 +873,12 @@ export const getCooked = (tags) => {
   let ref = null;
 
   for (const tag of tags) {
+    let tagType = parseTagType(tag);
     if (foundLast) throw new Error();
 
-    switch (tag.type) {
+    switch (tagType) {
       case ReferenceTag: {
-        ref = tag;
+        ref = parseTag(tag);
         if (depth === 1) {
           throw new Error('cookable nodes must not contain other nodes');
         }
@@ -768,7 +886,7 @@ export const getCooked = (tags) => {
       }
 
       case OpenNodeTag: {
-        const { flags, attributes, literalValue, selfClosing } = tag.value;
+        const { flags, attributes, literalValue, selfClosing } = parseTag(tag).value;
 
         depth += selfClosing ? 0 : 1;
 
@@ -776,7 +894,7 @@ export const getCooked = (tags) => {
           if (flags.token) {
             break;
           } else {
-            throw new Error(JSON.stringify(flags));
+            throw new Error(flags);
           }
         }
 
@@ -785,7 +903,7 @@ export const getCooked = (tags) => {
         }
 
         if (ref.type === '@') {
-          const { cooked: cookedValue } = tag.value.attributes;
+          const { cooked: cookedValue } = attributes;
 
           if (!cookedValue) throw new Error('cannot cook string: it contains uncooked escapes');
 
@@ -807,7 +925,7 @@ export const getCooked = (tags) => {
 
       case LiteralTag: {
         if (depth === 1) {
-          cooked += tag.value;
+          cooked += parseTag(tag).value;
         }
         break;
       }
@@ -830,13 +948,17 @@ export const printSource = (tags) => {
   if (!tags) return printed;
 
   for (const tag of tags) {
-    if (tag.type === OpenNodeTag && tag.value.literalValue) {
-      printed += tag.value.literalValue;
-    } else if (tag.type === LiteralTag) {
-      printed += tag.value;
-    } else if (tag.type === ShiftTag) {
+    let tagType = parseTagType(tag);
+    if (tagType === OpenNodeTag) {
+      let { literalValue } = parseTag(tag).value;
+      if (literalValue) {
+        printed += literalValue;
+      }
+    } else if (tagType === LiteralTag) {
+      printed += parseTag(tag).value;
+    } else if (tagType === ShiftTag) {
       held = true;
-    } else if (tag.type === GapTag) {
+    } else if (tagType === GapTag) {
       if (held) {
         held = false;
       } else {
@@ -850,12 +972,39 @@ export const printSource = (tags) => {
 
 export function* generateSourceTextFor(tags) {
   for (const tag of tags) {
-    if (tag.type === LiteralTag) {
-      yield* tag.value;
-    } else if (tag.type === GapTag) {
+    let tagType = parseTagType(tag);
+    if (tagType === LiteralTag) {
+      yield* parseTag(tag).value;
+    } else if (tagType === GapTag) {
       yield null;
     }
   }
 }
 
 export const sourceTextFor = printSource;
+
+export function* interpolateFragment(tags) {
+  let tags_ = isNode(tags) ? Tags.traverseInner(Tags.getTags(tags)) : tags;
+  let lastTag = null;
+  let first = true;
+  let dropFirst = false;
+  let depth = 0;
+  for (let tag of tags_) {
+    let tagType = parseTagType(tag);
+    if (lastTag) yield lastTag;
+
+    if (tagType === OpenNodeTag && !parseTag(tag).value.selfClosing) depth++;
+    if (tagType === CloseNodeTag) depth--;
+
+    dropFirst ||= first && tag === '<__>';
+    lastTag = first && dropFirst ? null : tag;
+    first = false;
+  }
+  if (dropFirst) {
+    if (lastTag !== '</>') throw new Error();
+  } else {
+    yield lastTag;
+  }
+
+  if (depth !== 0) throw new Error();
+}