@bablr/agast-helpers 0.8.0 → 0.10.0

package/lib/stream.js

@@ -1,160 +1,155 @@
 import { Coroutine } from '@bablr/coroutine';
 import emptyStack from '@iter-tools/imm-stack';
-import { printSelfClosingNodeTag, printTag } from './print.js';
-import { buildOpenNodeTag, buildTokenGroup } from './builders.js';
+import { getStreamIterator, StreamIterable, StreamGenerator, wait } from '@bablr/stream-iterator';
+import * as Tags from './tags.js';
+import { printTag } from './print.js';
+import {
+  buildChild,
+  buildGapTag,
+  buildNullTag,
+  buildOpenNodeTag,
+  buildProperty,
+  buildReference,
+} from './builders.js';
 import {
-  DoctypeTag,
   OpenNodeTag,
   CloseNodeTag,
   ReferenceTag,
-  ShiftTag,
   GapTag,
   NullTag,
-  InitializerTag,
   LiteralTag,
-  TokenGroup,
   BindingTag,
+  DoctypeTag,
+  ShiftTag,
   AttributeDefinition,
 } from './symbols.js';
+import { buildNode, isGapNode, Path } from './path.js';
+import { streamFromTree } from './tree.js';
 
-export * from './print.js';
-
-export const getStreamIterator = (obj) => {
-  return obj[Symbol.for('@@streamIterator')]?.() || obj[Symbol.iterator]?.();
-};
+export { getStreamIterator, StreamIterable, StreamGenerator, wait };
 
-export class SyncGenerator {
-  constructor(embeddedGenerator) {
-    if (!embeddedGenerator.next) throw new Error();
+export * from './print.js';
 
-    this.generator = embeddedGenerator;
+export const maybeWait = (maybePromise, callback) => {
+  if (maybePromise instanceof Promise) {
+    return maybePromise.then(callback);
+  } else {
+    return callback(maybePromise);
   }
+};
 
-  next(value) {
-    const step = this.generator.next(value);
+function* __flatMap(fn, tags) {
+  const co = new Coroutine(getStreamIterator(tags));
 
-    if (step instanceof Promise) {
-      throw new Error('invalid embedded generator');
-    }
+  for (;;) {
+    co.advance();
 
-    if (step.done) {
-      return step;
-    } else if (step.value instanceof Promise) {
-      throw new Error('sync generators cannot resolve promises');
-    } else {
-      return step;
+    if (co.current instanceof Promise) {
+      co.current = yield wait(co.current);
     }
-  }
+    if (co.done) break;
 
-  return(value) {
-    const step = this.generator.return(value);
-    if (step instanceof Promise) {
-      throw new Error('invalid embedded generator');
-    }
+    let { value } = co;
 
-    if (step.value instanceof Promise) {
-      throw new Error('sync generators cannot resolve promises');
-    }
-    return step;
+    yield* fn(value);
   }
 
-  [Symbol.iterator]() {
-    return this;
-  }
+  return true;
 }
 
-export class AsyncGenerator {
-  constructor(embeddedGenerator) {
-    this.generator = embeddedGenerator;
-  }
+export const flatMap = (fn, tags) => new StreamIterable(__flatMap(fn, tags));
 
-  next(value) {
-    const step = this.generator.next(value);
+export const evaluateReturnSync = (iterable) => {
+  const co = new Coroutine(iterable[Symbol.iterator]());
+  while (!co.done) co.advance();
+  return co.value;
+};
 
-    if (step instanceof Promise) {
-      throw new Error('invalid embedded generator');
-    }
+export const evaluateReturnAsync = async (iterable) => {
+  const co = new Coroutine(getStreamIterator(iterable));
+  while (!co.done) {
+    co.advance();
 
-    if (step.done) {
-      return Promise.resolve(step);
-    } else if (step.value instanceof Promise) {
-      return step.value.then((value) => {
-        return this.next(value);
-      });
-    } else {
-      return Promise.resolve(step);
+    if (co.current instanceof Promise) {
+      co.current = await co.current;
     }
   }
+  return co.value;
+};
 
-  return(value) {
-    const result = this.generator.return(value);
-    if (result instanceof Promise) {
-      throw new Error('sync generators cannot resolve promises');
-    }
-    return result;
-  }
+function* __treeFromStream(tags, options) {
+  let path = null;
+  const co = new Coroutine(getStreamIterator(tags));
+  const expressionsCo = new Coroutine(getStreamIterator(options.expressions || []));
 
-  [Symbol.asyncIterator]() {
-    return this;
-  }
-}
+  for (;;) {
+    co.advance();
 
-export class StreamGenerator {
-  constructor(embeddedGenerator) {
-    this.generator = embeddedGenerator;
-  }
+    if (co.current instanceof Promise) {
+      co.current = yield wait(co.current);
+    }
 
-  next(value) {
-    const step = this.generator.next(value);
+    if (co.done) break;
 
-    if (step.done) {
-      return step;
-    } else if (step.value instanceof Promise) {
-      return step.value.then((value) => {
-        return this.next(value);
-      });
-    } else {
-      return step;
+    let tag = co.value;
+
+    if (tag.type === 'Effect' || tag.type === DoctypeTag) {
+      continue;
     }
-  }
 
-  return(value) {
-    return this.generator.return(value);
-  }
+    if (!path) {
+      path = Path.from(buildNode(Tags.fromValues([tag])));
 
-  [Symbol.iterator]() {
-    return this;
-  }
+      // if (tag.value.selfClosing) {
+      //   path = null;
+      // }
 
-  [Symbol.for('@@streamIterator')]() {
-    return this;
-  }
-}
+      continue;
+    }
 
-export class StreamIterable {
-  constructor(embeddedStreamIterable) {
-    this.iterable = embeddedStreamIterable;
-  }
+    if (tag.type === GapTag && !path.held) {
+      if (!expressionsCo.current || !expressionsCo.done) {
+        expressionsCo.advance();
+      }
 
-  [Symbol.iterator]() {
-    return new SyncGenerator(this.iterable);
-  }
+      if (path.node.value.flags.token) {
+        if (expressionsCo.done || expressionsCo.value != null) {
+          let node = expressionsCo.value;
+          if (isGapNode(node)) {
+            throw new Error('not implemented');
+          } else {
+            path = path.advance(tag);
+          }
+        }
+      } else {
+        let node = expressionsCo.done
+          ? buildNode(buildGapTag())
+          : path == null || expressionsCo.value == null
+          ? buildNode(buildNullTag())
+          : expressionsCo.value;
 
-  [Symbol.asyncIterator]() {
-    return new AsyncGenerator(this.iterable);
+        path = path.advance(node);
+      }
+    } else {
+      path = path.advance(tag);
+    }
   }
 
-  [Symbol.for('@@streamIterator')]() {
-    return new StreamGenerator(this.iterable);
+  if (!path.done) {
+    throw new Error('imbalanced tag stack');
   }
+
+  return path?.node;
 }
 
-export const maybeWait = (maybePromise, callback) => {
-  if (maybePromise instanceof Promise) {
-    return maybePromise.then(callback);
-  } else {
-    return callback(maybePromise);
-  }
+export const treeFromStream = (tags, options = {}) => __treeFromStream(tags, options);
+
+export const treeFromStreamSync = (tokens, options = {}) => {
+  return evaluateReturnSync(treeFromStream(tokens, options));
+};
+
+export const treeFromStreamAsync = async (tokens, options = {}) => {
+  return evaluateReturnAsync(treeFromStream(tokens, options));
 };
 
 function* __isEmpty(tags) {
@@ -164,7 +159,7 @@ function* __isEmpty(tags) {
     co.advance();
 
     if (co.current instanceof Promise) {
-      co.current = yield co.current;
+      co.current = yield wait(co.current);
     }
     if (co.done) break;
 
@@ -181,7 +176,9 @@ function* __isEmpty(tags) {
       case OpenNodeTag:
         ++depth;
 
-        if (depth === 0 && ref.value.type === '@') {
+        if (tag.value.literalValue) return false;
+
+        if (depth === 0 && ref.type === '@') {
           return false;
         }
 
@@ -224,9 +221,9 @@ export const asyncStringFromStream = async (stream) => {
 
     if (co.done) break;
 
-    const tag = co.value;
+    const chr = co.value;
 
-    str += printTag(tag);
+    str += chr;
   }
 
   return str;
@@ -263,7 +260,7 @@ function* __generateCSTML(tags, options) {
     co.advance();
 
     if (co.current instanceof Promise) {
-      co.current = yield co.current;
+      co.current = yield wait(co.current);
     }
     if (co.done) break;
 
@@ -277,17 +274,10 @@ function* __generateCSTML(tags, options) {
       continue;
     }
 
-    if (tag.type === TokenGroup) {
-      const intrinsicValue = getCooked(tag.value);
-      yield* printSelfClosingNodeTag(tag.value[0], intrinsicValue);
-    } else {
-      yield* printTag(tag);
-    }
+    yield* printTag(tag);
 
     prevTag = tag;
   }
-
-  yield* '\n';
 }
 
 export const generateCSTML = (tags, options = {}) =>
@@ -305,24 +295,18 @@ function* __prettyGroupTags(tags) {
 
   const co = new Coroutine(getStreamIterator(tags));
 
-  let ref = null;
-
   for (;;) {
     co.advance();
 
-    if (co.done) break;
-
     if (co.current instanceof Promise) {
-      co.current = yield co.current;
+      co.current = yield wait(co.current);
     }
 
+    if (co.done) break;
+
     const tag = co.value;
     const isOpenClose = tag.type === CloseNodeTag || tag.type === OpenNodeTag;
 
-    if (tag.type === ReferenceTag) {
-      ref = tag;
-    }
-
     if (
       (tag.type === 'Effect' && tag.value.verb === 'write') ||
       [
@@ -331,11 +315,10 @@ function* __prettyGroupTags(tags) {
         BindingTag,
         GapTag,
         NullTag,
-        InitializerTag,
         ShiftTag,
         AttributeDefinition,
-      ].includes(tag.type) ||
-      (tag.type === OpenNodeTag && (!tag.value.type || ref?.value.type === '@'))
+        OpenNodeTag,
+      ].includes(tag.type)
     ) {
       state.broken = true;
 
@@ -353,8 +336,14 @@ function* __prettyGroupTags(tags) {
 
     if (tag.type === CloseNodeTag) {
       if (!state.broken && (isToken(state.open) || state.holding.length === 1)) {
-        state.holding.push(tag);
-        yield buildTokenGroup(state.holding);
+        let { flags, name, attributes } = state.holding[0].value;
+
+        let literal = state.holding
+          .slice(1)
+          .map((lit) => lit.value)
+          .join('');
+
+        yield buildOpenNodeTag(flags, name, literal, attributes, true);
       } else {
         if (state.holding.length) {
           yield* state.holding;
@@ -367,13 +356,304 @@ function* __prettyGroupTags(tags) {
     }
 
     if (tag.type === OpenNodeTag) {
-      states = states.push({ holding: [tag], broken: false, open: tag });
+      if (tag.value.selfClosing) {
+        yield tag;
+      } else {
+        states = states.push({ holding: [tag], broken: false, open: tag });
 
-      state = states.value;
+        state = states.value;
+      }
     }
   }
 }
 
+let pushFrame = (parent = null) => {
+  let heldProperties = parent ? [...parent.heldProperties] : [];
+  if (parent) {
+    parent.heldProperties = [];
+  }
+  return {
+    parent,
+    depth: !parent ? 0 : parent.depth + 1,
+    ref: null,
+    bindings: [],
+    open: null,
+    coverDepth: 0,
+    heldProperties,
+    expanded: false,
+    shifting: parent?.shifting || false,
+  };
+};
+
+let popFrame = (frame) => {
+  let { heldProperties } = frame;
+
+  if (frame.parent) {
+    frame.parent.heldProperties = heldProperties;
+    frame.parent.shifting = frame.shifting;
+  } else if (heldProperties.length) {
+    throw new Error();
+  }
+
+  return frame.parent;
+};
+
+function* emitHeld(frame, nextTag) {
+  if (frame.ref?.flags.expression && nextTag?.type === ShiftTag) return;
+
+  for (let property of frame.heldProperties) {
+    let { reference, bindings, node } = property;
+    yield buildChild(ReferenceTag, reference);
+    let isAnonymous = node.value.flags.token && !node.value.name;
+    if (!isAnonymous) {
+      for (let binding of bindings) {
+        yield buildChild(BindingTag, binding);
+      }
+    }
+    yield* streamFromTree(node);
+  }
+  frame.heldProperties = [];
+}
+
+export const hoistTrivia = (tags) => {
+  const co = new Coroutine(getStreamIterator(tags));
+
+  return new StreamIterable(__hoistTriviaOuter(co));
+};
+
+function* __hoistTriviaOuter(co) {
+  co.advance();
+
+  let firstTag = co.value;
+
+  co.advance();
+
+  yield* __hoistTrivia(co, firstTag);
+}
+
+function* __hoistTrivia(co, firstTag) {
+  if (co.current instanceof Promise) {
+    co.current = yield wait(co.current);
+  }
+  let frame = null;
+  let lastFrame = null;
+  let alreadyAdvanced = true;
+  let tag;
+  let nextTag;
+
+  tag = firstTag;
+  nextTag = co.value;
+
+  for (;;) {
+    if (co.done || !nextTag) break;
+
+    if (!alreadyAdvanced) {
+      co.advance();
+
+      if (co.current instanceof Promise) {
+        co.current = yield wait(co.current);
+      }
+    }
+    alreadyAdvanced = false;
+
+    nextTag = co.value;
+
+    if (tag.type === OpenNodeTag) {
+      let isCover = tag.value.type === Symbol.for('_');
+      let parentFrame = frame?.ref && !frame.open ? frame.parent : frame;
+      let done = false;
+
+      if (frame?.ref.type === '#' && !frame.ref.name) {
+        let { heldProperties } = frame;
+        if (heldProperties.length) {
+          let property = heldProperties[heldProperties.length - 1];
+
+          if (property.node) {
+            throw new Error();
+          }
+
+          let node = yield* treeFromStream(new StreamIterable(__hoistTrivia(co, tag)));
+
+          frame.heldProperties[heldProperties.length - 1] = buildProperty([
+            property.tags[0],
+            property.tags[1] || [],
+            node,
+          ]);
+
+          nextTag = co.value;
+
+          lastFrame = frame;
+
+          frame = popFrame(frame);
+
+          if (
+            !frame.ref.flags.expression &&
+            (frame.expanded || frame.parent?.open.value.type !== Symbol.for('_'))
+          ) {
+            yield* emitHeld(frame, nextTag);
+          }
+
+          done = true;
+        }
+      } else if (frame && !isCover) {
+        if (!frame.shifting && frame.ref && frame.ref.type !== '_') {
+          yield buildChild(ReferenceTag, frame.ref);
+          let isAnonymous = tag.value.flags.token && !tag.value.name;
+          if (!isAnonymous) {
+            for (let binding of frame.bindings) {
+              yield buildChild(BindingTag, binding);
+            }
+          }
+        }
+      }
+
+      if (!done) {
+        if (!frame?.ref || frame.open) {
+          frame = pushFrame(parentFrame);
+
+          frame.heldProperties = parentFrame?.heldProperties ? [...parentFrame.heldProperties] : [];
+
+          if (tag.value.type === Symbol.for('__')) {
+            frame.open = tag;
+            frame.ref = buildReference();
+          } else if (isCover) {
+            frame.ref = parentFrame?.ref || buildReference('_');
+            frame.bindings =
+              parentFrame && tag.value.type === Symbol.for('_') ? [...parentFrame.bindings] : [];
+            frame.shifting =
+              parentFrame && tag.value.type === Symbol.for('_') ? parentFrame.shifting : false;
+          } else {
+            frame.ref = buildReference();
+          }
+        }
+
+        frame.coverDepth = isCover ? (parentFrame?.coverDepth ?? 0) + 1 : 0;
+
+        frame.open = tag;
+
+        if (tag.value.selfClosing) {
+          lastFrame = frame;
+
+          frame = popFrame(frame);
+
+          if (!frame) {
+            yield tag;
+
+            break;
+          } else {
+            yield tag;
+          }
+        } else {
+          if (!isCover) {
+            yield tag;
+          }
+        }
+      }
+    } else if (tag.type === ReferenceTag) {
+      let parentFrame = frame;
+
+      frame = pushFrame(parentFrame);
+
+      if (tag.value.type === '#' && !tag.value.name) {
+        if (parentFrame.depth === 0 && parentFrame.coverDepth && !parentFrame.expanded) {
+          yield parentFrame.open;
+
+          parentFrame.expanded = true;
+        }
+
+        frame.heldProperties.push(buildProperty([tag]));
+      } else {
+        if (!parentFrame.shifting) {
+          yield* emitHeld(frame, nextTag);
+        }
+      }
+
+      frame.coverDepth = parentFrame.coverDepth;
+      frame.ref = tag.value.type === '_' ? parentFrame.ref : tag.value;
+      frame.bindings = parentFrame && tag.value.type === '_' ? [...parentFrame.bindings] : [];
+      frame.shifting = parentFrame && tag.value.type === '_' ? parentFrame.shifting : false;
+
+      if (tag.value.type === '_' && parentFrame.expanded) {
+        yield tag;
+      }
+    } else if (tag.type === ShiftTag) {
+      let doneFrame = frame;
+
+      frame = pushFrame(doneFrame);
+
+      frame.coverDepth = doneFrame?.coverDepth || 0;
+      frame.ref = lastFrame.ref;
+      frame.bindings = doneFrame && tag.value.type === '_' ? [...doneFrame.bindings] : [];
+      frame.shifting = true;
+      // frame.heldProperties = [...doneFrame.heldProperties];
+
+      yield tag;
+    } else if (tag.type === BindingTag) {
+      let { heldProperties } = frame;
+      if (heldProperties.length) {
+        let property = heldProperties[heldProperties.length - 1];
+        let { tags, node } = property;
+        if (!node) {
+          heldProperties[heldProperties.length - 1] = buildProperty([
+            tags[0],
+            [...(tags[1] || []), tag],
+          ]);
+          frame.bindings.push(tag.value);
+        }
+      }
+      if (!frame.ref) {
+        frame = pushFrame(frame);
+        frame.ref = buildReference();
+      }
+
+      frame.bindings.push(tag.value);
+    } else if (tag.type === CloseNodeTag) {
+      lastFrame = frame;
+
+      if (!lastFrame.open.value.type || !lastFrame.parent) {
+        yield* emitHeld(lastFrame, nextTag);
+      }
+
+      frame = popFrame(frame);
+
+      if (frame) {
+        frame.heldProperties = [...lastFrame.heldProperties];
+        if (!lastFrame?.coverDepth) {
+          yield tag;
+        }
+      } else {
+        if (lastFrame.expanded || lastFrame.open.value.type !== Symbol.for('_')) {
+          yield tag;
+        }
+        break;
+      }
+    } else if (tag.type === GapTag || tag.type === NullTag) {
+      if (frame.ref) {
+        yield buildChild(ReferenceTag, frame.ref);
+      }
+      lastFrame = frame;
+
+      frame = popFrame(frame);
+
+      if (tag.type === GapTag && frame.heldProperties.length) {
+        yield tag;
+
+        frame.shifting = false;
+
+        yield* emitHeld(frame, nextTag);
+      } else {
+        yield tag;
+      }
+    } else {
+      yield tag;
+    }
+
+    tag = nextTag;
+  }
+
+  if (frame) throw new Error();
+}
+
 function* __generatePrettyCSTML(tags, options) {
   let { indent = '  ', inline: inlineOption = true } = options;
 
@@ -382,7 +662,8 @@ function* __generatePrettyCSTML(tags, options) {
     return;
   }
 
-  const co = new Coroutine(getStreamIterator(prettyGroupTags(tags)));
+  const co = new Coroutine(getStreamIterator(prettyGroupTags(hoistTrivia(tags))));
+  // const co = new Coroutine(getStreamIterator(prettyGroupTags(tags)));
   let indentLevel = 0;
   let first = true;
   let inline = false;
@@ -391,19 +672,19 @@ function* __generatePrettyCSTML(tags, options) {
   for (;;) {
     co.advance();
 
-    if (co.done) break;
-
     if (co.current instanceof Promise) {
-      co.current = yield co.current;
+      co.current = yield wait(co.current);
     }
 
+    if (co.done) break;
+
     const tag = co.value;
 
     if (tag.type === 'Effect') {
       continue;
     }
 
-    if (tag.type === BindingTag && !tag.value.languagePath?.length) {
+    if (tag.type === BindingTag && !tag.value.segments?.length) {
       continue;
     }
 
@@ -414,8 +695,7 @@ function* __generatePrettyCSTML(tags, options) {
       (tag.type === NullTag ||
         tag.type === GapTag ||
         tag.type === BindingTag ||
-        tag.type === InitializerTag ||
-        tag.type === TokenGroup);
+        (tag.type === OpenNodeTag && tag.value.selfClosing));
 
     if (!first && !inline) {
       yield* '\n';
@@ -428,23 +708,12 @@ function* __generatePrettyCSTML(tags, options) {
     }
 
     if (!inline) {
-      yield* indent.repeat(indentLevel);
+      yield* indent.repeat(Math.max(0, indentLevel));
     } else {
       yield* ' ';
     }
 
-    if (tag.type === TokenGroup) {
-      ref = null;
-      const intrinsicValue = tag.value[0].value.flags.token ? getCooked(tag.value) : null;
-      let openTag = tag.value[0];
-      let { flags, type, attributes } = openTag.value;
-      yield* printSelfClosingNodeTag(buildOpenNodeTag(flags, type, attributes), intrinsicValue);
-    } else if (tag.type === OpenNodeTag) {
-      let { flags, type, attributes } = tag.value;
-      yield* printTag(buildOpenNodeTag(flags, type, attributes));
-    } else {
-      yield* printTag(tag);
-    }
+    yield* printTag(tag);
 
     if (tag.type === ReferenceTag) {
       inline = true;
@@ -452,7 +721,7 @@ function* __generatePrettyCSTML(tags, options) {
     }
 
     if (tag.type === OpenNodeTag) {
-      indentLevel++;
+      indentLevel += tag.value.selfClosing ? 0 : 1;
     }
 
     first = false;
@@ -494,9 +763,9 @@ export const getCooked = (tags) => {
       }
 
       case OpenNodeTag: {
-        const { flags, attributes } = tag.value;
+        const { flags, attributes, literalValue, selfClosing } = tag.value;
 
-        depth++;
+        depth += selfClosing ? 0 : 1;
 
         if (first) {
           if (flags.token) {
@@ -506,16 +775,18 @@ export const getCooked = (tags) => {
           }
         }
 
-        if (!(ref.value.type === '#' || (ref.value.type === '@' && attributes.cooked))) {
+        if (!(ref.type === '#' || (ref.type === '@' && attributes.cooked))) {
           throw new Error('cookable nodes must not contain other nodes');
         }
 
-        if (ref.value.type === '@') {
+        if (ref.type === '@') {
           const { cooked: cookedValue } = tag.value.attributes;
 
           if (!cookedValue) throw new Error('cannot cook string: it contains uncooked escapes');
 
           cooked += cookedValue;
+        } else if (literalValue) {
+          cooked += literalValue;
         }
 
         break;
@@ -553,7 +824,9 @@ export const printSource = (tags) => {
   if (!tags) return printed;
 
   for (const tag of tags) {
-    if (tag.type === LiteralTag) {
+    if (tag.type === OpenNodeTag && tag.value.literalValue) {
+      printed += tag.value.literalValue;
+    } else if (tag.type === LiteralTag) {
       printed += tag.value;
     } else if (tag.type === GapTag) {
       throw new Error('use generateSourceTextFor');