@bablr/agast-helpers 0.9.0 → 0.10.0

package/lib/stream.js

@@ -1,23 +1,32 @@
 import { Coroutine } from '@bablr/coroutine';
 import emptyStack from '@iter-tools/imm-stack';
-import { getStreamIterator, StreamIterable, StreamGenerator } from '@bablr/stream-iterator';
+import { getStreamIterator, StreamIterable, StreamGenerator, wait } from '@bablr/stream-iterator';
+import * as Tags from './tags.js';
 import { printTag } from './print.js';
-import { buildOpenNodeTag } from './builders.js';
+import {
+  buildChild,
+  buildGapTag,
+  buildNullTag,
+  buildOpenNodeTag,
+  buildProperty,
+  buildReference,
+} from './builders.js';
 import {
   OpenNodeTag,
   CloseNodeTag,
   ReferenceTag,
   GapTag,
   NullTag,
-  InitializerTag,
   LiteralTag,
   BindingTag,
   DoctypeTag,
   ShiftTag,
   AttributeDefinition,
 } from './symbols.js';
+import { buildNode, isGapNode, Path } from './path.js';
+import { streamFromTree } from './tree.js';
 
-export { getStreamIterator, StreamIterable, StreamGenerator };
+export { getStreamIterator, StreamIterable, StreamGenerator, wait };
 
 export * from './print.js';
 
@@ -29,6 +38,120 @@ export const maybeWait = (maybePromise, callback) => {
   }
 };
 
+function* __flatMap(fn, tags) {
+  const co = new Coroutine(getStreamIterator(tags));
+
+  for (;;) {
+    co.advance();
+
+    if (co.current instanceof Promise) {
+      co.current = yield wait(co.current);
+    }
+    if (co.done) break;
+
+    let { value } = co;
+
+    yield* fn(value);
+  }
+
+  return true;
+}
+
+export const flatMap = (fn, tags) => new StreamIterable(__flatMap(fn, tags));
+
+export const evaluateReturnSync = (iterable) => {
+  const co = new Coroutine(iterable[Symbol.iterator]());
+  while (!co.done) co.advance();
+  return co.value;
+};
+
+export const evaluateReturnAsync = async (iterable) => {
+  const co = new Coroutine(getStreamIterator(iterable));
+  while (!co.done) {
+    co.advance();
+
+    if (co.current instanceof Promise) {
+      co.current = await co.current;
+    }
+  }
+  return co.value;
+};
+
+function* __treeFromStream(tags, options) {
+  let path = null;
+  const co = new Coroutine(getStreamIterator(tags));
+  const expressionsCo = new Coroutine(getStreamIterator(options.expressions || []));
+
+  for (;;) {
+    co.advance();
+
+    if (co.current instanceof Promise) {
+      co.current = yield wait(co.current);
+    }
+
+    if (co.done) break;
+
+    let tag = co.value;
+
+    if (tag.type === 'Effect' || tag.type === DoctypeTag) {
+      continue;
+    }
+
+    if (!path) {
+      path = Path.from(buildNode(Tags.fromValues([tag])));
+
+      // if (tag.value.selfClosing) {
+      //   path = null;
+      // }
+
+      continue;
+    }
+
+    if (tag.type === GapTag && !path.held) {
+      if (!expressionsCo.current || !expressionsCo.done) {
+        expressionsCo.advance();
+      }
+
+      if (path.node.value.flags.token) {
+        if (expressionsCo.done || expressionsCo.value != null) {
+          let node = expressionsCo.value;
+          if (isGapNode(node)) {
+            throw new Error('not implemented');
+          } else {
+            path = path.advance(tag);
+          }
+        }
+      } else {
+        let node = expressionsCo.done
+          ? buildNode(buildGapTag())
+          : path == null || expressionsCo.value == null
+          ? buildNode(buildNullTag())
+          : expressionsCo.value;
+
+        path = path.advance(node);
+      }
+    } else {
+      path = path.advance(tag);
+    }
+  }
+
+  if (!path.done) {
+    throw new Error('imbalanced tag stack');
+  }
+
+  return path?.node;
+}
+
+export const treeFromStream = (tags, options = {}) => __treeFromStream(tags, options);
+
+export const treeFromStreamSync = (tokens, options = {}) => {
+  return evaluateReturnSync(treeFromStream(tokens, options));
+};
+
+export const treeFromStreamAsync = async (tokens, options = {}) => {
+  return evaluateReturnAsync(treeFromStream(tokens, options));
+};
+
 function* __isEmpty(tags) {
   const co = new Coroutine(getStreamIterator(tags));
 
@@ -36,7 +159,7 @@ function* __isEmpty(tags) {
     co.advance();
 
     if (co.current instanceof Promise) {
-      co.current = yield co.current;
+      co.current = yield wait(co.current);
     }
     if (co.done) break;
 
@@ -55,7 +178,7 @@ function* __isEmpty(tags) {
 
         if (tag.value.literalValue) return false;
 
-        if (depth === 0 && ref.value.type === '@') {
+        if (depth === 0 && ref.type === '@') {
           return false;
         }
 
@@ -137,7 +260,7 @@ function* __generateCSTML(tags, options) {
     co.advance();
 
     if (co.current instanceof Promise) {
-      co.current = yield co.current;
+      co.current = yield wait(co.current);
     }
     if (co.done) break;
 
@@ -155,8 +278,6 @@ function* __generateCSTML(tags, options) {
 
     prevTag = tag;
   }
-
-  yield* '\n';
 }
 
 export const generateCSTML = (tags, options = {}) =>
@@ -174,24 +295,18 @@ function* __prettyGroupTags(tags) {
 
   const co = new Coroutine(getStreamIterator(tags));
 
-  let ref = null;
-
   for (;;) {
     co.advance();
 
-    if (co.done) break;
-
     if (co.current instanceof Promise) {
-      co.current = yield co.current;
+      co.current = yield wait(co.current);
     }
 
+    if (co.done) break;
+
     const tag = co.value;
     const isOpenClose = tag.type === CloseNodeTag || tag.type === OpenNodeTag;
 
-    if (tag.type === ReferenceTag) {
-      ref = tag;
-    }
-
     if (
       (tag.type === 'Effect' && tag.value.verb === 'write') ||
       [
@@ -200,11 +315,10 @@ function* __prettyGroupTags(tags) {
         BindingTag,
         GapTag,
         NullTag,
-        InitializerTag,
         ShiftTag,
         AttributeDefinition,
-      ].includes(tag.type) ||
-      (tag.type === OpenNodeTag && (!tag.value.type || ref?.value.type === '@'))
+        OpenNodeTag,
+      ].includes(tag.type)
     ) {
       state.broken = true;
 
@@ -222,14 +336,14 @@ function* __prettyGroupTags(tags) {
 
     if (tag.type === CloseNodeTag) {
       if (!state.broken && (isToken(state.open) || state.holding.length === 1)) {
-        let { flags, type, attributes } = state.holding[0].value;
+        let { flags, name, attributes } = state.holding[0].value;
 
         let literal = state.holding
           .slice(1)
           .map((lit) => lit.value)
           .join('');
 
-        yield buildOpenNodeTag(flags, type, attributes, literal, true);
+        yield buildOpenNodeTag(flags, name, literal, attributes, true);
       } else {
         if (state.holding.length) {
           yield* state.holding;
@@ -242,7 +356,7 @@ function* __prettyGroupTags(tags) {
     }
 
     if (tag.type === OpenNodeTag) {
-      if (tag.value.literalValue) {
+      if (tag.value.selfClosing) {
         yield tag;
       } else {
         states = states.push({ holding: [tag], broken: false, open: tag });
@@ -253,6 +367,293 @@ function* __prettyGroupTags(tags) {
   }
 }
 
+let pushFrame = (parent = null) => {
+  let heldProperties = parent ? [...parent.heldProperties] : [];
+  if (parent) {
+    parent.heldProperties = [];
+  }
+  return {
+    parent,
+    depth: !parent ? 0 : parent.depth + 1,
+    ref: null,
+    bindings: [],
+    open: null,
+    coverDepth: 0,
+    heldProperties,
+    expanded: false,
+    shifting: parent?.shifting || false,
+  };
+};
+
+let popFrame = (frame) => {
+  let { heldProperties } = frame;
+
+  if (frame.parent) {
+    frame.parent.heldProperties = heldProperties;
+    frame.parent.shifting = frame.shifting;
+  } else if (heldProperties.length) {
+    throw new Error();
+  }
+
+  return frame.parent;
+};
+
+function* emitHeld(frame, nextTag) {
+  if (frame.ref?.flags.expression && nextTag?.type === ShiftTag) return;
+
+  for (let property of frame.heldProperties) {
+    let { reference, bindings, node } = property;
+    yield buildChild(ReferenceTag, reference);
+    let isAnonymous = node.value.flags.token && !node.value.name;
+    if (!isAnonymous) {
+      for (let binding of bindings) {
+        yield buildChild(BindingTag, binding);
+      }
+    }
+    yield* streamFromTree(node);
+  }
+  frame.heldProperties = [];
+}
+
+export const hoistTrivia = (tags) => {
+  const co = new Coroutine(getStreamIterator(tags));
+
+  return new StreamIterable(__hoistTriviaOuter(co));
+};
+
+function* __hoistTriviaOuter(co) {
+  co.advance();
+
+  let firstTag = co.value;
+
+  co.advance();
+
+  yield* __hoistTrivia(co, firstTag);
+}
+
+function* __hoistTrivia(co, firstTag) {
+  if (co.current instanceof Promise) {
+    co.current = yield wait(co.current);
+  }
+  let frame = null;
+  let lastFrame = null;
+  let alreadyAdvanced = true;
+  let tag;
+  let nextTag;
+
+  tag = firstTag;
+  nextTag = co.value;
+
+  for (;;) {
+    if (co.done || !nextTag) break;
+
+    if (!alreadyAdvanced) {
+      co.advance();
+
+      if (co.current instanceof Promise) {
+        co.current = yield wait(co.current);
+      }
+    }
+    alreadyAdvanced = false;
+
+    nextTag = co.value;
+
+    if (tag.type === OpenNodeTag) {
+      let isCover = tag.value.type === Symbol.for('_');
+      let parentFrame = frame?.ref && !frame.open ? frame.parent : frame;
+      let done = false;
+
+      if (frame?.ref.type === '#' && !frame.ref.name) {
+        let { heldProperties } = frame;
+        if (heldProperties.length) {
+          let property = heldProperties[heldProperties.length - 1];
+
+          if (property.node) {
+            throw new Error();
+          }
+
+          let node = yield* treeFromStream(new StreamIterable(__hoistTrivia(co, tag)));
+
+          frame.heldProperties[heldProperties.length - 1] = buildProperty([
+            property.tags[0],
+            property.tags[1] || [],
+            node,
+          ]);
+
+          nextTag = co.value;
+
+          lastFrame = frame;
+
+          frame = popFrame(frame);
+
+          if (
+            !frame.ref.flags.expression &&
+            (frame.expanded || frame.parent?.open.value.type !== Symbol.for('_'))
+          ) {
+            yield* emitHeld(frame, nextTag);
+          }
+
+          done = true;
+        }
+      } else if (frame && !isCover) {
+        if (!frame.shifting && frame.ref && frame.ref.type !== '_') {
+          yield buildChild(ReferenceTag, frame.ref);
+          let isAnonymous = tag.value.flags.token && !tag.value.name;
+          if (!isAnonymous) {
+            for (let binding of frame.bindings) {
+              yield buildChild(BindingTag, binding);
+            }
+          }
+        }
+      }
+
+      if (!done) {
+        if (!frame?.ref || frame.open) {
+          frame = pushFrame(parentFrame);
+
+          frame.heldProperties = parentFrame?.heldProperties ? [...parentFrame.heldProperties] : [];
+
+          if (tag.value.type === Symbol.for('__')) {
+            frame.open = tag;
+            frame.ref = buildReference();
+          } else if (isCover) {
+            frame.ref = parentFrame?.ref || buildReference('_');
+            frame.bindings =
+              parentFrame && tag.value.type === Symbol.for('_') ? [...parentFrame.bindings] : [];
+            frame.shifting =
+              parentFrame && tag.value.type === Symbol.for('_') ? parentFrame.shifting : false;
+          } else {
+            frame.ref = buildReference();
+          }
+        }
+
+        frame.coverDepth = isCover ? (parentFrame?.coverDepth ?? 0) + 1 : 0;
+
+        frame.open = tag;
+
+        if (tag.value.selfClosing) {
+          lastFrame = frame;
+
+          frame = popFrame(frame);
+
+          if (!frame) {
+            yield tag;
+
+            break;
+          } else {
+            yield tag;
+          }
+        } else {
+          if (!isCover) {
+            yield tag;
+          }
+        }
+      }
+    } else if (tag.type === ReferenceTag) {
+      let parentFrame = frame;
+
+      frame = pushFrame(parentFrame);
+
+      if (tag.value.type === '#' && !tag.value.name) {
+        if (parentFrame.depth === 0 && parentFrame.coverDepth && !parentFrame.expanded) {
+          yield parentFrame.open;
+
+          parentFrame.expanded = true;
+        }
+
+        frame.heldProperties.push(buildProperty([tag]));
+      } else {
+        if (!parentFrame.shifting) {
+          yield* emitHeld(frame, nextTag);
+        }
+      }
+
+      frame.coverDepth = parentFrame.coverDepth;
+      frame.ref = tag.value.type === '_' ? parentFrame.ref : tag.value;
+      frame.bindings = parentFrame && tag.value.type === '_' ? [...parentFrame.bindings] : [];
+      frame.shifting = parentFrame && tag.value.type === '_' ? parentFrame.shifting : false;
+
+      if (tag.value.type === '_' && parentFrame.expanded) {
+        yield tag;
+      }
+    } else if (tag.type === ShiftTag) {
+      let doneFrame = frame;
+
+      frame = pushFrame(doneFrame);
+
+      frame.coverDepth = doneFrame?.coverDepth || 0;
+      frame.ref = lastFrame.ref;
+      frame.bindings = doneFrame && tag.value.type === '_' ? [...doneFrame.bindings] : [];
+      frame.shifting = true;
+      // frame.heldProperties = [...doneFrame.heldProperties];
+
+      yield tag;
+    } else if (tag.type === BindingTag) {
+      let { heldProperties } = frame;
+      if (heldProperties.length) {
+        let property = heldProperties[heldProperties.length - 1];
+        let { tags, node } = property;
+        if (!node) {
+          heldProperties[heldProperties.length - 1] = buildProperty([
+            tags[0],
+            [...(tags[1] || []), tag],
+          ]);
+          frame.bindings.push(tag.value);
+        }
+      }
+      if (!frame.ref) {
+        frame = pushFrame(frame);
+        frame.ref = buildReference();
+      }
+
+      frame.bindings.push(tag.value);
+    } else if (tag.type === CloseNodeTag) {
+      lastFrame = frame;
+
+      if (!lastFrame.open.value.type || !lastFrame.parent) {
+        yield* emitHeld(lastFrame, nextTag);
+      }
+
+      frame = popFrame(frame);
+
+      if (frame) {
+        frame.heldProperties = [...lastFrame.heldProperties];
+        if (!lastFrame?.coverDepth) {
+          yield tag;
+        }
+      } else {
+        if (lastFrame.expanded || lastFrame.open.value.type !== Symbol.for('_')) {
+          yield tag;
+        }
+        break;
+      }
+    } else if (tag.type === GapTag || tag.type === NullTag) {
+      if (frame.ref) {
+        yield buildChild(ReferenceTag, frame.ref);
+      }
+      lastFrame = frame;
+
+      frame = popFrame(frame);
+
+      if (tag.type === GapTag && frame.heldProperties.length) {
+        yield tag;
+
+        frame.shifting = false;
+
+        yield* emitHeld(frame, nextTag);
+      } else {
+        yield tag;
+      }
+    } else {
+      yield tag;
+    }
+
+    tag = nextTag;
+  }
+
+  if (frame) throw new Error();
+}
+
 function* __generatePrettyCSTML(tags, options) {
   let { indent = '  ', inline: inlineOption = true } = options;
 
@@ -261,7 +662,8 @@ function* __generatePrettyCSTML(tags, options) {
     return;
   }
 
-  const co = new Coroutine(getStreamIterator(prettyGroupTags(tags)));
+  const co = new Coroutine(getStreamIterator(prettyGroupTags(hoistTrivia(tags))));
+  // const co = new Coroutine(getStreamIterator(prettyGroupTags(tags)));
   let indentLevel = 0;
   let first = true;
   let inline = false;
@@ -270,19 +672,19 @@ function* __generatePrettyCSTML(tags, options) {
   for (;;) {
     co.advance();
 
-    if (co.done) break;
-
     if (co.current instanceof Promise) {
-      co.current = yield co.current;
+      co.current = yield wait(co.current);
     }
 
+    if (co.done) break;
+
     const tag = co.value;
 
     if (tag.type === 'Effect') {
       continue;
     }
 
-    if (tag.type === BindingTag && !tag.value.languagePath?.length) {
+    if (tag.type === BindingTag && !tag.value.segments?.length) {
       continue;
     }
 
@@ -293,7 +695,6 @@ function* __generatePrettyCSTML(tags, options) {
       (tag.type === NullTag ||
         tag.type === GapTag ||
         tag.type === BindingTag ||
-        tag.type === InitializerTag ||
         (tag.type === OpenNodeTag && tag.value.selfClosing));
 
     if (!first && !inline) {
@@ -374,11 +775,11 @@ export const getCooked = (tags) => {
           }
         }
 
-        if (!(ref.value.type === '#' || (ref.value.type === '@' && attributes.cooked))) {
+        if (!(ref.type === '#' || (ref.type === '@' && attributes.cooked))) {
           throw new Error('cookable nodes must not contain other nodes');
         }
 
-        if (ref.value.type === '@') {
+        if (ref.type === '@') {
           const { cooked: cookedValue } = tag.value.attributes;
 
           if (!cookedValue) throw new Error('cannot cook string: it contains uncooked escapes');

package/lib/symbols.js

@@ -6,12 +6,13 @@ export const ShiftTag = Symbol.for('ShiftTag');
 export const GapTag = Symbol.for('GapTag');
 export const BindingTag = Symbol.for('BindingTag');
 export const NullTag = Symbol.for('NullTag');
-export const InitializerTag = Symbol.for('InitializerTag');
 export const AttributeDefinition = Symbol.for('AttributeDefinition');
 export const LiteralTag = Symbol.for('LiteralTag');
 
 export const Document = Symbol.for('Document');
+export const TreeNode = Symbol.for('TreeNode');
+export const NullNode = Symbol.for('NullNode');
+export const GapNode = Symbol.for('GapNode');
 export const Node = Symbol.for('Node');
 
 export const Property = Symbol.for('Property');
-export const PropertyWrapper = Symbol.for('PropertyWrapper');