@bablr/agast-helpers 0.1.5 → 0.2.0

package/lib/builders.js

@@ -1,5 +1,9 @@
 const { freeze } = Object;
 
+export const buildBeginningOfStreamToken = () => {
+  return freeze({ type: Symbol.for('@bablr/beginning-of-stream'), value: undefined });
+};
+
 export const buildReference = (name, isArray) => {
   return freeze({ type: 'Reference', value: freeze({ name, isArray }) });
 };
@@ -12,6 +16,10 @@ export const buildGap = () => {
   return freeze({ type: 'Gap', value: undefined });
 };
 
+export const buildShift = () => {
+  return freeze({ type: 'Shift', value: undefined });
+};
+
 export const buildEmbedded = (node) => {
   return freeze({ type: 'Embedded', value: node });
 };
@@ -23,11 +31,15 @@ export const buildDoctypeTag = (attributes) => {
   });
 };
 
-export const buildNodeOpenTag = (flags, language, type, intrinsicValue, attributes = {}) => {
+export const buildNodeOpenTag = (
+  flags = {},
+  language = null,
+  type = null,
+  intrinsicValue = null,
+  attributes = {},
+) => {
   let { token, trivia, escape, expression, intrinsic } = flags;
 
-  if (!type) throw new Error();
-
   token = !!token;
   trivia = !!trivia;
   escape = !!escape;
@@ -46,27 +58,10 @@ export const buildNodeOpenTag = (flags, language, type, intrinsicValue, attribut
   });
 };
 
-export const buildFragmentOpenTag = (flags = nodeFlags, language) => {
-  let { token, trivia, escape } = flags;
-
-  token = !!token;
-  trivia = !!trivia;
-  escape = !!escape;
-
-  return freeze({
-    type: 'OpenFragmentTag',
-    value: freeze({ flags: freeze({ token, trivia, escape }) }),
-  });
-};
-
 export const buildNodeCloseTag = (type = null, language = null) => {
   return freeze({ type: 'CloseNodeTag', value: freeze({ language, type }) });
 };
 
-export const buildFragmentCloseTag = () => {
-  return freeze({ type: 'CloseFragmentTag', value: freeze({}) });
-};
-
 const isString = (val) => typeof val === 'string';
 
 export const buildLiteral = (value) => {
@@ -227,3 +222,21 @@ export const buildTriviaNode = (language, type, children = [], properties = {},
     properties: freeze(properties),
     attributes: freeze(attributes),
   });
+
+export const buildEffect = (value) => {
+  return freeze({ type: 'Effect', value });
+};
+
+export const buildWriteEffect = (text, options = {}) => {
+  return buildEffect(freeze({ verb: 'write', value: freeze({ text, options }) }));
+};
+
+export const buildAnsiPushEffect = (spans = '') => {
+  return buildEffect(
+    freeze({ verb: 'ansi-push', value: { spans: spans === '' ? [] : spans.split(' ') } }),
+  );
+};
+
+export const buildAnsiPopEffect = () => {
+  return buildEffect(freeze({ verb: 'ansi-pop', value: undefined }));
+};

package/lib/path.js

@@ -2,7 +2,7 @@ export const parsePath = (str) => {
   const isArray = str.endsWith('[]');
   const name = isArray ? str.slice(0, -2) : str;
 
-  if (!/^\w+$/.test(name)) throw new Error();
+  if (!/^[a-zA-Z]+$/.test(name)) throw new Error();
 
   return { isArray, name };
 };

package/lib/print.js

@@ -3,6 +3,10 @@ const { isArray } = Array;
 const isString = (val) => typeof val === 'string';
 const isNumber = (val) => typeof val === 'number';
 const isObject = (val) => val && typeof val === 'object' && !isArray(val);
+const isFunction = (val) => typeof val === 'function';
+
+const when = (condition, value) =>
+  condition ? (isFunction(value) ? value() : value) : { *[Symbol.iterator]() {} };
 
 export const printExpression = (expr) => {
   if (isString(expr)) {
@@ -11,6 +15,7 @@ export const printExpression = (expr) => {
     return String(expr);
   } else if (isNumber(expr)) {
     if (!isFinite(expr)) {
+      if (isNaN(expr)) throw new Error();
       return expr === -Infinity ? '-Infinity' : '+Infinity';
     } else if (isInteger(expr)) {
       return String(expr);
@@ -41,7 +46,9 @@ export const printLanguage = (language) => {
 };
 
 export const printTagPath = (language, type) => {
-  return language?.length ? `${printLanguage(language)}:${type}` : type;
+  return [...when(language?.length, () => [printLanguage(language)]), ...when(type, [type])].join(
+    ':',
+  );
 };
 
 const escapeReplacer = (esc) => {
@@ -78,6 +85,12 @@ export const printGap = (terminal) => {
   return `<//>`;
 };
 
+export const printShift = (terminal) => {
+  if (terminal?.type !== 'Shift') throw new Error();
+
+  return `^^^`;
+};
+
 export const printReference = (terminal) => {
   if (terminal?.type !== 'Reference') throw new Error();
 
@@ -147,26 +160,12 @@ export const printOpenNodeTag = (terminal) => {
   )}${intrinsicFrag}${attributesFrag}${selfCloser}>`;
 };
 
-export const printOpenFragmentTag = (terminal) => {
-  if (terminal?.type !== 'OpenFragmentTag') throw new Error();
-
-  let { flags } = terminal.value;
-
-  return `<${printFlags(flags)}>`;
-};
-
 export const printCloseNodeTag = (terminal) => {
   if (terminal?.type !== 'CloseNodeTag') throw new Error();
 
   return `</>`;
 };
 
-export const printCloseFragmentTag = (terminal) => {
-  if (terminal?.type !== 'CloseFragmentTag') throw new Error();
-
-  return `</>`;
-};
-
 export const printTerminal = (terminal) => {
   if (!isObject(terminal)) throw new Error();
 
@@ -177,6 +176,9 @@ export const printTerminal = (terminal) => {
     case 'Gap':
       return printGap(terminal);
 
+    case 'Shift':
+      return printShift(terminal);
+
     case 'Literal':
       return printLiteral(terminal);
 
@@ -189,15 +191,9 @@ export const printTerminal = (terminal) => {
     case 'OpenNodeTag':
       return printOpenNodeTag(terminal);
 
-    case 'OpenFragmentTag':
-      return printOpenFragmentTag(terminal);
-
     case 'CloseNodeTag':
       return printCloseNodeTag(terminal);
 
-    case 'CloseFragmentTag':
-      return printCloseFragmentTag(terminal);
-
     default:
       throw new Error();
   }

package/lib/shorthand.js

@@ -3,9 +3,7 @@ import {
   buildGap,
   buildEmbedded,
   buildNodeOpenTag,
-  buildFragmentOpenTag,
   buildNodeCloseTag,
-  buildFragmentCloseTag,
   buildLiteral,
   buildNode,
   buildSyntacticNode,
@@ -47,9 +45,7 @@ export const lit = (str) => buildLiteral(stripArray(str));
 export const gap = buildGap;
 export const embedded = buildEmbedded;
 export const nodeOpen = buildNodeOpenTag;
-export const fragOpen = buildFragmentOpenTag;
 export const nodeClose = buildNodeCloseTag;
-export const fragClose = buildFragmentCloseTag;
 export const node = buildNode;
 export const s_node = buildSyntacticNode;
 export const s_i_node = buildSyntacticIntrinsicNode;

package/lib/stream.js

@@ -1,5 +1,6 @@
 import { Coroutine } from '@bablr/coroutine';
 import { printTerminal } from './print.js';
+import { buildWriteEffect } from './builders.js';
 export * from './print.js';
 
 export const getStreamIterator = (obj) => {
@@ -21,12 +22,11 @@ export class SyncGenerator {
     }
 
     if (step.done) {
-      return { value: undefined, done: true };
+      return step;
     } else if (step.value instanceof Promise) {
       throw new Error('sync generators cannot resolve promises');
     } else {
-      const { value } = step;
-      return { value, done: false };
+      return step;
     }
   }
 
@@ -60,14 +60,13 @@ export class AsyncGenerator {
     }
 
     if (step.done) {
-      return Promise.resolve({ value: undefined, done: true });
+      return Promise.resolve(step);
     } else if (step.value instanceof Promise) {
       return step.value.then((value) => {
         return this.next(value);
       });
     } else {
-      const { value } = step;
-      return Promise.resolve({ value, done: false });
+      return Promise.resolve(step);
     }
   }
 
@@ -93,14 +92,13 @@ export class StreamGenerator {
     const step = this.generator.next(value);
 
     if (step.done) {
-      return { value: undefined, done: true };
+      return step;
     } else if (step.value instanceof Promise) {
       return step.value.then((value) => {
         return this.next(value);
       });
     } else {
-      const { value } = step;
-      return { value, done: false };
+      return step;
     }
   }
 
@@ -139,9 +137,9 @@ export const maybeWait = (maybePromise, callback) => {
   }
 };
 
-function* __generateCSTML(terminals) {
+function* __generateCSTMLStrategy(terminals) {
   if (!terminals) {
-    yield* '<//>';
+    yield buildWriteEffect('<//>');
     return;
   }
 
@@ -160,23 +158,88 @@ function* __generateCSTML(terminals) {
     const terminal = co.value;
 
     if (terminal.type === 'Reference' && prevTerminal.type === 'Null') {
-      yield* ' ';
+      yield buildWriteEffect(' ');
     }
 
-    yield* printTerminal(terminal);
+    if (terminal.type !== 'Effect') {
+      buildWriteEffect(printTerminal(terminal));
 
-    prevTerminal = terminal;
+      prevTerminal = terminal;
+    } else {
+      yield terminal;
+    }
   }
 
-  yield* '\n';
+  yield buildWriteEffect('\n');
+}
+
+export const generateCSTMLStrategy = (terminals) =>
+  new StreamIterable(__generateCSTMLStrategy(terminals));
+
+function* __generateStandardOutput(terminals) {
+  const co = new Coroutine(getStreamIterator(terminals));
+
+  for (;;) {
+    co.advance();
+
+    if (co.current instanceof Promise) {
+      co.current = yield co.current;
+    }
+    if (co.done) break;
+
+    const terminal = co.value;
+
+    if (terminal.type === 'Effect') {
+      const effect = terminal.value;
+      if (effect.verb === 'write' && (effect.value.stream == null || effect.value.stream === 1)) {
+        yield* effect.value.text;
+      }
+    }
+  }
 }
 
-export const generateCSTML = (terminals) => new StreamIterable(__generateCSTML(terminals));
+export const generateStandardOutput = (terminals) =>
+  new StreamIterable(__generateStandardOutput(terminals));
+
+function* __generateAllOutput(terminals) {
+  const co = new Coroutine(getStreamIterator(terminals));
+
+  let currentStream = null;
+
+  for (;;) {
+    co.advance();
+
+    if (co.current instanceof Promise) {
+      co.current = yield co.current;
+    }
+    if (co.done) break;
+
+    const terminal = co.value;
+
+    if (terminal.type === 'Effect') {
+      const effect = terminal.value;
+      if (effect.verb === 'write') {
+        const prevStream = currentStream;
+        currentStream = effect.value.options.stream || 1;
+        if (prevStream && prevStream !== currentStream && !effect.value.text.startsWith('\n')) {
+          yield* '\n';
+        }
+        yield* effect.value.text;
+      }
+    }
+  }
+}
+
+export const generateAllOutput = (terminals) => new StreamIterable(__generateAllOutput(terminals));
 
 export const printCSTML = (terminals) => {
-  return stringFromStream(generateCSTML(terminals));
+  return stringFromStream(generateStandardOutput(generateCSTMLStrategy(terminals)));
 };
 
+function* __emptyStreamIterator() {}
+
+export const emptyStreamIterator = () => new StreamIterable(__emptyStreamIterator());
+
 export const asyncStringFromStream = async (stream) => {
   const co = new Coroutine(getStreamIterator(stream));
   let str = '';
@@ -215,9 +278,11 @@ export const stringFromStream = (stream) => {
   return str;
 };
 
-function* __generatePrettyCSTML(terminals, indent) {
+function* __generatePrettyCSTMLStrategy(terminals, options) {
+  const { indent = '  ', emitEffects = false, inline: inlineOption = true } = options;
+
   if (!terminals) {
-    yield* '<//>';
+    yield buildWriteEffect('<//>');
     return;
   }
 
@@ -236,18 +301,29 @@ function* __generatePrettyCSTML(terminals, indent) {
 
     const terminal = co.value;
 
+    if (terminal.type === 'Effect') {
+      const effect = terminal.value;
+      if (emitEffects && effect.verb === 'write') {
+        yield buildWriteEffect((first ? '' : '\n') + effect.value.text, effect.value.options);
+
+        first = false;
+      }
+      continue;
+    }
+
     const inline =
-      terminal.type === 'Null' ||
-      terminal.type === 'Gap' ||
-      (terminal.type === 'OpenNodeTag' &&
-        terminal.value.intrinsicValue &&
-        terminal.value.flags.intrinsic);
+      inlineOption &&
+      (terminal.type === 'Null' ||
+        terminal.type === 'Gap' ||
+        (terminal.type === 'OpenNodeTag' &&
+          terminal.value.intrinsicValue &&
+          terminal.value.flags.intrinsic));
 
     if (!first && !inline) {
-      yield* '\n';
+      yield buildWriteEffect('\n');
     }
 
-    if (['CloseNodeTag', 'CloseFragmentTag'].includes(terminal.type)) {
+    if (terminal.type === 'CloseNodeTag') {
       if (indentLevel === 0) {
         throw new Error('imbalanced tag stack');
       }
@@ -256,15 +332,15 @@ function* __generatePrettyCSTML(terminals, indent) {
     }
 
     if (!inline) {
-      yield* indent.repeat(indentLevel);
+      yield buildWriteEffect(indent.repeat(indentLevel));
     } else {
-      yield* ' ';
+      yield buildWriteEffect(' ');
     }
-    yield* printTerminal(terminal);
+    yield buildWriteEffect(printTerminal(terminal));
 
     if (
-      terminal.type === 'OpenFragmentTag' ||
-      (terminal.type === 'OpenNodeTag' && !terminal.value.intrinsicValue)
+      terminal.type === 'OpenNodeTag' &&
+      (!terminal.value.intrinsicValue || !terminal.value.type)
     ) {
       indentLevel++;
     }
@@ -272,15 +348,15 @@ function* __generatePrettyCSTML(terminals, indent) {
     first = false;
   }
 
-  yield* '\n';
+  yield buildWriteEffect('\n');
 }
 
-export const generatePrettyCSTML = (terminals, indent = '  ') => {
-  return new StreamIterable(__generatePrettyCSTML(terminals, indent));
+export const generatePrettyCSTMLStrategy = (terminals, options = {}) => {
+  return new StreamIterable(__generatePrettyCSTMLStrategy(terminals, options));
 };
 
-export const printPrettyCSTML = (terminals) => {
-  return stringFromStream(generatePrettyCSTML(terminals));
+export const printPrettyCSTML = (terminals, options = {}) => {
+  return stringFromStream(generateStandardOutput(generatePrettyCSTMLStrategy(terminals, options)));
 };
 
 export const getCooked = (terminals) => {
@@ -353,11 +429,11 @@ export function* generateSourceTextFor(terminals) {
 export const sourceTextFor = printSource;
 
 export const startsDocument = (terminal) => {
-  const { type } = terminal;
-  if (type === 'OpenFragmentTag' || type === 'DoctypeTag') {
+  const { type, value } = terminal;
+  if ((type === 'OpenNodeTag' && !value.type) || type === 'DoctypeTag') {
     return true;
   } else if (type === 'OpenNodeTag') {
-    const { flags } = terminal.value;
+    const { flags } = value;
 
     return flags.trivia || flags.escape;
   }

package/lib/tree.js

@@ -1,4 +1,5 @@
 import emptyStack from '@iter-tools/imm-stack';
+import { Coroutine } from '@bablr/coroutine';
 import {
   buildNodeCloseTag,
   buildNodeOpenTag,
@@ -6,13 +7,12 @@ import {
   buildEmbedded,
   nodeFlags,
   buildDoctypeTag,
-  buildFragmentOpenTag,
-  buildFragmentCloseTag,
-  buildReference,
 } from './builders.js';
 import {
   printPrettyCSTML as printPrettyCSTMLFromStream,
   printCSTML as printCSTMLFromStream,
+  getStreamIterator,
+  StreamIterable,
 } from './stream.js';
 export * from './builders.js';
 export * from './print.js';
@@ -38,134 +38,201 @@ const get = (node, path) => {
   }
 };
 
-const reduceTokenToNodes = (nodes, token) => {
-  switch (token.type) {
-    case 'OpenFragmentTag':
-    case 'Null': {
-      return nodes;
-    }
+function* __treeFromStream(tokens) {
+  let nodes = emptyStack;
+  let rootNode;
+  let held = null;
+  const co = new Coroutine(getStreamIterator(tokens));
 
-    case 'DoctypeTag': {
-      const { attributes } = token.value;
+  for (;;) {
+    co.advance();
 
-      return nodes.push(
-        freeze({
-          flags: nodeFlags,
-          type: Symbol.for('@bablr/fragment'),
-          children: [],
-          properties: {},
-          attributes: freeze(attributes),
-        }),
-      );
+    if (co.current instanceof Promise) {
+      co.current = yield co.current;
     }
 
-    case 'CloseFragmentTag': {
-      freeze(nodes.value.properties);
-      freeze(nodes.value.children);
+    if (co.done) break;
 
-      return nodes.pop();
-    }
+    const token = co.value;
 
-    case 'Literal':
-    case 'Gap':
-    case 'Reference': {
-      nodes.value.children.push(token);
-      return nodes;
+    if (token.type === 'Effect') {
+      continue;
     }
 
-    case 'OpenNodeTag': {
-      const buildStringTerminals = (str) => {
-        // do better
-        return [{ type: 'Literal', value: str }];
-      };
-
-      const { flags, language, type, intrinsicValue, attributes } = token.value;
-      const node = nodes.value;
-      const newNode = freeze({
-        flags,
-        language,
-        type,
-        children: intrinsicValue && flags.intrinsic ? buildStringTerminals(intrinsicValue) : [],
+    if (token.type === 'DoctypeTag') {
+      const { attributes } = token.value;
+
+      rootNode = freeze({
+        flags: nodeFlags,
+        type: null,
+        children: [],
         properties: {},
         attributes: freeze(attributes),
       });
+      nodes = nodes.push(rootNode);
+      continue;
+    }
 
-      if (node && !(flags.escape || flags.trivia)) {
-        if (nodes.size === 1) {
-          node.children.push(buildReference('root', false));
-        }
+    if (!nodes.size) {
+      throw new Error('imbalanced tag stack');
+    }
 
-        if (!node.children.length) {
-          throw new Error('Nodes must follow references');
-        }
+    switch (token.type) {
+      case 'Null': {
+        break;
+      }
+
+      case 'Literal':
+      case 'Reference': {
+        nodes.value.children.push(token);
+        break;
+      }
 
-        const { name, isArray } = arrayLast(node.children).value;
+      case 'Gap': {
+        if (held) {
+          const { children, properties } = nodes.value;
+          const ref = arrayLast(children);
 
-        if (isArray) {
-          if (!hasOwn(node.properties, name)) {
-            node.properties[name] = [];
+          if (ref.type !== 'Reference') throw new Error();
+
+          if (ref.value.isArray) {
+            if (!properties[ref.value.name]) {
+              properties[ref.value.name] = [];
+            }
+            properties[ref.value.name].push(held);
+          } else {
+            properties[ref.value.name] = held;
           }
-          const array = node.properties[name];
 
-          array.push(newNode);
-        } else {
-          node.properties[name] = newNode;
+          held = null;
         }
+        break;
       }
 
-      if (intrinsicValue && flags.intrinsic) {
-        freeze(newNode.children);
-        freeze(newNode.properties);
-        return nodes;
+      case 'Shift': {
+        const { children, properties } = nodes.value;
+
+        const ref = arrayLast(children);
+        let node = properties[ref.value.name];
+
+        if (ref.value.isArray) {
+          node = arrayLast(node);
+          properties[ref.value.name].pop();
+        } else {
+          properties[ref.value.name] = null;
+        }
+
+        held = node;
+        break;
       }
 
-      return nodes.push(newNode);
-    }
+      case 'OpenNodeTag': {
+        const buildStringTerminals = (str) => {
+          // do better
+          return [{ type: 'Literal', value: str }];
+        };
+
+        const { flags, language, type, intrinsicValue, attributes } = token.value;
+        const node = nodes.value;
 
-    case 'CloseNodeTag': {
-      const completedNode = nodes.value;
-      const { flags } = completedNode;
+        if (!type) {
+          break;
+        }
 
-      if (flags.escape || flags.trivia) {
-        const parentChildren = nodes.prev.value.children;
+        const newNode = freeze({
+          flags,
+          language,
+          type,
+          children: intrinsicValue && flags.intrinsic ? buildStringTerminals(intrinsicValue) : [],
+          properties: {},
+          attributes: freeze(attributes),
+        });
 
-        parentChildren.push(buildEmbedded(completedNode));
+        if (node && !(flags.escape || flags.trivia)) {
+          if (!node.children.length) {
+            throw new Error('Nodes must follow references');
+          }
+
+          const { name, isArray } = arrayLast(node.children).value;
+
+          if (isArray) {
+            if (!hasOwn(node.properties, name)) {
+              node.properties[name] = [];
+            }
+            const array = node.properties[name];
+
+            array.push(newNode);
+          } else {
+            node.properties[name] = newNode;
+          }
+        }
+
+        if (intrinsicValue && flags.intrinsic) {
+          freeze(newNode.children);
+          freeze(newNode.properties);
+          break;
+        }
+
+        nodes = nodes.push(newNode);
+        break;
       }
 
-      freeze(completedNode.properties);
-      freeze(completedNode.children);
+      case 'CloseNodeTag': {
+        const completedNode = nodes.value;
+        const { flags } = completedNode;
 
-      return nodes.pop();
-    }
+        if (flags.escape || flags.trivia) {
+          const parentChildren = nodes.prev.value.children;
 
-    default: {
-      throw new Error();
-    }
-  }
-};
+          parentChildren.push(buildEmbedded(completedNode));
+        }
 
-export const treeFromStreamSync = (tokens) => {
-  let nodes = emptyStack;
-  let rootNode;
+        freeze(completedNode.properties);
+        freeze(completedNode.children);
 
-  for (const token of tokens) {
-    nodes = reduceTokenToNodes(nodes, token);
-    rootNode = nodes.value || rootNode;
+        if (!completedNode.type && nodes.size !== 1) {
+          throw new Error('imbalanced tag stack');
+        }
+
+        nodes = nodes.pop();
+        break;
+      }
+
+      default: {
+        throw new Error();
+      }
+    }
   }
 
   return rootNode;
+}
+
+export const treeFromStream = (terminals) => new StreamIterable(__treeFromStream(terminals));
+
+export const treeFromStreamSync = (tokens) => {
+  return evaluateReturnSync(treeFromStream(tokens));
 };
 
 export const treeFromStreamAsync = async (tokens) => {
-  let nodes = emptyStack;
-  let rootNode;
+  return evaluateReturnAsync(treeFromStream(tokens));
+};
 
-  for await (const token of tokens) {
-    nodes = reduceTokenToNodes(nodes, token);
-    rootNode = nodes.value || rootNode;
-  }
+export const evaluateReturnSync = (generator) => {
+  const co = new Coroutine(generator[Symbol.iterator]());
+  while (!co.done) co.advance();
+  return co.value;
+};
 
-  return rootNode;
+export const evaluateReturnAsync = async (generator) => {
+  const co = new Coroutine(getStreamIterator(generator));
+  while (!co.done) {
+    co.advance();
+
+    if (co.current instanceof Promise) {
+      co.current = await co.current;
+    }
+  }
+  return co.value;
 };
 
 export function* streamFromTree(rootNode) {
@@ -174,7 +241,7 @@ export function* streamFromTree(rootNode) {
   }
 
   yield buildDoctypeTag(rootNode.attributes);
-  yield buildFragmentOpenTag();
+  yield buildNodeOpenTag();
 
   let stack = emptyStack.push(buildFrame(rootNode));
 
@@ -236,7 +303,7 @@ export function* streamFromTree(rootNode) {
 
     stack = stack.pop();
   }
-  yield buildFragmentCloseTag();
+  yield buildNodeCloseTag();
 }
 
 export const getCooked = (cookable) => {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@bablr/agast-helpers",
   "description": "Helper functions for working with agAST trees",
-  "version": "0.1.5",
+  "version": "0.2.0",
   "author": "Conrad Buck<conartist6@gmail.com>",
   "type": "module",
   "files": [