@bablr/boot 0.1.9 → 0.2.1

package/shorthand.macro.js

@@ -1,47 +1,29 @@
+const { spawnSync } = require('node:child_process');
 const t = require('@babel/types');
 const { expression } = require('@babel/template');
+const { diff } = require('jest-diff');
 const isObject = require('iter-tools-es/methods/is-object');
 const isUndefined = require('iter-tools-es/methods/is-undefined');
 const isNull = require('iter-tools-es/methods/is-null');
 const isString = require('iter-tools-es/methods/is-string');
 const concat = require('iter-tools-es/methods/concat');
 const { createMacro } = require('babel-plugin-macros');
-const { TemplateParser } = require('./lib/miniparser.js');
+const { TemplateParser, set, getAgASTValue } = require('./lib/index.js');
 const i = require('./lib/languages/instruction.js');
 const re = require('./lib/languages/regex.js');
 const spam = require('./lib/languages/spamex.js');
-const str = require('./lib/languages/string.js');
-const num = require('./lib/languages/number.js');
 const cstml = require('./lib/languages/cstml.js');
 const { addNamespace, addNamed } = require('@babel/helper-module-imports');
 const { PathResolver } = require('@bablr/boot-helpers/path');
+const { printPrettyCSTML } = require('./lib/print.js');
 
-const { hasOwn } = Object;
 const { isArray } = Array;
 const isNumber = (v) => typeof v === 'number';
 const isBoolean = (v) => typeof v === 'boolean';
-
 const isPlainObject = (v) => isObject(v) && !isArray(v);
+const printRef = (ref) => (ref.isArray ? `${ref.name}[]` : ref.name);
 
-const set = (obj, path, value) => {
-  const { pathName, pathIsArray } = path;
-  if (pathIsArray) {
-    if (!obj[pathName]) {
-      obj[pathName] = [];
-    }
-
-    if (!isArray(obj[pathName])) throw new Error('bad array value');
-
-    obj[pathName].push(value);
-  } else {
-    if (hasOwn(obj, pathName)) {
-      throw new Error('duplicate child name');
-    }
-    obj[pathName] = value;
-  }
-};
-
-const getASTValue = (v, exprs, bindings) => {
+const getBabelASTValue = (v, exprs, bindings) => {
   return isNull(v)
     ? t.nullLiteral()
     : isUndefined(v)
@@ -53,53 +35,52 @@ const getASTValue = (v, exprs, bindings) => {
     : isBoolean(v)
     ? t.booleanLiteral(v)
     : isArray(v)
-    ? t.arrayExpression(v.map((v) => getASTValue(v, exprs, bindings)))
+    ? t.arrayExpression(v.map((v) => getBabelASTValue(v, exprs, bindings)))
     : isPlainObject(v) && !v.language
     ? t.objectExpression(
         Object.entries(v).map(([k, v]) =>
-          t.objectProperty(t.identifier(k), getASTValue(v, exprs, bindings)),
+          t.objectProperty(t.identifier(k), getBabelASTValue(v, exprs, bindings)),
         ),
       )
-    : generateNode(v, exprs, bindings);
+    : generateBabelNode(v, exprs, bindings);
 };
 
-const escapeReplacer = (esc) => {
-  if (esc === '\r') {
-    return '\\r';
-  } else if (esc === '\n') {
-    return '\\n';
-  } else if (esc === '\0') {
-    return '\\0';
-  } else {
-    return `\\${esc}`;
-  }
-};
-const printTemplateString = (str) => {
-  return `\`${str.replace(/[`\\\0\r\n]/g, escapeReplacer)}\``;
-};
-
-const generateNodeChild = (child, bindings) => {
+const generateBabelNodeChild = (child, exprs, bindings) => {
   if (child.type === 'Reference') {
-    const { pathName, pathIsArray } = child.value;
-    const printedRef = pathIsArray ? `${pathName}[]` : pathName;
-    return expression(`%%t%%.ref\`${printedRef}\``)({ t: bindings.t });
+    return expression(`%%t%%.ref\`${printRef(child.value)}\``)({ t: bindings.t });
   } else if (child.type === 'Literal') {
-    return expression(`%%t%%.lit${printTemplateString(child.value)}`)({ t: bindings.t });
-  } else if (child.type === 'Trivia') {
-    return expression(`%%t%%.trivia${printTemplateString(child.value)}`)({ t: bindings.t });
-  } else if (child.type === 'Escape') {
-    const {cooked, raw} = child.value
-    return expression(`%%t%%.esc(${printTemplateString(raw)}, ${printTemplateString(cooked)})`)({
+    return expression(`%%t%%.lit(%%value%%)`)({
+      t: bindings.t,
+      value: getBabelASTValue(child.value, exprs, bindings),
+    });
+  } else if (child.type === 'Embedded') {
+    return expression(`%%t%%.embedded(%%value%%)`)({
       t: bindings.t,
+      value: generateBabelNode(child.value, exprs, bindings),
     });
   } else {
     throw new Error(`Unknown child type ${child.type}`);
   }
 };
 
-const generateNode = (node, exprs, bindings) => {
+const getAgastNodeType = (flags) => {
+  if (flags.intrinsic && flags.token) {
+    return 's_i_node';
+  } else if (flags.token && flags.trivia) {
+    return 's_t_node';
+  } else if (flags.token && flags.escape) {
+    return 's_e_node';
+  } else if (flags.token) {
+    return 's_node';
+  } else {
+    return 'node';
+  }
+};
+
+const generateBabelNode = (node, exprs, bindings) => {
   const resolver = new PathResolver(node);
-  const { children, type, language, attributes } = node;
+  const { flags = {}, children, type, language, attributes } = node;
+
   const properties_ = {};
   const children_ = [];
 
@@ -108,64 +89,122 @@ const generateNode = (node, exprs, bindings) => {
   }
 
   for (const child of children) {
-    children_.push(generateNodeChild(child, bindings));
-
     if (child.type === 'Reference') {
       const path = child.value;
-      const { pathIsArray } = path;
+      const { isArray: pathIsArray } = path;
       const resolved = resolver.get(path);
 
-      let embedded = resolved;
       if (resolved) {
-        embedded = generateNode(resolved, exprs, bindings);
+        set(properties_, path, generateBabelNode(resolved, exprs, bindings));
+        children_.push(generateBabelNodeChild(child, exprs, bindings));
       } else {
-        embedded = exprs.pop();
-        const { interpolateArray, interpolateString } = bindings;
+        // gap
+        const expr = exprs.pop();
+        const { interpolateArray, interpolateArrayChildren, interpolateString } = bindings;
 
         if (pathIsArray) {
-          embedded = expression('[...%%interpolateArray%%(%%embedded%%)]')({
-            interpolateArray,
-            embedded,
-          }).elements[0];
-        } else if (language === 'String' && type === 'String') {
-          embedded = expression('%%interpolateString%%(%%embedded%%)')({
-            interpolateString,
-            embedded,
-          });
+          set(
+            properties_,
+            path,
+            expression('[...%%interpolateArray%%(%%expr%%)]')({
+              interpolateArray,
+              expr,
+            }).elements[0],
+          );
+
+          children_.push(
+            t.spreadElement(
+              expression('%%interpolateArrayChildren%%(%%expr%%, %%ref%%, %%sep%%)')({
+                interpolateArrayChildren,
+                expr,
+                ref: expression(`%%t%%.ref\`${printRef(child.value)}\``)({ t: bindings.t }),
+
+                // Really really awful unsafe-as-heck hack, to be removed ASAP
+                // Fixing this requires having interpolation happen during parsing
+                // That way the grammar can deal with the separators!
+                sep: expression(
+                  "%%t%%.embedded(%%t%%.t_node(%%l%%.Comment, null, [%%t%%.embedded(%%t%%.t_node('Space', 'Space', [%%t%%.lit(' ')]))]))",
+                )({ t: bindings.t, l: bindings.l }),
+              }),
+            ),
+          );
+        } else if (language === cstml.canonicalURL && type === 'String') {
+          set(
+            properties_,
+            path,
+            expression('%%interpolateString%%(%%expr%%)')({
+              interpolateString,
+              expr,
+            }),
+          );
+
+          children_.push(generateBabelNodeChild(child, exprs, bindings));
+        } else {
+          set(properties_, path, expr);
+
+          children_.push(generateBabelNodeChild(child, exprs, bindings));
         }
       }
-
-      set(properties_, path, embedded);
+    } else {
+      children_.push(generateBabelNodeChild(child, exprs, bindings));
     }
   }
 
-  return expression(
-    `%%t%%.node(%%language%%, %%type%%, %%children%%, %%properties%%, %%attributes%%)`,
-  )({
+  const nodeType = getAgastNodeType(flags);
+
+  const propsAtts =
+    nodeType === 's_node' || nodeType === 's_i_node' ? '' : ', %%properties%%, %%attributes%%';
+  const propsAttsValue =
+    nodeType === 's_node' || nodeType === 's_i_node'
+      ? {}
+      : {
+          properties: t.objectExpression(
+            Object.entries(properties_).map(([key, value]) =>
+              t.objectProperty(
+                t.identifier(key),
+                isArray(value) ? t.arrayExpression(value) : value,
+              ),
+            ),
+          ),
+          attributes: t.objectExpression(
+            Object.entries(attributes).map(([key, value]) =>
+              t.objectProperty(t.identifier(key), getBabelASTValue(value, exprs, bindings)),
+            ),
+          ),
+        };
+
+  return expression(`%%t%%.%%nodeType%%(%%l%%.%%language%%, %%type%%, %%children%%${propsAtts})`)({
     t: bindings.t,
-    language: t.stringLiteral(language),
+    l: bindings.l,
+    language: t.identifier(namesFor[language]),
+    nodeType: t.identifier(nodeType),
     type: t.stringLiteral(type),
-    children: t.arrayExpression(children_),
-    properties: t.objectExpression(
-      Object.entries(properties_).map(([key, value]) =>
-        t.objectProperty(t.identifier(key), isArray(value) ? t.arrayExpression(value) : value),
-      ),
-    ),
-    attributes: t.objectExpression(
-      Object.entries(attributes).map(([key, value]) =>
-        t.objectProperty(t.identifier(key), getASTValue(value, exprs, bindings)),
-      ),
-    ),
+    children:
+      nodeType === 's_node' || nodeType === 's_i_node'
+        ? t.stringLiteral(children[0].value)
+        : t.arrayExpression(children_),
+    ...propsAttsValue,
   });
 };
 
+const getTopScope = (scope) => {
+  let top = scope;
+  while (top.parent) top = top.parent;
+  return top;
+};
+
+const namesFor = Object.fromEntries([
+  ...[i, re, spam, cstml].map((l) => [l.canonicalURL, l.name]),
+  ['https://bablr.org/languages/core/space-tab-newline', 'Space'],
+]);
+
 const languages = {
-  i,
-  re,
-  spam,
-  str,
-  num,
-  cst: cstml,
+  i: '@bablr/language-bablr-vm-instruction',
+  re: '@bablr/language-regex-vm-pattern',
+  spam: '@bablr/language-spamex',
+  str: '@bablr/language-cstml',
+  num: '@bablr/language-cstml',
+  cst: '@bablr/language-cstml',
 };
 
 const topTypes = {
@@ -177,10 +216,13 @@ const topTypes = {
   cst: 'Fragment',
 };
 
-const getTopScope = (scope) => {
-  let top = scope;
-  while (top.parent) top = top.parent;
-  return top;
+const miniLanguages = {
+  i,
+  re,
+  spam,
+  str: cstml,
+  num: cstml,
+  cst: cstml,
 };
 
 const shorthandMacro = ({ references }) => {
@@ -197,16 +239,30 @@ const shorthandMacro = ({ references }) => {
     references.cst,
   )) {
     if (!bindings.t) {
-      bindings.t = addNamespace(getTopScope(ref.scope).path, '@bablr/boot-helpers/types', {
+      bindings.t = addNamespace(getTopScope(ref.scope).path, '@bablr/agast-helpers/shorthand', {
         nameHint: 't',
       });
     }
 
+    if (!bindings.l) {
+      bindings.l = addNamespace(getTopScope(ref.scope).path, '@bablr/agast-vm-helpers/languages', {
+        nameHint: 'l',
+      });
+    }
+
     if (!bindings.interpolateArray) {
       bindings.interpolateArray = addNamed(
         getTopScope(ref.scope).path,
         'interpolateArray',
-        '@bablr/boot-helpers/template',
+        '@bablr/agast-helpers/template',
+      );
+    }
+
+    if (!bindings.interpolateArrayChildren) {
+      bindings.interpolateArrayChildren = addNamed(
+        getTopScope(ref.scope).path,
+        'interpolateArrayChildren',
+        '@bablr/agast-helpers/template',
       );
     }
 
@@ -214,7 +270,7 @@ const shorthandMacro = ({ references }) => {
       bindings.interpolateString = addNamed(
         getTopScope(ref.scope).path,
         'interpolateString',
-        '@bablr/boot-helpers/template',
+        '@bablr/agast-helpers/template',
       );
     }
 
@@ -230,15 +286,55 @@ const shorthandMacro = ({ references }) => {
         ? ref.parentPath.node.property.name
         : topTypes[tagName];
 
-    if (!language) throw new Error();
+    const streamText = quasis
+      .map((q) => `"${q.value.raw.replace(/[\\"]/g, '\\$&')}"`)
+      .join(' <//> ');
+
+    // console.log(streamText);
+
+    const miniLanguage = miniLanguages[tagName];
 
     const ast = new TemplateParser(
-      language,
+      miniLanguage,
       quasis.map((q) => q.value.raw),
       expressions.map(() => null),
-    ).eval({ language: language.name, type });
+    ).eval({ language: miniLanguage.name, type });
+
+    const agAST = getAgASTValue(miniLanguage, ast);
+    let referenceDocument = null;
+
+    const document = printPrettyCSTML(agAST);
+
+    // try {
+    //   const documentResult = spawnSync(
+    //     '../bablr-cli/bin/index.js',
+    //     ['-l', language, '-p', type, '-f'],
+    //     {
+    //       input: streamText,
+    //       encoding: 'utf8',
+    //     },
+    //   );
+
+    //   if (documentResult.status > 0) {
+    //     throw new Error('bablr CLI parse return non-zero exit');
+    //   }
+
+    //   if (documentResult.error) {
+    //     throw new Error(documentResult.error);
+    //   }
+
+    //   referenceDocument = documentResult.stdout.slice(0, -1);
+
+    //   if (!referenceDocument.length) {
+    //     throw new Error();
+    //   }
+
+    //   // secondaryAst = parse(cstml, 'Document', document);
+    // } catch (e) {
+    //   console.warn('  parse failure');
+    // }
 
-    taggedTemplate.replaceWith(generateNode(ast, expressions.reverse(), bindings));
+    taggedTemplate.replaceWith(generateBabelNode(agAST, expressions.reverse(), bindings));
   }
 };
 

package/lib/languages/number.js

@@ -1,38 +0,0 @@
-const { buildCovers } = require('../utils.js');
-const sym = require('../symbols.js');
-
-const name = 'Number';
-
-const dependencies = {};
-
-const covers = buildCovers({
-  [sym.node]: ['Number', 'Digit'],
-  Number: ['Integer', 'Infinity'],
-});
-
-const grammar = class NumberMiniparserGrammar {
-  // @Node
-  Integer(p) {
-    p.eatMatch('-', 'Punctuator', { path: 'negative' });
-    p.eatProduction('Digits', { path: 'digits[]' });
-  }
-
-  // @Node
-  Infinity(p) {
-    p.eatMatch('-', 'Punctuator', { path: 'negative' });
-    p.eat('Infinity', 'Keyword', { path: 'value' });
-  }
-
-  Digits(p) {
-    while (p.match(/\d/y)) {
-      p.eatProduction('Digit');
-    }
-  }
-
-  // @Node
-  Digit(p) {
-    p.eatLiteral(/\d/y);
-  }
-};
-
-module.exports = { name, dependencies, covers, grammar };

package/lib/languages/string.js

@@ -1,86 +0,0 @@
-const objectEntries = require('iter-tools-es/methods/object-entries');
-const { buildCovers } = require('../utils.js');
-const sym = require('../symbols.js');
-
-const name = 'String';
-
-const dependencies = {};
-
-const covers = buildCovers({
-  [sym.node]: ['String', 'Content'],
-});
-
-const PN = 'Punctuator';
-
-const escapables = new Map(
-  objectEntries({
-    n: '\n',
-    r: '\r',
-    t: '\t',
-    0: '\0',
-  }),
-);
-
-const cookEscape = (escape, span) => {
-  let hexMatch;
-
-  if (!escape.startsWith('\\')) {
-    throw new Error('string escape must start with \\');
-  }
-
-  if ((hexMatch = /\\u([0-9a-f]{4})/iy.exec(escape))) {
-    //continue
-  } else if ((hexMatch = /\\u{([0-9a-f]+)}/iy.exec(escape))) {
-    //continue
-  }
-
-  if (hexMatch) {
-    return String.fromCodePoint(parseInt(hexMatch[1], 16));
-  }
-
-  const litPattern = span === 'Single' ? /\\([\\nrt0'])/y : /\\([\\nrt0"])/y;
-  const litMatch = litPattern.exec(escape);
-
-  if (litMatch) {
-    return escapables.get(litMatch[1]) || litMatch[1];
-  }
-
-  throw new Error('unable to cook string escape');
-};
-
-const grammar = class StringMiniparserGrammar {
-  // @Node
-  String(p) {
-    const q = p.match(/['"]/y) || '"';
-
-    const span = q === '"' ? 'Double' : 'Single';
-
-    p.eat(q, PN, { path: 'open', startSpan: span, balanced: q });
-    while (p.match(/./sy) || p.atExpression) {
-      p.eatProduction('Content', { path: 'content' });
-    }
-    p.eat(q, PN, { path: 'close', endSpan: span, balancer: true });
-  }
-
-  // @Node
-  Content(p) {
-    let esc, lit;
-    let i = 0;
-    do {
-      esc =
-        p.span.type === 'Single'
-          ? p.eatMatchEscape(/\\(u(\{\d{1,6}\}|\d{4})|[\\nrt0'])/y)
-          : p.eatMatchEscape(/\\(u(\{\d{1,6}\}|\d{4})|[\\nrt0"])/y);
-      lit =
-        p.span.type === 'Single'
-          ? p.eatMatchLiteral(/[^\r\n\0\\']+/y)
-          : p.eatMatchLiteral(/[^\r\n\0\\"]+/y);
-      i++;
-    } while (esc || lit);
-    if (i === 1 && !esc && !lit) {
-      throw new Error('Invalid string content');
-    }
-  }
-};
-
-module.exports = { name, dependencies, covers, grammar, escapables, cookEscape };