@bablr/boot 0.5.0 → 0.6.1

package/lib/builders.js

@@ -2,8 +2,11 @@ const sym = require('@bablr/boot-helpers/symbols');
 const { freeze, getPrototypeOf } = Object;
 const { isArray } = Array;
 const {
+  DoctypeTag,
   OpenNodeTag,
   CloseNodeTag,
+  OpenFragmentTag,
+  CloseFragmentTag,
   ReferenceTag,
   GapTag,
   LiteralTag,
@@ -66,6 +69,13 @@ const interpolateString = (value) => {
   return buildNode('String', 'Content', children);
 };
 
+const buildDoctypeTag = (attributes) => {
+  return freeze({
+    type: DoctypeTag,
+    value: freeze({ doctype: 'cstml', version: 0, attributes }),
+  });
+};
+
 const buildReferenceTag = (name, isArray) => {
   return freeze({ type: ReferenceTag, value: freeze({ name, isArray }) });
 };
@@ -82,6 +92,19 @@ const buildNodeCloseTag = (type) => {
   return freeze({ type: CloseNodeTag, value: freeze({ type }) });
 };
 
+const buildFragmentOpenTag = (flags = {}) => {
+  return freeze({
+    type: OpenFragmentTag,
+    value: freeze({
+      flags: freeze(flags),
+    }),
+  });
+};
+
+const buildFragmentCloseTag = (type = null, language = null) => {
+  return freeze({ type: CloseFragmentTag, value: freeze({ language, type }) });
+};
+
 const buildLiteralTag = (value) => {
   return freeze({ type: LiteralTag, value });
 };
@@ -421,10 +444,13 @@ module.exports = {
   buildSpace,
   buildIdentifier,
   buildAttribute,
+  buildDoctypeTag,
   buildReferenceTag,
   buildGapTag,
   buildNodeOpenTag,
   buildNodeCloseTag,
+  buildFragmentOpenTag,
+  buildFragmentCloseTag,
   buildLiteralTag,
   buildNode,
   buildSyntacticNode,
@@ -432,6 +458,8 @@ module.exports = {
   buildSyntacticEscapeNode,
   buildTriviaNode,
   buildGapNode,
+  nodeFlags,
+
   ref,
   lit,
   gap,

package/lib/index.js

@@ -5,11 +5,16 @@ const regex = require('./languages/regex.js');
 const instruction = require('./languages/instruction.js');
 const {
   buildLiteralTag,
+  buildDoctypeTag,
   buildNodeOpenTag,
   buildNodeCloseTag,
+  buildFragmentOpenTag,
+  buildFragmentCloseTag,
   buildTriviaNode,
   buildGapNode,
   buildSyntacticEscapeNode,
+  nodeFlags,
+  buildReferenceTag,
 } = require('./builders.js');
 const { TemplateParser } = require('./miniparser.js');
 const { Resolver } = require('./print.js');
@@ -62,7 +67,7 @@ const add = (obj, path, value) => {
 
 const buildTag = (language, defaultType) => {
   const defaultTag = (quasis, ...exprs) => {
-    return getAgASTValue(
+    return getAgASTTree(
       language,
       new TemplateParser(language, quasis.raw, exprs).eval({
         language: language.name,
@@ -78,7 +83,7 @@ const buildTag = (language, defaultType) => {
 
     get(_, type) {
       return (quasis, ...exprs) => {
-        return getAgASTValue(
+        return getAgASTTree(
           language,
           new TemplateParser(language, quasis.raw, exprs).eval({
             language: language.name,
@@ -97,6 +102,23 @@ const parse = (language, type, sourceText) => {
   });
 };
 
+const getAgASTTree = (language, miniNode) => {
+  const attributes = { 'bablr-language': language.canonicalURL };
+  return {
+    flags: nodeFlags,
+    language: language.canonicalURL,
+    type: null,
+    children: [
+      buildDoctypeTag(attributes),
+      buildFragmentOpenTag(nodeFlags, null, null, null),
+      buildReferenceTag('.'),
+      buildFragmentCloseTag(),
+    ],
+    properties: { ['.']: getAgASTValue(language, miniNode) },
+    attributes,
+  };
+};
+
 const getAgASTValue = (language, miniNode) => {
   if (!miniNode) return miniNode;
 
@@ -109,7 +131,7 @@ const getAgASTValue = (language, miniNode) => {
     escape: !!miniNode.flags?.escape,
     trivia: !!miniNode.flags?.trivia,
     token: false,
-    intrinsic: false,
+    hasGap: false,
   };
   const properties = {};
   let children = [];
@@ -136,10 +158,6 @@ const getAgASTValue = (language, miniNode) => {
     flags.token = true;
   }
 
-  if (type === 'Punctuator' || type === 'Keyword') {
-    flags.intrinsic = true;
-  }
-
   children = btree.push(
     children,
     buildNodeOpenTag(flags, resolvedLanguage.canonicalURL, type, attributes),
@@ -202,7 +220,7 @@ const getAgASTValue = (language, miniNode) => {
 const str = buildTag(cstml, 'String');
 const num = buildTag(cstml, 'Integer');
 const cst = buildTag(cstml, 'Node');
-const spam = buildTag(spamex, 'Pattern');
+const spam = buildTag(spamex, 'Matcher');
 const re = buildTag(regex, 'Pattern');
 const i = buildTag(instruction, 'Call');
 

package/lib/languages/cstml.js

@@ -7,7 +7,6 @@ const _ = /\s+/y;
 const PN = 'Punctuator';
 const ID = 'Identifier';
 const KW = 'Keyword';
-const LIT = 'Literal';
 
 const name = 'CSTML';
 
@@ -67,9 +66,12 @@ const covers = buildCovers({
     'NullTag',
     'GapTag',
     'Node',
+    'Fragment',
     'IdentifierPath',
     'OpenNodeTag',
     'CloseNodeTag',
+    'OpenFragmentTag',
+    'CloseFragmentTag',
     'Tag',
     'Number',
     'Digit',
@@ -93,7 +95,7 @@ const grammar = class CSTMLMiniparserGrammar {
   Document(p) {
     p.eatProduction('DoctypeTag', { path: 'doctype' });
     p.eatMatchTrivia(_);
-    p.eatProduction('Node', { path: 'tree' });
+    p.eatProduction('Fragment', { path: 'tree' });
   }
 
   // @Node
@@ -105,7 +107,7 @@ const grammar = class CSTMLMiniparserGrammar {
 
     let sp = p.eatMatchTrivia(_);
 
-    if ((sp && p.match(/[a-zA-Z]+/y)) || p.atExpression) {
+    if ((sp && p.match(/!?[a-zA-Z]/y)) || p.atExpression) {
       p.eatProduction('Attributes');
       sp = p.eatMatchTrivia(_);
     }
@@ -156,7 +158,7 @@ const grammar = class CSTMLMiniparserGrammar {
     } else {
       if (p.match(/<\*?#/y)) {
         p.eatProduction('Node');
-      } else if (p.match(/[a-zA-Z]/y)) {
+      } else if (p.match(/[a-zA-Z]|\./y)) {
         p.eatProduction('Property');
       } else if (p.match(/['"]/y)) {
         p.eatProduction('LiteralTag');
@@ -164,6 +166,25 @@ const grammar = class CSTMLMiniparserGrammar {
     }
   }
 
+  // @Node
+  Fragment(p) {
+    let open = p.eatProduction('OpenFragmentTag', { path: 'open' });
+
+    p.eatMatchTrivia(_);
+
+    if (open.properties.flags?.token) {
+      p.eatProduction('NodeChild', { path: 'children[]' }, { token: true });
+      p.eatMatchTrivia(_);
+    }
+
+    while (!(p.match('</') || p.done)) {
+      p.eatProduction('NodeChild', { path: 'children[]' });
+      p.eatMatchTrivia(_);
+    }
+
+    p.eatProduction('CloseFragmentTag', { path: 'close' });
+  }
+
   // @Node
   Property(p) {
     p.eatProduction('ReferenceTag', { path: 'reference' });
@@ -184,10 +205,10 @@ const grammar = class CSTMLMiniparserGrammar {
   // @Node
   Flags(p) {
     let tr = p.eatMatch('#', PN, { path: 'triviaToken' });
-    p.eatMatch('~', PN, { path: 'intrinsicToken' });
     p.eatMatch('*', PN, { path: 'tokenToken' });
     let esc = p.eatMatch('@', PN, { path: 'escapeToken' });
     let exp = p.eatMatch('+', PN, { path: 'expressionToken' });
+    p.eatMatch('$', PN, { path: 'hasGapToken' });
 
     if ((tr && esc) || (exp && (tr || esc))) throw new Error();
   }
@@ -210,12 +231,18 @@ const grammar = class CSTMLMiniparserGrammar {
 
     let sp = p.eatMatchTrivia(_);
 
-    if (flags.properties.intrinsic && sp && (p.match(/['"/]/y) || p.atExpression)) {
-      p.eatProduction('String', { path: 'intrinsicValue' });
+    let iv;
+
+    if (sp && (p.match(/['"]/y) || p.atExpression)) {
+      iv = p.eatProduction('String', { path: 'intrinsicValue' });
 
       sp = p.eatMatchTrivia(_);
     }
 
+    if (!flags.properties.tokenToken && iv) {
+      throw new Error();
+    }
+
     if ((sp && p.match(/[a-zA-Z]+/y)) || p.atExpression) {
       p.eatProduction('Attributes');
       sp = p.eatMatchTrivia(_);
@@ -232,16 +259,37 @@ const grammar = class CSTMLMiniparserGrammar {
     p.eat('>', PN, { path: 'closeToken', endSpan: 'Tag', balancer: true });
   }
 
+  // @Node
+  OpenFragmentTag(p) {
+    p.eat('<', PN, { path: 'openToken', startSpan: 'Tag', balanced: '>' });
+
+    let flags = null;
+    if (!p.atExpression) {
+      flags = p.eatProduction('Flags', { path: 'flags' });
+    }
+
+    if (p.done) {
+      p.eat('>', PN, { path: 'closeToken', endSpan: 'Tag', balancer: true });
+      return;
+    }
+  }
+
+  // @Node
+  CloseFragmentTag(p) {
+    p.eat('</', PN, { path: 'openToken', startSpan: 'Tag', balanced: '>' });
+    p.eat('>', PN, { path: 'closeToken', endSpan: 'Tag', balancer: true });
+  }
+
   // @Fragment
   Attributes(p) {
     let sp = true;
-    while (sp && (p.match(/[a-zA-Z]+/y) || p.atExpression)) {
+    while (sp && (p.match(/!?[a-zA-Z]/y) || p.atExpression)) {
       if (p.atExpression) {
         p.eatProduction('Attributes'); // ??
       } else {
         p.eatProduction('Attribute', { path: 'attributes[]' });
       }
-      if (p.match(/\s+[a-zA-Z]/y) || (p.match(/\s+$/y) && !p.quasisDone)) {
+      if (p.match(/\s+!?[a-zA-Z]/y) || (p.match(/\s+$/y) && !p.quasisDone)) {
         sp = p.eatMatchTrivia(_);
       } else {
         sp = false;
@@ -251,7 +299,7 @@ const grammar = class CSTMLMiniparserGrammar {
 
   // @Cover
   Attribute(p) {
-    if (p.match(/[a-zA-Z][[a-zA-Z]-_]*\s*=/y)) {
+    if (p.match(/[a-zA-Z][a-zA-Z-_]*\s*=/y)) {
       p.eatProduction('MappingAttribute');
     } else {
       p.eatProduction('BooleanAttribute');
@@ -260,13 +308,13 @@ const grammar = class CSTMLMiniparserGrammar {
 
   // @Node
   BooleanAttribute(p) {
-    p.eat('!', KW, { path: 'negateToken' });
-    p.eat(/[a-zA-Z]+/y, ID, { path: 'key' });
+    p.eatMatch('!', KW, { path: 'negateToken' });
+    p.eat(/[a-zA-Z][a-zA-Z-_]*/y, ID, { path: 'key' });
   }
 
   // @Node
   MappingAttribute(p) {
-    p.eat(/[a-zA-Z]+/y, LIT, { path: 'key' });
+    p.eat(/[a-zA-Z][a-zA-Z-_]*/y, ID, { path: 'key' });
     p.eatMatchTrivia(_);
     p.eat('=', PN, { path: 'mapToken' });
     p.eatMatchTrivia(_);
@@ -283,7 +331,7 @@ const grammar = class CSTMLMiniparserGrammar {
   }
 
   TagType(p) {
-    if (p.match(/[[a-zA-Z].]+:/y)) {
+    if (p.match(/['"]|[a-zA-Z\.]+:/y)) {
       p.eatProduction('LanguageReference', { path: 'language' });
       p.eat(':', PN, { path: 'namespaceSeparatorToken' });
       p.eatProduction('Identifier', { path: 'type' });
@@ -324,7 +372,11 @@ const grammar = class CSTMLMiniparserGrammar {
 
   // @Node
   ReferenceTag(p) {
-    p.eatProduction('Identifier', { path: 'name' });
+    if (p.match('.')) {
+      p.eat('.', PN, { path: 'name' });
+    } else {
+      p.eatProduction('Identifier', { path: 'name' });
+    }
     p.eatMatchTrivia(_);
     p.eatMatch('[]', PN, { path: 'arrayToken' });
     p.eatMatchTrivia(_);

package/lib/languages/regex.js

@@ -154,7 +154,7 @@ const grammar = class RegexMiniparserGrammar {
   }
 
   Elements(p) {
-    while (p.match(/[^|]/y || p.atExpression)) {
+    while (p.match(/[^|]/y) || p.atExpression) {
       p.eatProduction('Element');
     }
   }

package/lib/languages/spamex.js

@@ -14,18 +14,22 @@ const canonicalURL = 'https://bablr.org/languages/core/en/spamex';
 const dependencies = { CSTML, Regex };
 
 const covers = buildCovers({
-  [sym.node]: ['Attribute', 'Identifier', 'Pattern', 'Matcher', 'Literal', 'CSTML:Flags'],
+  [sym.node]: [
+    'Attribute',
+    'Identifier',
+    'Matcher',
+    'OpenNodeMatcher',
+    'CloseNodeMatcher',
+    'Literal',
+    'CSTML:Flags',
+  ],
   Attribute: ['MappingAttribute', 'BooleanAttribute'],
   AttributeValue: ['CSTML:String', 'CSTML:Number'],
-  Matcher: ['NodeMatcher', 'OpenNodeMatcher', 'CloseNodeMatcher', 'StringMatcher'],
+  Matcher: ['NodeMatcher', 'StringMatcher'],
   StringMatcher: ['CSTML:String', 'Regex:Pattern'],
 });
 
 const grammar = class SpamexMiniparserGrammar {
-  Pattern(p) {
-    p.eatProduction('Matcher', { path: 'matcher' });
-  }
-
   // @Cover
   Matcher(p) {
     if (p.match(/<[^!/]/y)) {
@@ -67,10 +71,8 @@ const grammar = class SpamexMiniparserGrammar {
       p.eatProduction('CSTML:Flags', { path: 'flags' });
     }
 
-    if (p.match(/[a-zA-Z]+:/y)) {
-      p.eat(/[a-zA-Z]+/y, ID, { path: 'language' });
-      p.eat(':', PN, { path: 'namespaceSeparatorToken' });
-      p.eat(/[a-zA-Z]+/y, ID, { path: 'type' });
+    if (p.match(/['"]|[a-zA-Z]+:/y)) {
+      p.eatProduction('CSTML:TagType', { path: 'type' });
     } else if (p.match('?')) {
       p.eat('?', PN, { path: 'type' });
     } else if (p.match(' ')) {

package/lib/miniparser.js

@@ -178,9 +178,19 @@ class TemplateParser {
         const { properties, children } = parentPath.node;
         const path = parsePath(this.m.attrs.path);
 
-        children.push({ ...ref(path), id });
+        if (isArray(result)) {
+          for (const value of result) {
+            children.push(ref(path));
 
-        set(properties, path, result);
+            // TODO interpolate separators!
+
+            set(properties, path, value);
+          }
+        } else {
+          children.push(ref(path));
+
+          set(properties, path, result);
+        }
       }
     } else {
       if (isEmbedded) {
@@ -198,6 +208,7 @@ class TemplateParser {
         if (result?.attrs) {
           node.attributes = result.attrs;
         }
+
         if (parentPath?.node && !covers.has(type)) {
           const path = parsePath(this.m.attrs.path);
 

package/lib/print.js

@@ -161,17 +161,11 @@ class Resolver {
   }
 }
 
-const buildDoctypeTag = () => {
-  return freeze({ type: DoctypeTag, value: { doctype: 'cstml', version: 0 } });
-};
-
 function* streamFromTree(rootNode) {
   if (!rootNode || rootNode.type === GapTag) {
     return rootNode;
   }
 
-  yield buildDoctypeTag(rootNode.attributes);
-
   let stack = emptyStack.push(rootNode);
   const resolver = new Resolver();
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bablr/boot",
-  "version": "0.5.0",
+  "version": "0.6.1",
   "description": "Compile-time tools for bootstrapping BABLR VM",
   "engines": {
     "node": ">=12.0.0"
@@ -22,7 +22,7 @@
     "@babel/helper-module-imports": "^7.22.15",
     "@babel/template": "^7.22.15",
     "@babel/types": "7.23.0",
-    "@bablr/boot-helpers": "0.3.0",
+    "@bablr/boot-helpers": "0.3.1",
     "@iter-tools/imm-stack": "1.1.0",
     "escape-string-regexp": "4.0.0",
     "iter-tools": "^7.5.3",

package/shorthand.macro.js

@@ -96,9 +96,7 @@ const generateBabelNodeChild = (child, exprs, bindings) => {
 };
 
 const getAgastNodeType = (flags) => {
-  if (flags.intrinsic && flags.token) {
-    return 's_i_node';
-  } else if (flags.token && flags.trivia) {
+  if (flags.token && flags.trivia) {
     return 's_t_node';
   } else if (flags.token && flags.escape) {
     return 's_e_node';
@@ -140,7 +138,7 @@ const generateBabelNode = (node, exprs, bindings) => {
         } else {
           // gap
           const expr = exprs.pop();
-          const { interpolateArray, interpolateArrayChildren, interpolateString } = bindings;
+          const { interpolateArray, interpolateFragmentChildren, interpolateString } = bindings;
 
           if (pathIsArray) {
             add(
@@ -155,17 +153,10 @@ const generateBabelNode = (node, exprs, bindings) => {
             children_ = btree.push(
               children_,
               t.spreadElement(
-                expression('%%interpolateArrayChildren%%(%%expr%%, %%ref%%, %%sep%%)')({
-                  interpolateArrayChildren,
+                expression('%%interpolateFragmentChildren%%(%%expr%%, %%ref%%)')({
+                  interpolateFragmentChildren,
                   expr,
                   ref: expression(`%%t%%.ref\`${printRef(child.value)}\``)({ t: bindings.t }),
-
-                  // Really really awful unsafe-as-heck hack, to be removed ASAP
-                  // Fixing this requires having interpolation happen during parsing
-                  // That way the grammar can deal with the separators!
-                  sep: expression(
-                    "%%t%%.embedded(%%t%%.t_node(%%l%%.Comment, null, [%%t%%.embedded(%%t%%.t_node('Space', 'Space', [%%t%%.lit(' ')]))]))",
-                  )({ t: bindings.t, l: bindings.l }),
                 }),
               ),
             );
@@ -306,10 +297,10 @@ const shorthandMacro = ({ references }) => {
       );
     }
 
-    if (!bindings.interpolateArrayChildren) {
-      bindings.interpolateArrayChildren = addNamed(
+    if (!bindings.interpolateFragmentChildren) {
+      bindings.interpolateFragmentChildren = addNamed(
         getTopScope(ref.scope).path,
-        'interpolateArrayChildren',
+        'interpolateFragmentChildren',
         '@bablr/agast-helpers/template',
       );
     }