@bablr/boot 0.7.2 → 0.8.0

package/lib/index.js

@@ -4,7 +4,7 @@ import * as regex from './languages/regex.js';
 import * as instruction from './languages/instruction.js';
 
 import { TemplateParser } from './miniparser.js';
-import { Resolver, buildEmbeddedMatcher, buildEmbeddedRegex } from '@bablr/agast-helpers/tree';
+import { buildEmbeddedMatcher, buildEmbeddedRegex } from '@bablr/agast-vm-helpers/builders';
 
 export const buildTag = (language, defaultType) => {
   const defaultTag = (quasis, ...exprs) => {
@@ -57,4 +57,4 @@ export const re = new Proxy(re_, {
   },
 });
 
-export { TemplateParser, Resolver };
+export { TemplateParser };

package/lib/languages/cstml.js

@@ -1,4 +1,4 @@
-import * as sym from '@bablr/agast-helpers/symbols';
+import * as sym from '@bablr/agast-vm-helpers/symbols';
 import * as JSON from './json.js';
 
 const _ = /\s+/y;
@@ -24,6 +24,7 @@ export const covers = new Map([
       'ReferenceTag',
       'ReferenceFlags',
       'Identifier',
+      'IdentifierContent',
       'GlobalIdentifier',
       'NullTag',
       'GapTag',
@@ -49,7 +50,7 @@ export const grammar = class CSTMLMiniparserGrammar {
   Document(p) {
     p.eatProduction('DoctypeTag', { path: 'doctype' });
     p.eatMatchTrivia(_);
-    p.eatProduction('Node', { path: 'tree' });
+    p.eatProduction('Node', { path: 'tree' }, { fragment: true });
   }
 
   // @Node
@@ -80,8 +81,8 @@ export const grammar = class CSTMLMiniparserGrammar {
   }
 
   // @Node
-  Node(p) {
-    let open = p.eatProduction('OpenNodeTag', { path: 'open' });
+  Node(p, props) {
+    let open = p.eatProduction('OpenNodeTag', { path: 'open' }, props);
 
     p.eatMatchTrivia(_);
 
@@ -100,7 +101,7 @@ export const grammar = class CSTMLMiniparserGrammar {
     }
   }
 
-  NodeChild(p, _, props) {
+  NodeChild(p, props) {
     const { token } = props || {};
 
     if (token) {
@@ -112,7 +113,7 @@ export const grammar = class CSTMLMiniparserGrammar {
     } else {
       if (p.match(/<\*?#/y)) {
         p.eatProduction('Node');
-      } else if (p.match(/[a-zA-Z]|[.#@]/y)) {
+      } else if (p.match(/[a-zA-Z`\\\u{80}-\u{10ffff}.]|[.#@]/uy)) {
         p.eatProduction('Property');
       } else if (p.match(/['"]/y)) {
         p.eatProduction('LiteralTag');
@@ -141,10 +142,12 @@ export const grammar = class CSTMLMiniparserGrammar {
   NodeFlags(p) {
     p.eatMatch('*', PN, { path: 'tokenToken' });
     p.eatMatch('$', PN, { path: 'hasGapToken' });
+    p.eatMatch('_', PN, { path: 'fragmentToken' });
+    p.eatMatch('_', PN, { path: 'coverFragmentToken' });
   }
 
   // @Node
-  OpenNodeTag(p) {
+  OpenNodeTag(p, { fragment } = {}) {
     p.eat('<', PN, { path: 'openToken', startSpan: 'Tag', balanced: '>' });
 
     let flags = null;
@@ -153,9 +156,14 @@ export const grammar = class CSTMLMiniparserGrammar {
     }
 
     let sp = null;
-    let isFragment = !p.match(/['"a-zA-Z]/y);
 
-    if (!isFragment) {
+    if (fragment && !flags.properties.fragmentToken) throw new Error();
+
+    if (!flags.properties.fragmentToken && !p.match(/./sy)) {
+      throw new Error();
+    }
+
+    if (!flags.properties.fragmentToken) {
       p.eatProduction('TagType', { path: 'type' });
 
       sp = p.eatMatchTrivia(_);
@@ -190,7 +198,10 @@ export const grammar = class CSTMLMiniparserGrammar {
   }
 
   TagType(p) {
-    if (p.match(/['"]|[a-zA-Z.]+:/y) || (p.atExpression && p.quasis[p.quasiIdx + 1][0] === ':')) {
+    if (
+      p.match(/['"]|[a-zA-Z`\\\u{80}-\u{10ffff}.]+:/uy) ||
+      (p.atExpression && p.quasis[p.quasiIdx + 1][0] === ':')
+    ) {
       p.eatProduction('LanguageReference', { path: 'language' });
       p.eat(':', PN, { path: 'namespaceSeparatorToken' });
       p.eatProduction('Identifier', { path: 'type' });
@@ -217,7 +228,31 @@ export const grammar = class CSTMLMiniparserGrammar {
 
   // @Node
   Identifier(p) {
-    p.eatLiteral(/[a-zA-Z][a-zA-Z-_]*/y);
+    let res, q;
+    q = p.match('`');
+    if (q) p.eat('`', PN, { path: 'openToken', balanced: '`' });
+
+    p.eatProduction('IdentifierContent', { path: 'content', span: 'Identifier' }, { quoted: !!q });
+
+    if (q) p.eat('`', PN, { path: 'closeToken', balancer: true });
+  }
+
+  // @Node
+  IdentifierContent(p, props) {
+    let { quoted = false } = props || {};
+
+    let lit, esc;
+    do {
+      if ((esc = p.match('\\'))) {
+        p.eatMatchEscape(/\\(u(\{[0-9a-fA-F]\}|\d{4}))/y);
+      } else {
+        if (!quoted) {
+          lit = p.eatMatchLiteral(/[a-zA-Z\u{80}-\u{10ffff}][a-zA-Z0-9_\u{80}-\u{10ffff}-]*/uy);
+        } else {
+          lit = p.eatMatchLiteral(/[^`\r\n]+/uy);
+        }
+      }
+    } while (lit || esc);
   }
 
   // @Cover
@@ -263,9 +298,4 @@ export const grammar = class CSTMLMiniparserGrammar {
     p.eatMatch('+', PN, { path: 'expressionToken' });
     p.eatMatch('$', PN, { path: 'hasGapToken' });
   }
-
-  // @Node
-  Literal(p) {
-    p.eatProduction('JSON:String', { path: 'value' });
-  }
 };

package/lib/languages/instruction.js

@@ -1,4 +1,4 @@
-import * as sym from '@bablr/agast-helpers/symbols';
+import * as sym from '@bablr/agast-vm-helpers/symbols';
 import * as Spamex from './spamex.js';
 import * as CSTML from './cstml.js';
 import * as Regex from './regex.js';

package/lib/languages/json.js

@@ -1,5 +1,5 @@
 import objectEntries from 'iter-tools/methods/object-entries';
-import * as sym from '@bablr/agast-helpers/symbols';
+import * as sym from '@bablr/agast-vm-helpers/symbols';
 import * as Spamex from './spamex.js';
 import * as CSTML from './cstml.js';
 import * as Regex from './regex.js';
@@ -28,7 +28,8 @@ export const covers = new Map([
       'Number',
       'UnsignedInteger',
       'Infinity',
-      'NaN',
+      'NotANumber',
+      'Undefined',
       'Digit',
       'Integer',
       'String',
@@ -36,7 +37,7 @@ export const covers = new Map([
     ]),
   ],
   ['Expression', new Set(['Object', 'Array', 'Boolean', 'Null', 'Number', 'String'])],
-  ['Number', new Set(['Integer', 'Infinity', 'NaN'])],
+  ['Number', new Set(['Integer', 'Infinity', 'NotANumber'])],
 ]);
 
 export const escapables = new Map(
@@ -92,6 +93,8 @@ export const grammar = class JSONMiniparserGrammar {
       p.eatProduction('Boolean');
     } else if (p.match('null')) {
       p.eatProduction('Null');
+    } else if (p.match('undefined')) {
+      p.eatProduction('Undefined');
     } else if (p.match(/[+-]?(?:\d|Infinity)|NaN/y)) {
       p.eatProduction('Number');
     } else if (p.match(/['"]/y)) {
@@ -109,7 +112,7 @@ export const grammar = class JSONMiniparserGrammar {
     let sep;
     while (first || (sep && (p.match(/./y) || p.atExpression))) {
       p.eatProduction('Property', { path: 'properties[]' });
-      if (p.match(/\s*,?/y)) {
+      if ((sep = p.match(/\s*,/y))) {
         sep = p.eatProduction('ListSeparator', { path: 'separators[]' });
       }
       first = false;
@@ -139,7 +142,7 @@ export const grammar = class JSONMiniparserGrammar {
     let sep;
     while (first || (sep && (p.match(/./y) || p.atExpression))) {
       p.eatProduction('Expression', { path: 'elements[]' });
-      if (p.match(/\s*,?/y)) {
+      if (p.match(/\s*,/y)) {
         sep = p.eatProduction('ListSeparator', { path: 'separators[]' });
       }
 
@@ -161,7 +164,7 @@ export const grammar = class JSONMiniparserGrammar {
 
   ListSeparator(p) {
     p.eatMatchTrivia(_);
-    p.eatMatch(/,/y, PN, { path: 'separators[]' });
+    p.eat(/,/y, PN, { path: 'separators[]' });
     p.eatMatchTrivia(_);
   }
 
@@ -193,10 +196,15 @@ export const grammar = class JSONMiniparserGrammar {
   }
 
   // @Node
-  NaN(p) {
+  NotANumber(p) {
     p.eat('NaN', 'Keyword', { path: 'sigilToken' });
   }
 
+  // @Node
+  Undefined(p) {
+    p.eat('undefined', 'Keyword', { path: 'sigilToken' });
+  }
+
   Digits(p) {
     while (p.match(/\d/y)) {
       p.eatProduction('Digit');
@@ -228,8 +236,8 @@ export const grammar = class JSONMiniparserGrammar {
     do {
       esc =
         p.span.type === 'Single'
-          ? p.eatMatchEscape(/\\(u(\{\d{1,6}\}|\d{4})|[\\gnrt0'])/y)
-          : p.eatMatchEscape(/\\(u(\{\d{1,6}\}|\d{4})|[\\gnrt0"])/y);
+          ? p.eatMatchEscape(/\\(u(\{[0-9a-fA-F]{1,6}\}|\d{4})|[\\gnrt0'])/y)
+          : p.eatMatchEscape(/\\(u(\{[0-9a-fA-F]{1,6}\}|\d{4})|[\\gnrt0"])/y);
       lit =
         p.span.type === 'Single'
           ? p.eatMatchLiteral(/[^\r\n\0\\']+/y)

package/lib/languages/regex.js

@@ -1,4 +1,4 @@
-import * as sym from '@bablr/agast-helpers/symbols';
+import * as sym from '@bablr/agast-vm-helpers/symbols';
 import when from 'iter-tools/methods/when';
 import { escapables } from './json.js';
 
@@ -267,7 +267,7 @@ export const grammar = class RegexMiniparserGrammar {
       if (
         p.eatMatchEscape(
           new RegExp(
-            String.raw`\\(u(\{\d{1,6}\}|\d{4})|x[0-9a-fA-F]{2}|[nrt0]|${specialPattern.source})`,
+            String.raw`\\(u(\{[0-9a-fA-F]{1,6}\}|[0-9a-fA-F]{4})|x[0-9a-fA-F]{2}|[nrt0]|${specialPattern.source})`,
             'y',
           ),
         )
@@ -276,7 +276,7 @@ export const grammar = class RegexMiniparserGrammar {
       } else if (p.eatMatchEscape(new RegExp(String.raw`\\${specialPattern.source}`, 'y'))) {
         // done
       } else {
-        throw new Error('escape required');
+        throw new Error('escape not allowed');
       }
     } else {
       if (p.match(new RegExp(specialPattern, 'y'))) {
@@ -306,7 +306,7 @@ export const grammar = class RegexMiniparserGrammar {
 
   // @Cover
   CharacterClassElement(p, { first }) {
-    if (p.match(/.-[^\]\n]/y)) {
+    if (p.match(/(.|\\(u(\{[0-9a-fA-F]{1,6}\}|[0-9a-fA-F]{4})|x[0-9a-fA-F]{2}|\w))-[^\]\n]/y)) {
       p.eatProduction('CharacterClassRange', undefined, { first });
     } else if (p.match(/\\[dswp]/iy)) {
       p.eatProduction('CharacterSet');

package/lib/languages/spamex.js

@@ -1,4 +1,4 @@
-import * as sym from '@bablr/agast-helpers/symbols';
+import * as sym from '@bablr/agast-vm-helpers/symbols';
 import * as Regex from './regex.js';
 import * as CSTML from './cstml.js';
 import * as JSON from './json.js';
@@ -17,7 +17,7 @@ export const covers = new Map([
   [
     sym.node,
     new Set([
-      'Identifier',
+      'CSTML:Identifier',
       'PropertyMatcher',
       'JSON:String',
       'Regex:Pattern',
@@ -159,11 +159,7 @@ export const grammar = class SpamexMiniparserGrammar {
     } else if (p.match(' ')) {
       p.eatMatchTrivia(_);
     } else {
-      if (p.atExpression) {
-        p.eatProduction('Identifier', { path: 'type' });
-      } else {
-        p.eatMatch(/[a-zA-Z]+/y, ID, { path: 'type' });
-      }
+      p.eatProduction('CSTML:Identifier', { path: 'type' });
     }
 
     let sp = p.eatMatchTrivia(_);
@@ -197,9 +193,4 @@ export const grammar = class SpamexMiniparserGrammar {
       p.eatProduction('Regex:Pattern');
     }
   }
-
-  // @Node
-  Identifier(p) {
-    p.eatLiteral(/[a-zA-Z]+/y);
-  }
 };

package/lib/match.js

@@ -1,7 +1,7 @@
 import { createNode } from '@bablr/agast-helpers/tree';
 import { Path } from './path.js';
 import { resolveDependentLanguage } from './utils.js';
-import * as sym from '@bablr/agast-helpers/symbols';
+import * as sym from '@bablr/agast-vm-helpers/symbols';
 
 export class Match {
   constructor(parent, resolvedLanguage, id, attributes, path) {

package/lib/miniparser.js

@@ -2,11 +2,13 @@ import escapeRegex from 'escape-string-regexp';
 import arrayLast from 'iter-tools/methods/array-last';
 import isString from 'iter-tools/methods/is-string';
 import isObject from 'iter-tools/methods/is-object';
-import * as sym from '@bablr/agast-helpers/symbols';
+import find from 'iter-tools/methods/find';
+import every from 'iter-tools/methods/every';
+import * as sym from '@bablr/agast-vm-helpers/symbols';
 import { Match } from './match.js';
 import { parsePath } from './path.js';
 import { isRegex, isArray, getPrototypeOf } from './utils.js';
-import { ReferenceTag, LiteralTag, ShiftTag } from '@bablr/agast-helpers/symbols';
+import { ReferenceTag, LiteralTag } from '@bablr/agast-helpers/symbols';
 import {
   buildCloseNodeTag,
   buildLiteralTag,
@@ -14,8 +16,8 @@ import {
   buildReferenceTag,
   nodeFlags,
 } from '@bablr/agast-helpers/builders';
-import { add, buildGapTag, buildShiftTag, buildToken } from '@bablr/agast-helpers/tree';
-import * as btree from '@bablr/agast-helpers/btree';
+import { add, buildToken } from '@bablr/agast-helpers/tree';
+import * as sumtree from '@bablr/agast-helpers/sumtree';
 import { get } from '@bablr/agast-helpers/path';
 
 const Escape = Symbol.for('Escape');
@@ -173,19 +175,19 @@ export class TemplateParser {
       this.idx = 0;
 
       if (parentPath?.node && (isNode || covers.has(type))) {
-        const { properties, children } = parentPath.node;
+        const { node } = parentPath;
         const path = parsePath(this.m.attrs.path);
 
         if (isArray(result)) {
           for (const value of result) {
-            children.push(path);
+            node.children = sumtree.push(node.children, path);
 
-            add(parentPath.node, path, value);
+            add(node, path, value);
           }
         } else {
-          children.push(path);
+          node.children = sumtree.push(node.children, path);
 
-          add(parentPath.node, path, result);
+          add(node, path, result);
         }
       }
     } else {
@@ -199,7 +201,7 @@ export class TemplateParser {
 
       if (isNode) {
         let { node } = this.path;
-        node.children = btree.push(
+        node.children = sumtree.push(
           node.children,
           buildOpenNodeTag(nodeFlags, node.language, node.type),
         );
@@ -215,16 +217,16 @@ export class TemplateParser {
         const { node } = this.path;
         if (result?.attrs) {
           node.attributes = result.attrs;
-          node.children = btree.replaceAt(
+          node.children = sumtree.replaceAt(
             0,
             node.children,
             buildOpenNodeTag(nodeFlags, node.language, node.type, result.attrs),
           );
         }
 
-        node.children = btree.push(node.children, buildCloseNodeTag());
+        node.children = sumtree.push(node.children, buildCloseNodeTag());
 
-        if (parentPath?.node && !covers.has(type) && shift == null) {
+        if (parentPath?.node && !covers.has(type)) {
           const path = parsePath(this.m.attrs.path);
 
           add(parentPath.node, path, node, shift);
@@ -239,7 +241,7 @@ export class TemplateParser {
 
       const { children } = this.path.node;
 
-      if (children.find(isTag) && !children.every(isTag)) {
+      if (find(isTag, sumtree.traverse(children)) && every(isTag, sumtree.traverse(children))) {
         throw new Error('strings must be wrapped in nodes');
       }
     }
@@ -279,20 +281,17 @@ export class TemplateParser {
   }
 
   eatHeldProduction(type, attrs) {
-    const { children, properties } = this.node;
-
     if (!this.held) {
       throw new Error();
     }
 
-    const { held } = this;
+    const { held, node } = this;
 
     this.held = null;
 
     const path = parsePath(attrs.path);
 
-    children.push(path);
-    add(this.node, path, held);
+    add(node, path, held, 0);
 
     return held;
   }
@@ -304,7 +303,7 @@ export class TemplateParser {
 
     // get the most recently produced node and detach it from its parent
 
-    const ref = btree.getAt(-2, node.children);
+    const ref = sumtree.getAt(-2, node.children);
 
     if (!ref.value.flags.expression) throw new Error();
 
@@ -318,7 +317,7 @@ export class TemplateParser {
 
     const shifted = this.eval(id_, attrs, props, 1);
 
-    add(node, ref, shifted, 1);
+    // add(node, ref, shifted, 1);
 
     return shifted;
   }
@@ -448,7 +447,7 @@ export class TemplateParser {
 
     this.idx += result.length;
 
-    this.node.children.push(buildLiteralTag(result));
+    this.node.children = sumtree.push(this.node.children, buildLiteralTag(result));
 
     return result;
   }
@@ -459,7 +458,7 @@ export class TemplateParser {
     if (result) {
       this.idx += result.length;
 
-      this.node.children.push(buildLiteralTag(result));
+      this.node.children = sumtree.push(this.node.children, buildLiteralTag(result));
     }
 
     return result;

package/package.json

@@ -1,13 +1,18 @@
 {
   "name": "@bablr/boot",
-  "version": "0.7.2",
+  "version": "0.8.0",
   "description": "Compile-time tools for bootstrapping BABLR VM",
   "engines": {
     "node": ">=12.0.0"
   },
   "type": "module",
   "exports": {
-    ".": "./lib/index.js"
+    ".": "./lib/index.js",
+    "./languages/cstml": "./lib/languages/cstml.js",
+    "./languages/instruction": "./lib/languages/instruction.js",
+    "./languages/json": "./lib/languages/json.js",
+    "./languages/regex": "./lib/languages/regex.js",
+    "./languages/spamex": "./lib/languages/spamex.js"
   },
   "files": [
     "lib/**/*.js",
@@ -15,8 +20,8 @@
   ],
   "sideEffects": false,
   "dependencies": {
-    "@bablr/agast-helpers": "0.6.1",
-    "@bablr/agast-vm-helpers": "0.6.1",
+    "@bablr/agast-helpers": "0.7.0",
+    "@bablr/agast-vm-helpers": "0.7.0",
     "@iter-tools/imm-stack": "1.1.0",
     "escape-string-regexp": "5.0.0",
     "iter-tools": "^7.5.3"