@bablr/boot 0.1.9 → 0.2.1

package/README.md

@@ -1,3 +1,22 @@
 ## @bablr/boot
 
 What a great name. BABLR is defined in terms of BABLR, so this repo helps BABLR boot up without getting stuck in dependency cycles!
+
+### Bootstrapping
+
+This package's job is to break dependency cycles that arise when evolving the system by using code generation. The nature of the cycles is this:
+
+- The VM needs parsed instructions to process code
+- The parser for the instruction language needs a VM with the instruction language already defined
+- The instruction language now needs itself to parse itself
+
+Using code-generation to bypass this conundrum is a form of "bootstrapping", which is actually not a one-time event but a continuous process, since the dependency cycle cannot fully be eliminated. In general we use this package to define code generation that uses the last generation of parsers to codegen instructions that are valid definitions for the "core languages" in the next version of the parser.
+
+This allows us to do the following:
+
+- Import the "core grammars" compiled for the next vm, and the next VM into the helpers
+- Import template tags defined by the helpers into non-core grammars
+  - Define non-core grammar in terms of template tags like `` i`eat(' ')` ``
+    - These result of evaluating these template tags is a parse tree not a string
+- Execute the grammar, using the helper VM to parse template tag instructions on the fly
+  - Cache parsed instructions when they are static using `WeakMap` using `quasis` (first) argument to the template string iterpolator as the cache key, a trick that is possible because `quasis` is an array (a valid `WeakMap` key) whose identity does not change between subsequent invocations. This is how a pattern that looks slow can be fast! Using template string interpolation, e.g. `` i`eat(${something})` `` will prevent caching.

package/lib/builders.js

@@ -0,0 +1,422 @@
+const { freeze, getPrototypeOf } = Object;
+const { isArray } = Array;
+
+const spreads = new WeakMap();
+
+const spread = (arg) => {
+  const wrapper = { value: arg };
+  spreads.set(wrapper, true);
+  return wrapper;
+};
+
+const interpolateArray = (values, buildSeparator) => {
+  const children = [];
+  for (const value of values) {
+    if (spreads.has(value)) {
+      let first = true;
+
+      for (const element of value.value) {
+        if (!first && buildSeparator) {
+          children.push(buildSeparator());
+        }
+
+        children.push(element);
+
+        first = false;
+      }
+    } else {
+      children.push(value);
+    }
+  }
+  return children;
+};
+
+const buildTerminal = (term) => {
+  switch (term?.type) {
+    case 'Literal':
+      return term;
+
+    // case 'Escape':
+    //   return buildEscapeNode;
+
+    default:
+      throw new Error('Invalid terminal of type ' + term.type);
+  }
+};
+
+const interpolateString = (value) => {
+  const children = [];
+  if (isArray(value)) {
+    for (const element of value) {
+      children.push(buildTerminal(element));
+    }
+  } else {
+    // we can't safely interpolate strings here, though I wish we could
+    children.push(buildTerminal(value));
+  }
+
+  return buildNode('String', 'Content', children);
+};
+
+const buildReference = (name, isArray) => {
+  return freeze({ type: 'Reference', value: freeze({ name, isArray }) });
+};
+
+const buildGap = () => {
+  return freeze({ type: 'Gap', value: undefined });
+};
+
+const buildNodeOpenTag = (flags, type, attributes = {}) => {
+  return freeze({ type: 'OpenNodeTag', value: freeze({ flags, type, attributes }) });
+};
+
+const buildFragmentOpenTag = (flags = {}, language) => {
+  return freeze({ type: 'OpenFragmentTag', value: freeze({ flags: freeze(flags), language }) });
+};
+
+const buildNodeCloseTag = (type) => {
+  return freeze({ type: 'CloseNodeTag', value: freeze({ type }) });
+};
+
+const buildFragmentCloseTag = () => {
+  return freeze({ type: 'CloseFragmentTag', value: freeze({}) });
+};
+
+const buildLiteral = (value) => {
+  return freeze({ type: 'Literal', value });
+};
+
+const buildProperty = (key, value) => {
+  return node('Instruction', 'Property', [ref`key`, ref`mapOperator`, buildSpace(), ref`value`], {
+    key: buildIdentifier(key),
+    mapOperator: s_node('Instruction', 'Punctuator', ':'),
+    value: buildExpression(value),
+  });
+};
+
+const buildString = (value) => {
+  const terminals = [];
+  let literal = '';
+  for (const chr of value) {
+    if (chr === "'") {
+      if (literal)
+        terminals.push(
+          freeze({
+            type: 'Literal',
+            value: literal,
+          }),
+        );
+      terminals.push(
+        e_node(
+          'String',
+          'Escape',
+          [ref`escape`, ref`escapee`],
+          {
+            escape: s_node('String', 'Punctuator', '\\'),
+            escapee: s_node('String', 'Literal', "'"),
+          },
+          { cooked: chr },
+        ),
+      );
+    } else if (chr === '\\') {
+      if (literal)
+        terminals.push({
+          type: 'Literal',
+          value: literal,
+        });
+      terminals.push(
+        e_node(
+          'String',
+          'Escape',
+          [ref`escape`, ref`escapee`],
+          {
+            escape: s_node('String', 'Punctuator', '\\'),
+            escapee: s_node('String', 'Literal', '\\'),
+          },
+          { cooked: chr },
+        ),
+      );
+    } else {
+      literal += chr;
+    }
+  }
+  if (literal)
+    terminals.push(
+      freeze({
+        type: 'Literal',
+        value: literal,
+      }),
+    );
+  return node(
+    'String',
+    'String',
+    [ref`open`, ref`content`, ref`close`],
+    {
+      open: s_node('String', 'Punctuator', "'"),
+      content: interpolateString(terminals),
+      close: s_node('String', 'Punctuator', "'"),
+    },
+    {},
+  );
+};
+
+const buildBoolean = (value) => {
+  return value
+    ? node(
+        'Instruction',
+        'Boolean',
+        [ref`value`],
+        {
+          value: s_node('Instruction', 'Keyword', 'true'),
+        },
+        {},
+      )
+    : node(
+        'Instruction',
+        'Boolean',
+        [ref`value`],
+        {
+          value: s_node('Instruction', 'Keyword', 'false'),
+        },
+        {},
+      );
+};
+
+const buildNull = () => {
+  return node(
+    'Instruction',
+    'Null',
+    [ref`value`],
+    {
+      value: s_node('Instruction', 'Keyword', 'null'),
+    },
+    {},
+  );
+};
+
+const buildArray = (elements) => {
+  return node(
+    'Instruction',
+    'Array',
+    [ref`open`, ref`elements[]`, ref`close`],
+    {
+      open: s_node('Instruction', 'Punctuator', '['),
+      elements: [...interpolateArray(spread(elements, buildSpace))],
+      close: s_node('Instruction', 'Punctuator', ']'),
+    },
+    {},
+  );
+};
+
+const buildTuple = (elements) => {
+  return node(
+    'Instruction',
+    'Tuple',
+    [ref`open`, ref`values[]`, ref`close`],
+    {
+      open: s_node('Instruction', 'Punctuator', '('),
+      values: [...interpolateArray(spread(elements, buildSpace))],
+      close: s_node('Instruction', 'Punctuator', ')'),
+    },
+    {},
+  );
+};
+
+const buildObject = (properties) => {
+  return node(
+    'Instruction',
+    'Object',
+    [ref`open`, ref`properties[]`, ref`close`],
+    {
+      open: s_node('Instruction', 'Punctuator', '{'),
+      properties: [
+        ...interpolateArray(
+          spread(Object.entries(properties).map(([key, value]) => buildProperty(key, value))),
+        ),
+      ],
+      close: s_node('Instruction', 'Punctuator', '}'),
+    },
+    {},
+  );
+};
+
+const buildSpace = () => {
+  return t_node('Comment', null, [t_node('Space', 'Space', lit` `)]);
+};
+
+const buildIdentifier = (name) => {
+  return node('Instruction', 'Identifier', [buildLiteral(name)]);
+};
+
+const buildAttribute = (key, value) => {
+  return buildNode('CSTML', 'Attribute', [ref`key`, ref`mapOperator`, ref`value`], {
+    key: buildIdentifier(key),
+    mapOperator: s_node('CSTML', 'Punctuator', '='),
+    value: buildExpression(value),
+  });
+};
+
+const buildExpression = (expr) => {
+  if (expr == null) return buildNull();
+
+  switch (typeof expr) {
+    case 'boolean':
+      return buildBoolean(expr);
+    case 'string':
+      return buildString(expr);
+    case 'object': {
+      switch (getPrototypeOf(expr)) {
+        case Array.prototype:
+          return buildArray(expr);
+        case Object.prototype:
+          if (expr.type && expr.language && expr.children && expr.properties) {
+            return expr;
+          }
+          return buildObject(expr);
+        default:
+          throw new Error();
+      }
+    }
+    default:
+      throw new Error();
+  }
+};
+
+const nodeFlags = freeze({ syntactic: false, escape: false, trivia: false });
+
+const buildNode = (language, type, children = [], properties = {}, attributes = {}) =>
+  freeze({
+    flags: nodeFlags,
+    language,
+    type,
+    children: freeze(children),
+    properties: freeze(properties),
+    attributes: freeze(attributes),
+  });
+
+const syntacticFlags = freeze({ syntactic: true, escape: false, trivia: false });
+
+const buildSyntacticNode = (language, type, value, attributes = {}) =>
+  freeze({
+    flags: syntacticFlags,
+    language,
+    type,
+    children: [buildLiteral(value)],
+    properties: freeze({}),
+    attributes: freeze(attributes),
+  });
+
+const escapeFlags = freeze({ syntactic: false, escape: true, trivia: false });
+
+const buildEscapeNode = (language, type, children = [], properties = {}, attributes = {}) =>
+  freeze({
+    flags: escapeFlags,
+    language,
+    type,
+    children: freeze(children),
+    properties: freeze(properties),
+    attributes: freeze(attributes),
+  });
+
+const syntacticEscapeFlags = freeze({ syntactic: true, escape: true, trivia: false });
+
+const buildSyntacticEscapeNode = (
+  language,
+  type,
+  children = [],
+  properties = {},
+  attributes = {},
+) =>
+  freeze({
+    flags: syntacticEscapeFlags,
+    language,
+    type,
+    children: freeze(children),
+    properties: freeze(properties),
+    attributes: freeze(attributes),
+  });
+
+const triviaFlags = freeze({ syntactic: false, escape: false, trivia: true });
+
+const buildTriviaNode = (language, type, children = [], properties = {}, attributes = {}) =>
+  freeze({
+    flags: triviaFlags,
+    language,
+    type,
+    children: freeze(children),
+    properties: freeze(properties),
+    attributes: freeze(attributes),
+  });
+
+const stripArray = (val) => {
+  if (isArray(val)) {
+    if (val.length > 1) {
+      throw new Error();
+    }
+    return val[0];
+  } else {
+    return val;
+  }
+};
+
+const ref = (path) => {
+  if (isArray(path)) {
+    const pathIsArray = path[0].endsWith('[]');
+    const name = pathIsArray ? path[0].slice(0, -2) : path[0];
+    return buildReference(name, pathIsArray);
+  } else {
+    const { name, pathIsArray } = path;
+    return buildReference(name, pathIsArray);
+  }
+};
+
+const lit = (str) => buildLiteral(stripArray(str));
+
+const gap = buildGap;
+const nodeOpen = buildNodeOpenTag;
+const fragOpen = buildFragmentOpenTag;
+const nodeClose = buildNodeCloseTag;
+const fragClose = buildFragmentCloseTag;
+const node = buildNode;
+const s_node = buildSyntacticNode;
+const e_node = buildEscapeNode;
+const s_e_node = buildSyntacticEscapeNode;
+const t_node = buildTriviaNode;
+
+module.exports = {
+  buildProperty,
+  buildString,
+  buildBoolean,
+  buildNull,
+  buildArray,
+  buildTuple,
+  buildObject,
+  buildExpression,
+  buildSpace,
+  buildIdentifier,
+  buildAttribute,
+  buildReference,
+  buildGap,
+  buildNodeOpenTag,
+  buildFragmentOpenTag,
+  buildNodeCloseTag,
+  buildFragmentCloseTag,
+  buildLiteral,
+  buildNode,
+  buildSyntacticNode,
+  buildEscapeNode,
+  buildSyntacticEscapeNode,
+  buildTriviaNode,
+  ref,
+  lit,
+  gap,
+  nodeOpen,
+  fragOpen,
+  nodeClose,
+  fragClose,
+  node,
+  s_node,
+  e_node,
+  s_e_node,
+  t_node,
+};

package/lib/index.js

@@ -1,17 +1,40 @@
-const instruction = require('./languages/instruction.js');
-const regex = require('./languages/regex.js');
-const spamex = require('./languages/spamex.js');
-const string = require('./languages/string.js');
-const number = require('./languages/number.js');
+const { PathResolver } = require('@bablr/boot-helpers/path');
 const cstml = require('./languages/cstml.js');
+const regex = require('./languages/regex.js');
+const instruction = require('./languages/instruction.js');
+const { buildLiteral } = require('./builders.js');
 const { TemplateParser } = require('./miniparser.js');
 
+const { isArray } = Array;
+const { hasOwn } = Object;
+
+const set = (obj, path, value) => {
+  const { name, isArray: pathIsArray } = path;
+  if (pathIsArray) {
+    if (!obj[name]) {
+      obj[name] = [];
+    }
+
+    if (!isArray(obj[name])) throw new Error('bad array value');
+
+    obj[name].push(value);
+  } else {
+    if (hasOwn(obj, name)) {
+      throw new Error('duplicate child name');
+    }
+    obj[name] = value;
+  }
+};
+
 const buildTag = (language, defaultType) => {
   const defaultTag = (quasis, ...exprs) => {
-    return new TemplateParser(language, quasis.raw, exprs).eval({
-      language: language.name,
-      type: defaultType,
-    });
+    return getAgASTValue(
+      language,
+      new TemplateParser(language, quasis.raw, exprs).eval({
+        language: language.name,
+        type: defaultType,
+      }),
+    );
   };
 
   return new Proxy(defaultTag, {
@@ -21,20 +44,102 @@ const buildTag = (language, defaultType) => {
 
     get(_, type) {
       return (quasis, ...exprs) => {
-        return new TemplateParser(language, quasis.raw, exprs).eval({
-          language: language.name,
-          type,
-        });
+        return getAgASTValue(
+          language,
+          new TemplateParser(language, quasis.raw, exprs).eval({
+            language: language.name,
+            type,
+          }),
+        );
       };
     },
   });
 };
 
-const i = buildTag(instruction, 'Call');
-const spam = buildTag(spamex, 'Matcher');
-const re = buildTag(regex, 'Pattern');
-const str = buildTag(string, 'String');
-const num = buildTag(number, 'Integer');
+const parse = (language, type, sourceText) => {
+  return new TemplateParser(language, [sourceText], []).eval({
+    language: language.name,
+    type,
+  });
+};
+
+const getAgASTValue = (language, miniNode) => {
+  if (!miniNode) return miniNode;
+
+  const { language: languageName, type, attributes } = miniNode;
+  const flags = { escape: false, trivia: false, token: false, intrinsic: false };
+  const properties = {};
+  const children = [];
+  const resolver = new PathResolver(miniNode);
+  const resolvedLanguage =
+    languageName !== language.name ? language.dependencies[languageName] : language;
+
+  if (languageName.startsWith('https://')) {
+    return miniNode; // This node is already processed, possibly because it was interpolated
+  }
+
+  if (!resolvedLanguage) {
+    throw new Error();
+  }
+
+  if (
+    type === 'Punctuator' ||
+    type === 'Keyword' ||
+    type === 'Identifier' ||
+    type === 'StringContent'
+  ) {
+    flags.token = true;
+  }
+
+  if (type === 'Punctuator' || type === 'Keyword') {
+    flags.intrinsic = true;
+  }
+
+  for (const child of miniNode.children) {
+    if (child.type === 'Reference') {
+      const path = child.value;
+      const node = resolver.get(path);
+      set(properties, path, getAgASTValue(resolvedLanguage, node));
+      children.push(child);
+    } else if (child.type === 'Trivia') {
+      children.push({
+        type: 'Embedded',
+        value: {
+          flags: { escape: false, token: true, trivia: true, intrinsic: false },
+          language: 'https://bablr.org/languages/core/space-tab-newline',
+          type: 'Space',
+          children: [buildLiteral(child.value)],
+          properties: {},
+          attributes: {},
+        },
+      });
+    } else if (child.type === 'Escape') {
+      const { cooked, raw } = child.value;
+      const attributes = { cooked };
+
+      children.push({
+        type: 'Embedded',
+        value: {
+          flags: { escape: true, token: true, trivia: false, intrinsic: false },
+          language: cstml.canonicalURL,
+          type: 'Escape',
+          children: [buildLiteral(raw)],
+          properties: {},
+          attributes,
+        },
+      });
+    } else {
+      children.push(child);
+    }
+  }
+
+  return { flags, language: resolvedLanguage.canonicalURL, type, children, properties, attributes };
+};
+
+const str = buildTag(cstml, 'String');
+const num = buildTag(cstml, 'Integer');
 const cst = buildTag(cstml, 'Fragment');
+const re = buildTag(regex, 'Pattern');
+const i = buildTag(instruction, 'Call');
 
-module.exports = { re, spam, str, num, i, cst };
+module.exports = { str, num, cst, re, i, buildTag, set, getAgASTValue, TemplateParser, parse };