desmost 0.1.0

package/src/parser/desmost-parser.ts

@@ -0,0 +1,486 @@
+import { GenericParser } from "./generic-parser";
+import { Ast } from "./ast";
+
+import { RecoverableFail, UnrecoverableError } from "../errors";
+import type { Recoverable, Unrecoverable, MaybeRecoverable } from "../errors";
+
+import {
+  Incantation, ArgIncantation,
+  GLOBAL_INCANTATIONS, LOCAL_INCANTATIONS, EXPR_INCANTATIONS
+} from "../magic";
+import type { GLOBAL, LOCAL, EXPR } from "../magic";
+
+import type { DesmostOptions } from "../compiler";
+
+
+/**
+ * A stateful lazy parser for Desmost.
+ */
+export class DesmostParser extends GenericParser
+{
+  constructor(source: string, options?: DesmostOptions)
+  {
+    super(options?.ignore_trailing_blanks ? source : (source + "\n"));
+  }
+
+
+  // == TOP-LEVEL == //
+
+  /**
+   * Parse the next semantic block of source code, producing an `Ast` object, or `null` if the parser has successfully reached the end of the source code.
+   */
+  public parse_next(): Ast | null
+  {
+    this.consume_spaces();
+
+    if (this.out_of_bounds()) {
+      return null;
+    }
+
+    if (this.current === "/") {
+      let r = this.parse_pre_sep();
+
+      if ("global" in r) {
+        // 1 global
+        return r.global;
+      }
+
+      // 1+ locals + 1 expr
+      let incantations = r.local;
+
+      var expr = this.parse_post_sep();
+
+      for (let invocation of incantations) {
+        expr.incantations.push(invocation);
+      }
+    }
+    else {
+      // 1 expr
+      var expr = this.parse_post_sep();
+    }
+
+    this.consume_end_of_block();
+    return expr;
+  }
+
+  /**
+   * Parse Desmost syntax before the `::` separator, which may be:
+   * 
+   * - 1 global incantation
+   * - 1+ local incantations
+   */
+  parse_pre_sep():
+    Unrecoverable<
+      | { global: Ast.IncantationInvocation<GLOBAL>
+                | Ast.InvalidInvocation;
+      }
+      | { local: Array<Ast.IncantationInvocation<LOCAL>
+                      | Ast.InvalidInvocation>;
+      }
+    >
+  {
+    // 1 global incantation
+    try {
+      let r = this.try_parse_global_incantation();
+
+      this.consume_end_of_block(
+        `Received excess input after global incantation /${r.incantation.identifier}`
+      );
+
+      return { global: r };
+    }
+    catch (e) {
+      if (!(e instanceof RecoverableFail)) throw e;
+    }
+
+    let incantations = [];
+
+    // 1+ local incantations
+    while (this.current === "/") {
+      try {
+        incantations.push(this.try_parse_local_incantation());
+      }
+      catch (e) {
+        if (!(e instanceof RecoverableFail)) throw e;
+        break;
+      }
+    }
+
+    if (incantations.length > 0) {
+      this.parse_sep();
+    }
+
+    return { local: incantations };
+  }
+
+  /**
+   * Parse the Desmost `::` separator.
+   */
+  parse_sep(): Unrecoverable<void>
+  {
+    this.consume_whitespace();
+    this.consume("::",
+      `Expected \`::\` separator between local incantations and expression, but found: \`${this.preview()}\``
+    );
+  }
+
+  /**
+   * Parse Desmost syntax after the `::` separator, which may be:
+   * 
+   * - 1 line of LaTeX
+   * - 1 expression incantation
+   */
+  parse_post_sep(): Unrecoverable<Ast.Expression>
+  {
+    this.consume_spaces();
+
+    switch (this.current) {
+      // empty block
+      case "\n":
+        return {
+          kind: Ast.Kind.EXPRESSION,
+          data: { latex: ` ` },
+          incantations: [],
+        };
+
+      // expr incantation
+      case "/":
+        try {
+          return this.try_parse_expr_incantation();
+        }
+        catch (e) {
+          if (!(e instanceof RecoverableFail)) throw e;
+          // plain LaTeX (fallback)
+          return this.parse_latex_line();
+        }
+
+      // plain LaTeX
+      default:
+        return this.parse_latex_line();
+    }
+  }
+
+
+  // == LOW-LEVEL == //
+
+  /**
+   * Parse a single line of LaTeX.
+   */
+  parse_latex_line(): Unrecoverable<Ast.Expression>
+  {
+    let init = this.i;
+
+    while (!this.out_of_bounds() && this.current !== "\n") {
+      this.advance();
+    }
+
+    return {
+      kind: Ast.Kind.EXPRESSION,
+      data: { latex: this.source.slice(init, this.i) },
+      incantations: [],
+    };
+  }
+
+
+  /**
+   * Attempt to parse a global incantation invocation.
+   */
+  try_parse_global_incantation():
+    Recoverable<
+    | Ast.IncantationInvocation<GLOBAL>
+    | Ast.InvalidInvocation
+    >
+  {
+    let init = this.i;
+
+    this.try_consume("/");
+
+    try {
+      var incantation = this.try_parse_identifier(GLOBAL_INCANTATIONS);
+    }
+    catch (e) {
+      if (e instanceof RecoverableFail) {
+        this.i = init;
+      }
+      throw e;
+    }
+
+    let data = undefined;
+
+    if (incantation instanceof ArgIncantation) {
+      if (this.current === "{") {
+        data = this.parse_incantation_arg(incantation.arg_type);
+      }
+      else if (incantation.requires_arg) {
+        return {
+          kind: Ast.Kind.INVALID_INCANTATION,
+          incantation,
+          error: new UnrecoverableError.MissingInput(
+            `No argument provided for /${incantation.identifier}, which requires an argument of type: \`${incantation.arg_type}\``
+          )
+        };
+      }
+    }
+
+    this.consume_spaces();
+
+    return {
+      kind: Ast.Kind.INCANTATION_INVOCATION,
+      incantation,
+      arg_raw: data,
+    };
+  }
+
+  /**
+   * Attempt to parse a local incantation invocation.
+   */
+  try_parse_local_incantation():
+    Recoverable<
+    | Ast.IncantationInvocation<LOCAL>
+    | Ast.InvalidInvocation
+    >
+  {
+    let init = this.i;
+
+    this.try_consume("/");
+
+    try {
+      var incantation = this.try_parse_identifier(LOCAL_INCANTATIONS);
+    }
+    catch (e) {
+      if (e instanceof RecoverableFail) {
+        this.i = init;
+      }
+      throw e;
+    }
+
+    let arg_raw = undefined;
+
+    if (incantation instanceof ArgIncantation) {
+      if (this.current === "{") {
+        arg_raw = this.parse_incantation_arg(incantation.arg_type);
+      }
+      else if (incantation.requires_arg) {
+        return {
+          kind: Ast.Kind.INVALID_INCANTATION,
+          incantation,
+          error: new UnrecoverableError.MissingInput(
+            `No argument provided for /${incantation.identifier}, which requires an argument of type: \`${incantation.arg_type}\``
+          )
+        };
+      }
+    }
+
+    this.consume_whitespace();
+
+    return {
+      kind: Ast.Kind.INCANTATION_INVOCATION,
+      incantation,
+      arg_raw,
+    };
+  }
+
+  try_parse_expr_incantation(): MaybeRecoverable<Ast.Expression>
+  {
+    // TODO refactor with `backtrack()` helper
+    let init = this.i;
+
+    this.try_consume("/");
+
+    try {
+      var incantation = this.try_parse_identifier(EXPR_INCANTATIONS) as ArgIncantation<EXPR>;
+    }
+    catch (e) {
+      if (e instanceof RecoverableFail) {
+        // TODO maybe flag to user
+        this.i = init;
+      }
+      throw e;
+    }
+
+    let arg_raw = this.parse_incantation_arg(incantation.arg_type);
+
+    let data = {};
+    incantation.apply(data, arg_raw);
+
+    return {
+      kind: Ast.Kind.EXPRESSION,
+      data,
+      incantations: [],
+    };
+  }
+
+  /**
+   * Attempt to parse an incantation identifier.
+   */
+  try_parse_identifier<Effect extends Incantation.Effect>(
+    incantations: Incantation<Effect>[]
+  ): Recoverable<Incantation<Effect>>
+  {
+    for (let incantation of incantations) {
+      // yeah the duplication here is a little meh, unfortunately needing `return` means we can't extract it into a helper
+      try {
+        this.try_consume(incantation.identifier);
+        return incantation;
+      }
+      catch (e) {
+        if (!(e instanceof RecoverableFail)) throw e;
+      }
+
+      if (incantation.alias != undefined) {
+        try {
+          this.try_consume(incantation.alias);
+          return incantation;
+        }
+        catch (e) {
+          if (!(e instanceof RecoverableFail)) throw e;
+        }
+      }
+    }
+
+    throw new RecoverableFail();
+  }
+
+  /**
+   * Parse an argument to an incantation, enclosed in `{}`.
+   * 
+   * ```ts
+   * /incantation{ arg }
+   *             ^^^^^^^
+   * ```
+   * 
+   * ## Notes
+   * 
+   * We don't actually care for what type the argument is since we won't be evaluating it, we just want to find the closing `}`.
+   * 
+   * However, the argument itself might be *meant* to contain a `}`. So we need a way to accurately identify the actual closing brace:
+   * 
+   * ```ts
+   * /incantation{ field: { value: 1 } }
+   *                                 ^ ^
+   * ```
+   * 
+   * A well-formed argument always has matching pairs of `{}`, so we'll keep a stack. It starts with the opening `{`. When the stack is depleted, we must've reached the end of the argument.
+   * 
+   * However, there's still more edge cases:
+   * 
+   * ```ts
+   * /label{ text: "This }s weird" }
+   *              ^
+   * ```
+   * 
+   * `{` and `}` can appear in strings, and they won't necessarily be matched, so we'll ignore them by also tracking string contexts.
+   * 
+   * ```ts
+   * /latex{ \{ x, y ) }
+   *          ^
+   * ```
+   * 
+   * `{` and `}` in LaTeX can be escaped with `\{` or `\}`, and these won't necessarily be matched, so we'll ignore them by also tracking backslash escapes.
+   * 
+   * If the user truly mismatches `{}`, then, well ...parsing will fail catastrophically!
+   */
+  parse_incantation_arg(
+    /**
+     * The type of argument to parse.
+     * 
+     * Parsing strategy varies for different types:
+     * 
+     * - String / LaTeX: Track only balanced `{}` and escaped `\{\}`.
+     * - JavaScript object: The above, plus balanced string quotes.
+     */
+    arg_type: Incantation.ArgType,
+  ): Unrecoverable<string>
+  {
+    let init = this.i;
+
+    this.consume("{",
+      `Expected \`{\` to start incantation argument, but found: \`${this.preview()}\``
+    );
+
+    enum Ctx {
+      BLOCK  = Char.L_BRACE,
+      STR_1  = Char.QUOTE_SINGLE,
+      STR_2  = Char.QUOTE_DOUBLE,
+      STR_F  = Char.BACKTICK,
+      ESCAPE = Char.BACKSLASH,
+    }
+
+    let stack: Ctx[] = [Ctx.BLOCK];
+
+    /**
+     * Pop `ctx` if it is the currently active context, returning `true` if successful.
+     * 
+     * If `{ force: true }`, backtrack the stack and report errors for unterminated contexts.
+    */
+    function try_pop(ctx: Ctx, options?: { force: boolean }): boolean
+    {
+      if (stack.at(-1) === ctx) {
+        stack.pop();
+        return true;
+      }
+      else if (options?.force) {
+        if (stack.includes(ctx)) {
+          /* NOTE: We could report errors for unterminated contexts, but we'll leave that for the actual evaluation - in case we get something wrong ;) */
+          while (stack.at(-1) !== ctx) {
+            stack.pop();
+          }
+
+          return true;
+        }
+      }
+
+      return false;
+    }
+
+    while (stack.length > 0) {
+      if (try_pop(Ctx.ESCAPE)) {
+        this.advance(
+          `Unexpected end of input while parsing incantation argument, stack: ${JSON.stringify(stack)}`
+        );
+      }
+      
+      let top = stack.at(-1)!;
+
+      switch (this.current) {
+        case Ctx.ESCAPE: stack.push(Ctx.ESCAPE); break;
+
+        /* NOTE: *Currently* `{}` should be ignored in all contexts except `Ctx.BLOCK`. This might change if more contexts are added in future! */
+        case Char.L_BRACE:
+          if (top !== Ctx.BLOCK) break;
+          stack.push(Ctx.BLOCK); break;
+
+        case Char.R_BRACE:
+          if (top !== Ctx.BLOCK) break;
+          try_pop(Ctx.BLOCK, { force: true }); break;
+      }
+
+      if (arg_type === Incantation.ArgType.OBJECT) {
+        switch (this.current) {
+          case Ctx.STR_1: try_pop(Ctx.STR_1) || stack.push(Ctx.STR_1); break;
+          case Ctx.STR_2: try_pop(Ctx.STR_2) || stack.push(Ctx.STR_2); break;
+          case Ctx.STR_F: try_pop(Ctx.STR_F) || stack.push(Ctx.STR_F); break;
+        }
+      }
+
+      this.advance(
+        `Unexpected end of input while parsing incantation argument, stack: ${JSON.stringify(stack)}`
+      );
+    }
+
+    /* NOTE: Cut in by 1 on both sides to exclude {} braces */
+    return this.source.slice(init + 1, this.i - 1).trim();
+  }
+
+}
+
+
+enum Char
+{
+  L_BRACE      = "{",
+  R_BRACE      = "}",
+  QUOTE_SINGLE = `'`,
+  QUOTE_DOUBLE = `"`,
+  BACKTICK     = "`",
+  BACKSLASH    = "\\",
+}

package/src/parser/generic-parser.ts

@@ -0,0 +1,191 @@
+import { RecoverableFail, UnrecoverableError } from "../errors";
+import type { Recoverable, Unrecoverable } from "../errors";
+
+
+const IGNORED_CHARACTERS = new Set([
+  "\r",
+]);
+
+
+/**
+ * A stateful lazy parser, providing core methods for parsing.
+ * 
+ * Many methods come in `<verb>` and `try_<verb>` pairs. On parsing failure, the former throws an `UnrecoverableError`, but the latter instead backtracks and only throws a `RecoverableFail`.
+ */
+export class GenericParser
+{
+  /** The source code this parser is parsing. */
+  protected source: string;
+
+  /** The current position in the source code the parser is pointing to. */
+  protected i: number = 0;
+
+  /** The number of characters in the source code. */
+  protected length: number;
+
+
+  /** Create a parser for parsing `source`. */
+  constructor(source: string)
+  {
+    this.source = source;
+    this.length = source.length;
+  }
+
+
+  /**
+   * Is the parser currently pointing at an invalid character?
+   */
+  protected out_of_bounds(): boolean
+  {
+    return this.i >= this.length;
+  }
+
+  /**
+   * Get the character the parser is currently pointing at, or `undefined` if the parser is out-of-bounds.
+   */
+  protected get current(): string | undefined
+  {
+    return this.source.at(this.i);
+  }
+
+  /**
+   * Get the character *after* the current one that the parser points at.
+   */
+  protected next(): string | undefined
+  {
+    return this.source.at(this.i + 1);
+  }
+
+  /**
+   * Peek a snippet of the upcoming source text (for error messages).
+   */
+  public preview(): string
+  {
+    return this.source.slice(this.i, this.i + 20) + "...";
+  }
+
+  /**
+   * Advance to the next character, skipping ignored characters.
+   * 
+   * Errors if the parser is the out-of-bounds *before* advancing.
+   */
+  protected advance(error_msg?: string): Unrecoverable<void>
+  {
+    if (this.out_of_bounds()) {
+      throw new UnrecoverableError.UnexpectedEnd(error_msg ?? "Unexpected end of input");
+    }
+
+    this.i++;
+
+    // @ts-expect-error: `this.current == undefined` is a true negative
+    if (IGNORED_CHARACTERS.has(this.current)) {
+      this.advance();
+    }
+  }
+
+  /**
+   * Attempt to advance to the next character, skipping ignored characters.
+   */
+  protected try_advance(): Recoverable<void>
+  {
+    try {
+      this.advance();
+    }
+    catch {
+      throw new RecoverableFail();
+    }
+  }
+
+  /**
+   * Consume `raw`, erroring if `raw` was not found.
+   */
+  protected consume(raw: string, error_msg?: string): Unrecoverable<void>
+  {
+    if (this.out_of_bounds()) {
+      throw new UnrecoverableError.UnexpectedEnd(
+        `Unexpected end of input while trying to consume: \`${raw}\``
+      );
+    }
+
+    let ii = 0;
+
+    while (this.current === raw[ii]) {
+      this.advance(
+        `Unexpected end of input while trying to consume: \`${raw}\``
+      );
+
+      ii++;
+      if (ii === raw.length) return;
+    }
+
+    throw new UnrecoverableError.UnexpectedInput(
+      error_msg ?? `Expected: \`${raw}\`, but found: \`${this.preview()}\``
+    );
+  }
+
+  /**
+   * Attempt to consume `raw`, backtracking and throwing if `raw` was not found.
+   */
+  protected try_consume(raw: string): Recoverable<void>
+  {
+    let init = this.i;
+    let ii = 0;
+
+    while (this.current === raw[ii]) {
+      this.try_advance();
+      ii++;
+
+      if (ii === raw.length) return;
+      if (this.i === this.length) break;
+    }
+
+    this.i = init;
+    throw new RecoverableFail();
+  }
+
+  /**
+   * Consume 0 or more space characters.
+   */
+  protected consume_spaces(): void
+  {
+    /* NOTE: Callers should be able to assume this is safe to call even when at end of input, since it should just match 0 characters */
+    if (this.out_of_bounds()) return;
+
+    while (this.current === " ") {
+      this.advance();
+    }
+  }
+
+  /**
+   * Consume 0 or more whitespace characters.
+   * 
+   * Same as `.consume_spaces()`, but allows newlines.
+   */
+  protected consume_whitespace(): void
+  {
+    /* NOTE: Callers should be able to assume this is safe to call even when at end of input, since it should just match 0 characters */
+    if (this.i >= this.length) return;
+
+    while (this.current === " " || this.current === "\n") {
+      this.advance();
+    }
+  }
+
+  protected consume_end_of_block(error_msg?: string): Unrecoverable<void>
+  {
+    if (this.i >= this.length) return;
+
+    this.consume_spaces();
+
+    try {
+      this.try_consume("\n");
+    }
+    catch (e) {
+      if (!(e instanceof RecoverableFail)) throw e;
+
+      throw new UnrecoverableError.ExcessInput(
+        error_msg ?? `Expected end of block, but found: \`${this.preview()}\``
+      );
+    }
+  }
+}

package/src/parser/index.ts

@@ -0,0 +1,8 @@
+/**
+ * Implements `DesmostParser`, the Desmost parsing engine.
+ * 
+ * The parsing engine makes heavy use of exceptions for backtracking, which is *very unperformant*, but makes for *much more readable code*. Desmost isn't expecting to parse huge documents of source code, so I decided this is an acceptable trade-off.
+ */
+
+export { DesmostParser } from "./desmost-parser";
+export { Ast } from "./ast";