desmost 0.9.1 → 0.10.1

package/src/parser/desmost-parser.ts

@@ -2,8 +2,8 @@ import { GenericParser } from "./generic-parser";
 import { Ast } from "./ast";
 
 import type { DesmostOptions } from "../options";
-import { NO_MATCH, UnrecoverableError } from "../errors";
-import type { NoMatch, Unrecoverable } from "../errors";
+import { NO_MATCH, DesmostError } from "../errors";
+import type { NoMatch, Fallible } from "../errors";
 import * as utils from "../utils";
 
 import {
@@ -27,56 +27,79 @@ export class DesmostParser extends GenericParser
 	// == TOP-LEVEL == //
 
 	/**
-	 * Parse the next semantic block of source code, producing an `Ast` object, or `null` if the parser has successfully reached the end of the source code.
+	 * Parse the next semantic block of source code.
+	 * 
+	 * This returns:
+	 * 
+	 * ```ts
+	 * // an AST block(s) + any errors encountered
+	 * { blocks: Ast[], errors: [...] }
+	 * 
+	 * // ignored block + any errors encountered
+	 * { blocks: [], errors: [...] }
+	 * 
+	 * // parser has reached the end of the source
+	 * undefined
+	 * ```
+	 * 
+	 * `block: null` could be ignored content like comments, or it could be a parse that couldn't recover and so failed to produce a block.
 	 */
-	public parse_next(): Ast | null
+	public parse_next():
+		Fallible<{ blocks: Ast[], errors: DesmostError[] } | undefined>
 	{
-		this.consume_spaces();
+		this.errors = [];
+
+		this.eat_whitespace();
 
 		if (this.out_of_bounds()) {
-			return null;
+			return undefined;
 		}
 
+		let blocks = [];
+
 		if (this.current === "%") {
+			// comment
 			if (this.options?.ignore_comments) {
-				// ignore comment
 				this.parse_line();
-				this.consume_end_of_block();
-				return this.parse_next();
 			}
 			else {
-				// comment
-				var expr = this.parse_comment();
+				blocks.push(this.parse_comment());
 			}
 		}
 		else if (this.current === "/") {
 			let r = this.parse_pre_sep();
 
-			// 1 global
 			if ("global" in r) {
-				return r.global;
+				// 1 global
+				if (r.global !== null) {
+					blocks.push(r.global);
+				}
 			}
+			else {
+				// 1+ locals + 1 expr
+				let incantations = r.locals;
 
-			// 1+ locals + 1 expr
-			let incantations = r.local;
+				if (incantations.length > 0) {
+					this.parse_sep();
+				}
 
-			if (incantations.length > 0) {
-				this.parse_sep();
-			}
+				let block = this.parse_post_sep();
 
-			var expr = this.parse_post_sep();
+				for (let invocation of incantations) {
+					block.incantations.push(invocation);
+				}
 
-			for (let invocation of incantations) {
-				expr.incantations.push(invocation);
+				blocks.push(block);
 			}
 		}
 		else {
 			// 1 expr
-			var expr = this.parse_post_sep();
+			blocks.push(this.parse_post_sep());
 		}
 
-		this.consume_end_of_block();
-		return expr;
+		this.eat_end_of_block();
+
+		return { blocks, errors: this.errors };
 	}
 
 	/**
@@ -86,44 +109,82 @@ export class DesmostParser extends GenericParser
 	 * - 1+ local incantations
 	 */
 	parse_pre_sep():
-		Unrecoverable<
-		| { global: Ast.IncantationInvocation<GLOBAL> | Ast.InvalidInvocation }
-		| { local: Array<Ast.IncantationInvocation<LOCAL> | Ast.InvalidInvocation> }
+		Fallible<
+		| { global: Ast.IncantationInvocation<GLOBAL> | null }
+		| { locals: Ast.IncantationInvocation<LOCAL>[] }
 		>
 	{
 		// 1 global incantation
-		let r = this.try_parse_global_incantation();
-
-		if (r !== NO_MATCH) {
-			this.consume_end_of_block(
-				`Received excess input after global incantation /${r.incantation.identifier}`
-			);
+		try {
+			let r = this.try_parse_incantation(GLOBAL_INCANTATIONS);
 
-			return { global: r };
+			if (r !== NO_MATCH) {
+				this.eat_whitespace();
+				return { global: r };
+			}
+		}
+		catch (e) {
+			this.errors.push(e as Error);
+			return { global: null };
 		}
 
 		// 1+ local incantations
 		let incantations = [];
 
 		while (this.current === "/") {
-			let invocation = this.try_parse_local_incantation();
+			try {
+				var invocation = this.try_parse_incantation(LOCAL_INCANTATIONS);
+			}
+			catch (e) {
+				this.errors.push(e as Error);
+				continue;
+			}
+
 			if (invocation === NO_MATCH) break;
 			incantations.push(invocation);
+			this.eat_newlines();
 		}
 
-		return { local: incantations };
+		return { locals: incantations };
 	}
 
 	/**
 	 * Parse the Desmost `::` separator.
 	 */
-	parse_sep(): Unrecoverable<void>
+	parse_sep(): Fallible<void>
 	{
-		this.consume_whitespace();
-		this.consume("::",
-			`Expected \`::\` separator between local incantations and expression, but found: \`${this.preview()}\``
-		);
-		this.consume_whitespace();
+		this.eat_newlines();
+		
+		try {
+			this.eat("::", {
+				msg:  `Expected \`::\` separator between local incantations and expression, but found: \`${this.preview()}\``,
+				hint: `Use \`/incantation :: ${this.preview()}\``,
+			});
+		}
+		catch (e) {
+			if (!this.peek_line().includes("::")) throw e;
+
+			let init = this.i;
+
+			while (this.current !== ":" || this.peek() !== ":") {
+				this.advance();
+			}
+
+			const peek = 7;
+			const start = "...".length + Math.round(1.5 * peek);
+
+			this.errors.push(new DesmostError.ExcessInput({
+				msg: `While consuming \`::\` separator`,
+				show: {
+					text: `...${this.source.slice(init - peek, this.i + peek)}...`,
+					span: new utils.Range(start, start + this.i - init)
+				}
+			}));
+
+			this.eat("::");
+		}
+
+		this.eat_newlines();
 	}
 
 	/**
@@ -132,9 +193,9 @@ export class DesmostParser extends GenericParser
 	 * - 1 line of LaTeX
 	 * - 1 expression incantation
 	 */
-	parse_post_sep(): Unrecoverable<Ast.Expression>
+	parse_post_sep(): Fallible<Ast.Expression>
 	{
-		this.consume_spaces();
+		this.eat_whitespace();
 
 		switch (this.current) {
 			// empty block
@@ -161,7 +222,7 @@ export class DesmostParser extends GenericParser
 		}
 	}
 
-	parse_comment(): Unrecoverable<Ast.Expression>
+	parse_comment(): Fallible<Ast.Expression>
 	{
 		this.advance();
 		let text = this.parse_line();
@@ -179,7 +240,7 @@ export class DesmostParser extends GenericParser
 	/**
 	 * Parse a single line of arbitrary text.
 	 */
-	parse_line(): Unrecoverable<string>
+	parse_line(): Fallible<string>
 	{
 		let init = this.i;
 
@@ -191,64 +252,24 @@ export class DesmostParser extends GenericParser
 	}
 
 	/**
-	 * Attempt to parse a global incantation invocation.
+	 * Attempt to parse an incantation invocation allowed by `incantations`.
 	 */
-	try_parse_global_incantation():
-		| Ast.IncantationInvocation<GLOBAL>
-		| Ast.InvalidInvocation
-		| NoMatch
+	try_parse_incantation<Effect extends Incantation.Effect>(
+		incantations: Incantation<Effect>[],
+	): Fallible<Ast.IncantationInvocation<Effect> | NoMatch>
 	{
 		let init = this.i;
 
-		if (this.try_consume("/") === NO_MATCH) return NO_MATCH;
+		if (this.try_eat("/") === NO_MATCH) return NO_MATCH;
 
-		let incantation = this.try_parse_identifier(GLOBAL_INCANTATIONS);
-		if (incantation === NO_MATCH) {
+		let ident = this.try_eat_identifier();
+		if (ident === NO_MATCH) {
 			this.i = init;
 			return NO_MATCH;
 		}
 
-		let data = undefined;
-
-		if (incantation instanceof ArgIncantation) {
-			if (this.current === "{") {
-				data = this.parse_incantation_arg(incantation.arg_type);
-			}
-			else if (incantation.requires_arg) {
-				return {
-					kind: Ast.Kind.INVALID_INCANTATION,
-					incantation,
-					error: new UnrecoverableError.MissingInput(
-						`No argument provided for /${incantation.identifier}, which requires an argument of type: \`${incantation.arg_type}\``
-					)
-				};
-			}
-		}
-
-		this.consume_spaces();
-
-		// @ts-expect-error: FIXME
-		return {
-			kind: Ast.Kind.INCANTATION_INVOCATION,
-			incantation,
-			arg_raw: data,
-		};
-	}
-
-	/**
-	 * Attempt to parse a local incantation invocation.
-	 */
-	try_parse_local_incantation():
-		| Ast.IncantationInvocation<LOCAL>
-		| Ast.InvalidInvocation
-		| NoMatch
-	{
-		let init = this.i;
-
-		if (this.try_consume("/") === NO_MATCH) return NO_MATCH;
-
-		let incantation = this.try_parse_identifier(LOCAL_INCANTATIONS);
-		if (incantation === NO_MATCH) {
+		let incantation = incantations.find(inc => inc.identifier === ident || inc.alias === ident);
+		if (incantation == undefined) {
 			this.i = init;
 			return NO_MATCH;
 		}
@@ -260,19 +281,13 @@ export class DesmostParser extends GenericParser
 				arg_raw = this.parse_incantation_arg(incantation.arg_type);
 			}
 			else if (incantation.requires_arg) {
-				return {
-					kind: Ast.Kind.INVALID_INCANTATION,
-					incantation,
-					error: new UnrecoverableError.MissingInput(
-						`No argument provided for /${incantation.identifier}, which requires an argument of type: \`${incantation.arg_type}\``
-					)
-				};
+				throw new DesmostError.MissingInput({
+					msg:  `No argument provided for /${incantation.identifier}`,
+					hint: `/${incantation.identifier} requires an argument of type: \`${incantation.arg_type}\``,
+				});
 			}
 		}
 
-		this.consume_whitespace();
-
-		// @ts-expect-error: FIXME
 		return {
 			kind: Ast.Kind.INCANTATION_INVOCATION,
 			incantation,
@@ -283,19 +298,25 @@ export class DesmostParser extends GenericParser
 	/**
 	 * Attempt to parse an expression incantation invocation.
 	 */
-	try_parse_expr_incantation(): Unrecoverable<Ast.Expression | NoMatch>
+	try_parse_expr_incantation(): Fallible<Ast.Expression | NoMatch>
 	{
 		let init = this.i;
 
-		if (this.try_consume("/") === NO_MATCH) return NO_MATCH;
+		if (this.try_eat("/") === NO_MATCH) return NO_MATCH;
+
+		let ident = this.try_eat_identifier();
+		if (ident === NO_MATCH) {
+			this.i = init;
+			return NO_MATCH;
+		}
 
-		let incantation = this.try_parse_identifier(EXPR_INCANTATIONS);
-		if (incantation === NO_MATCH) {
-			// TODO maybe flag to user
+		let incantation = EXPR_INCANTATIONS.find(inc => inc.identifier === ident || inc.alias === ident);
+		if (incantation == undefined) {
 			this.i = init;
 			return NO_MATCH;
 		}
 
+		/* NOTE: All expression incantations currently require arguments, and all for the foreseeable future will too */
 		let arg_raw = this.parse_incantation_arg((incantation as ArgIncantation<EXPR>).arg_type);
 
 		let data = {};
@@ -308,27 +329,6 @@ export class DesmostParser extends GenericParser
 		};
 	}
 
-	/**
-	 * Attempt to parse an incantation identifier.
-	 */
-	try_parse_identifier<Effect extends Incantation.Effect>(
-		incantations: Incantation<Effect>[]
-	): Incantation<Effect> | NoMatch
-	{
-		for (let incantation of incantations) {
-			// yeah the duplication here is a little meh, unfortunately needing `return` means we can't extract it into a helper
-			let r = this.try_consume(incantation.identifier);
-			if (r !== NO_MATCH) return incantation;
-
-			if (incantation.alias != undefined) {
-				let r = this.try_consume(incantation.alias);
-				if (r !== NO_MATCH) return incantation;
-			}
-		}
-
-		return NO_MATCH;
-	}
-
 	/**
 	 * Parse an argument to an incantation, enclosed in `{}`.
 	 * 
@@ -394,13 +394,13 @@ export class DesmostParser extends GenericParser
 		 * - JavaScript object: The above, plus balanced string quotes.
 		 */
 		arg_type: Incantation.ArgType,
-	): Unrecoverable<string>
+	): Fallible<string>
 	{
 		let init = this.i;
 
-		this.consume("{",
-			`Expected \`{\` to start incantation argument, but found: \`${this.preview()}\``
-		);
+		this.eat("{", {
+			msg: `Expected \`{\` to start incantation argument, but found: \`${this.preview()}\``
+		});
 
 		/** The context stack to keep track of when the closing `}` is encountered. */
 		let stack = new ContextStack();
@@ -412,9 +412,7 @@ export class DesmostParser extends GenericParser
 			// If the last character was an escape, we don't care at all what the next character is!
 			// This handles \\ double escape perfectly fine, since the first negates the second
 			if (stack.try_pop(Ctx.ESCAPE)) {
-				this.advance(
-					`Unexpected end of input while parsing incantation argument, stack: ${stack.debug()}`
-				);
+				this.advance({ msg: `While parsing incantation argument`, debug: stack.debug() });
 			}
 			
 			let top = stack.top;
@@ -433,7 +431,7 @@ export class DesmostParser extends GenericParser
 					stack.push(Ctx.ENUM);
 					indices_to_insert_quotes.push(this.i);
 				}
-				else if (top === Ctx.ENUM && this.current?.match(/[^a-zA-Z]/)) {
+				else if (top === Ctx.ENUM && this.current?.match(/[^a-zA-Z-_]/)) {
 					stack.pop();
 					indices_to_insert_quotes.push(this.i);
 
@@ -462,9 +460,7 @@ export class DesmostParser extends GenericParser
 				}
 			}
 
-			this.advance(
-				`Unexpected end of input while parsing incantation argument, stack: ${JSON.stringify(stack)}`
-			);
+			this.advance({ msg: `While parsing incantation argument`, debug: stack.debug() });
 		}
 
 		/* NOTE: Cut in by 1 on both sides to exclude {} braces */
@@ -529,7 +525,7 @@ class ContextStack
 		}
 	}
 
-	/** Backtrack until `ctx` is popped. */
+	/** If `ctx` is in the context stack, backtrack until it is popped, returning `true` if so. */
 	force_pop(ctx: Ctx, options?: StackOperationOptions): boolean
 	{
 		if (this.#should_skip(options)) return false;

package/src/parser/generic-parser.ts

@@ -1,27 +1,30 @@
-import { NO_MATCH, UnrecoverableError } from "../errors";
-import type { NoMatch, Unrecoverable } from "../errors";
+import { NO_MATCH, DesmostError } from "../errors";
+import type { NoMatch, Fallible } from "../errors";
 
 
 const IGNORED_CHARACTERS = new Set([
-	"\r",
+	'\r',
 ]);
 
 
 /**
  * A stateful lazy parser, providing core (non-Desmost-specific) methods for parsing.
  * 
- * Many methods come in `<verb>` and `try_<verb>` pairs. On parsing failure, the former throws an `UnrecoverableError`, but the latter instead backtracks and only returns a `NoMatch`. The former is for expected parses, while the latter is for speculative parsing.
+ * Many methods come in `<verb>` and `try_<verb>` pairs. On parsing failure, the former throws an `DesmostError`, but the latter instead backtracks and only returns a `NoMatch`. The former is for expected parses, while the latter is for speculative parsing.
  */
 export class GenericParser
 {
-	/** The source code this parser is parsing. */
-	protected source: string;
+	/** The source code to parse. */
+	protected readonly source: string
 
 	/** The current position in the source code the parser is pointing to. */
-	protected i: number = 0;
+	protected i: number = 0
 
 	/** The number of characters in the source code. */
-	protected length: number;
+	protected readonly length: number
+
+	/** Accumulated errors in the current block being parsed. */
+	errors: DesmostError[]
 
 
 	/** Create a parser for parsing `source`. */
@@ -29,6 +32,7 @@ export class GenericParser
 	{
 		this.source = source;
 		this.length = source.length;
+		this.errors = [];
 	}
 
 
@@ -48,8 +52,21 @@ export class GenericParser
 		return this.source.at(this.i);
 	}
 
+	protected peek(): string | undefined
+	{
+		return this.source.at(this.i + 1);
+	}
+
+	/**
+	 * Peek the rest of the content in the current line.
+	 */
+	protected peek_line(): string
+	{
+		return this.source.slice(this.i, this.source.indexOf("\n", this.i));
+	}
+
 	/**
-	 * Peek a snippet of the upcoming source text.
+	 * Peek a snippet of the upcoming source text for display purposes.
 	 * 
 	 * Optionally start from the given `idx`.
 	 */
@@ -66,14 +83,13 @@ export class GenericParser
 	/**
 	 * Advance to the next character, skipping ignored characters.
 	 * 
-	 * Errors if the parser is the out-of-bounds *before* advancing.
+	 * Errors if the parser is already out-of-bounds *before* advancing.
 	 */
-	protected advance(error_msg?: string): Unrecoverable<void>
+	protected advance(error_data?: DesmostError.Data): Fallible<void>
 	{
 		if (this.out_of_bounds()) {
-			throw new UnrecoverableError.UnexpectedEnd(error_msg ?? "Unexpected end of input");
+			throw new DesmostError.UnexpectedEnd(error_data ?? {});
 		}
-
 		this.#advance();
 	}
 
@@ -84,7 +100,9 @@ export class GenericParser
 	 */
 	protected try_advance(): void | NoMatch
 	{
-		if (this.out_of_bounds()) return NO_MATCH;
+		if (this.out_of_bounds()) {
+			return NO_MATCH;
+		}
 		this.#advance();
 	}
 
@@ -93,50 +111,57 @@ export class GenericParser
 		this.i++;
 
 		// @ts-expect-error: `this.current == undefined` is a true negative
-		if (IGNORED_CHARACTERS.has(this.current)) {
-			this.advance();
+		while (IGNORED_CHARACTERS.has(this.current)) {
+			this.i++;
 		}
 	}
 
 	/**
 	 * Consume `raw`, erroring if `raw` was not found.
 	 */
-	protected consume(raw: string, error_msg?: string): Unrecoverable<void>
+	protected eat(raw: string, error_data?: DesmostError.Data): Fallible<void>
 	{
 		if (this.out_of_bounds()) {
-			throw new UnrecoverableError.UnexpectedEnd(
-				`Unexpected end of input while trying to consume: \`${raw}\``
-			);
+			throw new DesmostError.UnexpectedEnd({
+				msg: `While trying to consume: \`${raw}\``,
+				show: {
+					text: this.preview(this.i - 5),
+				}
+			});
 		}
 
 		let init = this.i;
 		let ii = 0;
 
 		while (this.current === raw[ii]) {
-			this.advance(
-				`Unexpected end of input while trying to consume: \`${raw}\`, at: \`${this.preview(init)}\``
-			);
+			this.advance({
+				msg: `While trying to consume: \`${raw}\``,
+				show: {
+					text: `\`${this.preview(init)}\``,
+				},
+			});
 
 			ii++;
 			if (ii === raw.length) return;
 		}
 
-		throw new UnrecoverableError.UnexpectedInput(
-			error_msg ?? `Expected: \`${raw}\`, but found: \`${this.preview(init)}\``
-		);
+		throw new DesmostError.UnexpectedInput({
+			msg: `Expected: \`${raw}\`, but found: \`${this.preview(init)}\``,
+			...error_data,
+		});
 	}
 
 	/**
 	 * Attempt to consume `raw`, backtracking if `raw` was not found.
 	 */
-	protected try_consume(raw: string): void | NoMatch
+	protected try_eat(raw: string): void | NoMatch
 	{
 		let init = this.i;
 		let ii = 0;
 
 		while (this.current === raw[ii]) {
 			let r = this.try_advance();
-			if (r === NO_MATCH) return NO_MATCH;  // FIXME?
+			if (r === NO_MATCH) break;
 
 			ii++;
 			if (ii === raw.length) return;
@@ -147,15 +172,31 @@ export class GenericParser
 		return NO_MATCH;
 	}
 
+	protected try_eat_identifier(): string | NoMatch
+	{
+		let init = this.i;
+
+		while (/[a-zA-Z-]/.test(this.current ?? "")) {
+			let r = this.try_advance();
+
+			if (r === NO_MATCH) {
+				this.i = init;
+				return NO_MATCH;
+			}
+		}
+
+		return this.source.slice(init, this.i);
+	}
+
 	/**
-	 * Consume 0 or more space characters.
+	 * Consume 0 or more space or tab characters.
 	 */
-	protected consume_spaces(): void
+	protected eat_whitespace(): void
 	{
 		/* NOTE: Callers should be able to assume this is safe to call even when at end of input, since it should just match 0 characters */
 		if (this.out_of_bounds()) return;
 
-		while (this.current === " ") {
+		while (this.current === ' ' || this.current === '\t') {
 			this.advance();
 		}
 	}
@@ -165,28 +206,29 @@ export class GenericParser
 	 * 
 	 * Same as `.consume_spaces()`, but allows tabs and newlines.
 	 */
-	protected consume_whitespace(): void
+	protected eat_newlines(): void
 	{
 		/* NOTE: Callers should be able to assume this is safe to call even when at end of input, since it should just match 0 characters */
 		if (this.i >= this.length) return;
 
 		while (
-			this.current === " "
-			|| this.current === "\n"
-			|| this.current === "\t"
+			this.current === ' '
+			|| this.current === '\n'
+			|| this.current === '\t'
 		) {
 			this.advance();
 		}
 	}
 
-	protected consume_end_of_block(error_msg?: string): Unrecoverable<void>
+	protected eat_end_of_block(error_data?: DesmostError.Data): Fallible<void>
 	{
 		if (this.i >= this.length) return;
 
-		this.consume_spaces();
+		this.eat_whitespace();
 
-		this.consume("\n",
-			error_msg ?? `Expected end of block, but found: \`${this.preview()}\``
-		);
+		this.eat("\n", {
+			msg: `Expected a newline at the end of a block, but found: \`${this.preview()}\``,
+			...error_data
+		});
 	}
 }