@mattwca/little-parser-lib 1.0.0

package/README.md

@@ -0,0 +1,311 @@
+# @mattwca/little-parser-lib
+
+A lightweight, flexible TypeScript library for building parsers using parser combinators. Create powerful parsers by combining simple, reusable parsing functions.
+
+## Features
+
+- 🚀 **Parser Combinators**: Build complex parsers from simple building blocks
+- 🔍 **Built-in Tokenizer**: Flexible tokenization with regex and string matching
+- 📝 **TypeScript First**: Full type safety and IntelliSense support
+- 🎯 **Backtracking Support**: Automatic position restoration on parse failures
+- 📦 **Zero Dependencies**: Lightweight with no external runtime dependencies
+- ✨ Packaged with [tsdown](https://tsdown.dev)
+
+## Installation
+
+```bash
+npm install @mattwca/little-parser-lib
+```
+
+## Quick Start
+
+```typescript
+import { Tokenizer, TokenStream, anyOf, and, many, runParser } from '@mattwca/little-parser-lib';
+
+// 1. Define your tokenizer
+const tokenizer = new Tokenizer()
+  .withTokenType('letter', /[a-zA-Z]/)
+  .withTokenType('digit', /[0-9]/)
+  .withTokenType('whitespace', /\s/);
+
+// 2. Tokenize your input
+const tokens = tokenizer.tokenize('hello123');
+const stream = new TokenStream(tokens);
+
+// 3. Create a parser using combinators
+const parser = and(
+  many(anyOf('letter')),
+  many(anyOf('digit'))
+);
+
+// 4. Run the parser
+const result = runParser(parser, stream);
+console.log(result); // { result: [[...letters], [...digits]] }
+```
+
+## Core Concepts
+
+### Tokenizer
+
+The `Tokenizer` class converts raw input strings into tokens. Each token has a type, value, and position.
+
+```typescript
+const tokenizer = new Tokenizer()
+  .withTokenType('number', /[0-9]/)
+  .withTokenType('operator', /[+\-*/]/)
+  .withTokenType('whitespace', /\s/);
+
+const tokens = tokenizer.tokenize('1 + 2');
+// [
+//   { type: 'number', value: '1', position: { line: 1, column: 1 } },
+//   { type: 'whitespace', value: ' ', position: { line: 1, column: 2 } },
+//   { type: 'operator', value: '+', position: { line: 1, column: 3 } },
+//   ...
+// ]
+```
+
+### Parser Functions
+
+A parser function (`ParseFn<T>`) takes a `TokenStream` and returns a `ParserResult<T>`, which can be either:
+- `SuccessfulParserResult<T>`: Contains the parsed result
+- `FailedParserResult`: Contains error message and position
+
+## Parser Combinators
+
+### `and(...parsers)`
+
+Combines multiple parsers in sequence. All parsers must succeed.
+
+```typescript
+const parser = and(
+  anyOf('keyword'),
+  anyOf('identifier'),
+  anyOf('semicolon')
+);
+```
+
+### `or(...parsers)`
+
+Tries parsers in order, returns the first successful result. If all fail, returns the deepest error.
+
+```typescript
+const parser = or(
+  anyOf('keyword'),
+  anyOf('identifier'),
+  anyOf('operator')
+);
+```
+
+### `many(parser)`
+
+Applies a parser repeatedly until it fails (requires at least one success).
+
+```typescript
+const parser = many(anyOf('digit')); // Parses one or more digits
+```
+
+### `optional(parser, shouldBacktrack?)`
+
+Makes a parser optional. Returns `null` if it fails.
+
+```typescript
+const parser = optional(anyOf('sign')); // Sign is optional
+```
+
+### `attempt(parser)`
+
+Wraps a parser with automatic backtracking on failure.
+
+```typescript
+const parser = attempt(
+  and(anyOf('keyword'), anyOf('identifier'))
+);
+```
+
+### `map(parser, mapFn)`
+
+Transforms the result of a parser using a mapping function.
+
+```typescript
+const digitParser = anyOf('digit');
+const numberParser = map(
+  many(digitParser),
+  (tokens) => parseInt(tokens.map(t => t.value).join(''))
+);
+```
+
+### `label(label, parser)`
+
+Adds a custom label to parser errors for better debugging.
+
+```typescript
+const parser = label(
+  'function declaration',
+  and(anyOf('function'), anyOf('identifier'))
+);
+```
+
+## Built-in Parsers
+
+### `anyOf(...types)`
+
+Parses any token matching the specified type(s).
+
+```typescript
+const parser = anyOf('letter', 'digit', 'underscore');
+```
+
+### `anyExcept(...types)`
+
+Parses any token NOT matching the specified type(s).
+
+```typescript
+const parser = anyExcept('whitespace', 'newline');
+```
+
+### `endOfInput()`
+
+Ensures the end of input has been reached.
+
+```typescript
+const parser = and(
+  myMainParser,
+  endOfInput() // Ensure nothing left to parse
+);
+```
+
+## Running Parsers
+
+### `runParser(parser, tokenStream)`
+
+Runs a parser on a token stream. Throws `ParsingError` on failure.
+
+```typescript
+try {
+  const result = runParser(myParser, tokenStream);
+  console.log(result.result);
+} catch (error) {
+  if (error instanceof ParsingError) {
+    console.error(`Parse error at ${error.position.line}:${error.position.column}`);
+  }
+}
+```
+
+### `runParserOnString(parser, input, tokenizer)`
+
+Convenience method to tokenize and parse in one step.
+
+```typescript
+const result = runParserOnString(myParser, 'input string', tokenizer);
+```
+
+## Example: Simple Expression Parser
+
+```typescript
+import { 
+  Tokenizer, 
+  TokenStream, 
+  anyOf, 
+  and, 
+  or, 
+  many, 
+  map, 
+  runParserOnString 
+} from '@mattwca/little-parser-lib';
+
+// Define tokenizer
+const tokenizer = new Tokenizer()
+  .withTokenType('digit', /[0-9]/)
+  .withTokenType('plus', '+')
+  .withTokenType('minus', '-')
+  .withTokenType('whitespace', /\s/);
+
+// Define parsers
+const digit = anyOf('digit');
+const number = map(
+  many(digit),
+  (tokens) => parseInt(tokens.map(t => t.value).join(''))
+);
+
+const operator = or(
+  anyOf('plus'),
+  anyOf('minus')
+);
+
+const expression = and(
+  number,
+  optional(anyOf('whitespace')),
+  operator,
+  optional(anyOf('whitespace')),
+  number
+);
+
+// Parse
+const result = runParserOnString(expression, '10 + 5', tokenizer);
+console.log(result.result); // [10, null, {...}, null, 5]
+```
+
+## Error Handling
+
+The library provides detailed error messages with position information:
+
+```typescript
+try {
+  const result = runParser(myParser, stream);
+} catch (error) {
+  if (error instanceof ParsingError) {
+    console.error(`
+      Error: ${error.message}
+      Line: ${error.position.line}
+      Column: ${error.position.column}
+      Position: ${error.position.position}
+    `);
+  }
+}
+```
+
+## API Reference
+
+### Classes
+
+- `Tokenizer`: Converts input strings into tokens
+- `TokenStream`: Manages token consumption and backtracking
+- `ParsingError`: Error thrown when parsing fails
+
+### Types
+
+- `Token`: Represents a single token with type, value, and position
+- `TokenType`: String identifier for token types
+- `ParseFn<T>`: Function that takes a TokenStream and returns ParserResult<T>
+- `ParserResult<T>`: Union of SuccessfulParserResult<T> and FailedParserResult
+
+### Combinators
+
+- `and(...parsers)`: Sequential combination
+- `or(...parsers)`: Alternative combination
+- `many(parser)`: One or more repetitions
+- `optional(parser)`: Optional parser
+- `attempt(parser)`: Parser with backtracking
+- `map(parser, fn)`: Transform parser result
+- `label(label, parser)`: Add error label
+
+### Parsers
+
+- `anyOf(...types)`: Match any of specified token types
+- `anyExcept(...types)`: Match any token except specified types
+- `endOfInput()`: Match end of input
+
+### Utilities
+
+- `runParser(parser, stream)`: Execute parser on token stream
+- `runParserOnString(parser, input, tokenizer)`: Execute parser on string
+- `isSuccessfulResult(result)`: Type guard for successful results
+- `isFailedResult(result)`: Type guard for failed results
+
+## License
+
+MIT
+
+## Author
+
+@mattwca

package/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "@mattwca/little-parser-lib",
+  "version": "1.0.0",
+  "description": "",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.mjs",
+      "require": "./dist/index.cjs"
+    }
+  },
+  "files": [
+    "dist",
+    "src",
+    "README.md"
+  ],
+  "scripts": {
+    "clean": "rm -rf dist",
+    "build": "npm run clean && tsdown",
+    "build:dev": "npm run clean && tsdown --dev"
+  },
+  "keywords": [
+    "parser",
+    "combinators",
+    "parser-combinators",
+    "parsing"
+  ],
+  "author": "@mattwca",
+  "license": "MIT",
+  "devDependencies": {
+    "@types/jest": "^30.0.0",
+    "@types/node": "^25.0.3",
+    "jest": "^30.2.0",
+    "ts-jest": "^29.4.6",
+    "tsdown": "^0.18.3",
+    "typescript": "^5.9.3"
+  },
+  "engines": {
+    "node": ">=14"
+  },
+  "repository": {
+    "url": "https://github.com/mattwca/little-parser-lib"
+  }
+}

package/src/index.ts

@@ -0,0 +1,2 @@
+export * from './tokenizer';
+export * from './parser';

package/src/parser/ParsingError.ts

@@ -0,0 +1,14 @@
+import { ParsingErrorPosition } from "./types";
+
+/**
+ * Represents a parsing error with a specific message.
+ */
+export class ParsingError extends Error {
+  public location: ParsingErrorPosition;
+
+  constructor(message: string, location: ParsingErrorPosition) {
+    super(`Parsing Error [${location.line}:${location.column}]: ${message}`);
+
+    this.location = location;
+  }
+}

package/src/parser/index.ts

@@ -0,0 +1,3 @@
+export * from './parser';
+export * from './ParsingError';
+export * from './types';

package/src/parser/parser.ts

@@ -0,0 +1,255 @@
+import { Tokenizer, TokenStream } from "../tokenizer";
+import { Token, TokenType } from "../tokenizer/types";
+import { ParsingError } from "./ParsingError";
+import { FailedParserResult, isFailedResult, isSuccessfulResult, ParseFn, ParserResult, SuccessfulParserResult } from "./types";
+
+/**
+ * Combines multiple parsers in sequence, returning an array of their results.
+ * If one of the parsers fails, the entire sequence fails.
+ */
+export function and(...parsers: ParseFn<any>[]): ParseFn<any[]> {
+  return (tokenStream: TokenStream) => {
+    const results: any[] = [];
+
+    for (const parser of parsers) {
+      const parseResult = parser(tokenStream);
+
+      if (isFailedResult(parseResult)) {
+        return parseResult;
+      }
+
+      results.push(parseResult.result);
+    }
+
+    return { result: results };
+  };
+}
+
+/**
+ * Attempts to run a parser, restoring the token position (backtracking) if it fails.
+ */
+export function attempt<T>(parser: ParseFn<T>): ParseFn<T> {
+  return (tokenStream: TokenStream) => {
+    tokenStream.storePosition();
+
+    const result = parser(tokenStream);
+
+    if (isSuccessfulResult(result)) {
+      tokenStream.clearPosition();
+    } else {
+      tokenStream.restorePosition();
+    }
+
+    return result;
+  };
+}
+
+/**
+ * Makes a given parser optional, returns `null` if it fails.
+ */
+export function optional<T>(parser: ParseFn<T>, shouldBacktrack: boolean = true): ParseFn<T | null> {
+  return (tokenStream: TokenStream) => {
+    const parseFn = shouldBacktrack ? attempt(parser) : parser;
+    const result = parseFn(tokenStream);
+
+    if (isSuccessfulResult(result)) {
+      return result;
+    }
+
+    return { result: null };
+  };
+}
+
+/**
+ * Tries multiple parsers in order, returning the result of the first successful parse.
+ * If all parsers fail, returns the error from the parser that got the furthest.
+ */
+export function or<T>(...parsers: ParseFn<T>[]): ParseFn<T> {
+  return (tokenStream: TokenStream) => {
+    let deepestError = null;
+    let deepestErrorPosition = -1;
+
+    for (const parser of parsers) {
+      const tryParse = attempt(parser);
+      const result = tryParse(tokenStream);
+
+      if (isSuccessfulResult(result)) {
+        return result as SuccessfulParserResult<T>;
+      }
+
+      if (isFailedResult(result)) {
+        const { position } = result;
+
+        if (position.position > deepestErrorPosition) {
+          deepestError = result;
+          deepestErrorPosition = position.position;
+        }
+      }
+    }
+
+    return deepestError as FailedParserResult;
+  };
+}
+
+/**
+ * Applies a parser repeatedly until it fails, collecting all successful results into an array.
+ * If the parser fails on the first attempt, returns a failure.
+ */
+export function many(parser: ParseFn<any>): ParseFn<any> {
+  return (tokenStream: TokenStream) => {
+    const results: any[] = [];
+
+    let parseFailure = null;
+
+    while (true) {
+      tokenStream.storePosition();
+
+      const result = parser(tokenStream);
+      if (isSuccessfulResult(result)) {
+        results.push(result.result);
+        tokenStream.clearPosition();
+      } else {
+        parseFailure = result;
+        tokenStream.restorePosition();
+        break;
+      }
+    }
+
+    if (parseFailure && results.length === 0) {
+      return parseFailure;
+    }
+
+    return { result: results };
+  };
+}
+
+/**
+ * Labels a parser with a custom error message for better context if it fails.
+ */
+export function label<T>(label: string, parser: ParseFn<T>): ParseFn<T> {
+  return (tokenStream: TokenStream) => {
+    const result = parser(tokenStream);
+
+    console.log('label parse result:', result);
+
+    if (isSuccessfulResult(result)) {
+      return result;
+    }
+
+    const errorMessage = `${label}: ${result.errorMessage}`;
+
+    console.log('labeled error message:', errorMessage);
+
+    return {
+      errorMessage,
+      position: result.position,
+    };
+  }
+}
+
+/**
+ * In-built utility parser that parses any token except those of the specified type(s).
+ */
+export function anyExcept(...types: TokenType[]): ParseFn<Token> {
+  return (tokenStream: TokenStream) => {
+    const token = tokenStream.consume();
+
+    if (!token || types.includes(token.type)) {
+      return {
+        errorMessage: `Expected any token not of type ${types.join(', ')}, but got ${token?.type || 'end of input'}`,
+        position: tokenStream.getPositionForError(),
+      };
+    }
+
+    return { result: token };
+  };
+}
+
+/**
+ * In-built utility parser that parses any token of the specified type(s).
+ */
+export function anyOf(...types: TokenType[]): ParseFn<Token> {
+  return (tokenStream: TokenStream) => {
+    const token = tokenStream.consume();
+
+    if (!token || !types.includes(token.type)) {
+      return {
+        errorMessage: `Expected token of type ${types.join(', ')}, but got ${token?.type || 'end of input'}`,
+        position: tokenStream.getPositionForError(),
+      };
+    }
+
+    return { result: token }
+  }
+}
+
+/**
+ * In-built utility parser that ensures the end of input has been reached.
+ */
+export function endOfInput(): ParseFn {
+  return (tokenStream: TokenStream) => {
+    const token = tokenStream.consume();
+
+    if (token && token.type !== 'end_of_input') {
+      return {
+        errorMessage: `Expected end of input, but got token of type ${token.type}`,
+        position: tokenStream.getPositionForError(),
+      };
+    }
+
+    return { result: null };
+  };
+}
+
+export function parseName(): ParseFn<any> {
+  return (tokenStream: TokenStream) => {
+    const validTokenTypes: TokenType[] = ['letter', 'digit', 'minus', 'underscore'];
+    const parseValidToken = anyOf(...validTokenTypes);
+
+    const parser = and(anyOf('letter'), many(parseValidToken));
+    const result = parser(tokenStream);
+
+    if (isSuccessfulResult(result)) {
+      console.log(result);
+    }
+
+    return result;
+  }
+}
+
+/**
+ * Transforms the result of a parser using a given mapping function.
+ */
+export function map<T, U>(parser: ParseFn<T>, mapFn: (value: T) => U): ParseFn<U> {
+  return (tokenStream: TokenStream) => {
+    const result = parser(tokenStream);
+
+    if (isSuccessfulResult(result)) {
+      return { result: mapFn(result.result) };
+    }
+
+    return result;
+  };
+}
+
+/**
+ * Runs a parser on a given TokenStream, throwing an error if parsing fails.
+ */
+export function runParser<T>(parser: ParseFn<T>, tokenStream: TokenStream): ParserResult<T> {
+  const test = parser(tokenStream);
+  if (isFailedResult(test)) {
+    throw new ParsingError(test.errorMessage, test.position);
+  }
+
+  return test;
+}
+
+/**
+ * Runs a parser on a given input string, using the provided tokenizer to generate tokens.
+ */
+export function runParserOnString<T>(parser: ParseFn<T>, input: string, tokenizer: Tokenizer): ParserResult<T> {
+  const tokens = tokenizer.tokenize(input);
+  const stream = new TokenStream(tokens);
+
+  return runParser<T>(parser, stream);
+}

package/src/parser/types.ts

@@ -0,0 +1,28 @@
+import { TokenStream } from "../tokenizer";
+
+export type ParsingErrorPosition = {
+  line: number;
+  column: number;
+  position: number;
+}
+
+export type SuccessfulParserResult<T> = {
+  result: T;
+}
+
+export type FailedParserResult = {
+  errorMessage: string;
+  position: ParsingErrorPosition;
+}
+
+export type ParserResult<T> = SuccessfulParserResult<T> | FailedParserResult;
+
+export type ParseFn<T = null> = (tokenStream: TokenStream) => ParserResult<T>;
+
+export const isSuccessfulResult = <T>(result: ParserResult<T>): result is SuccessfulParserResult<T> => {
+  return (result as SuccessfulParserResult<T>).result !== undefined;
+}
+
+export const isFailedResult = <T>(result: ParserResult<T>): result is FailedParserResult => {
+  return (result as FailedParserResult).errorMessage !== undefined;
+}

package/src/tokenizer/TokenStream.ts

@@ -0,0 +1,80 @@
+import { Token, TokenType } from "./types";
+
+/**
+ * Represents a stream of tokens for parsing, including methods to consume and peek tokens,
+ * as well as state and method for managing the parsing position.
+ */
+export class TokenStream {
+  public position: number;
+  public positionStack: number[];
+
+  constructor (private tokens: Token[]) {
+    this.position = 0;
+    this.positionStack = [];
+  }
+
+  /**
+   * Peeks at the next token in the stream, without consuming it.
+   * @returns The next token, or null if we're at the end of the stream.
+   */
+  public peek(): Token | null {
+    return this.tokens[this.position] || null;
+  }
+
+  /**
+   * Consumes and returns the next token in the stream.
+   * @returns The consumed token, or null if we're at the end of the stream.
+   */
+  public consume(): Token | null {
+    return this.tokens[this.position++] || null;
+  }
+
+  /**
+   * Consumes a token if it matches the expected type.
+   * @returns The consumed token, or null if the next token does not match the expected type.
+   */
+  public consumeIf(...types: TokenType[]): Token | null {
+    const token = this.peek();
+    if (token && types.includes(token.type)) {
+      return this.consume();
+    }
+    return null;
+  }
+
+  /**
+   * Stores the current position in the position stack.
+   */
+  public storePosition() {
+    this.positionStack.push(this.position);
+  }
+
+  /**
+   * Clears the last stored position without restoring it.
+   */
+  public clearPosition() {
+    this.positionStack.pop();
+  }
+
+  /**
+   * Restores the last stored position from the position stack.
+   */
+  public restorePosition() {
+    const pos = this.positionStack.pop();
+    if (pos !== undefined) {
+      this.position = pos;
+    }
+  }
+
+  public peekRemainder(): string {
+    return this.tokens.slice(this.position).map(t => t.value).join('');
+  }
+
+  public getPositionForError(): { line: number; column: number, position: number } {
+    const tokenToUse = this.peek() || this.tokens[this.tokens.length - 1];
+
+    return {
+      position: this.position,
+      ...tokenToUse.position,
+    };
+  }
+}

package/src/tokenizer/Tokenizer.ts

@@ -0,0 +1,54 @@
+import { Token, TokenType } from "./types";
+
+export const LetterRegex = /[a-zA-Z]/;
+export const DigitRegex = /[0-9]/;
+export const WhitespaceRegex = /\s+/;
+export const NewLineRegex = /\n/;
+
+export class Tokenizer {
+  constructor(
+    private tokenTypeMatchers: { matcher: string | RegExp, type: TokenType }[] = [],
+    private newLineTokenType: TokenType | null = null
+  ) {}
+
+  public withTokenType(type: TokenType, matcher: string | RegExp): Tokenizer {
+    this.tokenTypeMatchers.push({ matcher, type });
+    return this;
+  }
+
+  public tokenize(input: string): Token[] {
+    const position = {
+      line: 1,
+      column: 0,
+    }
+
+    return input.split('').reduce<Token[]>((tokens, char) => {
+      position.column += 1;
+
+      let matched = false;
+      for (const { matcher, type } of this.tokenTypeMatchers.values()) {
+        if (typeof matcher === 'string' && char === matcher) {
+          tokens.push({ type, value: char, position: { ...position }});
+          matched = true;
+          break;
+        }
+
+        if (matcher instanceof RegExp && matcher.test(char)) {
+          tokens.push({ type, value: char, position: { ...position }});
+          matched = true;
+          break;
+        }
+
+        if (matched && type === this.newLineTokenType) {
+          position.line += 1;
+        }
+      }
+
+      if (!matched) {
+        throw new Error(`No token type matched for character: ${char}`);
+      }
+
+      return tokens;
+    }, []);
+  }
+}

package/src/tokenizer/index.ts

@@ -0,0 +1,3 @@
+export * from "./Tokenizer";
+export * from "./TokenStream";
+export * from "./types";

package/src/tokenizer/types.ts

@@ -0,0 +1,12 @@
+export type TokenType = string | 'end_of_input';
+
+export type TokenPosition = {
+  line: number;
+  column: number;
+}
+
+export type Token = {
+  value: string;
+  type: TokenType;
+  position: TokenPosition;
+}