@altopelago/aeon-lexer 0.9.0

package/README.md

@@ -0,0 +1,24 @@
+# @altopelago/aeon-lexer
+
+Tokenization for AEON source text.
+
+## Installation
+
+```bash
+pnpm add @altopelago/aeon-lexer
+```
+
+## Usage
+
+```ts
+import { tokenize } from '@altopelago/aeon-lexer';
+
+const result = tokenize('answer = 42');
+
+for (const token of result.tokens) {
+  console.log(token.kind, token.raw);
+}
+```
+
+Use this package when you need direct token access for tooling, analysis, or editor features.
+If you want the stable application-facing entry point, prefer `@altopelago/aeon-core`.

package/dist/errors.d.ts

@@ -0,0 +1,56 @@
+import type { Span } from './tokens.js';
+/**
+ * Base class for all lexer errors
+ */
+export declare class LexerError extends Error {
+    readonly span: Span;
+    readonly code: string;
+    constructor(message: string, span: Span, code?: string);
+}
+/**
+ * Unexpected character encountered
+ */
+export declare class UnexpectedCharacterError extends LexerError {
+    readonly character: string;
+    constructor(character: string, span: Span);
+}
+/**
+ * Unterminated string literal
+ */
+export declare class UnterminatedStringError extends LexerError {
+    readonly delimiter: string;
+    constructor(delimiter: string, span: Span);
+}
+/**
+ * Invalid escape sequence
+ */
+export declare class InvalidEscapeSequenceError extends LexerError {
+    readonly sequence: string;
+    constructor(sequence: string, span: Span);
+}
+/**
+ * Invalid number literal
+ */
+export declare class InvalidNumberError extends LexerError {
+    readonly raw: string;
+    constructor(raw: string, span: Span);
+}
+export declare class InvalidTimeError extends LexerError {
+    readonly raw: string;
+    constructor(raw: string, span: Span);
+}
+export declare class InvalidDateError extends LexerError {
+    readonly raw: string;
+    constructor(raw: string, span: Span);
+}
+export declare class InvalidDateTimeError extends LexerError {
+    readonly raw: string;
+    constructor(raw: string, span: Span);
+}
+/**
+ * Unterminated block comment
+ */
+export declare class UnterminatedBlockCommentError extends LexerError {
+    constructor(span: Span);
+}
+//# sourceMappingURL=errors.d.ts.map

package/dist/errors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAExC;;GAEG;AACH,qBAAa,UAAW,SAAQ,KAAK;IACjC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;gBAEV,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,GAAE,MAAsB;CAMxE;AAED;;GAEG;AACH,qBAAa,wBAAyB,SAAQ,UAAU;IACpD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;gBAEf,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI;CAK5C;AAED;;GAEG;AACH,qBAAa,uBAAwB,SAAQ,UAAU;IACnD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;gBAEf,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI;CAK5C;AAED;;GAEG;AACH,qBAAa,0BAA2B,SAAQ,UAAU;IACtD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;gBAEd,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI;CAK3C;AAED;;GAEG;AACH,qBAAa,kBAAmB,SAAQ,UAAU;IAC9C,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;gBAET,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI;CAKtC;AAED,qBAAa,gBAAiB,SAAQ,UAAU;IAC5C,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;gBAET,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI;CAKtC;AAED,qBAAa,gBAAiB,SAAQ,UAAU;IAC5C,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;gBAET,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI;CAKtC;AAED,qBAAa,oBAAqB,SAAQ,UAAU;IAChD,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;gBAET,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI;CAKtC;AAED;;GAEG;AACH,qBAAa,6BAA8B,SAAQ,UAAU;gBAC7C,IAAI,EAAE,IAAI;CAIzB"}

package/dist/errors.js

@@ -0,0 +1,91 @@
+/**
+ * Base class for all lexer errors
+ */
+export class LexerError extends Error {
+    span;
+    code;
+    constructor(message, span, code = 'LEXER_ERROR') {
+        super(message);
+        this.name = 'LexerError';
+        this.span = span;
+        this.code = code;
+    }
+}
+/**
+ * Unexpected character encountered
+ */
+export class UnexpectedCharacterError extends LexerError {
+    character;
+    constructor(character, span) {
+        super(`Unexpected character: '${character}'`, span, 'UNEXPECTED_CHARACTER');
+        this.name = 'UnexpectedCharacterError';
+        this.character = character;
+    }
+}
+/**
+ * Unterminated string literal
+ */
+export class UnterminatedStringError extends LexerError {
+    delimiter;
+    constructor(delimiter, span) {
+        super(`Unterminated string literal (started with ${delimiter})`, span, 'UNTERMINATED_STRING');
+        this.name = 'UnterminatedStringError';
+        this.delimiter = delimiter;
+    }
+}
+/**
+ * Invalid escape sequence
+ */
+export class InvalidEscapeSequenceError extends LexerError {
+    sequence;
+    constructor(sequence, span) {
+        super(`Invalid escape sequence: '${sequence}'`, span, 'INVALID_ESCAPE');
+        this.name = 'InvalidEscapeSequenceError';
+        this.sequence = sequence;
+    }
+}
+/**
+ * Invalid number literal
+ */
+export class InvalidNumberError extends LexerError {
+    raw;
+    constructor(raw, span) {
+        super(`Invalid number literal: '${raw}'`, span, 'INVALID_NUMBER');
+        this.name = 'InvalidNumberError';
+        this.raw = raw;
+    }
+}
+export class InvalidTimeError extends LexerError {
+    raw;
+    constructor(raw, span) {
+        super(`Invalid time literal: '${raw}'`, span, 'INVALID_TIME');
+        this.name = 'InvalidTimeError';
+        this.raw = raw;
+    }
+}
+export class InvalidDateError extends LexerError {
+    raw;
+    constructor(raw, span) {
+        super(`Invalid date literal: '${raw}'`, span, 'INVALID_DATE');
+        this.name = 'InvalidDateError';
+        this.raw = raw;
+    }
+}
+export class InvalidDateTimeError extends LexerError {
+    raw;
+    constructor(raw, span) {
+        super(`Invalid datetime literal: '${raw}'`, span, 'INVALID_DATETIME');
+        this.name = 'InvalidDateTimeError';
+        this.raw = raw;
+    }
+}
+/**
+ * Unterminated block comment
+ */
+export class UnterminatedBlockCommentError extends LexerError {
+    constructor(span) {
+        super('Unterminated block comment', span, 'UNTERMINATED_BLOCK_COMMENT');
+        this.name = 'UnterminatedBlockCommentError';
+    }
+}
+//# sourceMappingURL=errors.js.map

package/dist/errors.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,MAAM,OAAO,UAAW,SAAQ,KAAK;IACxB,IAAI,CAAO;IACX,IAAI,CAAS;IAEtB,YAAY,OAAe,EAAE,IAAU,EAAE,OAAe,aAAa;QACjE,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,YAAY,CAAC;QACzB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACrB,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,OAAO,wBAAyB,SAAQ,UAAU;IAC3C,SAAS,CAAS;IAE3B,YAAY,SAAiB,EAAE,IAAU;QACrC,KAAK,CAAC,0BAA0B,SAAS,GAAG,EAAE,IAAI,EAAE,sBAAsB,CAAC,CAAC;QAC5E,IAAI,CAAC,IAAI,GAAG,0BAA0B,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC/B,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,OAAO,uBAAwB,SAAQ,UAAU;IAC1C,SAAS,CAAS;IAE3B,YAAY,SAAiB,EAAE,IAAU;QACrC,KAAK,CAAC,6CAA6C,SAAS,GAAG,EAAE,IAAI,EAAE,qBAAqB,CAAC,CAAC;QAC9F,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC;QACtC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC/B,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,OAAO,0BAA2B,SAAQ,UAAU;IAC7C,QAAQ,CAAS;IAE1B,YAAY,QAAgB,EAAE,IAAU;QACpC,KAAK,CAAC,6BAA6B,QAAQ,GAAG,EAAE,IAAI,EAAE,gBAAgB,CAAC,CAAC;QACxE,IAAI,CAAC,IAAI,GAAG,4BAA4B,CAAC;QACzC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC7B,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,OAAO,kBAAmB,SAAQ,UAAU;IACrC,GAAG,CAAS;IAErB,YAAY,GAAW,EAAE,IAAU;QAC/B,KAAK,CAAC,4BAA4B,GAAG,GAAG,EAAE,IAAI,EAAE,gBAAgB,CAAC,CAAC;QAClE,IAAI,CAAC,IAAI,GAAG,oBAAoB,CAAC;QACjC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACnB,CAAC;CACJ;AAED,MAAM,OAAO,gBAAiB,SAAQ,UAAU;IACnC,GAAG,CAAS;IAErB,YAAY,GAAW,EAAE,IAAU;QAC/B,KAAK,CAAC,0BAA0B,GAAG,GAAG,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;QAC9D,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;QAC/B,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACnB,CAAC;CACJ;AAED,MAAM,OAAO,gBAAiB,SAAQ,UAAU;IACnC,GAAG,CAAS;IAErB,YAAY,GAAW,EAAE,IAAU;QAC/B,KAAK,CAAC,0BAA0B,GAAG,GAAG,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;QAC9D,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;QAC/B,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACnB,CAAC;CACJ;AAED,MAAM,OAAO,oBAAqB,SAAQ,UAAU;IACvC,GAAG,CAAS;IAErB,YAAY,GAAW,EAAE,IAAU;QAC/B,KAAK,CAAC,8BAA8B,GAAG,GAAG,EAAE,IAAI,EAAE,kBAAkB,CAAC,CAAC;QACtE,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;QACnC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACnB,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,OAAO,6BAA8B,SAAQ,UAAU;IACzD,YAAY,IAAU;QAClB,KAAK,CAAC,4BAA4B,EAAE,IAAI,EAAE,4BAA4B,CAAC,CAAC;QACxE,IAAI,CAAC,IAAI,GAAG,+BAA+B,CAAC;IAChD,CAAC;CACJ"}

package/dist/index.d.ts

@@ -0,0 +1,9 @@
+/**
+ * @altopelago/aeon-lexer - AEON Lexer Package
+ *
+ * Tokenizes AEON documents into a stream of tokens with span information.
+ */
+export * from './tokens.js';
+export * from './lexer.js';
+export * from './errors.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC"}

package/dist/index.js

@@ -0,0 +1,9 @@
+/**
+ * @altopelago/aeon-lexer - AEON Lexer Package
+ *
+ * Tokenizes AEON documents into a stream of tokens with span information.
+ */
+export * from './tokens.js';
+export * from './lexer.js';
+export * from './errors.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC"}

package/dist/lexer.d.ts

@@ -0,0 +1,72 @@
+import { type Token } from './tokens.js';
+import { LexerError } from './errors.js';
+/**
+ * Lexer options
+ */
+export interface LexerOptions {
+    /** Include comment tokens in output (default: false) */
+    readonly includeComments?: boolean;
+    /** Include newline tokens in output (default: false) */
+    readonly includeNewlines?: boolean;
+}
+/**
+ * Result of lexing
+ */
+export interface LexResult {
+    readonly tokens: readonly Token[];
+    readonly errors: readonly LexerError[];
+}
+/**
+ * AEON Lexer
+ *
+ * Hand-written lexer for AEON documents. Produces a stream of tokens
+ * with accurate span information for error reporting.
+ */
+export declare class Lexer {
+    private readonly input;
+    private readonly options;
+    private offset;
+    private line;
+    private column;
+    private sawLeadingShebang;
+    private readonly tokens;
+    private readonly errors;
+    constructor(input: string, options?: LexerOptions);
+    /**
+     * Tokenize the input
+     */
+    tokenize(): LexResult;
+    private isAtEnd;
+    private currentPosition;
+    private peek;
+    private peekNext;
+    private peekN;
+    private advance;
+    private match;
+    private addToken;
+    private scanToken;
+    private scanString;
+    private scanEscapeSequence;
+    private scanUnicodeEscape;
+    private scanNumber;
+    private scanTime;
+    private scanDateOrDateTime;
+    private scanHexLiteral;
+    private scanRadixLiteral;
+    private scanEncodingLiteral;
+    private scanSeparatorLiteral;
+    private scanIdentifierOrKeyword;
+    private scanLineComment;
+    private scanShebangComment;
+    private scanBlockComment;
+    private scanSlashChannelBlockComment;
+    private addCommentToken;
+    private isLeadingShebangStart;
+    private isHostDirectiveSlot;
+    private classifyLineComment;
+}
+/**
+ * Tokenize an AEON document
+ */
+export declare function tokenize(input: string, options?: LexerOptions): LexResult;
+//# sourceMappingURL=lexer.d.ts.map

package/dist/lexer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lexer.d.ts","sourceRoot":"","sources":["../src/lexer.ts"],"names":[],"mappings":"AAAA,OAAO,EAIH,KAAK,KAAK,EAMb,MAAM,aAAa,CAAC;AACrB,OAAO,EACH,UAAU,EASb,MAAM,aAAa,CAAC;AAErB;;GAEG;AACH,MAAM,WAAW,YAAY;IACzB,wDAAwD;IACxD,QAAQ,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC;IACnC,wDAAwD;IACxD,QAAQ,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC;CACtC;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,QAAQ,CAAC,MAAM,EAAE,SAAS,UAAU,EAAE,CAAC;CAC1C;AA6FD;;;;;GAKG;AACH,qBAAa,KAAK;IACd,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAe;IACvC,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,IAAI,CAAa;IACzB,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,iBAAiB,CAAkB;IAC3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoB;gBAE/B,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB;IAKrD;;OAEG;IACH,QAAQ,IAAI,SAAS;IAmBrB,OAAO,CAAC,OAAO;IAIf,OAAO,CAAC,eAAe;IAIvB,OAAO,CAAC,IAAI;IAKZ,OAAO,CAAC,QAAQ;IAKhB,OAAO,CAAC,KAAK;IAMb,OAAO,CAAC,OAAO;IAYf,OAAO,CAAC,KAAK;IAOb,OAAO,CAAC,QAAQ;IAKhB,OAAO,CAAC,SAAS;IAwIjB,OAAO,CAAC,UAAU;IA0ClB,OAAO,CAAC,kBAAkB;IA6B1B,OAAO,CAAC,iBAAiB;IA2GzB,OAAO,CAAC,UAAU;IAsIlB,OAAO,CAAC,QAAQ;IAuBhB,OAAO,CAAC,kBAAkB;IAiE1B,OAAO,CAAC,cAAc;IAyBtB,OAAO,CAAC,gBAAgB;IAoBxB,OAAO,CAAC,mBAAmB;IA+B3B,OAAO,CAAC,oBAAoB;IA2D5B,OAAO,CAAC,uBAAuB;IAe/B,OAAO,CAAC,eAAe;IAYvB,OAAO,CAAC,kBAAkB;IAc1B,OAAO,CAAC,gBAAgB;IAsBxB,OAAO,CAAC,4BAA4B;IAuBpC,OAAO,CAAC,eAAe;IAKvB,OAAO,CAAC,qBAAqB;IAI7B,OAAO,CAAC,mBAAmB;IAU3B,OAAO,CAAC,mBAAmB;CAM9B;AA6SD;;GAEG;AACH,wBAAgB,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,SAAS,CAGzE"}