@harmoniclabs/pebble 0.1.0-dev0 → 0.1.0-dev2

package/dist/tokenizer/Tokenizer.d.ts

@@ -1,129 +0,0 @@
-import { Source } from "../ast/Source/Source.js";
-import { SourceRange } from "../ast/Source/SourceRange.js";
-import { DiagnosticEmitter } from "../diagnostics/DiagnosticEmitter.js";
-import { DiagnosticMessage } from "../diagnostics/DiagnosticMessage.js";
-import { IdentifierHandling } from "./IdentifierHandling.js";
-import { OnNewLine } from "./OnNewLine.js";
-import { Token } from "./Token.js";
-/** Comment kinds. */
-export declare enum CommentKind {
-    /** Line comment. */
-    Line = 0,
-    /** Triple-slash line comment. */
-    Triple = 1,
-    /** Block comment. */
-    Block = 2
-}
-export type CommentHandler = (kind: CommentKind, text: string, range: SourceRange) => void;
-/**
- * Takes a {@link Source} and tokenizes its contents.
- *
- * It acts as an iterator, so that the text is never broken down
- * but we can still get out single tokens.
-**/
-export declare class Tokenizer extends DiagnosticEmitter {
-    source: Source;
-    end: number;
-    pos: number;
-    token: Token;
-    tokenPos: number;
-    nextToken: Token;
-    nextTokenPos: number;
-    nextTokenOnNewLine: OnNewLine;
-    onComment: CommentHandler | undefined;
-    constructor(source: Source, diagnostics?: DiagnosticMessage[] | undefined);
-    /** advances the tokenizer and returns the new token */
-    next(identifierHandling?: IdentifierHandling): Token;
-    /**
-     * Retrieves the next token from the source text,
-     * handling identifiers and token length.
-     * This method is considered unsafe because
-     * it does not perform validation on the token.
-     *
-     * @param identifierHandling - Specifies how identifiers should be handled.
-     *
-     * @param maxTokenLength - The maximum length of the token to retrieve.
-     * Defaults to Number.MAX_SAFE_INTEGER.
-     *
-     * @returns The next token from the source text.
-     */
-    private unsafeNext;
-    peek(identifierHandling?: IdentifierHandling, maxCompoundLength?: number): Token;
-    isNextTokenOnNewLine(): boolean;
-    skipIdentifier(identifierHandling?: IdentifierHandling): boolean;
-    /**
-     *
-     * @param expectedToken The token type that the tokenizer should skip if it matches the current token.
-     * @param identifierHandling
-     * @returns {boolean} Returns true if the current token matches the expectedToken and the tokenizer's position is advanced.
-     * Returns false if the current token does not match the expectedToken.
-     *
-     * ### side effects
-     *
-     * - Advances the Tokenizer Position:
-     * If the current token matches the expectedToken,
-     * the tokenizer's position is advanced to the next token.
-     *
-     * - Updates Internal State:
-     * The method updates the internal state of the tokenizer,
-     * including the current token and its position.
-     *
-     */
-    skip(expectedToken: Token, identifierHandling?: IdentifierHandling): boolean;
-    eof(): boolean;
-    /** #__PURE__ gets the current state */
-    mark(): TokenizerState;
-    reset(state: TokenizerState): void;
-    /**
-     * ### Side Effects
-     * - Resets `nextToken`: Sets the nextToken property to an invalid state, typically -1.
-     * - Resets `nextTokenPos`: Sets the nextTokenPos property to 0 or another default value.
-     * - Resets `nextTokenOnNewLine`: Sets the nextTokenOnNewLine property to OnNewLine.
-     * Unknown or another default value.
-     */
-    clearNextToken(): void;
-    /**
-     * pure function for the same state
-    **/
-    range(start?: number, end?: number): SourceRange;
-    readIdentifier(): string;
-    readingTemplateString: boolean;
-    readStringStart: number;
-    readStringEnd: number;
-    readString(quote?: number, isTaggedTemplate?: boolean): string;
-    readEscapeSequence(isTaggedTemplate?: boolean): string;
-    readRegexpPattern(): string;
-    readRegexpFlags(): string;
-    testInteger(): boolean;
-    readInteger(): bigint;
-    readHexBytes(): Uint8Array;
-    readHexInteger(): bigint;
-    readDecimalInteger(): bigint;
-    readOctalInteger(): bigint;
-    readBinaryInteger(): bigint;
-    readFloat(): number;
-    readDecimalFloat(): number;
-    /** Reads past one section of a decimal float literal. Returns the number of separators encountered. */
-    private readDecimalFloatPartial;
-    readHexFloat(): number;
-    readHexadecimalEscape(remain?: number, startIfTaggedTemplate?: number): string;
-    checkForIdentifierStartAfterNumericLiteral(): void;
-    readUnicodeEscape(startIfTaggedTemplate?: number): string;
-    private readExtendedUnicodeEscape;
-}
-/** Tokenizer state as returned by {@link Tokenizer#mark} and consumed by {@link Tokenizer#reset}. */
-export declare class TokenizerState {
-    /** Current position. */
-    pos: number;
-    /** Current token. */
-    token: Token;
-    /** Current token's position. */
-    tokenPos: number;
-    constructor(
-    /** Current position. */
-    pos: number, 
-    /** Current token. */
-    token: Token, 
-    /** Current token's position. */
-    tokenPos: number);
-}