astn 0.110.19 → 0.110.20

package/dist/parse/string_iterator.d.ts

@@ -0,0 +1,47 @@
+import * as _et from 'exupery-core-types';
+export type Location = {
+    'absolute': number;
+    'relative': {
+        'line': number;
+        'column': number;
+    };
+};
+export type String_Iterator = {
+    'consume character': () => void;
+    'consume string': ($: string) => void;
+    /**
+     * returns the current character, or null if the end of the string has been reached.
+     * equivalent to `look ahead(0)`
+     */
+    'get current character': () => number | null;
+    'look ahead': ($: number) => number | null;
+    'create offset location info': (subtract: number) => Location;
+    'create location info': () => Location;
+    'create location info string': () => string;
+    /**
+     * if no non-whitespace character has been found yet on the current line,
+     * this will return the current column,
+     * otherwise it will return the offset of that first non-whitespace character
+     * (which is the indentation of the line)
+     */
+    'get line indentation': () => number;
+    'starts with': ($: string) => boolean;
+};
+export declare const WhitespaceChars: {
+    tab: number;
+    line_feed: number;
+    carriage_return: number;
+    space: number;
+};
+export declare const is_control_character: ($: number) => boolean;
+/**
+ * Creates a string iterator that allows iterating over characters in a string,
+ * while keeping track of line numbers, columns, and line indentation.
+ */
+export declare const create_string_iterator: ($: string, $p: {
+    "tab size": number;
+}) => String_Iterator;
+export type Token_Iterator<Expected, Token> = {
+    'get required token': (expected: _et.Array<Expected>) => Token;
+    'consume token': () => void;
+};

package/dist/parse/string_iterator.js

@@ -0,0 +1,139 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.create_string_iterator = exports.is_control_character = exports.WhitespaceChars = void 0;
+const _ea = __importStar(require("exupery-core-alg"));
+const op = {
+    'to character list': _ea.impure.text['to character list'],
+    'starts with': _ea.impure.text['starts with'],
+};
+exports.WhitespaceChars = {
+    tab: 0x09, // \t
+    line_feed: 0x0A, // \n
+    carriage_return: 0x0D, // \r
+    space: 0x20, //
+};
+const is_control_character = ($) => {
+    return ($ < 0x20 && $ !== exports.WhitespaceChars.tab && $ !== exports.WhitespaceChars.line_feed && $ !== exports.WhitespaceChars.carriage_return);
+};
+exports.is_control_character = is_control_character;
+/**
+ * Creates a string iterator that allows iterating over characters in a string,
+ * while keeping track of line numbers, columns, and line indentation.
+ */
+const create_string_iterator = ($, $p) => {
+    const source = $;
+    const characters = op['to character list']($);
+    const length = characters.__get_length();
+    let found_carriage_return_before = false;
+    let position = 0;
+    let relative_position = {
+        'line': 0,
+        'column': 0,
+        'line indentation': null
+    };
+    const consume_character = () => {
+        const c = characters.__get_element_at(position);
+        const start = relative_position;
+        relative_position = c.transform(($) => {
+            return $ === exports.WhitespaceChars.line_feed
+                ? {
+                    'line': relative_position.line + 1,
+                    'column': 0,
+                    'line indentation': null,
+                }
+                : found_carriage_return_before
+                    ? {
+                        'line': relative_position.line + 1,
+                        'column': 0,
+                        'line indentation': null,
+                    }
+                    : {
+                        'line': relative_position.line,
+                        'column': relative_position['column'] + ($ === exports.WhitespaceChars.tab
+                            ? $p['tab size']
+                            : 1),
+                        'line indentation': relative_position['line indentation'] !== null
+                            ? relative_position['line indentation']
+                            : $ === exports.WhitespaceChars.space || $ === exports.WhitespaceChars.tab
+                                ? null
+                                : relative_position['column'],
+                    };
+        }, () => relative_position //this is weird, we were already at the end of the list
+        );
+        found_carriage_return_before = c.transform(($) => $ === exports.WhitespaceChars.carriage_return, () => false);
+        position += 1;
+    };
+    return {
+        'consume string': ($) => {
+            op['to character list']($).__for_each(() => {
+                consume_character();
+            });
+        },
+        'consume character': consume_character,
+        'get current character': () => {
+            if (position === length) {
+                return null;
+            }
+            return characters.__get_element_at(position).transform(($) => $, () => null);
+        },
+        'look ahead': ($) => {
+            const next_position = position + $;
+            if (next_position >= length) {
+                return null;
+            }
+            return characters.__get_element_at(next_position).transform(($) => $, () => null);
+        },
+        'create location info': () => {
+            return {
+                'absolute': position,
+                'relative': {
+                    'line': relative_position.line,
+                    'column': relative_position['column'],
+                }
+            };
+        },
+        'create offset location info': (subtract) => {
+            return {
+                'absolute': position - subtract,
+                'relative': {
+                    'line': relative_position.line,
+                    'column': relative_position['column'] - subtract,
+                }
+            };
+        },
+        'create location info string': () => `${relative_position.line}:${relative_position['column']}`,
+        'get line indentation': () => {
+            return relative_position['line indentation'] !== null
+                ? relative_position['line indentation']
+                : relative_position['column'];
+        },
+        'starts with': ($) => {
+            return op['starts with'](source, $, position);
+        }
+    };
+};
+exports.create_string_iterator = create_string_iterator;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic3RyaW5nX2l0ZXJhdG9yLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL3BhcnNlL3N0cmluZ19pdGVyYXRvci50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7OztBQUFBLHNEQUF1QztBQXFDdkMsTUFBTSxFQUFFLEdBQUc7SUFDUCxtQkFBbUIsRUFBRSxHQUFHLENBQUMsTUFBTSxDQUFDLElBQUksQ0FBQyxtQkFBbUIsQ0FBQztJQUN6RCxhQUFhLEVBQUUsR0FBRyxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsYUFBYSxDQUFDO0NBQ2hELENBQUE7QUFFWSxRQUFBLGVBQWUsR0FBRztJQUMzQixHQUFHLEVBQUUsSUFBSSxFQUFtQixLQUFLO0lBQ2pDLFNBQVMsRUFBRSxJQUFJLEVBQWEsS0FBSztJQUNqQyxlQUFlLEVBQUUsSUFBSSxFQUFPLEtBQUs7SUFDakMsS0FBSyxFQUFFLElBQUksRUFBaUIsRUFBRTtDQUNqQyxDQUFBO0FBRU0sTUFBTSxvQkFBb0IsR0FBRyxDQUFDLENBQVMsRUFBVyxFQUFFO0lBQ3ZELE9BQU8sQ0FBQyxDQUFDLEdBQUcsSUFBSSxJQUFJLENBQUMsS0FBSyx1QkFBZSxDQUFDLEdBQUcsSUFBSSxDQUFDLEtBQUssdUJBQWUsQ0FBQyxTQUFTLElBQUksQ0FBQyxLQUFLLHVCQUFlLENBQUMsZUFBZSxDQUFDLENBQUE7QUFDOUgsQ0FBQyxDQUFBO0FBRlksUUFBQSxvQkFBb0Isd0JBRWhDO0FBRUQ7OztHQUdHO0FBRUksTUFBTSxzQkFBc0IsR0FBRyxDQUNsQyxDQUFTLEVBQ1QsRUFFQyxFQUNjLEVBQUU7SUFDakIsTUFBTSxNQUFNLEdBQUcsQ0FBQyxDQUFBO0lBQ2hCLE1BQU0sVUFBVSxHQUFHLEVBQUUsQ0FBQyxtQkFBbUIsQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFBO0lBQzdDLE1BQU0sTUFBTSxHQUFHLFVBQVUsQ0FBQyxZQUFZLEVBQUUsQ0FBQTtJQVF4QyxJQUFJLDRCQUE0QixHQUFHLEtBQUssQ0FBQTtJQUN4QyxJQUFJLFFBQVEsR0FBRyxDQUFDLENBQUE7SUFDaEIsSUFBSSxpQkFBaUIsR0FBa0M7UUFDbkQsTUFBTSxFQUFFLENBQUM7UUFDVCxRQUFRLEVBQUUsQ0FBQztRQUNYLGtCQUFrQixFQUFFLElBQUk7S0FDM0IsQ0FBQTtJQUVELE1BQU0saUJBQWlCLEdBQUcsR0FBRyxFQUFFO1FBQzNCLE1BQU0sQ0FBQyxHQUFHLFVBQVUsQ0FBQyxnQkFBZ0IsQ0FBQyxRQUFRLENBQUMsQ0FBQTtRQUMvQyxNQUFNLEtBQUssR0FBRyxpQkFBaUIsQ0FBQTtRQUMvQixpQkFBaUIsR0FBRyxDQUFDLENBQUMsU0FBUyxDQUMzQixDQUFDLENBQUMsRUFBRSxFQUFFO1lBQ0YsT0FBTyxDQUFDLEtBQUssdUJBQWUsQ0FBQyxTQUFTO2dCQUNsQyxDQUFDLENBQUM7b0JBQ0UsTUFBTSxFQUFFLGlCQUFpQixDQUFDLElBQUksR0FBRyxDQUFDO29CQUNsQyxRQUFRLEVBQUUsQ0FBQztvQkFDWCxrQkFBa0IsRUFBRSxJQUFJO2lCQUMzQjtnQkFDRCxDQUFDLENBQUMsNEJBQTRCO29CQUMxQixDQUFDLENBQUM7d0JBQ0UsTUFBTSxFQUFFLGlCQUFpQixDQUFDLElBQUksR0FBRyxDQUFDO3dCQUNsQyxRQUFRLEVBQUUsQ0FBQzt3QkFDWCxrQkFBa0IsRUFBRSxJQUFJO3FCQUMzQjtvQkFDRCxDQUFDLENBQUM7d0JBQ0UsTUFBTSxFQUFFLGlCQUFpQixDQUFDLElBQUk7d0JBQzlCLFFBQVEsRUFBRSxpQkFBaUIsQ0FBQyxRQUFRLENBQUMsR0FBRyxDQUFDLENBQUMsS0FBSyx1QkFBZSxDQUFDLEdBQUc7NEJBQzlELENBQUMsQ0FBQyxFQUFFLENBQUMsVUFBVSxDQUFDOzRCQUNoQixDQUFDLENBQUMsQ0FBQyxDQUFDO3dCQUNSLGtCQUFrQixFQUFFLGlCQUFpQixDQUFDLGtCQUFrQixDQUFDLEtBQUssSUFBSTs0QkFDOUQsQ0FBQyxDQUFDLGlCQUFpQixDQUFDLGtCQUFrQixDQUFDOzRCQUN2QyxDQUFDLENBQUMsQ0FBQyxLQUFLLHVCQUFlLENBQUMsS0FBSyxJQUFJLENBQUMsS0FBSyx1QkFBZSxDQUFDLEdBQUc7Z0NBQ3RELENBQUMsQ0FBQyxJQUFJO2dDQUNOLENBQUMsQ0FBQyxpQkFBaUIsQ0FBQyxRQUFRLENBQUM7cUJBQ3hDLENBQUE7UUFDYixDQUFDLEVBQ0QsR0FBRyxFQUFFLENBQUMsaUJBQWlCLENBQUMsdURBQXVEO1NBQ2xGLENBQUE7UUFFRCw0QkFBNEIsR0FBRyxDQUFDLENBQUMsU0FBUyxDQUN0QyxDQUFDLENBQUMsRUFBRSxFQUFFLENBQUMsQ0FBQyxLQUFLLHVCQUFlLENBQUMsZUFBZSxFQUM1QyxHQUFHLEVBQUUsQ0FBQyxLQUFLLENBQ2QsQ0FBQTtRQUNELFFBQVEsSUFBSSxDQUFDLENBQUE7SUFDakIsQ0FBQyxDQUFBO0lBRUQsT0FBTztRQUNILGdCQUFnQixFQUFFLENBQUMsQ0FBUyxFQUFFLEVBQUU7WUFDNUIsRUFBRSxDQUFDLG1CQUFtQixDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsVUFBVSxDQUFDLEdBQUcsRUFBRTtnQkFDdkMsaUJBQWlCLEVBQUUsQ0FBQTtZQUN2QixDQUFDLENBQUMsQ0FBQTtRQUNOLENBQUM7UUFDRCxtQkFBbUIsRUFBRSxpQkFBaUI7UUFDdEMsdUJBQXVCLEVBQUUsR0FBRyxFQUFFO1lBQzFCLElBQUksUUFBUSxLQUFLLE1BQU0sRUFBRSxDQUFDO2dCQUN0QixPQUFPLElBQUksQ0FBQTtZQUNmLENBQUM7WUFDRCxPQUFPLFVBQVUsQ0FBQyxnQkFBZ0IsQ0FBQyxRQUFRLENBQUMsQ0FBQyxTQUFTLENBQ2xELENBQUMsQ0FBQyxFQUFFLEVBQUUsQ0FBQyxDQUFDLEVBQ1IsR0FBRyxFQUFFLENBQUMsSUFBSSxDQUNiLENBQUE7UUFDTCxDQUFDO1FBQ0QsWUFBWSxFQUFFLENBQUMsQ0FBUyxFQUFFLEVBQUU7WUFDeEIsTUFBTSxhQUFhLEdBQUcsUUFBUSxHQUFHLENBQUMsQ0FBQztZQUNuQyxJQUFJLGFBQWEsSUFBSSxNQUFNLEVBQUUsQ0FBQztnQkFDMUIsT0FBTyxJQUFJLENBQUE7WUFDZixDQUFDO1lBQ0QsT0FBTyxVQUFVLENBQUMsZ0JBQWdCLENBQUMsYUFBYSxDQUFDLENBQUMsU0FBUyxDQUN2RCxDQUFDLENBQUMsRUFBRSxFQUFFLENBQUMsQ0FBQyxFQUNSLEdBQUcsRUFBRSxDQUFDLElBQUksQ0FDYixDQUFBO1FBQ0wsQ0FBQztRQUNELHNCQUFzQixFQUFFLEdBQUcsRUFBRTtZQUN6QixPQUFPO2dCQUNILFVBQVUsRUFBRSxRQUFRO2dCQUNwQixVQUFVLEVBQUU7b0JBQ1IsTUFBTSxFQUFFLGlCQUFpQixDQUFDLElBQUk7b0JBQzlCLFFBQVEsRUFBRSxpQkFBaUIsQ0FBQyxRQUFRLENBQUM7aUJBQ3hDO2FBQ0osQ0FBQTtRQUNMLENBQUM7UUFDRCw2QkFBNkIsRUFBRSxDQUFDLFFBQVEsRUFBRSxFQUFFO1lBQ3hDLE9BQU87Z0JBQ0gsVUFBVSxFQUFFLFFBQVEsR0FBRyxRQUFRO2dCQUMvQixVQUFVLEVBQUU7b0JBQ1IsTUFBTSxFQUFFLGlCQUFpQixDQUFDLElBQUk7b0JBQzlCLFFBQVEsRUFBRSxpQkFBaUIsQ0FBQyxRQUFRLENBQUMsR0FBRyxRQUFRO2lCQUNuRDthQUNKLENBQUE7UUFDTCxDQUFDO1FBQ0QsNkJBQTZCLEVBQUUsR0FBRyxFQUFFLENBQUMsR0FBRyxpQkFBaUIsQ0FBQyxJQUFJLElBQUksaUJBQWlCLENBQUMsUUFBUSxDQUFDLEVBQUU7UUFDL0Ysc0JBQXNCLEVBQUUsR0FBRyxFQUFFO1lBQ3pCLE9BQU8saUJBQWlCLENBQUMsa0JBQWtCLENBQUMsS0FBSyxJQUFJO2dCQUNqRCxDQUFDLENBQUMsaUJBQWlCLENBQUMsa0JBQWtCLENBQUM7Z0JBQ3ZDLENBQUMsQ0FBQyxpQkFBaUIsQ0FBQyxRQUFRLENBQUMsQ0FBQTtRQUNyQyxDQUFDO1FBQ0QsYUFBYSxFQUFFLENBQUMsQ0FBUyxFQUFFLEVBQUU7WUFDekIsT0FBTyxFQUFFLENBQUMsYUFBYSxDQUFDLENBQ3BCLE1BQU0sRUFDTixDQUFDLEVBQ0QsUUFBUSxDQUNYLENBQUE7UUFDTCxDQUFDO0tBQ0osQ0FBQTtBQUNMLENBQUMsQ0FBQTtBQXpIWSxRQUFBLHNCQUFzQiwwQkF5SGxDIn0=

package/dist/parse/token.d.ts

@@ -0,0 +1,9 @@
+import * as _out from "../generated/interface/schemas/token/unconstrained";
+import { String_Iterator } from "./string_iterator";
+export declare const Whitespace: (string_iterator: String_Iterator) => _out.Whitespace;
+export declare const Trivia: (string_iterator: String_Iterator) => _out.Trivia;
+export declare const Annotated_Token: (st: String_Iterator) => _out.Annotated_Token;
+export declare const Delimited_String: (string_iterator: String_Iterator, is_end_character: (character: number) => boolean, allow_newlines: boolean) => _out.Delimited_String;
+export declare const Tokenizer_Result: ($: null, $p: {
+    "string iterator": String_Iterator;
+}) => _out.Tokenizer_Result;