astn 0.113.6 → 0.113.7

package/dist/implementation/algorithms/refiners/token/text/iterator.js

@@ -0,0 +1,107 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.create_iterator = exports.is_control_character = void 0;
+const _ea = __importStar(require("exupery-core-alg"));
+const WhitespaceChars = {
+    tab: 0x09, // \t
+    line_feed: 0x0A, // \n
+    carriage_return: 0x0D, // \r
+    space: 0x20, //
+};
+const is_control_character = ($) => {
+    return ($ < 0x20 && $ !== WhitespaceChars.tab && $ !== WhitespaceChars.line_feed && $ !== WhitespaceChars.carriage_return);
+};
+exports.is_control_character = is_control_character;
+/**
+ * Creates a string iterator that allows iterating over characters in a string,
+ * while keeping track of line numbers, columns, and line indentation.
+ */
+const create_iterator = ($, $p) => {
+    const characters = _ea.text_to_character_list($);
+    let found_carriage_return_before = false;
+    let position = 0;
+    let relative_position = {
+        'line': 0,
+        'column': 0,
+        'line indentation': null
+    };
+    return {
+        'consume': () => {
+            const c = characters.__get_element_at(position);
+            const start = relative_position;
+            relative_position = c.transform(($) => {
+                return $ === WhitespaceChars.line_feed
+                    ? {
+                        'line': relative_position.line + 1,
+                        'column': 0,
+                        'line indentation': null,
+                    }
+                    : found_carriage_return_before
+                        ? {
+                            'line': relative_position.line + 1,
+                            'column': 0,
+                            'line indentation': null,
+                        }
+                        : {
+                            'line': relative_position.line,
+                            'column': relative_position['column'] + ($ === WhitespaceChars.tab
+                                ? $p['tab size']
+                                : 1),
+                            'line indentation': relative_position['line indentation'] !== null
+                                ? relative_position['line indentation']
+                                : $ === WhitespaceChars.space || $ === WhitespaceChars.tab
+                                    ? null
+                                    : relative_position['column'],
+                        };
+            }, () => relative_position //this is weird, we were already at the end of the list
+            );
+            found_carriage_return_before = c.transform(($) => $ === WhitespaceChars.carriage_return, () => false);
+            position += 1;
+        },
+        'get current': () => {
+            return characters.__get_element_at(position).transform(($) => _ea.set($), () => _ea.not_set());
+        },
+        'look ahead': (offset) => {
+            return characters.__get_element_at(position + offset).transform(($) => _ea.set($), () => _ea.not_set());
+        },
+        'get state': () => {
+            return {
+                'location': {
+                    'absolute': position,
+                    'relative': {
+                        'line': relative_position.line,
+                        'column': relative_position['column'],
+                    }
+                },
+                'line indentation': relative_position['line indentation'] !== null
+                    ? relative_position['line indentation']
+                    : relative_position['column']
+            };
+        }
+    };
+};
+exports.create_iterator = create_iterator;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaXRlcmF0b3IuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi8uLi8uLi8uLi9zcmMvaW1wbGVtZW50YXRpb24vYWxnb3JpdGhtcy9yZWZpbmVycy90b2tlbi90ZXh0L2l0ZXJhdG9yLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7O0FBQUEsc0RBQXVDO0FBNEJ2QyxNQUFNLGVBQWUsR0FBRztJQUNwQixHQUFHLEVBQUUsSUFBSSxFQUFtQixLQUFLO0lBQ2pDLFNBQVMsRUFBRSxJQUFJLEVBQWEsS0FBSztJQUNqQyxlQUFlLEVBQUUsSUFBSSxFQUFPLEtBQUs7SUFDakMsS0FBSyxFQUFFLElBQUksRUFBaUIsRUFBRTtDQUNqQyxDQUFBO0FBRU0sTUFBTSxvQkFBb0IsR0FBRyxDQUFDLENBQVMsRUFBVyxFQUFFO0lBQ3ZELE9BQU8sQ0FBQyxDQUFDLEdBQUcsSUFBSSxJQUFJLENBQUMsS0FBSyxlQUFlLENBQUMsR0FBRyxJQUFJLENBQUMsS0FBSyxlQUFlLENBQUMsU0FBUyxJQUFJLENBQUMsS0FBSyxlQUFlLENBQUMsZUFBZSxDQUFDLENBQUE7QUFDOUgsQ0FBQyxDQUFBO0FBRlksUUFBQSxvQkFBb0Isd0JBRWhDO0FBRUQ7OztHQUdHO0FBRUksTUFBTSxlQUFlLEdBQUcsQ0FDM0IsQ0FBUyxFQUNULEVBRUMsRUFDa0IsRUFBRTtJQUNyQixNQUFNLFVBQVUsR0FBRyxHQUFHLENBQUMsc0JBQXNCLENBQUMsQ0FBQyxDQUFDLENBQUE7SUFRaEQsSUFBSSw0QkFBNEIsR0FBRyxLQUFLLENBQUE7SUFDeEMsSUFBSSxRQUFRLEdBQUcsQ0FBQyxDQUFBO0lBQ2hCLElBQUksaUJBQWlCLEdBQWtDO1FBQ25ELE1BQU0sRUFBRSxDQUFDO1FBQ1QsUUFBUSxFQUFFLENBQUM7UUFDWCxrQkFBa0IsRUFBRSxJQUFJO0tBQzNCLENBQUE7SUFFRCxPQUFPO1FBQ0gsU0FBUyxFQUFFLEdBQUcsRUFBRTtZQUNaLE1BQU0sQ0FBQyxHQUFHLFVBQVUsQ0FBQyxnQkFBZ0IsQ0FBQyxRQUFRLENBQUMsQ0FBQTtZQUMvQyxNQUFNLEtBQUssR0FBRyxpQkFBaUIsQ0FBQTtZQUMvQixpQkFBaUIsR0FBRyxDQUFDLENBQUMsU0FBUyxDQUMzQixDQUFDLENBQUMsRUFBRSxFQUFFO2dCQUNGLE9BQU8sQ0FBQyxLQUFLLGVBQWUsQ0FBQyxTQUFTO29CQUNsQyxDQUFDLENBQUM7d0JBQ0UsTUFBTSxFQUFFLGlCQUFpQixDQUFDLElBQUksR0FBRyxDQUFDO3dCQUNsQyxRQUFRLEVBQUUsQ0FBQzt3QkFDWCxrQkFBa0IsRUFBRSxJQUFJO3FCQUMzQjtvQkFDRCxDQUFDLENBQUMsNEJBQTRCO3dCQUMxQixDQUFDLENBQUM7NEJBQ0UsTUFBTSxFQUFFLGlCQUFpQixDQUFDLElBQUksR0FBRyxDQUFDOzRCQUNsQyxRQUFRLEVBQUUsQ0FBQzs0QkFDWCxrQkFBa0IsRUFBRSxJQUFJO3lCQUMzQjt3QkFDRCxDQUFDLENBQUM7NEJBQ0UsTUFBTSxFQUFFLGlCQUFpQixDQUFDLElBQUk7NEJBQzlCLFFBQVEsRUFBRSxpQkFBaUIsQ0FBQyxRQUFRLENBQUMsR0FBRyxDQUFDLENBQUMsS0FBSyxlQUFlLENBQUMsR0FBRztnQ0FDOUQsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxVQUFVLENBQUM7Z0NBQ2hCLENBQUMsQ0FBQyxDQUFDLENBQUM7NEJBQ1Isa0JBQWtCLEVBQUUsaUJBQWlCLENBQUMsa0JBQWtCLENBQUMsS0FBSyxJQUFJO2dDQUM5RCxDQUFDLENBQUMsaUJBQWlCLENBQUMsa0JBQWtCLENBQUM7Z0NBQ3ZDLENBQUMsQ0FBQyxDQUFDLEtBQUssZUFBZSxDQUFDLEtBQUssSUFBSSxDQUFDLEtBQUssZUFBZSxDQUFDLEdBQUc7b0NBQ3RELENBQUMsQ0FBQyxJQUFJO29DQUNOLENBQUMsQ0FBQyxpQkFBaUIsQ0FBQyxRQUFRLENBQUM7eUJBQ3hDLENBQUE7WUFDYixDQUFDLEVBQ0QsR0FBRyxFQUFFLENBQUMsaUJBQWlCLENBQUMsdURBQXVEO2FBQ2xGLENBQUE7WUFFRCw0QkFBNEIsR0FBRyxDQUFDLENBQUMsU0FBUyxDQUN0QyxDQUFDLENBQUMsRUFBRSxFQUFFLENBQUMsQ0FBQyxLQUFLLGVBQWUsQ0FBQyxlQUFlLEVBQzVDLEdBQUcsRUFBRSxDQUFDLEtBQUssQ0FDZCxDQUFBO1lBQ0QsUUFBUSxJQUFJLENBQUMsQ0FBQTtRQUNqQixDQUFDO1FBQ0QsYUFBYSxFQUFFLEdBQUcsRUFBRTtZQUNoQixPQUFPLFVBQVUsQ0FBQyxnQkFBZ0IsQ0FBQyxRQUFRLENBQUMsQ0FBQyxTQUFTLENBQ2xELENBQUMsQ0FBQyxFQUFFLEVBQUUsQ0FBQyxHQUFHLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQyxFQUNqQixHQUFHLEVBQUUsQ0FBQyxHQUFHLENBQUMsT0FBTyxFQUFFLENBQ3RCLENBQUE7UUFDTCxDQUFDO1FBQ0QsWUFBWSxFQUFFLENBQUMsTUFBYyxFQUFFLEVBQUU7WUFDN0IsT0FBTyxVQUFVLENBQUMsZ0JBQWdCLENBQUMsUUFBUSxHQUFHLE1BQU0sQ0FBQyxDQUFDLFNBQVMsQ0FDM0QsQ0FBQyxDQUFDLEVBQUUsRUFBRSxDQUFDLEdBQUcsQ0FBQyxHQUFHLENBQUMsQ0FBQyxDQUFDLEVBQ2pCLEdBQUcsRUFBRSxDQUFDLEdBQUcsQ0FBQyxPQUFPLEVBQUUsQ0FDdEIsQ0FBQTtRQUNMLENBQUM7UUFDRCxXQUFXLEVBQUUsR0FBRyxFQUFFO1lBQ2QsT0FBTztnQkFDSCxVQUFVLEVBQUU7b0JBQ1IsVUFBVSxFQUFFLFFBQVE7b0JBQ3BCLFVBQVUsRUFBRTt3QkFDUixNQUFNLEVBQUUsaUJBQWlCLENBQUMsSUFBSTt3QkFDOUIsUUFBUSxFQUFFLGlCQUFpQixDQUFDLFFBQVEsQ0FBQztxQkFDeEM7aUJBQ0o7Z0JBQ0Qsa0JBQWtCLEVBQUUsaUJBQWlCLENBQUMsa0JBQWtCLENBQUMsS0FBSyxJQUFJO29CQUM5RCxDQUFDLENBQUMsaUJBQWlCLENBQUMsa0JBQWtCLENBQUM7b0JBQ3ZDLENBQUMsQ0FBQyxpQkFBaUIsQ0FBQyxRQUFRLENBQUM7YUFDcEMsQ0FBQTtRQUNMLENBQUM7S0FDSixDQUFBO0FBQ0wsQ0FBQyxDQUFBO0FBeEZZLFFBQUEsZUFBZSxtQkF3RjNCIn0=

package/dist/implementation/algorithms/refiners/token/text/refiners.d.ts

@@ -0,0 +1,9 @@
+import * as _ea from 'exupery-core-alg';
+import * as _out from "../../../../../interface/generated/pareto/schemas/token/data_types/target";
+import * as _parse_result from "../../../../../interface/generated/pareto/schemas/authoring_parse_result/data_types/target";
+import { Characters_Iterator } from "./iterator";
+export declare const Whitespace: (iterator: Characters_Iterator, abort: _ea.Abort<_parse_result.Parse_Error>) => _out.Whitespace;
+export declare const Trivia: (iterator: Characters_Iterator, abort: _ea.Abort<_parse_result.Parse_Error>) => _out.Trivia;
+export declare const Annotated_Token: (iterator: Characters_Iterator, abort: _ea.Abort<_parse_result.Parse_Error>) => _out.Annotated_Token;
+export declare const Delimited_String: (is_end_character: (character: number) => boolean, allow_newlines: boolean, iterator: Characters_Iterator, abort: _ea.Abort<_parse_result.Parse_Error>) => _out.Delimited_String;
+export declare const Tokenizer_Result: (iterator: Characters_Iterator, abort: _ea.Abort<_parse_result.Parse_Error>) => _out.Tokenizer_Result;