astn 0.113.5 → 0.113.7

package/dist/implementation/algorithms/refiners/authoring_parse_tree/token/refiners.js

@@ -0,0 +1,266 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Value = exports.Key_Value_Pairs = exports.Elements = exports.Document = exports.String = exports.Structural_Token = void 0;
+const _ea = __importStar(require("exupery-core-alg"));
+//this file contains the parser functionality, each function return a type from the 'ast' schema
+const Structural_Token = (token, //FIXME: this should be an iterator
+context) => {
+    return {
+        'trailing trivia': token['trailing trivia'],
+        'range': {
+            'start': token['start'],
+            'end': token['end']
+        }
+    };
+};
+exports.Structural_Token = Structural_Token;
+const String = (context) => {
+    const token = context.get_required_token(_ea.list_literal([['a string', null]]));
+    if (token.type[0] !== 'string') {
+        return context.unexpected_token(token, _ea.list_literal([['a string', null]]));
+    }
+    context.iterator['consume']();
+    return {
+        'range': {
+            'start': token['start'],
+            'end': token['end']
+        },
+        'value': token.type[1].value,
+        'type': token.type[1].type,
+        'trailing trivia': token['trailing trivia'],
+    };
+};
+exports.String = String;
+const Document = (context) => {
+    return {
+        'header': _ea.block(() => {
+            const token = context.get_required_token(_ea.list_literal([['!', null], ['a value', null]]));
+            if (token.type[0] !== '!') {
+                return _ea.not_set();
+            }
+            context.iterator['consume']();
+            return _ea.set({
+                '!': (0, exports.Structural_Token)(token, context),
+                'value': (0, exports.Value)(context)
+            });
+        }),
+        'content': (0, exports.Value)(context)
+    };
+};
+exports.Document = Document;
+const Elements = (end_reached, end_token, context) => {
+    return _ea.build_list(($i) => {
+        while (true) {
+            const current_token = context.get_required_token(_ea.list_literal([end_token, ['a value', null]]));
+            if (end_reached(current_token.type)) {
+                return;
+            }
+            $i['add element']({
+                'value': (0, exports.Value)(context)
+            });
+        }
+    });
+};
+exports.Elements = Elements;
+const Key_Value_Pairs = (end_reached, end_token, context) => {
+    return _ea.build_list(($i) => {
+        while (true) {
+            const current_token = context.get_required_token(_ea.list_literal([end_token, ['a string', null]]));
+            if (end_reached(current_token.type)) {
+                return;
+            }
+            $i['add element']({
+                'key': (0, exports.String)(context),
+                'value': _ea.block(() => {
+                    const candidate_colon = context.get_required_token(_ea.list_literal([['a string', null], [':', null], end_token]));
+                    if (candidate_colon.type[0] !== ':') {
+                        return _ea.not_set();
+                    }
+                    context.iterator['consume']();
+                    return _ea.set({
+                        ':': (0, exports.Structural_Token)(candidate_colon, context),
+                        'value': (0, exports.Value)(context)
+                    });
+                }),
+                ',': _ea.not_set()
+            });
+        }
+    });
+};
+exports.Key_Value_Pairs = Key_Value_Pairs;
+const Value = (context) => {
+    const token = context.get_required_token(_ea.list_literal([['a value', null]]));
+    return {
+        'type': _ea.cc(token.type, ($) => {
+            switch ($[0]) {
+                case 'string': return _ea.ss($, ($) => {
+                    return ['concrete', ['string', (0, exports.String)(context)]];
+                });
+                case '{': return _ea.ss($, ($) => {
+                    context.iterator['consume']();
+                    return ['concrete', ['indexed collection', ['dictionary', {
+                                    '{': (0, exports.Structural_Token)(token, context),
+                                    'entries': (0, exports.Key_Value_Pairs)(($) => $[0] === '}', ['}', null], context),
+                                    '}': _ea.block(() => {
+                                        const current_token = context.get_required_token(_ea.list_literal([['}', null]]));
+                                        context.iterator['consume']();
+                                        return (0, exports.Structural_Token)(current_token, context);
+                                    })
+                                }]]];
+                });
+                case '(': return _ea.ss($, ($) => {
+                    context.iterator['consume']();
+                    return ['concrete', ['indexed collection', ['verbose group', {
+                                    '(': (0, exports.Structural_Token)(token, context),
+                                    'entries': (0, exports.Key_Value_Pairs)(($) => $[0] === ')', [')', null], context),
+                                    ')': _ea.block(() => {
+                                        const current_token = context.get_required_token(_ea.list_literal([[')', null]]));
+                                        context.iterator['consume']();
+                                        return (0, exports.Structural_Token)(current_token, context);
+                                    })
+                                }]]];
+                });
+                case '[': return _ea.ss($, ($) => {
+                    context.iterator['consume']();
+                    return ['concrete', ['ordered collection', ['list', {
+                                    '[': (0, exports.Structural_Token)(token, context),
+                                    'elements': (0, exports.Elements)(($) => $[0] === ']', [']', null], context),
+                                    ']': _ea.block(() => {
+                                        const current_token = context.get_required_token(_ea.list_literal([[']', null]]));
+                                        context.iterator['consume']();
+                                        return (0, exports.Structural_Token)(current_token, context);
+                                    }),
+                                }]]];
+                });
+                case '<': return _ea.ss($, ($) => {
+                    context.iterator['consume']();
+                    return ['concrete', ['ordered collection', ['concise group', {
+                                    '<': (0, exports.Structural_Token)(token, context),
+                                    'elements': (0, exports.Elements)(($) => $[0] === '>', ['>', null], context),
+                                    '>': _ea.block(() => {
+                                        const current_token = context.get_required_token(_ea.list_literal([['>', null]]));
+                                        context.iterator['consume']();
+                                        return (0, exports.Structural_Token)(current_token, context);
+                                    }),
+                                }]]];
+                });
+                case '@': return _ea.ss($, ($) => {
+                    context.iterator['consume']();
+                    return ['include', {
+                            '@': (0, exports.Structural_Token)(token, context),
+                            'path': (0, exports.String)(context)
+                        }];
+                });
+                case '~': return _ea.ss($, ($) => {
+                    context.iterator['consume']();
+                    return ['concrete', ['not set', {
+                                '~': (0, exports.Structural_Token)(token, context),
+                            }]];
+                });
+                case '|': return _ea.ss($, ($) => {
+                    context.iterator['consume']();
+                    const token = context.get_required_token(_ea.list_literal([['a value', null], ['#', null]]));
+                    return ['concrete', ['tagged value', {
+                                '|': (0, exports.Structural_Token)(token, context),
+                                'status': _ea.cc(token.type, ($) => {
+                                    switch ($[0]) {
+                                        case 'string': return _ea.ss($, ($) => {
+                                            return ['set', {
+                                                    'state': (0, exports.String)(context),
+                                                    'value': (0, exports.Value)(context)
+                                                }];
+                                        });
+                                        case '#': return _ea.ss($, ($) => {
+                                            context.iterator['consume']();
+                                            return ['missing data', {
+                                                    '#': (0, exports.Structural_Token)(token, context),
+                                                }];
+                                        });
+                                        default: return context.unexpected_token(token, _ea.list_literal([
+                                            ['a value', null],
+                                            ['#', null],
+                                        ]));
+                                    }
+                                })
+                            }]];
+                });
+                case '*': return _ea.ss($, ($) => {
+                    context.iterator['consume']();
+                    return ['concrete', ['set optional value', {
+                                '*': (0, exports.Structural_Token)(token, context),
+                                'value': (0, exports.Value)(context)
+                            }]];
+                });
+                case '#': return _ea.ss($, ($) => {
+                    context.iterator['consume']();
+                    return ['missing data', {
+                            '#': (0, exports.Structural_Token)(token, context),
+                        }];
+                });
+                //unexpected tokens
+                case '!': return _ea.ss($, ($) => {
+                    context.iterator['consume']();
+                    return context.unexpected_token(token, _ea.list_literal([
+                        ['a value', null]
+                    ]));
+                });
+                case ':': return _ea.ss($, ($) => {
+                    context.iterator['consume']();
+                    return context.unexpected_token(token, _ea.list_literal([
+                        ['a value', null]
+                    ]));
+                });
+                case ')': return _ea.ss($, ($) => {
+                    context.iterator['consume']();
+                    return context.unexpected_token(token, _ea.list_literal([
+                        ['a value', null]
+                    ]));
+                });
+                case '>': return _ea.ss($, ($) => {
+                    context.iterator['consume']();
+                    return context.unexpected_token(token, _ea.list_literal([
+                        ['a value', null]
+                    ]));
+                });
+                case ']': return _ea.ss($, ($) => {
+                    context.iterator['consume']();
+                    return context.unexpected_token(token, _ea.list_literal([
+                        ['a value', null]
+                    ]));
+                });
+                case '}': return _ea.ss($, ($) => {
+                    context.iterator['consume']();
+                    return context.unexpected_token(token, _ea.list_literal([
+                        ['a value', null]
+                    ]));
+                });
+                default: return _ea.au($[0]);
+            }
+        })
+    };
+};
+exports.Value = Value;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicmVmaW5lcnMuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi8uLi8uLi8uLi9zcmMvaW1wbGVtZW50YXRpb24vYWxnb3JpdGhtcy9yZWZpbmVycy9hdXRob3JpbmdfcGFyc2VfdHJlZS90b2tlbi9yZWZpbmVycy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7OztBQUFBLHNEQUF1QztBQVN2QyxnR0FBZ0c7QUFFekYsTUFBTSxnQkFBZ0IsR0FBRyxDQUM1QixLQUE4QixFQUFFLG1DQUFtQztBQUNuRSxPQUE2QixFQUNMLEVBQUU7SUFDMUIsT0FBTztRQUNILGlCQUFpQixFQUFFLEtBQUssQ0FBQyxpQkFBaUIsQ0FBQztRQUMzQyxPQUFPLEVBQUU7WUFDTCxPQUFPLEVBQUUsS0FBSyxDQUFDLE9BQU8sQ0FBQztZQUN2QixLQUFLLEVBQUUsS0FBSyxDQUFDLEtBQUssQ0FBQztTQUN0QjtLQUNKLENBQUE7QUFDTCxDQUFDLENBQUE7QUFYWSxRQUFBLGdCQUFnQixvQkFXNUI7QUFFTSxNQUFNLE1BQU0sR0FBRyxDQUNsQixPQUE2QixFQUNmLEVBQUU7SUFDaEIsTUFBTSxLQUFLLEdBQUcsT0FBTyxDQUFDLGtCQUFrQixDQUFDLEdBQUcsQ0FBQyxZQUFZLENBQUMsQ0FBQyxDQUFDLFVBQVUsRUFBRSxJQUFJLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQTtJQUNoRixJQUFJLEtBQUssQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLEtBQUssUUFBUSxFQUFFLENBQUM7UUFDN0IsT0FBTyxPQUFPLENBQUMsZ0JBQWdCLENBQUMsS0FBSyxFQUFFLEdBQUcsQ0FBQyxZQUFZLENBQUMsQ0FBQyxDQUFDLFVBQVUsRUFBRSxJQUFJLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQTtJQUNsRixDQUFDO0lBQ0QsT0FBTyxDQUFDLFFBQVEsQ0FBQyxTQUFTLENBQUMsRUFBRSxDQUFBO0lBQzdCLE9BQU87UUFDSCxPQUFPLEVBQUU7WUFDTCxPQUFPLEVBQUUsS0FBSyxDQUFDLE9BQU8sQ0FBQztZQUN2QixLQUFLLEVBQUUsS0FBSyxDQUFDLEtBQUssQ0FBQztTQUN0QjtRQUNELE9BQU8sRUFBRSxLQUFLLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxDQUFDLEtBQUs7UUFDNUIsTUFBTSxFQUFFLEtBQUssQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLENBQUMsSUFBSTtRQUMxQixpQkFBaUIsRUFBRSxLQUFLLENBQUMsaUJBQWlCLENBQUM7S0FDOUMsQ0FBQTtBQUNMLENBQUMsQ0FBQTtBQWpCWSxRQUFBLE1BQU0sVUFpQmxCO0FBRU0sTUFBTSxRQUFRLEdBQUcsQ0FDcEIsT0FBNkIsRUFDYixFQUFFO0lBQ2xCLE9BQU87UUFDSCxRQUFRLEVBQUUsR0FBRyxDQUFDLEtBQUssQ0FBQyxHQUFHLEVBQUU7WUFDckIsTUFBTSxLQUFLLEdBQUcsT0FBTyxDQUFDLGtCQUFrQixDQUFDLEdBQUcsQ0FBQyxZQUFZLENBQUMsQ0FBQyxDQUFDLEdBQUcsRUFBRSxJQUFJLENBQUMsRUFBRSxDQUFDLFNBQVMsRUFBRSxJQUFJLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQTtZQUM1RixJQUFJLEtBQUssQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLEtBQUssR0FBRyxFQUFFLENBQUM7Z0JBQ3hCLE9BQU8sR0FBRyxDQUFDLE9BQU8sRUFBRSxDQUFBO1lBQ3hCLENBQUM7WUFDRCxPQUFPLENBQUMsUUFBUSxDQUFDLFNBQVMsQ0FBQyxFQUFFLENBQUE7WUFDN0IsT0FBTyxHQUFHLENBQUMsR0FBRyxDQUFDO2dCQUNYLEdBQUcsRUFBRSxJQUFBLHdCQUFnQixFQUFDLEtBQUssRUFBRSxPQUFPLENBQUM7Z0JBQ3JDLE9BQU8sRUFBRSxJQUFBLGFBQUssRUFBQyxPQUFPLENBQUM7YUFDMUIsQ0FBQyxDQUFBO1FBQ04sQ0FBQyxDQUFDO1FBQ0YsU0FBUyxFQUFFLElBQUEsYUFBSyxFQUFDLE9BQU8sQ0FBQztLQUM1QixDQUFBO0FBQ0wsQ0FBQyxDQUFBO0FBakJZLFFBQUEsUUFBUSxZQWlCcEI7QUFFTSxNQUFNLFFBQVEsR0FBRyxDQUNwQixXQUErQyxFQUMvQyxTQUFnRSxFQUNoRSxPQUE2QixFQUNiLEVBQUU7SUFDbEIsT0FBTyxHQUFHLENBQUMsVUFBVSxDQUFxQixDQUFDLEVBQUUsRUFBUSxFQUFFO1FBQ25ELE9BQU8sSUFBSSxFQUFFLENBQUM7WUFDVixNQUFNLGFBQWEsR0FBRyxPQUFPLENBQUMsa0JBQWtCLENBQUMsR0FBRyxDQUFDLFlBQVksQ0FBQyxDQUFDLFNBQVMsRUFBRSxDQUFDLFNBQVMsRUFBRSxJQUFJLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQTtZQUNsRyxJQUFJLFdBQVcsQ0FBQyxhQUFhLENBQUMsSUFBSSxDQUFDLEVBQUUsQ0FBQztnQkFDbEMsT0FBTTtZQUNWLENBQUM7WUFDRCxFQUFFLENBQUMsYUFBYSxDQUFDLENBQUM7Z0JBQ2QsT0FBTyxFQUFFLElBQUEsYUFBSyxFQUFDLE9BQU8sQ0FBQzthQUMxQixDQUFDLENBQUE7UUFDTixDQUFDO0lBQ0wsQ0FBQyxDQUFDLENBQUE7QUFDTixDQUFDLENBQUE7QUFoQlksUUFBQSxRQUFRLFlBZ0JwQjtBQUVNLE1BQU0sZUFBZSxHQUFHLENBQzNCLFdBQStDLEVBQy9DLFNBQWdFLEVBQ2hFLE9BQTZCLEVBQ04sRUFBRTtJQUN6QixPQUFPLEdBQUcsQ0FBQyxVQUFVLENBQTRCLENBQUMsRUFBRSxFQUFRLEVBQUU7UUFDMUQsT0FBTyxJQUFJLEVBQUUsQ0FBQztZQUNWLE1BQU0sYUFBYSxHQUFHLE9BQU8sQ0FBQyxrQkFBa0IsQ0FBQyxHQUFHLENBQUMsWUFBWSxDQUFDLENBQUMsU0FBUyxFQUFFLENBQUMsVUFBVSxFQUFFLElBQUksQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFBO1lBQ25HLElBQUksV0FBVyxDQUFDLGFBQWEsQ0FBQyxJQUFJLENBQUMsRUFBRSxDQUFDO2dCQUNsQyxPQUFNO1lBQ1YsQ0FBQztZQUVELEVBQUUsQ0FBQyxhQUFhLENBQUMsQ0FBQztnQkFDZCxLQUFLLEVBQUUsSUFBQSxjQUFNLEVBQUMsT0FBTyxDQUFDO2dCQUN0QixPQUFPLEVBQUUsR0FBRyxDQUFDLEtBQUssQ0FBQyxHQUFHLEVBQUU7b0JBQ3BCLE1BQU0sZUFBZSxHQUFHLE9BQU8sQ0FBQyxrQkFBa0IsQ0FBQyxHQUFHLENBQUMsWUFBWSxDQUFDLENBQUMsQ0FBQyxVQUFVLEVBQUUsSUFBSSxDQUFDLEVBQUUsQ0FBQyxHQUFHLEVBQUUsSUFBSSxDQUFDLEVBQUUsU0FBUyxDQUFDLENBQUMsQ0FBQyxDQUFBO29CQUNsSCxJQUFJLGVBQWUsQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLEtBQUssR0FBRyxFQUFFLENBQUM7d0JBQ2xDLE9BQU8sR0FBRyxDQUFDLE9BQU8sRUFBRSxDQUFBO29CQUN4QixDQUFDO29CQUNELE9BQU8sQ0FBQyxRQUFRLENBQUMsU0FBUyxDQUFDLEVBQUUsQ0FBQTtvQkFFN0IsT0FBTyxHQUFHLENBQUMsR0FBRyxDQUFDO3dCQUNYLEdBQUcsRUFBRSxJQUFBLHdCQUFnQixFQUFDLGVBQWUsRUFBRSxPQUFPLENBQUM7d0JBQy9DLE9BQU8sRUFBRSxJQUFBLGFBQUssRUFBQyxPQUFPLENBQUM7cUJBQzFCLENBQUMsQ0FBQTtnQkFDTixDQUFDLENBQUM7Z0JBQ0YsR0FBRyxFQUFFLEdBQUcsQ0FBQyxPQUFPLEVBQUU7YUFDckIsQ0FBQyxDQUFBO1FBQ04sQ0FBQztJQUNMLENBQUMsQ0FBQyxDQUFBO0FBQ04sQ0FBQyxDQUFBO0FBOUJZLFFBQUEsZUFBZSxtQkE4QjNCO0FBRU0sTUFBTSxLQUFLLEdBQUcsQ0FDakIsT0FBNkIsRUFDaEIsRUFBRTtJQUNmLE1BQU0sS0FBSyxHQUFHLE9BQU8sQ0FBQyxrQkFBa0IsQ0FBQyxHQUFHLENBQUMsWUFBWSxDQUFDLENBQUMsQ0FBQyxTQUFTLEVBQUUsSUFBSSxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUE7SUFDL0UsT0FBTztRQUNILE1BQU0sRUFBRSxHQUFHLENBQUMsRUFBRSxDQUFDLEtBQUssQ0FBQyxJQUFJLEVBQUUsQ0FBQyxDQUFDLEVBQXVCLEVBQUU7WUFFbEQsUUFBUSxDQUFDLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQztnQkFDWCxLQUFLLFFBQVEsQ0FBQyxDQUFDLE9BQU8sR0FBRyxDQUFDLEVBQUUsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxDQUFDLEVBQXVCLEVBQUU7b0JBRXZELE9BQU8sQ0FBQyxVQUFVLEVBQUUsQ0FBQyxRQUFRLEVBQUUsSUFBQSxjQUFNLEVBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQyxDQUFBO2dCQUNwRCxDQUFDLENBQUMsQ0FBQTtnQkFDRixLQUFLLEdBQUcsQ0FBQyxDQUFDLE9BQU8sR0FBRyxDQUFDLEVBQUUsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxDQUFDLEVBQUUsRUFBRTtvQkFDN0IsT0FBTyxDQUFDLFFBQVEsQ0FBQyxTQUFTLENBQUMsRUFBRSxDQUFBO29CQUM3QixPQUFPLENBQUMsVUFBVSxFQUFFLENBQUMsb0JBQW9CLEVBQUUsQ0FBQyxZQUFZLEVBQUU7b0NBQ3RELEdBQUcsRUFBRSxJQUFBLHdCQUFnQixFQUFDLEtBQUssRUFBRSxPQUFPLENBQUM7b0NBQ3JDLFNBQVMsRUFBRSxJQUFBLHVCQUFlLEVBQUMsQ0FBQyxDQUFDLEVBQUUsRUFBRSxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsS0FBSyxHQUFHLEVBQUUsQ0FBQyxHQUFHLEVBQUUsSUFBSSxDQUFDLEVBQUUsT0FBTyxDQUFDO29DQUNyRSxHQUFHLEVBQUUsR0FBRyxDQUFDLEtBQUssQ0FBQyxHQUFHLEVBQUU7d0NBQ2hCLE1BQU0sYUFBYSxHQUFHLE9BQU8sQ0FBQyxrQkFBa0IsQ0FBQyxHQUFHLENBQUMsWUFBWSxDQUFDLENBQUMsQ0FBQyxHQUFHLEVBQUUsSUFBSSxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUE7d0NBQ2pGLE9BQU8sQ0FBQyxRQUFRLENBQUMsU0FBUyxDQUFDLEVBQUUsQ0FBQTt3Q0FDN0IsT0FBTyxJQUFBLHdCQUFnQixFQUFDLGFBQWEsRUFBRSxPQUFPLENBQUMsQ0FBQTtvQ0FDbkQsQ0FBQyxDQUFDO2lDQUNMLENBQUMsQ0FBQyxDQUFDLENBQUE7Z0JBQ1IsQ0FBQyxDQUFDLENBQUE7Z0JBQ0YsS0FBSyxHQUFHLENBQUMsQ0FBQyxPQUFPLEdBQUcsQ0FBQyxFQUFFLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxFQUFFLEVBQUU7b0JBQzdCLE9BQU8sQ0FBQyxRQUFRLENBQUMsU0FBUyxDQUFDLEVBQUUsQ0FBQTtvQkFDN0IsT0FBTyxDQUFDLFVBQVUsRUFBRSxDQUFDLG9CQUFvQixFQUFFLENBQUMsZUFBZSxFQUFFO29DQUN6RCxHQUFHLEVBQUUsSUFBQSx3QkFBZ0IsRUFBQyxLQUFLLEVBQUUsT0FBTyxDQUFDO29DQUNyQyxTQUFTLEVBQUUsSUFBQSx1QkFBZSxFQUFDLENBQUMsQ0FBQyxFQUFFLEVBQUUsQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLEtBQUssR0FBRyxFQUFFLENBQUMsR0FBRyxFQUFFLElBQUksQ0FBQyxFQUFFLE9BQU8sQ0FBQztvQ0FDckUsR0FBRyxFQUFFLEdBQUcsQ0FBQyxLQUFLLENBQUMsR0FBRyxFQUFFO3dDQUNoQixNQUFNLGFBQWEsR0FBRyxPQUFPLENBQUMsa0JBQWtCLENBQUMsR0FBRyxDQUFDLFlBQVksQ0FBQyxDQUFDLENBQUMsR0FBRyxFQUFFLElBQUksQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFBO3dDQUNqRixPQUFPLENBQUMsUUFBUSxDQUFDLFNBQVMsQ0FBQyxFQUFFLENBQUE7d0NBQzdCLE9BQU8sSUFBQSx3QkFBZ0IsRUFBQyxhQUFhLEVBQUUsT0FBTyxDQUFDLENBQUE7b0NBQ25ELENBQUMsQ0FBQztpQ0FDTCxDQUFDLENBQUMsQ0FBQyxDQUFBO2dCQUNSLENBQUMsQ0FBQyxDQUFBO2dCQUNGLEtBQUssR0FBRyxDQUFDLENBQUMsT0FBTyxHQUFHLENBQUMsRUFBRSxDQUFDLENBQUMsRUFBRSxDQUFDLENBQUMsRUFBdUIsRUFBRTtvQkFDbEQsT0FBTyxDQUFDLFFBQVEsQ0FBQyxTQUFTLENBQUMsRUFBRSxDQUFBO29CQUM3QixPQUFPLENBQUMsVUFBVSxFQUFFLENBQUMsb0JBQW9CLEVBQUUsQ0FBQyxNQUFNLEVBQUU7b0NBQ2hELEdBQUcsRUFBRSxJQUFBLHdCQUFnQixFQUFDLEtBQUssRUFBRSxPQUFPLENBQUM7b0NBQ3JDLFVBQVUsRUFBRSxJQUFBLGdCQUFRLEVBQUMsQ0FBQyxDQUFDLEVBQUUsRUFBRSxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsS0FBSyxHQUFHLEVBQUUsQ0FBQyxHQUFHLEVBQUUsSUFBSSxDQUFDLEVBQUUsT0FBTyxDQUFDO29DQUMvRCxHQUFHLEVBQUUsR0FBRyxDQUFDLEtBQUssQ0FBQyxHQUFHLEVBQUU7d0NBQ2hCLE1BQU0sYUFBYSxHQUFHLE9BQU8sQ0FBQyxrQkFBa0IsQ0FBQyxHQUFHLENBQUMsWUFBWSxDQUFDLENBQUMsQ0FBQyxHQUFHLEVBQUUsSUFBSSxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUE7d0NBQ2pGLE9BQU8sQ0FBQyxRQUFRLENBQUMsU0FBUyxDQUFDLEVBQUUsQ0FBQTt3Q0FDN0IsT0FBTyxJQUFBLHdCQUFnQixFQUFDLGFBQWEsRUFBRSxPQUFPLENBQUMsQ0FBQTtvQ0FDbkQsQ0FBQyxDQUFDO2lDQUNMLENBQUMsQ0FBQyxDQUFDLENBQUE7Z0JBQ1IsQ0FBQyxDQUFDLENBQUE7Z0JBQ0YsS0FBSyxHQUFHLENBQUMsQ0FBQyxPQUFPLEdBQUcsQ0FBQyxFQUFFLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxFQUF1QixFQUFFO29CQUNsRCxPQUFPLENBQUMsUUFBUSxDQUFDLFNBQVMsQ0FBQyxFQUFFLENBQUE7b0JBQzdCLE9BQU8sQ0FBQyxVQUFVLEVBQUUsQ0FBQyxvQkFBb0IsRUFBRSxDQUFDLGVBQWUsRUFBRTtvQ0FDekQsR0FBRyxFQUFFLElBQUEsd0JBQWdCLEVBQUMsS0FBSyxFQUFFLE9BQU8sQ0FBQztvQ0FDckMsVUFBVSxFQUFFLElBQUEsZ0JBQVEsRUFBQyxDQUFDLENBQUMsRUFBRSxFQUFFLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxLQUFLLEdBQUcsRUFBRSxDQUFDLEdBQUcsRUFBRSxJQUFJLENBQUMsRUFBRSxPQUFPLENBQUM7b0NBQy9ELEdBQUcsRUFBRSxHQUFHLENBQUMsS0FBSyxDQUFDLEdBQUcsRUFBRTt3Q0FDaEIsTUFBTSxhQUFhLEdBQUcsT0FBTyxDQUFDLGtCQUFrQixDQUFDLEdBQUcsQ0FBQyxZQUFZLENBQUMsQ0FBQyxDQUFDLEdBQUcsRUFBRSxJQUFJLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQTt3Q0FDakYsT0FBTyxDQUFDLFFBQVEsQ0FBQyxTQUFTLENBQUMsRUFBRSxDQUFBO3dDQUM3QixPQUFPLElBQUEsd0JBQWdCLEVBQUMsYUFBYSxFQUFFLE9BQU8sQ0FBQyxDQUFBO29DQUNuRCxDQUFDLENBQUM7aUNBQ0wsQ0FBQyxDQUFDLENBQUMsQ0FBQTtnQkFDUixDQUFDLENBQUMsQ0FBQTtnQkFDRixLQUFLLEdBQUcsQ0FBQyxDQUFDLE9BQU8sR0FBRyxDQUFDLEVBQUUsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxDQUFDLEVBQUUsRUFBRTtvQkFDN0IsT0FBTyxDQUFDLFFBQVEsQ0FBQyxTQUFTLENBQUMsRUFBRSxDQUFBO29CQUM3QixPQUFPLENBQUMsU0FBUyxFQUFFOzRCQUNmLEdBQUcsRUFBRSxJQUFBLHdCQUFnQixFQUFDLEtBQUssRUFBRSxPQUFPLENBQUM7NEJBQ3JDLE1BQU0sRUFBRSxJQUFBLGNBQU0sRUFBQyxPQUFPLENBQUM7eUJBQzFCLENBQUMsQ0FBQTtnQkFDTixDQUFDLENBQUMsQ0FBQTtnQkFDRixLQUFLLEdBQUcsQ0FBQyxDQUFDLE9BQU8sR0FBRyxDQUFDLEVBQUUsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxDQUFDLEVBQUUsRUFBRTtvQkFDN0IsT0FBTyxDQUFDLFFBQVEsQ0FBQyxTQUFTLENBQUMsRUFBRSxDQUFBO29CQUM3QixPQUFPLENBQUMsVUFBVSxFQUFFLENBQUMsU0FBUyxFQUFFO2dDQUM1QixHQUFHLEVBQUUsSUFBQSx3QkFBZ0IsRUFBQyxLQUFLLEVBQUUsT0FBTyxDQUFDOzZCQUN4QyxDQUFDLENBQUMsQ0FBQTtnQkFDUCxDQUFDLENBQUMsQ0FBQTtnQkFDRixLQUFLLEdBQUcsQ0FBQyxDQUFDLE9BQU8sR0FBRyxDQUFDLEVBQUUsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxDQUFDLEVBQUUsRUFBRTtvQkFDN0IsT0FBTyxDQUFDLFFBQVEsQ0FBQyxTQUFTLENBQUMsRUFBRSxDQUFBO29CQUM3QixNQUFNLEtBQUssR0FBRyxPQUFPLENBQUMsa0JBQWtCLENBQUMsR0FBRyxDQUFDLFlBQVksQ0FBQyxDQUFDLENBQUMsU0FBUyxFQUFFLElBQUksQ0FBQyxFQUFFLENBQUMsR0FBRyxFQUFFLElBQUksQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFBO29CQUU1RixPQUFPLENBQUMsVUFBVSxFQUFFLENBQUMsY0FBYyxFQUFFO2dDQUNqQyxHQUFHLEVBQUUsSUFBQSx3QkFBZ0IsRUFBQyxLQUFLLEVBQUUsT0FBTyxDQUFDO2dDQUNyQyxRQUFRLEVBQUUsR0FBRyxDQUFDLEVBQUUsQ0FBQyxLQUFLLENBQUMsSUFBSSxFQUFFLENBQUMsQ0FBQyxFQUFpRCxFQUFFO29DQUM5RSxRQUFRLENBQUMsQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDO3dDQUNYLEtBQUssUUFBUSxDQUFDLENBQUMsT0FBTyxHQUFHLENBQUMsRUFBRSxDQUFDLENBQUMsRUFBRSxDQUFDLENBQUMsRUFBRSxFQUFFOzRDQUNsQyxPQUFPLENBQUMsS0FBSyxFQUFFO29EQUNYLE9BQU8sRUFBRSxJQUFBLGNBQU0sRUFBQyxPQUFPLENBQUM7b0RBQ3hCLE9BQU8sRUFBRSxJQUFBLGFBQUssRUFBQyxPQUFPLENBQUM7aURBQzFCLENBQUMsQ0FBQTt3Q0FDTixDQUFDLENBQUMsQ0FBQTt3Q0FDRixLQUFLLEdBQUcsQ0FBQyxDQUFDLE9BQU8sR0FBRyxDQUFDLEVBQUUsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxDQUFDLEVBQUUsRUFBRTs0Q0FDN0IsT0FBTyxDQUFDLFFBQVEsQ0FBQyxTQUFTLENBQUMsRUFBRSxDQUFBOzRDQUM3QixPQUFPLENBQUMsY0FBYyxFQUFFO29EQUNwQixHQUFHLEVBQUUsSUFBQSx3QkFBZ0IsRUFBQyxLQUFLLEVBQUUsT0FBTyxDQUFDO2lEQUN4QyxDQUFDLENBQUE7d0NBQ04sQ0FBQyxDQUFDLENBQUE7d0NBQ0YsT0FBTyxDQUFDLENBQUMsT0FBTyxPQUFPLENBQUMsZ0JBQWdCLENBQUMsS0FBSyxFQUFFLEdBQUcsQ0FBQyxZQUFZLENBQUM7NENBQzdELENBQUMsU0FBUyxFQUFFLElBQUksQ0FBQzs0Q0FDakIsQ0FBQyxHQUFHLEVBQUUsSUFBSSxDQUFDO3lDQUNkLENBQUMsQ0FBQyxDQUFBO29DQUNQLENBQUM7Z0NBQ0wsQ0FBQyxDQUFDOzZCQUNMLENBQUMsQ0FBQyxDQUFBO2dCQUNQLENBQUMsQ0FBQyxDQUFBO2dCQUNGLEtBQUssR0FBRyxDQUFDLENBQUMsT0FBTyxHQUFHLENBQUMsRUFBRSxDQUFDLENBQUMsRUFBRSxDQUFDLENBQUMsRUFBRSxFQUFFO29CQUM3QixPQUFPLENBQUMsUUFBUSxDQUFDLFNBQVMsQ0FBQyxFQUFFLENBQUE7b0JBQzdCLE9BQU8sQ0FBQyxVQUFVLEVBQUUsQ0FBQyxvQkFBb0IsRUFBRTtnQ0FDdkMsR0FBRyxFQUFFLElBQUEsd0JBQWdCLEVBQUMsS0FBSyxFQUFFLE9BQU8sQ0FBQztnQ0FDckMsT0FBTyxFQUFFLElBQUEsYUFBSyxFQUFDLE9BQU8sQ0FBQzs2QkFDMUIsQ0FBQyxDQUFDLENBQUE7Z0JBQ1AsQ0FBQyxDQUFDLENBQUE7Z0JBQ0YsS0FBSyxHQUFHLENBQUMsQ0FBQyxPQUFPLEdBQUcsQ0FBQyxFQUFFLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxFQUFFLEVBQUU7b0JBQzdCLE9BQU8sQ0FBQyxRQUFRLENBQUMsU0FBUyxDQUFDLEVBQUUsQ0FBQTtvQkFDN0IsT0FBTyxDQUFDLGNBQWMsRUFBRTs0QkFDcEIsR0FBRyxFQUFFLElBQUEsd0JBQWdCLEVBQUMsS0FBSyxFQUFFLE9BQU8sQ0FBQzt5QkFDeEMsQ0FBQyxDQUFBO2dCQUNOLENBQUMsQ0FBQyxDQUFBO2dCQUVGLG1CQUFtQjtnQkFFbkIsS0FBSyxHQUFHLENBQUMsQ0FBQyxPQUFPLEdBQUcsQ0FBQyxFQUFFLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxFQUFFLEVBQUU7b0JBQzdCLE9BQU8sQ0FBQyxRQUFRLENBQUMsU0FBUyxDQUFDLEVBQUUsQ0FBQTtvQkFDN0IsT0FBTyxPQUFPLENBQUMsZ0JBQWdCLENBQUMsS0FBSyxFQUFFLEdBQUcsQ0FBQyxZQUFZLENBQUM7d0JBQ3BELENBQUMsU0FBUyxFQUFFLElBQUksQ0FBQztxQkFDcEIsQ0FBQyxDQUFDLENBQUE7Z0JBQ1AsQ0FBQyxDQUFDLENBQUE7Z0JBQ0YsS0FBSyxHQUFHLENBQUMsQ0FBQyxPQUFPLEdBQUcsQ0FBQyxFQUFFLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxFQUFFLEVBQUU7b0JBQzdCLE9BQU8sQ0FBQyxRQUFRLENBQUMsU0FBUyxDQUFDLEVBQUUsQ0FBQTtvQkFDN0IsT0FBTyxPQUFPLENBQUMsZ0JBQWdCLENBQUMsS0FBSyxFQUFFLEdBQUcsQ0FBQyxZQUFZLENBQUM7d0JBQ3BELENBQUMsU0FBUyxFQUFFLElBQUksQ0FBQztxQkFDcEIsQ0FBQyxDQUFDLENBQUE7Z0JBQ1AsQ0FBQyxDQUFDLENBQUE7Z0JBQ0YsS0FBSyxHQUFHLENBQUMsQ0FBQyxPQUFPLEdBQUcsQ0FBQyxFQUFFLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxFQUFFLEVBQUU7b0JBQzdCLE9BQU8sQ0FBQyxRQUFRLENBQUMsU0FBUyxDQUFDLEVBQUUsQ0FBQTtvQkFDN0IsT0FBTyxPQUFPLENBQUMsZ0JBQWdCLENBQUMsS0FBSyxFQUFFLEdBQUcsQ0FBQyxZQUFZLENBQUM7d0JBQ3BELENBQUMsU0FBUyxFQUFFLElBQUksQ0FBQztxQkFDcEIsQ0FBQyxDQUFDLENBQUE7Z0JBQ1AsQ0FBQyxDQUFDLENBQUE7Z0JBQ0YsS0FBSyxHQUFHLENBQUMsQ0FBQyxPQUFPLEdBQUcsQ0FBQyxFQUFFLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxFQUFFLEVBQUU7b0JBQzdCLE9BQU8sQ0FBQyxRQUFRLENBQUMsU0FBUyxDQUFDLEVBQUUsQ0FBQTtvQkFDN0IsT0FBTyxPQUFPLENBQUMsZ0JBQWdCLENBQUMsS0FBSyxFQUFFLEdBQUcsQ0FBQyxZQUFZLENBQUM7d0JBQ3BELENBQUMsU0FBUyxFQUFFLElBQUksQ0FBQztxQkFDcEIsQ0FBQyxDQUFDLENBQUE7Z0JBQ1AsQ0FBQyxDQUFDLENBQUE7Z0JBQ0YsS0FBSyxHQUFHLENBQUMsQ0FBQyxPQUFPLEdBQUcsQ0FBQyxFQUFFLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxFQUFFLEVBQUU7b0JBQzdCLE9BQU8sQ0FBQyxRQUFRLENBQUMsU0FBUyxDQUFDLEVBQUUsQ0FBQTtvQkFDN0IsT0FBTyxPQUFPLENBQUMsZ0JBQWdCLENBQUMsS0FBSyxFQUFFLEdBQUcsQ0FBQyxZQUFZLENBQUM7d0JBQ3BELENBQUMsU0FBUyxFQUFFLElBQUksQ0FBQztxQkFDcEIsQ0FBQyxDQUFDLENBQUE7Z0JBQ1AsQ0FBQyxDQUFDLENBQUE7Z0JBQ0YsS0FBSyxHQUFHLENBQUMsQ0FBQyxPQUFPLEdBQUcsQ0FBQyxFQUFFLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxFQUFFLEVBQUU7b0JBQzdCLE9BQU8sQ0FBQyxRQUFRLENBQUMsU0FBUyxDQUFDLEVBQUUsQ0FBQTtvQkFDN0IsT0FBTyxPQUFPLENBQUMsZ0JBQWdCLENBQUMsS0FBSyxFQUFFLEdBQUcsQ0FBQyxZQUFZLENBQUM7d0JBQ3BELENBQUMsU0FBUyxFQUFFLElBQUksQ0FBQztxQkFDcEIsQ0FBQyxDQUFDLENBQUE7Z0JBQ1AsQ0FBQyxDQUFDLENBQUE7Z0JBRUYsT0FBTyxDQUFDLENBQUMsT0FBTyxHQUFHLENBQUMsRUFBRSxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFBO1lBQ2hDLENBQUM7UUFDTCxDQUFDLENBQUM7S0FDTCxDQUFBO0FBQ0wsQ0FBQyxDQUFBO0FBOUpZLFFBQUEsS0FBSyxTQThKakIifQ==

package/dist/implementation/algorithms/refiners/token/text/context.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/implementation/algorithms/refiners/token/text/context.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+// export type Refinement_Context = {
+//     iterator: Characters_Iterator
+//     lexer_error: (
+//         type: d_parse_result.Parse_Error._type.SG.lexer,
+//         range: _source.Range
+//     ) => never
+// }
+// export const create_context = (
+//     token_iterator: Characters_Iterator,
+//     abort: _ea.Abort<d_parse_result.Parse_Error>,
+// ): Refinement_Context => {
+//     return {
+//         iterator: token_iterator,
+//         lexer_error: (type, range) => {
+//             abort(sh.lexer_error(
+//                 type,
+//                 range
+//             ))
+//         }
+//     }
+// }
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY29udGV4dC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uLy4uLy4uLy4uL3NyYy9pbXBsZW1lbnRhdGlvbi9hbGdvcml0aG1zL3JlZmluZXJzL3Rva2VuL3RleHQvY29udGV4dC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOztBQVNBLHFDQUFxQztBQUNyQyxvQ0FBb0M7QUFDcEMscUJBQXFCO0FBQ3JCLDJEQUEyRDtBQUMzRCwrQkFBK0I7QUFDL0IsaUJBQWlCO0FBQ2pCLElBQUk7QUFFSixrQ0FBa0M7QUFDbEMsMkNBQTJDO0FBQzNDLG9EQUFvRDtBQUNwRCw2QkFBNkI7QUFDN0IsZUFBZTtBQUNmLG9DQUFvQztBQUNwQywwQ0FBMEM7QUFDMUMsb0NBQW9DO0FBQ3BDLHdCQUF3QjtBQUN4Qix3QkFBd0I7QUFDeEIsaUJBQWlCO0FBQ2pCLFlBQVk7QUFDWixRQUFRO0FBQ1IsSUFBSSJ9

package/dist/implementation/algorithms/refiners/token/text/iterator.d.ts

@@ -0,0 +1,27 @@
+import { Iterator } from '../../../../../core/iterator';
+export type Iterator_Location = {
+    'absolute': number;
+    'relative': {
+        'line': number;
+        'column': number;
+    };
+};
+export type Token_Iterator_State = {
+    'location': Iterator_Location;
+    /**
+     * if no non-whitespace character has been found yet on the current line,
+     * this will return the current column,
+     * otherwise it will return the offset of that first non-whitespace character
+     * (which is the indentation of the line)
+     */
+    'line indentation': number;
+};
+export type Characters_Iterator = Iterator<number, Token_Iterator_State>;
+export declare const is_control_character: ($: number) => boolean;
+/**
+ * Creates a string iterator that allows iterating over characters in a string,
+ * while keeping track of line numbers, columns, and line indentation.
+ */
+export declare const create_iterator: ($: string, $p: {
+    "tab size": number;
+}) => Characters_Iterator;

package/dist/implementation/algorithms/refiners/token/text/iterator.js

@@ -0,0 +1,107 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.create_iterator = exports.is_control_character = void 0;
+const _ea = __importStar(require("exupery-core-alg"));
+const WhitespaceChars = {
+    tab: 0x09, // \t
+    line_feed: 0x0A, // \n
+    carriage_return: 0x0D, // \r
+    space: 0x20, //
+};
+const is_control_character = ($) => {
+    return ($ < 0x20 && $ !== WhitespaceChars.tab && $ !== WhitespaceChars.line_feed && $ !== WhitespaceChars.carriage_return);
+};
+exports.is_control_character = is_control_character;
+/**
+ * Creates a string iterator that allows iterating over characters in a string,
+ * while keeping track of line numbers, columns, and line indentation.
+ */
+const create_iterator = ($, $p) => {
+    const characters = _ea.text_to_character_list($);
+    let found_carriage_return_before = false;
+    let position = 0;
+    let relative_position = {
+        'line': 0,
+        'column': 0,
+        'line indentation': null
+    };
+    return {
+        'consume': () => {
+            const c = characters.__get_element_at(position);
+            const start = relative_position;
+            relative_position = c.transform(($) => {
+                return $ === WhitespaceChars.line_feed
+                    ? {
+                        'line': relative_position.line + 1,
+                        'column': 0,
+                        'line indentation': null,
+                    }
+                    : found_carriage_return_before
+                        ? {
+                            'line': relative_position.line + 1,
+                            'column': 0,
+                            'line indentation': null,
+                        }
+                        : {
+                            'line': relative_position.line,
+                            'column': relative_position['column'] + ($ === WhitespaceChars.tab
+                                ? $p['tab size']
+                                : 1),
+                            'line indentation': relative_position['line indentation'] !== null
+                                ? relative_position['line indentation']
+                                : $ === WhitespaceChars.space || $ === WhitespaceChars.tab
+                                    ? null
+                                    : relative_position['column'],
+                        };
+            }, () => relative_position //this is weird, we were already at the end of the list
+            );
+            found_carriage_return_before = c.transform(($) => $ === WhitespaceChars.carriage_return, () => false);
+            position += 1;
+        },
+        'get current': () => {
+            return characters.__get_element_at(position).transform(($) => _ea.set($), () => _ea.not_set());
+        },
+        'look ahead': (offset) => {
+            return characters.__get_element_at(position + offset).transform(($) => _ea.set($), () => _ea.not_set());
+        },
+        'get state': () => {
+            return {
+                'location': {
+                    'absolute': position,
+                    'relative': {
+                        'line': relative_position.line,
+                        'column': relative_position['column'],
+                    }
+                },
+                'line indentation': relative_position['line indentation'] !== null
+                    ? relative_position['line indentation']
+                    : relative_position['column']
+            };
+        }
+    };
+};
+exports.create_iterator = create_iterator;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaXRlcmF0b3IuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi8uLi8uLi8uLi9zcmMvaW1wbGVtZW50YXRpb24vYWxnb3JpdGhtcy9yZWZpbmVycy90b2tlbi90ZXh0L2l0ZXJhdG9yLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7O0FBQUEsc0RBQXVDO0FBNEJ2QyxNQUFNLGVBQWUsR0FBRztJQUNwQixHQUFHLEVBQUUsSUFBSSxFQUFtQixLQUFLO0lBQ2pDLFNBQVMsRUFBRSxJQUFJLEVBQWEsS0FBSztJQUNqQyxlQUFlLEVBQUUsSUFBSSxFQUFPLEtBQUs7SUFDakMsS0FBSyxFQUFFLElBQUksRUFBaUIsRUFBRTtDQUNqQyxDQUFBO0FBRU0sTUFBTSxvQkFBb0IsR0FBRyxDQUFDLENBQVMsRUFBVyxFQUFFO0lBQ3ZELE9BQU8sQ0FBQyxDQUFDLEdBQUcsSUFBSSxJQUFJLENBQUMsS0FBSyxlQUFlLENBQUMsR0FBRyxJQUFJLENBQUMsS0FBSyxlQUFlLENBQUMsU0FBUyxJQUFJLENBQUMsS0FBSyxlQUFlLENBQUMsZUFBZSxDQUFDLENBQUE7QUFDOUgsQ0FBQyxDQUFBO0FBRlksUUFBQSxvQkFBb0Isd0JBRWhDO0FBRUQ7OztHQUdHO0FBRUksTUFBTSxlQUFlLEdBQUcsQ0FDM0IsQ0FBUyxFQUNULEVBRUMsRUFDa0IsRUFBRTtJQUNyQixNQUFNLFVBQVUsR0FBRyxHQUFHLENBQUMsc0JBQXNCLENBQUMsQ0FBQyxDQUFDLENBQUE7SUFRaEQsSUFBSSw0QkFBNEIsR0FBRyxLQUFLLENBQUE7SUFDeEMsSUFBSSxRQUFRLEdBQUcsQ0FBQyxDQUFBO0lBQ2hCLElBQUksaUJBQWlCLEdBQWtDO1FBQ25ELE1BQU0sRUFBRSxDQUFDO1FBQ1QsUUFBUSxFQUFFLENBQUM7UUFDWCxrQkFBa0IsRUFBRSxJQUFJO0tBQzNCLENBQUE7SUFFRCxPQUFPO1FBQ0gsU0FBUyxFQUFFLEdBQUcsRUFBRTtZQUNaLE1BQU0sQ0FBQyxHQUFHLFVBQVUsQ0FBQyxnQkFBZ0IsQ0FBQyxRQUFRLENBQUMsQ0FBQTtZQUMvQyxNQUFNLEtBQUssR0FBRyxpQkFBaUIsQ0FBQTtZQUMvQixpQkFBaUIsR0FBRyxDQUFDLENBQUMsU0FBUyxDQUMzQixDQUFDLENBQUMsRUFBRSxFQUFFO2dCQUNGLE9BQU8sQ0FBQyxLQUFLLGVBQWUsQ0FBQyxTQUFTO29CQUNsQyxDQUFDLENBQUM7d0JBQ0UsTUFBTSxFQUFFLGlCQUFpQixDQUFDLElBQUksR0FBRyxDQUFDO3dCQUNsQyxRQUFRLEVBQUUsQ0FBQzt3QkFDWCxrQkFBa0IsRUFBRSxJQUFJO3FCQUMzQjtvQkFDRCxDQUFDLENBQUMsNEJBQTRCO3dCQUMxQixDQUFDLENBQUM7NEJBQ0UsTUFBTSxFQUFFLGlCQUFpQixDQUFDLElBQUksR0FBRyxDQUFDOzRCQUNsQyxRQUFRLEVBQUUsQ0FBQzs0QkFDWCxrQkFBa0IsRUFBRSxJQUFJO3lCQUMzQjt3QkFDRCxDQUFDLENBQUM7NEJBQ0UsTUFBTSxFQUFFLGlCQUFpQixDQUFDLElBQUk7NEJBQzlCLFFBQVEsRUFBRSxpQkFBaUIsQ0FBQyxRQUFRLENBQUMsR0FBRyxDQUFDLENBQUMsS0FBSyxlQUFlLENBQUMsR0FBRztnQ0FDOUQsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxVQUFVLENBQUM7Z0NBQ2hCLENBQUMsQ0FBQyxDQUFDLENBQUM7NEJBQ1Isa0JBQWtCLEVBQUUsaUJBQWlCLENBQUMsa0JBQWtCLENBQUMsS0FBSyxJQUFJO2dDQUM5RCxDQUFDLENBQUMsaUJBQWlCLENBQUMsa0JBQWtCLENBQUM7Z0NBQ3ZDLENBQUMsQ0FBQyxDQUFDLEtBQUssZUFBZSxDQUFDLEtBQUssSUFBSSxDQUFDLEtBQUssZUFBZSxDQUFDLEdBQUc7b0NBQ3RELENBQUMsQ0FBQyxJQUFJO29DQUNOLENBQUMsQ0FBQyxpQkFBaUIsQ0FBQyxRQUFRLENBQUM7eUJBQ3hDLENBQUE7WUFDYixDQUFDLEVBQ0QsR0FBRyxFQUFFLENBQUMsaUJBQWlCLENBQUMsdURBQXVEO2FBQ2xGLENBQUE7WUFFRCw0QkFBNEIsR0FBRyxDQUFDLENBQUMsU0FBUyxDQUN0QyxDQUFDLENBQUMsRUFBRSxFQUFFLENBQUMsQ0FBQyxLQUFLLGVBQWUsQ0FBQyxlQUFlLEVBQzVDLEdBQUcsRUFBRSxDQUFDLEtBQUssQ0FDZCxDQUFBO1lBQ0QsUUFBUSxJQUFJLENBQUMsQ0FBQTtRQUNqQixDQUFDO1FBQ0QsYUFBYSxFQUFFLEdBQUcsRUFBRTtZQUNoQixPQUFPLFVBQVUsQ0FBQyxnQkFBZ0IsQ0FBQyxRQUFRLENBQUMsQ0FBQyxTQUFTLENBQ2xELENBQUMsQ0FBQyxFQUFFLEVBQUUsQ0FBQyxHQUFHLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQyxFQUNqQixHQUFHLEVBQUUsQ0FBQyxHQUFHLENBQUMsT0FBTyxFQUFFLENBQ3RCLENBQUE7UUFDTCxDQUFDO1FBQ0QsWUFBWSxFQUFFLENBQUMsTUFBYyxFQUFFLEVBQUU7WUFDN0IsT0FBTyxVQUFVLENBQUMsZ0JBQWdCLENBQUMsUUFBUSxHQUFHLE1BQU0sQ0FBQyxDQUFDLFNBQVMsQ0FDM0QsQ0FBQyxDQUFDLEVBQUUsRUFBRSxDQUFDLEdBQUcsQ0FBQyxHQUFHLENBQUMsQ0FBQyxDQUFDLEVBQ2pCLEdBQUcsRUFBRSxDQUFDLEdBQUcsQ0FBQyxPQUFPLEVBQUUsQ0FDdEIsQ0FBQTtRQUNMLENBQUM7UUFDRCxXQUFXLEVBQUUsR0FBRyxFQUFFO1lBQ2QsT0FBTztnQkFDSCxVQUFVLEVBQUU7b0JBQ1IsVUFBVSxFQUFFLFFBQVE7b0JBQ3BCLFVBQVUsRUFBRTt3QkFDUixNQUFNLEVBQUUsaUJBQWlCLENBQUMsSUFBSTt3QkFDOUIsUUFBUSxFQUFFLGlCQUFpQixDQUFDLFFBQVEsQ0FBQztxQkFDeEM7aUJBQ0o7Z0JBQ0Qsa0JBQWtCLEVBQUUsaUJBQWlCLENBQUMsa0JBQWtCLENBQUMsS0FBSyxJQUFJO29CQUM5RCxDQUFDLENBQUMsaUJBQWlCLENBQUMsa0JBQWtCLENBQUM7b0JBQ3ZDLENBQUMsQ0FBQyxpQkFBaUIsQ0FBQyxRQUFRLENBQUM7YUFDcEMsQ0FBQTtRQUNMLENBQUM7S0FDSixDQUFBO0FBQ0wsQ0FBQyxDQUFBO0FBeEZZLFFBQUEsZUFBZSxtQkF3RjNCIn0=

package/dist/implementation/algorithms/refiners/token/text/refiners.d.ts

@@ -0,0 +1,9 @@
+import * as _ea from 'exupery-core-alg';
+import * as _out from "../../../../../interface/generated/pareto/schemas/token/data_types/target";
+import * as _parse_result from "../../../../../interface/generated/pareto/schemas/authoring_parse_result/data_types/target";
+import { Characters_Iterator } from "./iterator";
+export declare const Whitespace: (iterator: Characters_Iterator, abort: _ea.Abort<_parse_result.Parse_Error>) => _out.Whitespace;
+export declare const Trivia: (iterator: Characters_Iterator, abort: _ea.Abort<_parse_result.Parse_Error>) => _out.Trivia;
+export declare const Annotated_Token: (iterator: Characters_Iterator, abort: _ea.Abort<_parse_result.Parse_Error>) => _out.Annotated_Token;
+export declare const Delimited_String: (is_end_character: (character: number) => boolean, allow_newlines: boolean, iterator: Characters_Iterator, abort: _ea.Abort<_parse_result.Parse_Error>) => _out.Delimited_String;
+export declare const Tokenizer_Result: (iterator: Characters_Iterator, abort: _ea.Abort<_parse_result.Parse_Error>) => _out.Tokenizer_Result;