@retorquere/bibtex-parser 9.0.1 → 9.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -1
- package/sentence-case.js +17 -8
- package/sentence-case.js.map +1 -1
- package/tokenizer.d.ts +1 -0
- package/tokenizer.js +24 -15
- package/tokenizer.js.map +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@retorquere/bibtex-parser",
|
|
3
|
-
"version": "9.0.
|
|
3
|
+
"version": "9.0.2",
|
|
4
4
|
"description": "BibTeX parser",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"typings": "./index.d.ts",
|
|
@@ -65,6 +65,7 @@
|
|
|
65
65
|
"i": "^0.3.7",
|
|
66
66
|
"lodash.merge": "^4.6.2",
|
|
67
67
|
"moo": "^0.5.2",
|
|
68
|
+
"nearley": "^2.20.1",
|
|
68
69
|
"npm": "^10.5.2",
|
|
69
70
|
"unicode2latex": "^6.0.4",
|
|
70
71
|
"wink-eng-lite-web-model": "^1.6.0",
|
package/sentence-case.js
CHANGED
|
@@ -13,9 +13,11 @@ const merge_1 = require("./merge");
|
|
|
13
13
|
function titleCase(s) {
|
|
14
14
|
return s.replace(/^(.)(.+)/, (match, car, cdr) => `${car}${cdr.toLowerCase()}`);
|
|
15
15
|
}
|
|
16
|
-
const connectedWord = (
|
|
16
|
+
// const connectedWord = XRegExp('(^|-)\\p{Lu}\\p{Ll}*(?=-|$)', 'g')
|
|
17
17
|
const connectedInnerWord = (0, xregexp_1.default)('-\\p{Lu}\\p{Ll}*(?=-|$)', 'g');
|
|
18
|
-
function wordSC(token, allCaps, subSentence) {
|
|
18
|
+
function wordSC(token, allCaps, subSentence, hyphenated) {
|
|
19
|
+
if (token.type === 'domain')
|
|
20
|
+
return token.text.toLowerCase();
|
|
19
21
|
if (token.type !== 'word')
|
|
20
22
|
return token.text;
|
|
21
23
|
if (token.text.match(/^I'/))
|
|
@@ -29,10 +31,9 @@ function wordSC(token, allCaps, subSentence) {
|
|
|
29
31
|
return token.text.toLowerCase();
|
|
30
32
|
if (token.subtype === 'acronym')
|
|
31
33
|
return token.text;
|
|
32
|
-
if (!allCaps && token.shape.match(/^[Xxd]+(-[Xxd]+)+/))
|
|
33
|
-
return xregexp_1.default.replace(token.text, connectedWord, match => match.toLowerCase());
|
|
34
|
+
// if (!allCaps && token.shape.match(/^[Xxd]+(-[Xxd]+)+/)) return XRegExp.replace(token.text, connectedWord, match => match.toLowerCase())
|
|
34
35
|
if (token.text.match(/^[B-Z]$/))
|
|
35
|
-
return token.text;
|
|
36
|
+
return hyphenated ? token.text.toLowerCase() : token.text;
|
|
36
37
|
if (!allCaps && token.shape.match(/^[-X]+$/))
|
|
37
38
|
return token.text;
|
|
38
39
|
const shape = token.shape.replace(/[^Xxd]/g, '');
|
|
@@ -83,11 +84,19 @@ function toSentenceCase(title, options = {}) {
|
|
|
83
84
|
title = title.normalize('NFC'); // https://github.com/winkjs/wink-nlp/issues/134
|
|
84
85
|
if (allCaps && !title.match(/\s/))
|
|
85
86
|
return title;
|
|
86
|
-
let sentenceCased = title;
|
|
87
87
|
const tokens = (0, tokenizer_1.tokenize)(title, options.markup);
|
|
88
|
+
let sentenceCased = '';
|
|
88
89
|
for (const token of tokens) {
|
|
89
|
-
if (token.
|
|
90
|
-
sentenceCased
|
|
90
|
+
if (token.hyphenated && token.shape[0] === 'x') {
|
|
91
|
+
sentenceCased += token.text;
|
|
92
|
+
}
|
|
93
|
+
else if (token.hyphenated) {
|
|
94
|
+
for (const t of token.hyphenated) {
|
|
95
|
+
sentenceCased += wordSC(t, allCaps, options.subSentenceCapitalization, true);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
else {
|
|
99
|
+
sentenceCased += wordSC(token, allCaps, options.subSentenceCapitalization, false);
|
|
91
100
|
}
|
|
92
101
|
}
|
|
93
102
|
if (options.markup) {
|
package/sentence-case.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sentence-case.js","sourceRoot":"","sources":["sentence-case.ts"],"names":[],"mappings":";AAAA,sDAAsD;;;;;;AAEtD,sDAA6B;AAC7B,2CAA6C;AAC7C,mCAA+B;AAE/B,4CAA4C;AAC5C,mKAAmK;AAEnK,SAAS,SAAS,CAAC,CAAS;IAC1B,OAAO,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC,CAAA;AACjF,CAAC;AAED,MAAM,
|
|
1
|
+
{"version":3,"file":"sentence-case.js","sourceRoot":"","sources":["sentence-case.ts"],"names":[],"mappings":";AAAA,sDAAsD;;;;;;AAEtD,sDAA6B;AAC7B,2CAA6C;AAC7C,mCAA+B;AAE/B,4CAA4C;AAC5C,mKAAmK;AAEnK,SAAS,SAAS,CAAC,CAAS;IAC1B,OAAO,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC,CAAA;AACjF,CAAC;AAED,oEAAoE;AACpE,MAAM,kBAAkB,GAAG,IAAA,iBAAO,EAAC,yBAAyB,EAAE,GAAG,CAAC,CAAA;AAElE,SAAS,MAAM,CAAC,KAAY,EAAE,OAAgB,EAAE,WAAoB,EAAE,UAAmB;IACvF,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAA;IAC5D,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,KAAK,CAAC,IAAI,CAAA;IAC5C,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IACzD,IAAI,WAAW,IAAI,KAAK,CAAC,gBAAgB,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,GAAG,CAAA;IAEjF,IAAI,CAAC,WAAW,IAAI,KAAK,CAAC,gBAAgB,CAAC,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;QACnE,OAAO,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,iBAAO,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,kBAAkB,EAAE,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAA;IACxH,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,KAAK,aAAa;QAAE,OAAO,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAA;IACpE,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC,IAAI,CAAA;IAElD,0IAA0I;IAE1I,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;QAAE,OAAO,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAA;IAE1F,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC;QAAE,OAAO,KAAK,CAAC,IAAI,CAAA;IAE/D,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;IAChD,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC;QAAE,OAAO,KAAK,CAAC,IAAI,CAAA;IAC1D,IAAI,KAAK,CAAC,KAAK,CAAC,oBAAoB,CAAC;QAAE,OAAO,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAA;IACtE,IAAI,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC;QAAE,OAAO,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAA;IAElF,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC,IAAI,CAAA;IAC/C,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,KAAK,CAAC,IAAI,CAAA;IAE1C,OAAO,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAA;AACjC,CAAC;AAUD,SAAgB,cAAc,CAAC,KAAa,EAAE,UAAmB,EAAE;IACjE,OAAO,GAAG,IAAA,aAAK,EAAC,OAAO,EAAE;QACvB,cAAc,EAAE,IAAI;QACpB,yBAAyB,EAAE,IAAI;QAC/B,KAAK,EAAE,KAAK;KACb,CAAC,CAAA;IAEF,MAAM,OAAO,GAAG,KAAK,KAAK,KAAK,CAAC,WAAW,EAAE,CAAA;IAC7C,MAAM,QAAQ,GAAG,KAAK,KAAK,KAAK,CAAC,WAAW,EAAE,CAAA;IAC9C,IAAI,OAAO,CAAC,KAAK,IAAI,CAAC,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC3C,IAAI,MAAM,GAAG,KAAK,CAAA;QAClB,IAAI,OAAO,CAAC,MAAM;YAAE,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;QAClG,IAAI,OAAO,CAAC,MAAM;YAAE,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;QAC/D,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YAC3B,KAAK,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;gBAC/C,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;YAChC,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG;YACZ,KAAK,EAAE,IAAA,oBAAQ,EAAC,MAAM,CAAC;YACvB,EAAE,EAAE,CAAC;YACL,KAAK,EAAE,CAAC;SACT,CAAA;QAED,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;YAC/B,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,GAAC,CAAC,CAAC,CAAC,IAAI,KAAK,YAAY,CAAC,EAAE,CAAC;gBACjK,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;YACjE,CAAC;QACH,CAAC,CAAC,CAAA;QACF,IAAI,KAAK,CAAC,EAAE,IAAI,KAAK,CAAC,EAAE,IAAI,KAAK,CAAC,KAAK;YAAE,OAAO,KAAK,CAAA;IACvD,CAAC;IAED,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA,CAAC,gDAAgD;IAC/E,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAA;IAE/C,MAAM,MAAM,GAAG,IAAA,oBAAQ,EAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;IAE9C,IAAI,aAAa,GAAG,EAAE,CAAA;IACtB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YAC/C,aAAa,IAAI,KAAK,CAAC,IAAI,CAAA;QAC7B,CAAC;aACI,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YAC1B,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;gBACjC,aAAa,IAAI,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,yBAAyB,EAAE,IAAI,CAAC,CAAA;YAC9E,CAAC;QACH,CAAC;aACI,CAAC;YACJ,aAAa,IAAI,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,yBAAyB,EAAE,KAAK,CAAC,CAAA;QACnF,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACnD,aAAa,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAA;QAC7H,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,aAAa,GAAG,aAAa;aAC1B,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,GAAW,EAAE,MAAc,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;IAC5H,CAAC;IAED,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;YAC/C,aAAa,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,KAAa,EAAE,MAAc,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;QAC7H,CAAC;IACH,CAAC;IAED,OAAO,aAAa,CAAA;AACtB,CAAC;AAvED,wCAuEC"}
|
package/tokenizer.d.ts
CHANGED
package/tokenizer.js
CHANGED
|
@@ -19,12 +19,12 @@ const P = new RegExp(re(charCategories.filter(cat => cat.name.match(/^P/))));
|
|
|
19
19
|
const Lu = re(charCategories.filter(cat => cat.name === 'Lu'), '\u2060');
|
|
20
20
|
const Acronym = new RegExp(`(?:(?:(?:${Lu}[.]){2,}${B})|(?:(?:vs?[.])(?=[ \t\n\r\u00A0])))`);
|
|
21
21
|
const Contraction = new RegExp(`${W}['’]${W}${B}`);
|
|
22
|
-
const Compound = new RegExp(`${W}(?:-${W})+${B}`);
|
|
23
22
|
const Whitespace = /[ \t\n\r\u00A0]+/;
|
|
24
23
|
const Ordinal = new RegExp(`\\d+(?:st|nd|rd|th)${B}`);
|
|
25
24
|
const Email = new RegExp(`[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:[.][A-Za-z0-9-]+)+${B}`);
|
|
26
25
|
const Handle = new RegExp(`@[A-Za-z0-9-]{2,}${B}`);
|
|
27
26
|
const Int = new RegExp(`[0-9]+${B}`);
|
|
27
|
+
const Domain = new RegExp(`${W}(?:[.]${W})+${B}`);
|
|
28
28
|
const ComplexPreposition = /^([^ \t\n\r\u00A0]+)([ \t\n\r\u00A0]+)([^ \t\n\r\u00A0]+)(?:([ \t\n\r\u00A0]+)([^ \t\n\r\u00A0]+))?$/;
|
|
29
29
|
function ci(s) {
|
|
30
30
|
return s
|
|
@@ -37,10 +37,10 @@ const lexer = moo_1.default.compile({
|
|
|
37
37
|
'word-preposition': Preposition,
|
|
38
38
|
'word-acronym': Acronym,
|
|
39
39
|
'word-contraction': Contraction,
|
|
40
|
-
'word-compound': Compound,
|
|
41
40
|
'word-ordinal': Ordinal,
|
|
42
41
|
email: Email,
|
|
43
42
|
handle: Handle,
|
|
43
|
+
domain: Domain,
|
|
44
44
|
word: Word,
|
|
45
45
|
number: Int, // eslint-disable-line id-blacklist
|
|
46
46
|
'punctuation-end': /[?.!](?=[ \t\n\r\u00A0]|$)/,
|
|
@@ -58,6 +58,22 @@ const shape = [
|
|
|
58
58
|
[/–/g, '-'],
|
|
59
59
|
[/[\u2060\u00AD]/g, ''],
|
|
60
60
|
];
|
|
61
|
+
function combine(tokens) {
|
|
62
|
+
const combined = Object.assign({}, tokens[0]);
|
|
63
|
+
for (const t of tokens.slice(1)) {
|
|
64
|
+
combined.text += t.text;
|
|
65
|
+
combined.end = t.end;
|
|
66
|
+
combined.shape += t.shape;
|
|
67
|
+
}
|
|
68
|
+
return combined;
|
|
69
|
+
}
|
|
70
|
+
function hyphenate(t) {
|
|
71
|
+
if (t.type === 'word')
|
|
72
|
+
return 'w';
|
|
73
|
+
if (t.text === '-')
|
|
74
|
+
return '-';
|
|
75
|
+
return ' ';
|
|
76
|
+
}
|
|
61
77
|
function tokenize(title, markup) {
|
|
62
78
|
if (markup)
|
|
63
79
|
title = title.replace(markup, match => '\u2060'.repeat(match.length));
|
|
@@ -92,14 +108,6 @@ function tokenize(title, markup) {
|
|
|
92
108
|
}
|
|
93
109
|
}
|
|
94
110
|
const stack = tokens.splice(0);
|
|
95
|
-
const combine = (ts) => {
|
|
96
|
-
for (const t of ts.slice(1)) {
|
|
97
|
-
ts[0].text += t.text;
|
|
98
|
-
ts[0].end = t.end;
|
|
99
|
-
ts[0].shape += t.shape;
|
|
100
|
-
}
|
|
101
|
-
return ts[0];
|
|
102
|
-
};
|
|
103
111
|
let cpt;
|
|
104
112
|
let cps;
|
|
105
113
|
while (stack.length) {
|
|
@@ -117,15 +125,16 @@ function tokenize(title, markup) {
|
|
|
117
125
|
}
|
|
118
126
|
continue;
|
|
119
127
|
}
|
|
120
|
-
//
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
128
|
+
// hyphenated words
|
|
129
|
+
const pat = stack.map(t => hyphenate(t)).join('');
|
|
130
|
+
if (cpt = pat.match(/^w(-w)+(?= |$)/)) {
|
|
131
|
+
const hyphenated = stack.splice(0, cpt[0].length);
|
|
132
|
+
tokens.push(Object.assign(Object.assign({}, combine(hyphenated)), { hyphenated }));
|
|
124
133
|
continue;
|
|
125
134
|
}
|
|
126
135
|
tokens.push(stack.shift());
|
|
127
136
|
}
|
|
128
|
-
return tokens;
|
|
137
|
+
return markup ? tokens.map(token => (Object.assign(Object.assign({}, token), { text: title.substring(token.start, token.end + 1) }))) : tokens;
|
|
129
138
|
}
|
|
130
139
|
exports.tokenize = tokenize;
|
|
131
140
|
//# sourceMappingURL=tokenizer.js.map
|
package/tokenizer.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["tokenizer.ts"],"names":[],"mappings":";;;;;;AAAA,8CAAqB;AACrB,sDAA6B;AAa7B,MAAM,cAAc,GAAmB,OAAO,CAAC,iCAAiC,CAAC,CAAA;AAEjF,SAAS,EAAE,CAAC,IAAoB,EAAE,KAAc,EAAE,GAAG,GAAC,KAAK;IACzD,OAAO,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,KAAK,IAAI,EAAE,GAAG,CAAA;AAChF,CAAC;AACD,MAAM,CAAC,GAAW,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAA;AACpE,MAAM,GAAG,GAAW,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,cAAc,CAAC,CAAA;AAC9F,MAAM,CAAC,GAAG,GAAG,GAAG,KAAK,CAAC,GAAG,GAAG,GAAG,CAAA;AAC/B,MAAM,CAAC,GAAG,SAAS,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAA;AAEvH,MAAM,IAAI,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;AACnC,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;AAE5E,MAAM,EAAE,GAAW,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,QAAQ,CAAC,CAAA;AAChF,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,YAAY,EAAE,WAAW,CAAC,sCAAsC,CAAC,CAAA;AAE5F,MAAM,WAAW,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;AAClD,MAAM,
|
|
1
|
+
{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["tokenizer.ts"],"names":[],"mappings":";;;;;;AAAA,8CAAqB;AACrB,sDAA6B;AAa7B,MAAM,cAAc,GAAmB,OAAO,CAAC,iCAAiC,CAAC,CAAA;AAEjF,SAAS,EAAE,CAAC,IAAoB,EAAE,KAAc,EAAE,GAAG,GAAC,KAAK;IACzD,OAAO,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,KAAK,IAAI,EAAE,GAAG,CAAA;AAChF,CAAC;AACD,MAAM,CAAC,GAAW,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAA;AACpE,MAAM,GAAG,GAAW,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,cAAc,CAAC,CAAA;AAC9F,MAAM,CAAC,GAAG,GAAG,GAAG,KAAK,CAAC,GAAG,GAAG,GAAG,CAAA;AAC/B,MAAM,CAAC,GAAG,SAAS,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAA;AAEvH,MAAM,IAAI,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;AACnC,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;AAE5E,MAAM,EAAE,GAAW,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,QAAQ,CAAC,CAAA;AAChF,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,YAAY,EAAE,WAAW,CAAC,sCAAsC,CAAC,CAAA;AAE5F,MAAM,WAAW,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;AAClD,MAAM,UAAU,GAAG,kBAAkB,CAAA;AACrC,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAA;AACrD,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,uDAAuD,CAAC,EAAE,CAAC,CAAA;AACpF,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAA;AAClD,MAAM,GAAG,GAAG,IAAI,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;AACpC,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;AAEjD,MAAM,kBAAkB,GAAG,sGAAsG,CAAA;AAEjI,SAAS,EAAE,CAAC,CAAS;IACnB,OAAO,CAAC;SACL,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,WAAW,EAAE,GAAG,KAAK,CAAC,WAAW,EAAE,GAAG,CAAC;SAC7E,OAAO,CAAC,GAAG,EAAE,UAAU,CAAC,MAAM,CAAE,CAAA;AACrC,CAAC;AACD,MAAM,YAAY,GAAW,OAAO,CAAC,qBAAqB,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AAC9F,MAAM,WAAW,GAAG,IAAI,MAAM,CAAC,MAAM,YAAY,IAAI,CAAC,EAAE,CAAC,CAAA;AAEzD,MAAM,KAAK,GAAG,aAAG,CAAC,OAAO,CAAC;IACxB,kBAAkB,EAAM,WAAW;IACnC,cAAc,EAAU,OAAO;IAC/B,kBAAkB,EAAM,WAAW;IACnC,cAAc,EAAU,OAAO;IAC/B,KAAK,EAAmB,KAAK;IAC7B,MAAM,EAAkB,MAAM;IAC9B,MAAM,EAAkB,MAAM;IAC9B,IAAI,EAAoB,IAAI;IAC5B,MAAM,EAAkB,GAAG,EAAE,mCAAmC;IAChE,iBAAiB,EAAO,4BAA4B;IACpD,mBAAmB,EAAK,sBAAsB;IAC9C,sBAAsB,EAAE,WAAW;IACnC,WAAW,EAAa,CAAC;IACzB,UAAU,EAAc,EAAE,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,IAAI,EAAE;IACtE,KAAK,EAAmB,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,EAAE;CAC9D,CAAC,CAAA;AAEF,MAAM,KAAK,GAAwB;IACjC,CAAE,IAAA,iBAAO,EAAC,SAAS,CAAC,EAAE,GAAG,EAAE,KAAK,CAAE;IAClC,CAAE,IAAI,MAAM,CAAC,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAE;IACrF,CAAE,IAAA,iBAAO,EAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,KAAK,CAAE;IACjC,CAAE,IAAI,EAAE,GAAG,CAAE;IACb,CAAE,IAAI,EAAE,GAAG,CAAE;IACb,CAAE,iBAAiB,EAAE,EAAE,CAAE;CAC1B,CAAA;AAcD,SAAS,OAAO,CAAC,MAAe;IAC9B,MAAM,QAAQ,qBAAQ,MAAM,CAAC,CAAC,CAAC,CAAE,CAAA;IACjC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAChC,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,CAAA;QACvB,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAA;QACpB,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,CAAA;IAC3B,CAAC;IACD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,SAAS,SAAS,CAAC,CAAQ;IACzB,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,GAAG,CAAA;IACjC,IAAI,CAAC,CAAC,IAAI,KAAK,GAAG;QAAE,OAAO,GAAG,CAAA;IAC9B,OAAO,GAAG,CAAA;AACZ,CAAC;AAED,SAAgB,QAAQ,CAAC,KAAa,EAAE,MAAe;IACrD,IAAI,MAAM;QAAE,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;IAEjF,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IAClB,MAAM,MAAM,GAAY,EAAE,CAAA;IAC1B,IAAI,aAAa,GAAG,IAAI,CAAA;IACxB,IAAI,gBAAgB,GAAG,KAAK,CAAA;IAC5B,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,CAAE,IAAI,EAAE,OAAO,CAAE,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QAE/F,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,GAAG,EAAE,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;YACzC,KAAK,EAAE,iBAAO,CAAC,WAAW,CAAS,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC;YACrD,aAAa,EAAE,IAAI,KAAK,MAAM,IAAI,aAAa;YAC/C,gBAAgB,EAAE,IAAI,KAAK,MAAM,IAAI,gBAAgB;SACtD,CAAC,CAAA;QAEF,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,iBAAiB;gBACpB,aAAa,GAAG,IAAI,CAAA;gBACpB,MAAK;YACP,KAAK,mBAAmB;gBACtB,gBAAgB,GAAG,IAAI,CAAA;gBACvB,MAAK;YACP;gBACE,IAAI,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,CAAC;oBAC9B,aAAa,GAAG,KAAK,CAAA;oBACrB,gBAAgB,GAAG,KAAK,CAAA;gBAC1B,CAAC;gBACD,MAAK;QACT,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;IAE9B,IAAI,GAAqB,CAAA;IACzB,IAAI,GAAqB,CAAA;IACzB,OAAO,KAAK,CAAC,MAAM,EAAE,CAAC;QACpB,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,aAAa,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,EAAE,CAAC;YAC9I,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,EAAE,CAAA;YAE7B,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK,CAAA;YACzB,IAAI,GAAG,CAAA;YACP,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,EAAC,MAAM,EAAE,CAAC,EAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAC,CAAC,CAAC,EAAE,CAAC;gBACvD,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;oBAAE,MAAK;gBAElB,GAAG,GAAG,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAA;gBAC/B,MAAM,CAAC,IAAI,iCACN,OAAO,KACV,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,EACZ,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,EACb,KAAK;oBACL,GAAG,EACH,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,EACzC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAClC,CAAA;gBACF,KAAK,GAAG,GAAG,GAAG,CAAC,CAAA;YACjB,CAAC;YACD,SAAQ;QACV,CAAC;QAED,mBAAmB;QACnB,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACjD,IAAI,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACtC,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAA;YACjD,MAAM,CAAC,IAAI,iCAAM,OAAO,CAAC,UAAU,CAAC,KAAE,UAAU,IAAG,CAAA;YACnD,SAAQ;QACV,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAA;IAC5B,CAAC;IAED,OAAO,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,iCAAM,KAAK,KAAE,IAAI,EAAE,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,IAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;AACjH,CAAC;AA5ED,4BA4EC"}
|