@retorquere/bibtex-parser 9.0.1 → 9.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -1
- package/sentence-case.js +17 -14
- package/sentence-case.js.map +1 -1
- package/tokenizer.d.ts +2 -0
- package/tokenizer.js +45 -28
- package/tokenizer.js.map +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@retorquere/bibtex-parser",
|
|
3
|
-
"version": "9.0.
|
|
3
|
+
"version": "9.0.3",
|
|
4
4
|
"description": "BibTeX parser",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"typings": "./index.d.ts",
|
|
@@ -65,6 +65,7 @@
|
|
|
65
65
|
"i": "^0.3.7",
|
|
66
66
|
"lodash.merge": "^4.6.2",
|
|
67
67
|
"moo": "^0.5.2",
|
|
68
|
+
"nearley": "^2.20.1",
|
|
68
69
|
"npm": "^10.5.2",
|
|
69
70
|
"unicode2latex": "^6.0.4",
|
|
70
71
|
"wink-eng-lite-web-model": "^1.6.0",
|
package/sentence-case.js
CHANGED
|
@@ -1,11 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/* eslint-disable @typescript-eslint/unbound-method */
|
|
3
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
4
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
5
|
-
};
|
|
6
3
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
4
|
exports.toSentenceCase = void 0;
|
|
8
|
-
const xregexp_1 = __importDefault(require("xregexp"));
|
|
9
5
|
const tokenizer_1 = require("./tokenizer");
|
|
10
6
|
const merge_1 = require("./merge");
|
|
11
7
|
// eslint-disable-next-line no-magic-numbers
|
|
@@ -13,9 +9,9 @@ const merge_1 = require("./merge");
|
|
|
13
9
|
function titleCase(s) {
|
|
14
10
|
return s.replace(/^(.)(.+)/, (match, car, cdr) => `${car}${cdr.toLowerCase()}`);
|
|
15
11
|
}
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
12
|
+
function wordSC(token, allCaps, subSentence, hyphenated) {
|
|
13
|
+
if (token.type === 'domain')
|
|
14
|
+
return token.text.toLowerCase();
|
|
19
15
|
if (token.type !== 'word')
|
|
20
16
|
return token.text;
|
|
21
17
|
if (token.text.match(/^I'/))
|
|
@@ -23,16 +19,15 @@ function wordSC(token, allCaps, subSentence) {
|
|
|
23
19
|
if (subSentence && token.subSentenceStart && token.text.match(/^a$/i))
|
|
24
20
|
return 'a';
|
|
25
21
|
if ((subSentence && token.subSentenceStart) || token.sentenceStart) {
|
|
26
|
-
return allCaps ? titleCase(token.text) :
|
|
22
|
+
return allCaps ? titleCase(token.text) : token.text.replace(tokenizer_1.connectedInnerWord, match => match.toLowerCase());
|
|
27
23
|
}
|
|
28
24
|
if (token.subtype === 'preposition')
|
|
29
25
|
return token.text.toLowerCase();
|
|
30
26
|
if (token.subtype === 'acronym')
|
|
31
27
|
return token.text;
|
|
32
|
-
if (!allCaps && token.shape.match(/^[Xxd]+(-[Xxd]+)+/))
|
|
33
|
-
return xregexp_1.default.replace(token.text, connectedWord, match => match.toLowerCase());
|
|
28
|
+
// if (!allCaps && token.shape.match(/^[Xxd]+(-[Xxd]+)+/)) return XRegExp.replace(token.text, connectedWord, match => match.toLowerCase())
|
|
34
29
|
if (token.text.match(/^[B-Z]$/))
|
|
35
|
-
return token.text;
|
|
30
|
+
return hyphenated ? token.text.toLowerCase() : token.text;
|
|
36
31
|
if (!allCaps && token.shape.match(/^[-X]+$/))
|
|
37
32
|
return token.text;
|
|
38
33
|
const shape = token.shape.replace(/[^Xxd]/g, '');
|
|
@@ -83,11 +78,19 @@ function toSentenceCase(title, options = {}) {
|
|
|
83
78
|
title = title.normalize('NFC'); // https://github.com/winkjs/wink-nlp/issues/134
|
|
84
79
|
if (allCaps && !title.match(/\s/))
|
|
85
80
|
return title;
|
|
86
|
-
let sentenceCased = title;
|
|
87
81
|
const tokens = (0, tokenizer_1.tokenize)(title, options.markup);
|
|
82
|
+
let sentenceCased = '';
|
|
88
83
|
for (const token of tokens) {
|
|
89
|
-
if (token.
|
|
90
|
-
sentenceCased
|
|
84
|
+
if (token.hyphenated && token.shape[0] === 'x') {
|
|
85
|
+
sentenceCased += token.text;
|
|
86
|
+
}
|
|
87
|
+
else if (token.hyphenated) {
|
|
88
|
+
for (const t of token.hyphenated) {
|
|
89
|
+
sentenceCased += wordSC(t, allCaps, options.subSentenceCapitalization, true);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
else {
|
|
93
|
+
sentenceCased += wordSC(token, allCaps, options.subSentenceCapitalization, false);
|
|
91
94
|
}
|
|
92
95
|
}
|
|
93
96
|
if (options.markup) {
|
package/sentence-case.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sentence-case.js","sourceRoot":"","sources":["sentence-case.ts"],"names":[],"mappings":";AAAA,sDAAsD
|
|
1
|
+
{"version":3,"file":"sentence-case.js","sourceRoot":"","sources":["sentence-case.ts"],"names":[],"mappings":";AAAA,sDAAsD;;;AAEtD,2CAAiE;AACjE,mCAA+B;AAE/B,4CAA4C;AAC5C,mKAAmK;AAEnK,SAAS,SAAS,CAAC,CAAS;IAC1B,OAAO,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC,CAAA;AACjF,CAAC;AAED,SAAS,MAAM,CAAC,KAAY,EAAE,OAAgB,EAAE,WAAoB,EAAE,UAAmB;IACvF,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAA;IAC5D,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,KAAK,CAAC,IAAI,CAAA;IAC5C,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IACzD,IAAI,WAAW,IAAI,KAAK,CAAC,gBAAgB,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,GAAG,CAAA;IAEjF,IAAI,CAAC,WAAW,IAAI,KAAK,CAAC,gBAAgB,CAAC,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;QACnE,OAAO,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,8BAAkB,EAAE,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAA;IAC/G,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,KAAK,aAAa;QAAE,OAAO,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAA;IACpE,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC,IAAI,CAAA;IAElD,0IAA0I;IAE1I,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;QAAE,OAAO,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAA;IAE1F,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC;QAAE,OAAO,KAAK,CAAC,IAAI,CAAA;IAE/D,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;IAChD,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC;QAAE,OAAO,KAAK,CAAC,IAAI,CAAA;IAC1D,IAAI,KAAK,CAAC,KAAK,CAAC,oBAAoB,CAAC;QAAE,OAAO,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAA;IACtE,IAAI,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC;QAAE,OAAO,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAA;IAElF,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC,IAAI,CAAA;IAC/C,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,KAAK,CAAC,IAAI,CAAA;IAE1C,OAAO,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAA;AACjC,CAAC;AAUD,SAAgB,cAAc,CAAC,KAAa,EAAE,UAAmB,EAAE;IACjE,OAAO,GAAG,IAAA,aAAK,EAAC,OAAO,EAAE;QACvB,cAAc,EAAE,IAAI;QACpB,yBAAyB,EAAE,IAAI;QAC/B,KAAK,EAAE,KAAK;KACb,CAAC,CAAA;IAEF,MAAM,OAAO,GAAG,KAAK,KAAK,KAAK,CAAC,WAAW,EAAE,CAAA;IAC7C,MAAM,QAAQ,GAAG,KAAK,KAAK,KAAK,CAAC,WAAW,EAAE,CAAA;IAC9C,IAAI,OAAO,CAAC,KAAK,IAAI,CAAC,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC3C,IAAI,MAAM,GAAG,KAAK,CAAA;QAClB,IAAI,OAAO,CAAC,MAAM;YAAE,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;QAClG,IAAI,OAAO,CAAC,MAAM;YAAE,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;QAC/D,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YAC3B,KAAK,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;gBAC/C,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;YAChC,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG;YACZ,KAAK,EAAE,IAAA,oBAAQ,EAAC,MAAM,CAAC;YACvB,EAAE,EAAE,CAAC;YACL,KAAK,EAAE,CAAC;SACT,CAAA;QAED,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;YAC/B,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,GAAC,CAAC,CAAC,CAAC,IAAI,KAAK,YAAY,CAAC,EAAE,CAAC;gBACjK,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;YACjE,CAAC;QACH,CAAC,CAAC,CAAA;QACF,IAAI,KAAK,CAAC,EAAE,IAAI,KAAK,CAAC,EAAE,IAAI,KAAK,CAAC,KAAK;YAAE,OAAO,KAAK,CAAA;IACvD,CAAC;IAED,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA,CAAC,gDAAgD;IAC/E,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAA;IAE/C,MAAM,MAAM,GAAG,IAAA,oBAAQ,EAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;IAE9C,IAAI,aAAa,GAAG,EAAE,CAAA;IACtB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YAC/C,aAAa,IAAI,KAAK,CAAC,IAAI,CAAA;QAC7B,CAAC;aACI,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YAC1B,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;gBACjC,aAAa,IAAI,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,yBAAyB,EAAE,IAAI,CAAC,CAAA;YAC9E,CAAC;QACH,CAAC;aACI,CAAC;YACJ,aAAa,IAAI,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,yBAAyB,EAAE,KAAK,CAAC,CAAA;QACnF,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACnD,aAAa,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAA;QAC7H,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,aAAa,GAAG,aAAa;aAC1B,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,GAAW,EAAE,MAAc,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;IAC5H,CAAC;IAED,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;YAC/C,aAAa,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,KAAa,EAAE,MAAc,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;QAC7H,CAAC;IACH,CAAC;IAED,OAAO,aAAa,CAAA;AACtB,CAAC;AAvED,wCAuEC"}
|
package/tokenizer.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export declare const connectedInnerWord: RegExp;
|
|
1
2
|
export type Token = {
|
|
2
3
|
type: string;
|
|
3
4
|
subtype: string;
|
|
@@ -7,6 +8,7 @@ export type Token = {
|
|
|
7
8
|
shape: string;
|
|
8
9
|
sentenceStart: boolean;
|
|
9
10
|
subSentenceStart: boolean;
|
|
11
|
+
hyphenated?: Token[];
|
|
10
12
|
};
|
|
11
13
|
export declare function tokenize(title: string, markup?: RegExp): Token[];
|
|
12
14
|
//# sourceMappingURL=tokenizer.d.ts.map
|
package/tokenizer.js
CHANGED
|
@@ -3,28 +3,30 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.tokenize = void 0;
|
|
6
|
+
exports.tokenize = exports.connectedInnerWord = void 0;
|
|
7
7
|
const moo_1 = __importDefault(require("moo"));
|
|
8
|
-
const xregexp_1 = __importDefault(require("xregexp"));
|
|
9
8
|
const charCategories = require('xregexp/tools/output/categories');
|
|
10
|
-
function
|
|
9
|
+
function char(cats, extra, neg = false) {
|
|
11
10
|
return `[${neg ? '^' : ''}${cats.map(cat => cat.bmp).join('')}${extra || ''}]`;
|
|
12
11
|
}
|
|
13
|
-
const
|
|
14
|
-
const
|
|
12
|
+
const ciwLu = char(charCategories.filter(cat => cat.name === 'Lu'));
|
|
13
|
+
const ciwLl = char(charCategories.filter(cat => cat.name === 'Ll'));
|
|
14
|
+
exports.connectedInnerWord = new RegExp(`-${ciwLu}${ciwLl}*(?=-|$)`, 'g');
|
|
15
|
+
const L = char(charCategories.filter(cat => cat.name === 'L'));
|
|
16
|
+
const LNM = char(charCategories.filter(cat => cat.name.match(/^[LNM]/)), '\u00AD\u2060');
|
|
15
17
|
const W = `${LNM}*?${L}${LNM}*`;
|
|
16
|
-
const B = `(?=(?:${
|
|
18
|
+
const B = `(?=(?:${char(charCategories.filter(cat => cat.name.match(/^[LNM]/)), '\u00AD\u2060').replace(/^./, '[^')}|$))`;
|
|
17
19
|
const Word = new RegExp(`${W}${B}`);
|
|
18
|
-
const P = new RegExp(
|
|
19
|
-
const Lu =
|
|
20
|
+
const P = new RegExp(char(charCategories.filter(cat => cat.name.match(/^P/))));
|
|
21
|
+
const Lu = char(charCategories.filter(cat => cat.name === 'Lu'), '\u2060');
|
|
20
22
|
const Acronym = new RegExp(`(?:(?:(?:${Lu}[.]){2,}${B})|(?:(?:vs?[.])(?=[ \t\n\r\u00A0])))`);
|
|
21
23
|
const Contraction = new RegExp(`${W}['’]${W}${B}`);
|
|
22
|
-
const Compound = new RegExp(`${W}(?:-${W})+${B}`);
|
|
23
24
|
const Whitespace = /[ \t\n\r\u00A0]+/;
|
|
24
25
|
const Ordinal = new RegExp(`\\d+(?:st|nd|rd|th)${B}`);
|
|
25
26
|
const Email = new RegExp(`[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:[.][A-Za-z0-9-]+)+${B}`);
|
|
26
27
|
const Handle = new RegExp(`@[A-Za-z0-9-]{2,}${B}`);
|
|
27
28
|
const Int = new RegExp(`[0-9]+${B}`);
|
|
29
|
+
const Domain = new RegExp(`${W}(?:[.]${W})+${B}`);
|
|
28
30
|
const ComplexPreposition = /^([^ \t\n\r\u00A0]+)([ \t\n\r\u00A0]+)([^ \t\n\r\u00A0]+)(?:([ \t\n\r\u00A0]+)([^ \t\n\r\u00A0]+))?$/;
|
|
29
31
|
function ci(s) {
|
|
30
32
|
return s
|
|
@@ -37,10 +39,10 @@ const lexer = moo_1.default.compile({
|
|
|
37
39
|
'word-preposition': Preposition,
|
|
38
40
|
'word-acronym': Acronym,
|
|
39
41
|
'word-contraction': Contraction,
|
|
40
|
-
'word-compound': Compound,
|
|
41
42
|
'word-ordinal': Ordinal,
|
|
42
43
|
email: Email,
|
|
43
44
|
handle: Handle,
|
|
45
|
+
domain: Domain,
|
|
44
46
|
word: Word,
|
|
45
47
|
number: Int, // eslint-disable-line id-blacklist
|
|
46
48
|
'punctuation-end': /[?.!](?=[ \t\n\r\u00A0]|$)/,
|
|
@@ -50,14 +52,36 @@ const lexer = moo_1.default.compile({
|
|
|
50
52
|
whitespace: { match: /[ \t\n\r\u00A0]/, lineBreaks: true },
|
|
51
53
|
other: { match: /[\s\S]/, lineBreaks: true },
|
|
52
54
|
});
|
|
53
|
-
const
|
|
54
|
-
[(
|
|
55
|
-
[new RegExp(
|
|
56
|
-
[(0
|
|
55
|
+
const shaper = [
|
|
56
|
+
[new RegExp(char(charCategories.filter(cat => cat.name === 'Lu')), 'g'), 'X'],
|
|
57
|
+
[new RegExp(char(charCategories.filter(cat => cat.name.match(/^L[^Cu]/))), 'g'), 'x'],
|
|
58
|
+
[new RegExp(char(charCategories.filter(cat => cat.name[0] === 'N')), 'g'), 'd'],
|
|
57
59
|
[/’/g, "'"],
|
|
58
60
|
[/–/g, '-'],
|
|
59
61
|
[/[\u2060\u00AD]/g, ''],
|
|
60
62
|
];
|
|
63
|
+
function shape(s) {
|
|
64
|
+
for (const [re, repl] of shaper) {
|
|
65
|
+
s = s.replace(re, repl);
|
|
66
|
+
}
|
|
67
|
+
return s;
|
|
68
|
+
}
|
|
69
|
+
function combine(tokens) {
|
|
70
|
+
const combined = Object.assign({}, tokens[0]);
|
|
71
|
+
for (const t of tokens.slice(1)) {
|
|
72
|
+
combined.text += t.text;
|
|
73
|
+
combined.end = t.end;
|
|
74
|
+
combined.shape += t.shape;
|
|
75
|
+
}
|
|
76
|
+
return combined;
|
|
77
|
+
}
|
|
78
|
+
function hyphenate(t) {
|
|
79
|
+
if (t.type === 'word')
|
|
80
|
+
return 'w';
|
|
81
|
+
if (t.text === '-')
|
|
82
|
+
return '-';
|
|
83
|
+
return ' ';
|
|
84
|
+
}
|
|
61
85
|
function tokenize(title, markup) {
|
|
62
86
|
if (markup)
|
|
63
87
|
title = title.replace(markup, match => '\u2060'.repeat(match.length));
|
|
@@ -72,7 +96,7 @@ function tokenize(title, markup) {
|
|
|
72
96
|
text: token.text,
|
|
73
97
|
start: token.offset,
|
|
74
98
|
end: token.offset + token.text.length - 1,
|
|
75
|
-
shape:
|
|
99
|
+
shape: shape(token.text),
|
|
76
100
|
sentenceStart: type === 'word' && sentenceStart,
|
|
77
101
|
subSentenceStart: type === 'word' && subSentenceStart,
|
|
78
102
|
});
|
|
@@ -92,14 +116,6 @@ function tokenize(title, markup) {
|
|
|
92
116
|
}
|
|
93
117
|
}
|
|
94
118
|
const stack = tokens.splice(0);
|
|
95
|
-
const combine = (ts) => {
|
|
96
|
-
for (const t of ts.slice(1)) {
|
|
97
|
-
ts[0].text += t.text;
|
|
98
|
-
ts[0].end = t.end;
|
|
99
|
-
ts[0].shape += t.shape;
|
|
100
|
-
}
|
|
101
|
-
return ts[0];
|
|
102
|
-
};
|
|
103
119
|
let cpt;
|
|
104
120
|
let cps;
|
|
105
121
|
while (stack.length) {
|
|
@@ -117,15 +133,16 @@ function tokenize(title, markup) {
|
|
|
117
133
|
}
|
|
118
134
|
continue;
|
|
119
135
|
}
|
|
120
|
-
//
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
136
|
+
// hyphenated words
|
|
137
|
+
const pat = stack.map(t => hyphenate(t)).join('');
|
|
138
|
+
if (cpt = pat.match(/^w(-w)+(?= |$)/)) {
|
|
139
|
+
const hyphenated = stack.splice(0, cpt[0].length);
|
|
140
|
+
tokens.push(Object.assign(Object.assign({}, combine(hyphenated)), { hyphenated }));
|
|
124
141
|
continue;
|
|
125
142
|
}
|
|
126
143
|
tokens.push(stack.shift());
|
|
127
144
|
}
|
|
128
|
-
return tokens;
|
|
145
|
+
return markup ? tokens.map(token => (Object.assign(Object.assign({}, token), { text: title.substring(token.start, token.end + 1) }))) : tokens;
|
|
129
146
|
}
|
|
130
147
|
exports.tokenize = tokenize;
|
|
131
148
|
//# sourceMappingURL=tokenizer.js.map
|
package/tokenizer.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["tokenizer.ts"],"names":[],"mappings":";;;;;;AAAA,8CAAqB;
|
|
1
|
+
{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["tokenizer.ts"],"names":[],"mappings":";;;;;;AAAA,8CAAqB;AAYrB,MAAM,cAAc,GAAmB,OAAO,CAAC,iCAAiC,CAAC,CAAA;AAEjF,SAAS,IAAI,CAAC,IAAoB,EAAE,KAAc,EAAE,GAAG,GAAC,KAAK;IAC3D,OAAO,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,KAAK,IAAI,EAAE,GAAG,CAAA;AAChF,CAAC;AAED,MAAM,KAAK,GAAW,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAA;AAC3E,MAAM,KAAK,GAAW,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAA;AAC9D,QAAA,kBAAkB,GAAG,IAAI,MAAM,CAAC,IAAI,KAAK,GAAG,KAAK,UAAU,EAAE,GAAG,CAAC,CAAA;AAE9E,MAAM,CAAC,GAAW,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAA;AACtE,MAAM,GAAG,GAAW,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,cAAc,CAAC,CAAA;AAChG,MAAM,CAAC,GAAG,GAAG,GAAG,KAAK,CAAC,GAAG,GAAG,GAAG,CAAA;AAC/B,MAAM,CAAC,GAAG,SAAS,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAA;AAEzH,MAAM,IAAI,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;AACnC,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;AAE9E,MAAM,EAAE,GAAW,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,QAAQ,CAAC,CAAA;AAClF,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,YAAY,EAAE,WAAW,CAAC,sCAAsC,CAAC,CAAA;AAE5F,MAAM,WAAW,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;AAClD,MAAM,UAAU,GAAG,kBAAkB,CAAA;AACrC,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAA;AACrD,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,uDAAuD,CAAC,EAAE,CAAC,CAAA;AACpF,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAA;AAClD,MAAM,GAAG,GAAG,IAAI,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;AACpC,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;AAEjD,MAAM,kBAAkB,GAAG,sGAAsG,CAAA;AAEjI,SAAS,EAAE,CAAC,CAAS;IACnB,OAAO,CAAC;SACL,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,WAAW,EAAE,GAAG,KAAK,CAAC,WAAW,EAAE,GAAG,CAAC;SAC7E,OAAO,CAAC,GAAG,EAAE,UAAU,CAAC,MAAM,CAAE,CAAA;AACrC,CAAC;AACD,MAAM,YAAY,GAAW,OAAO,CAAC,qBAAqB,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AAC9F,MAAM,WAAW,GAAG,IAAI,MAAM,CAAC,MAAM,YAAY,IAAI,CAAC,EAAE,CAAC,CAAA;AAEzD,MAAM,KAAK,GAAG,aAAG,CAAC,OAAO,CAAC;IACxB,kBAAkB,EAAM,WAAW;IACnC,cAAc,EAAU,OAAO;IAC/B,kBAAkB,EAAM,WAAW;IACnC,cAAc,EAAU,OAAO;IAC/B,KAAK,EAAmB,KAAK;IAC7B,MAAM,EAAkB,MAAM;IAC9B,MAAM,EAAkB,MAAM;IAC9B,IAAI,EAAoB,IAAI;IAC5B,MAAM,EAAkB,GAAG,EAAE,mCAAmC;IAChE,iBAAiB,EAAO,4BAA4B;IACpD,mBAAmB,EAAK,sBAAsB;IAC9C,sBAAsB,EAAE,WAAW;IACnC,WAAW,EAAa,CAAC;IACzB,UAAU,EAAc,EAAE,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,IAAI,EAAE;IACtE,KAAK,EAAmB,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,EAAE;CAC9D,CAAC,CAAA;AAEF,MAAM,MAAM,GAAuB;IACjC,CAAE,IAAI,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAE;IAC/E,CAAE,IAAI,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAE;IACvF,CAAE,IAAI,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAE;IACjF,CAAE,IAAI,EAAE,GAAG,CAAE;IACb,CAAE,IAAI,EAAE,GAAG,CAAE;IACb,CAAE,iBAAiB,EAAE,EAAE,CAAE;CAC1B,CAAA;AACD,SAAS,KAAK,CAAC,CAAS;IACtB,KAAK,MAAM,CAAE,EAAE,EAAE,IAAI,CAAE,IAAI,MAAM,EAAE,CAAC;QAClC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,CAAA;IACzB,CAAC;IACD,OAAO,CAAC,CAAA;AACV,CAAC;AAcD,SAAS,OAAO,CAAC,MAAe;IAC9B,MAAM,QAAQ,qBAAQ,MAAM,CAAC,CAAC,CAAC,CAAE,CAAA;IACjC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAChC,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,CAAA;QACvB,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAA;QACpB,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,CAAA;IAC3B,CAAC;IACD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,SAAS,SAAS,CAAC,CAAQ;IACzB,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,GAAG,CAAA;IACjC,IAAI,CAAC,CAAC,IAAI,KAAK,GAAG;QAAE,OAAO,GAAG,CAAA;IAC9B,OAAO,GAAG,CAAA;AACZ,CAAC;AAED,SAAgB,QAAQ,CAAC,KAAa,EAAE,MAAe;IACrD,IAAI,MAAM;QAAE,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;IAEjF,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IAClB,MAAM,MAAM,GAAY,EAAE,CAAA;IAC1B,IAAI,aAAa,GAAG,IAAI,CAAA;IACxB,IAAI,gBAAgB,GAAG,KAAK,CAAA;IAC5B,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,CAAE,IAAI,EAAE,OAAO,CAAE,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QAE/F,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,GAAG,EAAE,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;YACzC,KAAK,EAAE,KAAK,CAAS,KAAK,CAAC,IAAI,CAAC;YAChC,aAAa,EAAE,IAAI,KAAK,MAAM,IAAI,aAAa;YAC/C,gBAAgB,EAAE,IAAI,KAAK,MAAM,IAAI,gBAAgB;SACtD,CAAC,CAAA;QAEF,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,iBAAiB;gBACpB,aAAa,GAAG,IAAI,CAAA;gBACpB,MAAK;YACP,KAAK,mBAAmB;gBACtB,gBAAgB,GAAG,IAAI,CAAA;gBACvB,MAAK;YACP;gBACE,IAAI,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,CAAC;oBAC9B,aAAa,GAAG,KAAK,CAAA;oBACrB,gBAAgB,GAAG,KAAK,CAAA;gBAC1B,CAAC;gBACD,MAAK;QACT,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;IAE9B,IAAI,GAAqB,CAAA;IACzB,IAAI,GAAqB,CAAA;IACzB,OAAO,KAAK,CAAC,MAAM,EAAE,CAAC;QACpB,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,aAAa,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,EAAE,CAAC;YAC9I,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,EAAE,CAAA;YAE7B,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK,CAAA;YACzB,IAAI,GAAG,CAAA;YACP,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,EAAC,MAAM,EAAE,CAAC,EAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAC,CAAC,CAAC,EAAE,CAAC;gBACvD,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;oBAAE,MAAK;gBAElB,GAAG,GAAG,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAA;gBAC/B,MAAM,CAAC,IAAI,iCACN,OAAO,KACV,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,EACZ,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,EACb,KAAK;oBACL,GAAG,EACH,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,EACzC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAClC,CAAA;gBACF,KAAK,GAAG,GAAG,GAAG,CAAC,CAAA;YACjB,CAAC;YACD,SAAQ;QACV,CAAC;QAED,mBAAmB;QACnB,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACjD,IAAI,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACtC,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAA;YACjD,MAAM,CAAC,IAAI,iCAAM,OAAO,CAAC,UAAU,CAAC,KAAE,UAAU,IAAG,CAAA;YACnD,SAAQ;QACV,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAA;IAC5B,CAAC;IAED,OAAO,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,iCAAM,KAAK,KAAE,IAAI,EAAE,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,IAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;AACjH,CAAC;AA5ED,4BA4EC"}
|