@retorquere/bibtex-parser 9.0.3 → 9.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/sentence-case.js +42 -15
- package/sentence-case.js.map +1 -1
- package/tokenizer.d.ts +0 -1
- package/tokenizer.js +74 -30
- package/tokenizer.js.map +1 -1
package/package.json
CHANGED
package/sentence-case.js
CHANGED
|
@@ -1,44 +1,76 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/* eslint-disable @typescript-eslint/unbound-method */
|
|
3
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
4
|
+
if (k2 === undefined) k2 = k;
|
|
5
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
6
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
7
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
8
|
+
}
|
|
9
|
+
Object.defineProperty(o, k2, desc);
|
|
10
|
+
}) : (function(o, m, k, k2) {
|
|
11
|
+
if (k2 === undefined) k2 = k;
|
|
12
|
+
o[k2] = m[k];
|
|
13
|
+
}));
|
|
14
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
15
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
16
|
+
}) : function(o, v) {
|
|
17
|
+
o["default"] = v;
|
|
18
|
+
});
|
|
19
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
20
|
+
if (mod && mod.__esModule) return mod;
|
|
21
|
+
var result = {};
|
|
22
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
23
|
+
__setModuleDefault(result, mod);
|
|
24
|
+
return result;
|
|
25
|
+
};
|
|
3
26
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
27
|
exports.toSentenceCase = void 0;
|
|
5
28
|
const tokenizer_1 = require("./tokenizer");
|
|
6
29
|
const merge_1 = require("./merge");
|
|
30
|
+
const rx = __importStar(require("./re"));
|
|
31
|
+
const Lu = rx.match(rx.categories.filter(cat => cat.name === 'Lu' || cat.name === 'Lt'));
|
|
32
|
+
const Ll = rx.match(rx.categories.filter(cat => cat.name === 'Ll'));
|
|
33
|
+
const connectedInnerWord = new RegExp(`-${Lu}${Ll}*(?=-|$)`, 'g');
|
|
7
34
|
// eslint-disable-next-line no-magic-numbers
|
|
8
35
|
// const show = (obj: any): string => JSON.stringify(obj, null, 2).replace(/[\u007F-\uFFFF]/g, chr => `\\u${(`0000${chr.charCodeAt(0).toString(16)}`).substr(-4)}`)
|
|
9
36
|
function titleCase(s) {
|
|
10
37
|
return s.replace(/^(.)(.+)/, (match, car, cdr) => `${car}${cdr.toLowerCase()}`);
|
|
11
38
|
}
|
|
12
39
|
function wordSC(token, allCaps, subSentence, hyphenated) {
|
|
40
|
+
// domains are case-insensitive
|
|
13
41
|
if (token.type === 'domain')
|
|
14
42
|
return token.text.toLowerCase();
|
|
43
|
+
// punctuation needs no handling, and websites can have case-sensitive stuff in the URL
|
|
15
44
|
if (token.type !== 'word')
|
|
16
45
|
return token.text;
|
|
46
|
+
// I'll, I'm
|
|
17
47
|
if (token.text.match(/^I'/))
|
|
18
48
|
return titleCase(token.text);
|
|
49
|
+
// starting 'a'
|
|
19
50
|
if (subSentence && token.subSentenceStart && token.text.match(/^a$/i))
|
|
20
51
|
return 'a';
|
|
52
|
+
// (sub)sentence start
|
|
21
53
|
if ((subSentence && token.subSentenceStart) || token.sentenceStart) {
|
|
22
|
-
return allCaps ? titleCase(token.text) : token.text.replace(
|
|
54
|
+
return allCaps ? titleCase(token.text) : token.text.replace(connectedInnerWord, match => match.toLowerCase());
|
|
23
55
|
}
|
|
56
|
+
// prepositions should already have been lowercased, but OK
|
|
24
57
|
if (token.subtype === 'preposition')
|
|
25
58
|
return token.text.toLowerCase();
|
|
59
|
+
// keep acronyms
|
|
26
60
|
if (token.subtype === 'acronym')
|
|
27
61
|
return token.text;
|
|
28
62
|
// if (!allCaps && token.shape.match(/^[Xxd]+(-[Xxd]+)+/)) return XRegExp.replace(token.text, connectedWord, match => match.toLowerCase())
|
|
63
|
+
// single-letter capitals are usually some kind of noun, but not when they're part of an hyphenated word
|
|
29
64
|
if (token.text.match(/^[B-Z]$/))
|
|
30
65
|
return hyphenated ? token.text.toLowerCase() : token.text;
|
|
31
|
-
if (!allCaps && token.shape.match(/^[-X]+$/))
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
if (
|
|
35
|
-
|
|
36
|
-
if (shape.match(/^X[xd]*(-[Xxd]*)*$/))
|
|
37
|
-
return token.text.toLowerCase();
|
|
66
|
+
// ?? if (!allCaps && token.shape.match(/^[-X]+$/)) return token.text
|
|
67
|
+
const shape = token.shape.replace(/[^-Xxd]/g, '');
|
|
68
|
+
// ?? if (!allCaps && shape.match(/^[Xd-]+$/)) return token.text
|
|
69
|
+
// hyphenated ?? if (shape.match(/^X[xd]*(-[Xxd]*)*$/)) return token.text.toLowerCase()
|
|
70
|
+
// all-caps and numbers
|
|
38
71
|
if (shape.match(/^[Xd]+$/))
|
|
39
72
|
return allCaps ? token.text.toLowerCase() : token.text;
|
|
40
|
-
if (token.text.includes('.'))
|
|
41
|
-
return token.text;
|
|
73
|
+
// domain ?? if (token.text.includes('.')) return token.text
|
|
42
74
|
if (shape.match(/x.*X/))
|
|
43
75
|
return token.text;
|
|
44
76
|
return token.text.toLowerCase();
|
|
@@ -57,11 +89,6 @@ function toSentenceCase(title, options = {}) {
|
|
|
57
89
|
$title = $title.replace(options.nocase, match => match.match(/\s/) ? ' ' : '');
|
|
58
90
|
if (options.markup)
|
|
59
91
|
$title = $title.replace(options.markup, '');
|
|
60
|
-
if (options.preserveQuoted) {
|
|
61
|
-
for (const q of [/“.*?”/g, /‘.*?’/g, /".*?"/g]) {
|
|
62
|
-
$title = $title.replace(q, '');
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
92
|
const guess = {
|
|
66
93
|
words: (0, tokenizer_1.tokenize)($title),
|
|
67
94
|
sc: 0,
|
package/sentence-case.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sentence-case.js","sourceRoot":"","sources":["sentence-case.ts"],"names":[],"mappings":";AAAA,sDAAsD
|
|
1
|
+
{"version":3,"file":"sentence-case.js","sourceRoot":"","sources":["sentence-case.ts"],"names":[],"mappings":";AAAA,sDAAsD;;;;;;;;;;;;;;;;;;;;;;;;;;AAEtD,2CAA6C;AAC7C,mCAA+B;AAE/B,yCAA0B;AAE1B,MAAM,EAAE,GAAW,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAA;AAChG,MAAM,EAAE,GAAW,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAA;AAC3E,MAAM,kBAAkB,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG,CAAC,CAAA;AAEjE,4CAA4C;AAC5C,mKAAmK;AAEnK,SAAS,SAAS,CAAC,CAAS;IAC1B,OAAO,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC,CAAA;AACjF,CAAC;AAED,SAAS,MAAM,CAAC,KAAY,EAAE,OAAgB,EAAE,WAAoB,EAAE,UAAmB;IAEvF,+BAA+B;IAC/B,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAA;IAE5D,uFAAuF;IACvF,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,KAAK,CAAC,IAAI,CAAA;IAE5C,YAAY;IACZ,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAEzD,eAAe;IACf,IAAI,WAAW,IAAI,KAAK,CAAC,gBAAgB,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,GAAG,CAAA;IAEjF,sBAAsB;IACtB,IAAI,CAAC,WAAW,IAAI,KAAK,CAAC,gBAAgB,CAAC,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;QACnE,OAAO,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAA;IAC/G,CAAC;IAED,2DAA2D;IAC3D,IAAI,KAAK,CAAC,OAAO,KAAK,aAAa;QAAE,OAAO,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAA;IAEpE,gBAAgB;IAChB,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC,IAAI,CAAA;IAElD,0IAA0I;IAE1I,wGAAwG;IACxG,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;QAAE,OAAO,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAA;IAE1F,qEAAqE;IAErE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAA;IACjD,gEAAgE;IAEhE,uFAAuF;IAEvF,uBAAuB;IACvB,IAAI,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC;QAAE,OAAO,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAA;IAElF,4DAA4D;IAC5D,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,KAAK,CAAC,IAAI,CAAA;IAE1C,OAAO,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAA;AACjC,CAAC;AAUD,SAAgB,cAAc,CAAC,KAAa,EAAE,UAAmB,EAAE;IACjE,OAAO,GAAG,IAAA,aAAK,EAAC,OAAO,EAAE;QACvB,cAAc,EAAE,IAAI;QACpB,yBAAyB,EAAE,IAAI;QAC/B,KAAK,EAAE,KAAK;KACb,CAAC,CAAA;IAEF,MAAM,OAAO,GAAG,KAAK,KAAK,KAAK,CAAC,WAAW,EAAE,CAAA;IAC7C,MAAM,QAAQ,GAAG,KAAK,KAAK,KAAK,CAAC,WAAW,EAAE,CAAA;IAC9C,IAAI,OAAO,CAAC,KAAK,IAAI,CAAC,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC3C,IAAI,MAAM,GAAG,KAAK,CAAA;QAClB,IAAI,OAAO,CAAC,MAAM;YAAE,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;QAClG,IAAI,OAAO,CAAC,MAAM;YAAE,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;QAE/D,MAAM,KAAK,GAAG;YACZ,KAAK,EAAE,IAAA,oBAAQ,EAAC,MAAM,CAAC;YACvB,EAAE,EAAE,CAAC;YACL,KAAK,EAAE,CAAC;SACT,CAAA;QAED,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;YAC/B,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,GAAC,CAAC,CAAC,CAAC,IAAI,KAAK,YAAY,CAAC,EAAE,CAAC;gBACjK,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;YACjE,CAAC;QACH,CAAC,CAAC,CAAA;QACF,IAAI,KAAK,CAAC,EAAE,IAAI,KAAK,CAAC,EAAE,IAAI,KAAK,CAAC,KAAK;YAAE,OAAO,KAAK,CAAA;IACvD,CAAC;IAED,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA,CAAC,gDAAgD;IAC/E,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAA;IAE/C,MAAM,MAAM,GAAG,IAAA,oBAAQ,EAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;IAE9C,IAAI,aAAa,GAAG,EAAE,CAAA;IACtB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YAC/C,aAAa,IAAI,KAAK,CAAC,IAAI,CAAA;QAC7B,CAAC;aACI,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YAC1B,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;gBACjC,aAAa,IAAI,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,yBAAyB,EAAE,IAAI,CAAC,CAAA;YAC9E,CAAC;QACH,CAAC;aACI,CAAC;YACJ,aAAa,IAAI,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,yBAAyB,EAAE,KAAK,CAAC,CAAA;QACnF,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACnD,aAAa,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAA;QAC7H,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,aAAa,GAAG,aAAa;aAC1B,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,GAAW,EAAE,MAAc,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;IAC5H,CAAC;IAED,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;YAC/C,aAAa,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,KAAa,EAAE,MAAc,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;QAC7H,CAAC;IACH,CAAC;IAED,OAAO,aAAa,CAAA;AACtB,CAAC;AAlED,wCAkEC"}
|
package/tokenizer.d.ts
CHANGED
package/tokenizer.js
CHANGED
|
@@ -1,32 +1,52 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
2
25
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
26
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
27
|
};
|
|
5
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.tokenize =
|
|
29
|
+
exports.tokenize = void 0;
|
|
7
30
|
const moo_1 = __importDefault(require("moo"));
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
const
|
|
13
|
-
const ciwLl = char(charCategories.filter(cat => cat.name === 'Ll'));
|
|
14
|
-
exports.connectedInnerWord = new RegExp(`-${ciwLu}${ciwLl}*(?=-|$)`, 'g');
|
|
15
|
-
const L = char(charCategories.filter(cat => cat.name === 'L'));
|
|
16
|
-
const LNM = char(charCategories.filter(cat => cat.name.match(/^[LNM]/)), '\u00AD\u2060');
|
|
31
|
+
// eslint-disable-next-line no-magic-numbers
|
|
32
|
+
// const show = (obj: any): string => JSON.stringify(obj, null, 2).replace(/[\u007F-\uFFFF]/g, chr => `\\u${(`0000${chr.charCodeAt(0).toString(16)}`).substr(-4)}`)
|
|
33
|
+
const rx = __importStar(require("./re"));
|
|
34
|
+
const L = rx.match(rx.categories.filter(cat => cat.name === 'L'));
|
|
35
|
+
const LNM = rx.match(rx.categories.filter(cat => cat.name.match(/^[LNM]/)), '\u00AD\u2060');
|
|
17
36
|
const W = `${LNM}*?${L}${LNM}*`;
|
|
18
|
-
const B = `(?=(?:${
|
|
37
|
+
const B = `(?=(?:${rx.match(rx.categories.filter(cat => cat.name.match(/^[LNM]/)), '\u00AD\u2060').replace(/^./, '[^')}|$))`;
|
|
19
38
|
const Word = new RegExp(`${W}${B}`);
|
|
20
|
-
const P = new RegExp(
|
|
21
|
-
const Lu =
|
|
39
|
+
const P = new RegExp(rx.match(rx.categories.filter(cat => cat.name.match(/^P/))));
|
|
40
|
+
const Lu = rx.match(rx.categories.filter(cat => cat.name === 'Lu' || cat.name === 'Lt'), '\u2060');
|
|
22
41
|
const Acronym = new RegExp(`(?:(?:(?:${Lu}[.]){2,}${B})|(?:(?:vs?[.])(?=[ \t\n\r\u00A0])))`);
|
|
23
42
|
const Contraction = new RegExp(`${W}['’]${W}${B}`);
|
|
24
43
|
const Whitespace = /[ \t\n\r\u00A0]+/;
|
|
25
44
|
const Ordinal = new RegExp(`\\d+(?:st|nd|rd|th)${B}`);
|
|
26
45
|
const Email = new RegExp(`[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:[.][A-Za-z0-9-]+)+${B}`);
|
|
27
46
|
const Handle = new RegExp(`@[A-Za-z0-9-]{2,}${B}`);
|
|
28
|
-
const Int = new RegExp(
|
|
47
|
+
const Int = new RegExp(`\\d+${B}`);
|
|
29
48
|
const Domain = new RegExp(`${W}(?:[.]${W})+${B}`);
|
|
49
|
+
const Website = new RegExp(`https?://${W}(?:[.]${W})+(?:[^.!? \t\n\r\u00A0]+|[.!?]${LNM})+`);
|
|
30
50
|
const ComplexPreposition = /^([^ \t\n\r\u00A0]+)([ \t\n\r\u00A0]+)([^ \t\n\r\u00A0]+)(?:([ \t\n\r\u00A0]+)([^ \t\n\r\u00A0]+))?$/;
|
|
31
51
|
function ci(s) {
|
|
32
52
|
return s
|
|
@@ -42,6 +62,7 @@ const lexer = moo_1.default.compile({
|
|
|
42
62
|
'word-ordinal': Ordinal,
|
|
43
63
|
email: Email,
|
|
44
64
|
handle: Handle,
|
|
65
|
+
website: Website,
|
|
45
66
|
domain: Domain,
|
|
46
67
|
word: Word,
|
|
47
68
|
number: Int, // eslint-disable-line id-blacklist
|
|
@@ -52,20 +73,43 @@ const lexer = moo_1.default.compile({
|
|
|
52
73
|
whitespace: { match: /[ \t\n\r\u00A0]/, lineBreaks: true },
|
|
53
74
|
other: { match: /[\s\S]/, lineBreaks: true },
|
|
54
75
|
});
|
|
55
|
-
const
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
function shape(s) {
|
|
64
|
-
for (const [re, repl] of shaper) {
|
|
65
|
-
s = s.replace(re, repl);
|
|
76
|
+
const Shape = new class {
|
|
77
|
+
constructor() {
|
|
78
|
+
this.shapes = {};
|
|
79
|
+
this.re = {
|
|
80
|
+
X: new RegExp(rx.match(rx.categories.filter(cat => cat.name === 'Lu' || cat.name === 'Lt'))),
|
|
81
|
+
x: new RegExp(rx.match(rx.categories.filter(cat => cat.name.match(/^L[^Cut]/)))),
|
|
82
|
+
d: new RegExp(rx.match(rx.categories.filter(cat => cat.name[0] === 'N'))),
|
|
83
|
+
};
|
|
66
84
|
}
|
|
67
|
-
|
|
68
|
-
|
|
85
|
+
match(c) {
|
|
86
|
+
if (c.match(this.re.d))
|
|
87
|
+
return 'd';
|
|
88
|
+
if (c.toLowerCase() === c.toUpperCase())
|
|
89
|
+
return c;
|
|
90
|
+
if (c.match(this.re.X))
|
|
91
|
+
return 'X';
|
|
92
|
+
if (c.match(this.re.x))
|
|
93
|
+
return 'x';
|
|
94
|
+
if (c === '’')
|
|
95
|
+
return "'";
|
|
96
|
+
if (c === '–')
|
|
97
|
+
return '-';
|
|
98
|
+
if (c === '\u2060' || c === '\u00AD')
|
|
99
|
+
return '';
|
|
100
|
+
return c;
|
|
101
|
+
}
|
|
102
|
+
fetch(c) {
|
|
103
|
+
if (typeof this.shapes[c] === 'undefined')
|
|
104
|
+
this.shapes[c] = this.match(c);
|
|
105
|
+
return this.shapes[c];
|
|
106
|
+
}
|
|
107
|
+
shape(t) {
|
|
108
|
+
if (!this.shapes[t])
|
|
109
|
+
this.shapes[t] = Array.from(t).map(c => this.fetch(c)).join('');
|
|
110
|
+
return this.shapes[t];
|
|
111
|
+
}
|
|
112
|
+
};
|
|
69
113
|
function combine(tokens) {
|
|
70
114
|
const combined = Object.assign({}, tokens[0]);
|
|
71
115
|
for (const t of tokens.slice(1)) {
|
|
@@ -78,7 +122,7 @@ function combine(tokens) {
|
|
|
78
122
|
function hyphenate(t) {
|
|
79
123
|
if (t.type === 'word')
|
|
80
124
|
return 'w';
|
|
81
|
-
if (t.text === '-')
|
|
125
|
+
if (t.text === '-' || t.text === '–')
|
|
82
126
|
return '-';
|
|
83
127
|
return ' ';
|
|
84
128
|
}
|
|
@@ -96,7 +140,7 @@ function tokenize(title, markup) {
|
|
|
96
140
|
text: token.text,
|
|
97
141
|
start: token.offset,
|
|
98
142
|
end: token.offset + token.text.length - 1,
|
|
99
|
-
shape: shape(token.text),
|
|
143
|
+
shape: Shape.shape(token.text),
|
|
100
144
|
sentenceStart: type === 'word' && sentenceStart,
|
|
101
145
|
subSentenceStart: type === 'word' && subSentenceStart,
|
|
102
146
|
});
|
|
@@ -108,7 +152,7 @@ function tokenize(title, markup) {
|
|
|
108
152
|
subSentenceStart = true;
|
|
109
153
|
break;
|
|
110
154
|
default:
|
|
111
|
-
if (type.match(/word|number/)) {
|
|
155
|
+
if (type.match(/word|number|handle|domain|website/)) {
|
|
112
156
|
sentenceStart = false;
|
|
113
157
|
subSentenceStart = false;
|
|
114
158
|
}
|
package/tokenizer.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["tokenizer.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["tokenizer.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,8CAAqB;AAErB,4CAA4C;AAC5C,mKAAmK;AAEnK,yCAA0B;AAE1B,MAAM,CAAC,GAAW,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAA;AACzE,MAAM,GAAG,GAAW,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,cAAc,CAAC,CAAA;AACnG,MAAM,CAAC,GAAG,GAAG,GAAG,KAAK,CAAC,GAAG,GAAG,GAAG,CAAA;AAC/B,MAAM,CAAC,GAAG,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAA;AAE5H,MAAM,IAAI,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;AACnC,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;AAEjF,MAAM,EAAE,GAAW,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,QAAQ,CAAC,CAAA;AAC1G,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,YAAY,EAAE,WAAW,CAAC,sCAAsC,CAAC,CAAA;AAE5F,MAAM,WAAW,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;AAClD,MAAM,UAAU,GAAG,kBAAkB,CAAA;AACrC,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAA;AACrD,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,uDAAuD,CAAC,EAAE,CAAC,CAAA;AACpF,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAA;AAClD,MAAM,GAAG,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;AAClC,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;AACjD,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,YAAY,CAAC,SAAS,CAAC,kCAAkC,GAAG,IAAI,CAAC,CAAA;AAE5F,MAAM,kBAAkB,GAAG,sGAAsG,CAAA;AAEjI,SAAS,EAAE,CAAC,CAAS;IACnB,OAAO,CAAC;SACL,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,WAAW,EAAE,GAAG,KAAK,CAAC,WAAW,EAAE,GAAG,CAAC;SAC7E,OAAO,CAAC,GAAG,EAAE,UAAU,CAAC,MAAM,CAAE,CAAA;AACrC,CAAC;AACD,MAAM,YAAY,GAAW,OAAO,CAAC,qBAAqB,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AAC9F,MAAM,WAAW,GAAG,IAAI,MAAM,CAAC,MAAM,YAAY,IAAI,CAAC,EAAE,CAAC,CAAA;AAEzD,MAAM,KAAK,GAAG,aAAG,CAAC,OAAO,CAAC;IACxB,kBAAkB,EAAM,WAAW;IACnC,cAAc,EAAU,OAAO;IAC/B,kBAAkB,EAAM,WAAW;IACnC,cAAc,EAAU,OAAO;IAC/B,KAAK,EAAmB,KAAK;IAC7B,MAAM,EAAkB,MAAM;IAC9B,OAAO,EAAiB,OAAO;IAC/B,MAAM,EAAkB,MAAM;IAC9B,IAAI,EAAoB,IAAI;IAC5B,MAAM,EAAkB,GAAG,EAAE,mCAAmC;IAChE,iBAAiB,EAAO,4BAA4B;IACpD,mBAAmB,EAAK,sBAAsB;IAC9C,sBAAsB,EAAE,WAAW;IACnC,WAAW,EAAa,CAAC;IACzB,UAAU,EAAc,EAAE,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,IAAI,EAAE;IACtE,KAAK,EAAmB,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,EAAE;CAC9D,CAAC,CAAA;AAEF,MAAM,KAAK,GAAG,IAAI;IAAA;QACR,WAAM,GAA2B,EAAE,CAAA;QACnC,OAAE,GAA2B;YACnC,CAAC,EAAE,IAAI,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC;YAC5F,CAAC,EAAE,IAAI,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAChF,CAAC,EAAE,IAAI,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;SAC1E,CAAA;IAsBH,CAAC;IApBS,KAAK,CAAC,CAAS;QACrB,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YAAE,OAAO,GAAG,CAAA;QAClC,IAAI,CAAC,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,WAAW,EAAE;YAAE,OAAO,CAAC,CAAA;QACjD,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YAAE,OAAO,GAAG,CAAA;QAClC,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YAAE,OAAO,GAAG,CAAA;QAClC,IAAI,CAAC,KAAK,GAAG;YAAE,OAAO,GAAG,CAAA;QACzB,IAAI,CAAC,KAAK,GAAG;YAAE,OAAO,GAAG,CAAA;QACzB,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,QAAQ;YAAE,OAAO,EAAE,CAAA;QAC/C,OAAO,CAAC,CAAA;IACV,CAAC;IAEO,KAAK,CAAC,CAAS;QACrB,IAAI,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,WAAW;YAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;QACzE,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;IACvB,CAAC;IAED,KAAK,CAAC,CAAS;QACb,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;YAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACpF,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;IACvB,CAAC;CACF,CAAA;AAcD,SAAS,OAAO,CAAC,MAAe;IAC9B,MAAM,QAAQ,qBAAQ,MAAM,CAAC,CAAC,CAAC,CAAE,CAAA;IACjC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAChC,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,CAAA;QACvB,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAA;QACpB,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,CAAA;IAC3B,CAAC;IACD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,SAAS,SAAS,CAAC,CAAQ;IACzB,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,GAAG,CAAA;IACjC,IAAI,CAAC,CAAC,IAAI,KAAK,GAAG,IAAI,CAAC,CAAC,IAAI,KAAK,GAAG;QAAE,OAAO,GAAG,CAAA;IAChD,OAAO,GAAG,CAAA;AACZ,CAAC;AAED,SAAgB,QAAQ,CAAC,KAAa,EAAE,MAAe;IACrD,IAAI,MAAM;QAAE,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;IAEjF,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IAClB,MAAM,MAAM,GAAY,EAAE,CAAA;IAC1B,IAAI,aAAa,GAAG,IAAI,CAAA;IACxB,IAAI,gBAAgB,GAAG,KAAK,CAAA;IAC5B,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,CAAE,IAAI,EAAE,OAAO,CAAE,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QAE/F,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,GAAG,EAAE,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;YACzC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAS,KAAK,CAAC,IAAI,CAAC;YACtC,aAAa,EAAE,IAAI,KAAK,MAAM,IAAI,aAAa;YAC/C,gBAAgB,EAAE,IAAI,KAAK,MAAM,IAAI,gBAAgB;SACtD,CAAC,CAAA;QAEF,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,iBAAiB;gBACpB,aAAa,GAAG,IAAI,CAAA;gBACpB,MAAK;YACP,KAAK,mBAAmB;gBACtB,gBAAgB,GAAG,IAAI,CAAA;gBACvB,MAAK;YACP;gBACE,IAAI,IAAI,CAAC,KAAK,CAAC,mCAAmC,CAAC,EAAE,CAAC;oBACpD,aAAa,GAAG,KAAK,CAAA;oBACrB,gBAAgB,GAAG,KAAK,CAAA;gBAC1B,CAAC;gBACD,MAAK;QACT,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;IAE9B,IAAI,GAAqB,CAAA;IACzB,IAAI,GAAqB,CAAA;IACzB,OAAO,KAAK,CAAC,MAAM,EAAE,CAAC;QACpB,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,aAAa,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,EAAE,CAAC;YAC9I,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,EAAE,CAAA;YAE7B,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK,CAAA;YACzB,IAAI,GAAG,CAAA;YACP,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,EAAC,MAAM,EAAE,CAAC,EAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAC,CAAC,CAAC,EAAE,CAAC;gBACvD,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;oBAAE,MAAK;gBAElB,GAAG,GAAG,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAA;gBAC/B,MAAM,CAAC,IAAI,iCACN,OAAO,KACV,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,EACZ,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,EACb,KAAK;oBACL,GAAG,EACH,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,EACzC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAClC,CAAA;gBACF,KAAK,GAAG,GAAG,GAAG,CAAC,CAAA;YACjB,CAAC;YACD,SAAQ;QACV,CAAC;QAED,mBAAmB;QACnB,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACjD,IAAI,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACtC,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAA;YACjD,MAAM,CAAC,IAAI,iCAAM,OAAO,CAAC,UAAU,CAAC,KAAE,UAAU,IAAG,CAAA;YACnD,SAAQ;QACV,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAA;IAC5B,CAAC;IAED,OAAO,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,iCAAM,KAAK,KAAE,IAAI,EAAE,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,IAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;AACjH,CAAC;AA5ED,4BA4EC"}
|