@retorquere/bibtex-parser 9.0.2 → 9.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/sentence-case.js +41 -20
- package/sentence-case.js.map +1 -1
- package/tokenizer.js +45 -16
- package/tokenizer.js.map +1 -1
package/package.json
CHANGED
package/sentence-case.js
CHANGED
|
@@ -1,50 +1,76 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/* eslint-disable @typescript-eslint/unbound-method */
|
|
3
|
-
var
|
|
4
|
-
|
|
3
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
4
|
+
if (k2 === undefined) k2 = k;
|
|
5
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
6
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
7
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
8
|
+
}
|
|
9
|
+
Object.defineProperty(o, k2, desc);
|
|
10
|
+
}) : (function(o, m, k, k2) {
|
|
11
|
+
if (k2 === undefined) k2 = k;
|
|
12
|
+
o[k2] = m[k];
|
|
13
|
+
}));
|
|
14
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
15
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
16
|
+
}) : function(o, v) {
|
|
17
|
+
o["default"] = v;
|
|
18
|
+
});
|
|
19
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
20
|
+
if (mod && mod.__esModule) return mod;
|
|
21
|
+
var result = {};
|
|
22
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
23
|
+
__setModuleDefault(result, mod);
|
|
24
|
+
return result;
|
|
5
25
|
};
|
|
6
26
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
27
|
exports.toSentenceCase = void 0;
|
|
8
|
-
const xregexp_1 = __importDefault(require("xregexp"));
|
|
9
28
|
const tokenizer_1 = require("./tokenizer");
|
|
10
29
|
const merge_1 = require("./merge");
|
|
30
|
+
const rx = __importStar(require("./re"));
|
|
31
|
+
const Lu = rx.match(rx.categories.filter(cat => cat.name === 'Lu'));
|
|
32
|
+
const Ll = rx.match(rx.categories.filter(cat => cat.name === 'Ll'));
|
|
33
|
+
const connectedInnerWord = new RegExp(`-${Lu}${Ll}*(?=-|$)`, 'g');
|
|
11
34
|
// eslint-disable-next-line no-magic-numbers
|
|
12
35
|
// const show = (obj: any): string => JSON.stringify(obj, null, 2).replace(/[\u007F-\uFFFF]/g, chr => `\\u${(`0000${chr.charCodeAt(0).toString(16)}`).substr(-4)}`)
|
|
13
36
|
function titleCase(s) {
|
|
14
37
|
return s.replace(/^(.)(.+)/, (match, car, cdr) => `${car}${cdr.toLowerCase()}`);
|
|
15
38
|
}
|
|
16
|
-
// const connectedWord = XRegExp('(^|-)\\p{Lu}\\p{Ll}*(?=-|$)', 'g')
|
|
17
|
-
const connectedInnerWord = (0, xregexp_1.default)('-\\p{Lu}\\p{Ll}*(?=-|$)', 'g');
|
|
18
39
|
function wordSC(token, allCaps, subSentence, hyphenated) {
|
|
40
|
+
// domains are case-insensitive
|
|
19
41
|
if (token.type === 'domain')
|
|
20
42
|
return token.text.toLowerCase();
|
|
43
|
+
// punctuation needs no handling, and websites can have case-sensitive stuff in the URL
|
|
21
44
|
if (token.type !== 'word')
|
|
22
45
|
return token.text;
|
|
46
|
+
// I'll, I'm
|
|
23
47
|
if (token.text.match(/^I'/))
|
|
24
48
|
return titleCase(token.text);
|
|
49
|
+
// starting 'a'
|
|
25
50
|
if (subSentence && token.subSentenceStart && token.text.match(/^a$/i))
|
|
26
51
|
return 'a';
|
|
52
|
+
// (sub)sentence start
|
|
27
53
|
if ((subSentence && token.subSentenceStart) || token.sentenceStart) {
|
|
28
|
-
return allCaps ? titleCase(token.text) :
|
|
54
|
+
return allCaps ? titleCase(token.text) : token.text.replace(connectedInnerWord, match => match.toLowerCase());
|
|
29
55
|
}
|
|
56
|
+
// prepositions should already have been lowercased, but OK
|
|
30
57
|
if (token.subtype === 'preposition')
|
|
31
58
|
return token.text.toLowerCase();
|
|
59
|
+
// keep acronyms
|
|
32
60
|
if (token.subtype === 'acronym')
|
|
33
61
|
return token.text;
|
|
34
62
|
// if (!allCaps && token.shape.match(/^[Xxd]+(-[Xxd]+)+/)) return XRegExp.replace(token.text, connectedWord, match => match.toLowerCase())
|
|
63
|
+
// single-letter capitals are usually some kind of noun, but not when they're part of an hyphenated word
|
|
35
64
|
if (token.text.match(/^[B-Z]$/))
|
|
36
65
|
return hyphenated ? token.text.toLowerCase() : token.text;
|
|
37
|
-
if (!allCaps && token.shape.match(/^[-X]+$/))
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
if (
|
|
41
|
-
|
|
42
|
-
if (shape.match(/^X[xd]*(-[Xxd]*)*$/))
|
|
43
|
-
return token.text.toLowerCase();
|
|
66
|
+
// ?? if (!allCaps && token.shape.match(/^[-X]+$/)) return token.text
|
|
67
|
+
const shape = token.shape.replace(/[^-Xxd]/g, '');
|
|
68
|
+
// ?? if (!allCaps && shape.match(/^[Xd-]+$/)) return token.text
|
|
69
|
+
// hyphenated ?? if (shape.match(/^X[xd]*(-[Xxd]*)*$/)) return token.text.toLowerCase()
|
|
70
|
+
// all-caps and numbers
|
|
44
71
|
if (shape.match(/^[Xd]+$/))
|
|
45
72
|
return allCaps ? token.text.toLowerCase() : token.text;
|
|
46
|
-
if (token.text.includes('.'))
|
|
47
|
-
return token.text;
|
|
73
|
+
// domain ?? if (token.text.includes('.')) return token.text
|
|
48
74
|
if (shape.match(/x.*X/))
|
|
49
75
|
return token.text;
|
|
50
76
|
return token.text.toLowerCase();
|
|
@@ -63,11 +89,6 @@ function toSentenceCase(title, options = {}) {
|
|
|
63
89
|
$title = $title.replace(options.nocase, match => match.match(/\s/) ? ' ' : '');
|
|
64
90
|
if (options.markup)
|
|
65
91
|
$title = $title.replace(options.markup, '');
|
|
66
|
-
if (options.preserveQuoted) {
|
|
67
|
-
for (const q of [/“.*?”/g, /‘.*?’/g, /".*?"/g]) {
|
|
68
|
-
$title = $title.replace(q, '');
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
92
|
const guess = {
|
|
72
93
|
words: (0, tokenizer_1.tokenize)($title),
|
|
73
94
|
sc: 0,
|
package/sentence-case.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sentence-case.js","sourceRoot":"","sources":["sentence-case.ts"],"names":[],"mappings":";AAAA,sDAAsD
|
|
1
|
+
{"version":3,"file":"sentence-case.js","sourceRoot":"","sources":["sentence-case.ts"],"names":[],"mappings":";AAAA,sDAAsD;;;;;;;;;;;;;;;;;;;;;;;;;;AAEtD,2CAA6C;AAC7C,mCAA+B;AAE/B,yCAA0B;AAE1B,MAAM,EAAE,GAAW,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAA;AAC3E,MAAM,EAAE,GAAW,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAA;AAC3E,MAAM,kBAAkB,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG,CAAC,CAAA;AAEjE,4CAA4C;AAC5C,mKAAmK;AAEnK,SAAS,SAAS,CAAC,CAAS;IAC1B,OAAO,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC,CAAA;AACjF,CAAC;AAED,SAAS,MAAM,CAAC,KAAY,EAAE,OAAgB,EAAE,WAAoB,EAAE,UAAmB;IAEvF,+BAA+B;IAC/B,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAA;IAE5D,uFAAuF;IACvF,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,KAAK,CAAC,IAAI,CAAA;IAE5C,YAAY;IACZ,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAEzD,eAAe;IACf,IAAI,WAAW,IAAI,KAAK,CAAC,gBAAgB,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,GAAG,CAAA;IAEjF,sBAAsB;IACtB,IAAI,CAAC,WAAW,IAAI,KAAK,CAAC,gBAAgB,CAAC,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;QACnE,OAAO,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAA;IAC/G,CAAC;IAED,2DAA2D;IAC3D,IAAI,KAAK,CAAC,OAAO,KAAK,aAAa;QAAE,OAAO,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAA;IAEpE,gBAAgB;IAChB,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC,IAAI,CAAA;IAElD,0IAA0I;IAE1I,wGAAwG;IACxG,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;QAAE,OAAO,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAA;IAE1F,qEAAqE;IAErE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAA;IACjD,gEAAgE;IAEhE,uFAAuF;IAEvF,uBAAuB;IACvB,IAAI,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC;QAAE,OAAO,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAA;IAElF,4DAA4D;IAC5D,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,KAAK,CAAC,IAAI,CAAA;IAE1C,OAAO,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAA;AACjC,CAAC;AAUD,SAAgB,cAAc,CAAC,KAAa,EAAE,UAAmB,EAAE;IACjE,OAAO,GAAG,IAAA,aAAK,EAAC,OAAO,EAAE;QACvB,cAAc,EAAE,IAAI;QACpB,yBAAyB,EAAE,IAAI;QAC/B,KAAK,EAAE,KAAK;KACb,CAAC,CAAA;IAEF,MAAM,OAAO,GAAG,KAAK,KAAK,KAAK,CAAC,WAAW,EAAE,CAAA;IAC7C,MAAM,QAAQ,GAAG,KAAK,KAAK,KAAK,CAAC,WAAW,EAAE,CAAA;IAC9C,IAAI,OAAO,CAAC,KAAK,IAAI,CAAC,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC3C,IAAI,MAAM,GAAG,KAAK,CAAA;QAClB,IAAI,OAAO,CAAC,MAAM;YAAE,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;QAClG,IAAI,OAAO,CAAC,MAAM;YAAE,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;QAE/D,MAAM,KAAK,GAAG;YACZ,KAAK,EAAE,IAAA,oBAAQ,EAAC,MAAM,CAAC;YACvB,EAAE,EAAE,CAAC;YACL,KAAK,EAAE,CAAC;SACT,CAAA;QAED,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;YAC/B,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,GAAC,CAAC,CAAC,CAAC,IAAI,KAAK,YAAY,CAAC,EAAE,CAAC;gBACjK,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;YACjE,CAAC;QACH,CAAC,CAAC,CAAA;QACF,IAAI,KAAK,CAAC,EAAE,IAAI,KAAK,CAAC,EAAE,IAAI,KAAK,CAAC,KAAK;YAAE,OAAO,KAAK,CAAA;IACvD,CAAC;IAED,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA,CAAC,gDAAgD;IAC/E,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAA;IAE/C,MAAM,MAAM,GAAG,IAAA,oBAAQ,EAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;IAE9C,IAAI,aAAa,GAAG,EAAE,CAAA;IACtB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YAC/C,aAAa,IAAI,KAAK,CAAC,IAAI,CAAA;QAC7B,CAAC;aACI,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YAC1B,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;gBACjC,aAAa,IAAI,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,yBAAyB,EAAE,IAAI,CAAC,CAAA;YAC9E,CAAC;QACH,CAAC;aACI,CAAC;YACJ,aAAa,IAAI,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,yBAAyB,EAAE,KAAK,CAAC,CAAA;QACnF,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACnD,aAAa,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAA;QAC7H,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,aAAa,GAAG,aAAa;aAC1B,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,GAAW,EAAE,MAAc,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;IAC5H,CAAC;IAED,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;YAC/C,aAAa,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,KAAa,EAAE,MAAc,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;QAC7H,CAAC;IACH,CAAC;IAED,OAAO,aAAa,CAAA;AACtB,CAAC;AAlED,wCAkEC"}
|
package/tokenizer.js
CHANGED
|
@@ -1,22 +1,43 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
2
25
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
26
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
27
|
};
|
|
5
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
29
|
exports.tokenize = void 0;
|
|
7
30
|
const moo_1 = __importDefault(require("moo"));
|
|
8
|
-
|
|
9
|
-
const
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
const L = re(charCategories.filter(cat => cat.name === 'L'));
|
|
14
|
-
const LNM = re(charCategories.filter(cat => cat.name.match(/^[LNM]/)), '\u00AD\u2060');
|
|
31
|
+
// eslint-disable-next-line no-magic-numbers
|
|
32
|
+
// const show = (obj: any): string => JSON.stringify(obj, null, 2).replace(/[\u007F-\uFFFF]/g, chr => `\\u${(`0000${chr.charCodeAt(0).toString(16)}`).substr(-4)}`)
|
|
33
|
+
const rx = __importStar(require("./re"));
|
|
34
|
+
const L = rx.match(rx.categories.filter(cat => cat.name === 'L'));
|
|
35
|
+
const LNM = rx.match(rx.categories.filter(cat => cat.name.match(/^[LNM]/)), '\u00AD\u2060');
|
|
15
36
|
const W = `${LNM}*?${L}${LNM}*`;
|
|
16
|
-
const B = `(?=(?:${
|
|
37
|
+
const B = `(?=(?:${rx.match(rx.categories.filter(cat => cat.name.match(/^[LNM]/)), '\u00AD\u2060').replace(/^./, '[^')}|$))`;
|
|
17
38
|
const Word = new RegExp(`${W}${B}`);
|
|
18
|
-
const P = new RegExp(
|
|
19
|
-
const Lu =
|
|
39
|
+
const P = new RegExp(rx.match(rx.categories.filter(cat => cat.name.match(/^P/))));
|
|
40
|
+
const Lu = rx.match(rx.categories.filter(cat => cat.name === 'Lu'), '\u2060');
|
|
20
41
|
const Acronym = new RegExp(`(?:(?:(?:${Lu}[.]){2,}${B})|(?:(?:vs?[.])(?=[ \t\n\r\u00A0])))`);
|
|
21
42
|
const Contraction = new RegExp(`${W}['’]${W}${B}`);
|
|
22
43
|
const Whitespace = /[ \t\n\r\u00A0]+/;
|
|
@@ -25,6 +46,7 @@ const Email = new RegExp(`[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:[.][A-Za-z0-9-]+)+${
|
|
|
25
46
|
const Handle = new RegExp(`@[A-Za-z0-9-]{2,}${B}`);
|
|
26
47
|
const Int = new RegExp(`[0-9]+${B}`);
|
|
27
48
|
const Domain = new RegExp(`${W}(?:[.]${W})+${B}`);
|
|
49
|
+
const Website = new RegExp(`https?://${W}(?:[.]${W})+(?:[^.!? \t\n\r\u00A0]+|[.!?]${LNM})+`);
|
|
28
50
|
const ComplexPreposition = /^([^ \t\n\r\u00A0]+)([ \t\n\r\u00A0]+)([^ \t\n\r\u00A0]+)(?:([ \t\n\r\u00A0]+)([^ \t\n\r\u00A0]+))?$/;
|
|
29
51
|
function ci(s) {
|
|
30
52
|
return s
|
|
@@ -40,6 +62,7 @@ const lexer = moo_1.default.compile({
|
|
|
40
62
|
'word-ordinal': Ordinal,
|
|
41
63
|
email: Email,
|
|
42
64
|
handle: Handle,
|
|
65
|
+
website: Website,
|
|
43
66
|
domain: Domain,
|
|
44
67
|
word: Word,
|
|
45
68
|
number: Int, // eslint-disable-line id-blacklist
|
|
@@ -50,14 +73,20 @@ const lexer = moo_1.default.compile({
|
|
|
50
73
|
whitespace: { match: /[ \t\n\r\u00A0]/, lineBreaks: true },
|
|
51
74
|
other: { match: /[\s\S]/, lineBreaks: true },
|
|
52
75
|
});
|
|
53
|
-
const
|
|
54
|
-
[(
|
|
55
|
-
[new RegExp(
|
|
56
|
-
[(0
|
|
76
|
+
const shaper = [
|
|
77
|
+
[new RegExp(rx.match(rx.categories.filter(cat => cat.name === 'Lu')), 'g'), 'X'],
|
|
78
|
+
[new RegExp(rx.match(rx.categories.filter(cat => cat.name.match(/^L[^Cu]/))), 'g'), 'x'],
|
|
79
|
+
[new RegExp(rx.match(rx.categories.filter(cat => cat.name[0] === 'N')), 'g'), 'd'],
|
|
57
80
|
[/’/g, "'"],
|
|
58
81
|
[/–/g, '-'],
|
|
59
82
|
[/[\u2060\u00AD]/g, ''],
|
|
60
83
|
];
|
|
84
|
+
function shape(s) {
|
|
85
|
+
for (const [re, repl] of shaper) {
|
|
86
|
+
s = s.replace(re, repl);
|
|
87
|
+
}
|
|
88
|
+
return s;
|
|
89
|
+
}
|
|
61
90
|
function combine(tokens) {
|
|
62
91
|
const combined = Object.assign({}, tokens[0]);
|
|
63
92
|
for (const t of tokens.slice(1)) {
|
|
@@ -88,7 +117,7 @@ function tokenize(title, markup) {
|
|
|
88
117
|
text: token.text,
|
|
89
118
|
start: token.offset,
|
|
90
119
|
end: token.offset + token.text.length - 1,
|
|
91
|
-
shape:
|
|
120
|
+
shape: shape(token.text),
|
|
92
121
|
sentenceStart: type === 'word' && sentenceStart,
|
|
93
122
|
subSentenceStart: type === 'word' && subSentenceStart,
|
|
94
123
|
});
|
|
@@ -100,7 +129,7 @@ function tokenize(title, markup) {
|
|
|
100
129
|
subSentenceStart = true;
|
|
101
130
|
break;
|
|
102
131
|
default:
|
|
103
|
-
if (type.match(/word|number/)) {
|
|
132
|
+
if (type.match(/word|number|handle|domain|website/)) {
|
|
104
133
|
sentenceStart = false;
|
|
105
134
|
subSentenceStart = false;
|
|
106
135
|
}
|
package/tokenizer.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["tokenizer.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["tokenizer.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,8CAAqB;AAErB,4CAA4C;AAC5C,mKAAmK;AAEnK,yCAA0B;AAE1B,MAAM,CAAC,GAAW,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAA;AACzE,MAAM,GAAG,GAAW,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,cAAc,CAAC,CAAA;AACnG,MAAM,CAAC,GAAG,GAAG,GAAG,KAAK,CAAC,GAAG,GAAG,GAAG,CAAA;AAC/B,MAAM,CAAC,GAAG,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAA;AAE5H,MAAM,IAAI,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;AACnC,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;AAEjF,MAAM,EAAE,GAAW,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,QAAQ,CAAC,CAAA;AACrF,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,YAAY,EAAE,WAAW,CAAC,sCAAsC,CAAC,CAAA;AAE5F,MAAM,WAAW,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;AAClD,MAAM,UAAU,GAAG,kBAAkB,CAAA;AACrC,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAA;AACrD,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,uDAAuD,CAAC,EAAE,CAAC,CAAA;AACpF,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAA;AAClD,MAAM,GAAG,GAAG,IAAI,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;AACpC,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;AACjD,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,YAAY,CAAC,SAAS,CAAC,kCAAkC,GAAG,IAAI,CAAC,CAAA;AAE5F,MAAM,kBAAkB,GAAG,sGAAsG,CAAA;AAEjI,SAAS,EAAE,CAAC,CAAS;IACnB,OAAO,CAAC;SACL,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,WAAW,EAAE,GAAG,KAAK,CAAC,WAAW,EAAE,GAAG,CAAC;SAC7E,OAAO,CAAC,GAAG,EAAE,UAAU,CAAC,MAAM,CAAE,CAAA;AACrC,CAAC;AACD,MAAM,YAAY,GAAW,OAAO,CAAC,qBAAqB,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AAC9F,MAAM,WAAW,GAAG,IAAI,MAAM,CAAC,MAAM,YAAY,IAAI,CAAC,EAAE,CAAC,CAAA;AAEzD,MAAM,KAAK,GAAG,aAAG,CAAC,OAAO,CAAC;IACxB,kBAAkB,EAAM,WAAW;IACnC,cAAc,EAAU,OAAO;IAC/B,kBAAkB,EAAM,WAAW;IACnC,cAAc,EAAU,OAAO;IAC/B,KAAK,EAAmB,KAAK;IAC7B,MAAM,EAAkB,MAAM;IAC9B,OAAO,EAAiB,OAAO;IAC/B,MAAM,EAAkB,MAAM;IAC9B,IAAI,EAAoB,IAAI;IAC5B,MAAM,EAAkB,GAAG,EAAE,mCAAmC;IAChE,iBAAiB,EAAO,4BAA4B;IACpD,mBAAmB,EAAK,sBAAsB;IAC9C,sBAAsB,EAAE,WAAW;IACnC,WAAW,EAAa,CAAC;IACzB,UAAU,EAAc,EAAE,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,IAAI,EAAE;IACtE,KAAK,EAAmB,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,EAAE;CAC9D,CAAC,CAAA;AAEF,MAAM,MAAM,GAAuB;IACjC,CAAE,IAAI,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAE;IAClF,CAAE,IAAI,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAE;IAC1F,CAAE,IAAI,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAE;IACpF,CAAE,IAAI,EAAE,GAAG,CAAE;IACb,CAAE,IAAI,EAAE,GAAG,CAAE;IACb,CAAE,iBAAiB,EAAE,EAAE,CAAE;CAC1B,CAAA;AACD,SAAS,KAAK,CAAC,CAAS;IACtB,KAAK,MAAM,CAAE,EAAE,EAAE,IAAI,CAAE,IAAI,MAAM,EAAE,CAAC;QAClC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,CAAA;IACzB,CAAC;IACD,OAAO,CAAC,CAAA;AACV,CAAC;AAcD,SAAS,OAAO,CAAC,MAAe;IAC9B,MAAM,QAAQ,qBAAQ,MAAM,CAAC,CAAC,CAAC,CAAE,CAAA;IACjC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAChC,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,CAAA;QACvB,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAA;QACpB,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,CAAA;IAC3B,CAAC;IACD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,SAAS,SAAS,CAAC,CAAQ;IACzB,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,GAAG,CAAA;IACjC,IAAI,CAAC,CAAC,IAAI,KAAK,GAAG;QAAE,OAAO,GAAG,CAAA;IAC9B,OAAO,GAAG,CAAA;AACZ,CAAC;AAED,SAAgB,QAAQ,CAAC,KAAa,EAAE,MAAe;IACrD,IAAI,MAAM;QAAE,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;IAEjF,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IAClB,MAAM,MAAM,GAAY,EAAE,CAAA;IAC1B,IAAI,aAAa,GAAG,IAAI,CAAA;IACxB,IAAI,gBAAgB,GAAG,KAAK,CAAA;IAC5B,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,CAAE,IAAI,EAAE,OAAO,CAAE,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QAE/F,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,GAAG,EAAE,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;YACzC,KAAK,EAAE,KAAK,CAAS,KAAK,CAAC,IAAI,CAAC;YAChC,aAAa,EAAE,IAAI,KAAK,MAAM,IAAI,aAAa;YAC/C,gBAAgB,EAAE,IAAI,KAAK,MAAM,IAAI,gBAAgB;SACtD,CAAC,CAAA;QAEF,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,iBAAiB;gBACpB,aAAa,GAAG,IAAI,CAAA;gBACpB,MAAK;YACP,KAAK,mBAAmB;gBACtB,gBAAgB,GAAG,IAAI,CAAA;gBACvB,MAAK;YACP;gBACE,IAAI,IAAI,CAAC,KAAK,CAAC,mCAAmC,CAAC,EAAE,CAAC;oBACpD,aAAa,GAAG,KAAK,CAAA;oBACrB,gBAAgB,GAAG,KAAK,CAAA;gBAC1B,CAAC;gBACD,MAAK;QACT,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;IAE9B,IAAI,GAAqB,CAAA;IACzB,IAAI,GAAqB,CAAA;IACzB,OAAO,KAAK,CAAC,MAAM,EAAE,CAAC;QACpB,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,aAAa,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,EAAE,CAAC;YAC9I,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,EAAE,CAAA;YAE7B,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK,CAAA;YACzB,IAAI,GAAG,CAAA;YACP,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,EAAC,MAAM,EAAE,CAAC,EAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAC,CAAC,CAAC,EAAE,CAAC;gBACvD,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;oBAAE,MAAK;gBAElB,GAAG,GAAG,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAA;gBAC/B,MAAM,CAAC,IAAI,iCACN,OAAO,KACV,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,EACZ,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,EACb,KAAK;oBACL,GAAG,EACH,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,EACzC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAClC,CAAA;gBACF,KAAK,GAAG,GAAG,GAAG,CAAC,CAAA;YACjB,CAAC;YACD,SAAQ;QACV,CAAC;QAED,mBAAmB;QACnB,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACjD,IAAI,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACtC,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAA;YACjD,MAAM,CAAC,IAAI,iCAAM,OAAO,CAAC,UAAU,CAAC,KAAE,UAAU,IAAG,CAAA;YACnD,SAAQ;QACV,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAA;IAC5B,CAAC;IAED,OAAO,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,iCAAM,KAAK,KAAE,IAAI,EAAE,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,IAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;AACjH,CAAC;AA5ED,4BA4EC"}
|