@retorquere/bibtex-parser 12.1.11 → 13.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -10
- package/dist/esm/crossref.d.ts.map +1 -0
- package/dist/esm/crossref.js.map +1 -1
- package/dist/esm/fields.d.ts.map +1 -0
- package/dist/esm/fields.js.map +1 -1
- package/dist/{types → esm}/index.d.ts +41 -12
- package/dist/esm/index.d.ts.map +1 -0
- package/dist/esm/index.js +27 -3
- package/dist/esm/index.js.map +3 -3
- package/dist/{types → esm}/jabref.d.ts +9 -21
- package/dist/esm/jabref.d.ts.map +1 -0
- package/dist/esm/jabref.js +2 -3
- package/dist/esm/jabref.js.map +3 -3
- package/dist/esm/merge.d.ts.map +1 -0
- package/dist/esm/merge.js.map +1 -1
- package/dist/esm/prepositions.d.ts.map +1 -0
- package/dist/esm/prepositions.js.map +1 -1
- package/dist/{types → esm}/sentence-case.d.ts +1 -0
- package/dist/esm/sentence-case.d.ts.map +1 -0
- package/dist/esm/sentence-case.js +126 -28
- package/dist/esm/sentence-case.js.map +3 -3
- package/dist/esm/tokenizer.d.ts.map +1 -0
- package/dist/esm/tokenizer.js.map +1 -1
- package/dist/esm/verbatim.d.ts +90 -0
- package/dist/esm/verbatim.d.ts.map +1 -0
- package/dist/esm/verbatim.js +10 -0
- package/dist/esm/verbatim.js.map +3 -3
- package/dist/esm/yield.d.ts.map +1 -0
- package/dist/esm/yield.js.map +1 -1
- package/dist/wasm/bibtex_parser.d.ts +12 -9
- package/dist/wasm/bibtex_parser.js +13 -23
- package/dist/wasm/bibtex_parser_bg.wasm +0 -0
- package/dist/wasm/bibtex_parser_bg.wasm.d.ts +2 -3
- package/dist/wasm/english.json +27 -0
- package/dist/wasm/index.d.ts +105 -29
- package/dist/wasm/index.js +54 -51
- package/dist/wasm/jabref.d.ts +9 -21
- package/dist/wasm/jabref.js +3 -3
- package/package.json +25 -26
- package/dist/cjs/crossref.js +0 -201
- package/dist/cjs/crossref.js.map +0 -7
- package/dist/cjs/fields.js +0 -7186
- package/dist/cjs/fields.js.map +0 -7
- package/dist/cjs/index.js +0 -1126
- package/dist/cjs/index.js.map +0 -7
- package/dist/cjs/jabref.js +0 -135
- package/dist/cjs/jabref.js.map +0 -7
- package/dist/cjs/merge.js +0 -37
- package/dist/cjs/merge.js.map +0 -7
- package/dist/cjs/prepositions.js +0 -111
- package/dist/cjs/prepositions.js.map +0 -7
- package/dist/cjs/sentence-case.js +0 -98
- package/dist/cjs/sentence-case.js.map +0 -7
- package/dist/cjs/tokenizer.js +0 -194
- package/dist/cjs/tokenizer.js.map +0 -7
- package/dist/cjs/verbatim.js +0 -403
- package/dist/cjs/verbatim.js.map +0 -7
- package/dist/cjs/yield.js +0 -26
- package/dist/cjs/yield.js.map +0 -7
- package/dist/types/crossref.d.ts.map +0 -1
- package/dist/types/fields.d.ts.map +0 -1
- package/dist/types/index.d.ts.map +0 -1
- package/dist/types/jabref.d.ts.map +0 -1
- package/dist/types/merge.d.ts.map +0 -1
- package/dist/types/prepositions.d.ts.map +0 -1
- package/dist/types/sentence-case.d.ts.map +0 -1
- package/dist/types/tokenizer.d.ts.map +0 -1
- package/dist/types/verbatim.d.ts +0 -66
- package/dist/types/verbatim.d.ts.map +0 -1
- package/dist/types/yield.d.ts.map +0 -1
- /package/dist/{types → esm}/crossref.d.ts +0 -0
- /package/dist/{types → esm}/fields.d.ts +0 -0
- /package/dist/{types → esm}/merge.d.ts +0 -0
- /package/dist/{types → esm}/prepositions.d.ts +0 -0
- /package/dist/{types → esm}/tokenizer.d.ts +0 -0
- /package/dist/{types → esm}/yield.d.ts +0 -0
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"version": 3,
|
|
3
|
-
"sources": ["../../sentence-case.ts"],
|
|
4
|
-
"sourcesContent": ["import { merge } from './merge.js'\nimport type { Token } from './tokenizer.js'\nimport { tokenize } from './tokenizer.js'\n\nconst connectedInnerWord = /-\\p{Lu}\\p{Ll}*(?=-|$)/ug\n\n// const show = (obj: any): string => JSON.stringify(obj, null, 2).replace(/[\\u007F-\\uFFFF]/g, chr => `\\\\u${(`0000${chr.charCodeAt(0).toString(16)}`).substr(-4)}`)\n\nfunction titleCase(s: string): string {\n return s.replace(/^(.)(.+)/, (match, car, cdr) => `${car}${cdr.toLowerCase()}`)\n}\n\nfunction wordSC(token: Token, allCaps: boolean, subSentence: boolean, hyphenated: boolean): string {\n // domains are case-insensitive\n if (token.type === 'domain') return token.text.toLowerCase()\n\n // punctuation needs no handling, and websites can have case-sensitive stuff in the URL\n if (token.type !== 'word') return token.text\n\n // I'll, I'm\n if (token.text.match(/^I'/)) return titleCase(token.text)\n\n // starting 'a'\n if (subSentence && token.subSentenceStart && token.text.match(/^a$/i)) return 'a'\n\n // (sub)sentence start\n if ((subSentence && token.subSentenceStart) || token.sentenceStart) {\n return allCaps ? titleCase(token.text) : token.text.replace(connectedInnerWord, match => match.toLowerCase())\n }\n\n // prepositions should already have been lowercased, but OK\n if (token.subtype === 'preposition') return token.text.toLowerCase()\n\n // keep acronyms\n if (token.subtype === 'acronym') return token.text\n\n // if (!allCaps && token.shape.match(/^[Xxd]+(-[Xxd]+)+/)) return XRegExp.replace(token.text, connectedWord, match => match.toLowerCase())\n\n // single-letter capitals are usually some kind of noun, but not when they're part of an hyphenated word\n if (token.text.match(/^[B-Z]$/)) return hyphenated ? token.text.toLowerCase() : token.text\n\n // ?? if (!allCaps && token.shape.match(/^[-X]+$/)) return token.text\n\n const shape = token.shape.replace(/[^-Xxd]/g, '')\n // ?? if (!allCaps && shape.match(/^[Xd-]+$/)) return token.text\n\n // hyphenated ?? if (shape.match(/^X[xd]*(-[Xxd]*)*$/)) return token.text.toLowerCase()\n\n // all-caps and numbers\n if (shape.match(/^[Xd]+$/)) return allCaps ? token.text.toLowerCase() : token.text\n\n // domain ?? if (token.text.includes('.')) return token.text\n if (shape.match(/x.*X/)) return token.text\n\n return token.text.toLowerCase()\n}\n\nexport type Options = {\n preserveQuoted?: boolean\n subSentenceCapitalization?: boolean\n markup?: RegExp\n nocase?: RegExp\n guess?: boolean\n}\n\nexport function toSentenceCase(title: string, options: Options = {}): string {\n options = merge(options, {\n preserveQuoted: true,\n subSentenceCapitalization: true,\n guess: false,\n })\n\n const allCaps = title === title.toUpperCase()\n const allLower = title === title.toLowerCase()\n if (options.guess && !allCaps && !allLower) {\n let $title = title\n if (options.nocase) $title = $title.replace(options.nocase, match => match.match(/\\s/) ? ' ' : '')\n if (options.markup) $title = $title.replace(options.markup, '')\n\n const guess = {\n words: tokenize($title),\n sc: 0,\n other: 0,\n }\n\n guess.words.forEach((token, i) => {\n if (token.type === 'word' && token.text.length > 1 && !token.subtype.match(/preposition|acronym|ordinal/) && (i === 0 || guess.words[i - 1].type === 'whitespace')) {\n guess[token.shape.match(/^[^X]*x[^X]*$/) ? 'sc' : 'other'] += 1\n }\n })\n if (guess.sc && guess.sc >= guess.other) return title\n }\n\n if (allCaps && !title.match(/\\s/)) return title\n\n const tokens = tokenize(title, options.markup)\n\n let sentenceCased = ''\n for (const token of tokens) {\n if (token.hyphenated && token.shape[0] === 'x') {\n sentenceCased += token.text\n }\n else if (token.hyphenated) {\n for (const t of token.hyphenated) {\n sentenceCased += wordSC(t, allCaps, options.subSentenceCapitalization!, true)\n }\n }\n else {\n sentenceCased += wordSC(token, allCaps, options.subSentenceCapitalization!, false)\n }\n }\n\n if (options.markup) {\n for (const match of title.matchAll(options.markup)) {\n sentenceCased = sentenceCased.substring(0, match.index) + match[0] + sentenceCased.substring(match.index + match[0].length)\n }\n }\n\n if (options.nocase) {\n sentenceCased = sentenceCased\n .replace(options.nocase, (match: string, tag: string, offset: number) => title.substring(offset, offset + match.length))\n }\n\n if (options.preserveQuoted) {\n for (const q of [/\u201C.*?\u201D/g, /\u2018.*?\u2019/g, /\".*?\"/g]) {\n sentenceCased = sentenceCased.replace(q, (match: string, offset: number) => title.substring(offset, offset + match.length))\n }\n }\n\n return sentenceCased\n}\n"],
|
|
5
|
-
"mappings": ";;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,mBAAsB;AAEtB,uBAAyB;AAEzB,MAAM,qBAAqB,WAAC,2BAAsB,IAAE;AAIpD,SAAS,UAAU,GAAmB;AACpC,SAAO,EAAE,QAAQ,YAAY,CAAC,OAAO,KAAK,QAAQ,GAAG,GAAG,GAAG,IAAI,YAAY,CAAC,EAAE;AAChF;AAEA,SAAS,OAAO,OAAc,SAAkB,aAAsB,YAA6B;AAEjG,MAAI,MAAM,SAAS,SAAU,QAAO,MAAM,KAAK,YAAY;AAG3D,MAAI,MAAM,SAAS,OAAQ,QAAO,MAAM;AAGxC,MAAI,MAAM,KAAK,MAAM,KAAK,EAAG,QAAO,UAAU,MAAM,IAAI;AAGxD,MAAI,eAAe,MAAM,oBAAoB,MAAM,KAAK,MAAM,MAAM,EAAG,QAAO;AAG9E,MAAK,eAAe,MAAM,oBAAqB,MAAM,eAAe;AAClE,WAAO,UAAU,UAAU,MAAM,IAAI,IAAI,MAAM,KAAK,QAAQ,oBAAoB,WAAS,MAAM,YAAY,CAAC;AAAA,EAC9G;AAGA,MAAI,MAAM,YAAY,cAAe,QAAO,MAAM,KAAK,YAAY;AAGnE,MAAI,MAAM,YAAY,UAAW,QAAO,MAAM;AAK9C,MAAI,MAAM,KAAK,MAAM,SAAS,EAAG,QAAO,aAAa,MAAM,KAAK,YAAY,IAAI,MAAM;AAItF,QAAM,QAAQ,MAAM,MAAM,QAAQ,YAAY,EAAE;AAMhD,MAAI,MAAM,MAAM,SAAS,EAAG,QAAO,UAAU,MAAM,KAAK,YAAY,IAAI,MAAM;AAG9E,MAAI,MAAM,MAAM,MAAM,EAAG,QAAO,MAAM;AAEtC,SAAO,MAAM,KAAK,YAAY;AAChC;AAUO,SAAS,eAAe,OAAe,UAAmB,CAAC,GAAW;AAC3E,gBAAU,oBAAM,SAAS;AAAA,IACvB,gBAAgB;AAAA,IAChB,2BAA2B;AAAA,IAC3B,OAAO;AAAA,EACT,CAAC;AAED,QAAM,UAAU,UAAU,MAAM,YAAY;AAC5C,QAAM,WAAW,UAAU,MAAM,YAAY;AAC7C,MAAI,QAAQ,SAAS,CAAC,WAAW,CAAC,UAAU;AAC1C,QAAI,SAAS;AACb,QAAI,QAAQ,OAAQ,UAAS,OAAO,QAAQ,QAAQ,QAAQ,WAAS,MAAM,MAAM,IAAI,IAAI,MAAM,EAAE;AACjG,QAAI,QAAQ,OAAQ,UAAS,OAAO,QAAQ,QAAQ,QAAQ,EAAE;AAE9D,UAAM,QAAQ;AAAA,MACZ,WAAO,2BAAS,MAAM;AAAA,MACtB,IAAI;AAAA,MACJ,OAAO;AAAA,IACT;AAEA,UAAM,MAAM,QAAQ,CAAC,OAAO,MAAM;AAChC,UAAI,MAAM,SAAS,UAAU,MAAM,KAAK,SAAS,KAAK,CAAC,MAAM,QAAQ,MAAM,6BAA6B,MAAM,MAAM,KAAK,MAAM,MAAM,IAAI,CAAC,EAAE,SAAS,eAAe;AAClK,cAAM,MAAM,MAAM,MAAM,eAAe,IAAI,OAAO,OAAO,KAAK;AAAA,MAChE;AAAA,IACF,CAAC;AACD,QAAI,MAAM,MAAM,MAAM,MAAM,MAAM,MAAO,QAAO;AAAA,EAClD;AAEA,MAAI,WAAW,CAAC,MAAM,MAAM,IAAI,EAAG,QAAO;AAE1C,QAAM,aAAS,2BAAS,OAAO,QAAQ,MAAM;AAE7C,MAAI,gBAAgB;AACpB,aAAW,SAAS,QAAQ;AAC1B,QAAI,MAAM,cAAc,MAAM,MAAM,CAAC,MAAM,KAAK;AAC9C,uBAAiB,MAAM;AAAA,IACzB,WACS,MAAM,YAAY;AACzB,iBAAW,KAAK,MAAM,YAAY;AAChC,yBAAiB,OAAO,GAAG,SAAS,QAAQ,2BAA4B,IAAI;AAAA,MAC9E;AAAA,IACF,OACK;AACH,uBAAiB,OAAO,OAAO,SAAS,QAAQ,2BAA4B,KAAK;AAAA,IACnF;AAAA,EACF;AAEA,MAAI,QAAQ,QAAQ;AAClB,eAAW,SAAS,MAAM,SAAS,QAAQ,MAAM,GAAG;AAClD,sBAAgB,cAAc,UAAU,GAAG,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,cAAc,UAAU,MAAM,QAAQ,MAAM,CAAC,EAAE,MAAM;AAAA,IAC5H;AAAA,EACF;AAEA,MAAI,QAAQ,QAAQ;AAClB,oBAAgB,cACb,QAAQ,QAAQ,QAAQ,CAAC,OAAe,KAAa,WAAmB,MAAM,UAAU,QAAQ,SAAS,MAAM,MAAM,CAAC;AAAA,EAC3H;AAEA,MAAI,QAAQ,gBAAgB;AAC1B,eAAW,KAAK,CAAC,UAAU,UAAU,QAAQ,GAAG;AAC9C,sBAAgB,cAAc,QAAQ,GAAG,CAAC,OAAe,WAAmB,MAAM,UAAU,QAAQ,SAAS,MAAM,MAAM,CAAC;AAAA,IAC5H;AAAA,EACF;AAEA,SAAO;AACT;",
|
|
6
|
-
"names": []
|
|
7
|
-
}
|
package/dist/cjs/tokenizer.js
DELETED
|
@@ -1,194 +0,0 @@
|
|
|
1
|
-
var __create = Object.create;
|
|
2
|
-
var __defProp = Object.defineProperty;
|
|
3
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
6
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
7
|
-
var __export = (target, all) => {
|
|
8
|
-
for (var name in all)
|
|
9
|
-
__defProp(target, name, { get: all[name], enumerable: true });
|
|
10
|
-
};
|
|
11
|
-
var __copyProps = (to, from, except, desc) => {
|
|
12
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
13
|
-
for (let key of __getOwnPropNames(from))
|
|
14
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
15
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
16
|
-
}
|
|
17
|
-
return to;
|
|
18
|
-
};
|
|
19
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
20
|
-
// If the importer is in node compatibility mode or this is not an ESM
|
|
21
|
-
// file that has been converted to a CommonJS file using a Babel-
|
|
22
|
-
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
23
|
-
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
24
|
-
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
25
|
-
mod
|
|
26
|
-
));
|
|
27
|
-
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
28
|
-
var tokenizer_exports = {};
|
|
29
|
-
__export(tokenizer_exports, {
|
|
30
|
-
tokenize: () => tokenize
|
|
31
|
-
});
|
|
32
|
-
module.exports = __toCommonJS(tokenizer_exports);
|
|
33
|
-
var import_moo = __toESM(require("moo"), 1);
|
|
34
|
-
var import_prepositions = __toESM(require("./prepositions.js"), 1);
|
|
35
|
-
const RE = new class {
|
|
36
|
-
Acronym;
|
|
37
|
-
Word;
|
|
38
|
-
Contraction;
|
|
39
|
-
Whitespace = /[ \t\n\r\u00A0]+/u;
|
|
40
|
-
Ordinal;
|
|
41
|
-
Email;
|
|
42
|
-
Handle;
|
|
43
|
-
IntOrVersion;
|
|
44
|
-
Domain;
|
|
45
|
-
Website;
|
|
46
|
-
Preposition;
|
|
47
|
-
ComplexPreposition = /^([^ \t\n\r\u00A0]+)([ \t\n\r\u00A0]+)([^ \t\n\r\u00A0]+)(?:([ \t\n\r\u00A0]+)([^ \t\n\r\u00A0]+))?$/u;
|
|
48
|
-
P = new RegExp("\\p{P}", "u");
|
|
49
|
-
constructor() {
|
|
50
|
-
const B = "(?=(?:[^\\p{L}\\p{N}\\p{M}\\u00AD\\u2060]|$))";
|
|
51
|
-
const LNM = "[\\p{L}\\p{N}\\p{M}\xAD\u2060]";
|
|
52
|
-
const W = `${LNM}*?\\p{L}${LNM}*`;
|
|
53
|
-
this.Acronym = new RegExp(`(?:(?:(?:[\\p{Lu}\\p{Lt}][.]){2,}${B})|(?:(?:vs?[.])(?=[
|
|
54
|
-
\r\xA0])))`, "u");
|
|
55
|
-
this.Word = new RegExp(`${W}${B}`, "u");
|
|
56
|
-
this.Contraction = new RegExp(`${W}['\u2019]${W}${B}`, "u");
|
|
57
|
-
this.Ordinal = new RegExp(`\\d+(?:st|nd|rd|th)${B}`, "u");
|
|
58
|
-
this.Email = new RegExp(`[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:[.][A-Za-z0-9-]+)+${B}`, "u");
|
|
59
|
-
this.Handle = new RegExp(`@[A-Za-z0-9-]{2,}${B}`, "u");
|
|
60
|
-
this.IntOrVersion = new RegExp(`v?\\d+(?:\\.\\d+)*${B}`, "u");
|
|
61
|
-
this.Domain = new RegExp(`${W}(?:[.]${W})+${B}`, "u");
|
|
62
|
-
this.Website = new RegExp(`https?://${W}(?:[.]${W})+(?:[^.!?
|
|
63
|
-
\r\xA0]+|[.!?]${LNM})+`, "u");
|
|
64
|
-
const ci = (s) => s.replace(/[a-z]/ig, (match) => `[${match.toUpperCase()}${match.toLowerCase()}]`).replace(" ", this.Whitespace.source);
|
|
65
|
-
this.Preposition = new RegExp(`(?:${import_prepositions.default.sort().reverse().map(ci).join("|")})${B}`, "u");
|
|
66
|
-
}
|
|
67
|
-
}();
|
|
68
|
-
const lexer = import_moo.default.compile({
|
|
69
|
-
"word-preposition": RE.Preposition,
|
|
70
|
-
"word-acronym": RE.Acronym,
|
|
71
|
-
"word-contraction": RE.Contraction,
|
|
72
|
-
"word-ordinal": RE.Ordinal,
|
|
73
|
-
email: RE.Email,
|
|
74
|
-
handle: RE.Handle,
|
|
75
|
-
website: RE.Website,
|
|
76
|
-
domain: RE.Domain,
|
|
77
|
-
word: RE.Word,
|
|
78
|
-
number: RE.IntOrVersion,
|
|
79
|
-
// eslint-disable-line id-blacklist
|
|
80
|
-
"punctuation-end": /[?.!](?=[ \t\n\r\u00A0]|$)/u,
|
|
81
|
-
"punctuation-colon": /:(?=[ \t\n\r\u00A0])/u,
|
|
82
|
-
"punctuation-ellipsis": /[.][.][.]/u,
|
|
83
|
-
punctuation: RE.P,
|
|
84
|
-
whitespace: { match: /[ \t\n\r\u00A0]/u, lineBreaks: true },
|
|
85
|
-
other: { match: /[\s\S]/u, lineBreaks: true }
|
|
86
|
-
});
|
|
87
|
-
const Shape = new class {
|
|
88
|
-
shapes = /* @__PURE__ */ new Map();
|
|
89
|
-
re = {
|
|
90
|
-
X: /[\p{Lu}\p{Lt}]/u,
|
|
91
|
-
x: /[\p{Ll}\p{Lm}\p{Lo}]/u,
|
|
92
|
-
d: new RegExp("\\p{N}", "u")
|
|
93
|
-
};
|
|
94
|
-
match(c) {
|
|
95
|
-
if (c.match(this.re.d)) return "d";
|
|
96
|
-
if (c.toLowerCase() === c.toUpperCase()) return c;
|
|
97
|
-
if (c.match(this.re.X)) return "X";
|
|
98
|
-
if (c.match(this.re.x)) return "x";
|
|
99
|
-
if (c === "\u2019") return "'";
|
|
100
|
-
if (c === "\u2013") return "-";
|
|
101
|
-
if (c === "\u2060" || c === "\xAD") return "";
|
|
102
|
-
return c;
|
|
103
|
-
}
|
|
104
|
-
fetch(c) {
|
|
105
|
-
if (!this.shapes.has(c)) this.shapes.set(c, this.match(c));
|
|
106
|
-
return this.shapes.get(c);
|
|
107
|
-
}
|
|
108
|
-
shape(t) {
|
|
109
|
-
if (!this.shapes.has(t)) this.shapes.set(t, Array.from(t).map((c) => this.fetch(c)).join(""));
|
|
110
|
-
return this.shapes.get(t);
|
|
111
|
-
}
|
|
112
|
-
}();
|
|
113
|
-
function combine(tokens) {
|
|
114
|
-
const combined = { ...tokens[0] };
|
|
115
|
-
for (const t of tokens.slice(1)) {
|
|
116
|
-
combined.text += t.text;
|
|
117
|
-
combined.end = t.end;
|
|
118
|
-
combined.shape += t.shape;
|
|
119
|
-
}
|
|
120
|
-
return combined;
|
|
121
|
-
}
|
|
122
|
-
function hyphenate(t) {
|
|
123
|
-
if (t.type === "word") return "w";
|
|
124
|
-
if (t.text === "-" || t.text === "\u2013") return "-";
|
|
125
|
-
return " ";
|
|
126
|
-
}
|
|
127
|
-
function tokenize(title, markup) {
|
|
128
|
-
if (markup) title = title.replace(markup, (match) => "\u2060".repeat(match.length));
|
|
129
|
-
lexer.reset(title);
|
|
130
|
-
const tokens = [];
|
|
131
|
-
let sentenceStart = true;
|
|
132
|
-
let subSentenceStart = false;
|
|
133
|
-
for (const token of lexer) {
|
|
134
|
-
const [type, subtype] = (token.type.includes("-") ? token.type : `${token.type}-`).split("-");
|
|
135
|
-
tokens.push({
|
|
136
|
-
type,
|
|
137
|
-
subtype,
|
|
138
|
-
text: token.text,
|
|
139
|
-
start: token.offset,
|
|
140
|
-
end: token.offset + token.text.length - 1,
|
|
141
|
-
shape: Shape.shape(token.text),
|
|
142
|
-
sentenceStart: type === "word" && sentenceStart,
|
|
143
|
-
subSentenceStart: type === "word" && subSentenceStart
|
|
144
|
-
});
|
|
145
|
-
switch (token.type) {
|
|
146
|
-
case "punctuation-end":
|
|
147
|
-
sentenceStart = true;
|
|
148
|
-
break;
|
|
149
|
-
case "punctuation-colon":
|
|
150
|
-
subSentenceStart = true;
|
|
151
|
-
break;
|
|
152
|
-
default:
|
|
153
|
-
if (type.match(/word|number|handle|domain|website/)) {
|
|
154
|
-
sentenceStart = false;
|
|
155
|
-
subSentenceStart = false;
|
|
156
|
-
}
|
|
157
|
-
break;
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
const stack = tokens.splice(0);
|
|
161
|
-
let cpt;
|
|
162
|
-
let cps;
|
|
163
|
-
while (stack.length) {
|
|
164
|
-
if (stack[0].subtype === "preposition" && (cpt = stack[0].text.match(RE.ComplexPreposition)) && (cps = stack[0].shape.match(RE.ComplexPreposition))) {
|
|
165
|
-
const complex = stack.shift();
|
|
166
|
-
let start = complex.start;
|
|
167
|
-
let end;
|
|
168
|
-
for (const i of Array.from({ length: 5 }, (_, n) => n + 1)) {
|
|
169
|
-
if (!cpt[i]) break;
|
|
170
|
-
end = start + cps[i].length - 1;
|
|
171
|
-
tokens.push({
|
|
172
|
-
...complex,
|
|
173
|
-
text: cpt[i],
|
|
174
|
-
shape: cps[i],
|
|
175
|
-
start,
|
|
176
|
-
end,
|
|
177
|
-
type: i % 2 ? complex.type : "whitespace",
|
|
178
|
-
subtype: i % 2 ? complex.type : ""
|
|
179
|
-
});
|
|
180
|
-
start = end + 1;
|
|
181
|
-
}
|
|
182
|
-
continue;
|
|
183
|
-
}
|
|
184
|
-
const pat = stack.map((t) => hyphenate(t)).join("");
|
|
185
|
-
if (cpt = pat.match(/^w(-w)+(?= |$)/)) {
|
|
186
|
-
const hyphenated = stack.splice(0, cpt[0].length);
|
|
187
|
-
tokens.push({ ...combine(hyphenated), hyphenated });
|
|
188
|
-
continue;
|
|
189
|
-
}
|
|
190
|
-
tokens.push(stack.shift());
|
|
191
|
-
}
|
|
192
|
-
return markup ? tokens.map((token) => ({ ...token, text: title.substring(token.start, token.end + 1) })) : tokens;
|
|
193
|
-
}
|
|
194
|
-
//# sourceMappingURL=tokenizer.js.map
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"version": 3,
|
|
3
|
-
"sources": ["../../tokenizer.ts"],
|
|
4
|
-
"sourcesContent": ["import moo from 'moo'\n\nimport prepositions from './prepositions.js'\n\n// const show = (obj: any): string => JSON.stringify(obj, null, 2).replace(/[\\u007F-\\uFFFF]/g, chr => `\\\\u${(`0000${chr.charCodeAt(0).toString(16)}`).substr(-4)}`)\n\nconst RE = new class {\n public Acronym: RegExp\n public Word: RegExp\n public Contraction: RegExp\n public Whitespace = /[ \\t\\n\\r\\u00A0]+/u\n public Ordinal: RegExp\n public Email: RegExp\n public Handle: RegExp\n public IntOrVersion: RegExp\n public Domain: RegExp\n public Website: RegExp\n public Preposition: RegExp\n public ComplexPreposition = /^([^ \\t\\n\\r\\u00A0]+)([ \\t\\n\\r\\u00A0]+)([^ \\t\\n\\r\\u00A0]+)(?:([ \\t\\n\\r\\u00A0]+)([^ \\t\\n\\r\\u00A0]+))?$/u\n public P = /\\p{P}/u\n\n constructor() {\n const B = '(?=(?:[^\\\\p{L}\\\\p{N}\\\\p{M}\\\\u00AD\\\\u2060]|$))'\n const LNM = '[\\\\p{L}\\\\p{N}\\\\p{M}\\u00AD\\u2060]'\n const W = `${LNM}*?\\\\p{L}${LNM}*`\n\n this.Acronym = new RegExp(`(?:(?:(?:[\\\\p{Lu}\\\\p{Lt}][.]){2,}${B})|(?:(?:vs?[.])(?=[ \\t\\n\\r\\u00A0])))`, 'u')\n this.Word = new RegExp(`${W}${B}`, 'u')\n\n this.Contraction = new RegExp(`${W}['\u2019]${W}${B}`, 'u')\n this.Ordinal = new RegExp(`\\\\d+(?:st|nd|rd|th)${B}`, 'u')\n this.Email = new RegExp(`[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:[.][A-Za-z0-9-]+)+${B}`, 'u')\n this.Handle = new RegExp(`@[A-Za-z0-9-]{2,}${B}`, 'u')\n this.IntOrVersion = new RegExp(`v?\\\\d+(?:\\\\.\\\\d+)*${B}`, 'u')\n this.Domain = new RegExp(`${W}(?:[.]${W})+${B}`, 'u')\n this.Website = new RegExp(`https?://${W}(?:[.]${W})+(?:[^.!? \\t\\n\\r\\u00A0]+|[.!?]${LNM})+`, 'u')\n\n const ci = (s: string): string =>\n s\n .replace(/[a-z]/ig, match => `[${match.toUpperCase()}${match.toLowerCase()}]`)\n .replace(' ', this.Whitespace.source)\n\n this.Preposition = new RegExp(`(?:${prepositions.sort().reverse().map(ci).join('|')})${B}`, 'u')\n }\n}()\n\nconst lexer = moo.compile({\n 'word-preposition': RE.Preposition,\n 'word-acronym': RE.Acronym,\n 'word-contraction': RE.Contraction,\n 'word-ordinal': RE.Ordinal,\n email: RE.Email,\n handle: RE.Handle,\n website: RE.Website,\n domain: RE.Domain,\n word: RE.Word,\n number: RE.IntOrVersion, // eslint-disable-line id-blacklist\n 'punctuation-end': /[?.!](?=[ \\t\\n\\r\\u00A0]|$)/u,\n 'punctuation-colon': /:(?=[ \\t\\n\\r\\u00A0])/u,\n 'punctuation-ellipsis': /[.][.][.]/u,\n punctuation: RE.P,\n whitespace: { match: /[ \\t\\n\\r\\u00A0]/u, lineBreaks: true },\n other: { match: /[\\s\\S]/u, lineBreaks: true },\n})\n\nconst Shape = new class {\n private shapes: Map<string, string> = new Map()\n private re: Record<string, RegExp> = {\n X: /[\\p{Lu}\\p{Lt}]/u,\n x: /[\\p{Ll}\\p{Lm}\\p{Lo}]/u,\n d: /\\p{N}/u,\n }\n\n private match(c: string): string {\n if (c.match(this.re.d)) return 'd'\n if (c.toLowerCase() === c.toUpperCase()) return c\n if (c.match(this.re.X)) return 'X'\n if (c.match(this.re.x)) return 'x'\n if (c === '\u2019') return \"'\"\n if (c === '\u2013') return '-'\n if (c === '\\u2060' || c === '\\u00AD') return ''\n return c\n }\n\n private fetch(c: string): string {\n if (!this.shapes.has(c)) this.shapes.set(c, this.match(c))\n return this.shapes.get(c)!\n }\n\n shape(t: string) {\n if (!this.shapes.has(t)) this.shapes.set(t, Array.from(t).map(c => this.fetch(c)).join(''))\n return this.shapes.get(t)!\n }\n}()\n\nexport type Token = {\n type: 'word' | 'domain' | 'whitespace'\n subtype: '' | 'preposition' | 'acronym' | 'whitespace' | 'word' | 'domain'\n text: string\n start: number\n end: number\n shape: string\n sentenceStart: boolean\n subSentenceStart: boolean\n hyphenated?: Token[]\n}\n\nfunction combine(tokens: Token[]) {\n const combined = { ...tokens[0] }\n for (const t of tokens.slice(1)) {\n combined.text += t.text\n combined.end = t.end\n combined.shape += t.shape\n }\n return combined\n}\n\nfunction hyphenate(t: Token) {\n if (t.type === 'word') return 'w'\n if (t.text === '-' || t.text === '\u2013') return '-'\n return ' '\n}\n\nexport function tokenize(title: string, markup?: RegExp): Token[] {\n if (markup) title = title.replace(markup, match => '\\u2060'.repeat(match.length))\n\n lexer.reset(title)\n const tokens: Token[] = []\n let sentenceStart = true\n let subSentenceStart = false\n for (const token of lexer) {\n const [type, subtype] = (token.type.includes('-') ? token.type : `${token.type}-`).split('-')\n\n tokens.push({\n type,\n subtype,\n text: token.text,\n start: token.offset,\n end: token.offset + token.text.length - 1,\n shape: Shape.shape(<string> token.text)!,\n sentenceStart: type === 'word' && sentenceStart,\n subSentenceStart: type === 'word' && subSentenceStart,\n })\n\n switch (token.type) {\n case 'punctuation-end':\n sentenceStart = true\n break\n case 'punctuation-colon':\n subSentenceStart = true\n break\n default:\n if (type.match(/word|number|handle|domain|website/)) {\n sentenceStart = false\n subSentenceStart = false\n }\n break\n }\n }\n\n const stack = tokens.splice(0)\n\n let cpt: RegExpMatchArray | null\n let cps: RegExpMatchArray | null\n while (stack.length) {\n if (stack[0].subtype === 'preposition' && (cpt = stack[0].text.match(RE.ComplexPreposition)) && (cps = stack[0].shape.match(RE.ComplexPreposition))) {\n const complex = stack.shift()!\n\n let start = complex.start\n let end\n for (const i of Array.from({ length: 5 }, (_, n) => n + 1)) {\n if (!cpt[i]) break\n\n end = start + cps[i].length - 1\n tokens.push({\n ...complex,\n text: cpt[i],\n shape: cps[i],\n start,\n end,\n type: i % 2 ? complex.type : 'whitespace',\n subtype: i % 2 ? complex.type : '',\n })\n start = end + 1\n }\n continue\n }\n\n // hyphenated words\n const pat = stack.map(t => hyphenate(t)).join('')\n if (cpt = pat.match(/^w(-w)+(?= |$)/)) {\n const hyphenated = stack.splice(0, cpt[0].length)\n tokens.push({ ...combine(hyphenated), hyphenated })\n continue\n }\n\n tokens.push(stack.shift()!)\n }\n\n return markup ? tokens.map(token => ({ ...token, text: title.substring(token.start, token.end + 1) })) : tokens\n}\n"],
|
|
5
|
-
"mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,iBAAgB;AAEhB,0BAAyB;AAIzB,MAAM,KAAK,IAAI,MAAM;AAAA,EACZ;AAAA,EACA;AAAA,EACA;AAAA,EACA,aAAa;AAAA,EACb;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,qBAAqB;AAAA,EACrB,IAAI,WAAC,UAAM,GAAC;AAAA,EAEnB,cAAc;AACZ,UAAM,IAAI;AACV,UAAM,MAAM;AACZ,UAAM,IAAI,GAAG,GAAG,WAAW,GAAG;AAE9B,SAAK,UAAU,IAAI,OAAO,oCAAoC,CAAC;AAAA,aAAwC,GAAG;AAC1G,SAAK,OAAO,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG;AAEtC,SAAK,cAAc,IAAI,OAAO,GAAG,CAAC,YAAO,CAAC,GAAG,CAAC,IAAI,GAAG;AACrD,SAAK,UAAU,IAAI,OAAO,sBAAsB,CAAC,IAAI,GAAG;AACxD,SAAK,QAAQ,IAAI,OAAO,uDAAuD,CAAC,IAAI,GAAG;AACvF,SAAK,SAAS,IAAI,OAAO,oBAAoB,CAAC,IAAI,GAAG;AACrD,SAAK,eAAe,IAAI,OAAO,qBAAqB,CAAC,IAAI,GAAG;AAC5D,SAAK,SAAS,IAAI,OAAO,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,GAAG;AACpD,SAAK,UAAU,IAAI,OAAO,YAAY,CAAC,SAAS,CAAC;AAAA,gBAAkC,GAAG,MAAM,GAAG;AAE/F,UAAM,KAAK,CAAC,MACV,EACG,QAAQ,WAAW,WAAS,IAAI,MAAM,YAAY,CAAC,GAAG,MAAM,YAAY,CAAC,GAAG,EAC5E,QAAQ,KAAK,KAAK,WAAW,MAAM;AAExC,SAAK,cAAc,IAAI,OAAO,MAAM,oBAAAA,QAAa,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,KAAK,GAAG,CAAC,IAAI,CAAC,IAAI,GAAG;AAAA,EACjG;AACF,EAAE;AAEF,MAAM,QAAQ,WAAAC,QAAI,QAAQ;AAAA,EACxB,oBAAoB,GAAG;AAAA,EACvB,gBAAgB,GAAG;AAAA,EACnB,oBAAoB,GAAG;AAAA,EACvB,gBAAgB,GAAG;AAAA,EACnB,OAAO,GAAG;AAAA,EACV,QAAQ,GAAG;AAAA,EACX,SAAS,GAAG;AAAA,EACZ,QAAQ,GAAG;AAAA,EACX,MAAM,GAAG;AAAA,EACT,QAAQ,GAAG;AAAA;AAAA,EACX,mBAAmB;AAAA,EACnB,qBAAqB;AAAA,EACrB,wBAAwB;AAAA,EACxB,aAAa,GAAG;AAAA,EAChB,YAAY,EAAE,OAAO,oBAAoB,YAAY,KAAK;AAAA,EAC1D,OAAO,EAAE,OAAO,WAAW,YAAY,KAAK;AAC9C,CAAC;AAED,MAAM,QAAQ,IAAI,MAAM;AAAA,EACd,SAA8B,oBAAI,IAAI;AAAA,EACtC,KAA6B;AAAA,IACnC,GAAG;AAAA,IACH,GAAG;AAAA,IACH,GAAG,WAAC,UAAM,GAAC;AAAA,EACb;AAAA,EAEQ,MAAM,GAAmB;AAC/B,QAAI,EAAE,MAAM,KAAK,GAAG,CAAC,EAAG,QAAO;AAC/B,QAAI,EAAE,YAAY,MAAM,EAAE,YAAY,EAAG,QAAO;AAChD,QAAI,EAAE,MAAM,KAAK,GAAG,CAAC,EAAG,QAAO;AAC/B,QAAI,EAAE,MAAM,KAAK,GAAG,CAAC,EAAG,QAAO;AAC/B,QAAI,MAAM,SAAK,QAAO;AACtB,QAAI,MAAM,SAAK,QAAO;AACtB,QAAI,MAAM,YAAY,MAAM,OAAU,QAAO;AAC7C,WAAO;AAAA,EACT;AAAA,EAEQ,MAAM,GAAmB;AAC/B,QAAI,CAAC,KAAK,OAAO,IAAI,CAAC,EAAG,MAAK,OAAO,IAAI,GAAG,KAAK,MAAM,CAAC,CAAC;AACzD,WAAO,KAAK,OAAO,IAAI,CAAC;AAAA,EAC1B;AAAA,EAEA,MAAM,GAAW;AACf,QAAI,CAAC,KAAK,OAAO,IAAI,CAAC,EAAG,MAAK,OAAO,IAAI,GAAG,MAAM,KAAK,CAAC,EAAE,IAAI,OAAK,KAAK,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;AAC1F,WAAO,KAAK,OAAO,IAAI,CAAC;AAAA,EAC1B;AACF,EAAE;AAcF,SAAS,QAAQ,QAAiB;AAChC,QAAM,WAAW,EAAE,GAAG,OAAO,CAAC,EAAE;AAChC,aAAW,KAAK,OAAO,MAAM,CAAC,GAAG;AAC/B,aAAS,QAAQ,EAAE;AACnB,aAAS,MAAM,EAAE;AACjB,aAAS,SAAS,EAAE;AAAA,EACtB;AACA,SAAO;AACT;AAEA,SAAS,UAAU,GAAU;AAC3B,MAAI,EAAE,SAAS,OAAQ,QAAO;AAC9B,MAAI,EAAE,SAAS,OAAO,EAAE,SAAS,SAAK,QAAO;AAC7C,SAAO;AACT;AAEO,SAAS,SAAS,OAAe,QAA0B;AAChE,MAAI,OAAQ,SAAQ,MAAM,QAAQ,QAAQ,WAAS,SAAS,OAAO,MAAM,MAAM,CAAC;AAEhF,QAAM,MAAM,KAAK;AACjB,QAAM,SAAkB,CAAC;AACzB,MAAI,gBAAgB;AACpB,MAAI,mBAAmB;AACvB,aAAW,SAAS,OAAO;AACzB,UAAM,CAAC,MAAM,OAAO,KAAK,MAAM,KAAK,SAAS,GAAG,IAAI,MAAM,OAAO,GAAG,MAAM,IAAI,KAAK,MAAM,GAAG;AAE5F,WAAO,KAAK;AAAA,MACV;AAAA,MACA;AAAA,MACA,MAAM,MAAM;AAAA,MACZ,OAAO,MAAM;AAAA,MACb,KAAK,MAAM,SAAS,MAAM,KAAK,SAAS;AAAA,MACxC,OAAO,MAAM,MAAe,MAAM,IAAI;AAAA,MACtC,eAAe,SAAS,UAAU;AAAA,MAClC,kBAAkB,SAAS,UAAU;AAAA,IACvC,CAAC;AAED,YAAQ,MAAM,MAAM;AAAA,MAClB,KAAK;AACH,wBAAgB;AAChB;AAAA,MACF,KAAK;AACH,2BAAmB;AACnB;AAAA,MACF;AACE,YAAI,KAAK,MAAM,mCAAmC,GAAG;AACnD,0BAAgB;AAChB,6BAAmB;AAAA,QACrB;AACA;AAAA,IACJ;AAAA,EACF;AAEA,QAAM,QAAQ,OAAO,OAAO,CAAC;AAE7B,MAAI;AACJ,MAAI;AACJ,SAAO,MAAM,QAAQ;AACnB,QAAI,MAAM,CAAC,EAAE,YAAY,kBAAkB,MAAM,MAAM,CAAC,EAAE,KAAK,MAAM,GAAG,kBAAkB,OAAO,MAAM,MAAM,CAAC,EAAE,MAAM,MAAM,GAAG,kBAAkB,IAAI;AACnJ,YAAM,UAAU,MAAM,MAAM;AAE5B,UAAI,QAAQ,QAAQ;AACpB,UAAI;AACJ,iBAAW,KAAK,MAAM,KAAK,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,MAAM,IAAI,CAAC,GAAG;AAC1D,YAAI,CAAC,IAAI,CAAC,EAAG;AAEb,cAAM,QAAQ,IAAI,CAAC,EAAE,SAAS;AAC9B,eAAO,KAAK;AAAA,UACV,GAAG;AAAA,UACH,MAAM,IAAI,CAAC;AAAA,UACX,OAAO,IAAI,CAAC;AAAA,UACZ;AAAA,UACA;AAAA,UACA,MAAM,IAAI,IAAI,QAAQ,OAAO;AAAA,UAC7B,SAAS,IAAI,IAAI,QAAQ,OAAO;AAAA,QAClC,CAAC;AACD,gBAAQ,MAAM;AAAA,MAChB;AACA;AAAA,IACF;AAGA,UAAM,MAAM,MAAM,IAAI,OAAK,UAAU,CAAC,CAAC,EAAE,KAAK,EAAE;AAChD,QAAI,MAAM,IAAI,MAAM,gBAAgB,GAAG;AACrC,YAAM,aAAa,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,MAAM;AAChD,aAAO,KAAK,EAAE,GAAG,QAAQ,UAAU,GAAG,WAAW,CAAC;AAClD;AAAA,IACF;AAEA,WAAO,KAAK,MAAM,MAAM,CAAE;AAAA,EAC5B;AAEA,SAAO,SAAS,OAAO,IAAI,YAAU,EAAE,GAAG,OAAO,MAAM,MAAM,UAAU,MAAM,OAAO,MAAM,MAAM,CAAC,EAAE,EAAE,IAAI;AAC3G;",
|
|
6
|
-
"names": ["prepositions", "moo"]
|
|
7
|
-
}
|
package/dist/cjs/verbatim.js
DELETED
|
@@ -1,403 +0,0 @@
|
|
|
1
|
-
var __defProp = Object.defineProperty;
|
|
2
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
3
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
4
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
5
|
-
var __export = (target, all) => {
|
|
6
|
-
for (var name in all)
|
|
7
|
-
__defProp(target, name, { get: all[name], enumerable: true });
|
|
8
|
-
};
|
|
9
|
-
var __copyProps = (to, from, except, desc) => {
|
|
10
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
11
|
-
for (let key of __getOwnPropNames(from))
|
|
12
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
13
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
14
|
-
}
|
|
15
|
-
return to;
|
|
16
|
-
};
|
|
17
|
-
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
18
|
-
var verbatim_exports = {};
|
|
19
|
-
__export(verbatim_exports, {
|
|
20
|
-
Library: () => Library,
|
|
21
|
-
parse: () => parse,
|
|
22
|
-
promises: () => promises
|
|
23
|
-
});
|
|
24
|
-
module.exports = __toCommonJS(verbatim_exports);
|
|
25
|
-
var import_yield = require("./yield.js");
|
|
26
|
-
class ParsingError extends Error {
|
|
27
|
-
source;
|
|
28
|
-
constructor(message, parser) {
|
|
29
|
-
message += ` at ${parser.location()}`;
|
|
30
|
-
if (parser.parsing) message += ` in ${JSON.stringify(parser.parsing)}`;
|
|
31
|
-
super(message);
|
|
32
|
-
this.name = "ParsingError";
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
const letter = /[\p{Lu}\p{Lt}\p{Ll}\p{Lm}\p{Lo}]/u;
|
|
36
|
-
class Library {
|
|
37
|
-
parsing;
|
|
38
|
-
entries = [];
|
|
39
|
-
strings = {};
|
|
40
|
-
comments = [];
|
|
41
|
-
errors = [];
|
|
42
|
-
preambles = [];
|
|
43
|
-
default_strings = {
|
|
44
|
-
JAN: "01",
|
|
45
|
-
JANUARY: "01",
|
|
46
|
-
FEB: "02",
|
|
47
|
-
FEBRUARY: "02",
|
|
48
|
-
MAR: "03",
|
|
49
|
-
MARCH: "03",
|
|
50
|
-
APR: "04",
|
|
51
|
-
APRIL: "04",
|
|
52
|
-
MAY: "05",
|
|
53
|
-
JUN: "06",
|
|
54
|
-
JUNE: "06",
|
|
55
|
-
JUL: "07",
|
|
56
|
-
JULY: "07",
|
|
57
|
-
AUG: "08",
|
|
58
|
-
AUGUST: "08",
|
|
59
|
-
SEP: "09",
|
|
60
|
-
SEPTEMBER: "09",
|
|
61
|
-
OCT: "10",
|
|
62
|
-
OCTOBER: "10",
|
|
63
|
-
NOV: "11",
|
|
64
|
-
NOVEMBER: "11",
|
|
65
|
-
DEC: "12",
|
|
66
|
-
DECEMBER: "12",
|
|
67
|
-
ACMCS: "ACM Computing Surveys",
|
|
68
|
-
ACTA: "Acta Informatica",
|
|
69
|
-
CACM: "Communications of the ACM",
|
|
70
|
-
IBMJRD: "IBM Journal of Research and Development",
|
|
71
|
-
IBMSJ: "IBM Systems Journal",
|
|
72
|
-
IEEESE: "IEEE Transactions on Software Engineering",
|
|
73
|
-
IEEETC: "IEEE Transactions on Computers",
|
|
74
|
-
IEEETCAD: "IEEE Transactions on Computer-Aided Design of Integrated Circuits",
|
|
75
|
-
IPL: "Information Processing Letters",
|
|
76
|
-
JACM: "Journal of the ACM",
|
|
77
|
-
JCSS: "Journal of Computer and System Sciences",
|
|
78
|
-
SCP: "Science of Computer Programming",
|
|
79
|
-
SICOMP: "SIAM Journal on Computing",
|
|
80
|
-
TOCS: "ACM Transactions on Computer Systems",
|
|
81
|
-
TODS: "ACM Transactions on Database Systems",
|
|
82
|
-
TOG: "ACM Transactions on Graphics",
|
|
83
|
-
TOMS: "ACM Transactions on Mathematical Software",
|
|
84
|
-
TOOIS: "ACM Transactions on Office Information Systems",
|
|
85
|
-
TOPLAS: "ACM Transactions on Programming Languages and Systems",
|
|
86
|
-
TCS: "Theoretical Computer Science"
|
|
87
|
-
};
|
|
88
|
-
pos = 0;
|
|
89
|
-
linebreaks = [];
|
|
90
|
-
input;
|
|
91
|
-
max_entries;
|
|
92
|
-
constructor(input, options = {}) {
|
|
93
|
-
this.max_entries = options.max_entries || 0;
|
|
94
|
-
this.input = input;
|
|
95
|
-
this.parsing = null;
|
|
96
|
-
if (typeof options.strings === "string") {
|
|
97
|
-
this.input = options.strings + "\n" + input;
|
|
98
|
-
} else if (options.strings) {
|
|
99
|
-
for (const [k, v] of Object.entries(options.strings)) {
|
|
100
|
-
this.default_strings[k.toUpperCase()] = v;
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
let pos = input.indexOf("\n");
|
|
104
|
-
while (pos !== -1) {
|
|
105
|
-
this.linebreaks.push(pos);
|
|
106
|
-
pos = input.indexOf("\n", pos + 1);
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
parse() {
|
|
110
|
-
this.bibtex();
|
|
111
|
-
this.entries.reverse();
|
|
112
|
-
}
|
|
113
|
-
async parseAsync() {
|
|
114
|
-
await this.bibtexAsync();
|
|
115
|
-
this.entries.reverse();
|
|
116
|
-
}
|
|
117
|
-
isWhitespace(s, horizontalOnly = false) {
|
|
118
|
-
return s === " " || s === " " || !horizontalOnly && (s === "\r" || s === "\n");
|
|
119
|
-
}
|
|
120
|
-
match(s, sws = true) {
|
|
121
|
-
this.skipWhitespace();
|
|
122
|
-
if (this.input.substr(this.pos, s.length) !== s) {
|
|
123
|
-
throw new ParsingError(`Token mismatch, expected ${JSON.stringify(s)}, found ${JSON.stringify(this.input.substr(this.pos, 20))}...`, this);
|
|
124
|
-
}
|
|
125
|
-
this.pos += s.length;
|
|
126
|
-
if (sws) this.skipWhitespace();
|
|
127
|
-
}
|
|
128
|
-
tryMatch(s, consume = false) {
|
|
129
|
-
this.skipWhitespace();
|
|
130
|
-
const match = this.input.substr(this.pos, s.length) === s;
|
|
131
|
-
if (match && consume) this.pos += s.length;
|
|
132
|
-
return match;
|
|
133
|
-
}
|
|
134
|
-
skipWhitespace() {
|
|
135
|
-
while (this.pos < this.input.length && this.input[this.pos].match(/[ \t\r\n%]/)) {
|
|
136
|
-
if (this.input[this.pos] === "%") {
|
|
137
|
-
while (this.pos < this.input.length && this.input[this.pos] !== "\n") this.pos++;
|
|
138
|
-
} else {
|
|
139
|
-
this.pos++;
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
/*
|
|
144
|
-
private fix_path() {
|
|
145
|
-
// workaround for https://github.com/siefkenj/unified-latex/issues/94
|
|
146
|
-
const path = '\\path|'
|
|
147
|
-
const verb = path.replace('path', 'verb')
|
|
148
|
-
if (this.input.substring(this.pos).startsWith(path)) {
|
|
149
|
-
// eslint-disable-next-line prefer-template
|
|
150
|
-
this.input = this.input.substring(0, this.pos) + verb + this.input.substring(this.pos + path.length)
|
|
151
|
-
}
|
|
152
|
-
}
|
|
153
|
-
*/
|
|
154
|
-
value_braces() {
|
|
155
|
-
let bracecount = 0;
|
|
156
|
-
this.match("{", false);
|
|
157
|
-
const start = this.pos;
|
|
158
|
-
let math = false;
|
|
159
|
-
while (true) {
|
|
160
|
-
switch (this.input[this.pos]) {
|
|
161
|
-
case "\\":
|
|
162
|
-
this.pos += 1;
|
|
163
|
-
break;
|
|
164
|
-
case "{":
|
|
165
|
-
bracecount++;
|
|
166
|
-
break;
|
|
167
|
-
case "}":
|
|
168
|
-
if (bracecount === 0) {
|
|
169
|
-
if (math) throw new ParsingError("Unclosed math section", this);
|
|
170
|
-
this.pos++;
|
|
171
|
-
return this.input.substring(start, this.pos - 1);
|
|
172
|
-
}
|
|
173
|
-
bracecount--;
|
|
174
|
-
break;
|
|
175
|
-
case "$":
|
|
176
|
-
math = !math;
|
|
177
|
-
break;
|
|
178
|
-
}
|
|
179
|
-
this.pos++;
|
|
180
|
-
if (this.pos >= this.input.length) {
|
|
181
|
-
throw new ParsingError(`Unterminated brace-value ${JSON.stringify(this.input.substr(start, 20))}`, this);
|
|
182
|
-
}
|
|
183
|
-
}
|
|
184
|
-
}
|
|
185
|
-
value_quotes() {
|
|
186
|
-
this.match('"', false);
|
|
187
|
-
const start = this.pos;
|
|
188
|
-
let bracecount = 0;
|
|
189
|
-
while (true) {
|
|
190
|
-
switch (this.input[this.pos]) {
|
|
191
|
-
case "\\":
|
|
192
|
-
this.pos += 1;
|
|
193
|
-
break;
|
|
194
|
-
case "{":
|
|
195
|
-
bracecount++;
|
|
196
|
-
break;
|
|
197
|
-
case "}":
|
|
198
|
-
bracecount--;
|
|
199
|
-
break;
|
|
200
|
-
case '"':
|
|
201
|
-
if (bracecount <= 0) {
|
|
202
|
-
this.pos++;
|
|
203
|
-
return this.input.substring(start, this.pos - 1);
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
this.pos++;
|
|
207
|
-
if (this.pos >= this.input.length) {
|
|
208
|
-
throw new ParsingError(`Unterminated quote-value ${JSON.stringify(this.input.substr(start, 20))}`, this);
|
|
209
|
-
}
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
single_value() {
|
|
213
|
-
if (this.tryMatch("{")) {
|
|
214
|
-
return this.value_braces();
|
|
215
|
-
} else if (this.tryMatch('"')) {
|
|
216
|
-
return this.value_quotes();
|
|
217
|
-
} else {
|
|
218
|
-
const bare = this.key();
|
|
219
|
-
if (bare.match(/^\d+$/)) return bare;
|
|
220
|
-
const u_bare = bare.toUpperCase();
|
|
221
|
-
const resolved = this.strings[u_bare] || this.default_strings[u_bare];
|
|
222
|
-
if (typeof resolved === "undefined") {
|
|
223
|
-
this.error({
|
|
224
|
-
error: `unresolved @string reference ${JSON.stringify(bare)}`,
|
|
225
|
-
input: ""
|
|
226
|
-
});
|
|
227
|
-
}
|
|
228
|
-
return resolved || `{{${bare}}}`;
|
|
229
|
-
}
|
|
230
|
-
}
|
|
231
|
-
location() {
|
|
232
|
-
const lines = this.input.substring(0, this.pos).split("\n");
|
|
233
|
-
return `line ${lines.length}, column ${lines[lines.length - 1].length + 1}`;
|
|
234
|
-
}
|
|
235
|
-
error(err) {
|
|
236
|
-
if (this.errors.find((e) => e.error === err.error)) return;
|
|
237
|
-
this.errors.push(err);
|
|
238
|
-
}
|
|
239
|
-
value() {
|
|
240
|
-
const values = [];
|
|
241
|
-
values.push(this.single_value());
|
|
242
|
-
while (this.tryMatch("#")) {
|
|
243
|
-
this.match("#");
|
|
244
|
-
values.push(this.single_value());
|
|
245
|
-
}
|
|
246
|
-
return values.join("");
|
|
247
|
-
}
|
|
248
|
-
key(allow = "") {
|
|
249
|
-
const start = this.pos;
|
|
250
|
-
while (true) {
|
|
251
|
-
if (this.pos === this.input.length) {
|
|
252
|
-
throw new ParsingError("Runaway key", this);
|
|
253
|
-
}
|
|
254
|
-
if (this.input[this.pos].match(/[+'a-zA-Z0-9&;_:\\./-]/) || this.input[this.pos].match(letter) || allow.includes(this.input[this.pos])) {
|
|
255
|
-
this.pos++;
|
|
256
|
-
} else {
|
|
257
|
-
return this.input.substring(start, this.pos);
|
|
258
|
-
}
|
|
259
|
-
}
|
|
260
|
-
}
|
|
261
|
-
key_equals_value(key) {
|
|
262
|
-
key = key || this.key();
|
|
263
|
-
if (!key) return;
|
|
264
|
-
if (!this.tryMatch("=")) {
|
|
265
|
-
throw new ParsingError(`${key} = value expected, equals sign missing: ${JSON.stringify(this.input.substr(this.pos, 20))}...`, this);
|
|
266
|
-
}
|
|
267
|
-
this.match("=");
|
|
268
|
-
const val = this.value();
|
|
269
|
-
if (this.parsing === "string") {
|
|
270
|
-
this.strings[key.toUpperCase()] = val;
|
|
271
|
-
} else {
|
|
272
|
-
const bare = key.toLowerCase();
|
|
273
|
-
let postfix = 0;
|
|
274
|
-
while (typeof this.entries[0].fields[key = postfix ? `${bare}+duplicate-${postfix}` : bare] === "string") {
|
|
275
|
-
postfix++;
|
|
276
|
-
}
|
|
277
|
-
this.entries[0].fields[key] = val;
|
|
278
|
-
}
|
|
279
|
-
}
|
|
280
|
-
entry(d, guard) {
|
|
281
|
-
if (this.tryMatch(guard)) return;
|
|
282
|
-
let key;
|
|
283
|
-
if (this.tryMatch(",")) {
|
|
284
|
-
key = "";
|
|
285
|
-
} else {
|
|
286
|
-
key = this.key('[]*"');
|
|
287
|
-
}
|
|
288
|
-
if (this.tryMatch("=")) {
|
|
289
|
-
this.key_equals_value(key);
|
|
290
|
-
} else {
|
|
291
|
-
this.entries[0].key = key;
|
|
292
|
-
}
|
|
293
|
-
this.tryMatch(",", true);
|
|
294
|
-
if (this.tryMatch(guard)) return;
|
|
295
|
-
this.key_equals_value();
|
|
296
|
-
while (this.tryMatch(",", true)) {
|
|
297
|
-
if (this.tryMatch(guard)) return;
|
|
298
|
-
this.key_equals_value();
|
|
299
|
-
}
|
|
300
|
-
}
|
|
301
|
-
directive() {
|
|
302
|
-
this.match("@");
|
|
303
|
-
return this.key().toLowerCase();
|
|
304
|
-
}
|
|
305
|
-
string() {
|
|
306
|
-
this.key_equals_value();
|
|
307
|
-
}
|
|
308
|
-
preamble() {
|
|
309
|
-
this.preambles.push(this.value());
|
|
310
|
-
}
|
|
311
|
-
comment() {
|
|
312
|
-
while (this.isWhitespace(this.input[this.pos], true)) this.pos++;
|
|
313
|
-
if (this.input[this.pos] === "{") {
|
|
314
|
-
this.comments.push(this.value_braces());
|
|
315
|
-
} else {
|
|
316
|
-
const start = this.pos;
|
|
317
|
-
while (this.input[this.pos] !== "\n" && this.pos < this.input.length) this.pos++;
|
|
318
|
-
this.comments.push(this.input.substring(start, this.pos));
|
|
319
|
-
}
|
|
320
|
-
}
|
|
321
|
-
hasMore() {
|
|
322
|
-
if (this.max_entries && this.entries.length >= this.max_entries) return false;
|
|
323
|
-
return this.pos < this.input.length;
|
|
324
|
-
}
|
|
325
|
-
bibtex() {
|
|
326
|
-
while (this.hasMore()) {
|
|
327
|
-
this.parseNext();
|
|
328
|
-
}
|
|
329
|
-
}
|
|
330
|
-
async bibtexAsync() {
|
|
331
|
-
let n = 1;
|
|
332
|
-
while (this.hasMore()) {
|
|
333
|
-
this.parseNext();
|
|
334
|
-
if (n++ % 1e3 === 0) await (0, import_yield.playnice)();
|
|
335
|
-
}
|
|
336
|
-
}
|
|
337
|
-
matchGuard() {
|
|
338
|
-
for (const guard of ["{}", "()"]) {
|
|
339
|
-
if (this.tryMatch(guard[0])) {
|
|
340
|
-
this.match(guard[0]);
|
|
341
|
-
return guard[1];
|
|
342
|
-
}
|
|
343
|
-
}
|
|
344
|
-
throw new ParsingError(`Token mismatch, expected '{' or '(', found ${JSON.stringify(this.input.substr(this.pos, 20))}...`, this);
|
|
345
|
-
}
|
|
346
|
-
parseNext() {
|
|
347
|
-
this.skipWhitespace();
|
|
348
|
-
while (this.pos < this.input.length && this.input[this.pos] !== "@") {
|
|
349
|
-
if (this.input[this.pos] === "%") {
|
|
350
|
-
while (this.pos < this.input.length && this.input[this.pos] !== "\n") this.pos++;
|
|
351
|
-
} else {
|
|
352
|
-
this.pos++;
|
|
353
|
-
}
|
|
354
|
-
}
|
|
355
|
-
if (this.pos >= this.input.length) return;
|
|
356
|
-
let guard = "";
|
|
357
|
-
const start = this.pos;
|
|
358
|
-
try {
|
|
359
|
-
const d = this.parsing = this.directive();
|
|
360
|
-
switch (d) {
|
|
361
|
-
case "string":
|
|
362
|
-
guard = this.matchGuard();
|
|
363
|
-
this.string();
|
|
364
|
-
this.match(guard);
|
|
365
|
-
break;
|
|
366
|
-
case "preamble":
|
|
367
|
-
this.preamble();
|
|
368
|
-
break;
|
|
369
|
-
case "comment":
|
|
370
|
-
this.comment();
|
|
371
|
-
break;
|
|
372
|
-
default:
|
|
373
|
-
guard = this.matchGuard();
|
|
374
|
-
this.entries.unshift({ input: "", type: d, key: "", fields: {} });
|
|
375
|
-
this.entry(d, guard);
|
|
376
|
-
this.match(guard);
|
|
377
|
-
this.entries[0].input = this.input.substring(start, this.pos).trim();
|
|
378
|
-
break;
|
|
379
|
-
}
|
|
380
|
-
} catch (err) {
|
|
381
|
-
if (err.name !== "ParsingError") throw err;
|
|
382
|
-
this.pos = start + 1;
|
|
383
|
-
while (this.pos < this.input.length && this.input[this.pos] !== "@") this.pos++;
|
|
384
|
-
this.error({
|
|
385
|
-
error: err.message,
|
|
386
|
-
input: this.input.substring(start, this.pos)
|
|
387
|
-
});
|
|
388
|
-
}
|
|
389
|
-
}
|
|
390
|
-
}
|
|
391
|
-
function parse(input, options = {}) {
|
|
392
|
-
const parser = new Library(input, options);
|
|
393
|
-
parser.parse();
|
|
394
|
-
return parser;
|
|
395
|
-
}
|
|
396
|
-
const promises = {
|
|
397
|
-
async parse(input, options = {}) {
|
|
398
|
-
const parser = new Library(input, options);
|
|
399
|
-
await parser.parseAsync();
|
|
400
|
-
return parser;
|
|
401
|
-
}
|
|
402
|
-
};
|
|
403
|
-
//# sourceMappingURL=verbatim.js.map
|