@retorquere/bibtex-parser 8.0.12 → 8.0.13

package/index.js

@@ -57,6 +57,7 @@ const JabRef = __importStar(require("./jabref"));
 var sentence_case_1 = require("./sentence-case");
 Object.defineProperty(exports, "toSentenceCase", { enumerable: true, get: function () { return sentence_case_1.toSentenceCase; } });
 const sentence_case_2 = require("./sentence-case");
+const tokenizer_1 = require("./tokenizer");
 const crossref_json_1 = __importDefault(require("./crossref.json"));
 const fields_json_1 = __importDefault(require("./fields.json"));
 const unabbreviations = require('./unabbrev.json');
@@ -753,14 +754,13 @@ class BibTeXParser {
         if (FieldAction.parseInt.includes(field) && value.trim().match(/^-?\d+$/))
             return `${parseInt(value)}`;
         if (mode === 'title' && sentenceCase) {
-            if (!this.options.sentenceCase.guess || !(0, sentence_case_2.guessSentenceCased)(value, /\x0E\/?([a-z]+)\x0F/ig)) {
-                value = (0, sentence_case_2.toSentenceCase)(value, {
-                    preserveQuoted: this.options.sentenceCase.preserveQuoted,
-                    subSentenceCapitalization: this.options.sentenceCase.subSentence,
-                    markup: /\x0E\/?([a-z]+)\x0F/ig,
-                    nocase: /\x0E(ncx?)\x0F.*?\x0E\/\1\x0F/ig,
-                });
-            }
+            value = (0, sentence_case_2.toSentenceCase)(value, {
+                preserveQuoted: this.options.sentenceCase.preserveQuoted,
+                subSentenceCapitalization: this.options.sentenceCase.subSentence,
+                markup: /\x0E\/?([a-z]+)\x0F/ig,
+                nocase: /\x0E(ncx?)\x0F.*?\x0E\/\1\x0F/ig,
+                guess: this.options.sentenceCase.guess && !value.match(/\x0E(nc)\x0F/),
+            });
             let cancel = (_match, stripped) => stripped;
             switch (this.options.caseProtection) {
                 case 'strict':
@@ -768,7 +768,7 @@ class BibTeXParser {
                     break;
                 case 'as-needed':
                     cancel = (match, stripped) => {
-                        const words = (0, sentence_case_2.tokenize)(stripped, /\x0E\/?([a-z]+)\x0F/ig);
+                        const words = (0, tokenizer_1.tokenize)(stripped, /\x0E\/?([a-z]+)\x0F/ig);
                         return words.find(w => w.shape.match(/^(?!.*X).*x.*$/)) ? match : this.wrap(stripped, 'ncx');
                     };
                     break;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@retorquere/bibtex-parser",
-  "version": "8.0.12",
+  "version": "8.0.13",
   "description": "BibTeX parser",
   "main": "index.js",
   "typings": "./index.d.ts",
@@ -23,7 +23,7 @@
     "update-cases": "curl -s https://raw.githubusercontent.com/retorquere/zotero-better-bibtex/master/test/loaded.json | jq -r '.[] | select(test(\"\\\\.bib((la)?tex)?$\"))' | rsync -av --delete -m --files-from=- ../better-bibtex/test/fixtures/ test/better-bibtex/",
     "predocs": "npm start",
     "docs": "typedoc",
-    "ncu": "./build/submodules.py && ./build/crossref.py && ncu -u && npm i && git add package.json package-lock.json && npm audit fix && git commit -m ncu"
+    "ncu": "./build/submodules.py && ./build/crossref.py && ncu -u -x eslint && npm i && git add package.json package-lock.json && npm audit fix && git commit -m ncu"
   },
   "repository": {
     "type": "git",
@@ -35,9 +35,9 @@
   "author": "Emiliano Heyns",
   "license": "ISC",
   "devDependencies": {
-    "@types/node": "^20.12.2",
-    "@typescript-eslint/eslint-plugin": "^7.4.0",
-    "@typescript-eslint/parser": "^7.4.0",
+    "@types/node": "^20.12.7",
+    "@typescript-eslint/eslint-plugin": "^7.6.0",
+    "@typescript-eslint/parser": "^7.6.0",
     "command-line-args": "^5.2.1",
     "csv": "^6.3.8",
     "digital-tree": "^2.0.4",
@@ -54,15 +54,15 @@
     "tap": "^18.7.2",
     "ts-node": "^10.9.2",
     "tslib": "^2.6.2",
-    "typedoc": "^0.25.12",
-    "typescript": "^5.4.3"
+    "typedoc": "^0.25.13",
+    "typescript": "^5.4.5"
   },
   "dependencies": {
     "@unified-latex/unified-latex-util-pegjs": "^1.7.1",
     "@unified-latex/unified-latex-util-print-raw": "^1.7.1",
     "@unified-latex/unified-latex-util-replace": "^1.7.1",
     "@unified-latex/unified-latex-util-visit": "^1.7.1",
-    "compromise": "^14.13.0",
+    "moo": "^0.5.2",
     "unicode2latex": "^6.0.4",
     "wink-eng-lite-web-model": "^1.6.0",
     "wink-nlp": "^2.2.0",
@@ -84,7 +84,8 @@
     "sentence-case.d.ts",
     "fields.json",
     "strings.bib",
-    "unabbrev.json"
+    "unabbrev.json",
+    "prepositions.json"
   ],
   "publishConfig": {
     "access": "public"

package/prepositions.json

@@ -0,0 +1,88 @@
+[
+  "a",
+  "about",
+  "above",
+  "according to",
+  "across",
+  "after",
+  "against",
+  "ahead of",
+  "along",
+  "although",
+  "among",
+  "an",
+  "and",
+  "apart from",
+  "around",
+  "as for",
+  "as if",
+  "as long as",
+  "as of",
+  "as per",
+  "as regards",
+  "as soon as",
+  "as",
+  "aside from",
+  "at",
+  "back to",
+  "because of",
+  "because",
+  "before",
+  "behind",
+  "below",
+  "beneath",
+  "beside",
+  "between",
+  "but",
+  "by",
+  "close to",
+  "down",
+  "due to",
+  "during",
+  "except for",
+  "far from",
+  "for",
+  "from",
+  "if",
+  "in",
+  "inside of",
+  "inside",
+  "instead of",
+  "into",
+  "like",
+  "near",
+  "next to",
+  "nor",
+  "of",
+  "off",
+  "on",
+  "onto",
+  "or",
+  "out",
+  "outside of",
+  "over",
+  "owing to",
+  "prior to",
+  "pursuant to",
+  "regardless of",
+  "right of",
+  "since",
+  "so",
+  "subsequent to",
+  "the",
+  "through",
+  "to",
+  "towards",
+  "under",
+  "underneath",
+  "unless",
+  "until",
+  "up",
+  "upon",
+  "when",
+  "while",
+  "with",
+  "within",
+  "without",
+  "yet"
+]

package/sentence-case.d.ts

@@ -1,33 +1,9 @@
-type WordToken = {
-    type: 'word';
-    text: string;
-    shape: string;
-    start: number;
-    end: number;
-    sentenceStart?: boolean;
-    subSentenceStart?: boolean;
-};
-type PunctuationToken = {
-    type: 'punctuation';
-    text: string;
-    shape: string;
-    start: number;
-    end: number;
-};
-type WhitespaceToken = {
-    type: 'whitespace';
-    text: string;
-    shape: string;
-};
-type Token = WordToken | PunctuationToken | WhitespaceToken;
-export declare function tokenize(title: string, markup?: RegExp): Token[];
 export type Options = {
     preserveQuoted?: boolean;
     subSentenceCapitalization?: boolean;
     markup?: RegExp;
     nocase?: RegExp;
+    guess?: boolean;
 };
 export declare function toSentenceCase(title: string, options?: Options): string;
-export declare function guessSentenceCased(title: string, markup?: RegExp): boolean;
-export {};
 //# sourceMappingURL=sentence-case.d.ts.map

package/sentence-case.js

@@ -4,34 +4,17 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.guessSentenceCased = exports.toSentenceCase = exports.tokenize = void 0;
+exports.toSentenceCase = void 0;
 const xregexp_1 = __importDefault(require("xregexp"));
-const one_1 = __importDefault(require("compromise/one"));
-const unicode2latex_1 = require("unicode2latex");
-function gather(tokens) {
-    return {
-        start: tokens[0].start,
-        end: tokens[tokens.length - 1].end,
-        text: tokens.map(c => c.text).join(''),
-        type: 'word',
-        shape: tokens.map(c => c.shape).join(''),
-        sentenceStart: !!tokens.find(t => t.type === 'word' && t.sentenceStart) || false,
-        subSentenceStart: !!tokens.find(t => t.type === 'word' && t.subSentenceStart) || false,
-    };
-}
-const preposition = {
-    simple: /^(a|about|above|across|after|against|along|among|an|and|around|as|at|before|behind|below|beneath|beside|between|but|by|down|during|for|from|in|inside|into|like|near|nor|of|off|on|onto|or|out|over|so|the|through|to|towards|under|underneath|until|up|upon|with|within|without|yet)$/i,
-    complex: /^((according to)|(ahead of)|(apart from)|(as for)|(as of)|(as per)|(as regards)|(aside from)|(back to)|(because of)|(close to)|(due to)|(except for)|(far from)|(inside of)|(instead of)|(next to)|(outside of)|(owing to)|(prior to)|(pursuant to)|(regardless of)|(right of)|(subsequent to))$/i,
-};
+const tokenizer_1 = require("./tokenizer");
+// eslint-disable-next-line no-magic-numbers
+// const show = (obj: any): string => JSON.stringify(obj, null, 2).replace(/[\u007F-\uFFFF]/g, chr => `\\u${(`0000${chr.charCodeAt(0).toString(16)}`).substr(-4)}`)
 function titleCase(s) {
     return s.replace(/^(.)(.+)/, (match, car, cdr) => `${car}${cdr.toLowerCase()}`);
 }
-const Lu = (0, xregexp_1.default)('\\p{Lu}');
-const Ll = (0, xregexp_1.default)('\\p{Ll}');
 const connectedWord = (0, xregexp_1.default)('(^|-)\\p{Lu}\\p{Ll}*(?=-|$)', 'g');
 const connectedInnerWord = (0, xregexp_1.default)('-\\p{Lu}\\p{Ll}*(?=-|$)', 'g');
-const strayCC = new RegExp(`^(${unicode2latex_1.combining.regex})`);
-function wordSC(token, succ, allCaps, subSentence) {
+function wordSC(token, allCaps, subSentence) {
     if (token.type !== 'word')
         return token.text;
     if (token.text.match(/^I'/))
@@ -41,18 +24,16 @@ function wordSC(token, succ, allCaps, subSentence) {
     if ((subSentence && token.subSentenceStart) || token.sentenceStart) {
         return allCaps ? titleCase(token.text) : xregexp_1.default.replace(token.text, connectedInnerWord, match => match.toLowerCase());
     }
+    if (token.subtype === 'preposition')
+        return token.text.toLowerCase();
+    if (token.subtype === 'acronym')
+        return token.text;
     if (!allCaps && token.shape.match(/^[Xxd]+(-[Xxd]+)+/))
         return xregexp_1.default.replace(token.text, connectedWord, match => match.toLowerCase());
     if (token.text.match(/^[B-Z]$/))
         return token.text;
     if (!allCaps && token.shape.match(/^[-X]+$/))
         return token.text;
-    if (succ && `${token.text} ${succ.text}`.match(preposition.complex)) {
-        succ.text = succ.text.toLowerCase();
-        return token.text.toLowerCase();
-    }
-    if (token.text.match(preposition.simple))
-        return token.text.toLowerCase();
     const shape = token.shape.replace(/[^Xxd]/g, '');
     if (!allCaps && shape.match(/^[Xd-]+$/))
         return token.text;
@@ -66,110 +47,48 @@ function wordSC(token, succ, allCaps, subSentence) {
         return token.text;
     return token.text.toLowerCase();
 }
-function tokentype(token) {
-    if (token.type === 'word' && token.text.match(/^\u2060+$/))
-        return 'm';
-    if (token.type === 'word')
-        return 'w';
-    if (token.shape === '-')
-        return '-';
-    return ' ';
-}
-function prepost(s, offset) {
-    const tokens = [];
-    for (const text of s.match(/(\s+|\S+)/g)) {
-        if (text.match(/\s/)) {
-            tokens.push({ type: 'whitespace', text, shape: text });
-        }
-        else if (text) {
-            tokens.push({ type: 'punctuation', text, shape: text, start: offset, end: offset + text.length - 1 });
-        }
-        offset += text.length;
-    }
-    return tokens;
-}
-function tokenize(title, markup) {
-    if (markup)
-        title = title.replace(markup, match => '\u2060'.repeat(match.length));
-    const doc = (0, one_1.default)(title);
-    const tokens = [];
-    for (const sentence of doc.json({ offset: true })) {
-        const words = [];
-        let sentenceStart = true;
-        let subSentenceStart = false;
-        for (const term of sentence.terms) {
-            const m = term.post.match(strayCC);
-            if (m) {
-                term.text += m[0];
-                term.post = term.post.substring(m[0].length);
-                term.offset.length += m[0].length;
+function toSentenceCase(title, options = {}) {
+    options = Object.assign({ preserveQuoted: true, subSentenceCapitalization: true }, options);
+    const allCaps = title === title.toUpperCase();
+    const allLower = title === title.toLowerCase();
+    if (options.guess && !allCaps && !allLower) {
+        let $title = title;
+        if (options.nocase)
+            $title = $title.replace(options.nocase, match => match.match(/\s/) ? ' ' : '');
+        if (options.markup)
+            $title = $title.replace(options.markup, '');
+        if (options.preserveQuoted) {
+            for (const q of [/“.*?”/g, /‘.*?’/g, /".*?"/g]) {
+                $title = $title.replace(q, '');
             }
-            if (term.pre)
-                words.push(...prepost(term.pre, term.offset.start - term.pre.length));
-            words.push({
-                start: term.offset.start,
-                end: term.offset.start + term.offset.length - 1,
-                text: term.text,
-                type: 'word',
-                shape: xregexp_1.default.replaceEach(term.text, [[Lu, 'X', 'all'], [Ll, 'x', 'all'], [/\d/g, 'd']]),
-                sentenceStart,
-                subSentenceStart,
-            });
-            sentenceStart = false;
-            subSentenceStart = term.post.includes(':');
-            if (term.post)
-                words.push(...prepost(term.post, term.offset.start + term.offset.length));
         }
-        subSentenceStart = true;
-        while (words.length) {
-            if (words[0].type === 'word' && words[0].text.includes('/')) {
-                const multiple = words.shift();
-                let start = multiple.start;
-                words.unshift(...multiple.text.match(/([/]|[^/]+)/g).map((w) => {
-                    const started = start;
-                    start += w.length;
-                    const ss = w === '/' ? [false, false] : [multiple.sentenceStart, multiple.subSentenceStart];
-                    if (w !== '/')
-                        multiple.sentenceStart = multiple.subSentenceStart = false;
-                    return {
-                        start: started,
-                        end: start - 1,
-                        text: w,
-                        type: w === '/' ? 'punctuation' : 'word',
-                        shape: xregexp_1.default.replaceEach(w, [[Lu, 'X', 'all'], [Ll, 'x', 'all'], [/\d/g, 'd']]),
-                        sentenceStart: ss[0],
-                        subSentenceStart: ss[1],
-                    };
-                }));
-                continue;
+        let sc = 0;
+        const words = (0, tokenizer_1.tokenize)($title);
+        const minsc = 2;
+        words.forEach((token, i) => {
+            // reset for each subsentence
+            if (options.subSentenceCapitalization && token.type === 'punctuation' && token.subtype === 'end' && sc < minsc) {
+                sc = 0;
             }
-            const type = words.map(tokentype).join('');
-            let m;
-            if (m = type.match(/^m?[wc]m?(-m?[wc]m?)+/)) {
-                tokens.push(gather(words.splice(0, m[0].length)));
-                continue;
+            if (token.type === 'word' && token.text.length > 1 && !token.subtype.match(/preposition|acronym|ordinal/) && token.shape.match(/^[^X]*x[^X]*$/)) {
+                if (i === 0 || words[i - 1].type === 'whitespace') {
+                    sc += 1;
+                }
             }
-            const word = words.shift();
-            if (word.type !== 'whitespace')
-                tokens.push(word);
-        }
+        });
+        if (sc >= minsc)
+            return title;
     }
-    return tokens;
-}
-exports.tokenize = tokenize;
-function toSentenceCase(title, options = {}) {
-    options = Object.assign({ preserveQuoted: true, subSentenceCapitalization: true }, options);
     title = title.normalize('NFC'); // https://github.com/winkjs/wink-nlp/issues/134
-    const allCaps = title === title.toUpperCase();
     if (allCaps && !title.match(/\s/))
         return title;
     let sentenceCased = title;
-    const tokens = tokenize(title, options.markup);
-    tokens.forEach((token, i) => {
+    const tokens = (0, tokenizer_1.tokenize)(title, options.markup);
+    for (const token of tokens) {
         if (token.type !== 'whitespace') {
-            sentenceCased = sentenceCased.substring(0, token.start) + wordSC(token, tokens[i + 1], allCaps, options.subSentenceCapitalization) + sentenceCased.substring(token.end + 1);
+            sentenceCased = sentenceCased.substring(0, token.start) + wordSC(token, allCaps, options.subSentenceCapitalization) + sentenceCased.substring(token.end + 1);
         }
-    });
+    }
     if (options.markup) {
         for (const match of title.matchAll(options.markup)) {
             sentenceCased = sentenceCased.substring(0, match.index) + match[0] + sentenceCased.substring(match.index + match[0].length);
@@ -187,17 +106,4 @@ function toSentenceCase(title, options = {}) {
     return sentenceCased;
 }
 exports.toSentenceCase = toSentenceCase;
-function guessSentenceCased(title, markup = /<\/?(?:i|b|sup|sub|ncx?)>/g) {
-    const noMarkup = title.replace(markup, '');
-    if (noMarkup === noMarkup.toUpperCase())
-        return false;
-    if (noMarkup === noMarkup.toLowerCase())
-        return false;
-    const words = tokenize(title, markup).filter(token => token.type === 'word' && token.shape.match(/x/i));
-    if (!words.length)
-        return true;
-    const titleCased = words.filter(word => word.shape.match(/^X+$|^X.*x|[^Xx]+-X/));
-    return (titleCased.length / words.length) < 0.5; // eslint-disable-line no-magic-numbers
-}
-exports.guessSentenceCased = guessSentenceCased;
 //# sourceMappingURL=sentence-case.js.map