@retorquere/bibtex-parser 7.0.16 → 8.0.1

package/package.json

@@ -1,15 +1,15 @@
 {
   "name": "@retorquere/bibtex-parser",
-  "version": "7.0.16",
+  "version": "8.0.1",
   "description": "BibTeX parser",
   "main": "index.js",
   "typings": "./index.d.ts",
   "scripts": {
+    "unified": "tsx unified.ts",
     "lint": "eslint . --ext .ts --cache --cache-location .eslintcache/",
-    "prestart": "npm run lint && peggy grammar.peggy",
-    "start": "tsc -d",
-    "tsc": "tsc -d",
-    "poststart": "npm run docs",
+    "prestart": "npm run lint",
+    "start": "npm run tsc",
+    "tsc": "peggy grammar.peggy && tsc -d",
     "pretest": "npm start",
     "test": "tap --timeout 0 --disable-coverage --allow-empty-coverage",
     "snap": "npm test --snapshot",
@@ -18,10 +18,11 @@
     "list": "tar tvf $(npm pack)",
     "prepublishOnly": "npm start",
     "abbr": "./build/abbr.py",
-    "preversion": "npm test && npm run abbr",
+    "preversion": "./console.py r && npm test && npm run abbr",
     "postversion": "git push --follow-tags",
     "update-cases": "curl -s https://raw.githubusercontent.com/retorquere/zotero-better-bibtex/master/test/loaded.json | jq -r '.[] | select(test(\"\\\\.bib((la)?tex)?$\"))' | rsync -av --delete -m --files-from=- ../better-bibtex/test/fixtures/ test/better-bibtex/",
     "x-docs": "rm -rf docs temp && typedoc --name 'BibTeX parser' index.ts && touch docs/.nojekyll",
+    "predocs": "npm start",
     "docs": "mkdir -p etc temp && npm run api-extractor && npm run api-documenter",
     "ncu": "./build/submodules.py && ./build/crossref.py && ncu -u && npm i && git add package.json package-lock.json && npm audit fix && git commit -m ncu",
     "api-documenter": "api-documenter markdown --input temp --output docs",
@@ -37,39 +38,47 @@
   "author": "Emiliano Heyns",
   "license": "ISC",
   "devDependencies": {
-    "@microsoft/api-documenter": "^7.23.31",
-    "@microsoft/api-extractor": "^7.41.0",
-    "@types/node": "^20.11.20",
-    "@typescript-eslint/eslint-plugin": "^7.0.2",
-    "@typescript-eslint/parser": "^7.0.2",
+    "@microsoft/api-documenter": "^7.24.1",
+    "@microsoft/api-extractor": "^7.43.0",
+    "@types/node": "^20.12.2",
+    "@typescript-eslint/eslint-plugin": "^7.4.0",
+    "@typescript-eslint/parser": "^7.4.0",
     "command-line-args": "^5.2.1",
-    "csv": "^6.3.6",
+    "csv": "^6.3.8",
     "digital-tree": "^2.0.4",
     "ejs": "^3.1.9",
+    "esbuild": "^0.20.2",
     "eslint": "^8.57.0",
     "eslint-plugin-import": "^2.29.1",
     "eslint-plugin-prefer-arrow": "^1.2.3",
-    "glob": "^10.3.10",
+    "glob": "^10.3.12",
     "mkdirp": "^3.0.1",
     "node-fetch": "^3.3.2",
-    "peggy": "^4.0.1",
-    "piscina": "^4.3.2",
-    "slug": "^8.2.3",
-    "tap": "^18.7.0",
+    "peggy": "^4.0.2",
+    "piscina": "^4.4.0",
+    "slug": "^9.0.0",
+    "tap": "^18.7.2",
     "ts-node": "^10.9.2",
     "tslib": "^2.6.2",
-    "typescript": "^5.3.3"
+    "typescript": "^5.4.3"
   },
   "dependencies": {
-    "unicode2latex": "^5.0.13",
+    "@unified-latex/unified-latex-util-pegjs": "^1.7.1",
+    "@unified-latex/unified-latex-util-print-raw": "^1.7.1",
+    "@unified-latex/unified-latex-util-replace": "^1.7.1",
+    "@unified-latex/unified-latex-util-visit": "^1.7.1",
+    "compromise": "^14.13.0",
+    "unicode2latex": "^6.0.4",
+    "wink-eng-lite-web-model": "^1.6.0",
+    "wink-nlp": "^2.2.0",
     "xregexp": "^5.1.1"
   },
   "files": [
     "package.json",
     "grammar.js",
     "grammar.d.ts",
-    "chunker.d.ts",
-    "chunker.js",
+    "verbatim.d.ts",
+    "verbatim.js",
     "index.d.ts",
     "index.js",
     "jabref.d.ts",

package/sentence-case.d.ts

@@ -1,2 +1,33 @@
-export declare function toSentenceCase(sentence: string, preserveQuoted?: boolean): string;
+type WordToken = {
+    type: 'word';
+    text: string;
+    shape: string;
+    start: number;
+    end: number;
+    sentenceStart?: boolean;
+    subSentenceStart?: boolean;
+};
+type PunctuationToken = {
+    type: 'punctuation';
+    text: string;
+    shape: string;
+    start: number;
+    end: number;
+};
+type WhitespaceToken = {
+    type: 'whitespace';
+    text: string;
+    shape: string;
+};
+type Token = WordToken | PunctuationToken | WhitespaceToken;
+export declare function tokenize(title: string, markup?: RegExp): Token[];
+export type Options = {
+    preserveQuoted?: boolean;
+    subSentenceCapitalization?: boolean;
+    markup?: RegExp;
+    nocase?: RegExp;
+};
+export declare function toSentenceCase(title: string, options?: Options): string;
+export declare function guessSentenceCased(title: string, markup?: RegExp): boolean;
+export {};
 //# sourceMappingURL=sentence-case.d.ts.map

package/sentence-case.js

@@ -1,90 +1,203 @@
 "use strict";
+/* eslint-disable @typescript-eslint/unbound-method */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.toSentenceCase = void 0;
-/* eslint-disable @typescript-eslint/no-unsafe-argument */
-const categories = require("xregexp/tools/output/categories");
-const cat = name => categories.find(c => c.name === name).bmp;
-const L = cat('L');
-const Lu = cat('Lu');
-const Ll = cat('Ll');
-const N = cat('N');
-const No = cat('No');
-const Pc = cat('Pc');
-const re = {
-    acronym: new RegExp(`([${Lu}][.])+$`),
-    innerCaps: new RegExp(`.[${Lu}]`),
-    ident: new RegExp(`^[$p{L}]+[${N}${No}][${L}${N}${No}]*$`),
-    allCaps: new RegExp(`^[${Lu}${N}${No}]+$`),
-    skipWords: /^(but|or|yet|so|for|and|nor|a|an|the|at|by|from|in|into|of|on|to|with|updown|as)$/i,
-    // chemElements: /^(H|He|Li|Be|B|C|N|O|F|Ne|Na|Mg|Al|Si|P|S|Cl|Ar|K|Ca|Sc|Ti|V|Cr|Mn|Fe|Co|Ni|Cu|Zn|Ga|Ge|As|Se|Br|Kr|Rb|Sr|Y|Zr|Nb|Mo|Tc|Ru|Rh|Pb|Ag|Cd|In|Sn|Sb|Te|I|Xe|Cs|Ba|La|Hf|Ta|W|Re|Os|Ir|Pt|Au|Hg|Tl|Pb|Bi|Po|At|Rn|Fr|Ra|Ac|Rf|Db|Sg|Bh|Hs|Mt|Ds|Rg|Cn|Nh|Fl|Mc|Lv|Ts|Og|La|Ce|Pr|Nd|Pm|Sm|Eu|Gd|Tb|Dy|Ho|Er|Tm|Yb|Lu|Ac|Th|Pa|U|Np|Pu|Am|Cm|Bk|Cf|Es|Fm|Md|No|Lr)$/,
-    words: new RegExp(`([\uFFFD${L}${N}${No}]+([\uFFFD${Pc}${L}${N}${No}]*))|(\\s([\uFFFD${Lu}]+[.]){2,})?`, 'g'), // compound words and acronyms
-    titleCase: new RegExp(`^[${Lu}][${Ll}${N}${No}]+$`),
-    // private aint = XRegExp("^\\p{L}n't(?=$|[\\P{L}])") // isn't
-    // private word = XRegExp('^\\p{L}+([-.]\\p{L}+)*') // also match gallium-nitride as one word
-    // private and = XRegExp('^\\p{Lu}&\\p{Lu}(?=$|[\\P{L}])') // Q&A
-    subSentenceStart: new RegExp(`([.?!][\\s]+)(<[^>]+>)?([${Lu}])`, 'g'),
-    sentenceStart: new RegExp(`^(<[^>]+>)?([${Lu}])`, 'g'),
+exports.guessSentenceCased = exports.toSentenceCase = exports.tokenize = void 0;
+const xregexp_1 = __importDefault(require("xregexp"));
+const one_1 = __importDefault(require("compromise/one"));
+const unicode2latex_1 = require("unicode2latex");
+function gather(tokens) {
+    return {
+        start: tokens[0].start,
+        end: tokens[tokens.length - 1].end,
+        text: tokens.map(c => c.text).join(''),
+        type: 'word',
+        shape: tokens.map(c => c.shape).join(''),
+        sentenceStart: !!tokens.find(t => t.type === 'word' && t.sentenceStart) || false,
+        subSentenceStart: !!tokens.find(t => t.type === 'word' && t.subSentenceStart) || false,
+    };
+}
+const preposition = {
+    simple: /^(a|about|above|across|after|against|along|among|an|and|around|as|at|before|behind|below|beneath|beside|between|but|by|down|during|for|from|in|inside|into|like|near|nor|of|off|on|onto|or|out|over|so|the|through|to|towards|under|underneath|until|up|upon|with|within|without|yet)$/i,
+    complex: /^((according to)|(ahead of)|(apart from)|(as for)|(as of)|(as per)|(as regards)|(aside from)|(back to)|(because of)|(close to)|(due to)|(except for)|(far from)|(inside of)|(instead of)|(next to)|(outside of)|(owing to)|(prior to)|(pursuant to)|(regardless of)|(right of)|(subsequent to))$/i,
 };
-function lowercase(word, allcaps) {
-    if (!word)
-        return word;
-    if (allcaps)
-        return word.toLowerCase();
-    const unmasked = word.replace(/\uFFFD/g, '');
-    if (unmasked.match(re.skipWords))
-        return word.toLowerCase();
-    if (unmasked.match(re.titleCase))
-        return word.toLowerCase();
-    // if (unmasked.match(re.chemElements)) return word
-    if (unmasked.length === 1)
-        return unmasked === 'A' ? word.toLowerCase() : word;
-    if (unmasked.match(re.innerCaps))
-        return word;
-    if (unmasked.match(re.ident) || unmasked.match(re.allCaps))
-        return word;
-    return word.toLowerCase();
+function titleCase(s) {
+    return s.replace(/^(.)(.+)/, (match, car, cdr) => `${car}${cdr.toLowerCase()}`);
+}
+const Lu = (0, xregexp_1.default)('\\p{Lu}');
+const Ll = (0, xregexp_1.default)('\\p{Ll}');
+const connectedWord = (0, xregexp_1.default)('(^|-)\\p{Lu}\\p{Ll}*(?=-|$)', 'g');
+const connectedInnerWord = (0, xregexp_1.default)('-\\p{Lu}\\p{Ll}*(?=-|$)', 'g');
+const strayCC = new RegExp(`^(${unicode2latex_1.combining.regex})`);
+function wordSC(token, succ, allCaps, subSentence) {
+    if (token.type !== 'word')
+        return token.text;
+    if (token.text.match(/^I'/))
+        return titleCase(token.text);
+    if (subSentence && token.subSentenceStart && token.text.match(/^a$/i))
+        return 'a';
+    if ((subSentence && token.subSentenceStart) || token.sentenceStart) {
+        return allCaps ? titleCase(token.text) : xregexp_1.default.replace(token.text, connectedInnerWord, match => match.toLowerCase());
+    }
+    if (!allCaps && token.shape.match(/^[Xxd]+(-[Xxd]+)+/))
+        return xregexp_1.default.replace(token.text, connectedWord, match => match.toLowerCase());
+    if (token.text.match(/^[B-Z]$/))
+        return token.text;
+    if (!allCaps && token.shape.match(/^[-X]+$/))
+        return token.text;
+    if (succ && `${token.text} ${succ.text}`.match(preposition.complex)) {
+        succ.text = succ.text.toLowerCase();
+        return token.text.toLowerCase();
+    }
+    if (token.text.match(preposition.simple))
+        return token.text.toLowerCase();
+    const shape = token.shape.replace(/[^Xxd]/g, '');
+    if (!allCaps && shape.match(/^[Xd-]+$/))
+        return token.text;
+    if (shape.match(/^X[xd]*(-[Xxd]*)*$/))
+        return token.text.toLowerCase();
+    if (shape.match(/^[Xd]+$/))
+        return allCaps ? token.text.toLowerCase() : token.text;
+    if (token.text.includes('.'))
+        return token.text;
+    if (shape.match(/x.*X/))
+        return token.text;
+    return token.text.toLowerCase();
+}
+function tokentype(token) {
+    if (token.type === 'word' && token.text.match(/^\u2060+$/))
+        return 'm';
+    if (token.type === 'word')
+        return 'w';
+    if (token.shape === '-')
+        return '-';
+    return ' ';
 }
-function toSentenceCase(sentence, preserveQuoted = false) {
-    const allcaps = sentence === sentence.toUpperCase();
-    const preserve = [];
-    sentence.replace(re.subSentenceStart, (match, end, markup, char, i) => {
-        if (!sentence.substring(0, i + 1).match(re.acronym)) {
-            preserve.push({ pos: i + end.length + ((markup === null || markup === void 0 ? void 0 : markup.length) || 0), text: char, description: 'sub-sentence-start' });
+function prepost(s, offset) {
+    const tokens = [];
+    for (const text of s.match(/(\s+|\S+)/g)) {
+        if (text.match(/\s/)) {
+            tokens.push({ type: 'whitespace', text, shape: text });
         }
-        return '';
-    });
-    sentence.replace(re.sentenceStart, (match, markup, char) => {
-        preserve.push({ pos: ((markup === null || markup === void 0 ? void 0 : markup.length) || 0), text: char, description: 'sentence-start' });
-        return '';
-    });
-    sentence.replace(/<span class="nocase">.*?<\/span>|<nc>.*?<\/nc>/gi, (text, pos) => {
-        preserve.push({ pos, text, description: 'nocase' });
-        return '';
-    });
-    let masked = sentence.replace(/<[^>]+>/g, (text, pos) => {
-        preserve.push({ pos, text, description: 'markup' });
-        return '\uFFFD'.repeat(text.length);
-    });
-    // last because we're potentially modifying the original
-    for (const q of [/(“)(.*?)”/g, /(‘)(.*?)’/g, /(["])(.*?)\1/g]) {
-        sentence.replace(q, (text, quote, quoted, pos) => {
-            preserve.push({
-                pos: pos + (preserveQuoted ? 0 : quote.length),
-                text: preserveQuoted ? text : toSentenceCase(quoted, preserveQuoted),
-                description: `quoted with ${q}`,
+        else if (text) {
+            tokens.push({ type: 'punctuation', text, shape: text, start: offset, end: offset + text.length - 1 });
+        }
+        offset += text.length;
+    }
+    return tokens;
+}
+function tokenize(title, markup) {
+    if (markup)
+        title = title.replace(markup, match => '\u2060'.repeat(match.length));
+    const doc = (0, one_1.default)(title);
+    const tokens = [];
+    for (const sentence of doc.json({ offset: true })) {
+        const words = [];
+        let sentenceStart = true;
+        let subSentenceStart = false;
+        for (const term of sentence.terms) {
+            const m = term.post.match(strayCC);
+            if (m) {
+                term.text += m[0];
+                term.post = term.post.substring(m[0].length);
+                term.offset.length += m[0].length;
+            }
+            if (term.pre)
+                words.push(...prepost(term.pre, term.offset.start - term.pre.length));
+            words.push({
+                start: term.offset.start,
+                end: term.offset.start + term.offset.length - 1,
+                text: term.text,
+                type: 'word',
+                shape: xregexp_1.default.replaceEach(term.text, [[Lu, 'X', 'all'], [Ll, 'x', 'all'], [/\d/g, 'd']]),
+                sentenceStart,
+                subSentenceStart,
             });
-            return '';
-        });
+            sentenceStart = false;
+            subSentenceStart = term.post.includes(':');
+            if (term.post)
+                words.push(...prepost(term.post, term.offset.start + term.offset.length));
+        }
+        subSentenceStart = true;
+        while (words.length) {
+            if (words[0].type === 'word' && words[0].text.includes('/')) {
+                const multiple = words.shift();
+                let start = multiple.start;
+                words.unshift(...multiple.text.match(/([/]|[^/]+)/g).map((w) => {
+                    const started = start;
+                    start += w.length;
+                    const ss = w === '/' ? [false, false] : [multiple.sentenceStart, multiple.subSentenceStart];
+                    if (w !== '/')
+                        multiple.sentenceStart = multiple.subSentenceStart = false;
+                    return {
+                        start: started,
+                        end: start - 1,
+                        text: w,
+                        type: w === '/' ? 'punctuation' : 'word',
+                        shape: xregexp_1.default.replaceEach(w, [[Lu, 'X', 'all'], [Ll, 'x', 'all'], [/\d/g, 'd']]),
+                        sentenceStart: ss[0],
+                        subSentenceStart: ss[1],
+                    };
+                }));
+                continue;
+            }
+            const type = words.map(tokentype).join('');
+            let m;
+            if (m = type.match(/^m?[wc]m?(-m?[wc]m?)+/)) {
+                tokens.push(gather(words.splice(0, m[0].length)));
+                continue;
+            }
+            const word = words.shift();
+            if (word.type !== 'whitespace')
+                tokens.push(word);
+        }
     }
-    masked = masked
-        .replace(/[;:]\uFFFD*\s+\uFFFD*A\s/g, match => match.toLowerCase())
-        .replace(/[–—]\uFFFD*\s*\uFFFD*A\s/g, match => match.toLowerCase())
-        .replace(re.words, word => lowercase(word, allcaps));
-    sentence = masked;
-    for (const { pos, text } of preserve) {
-        sentence = sentence.substring(0, pos) + text + sentence.substring(pos + text.length);
+    return tokens;
+}
+exports.tokenize = tokenize;
+function toSentenceCase(title, options = {}) {
+    options = Object.assign({ preserveQuoted: true, subSentenceCapitalization: true }, options);
+    title = title.normalize('NFC'); // https://github.com/winkjs/wink-nlp/issues/134
+    const allCaps = title === title.toUpperCase();
+    if (allCaps && !title.match(/\s/))
+        return title;
+    let sentenceCased = title;
+    const tokens = tokenize(title, options.markup);
+    tokens.forEach((token, i) => {
+        if (token.type !== 'whitespace') {
+            sentenceCased = sentenceCased.substring(0, token.start) + wordSC(token, tokens[i + 1], allCaps, options.subSentenceCapitalization) + sentenceCased.substring(token.end + 1);
+        }
+    });
+    if (options.markup) {
+        for (const match of title.matchAll(options.markup)) {
+            sentenceCased = sentenceCased.substring(0, match.index) + match[0] + sentenceCased.substring(match.index + match[0].length);
+        }
+    }
+    if (options.nocase) {
+        sentenceCased = sentenceCased
+            .replace(options.nocase, (match, tag, offset) => title.substring(offset, offset + match.length));
     }
-    return sentence;
+    if (options.preserveQuoted) {
+        for (const q of [/“.*?”/g, /‘.*?’/g, /".*?"/g]) {
+            sentenceCased = sentenceCased.replace(q, (match, offset) => title.substring(offset, offset + match.length));
+        }
+    }
+    return sentenceCased;
 }
 exports.toSentenceCase = toSentenceCase;
+function guessSentenceCased(title, markup = /<\/?(?:i|b|sup|sub|ncx?)>/g) {
+    const noMarkup = title.replace(markup, '');
+    if (noMarkup === noMarkup.toUpperCase())
+        return false;
+    if (noMarkup === noMarkup.toLowerCase())
+        return false;
+    const words = tokenize(title, markup).filter(token => token.type === 'word' && token.shape.match(/x/i));
+    if (!words.length)
+        return true;
+    const titleCased = words.filter(word => word.shape.match(/^X+$|^X.*x|[^Xx]+-X/));
+    return (titleCased.length / words.length) < 0.5; // eslint-disable-line no-magic-numbers
+}
+exports.guessSentenceCased = guessSentenceCased;
 //# sourceMappingURL=sentence-case.js.map