@chaffjs/lang-en 0.35.0 → 0.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/citation.d.ts.map +1 -1
- package/dist/citation.js +8 -2
- package/dist/citation.js.map +1 -1
- package/dist/names-read-as-verbs.d.ts +21 -0
- package/dist/names-read-as-verbs.d.ts.map +1 -0
- package/dist/names-read-as-verbs.js +40 -0
- package/dist/names-read-as-verbs.js.map +1 -0
- package/dist/pos.d.ts.map +1 -1
- package/dist/pos.js +30 -1
- package/dist/pos.js.map +1 -1
- package/dist/structure.d.ts.map +1 -1
- package/dist/structure.js +13 -0
- package/dist/structure.js.map +1 -1
- package/lexicons/ai-tell.yaml +55 -27
- package/lexicons/announcing-opener.yaml +7 -0
- package/lexicons/article-sound.yaml +1 -0
- package/lexicons/citation-series.yaml +1 -0
- package/lexicons/clause-conjunction.yaml +7 -0
- package/lexicons/closing-cliche.yaml +11 -0
- package/lexicons/closing-exhortation.yaml +11 -0
- package/lexicons/cold-open.yaml +30 -0
- package/lexicons/common-acronym.yaml +15 -0
- package/lexicons/comparative-baseline.yaml +14 -0
- package/lexicons/condition-heading.yaml +21 -0
- package/lexicons/connective-opener.yaml +23 -0
- package/lexicons/contraction-head.yaml +10 -0
- package/lexicons/contrast-amount.yaml +13 -0
- package/lexicons/contrast-lead.yaml +38 -0
- package/lexicons/contrast-turn.yaml +5 -0
- package/lexicons/drug-stem.yaml +48 -0
- package/lexicons/elision-mark.yaml +9 -0
- package/lexicons/email-closing.yaml +7 -1
- package/lexicons/email-salutation-end.yaml +7 -0
- package/lexicons/email-stock-courtesy.yaml +29 -0
- package/lexicons/everyday-name.yaml +24 -0
- package/lexicons/extra-word.yaml +15 -1
- package/lexicons/honesty-marker.yaml +14 -0
- package/lexicons/honorific.yaml +3 -0
- package/lexicons/indefinite-article.yaml +8 -0
- package/lexicons/intent-phrase.yaml +21 -0
- package/lexicons/interviewer-mark.yaml +10 -0
- package/lexicons/list-continues.yaml +9 -0
- package/lexicons/name-capitals-prefix.yaml +16 -0
- package/lexicons/name-direction-word.yaml +10 -0
- package/lexicons/name-title-head.yaml +23 -0
- package/lexicons/name-word-ending.yaml +10 -0
- package/lexicons/navigation-line.yaml +11 -0
- package/lexicons/open-list-end.yaml +10 -0
- package/lexicons/option-choice-label.yaml +8 -0
- package/lexicons/over-general-claim.yaml +57 -6
- package/lexicons/padded-intro.yaml +14 -0
- package/lexicons/product-name.yaml +9 -0
- package/lexicons/prohibition-opener.yaml +6 -0
- package/lexicons/quote-not-attribution.yaml +13 -0
- package/lexicons/request-deadline-section.yaml +1 -0
- package/lexicons/sql-keyword.yaml +28 -0
- package/lexicons/superlative-goal.yaml +40 -0
- package/lexicons/superlative-part.yaml +22 -0
- package/lexicons/superlative-phrase.yaml +8 -0
- package/lexicons/superlative-purpose.yaml +7 -0
- package/lexicons/superlative-referent.yaml +15 -0
- package/lexicons/superlative-scope.yaml +5 -1
- package/lexicons/template-heading.yaml +35 -0
- package/lexicons/text-message-verb.yaml +11 -0
- package/lexicons/uninflected-word.yaml +22 -0
- package/lexicons/unprompted-denial.yaml +113 -0
- package/lexicons/verbatim-lead.yaml +11 -0
- package/lexicons/word-suffix.yaml +29 -1
- package/package.json +1 -1
- package/src/citation.ts +9 -2
- package/src/names-read-as-verbs.ts +65 -0
- package/src/pos.ts +33 -1
- package/src/structure.ts +14 -0
package/src/pos.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { capitalisedGuessedVerbsAsNames, isOrdinaryWord, lowercasedAt, properNou
|
|
|
6
6
|
import { isStativeParticiple, stativeVocabulary } from "./stative-participle.ts";
|
|
7
7
|
import { isCapitalisedOrdinaryWord, isEmphasisedAdverb, isEmphasisedWord, isHyphenated, standsAlone } from "./emphasis.ts";
|
|
8
8
|
import { gerundFeatures } from "./gerund.ts";
|
|
9
|
+
import { namesReadAsVerbs, rereadAsNames, type NameReadings } from "./names-read-as-verbs.ts";
|
|
9
10
|
import { winkLexiconWords, winkPosTagger, winkWordNet } from "./wink-modules.ts";
|
|
10
11
|
|
|
11
12
|
/** wink は CommonJS で、辞書を同期に持つ。初期化は 130 ms ほど。 */
|
|
@@ -164,6 +165,12 @@ const OBJECT_OR_POSSESSIVE = wordsOf("object-or-possessive");
|
|
|
164
165
|
|
|
165
166
|
const POSSESSIVE_TAG = new Set(["PRP$", "WP$"]);
|
|
166
167
|
|
|
168
|
+
const NAME_READINGS: NameReadings = {
|
|
169
|
+
contractionHeads: wordsOf("contraction-head"),
|
|
170
|
+
opensNounPhrase: (word) =>
|
|
171
|
+
ARTICLES.has(word.value.toLowerCase()) || word.pos === "POS" || (word.pos === "PRP$" && !OBJECT_OR_POSSESSIVE.has(word.value.toLowerCase())),
|
|
172
|
+
};
|
|
173
|
+
|
|
167
174
|
type Features = { features?: Readonly<Record<string, string>> };
|
|
168
175
|
|
|
169
176
|
/** 冠詞(PronType=Art)と所有の語(Poss=Yes)は、名詞の前に一つしか立たない。二つ並ぶ our the platform は書き損じ。 */
|
|
@@ -306,10 +313,35 @@ const withSentenceInitialCase = (tagger: Tagger, text: string, entries: readonly
|
|
|
306
313
|
return lowered === undefined ? entries : rereadAt(entries, at, tagged(tagger, lowered));
|
|
307
314
|
};
|
|
308
315
|
|
|
316
|
+
/** A word the tagger's vocabulary does not hold. The tagger reads such capitals as a name, as it reads UN. */
|
|
317
|
+
const UNKNOWN_NAME = "XQZ";
|
|
318
|
+
|
|
319
|
+
/** text with each word at positions replaced by UNKNOWN_NAME, found in order as locate finds them. */
|
|
320
|
+
const withUnknownNamesAt = (text: string, entries: readonly Tagged[], positions: readonly number[]): string => {
|
|
321
|
+
const parts: string[] = [];
|
|
322
|
+
let cursor = 0;
|
|
323
|
+
entries.forEach((entry, at) => {
|
|
324
|
+
const start = text.indexOf(entry.value, cursor);
|
|
325
|
+
if (start === -1) return;
|
|
326
|
+
parts.push(text.slice(cursor, start), positions.includes(at) ? UNKNOWN_NAME : entry.value);
|
|
327
|
+
cursor = start + entry.value.length;
|
|
328
|
+
});
|
|
329
|
+
parts.push(text.slice(cursor));
|
|
330
|
+
return parts.join("");
|
|
331
|
+
};
|
|
332
|
+
|
|
333
|
+
/** Names the tagger read as a verb or a pronoun (AI, the US): the sentence is read again with a name the tagger does not know in their place. */
|
|
334
|
+
const withNamesReread = (tagger: Tagger, text: string, entries: readonly Tagged[]): readonly Tagged[] => {
|
|
335
|
+
const positions = namesReadAsVerbs(entries, NAME_READINGS);
|
|
336
|
+
if (positions.length === 0) return entries;
|
|
337
|
+
return rereadAsNames(entries, tagged(tagger, withUnknownNamesAt(text, entries, positions)), positions, UNKNOWN_NAME);
|
|
338
|
+
};
|
|
339
|
+
|
|
309
340
|
export const tokenize = (text: string): Token[] | undefined => {
|
|
310
341
|
const tagger = state.ready;
|
|
311
342
|
if (tagger === undefined) return undefined;
|
|
312
343
|
// 語は解析させた字(don't)で返す。語彙表の語と同じ字で比べられる。span は本文を指したまま。
|
|
313
344
|
const words = straightApostrophes(blankLongRuns(text, RUN_LIMIT));
|
|
314
|
-
|
|
345
|
+
const read = withNamesReread(tagger, words, tagged(tagger, words));
|
|
346
|
+
return locate(words, capitalisedGuessedVerbsAsNames(withSentenceInitialCase(tagger, words, read), state.vocabulary, state.senses));
|
|
315
347
|
};
|
package/src/structure.ts
CHANGED
|
@@ -29,9 +29,23 @@ const numberOf = (text: string | undefined): string | undefined => {
|
|
|
29
29
|
const titleOf = (rest: string): string | undefined => {
|
|
30
30
|
const trimmed = rest.replace(/^[.:\-–—\s]+/u, "").trim();
|
|
31
31
|
if (trimmed !== "" && /^\p{Ll}/u.test(trimmed)) return undefined;
|
|
32
|
+
if (citesInSentence(rest)) return undefined;
|
|
32
33
|
return trimmed;
|
|
33
34
|
};
|
|
34
35
|
|
|
36
|
+
/** "(c) is brief", "(a) of the Act, 50 U.S.C.", "(15 USCA § 46), among other things": subdivisions or a bracketed citation. */
|
|
37
|
+
const LEADING_BRACKETS = /^(?:\s*\([^()]{1,40}\))+/u;
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* An opinion's paragraph or footnote that opens with a citation ("§ 331, 7 U.S.C. § 1331.", "Section 9(c) is brief…")
|
|
41
|
+
* cites another Act's section: a comma, or a lowercase word, right after the number and its bracketed parts. A heading's
|
|
42
|
+
* title follows its number without one ("Section 5 (Reserved).", "§ 3(a) Definitions").
|
|
43
|
+
*/
|
|
44
|
+
const citesInSentence = (rest: string): boolean => {
|
|
45
|
+
const after = rest.replace(LEADING_BRACKETS, "").trimStart();
|
|
46
|
+
return after.startsWith(",") || (after !== rest.trimStart() && /^\p{Ll}/u.test(after));
|
|
47
|
+
};
|
|
48
|
+
|
|
35
49
|
const ARTICLE = /^\s{0,3}(?:ARTICLE|Article)\s+(?<n>\d{1,3}|[IVXLC]{1,7})\b(?<rest>.*)$/u;
|
|
36
50
|
const SECTION = /^\s{0,3}(?:SECTION|Section|§)\s*(?<n>\d{1,3}(?:\.\d{1,3}){0,5})\b(?<rest>.*)$/u;
|
|
37
51
|
/**
|