@chaffjs/lang-en 0.35.0 → 0.37.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/dist/citation.d.ts.map +1 -1
  2. package/dist/citation.js +8 -2
  3. package/dist/citation.js.map +1 -1
  4. package/dist/names-read-as-verbs.d.ts +21 -0
  5. package/dist/names-read-as-verbs.d.ts.map +1 -0
  6. package/dist/names-read-as-verbs.js +40 -0
  7. package/dist/names-read-as-verbs.js.map +1 -0
  8. package/dist/pos.d.ts.map +1 -1
  9. package/dist/pos.js +30 -1
  10. package/dist/pos.js.map +1 -1
  11. package/dist/structure.d.ts.map +1 -1
  12. package/dist/structure.js +13 -0
  13. package/dist/structure.js.map +1 -1
  14. package/lexicons/ai-tell.yaml +55 -27
  15. package/lexicons/announcing-opener.yaml +7 -0
  16. package/lexicons/article-sound.yaml +1 -0
  17. package/lexicons/citation-series.yaml +1 -0
  18. package/lexicons/clause-conjunction.yaml +7 -0
  19. package/lexicons/closing-cliche.yaml +11 -0
  20. package/lexicons/closing-exhortation.yaml +11 -0
  21. package/lexicons/cold-open.yaml +30 -0
  22. package/lexicons/common-acronym.yaml +15 -0
  23. package/lexicons/comparative-baseline.yaml +14 -0
  24. package/lexicons/condition-heading.yaml +21 -0
  25. package/lexicons/connective-opener.yaml +23 -0
  26. package/lexicons/contraction-head.yaml +10 -0
  27. package/lexicons/contrast-amount.yaml +13 -0
  28. package/lexicons/contrast-lead.yaml +38 -0
  29. package/lexicons/contrast-turn.yaml +5 -0
  30. package/lexicons/drug-stem.yaml +48 -0
  31. package/lexicons/elision-mark.yaml +9 -0
  32. package/lexicons/email-closing.yaml +7 -1
  33. package/lexicons/email-salutation-end.yaml +7 -0
  34. package/lexicons/email-stock-courtesy.yaml +29 -0
  35. package/lexicons/everyday-name.yaml +24 -0
  36. package/lexicons/extra-word.yaml +15 -1
  37. package/lexicons/honesty-marker.yaml +14 -0
  38. package/lexicons/honorific.yaml +3 -0
  39. package/lexicons/indefinite-article.yaml +8 -0
  40. package/lexicons/intent-phrase.yaml +21 -0
  41. package/lexicons/interviewer-mark.yaml +10 -0
  42. package/lexicons/list-continues.yaml +9 -0
  43. package/lexicons/name-capitals-prefix.yaml +16 -0
  44. package/lexicons/name-direction-word.yaml +10 -0
  45. package/lexicons/name-title-head.yaml +23 -0
  46. package/lexicons/name-word-ending.yaml +10 -0
  47. package/lexicons/navigation-line.yaml +11 -0
  48. package/lexicons/open-list-end.yaml +10 -0
  49. package/lexicons/option-choice-label.yaml +8 -0
  50. package/lexicons/over-general-claim.yaml +57 -6
  51. package/lexicons/padded-intro.yaml +14 -0
  52. package/lexicons/product-name.yaml +9 -0
  53. package/lexicons/prohibition-opener.yaml +6 -0
  54. package/lexicons/quote-not-attribution.yaml +13 -0
  55. package/lexicons/request-deadline-section.yaml +1 -0
  56. package/lexicons/sql-keyword.yaml +28 -0
  57. package/lexicons/superlative-goal.yaml +40 -0
  58. package/lexicons/superlative-part.yaml +22 -0
  59. package/lexicons/superlative-phrase.yaml +8 -0
  60. package/lexicons/superlative-purpose.yaml +7 -0
  61. package/lexicons/superlative-referent.yaml +15 -0
  62. package/lexicons/superlative-scope.yaml +5 -1
  63. package/lexicons/template-heading.yaml +35 -0
  64. package/lexicons/text-message-verb.yaml +11 -0
  65. package/lexicons/uninflected-word.yaml +22 -0
  66. package/lexicons/unprompted-denial.yaml +113 -0
  67. package/lexicons/verbatim-lead.yaml +11 -0
  68. package/lexicons/word-suffix.yaml +29 -1
  69. package/package.json +1 -1
  70. package/src/citation.ts +9 -2
  71. package/src/names-read-as-verbs.ts +65 -0
  72. package/src/pos.ts +33 -1
  73. package/src/structure.ts +14 -0
package/src/pos.ts CHANGED
@@ -6,6 +6,7 @@ import { capitalisedGuessedVerbsAsNames, isOrdinaryWord, lowercasedAt, properNou
6
6
  import { isStativeParticiple, stativeVocabulary } from "./stative-participle.ts";
7
7
  import { isCapitalisedOrdinaryWord, isEmphasisedAdverb, isEmphasisedWord, isHyphenated, standsAlone } from "./emphasis.ts";
8
8
  import { gerundFeatures } from "./gerund.ts";
9
+ import { namesReadAsVerbs, rereadAsNames, type NameReadings } from "./names-read-as-verbs.ts";
9
10
  import { winkLexiconWords, winkPosTagger, winkWordNet } from "./wink-modules.ts";
10
11
 
11
12
  /** wink は CommonJS で、辞書を同期に持つ。初期化は 130 ms ほど。 */
@@ -164,6 +165,12 @@ const OBJECT_OR_POSSESSIVE = wordsOf("object-or-possessive");
164
165
 
165
166
  const POSSESSIVE_TAG = new Set(["PRP$", "WP$"]);
166
167
 
168
+ const NAME_READINGS: NameReadings = {
169
+ contractionHeads: wordsOf("contraction-head"),
170
+ opensNounPhrase: (word) =>
171
+ ARTICLES.has(word.value.toLowerCase()) || word.pos === "POS" || (word.pos === "PRP$" && !OBJECT_OR_POSSESSIVE.has(word.value.toLowerCase())),
172
+ };
173
+
167
174
  type Features = { features?: Readonly<Record<string, string>> };
168
175
 
169
176
  /** 冠詞(PronType=Art)と所有の語(Poss=Yes)は、名詞の前に一つしか立たない。二つ並ぶ our the platform は書き損じ。 */
@@ -306,10 +313,35 @@ const withSentenceInitialCase = (tagger: Tagger, text: string, entries: readonly
306
313
  return lowered === undefined ? entries : rereadAt(entries, at, tagged(tagger, lowered));
307
314
  };
308
315
 
316
+ /** A word the tagger's vocabulary does not hold. The tagger reads such capitals as a name, as it reads UN. */
317
+ const UNKNOWN_NAME = "XQZ";
318
+
319
+ /** text with each word at positions replaced by UNKNOWN_NAME, found in order as locate finds them. */
320
+ const withUnknownNamesAt = (text: string, entries: readonly Tagged[], positions: readonly number[]): string => {
321
+ const parts: string[] = [];
322
+ let cursor = 0;
323
+ entries.forEach((entry, at) => {
324
+ const start = text.indexOf(entry.value, cursor);
325
+ if (start === -1) return;
326
+ parts.push(text.slice(cursor, start), positions.includes(at) ? UNKNOWN_NAME : entry.value);
327
+ cursor = start + entry.value.length;
328
+ });
329
+ parts.push(text.slice(cursor));
330
+ return parts.join("");
331
+ };
332
+
333
+ /** Names the tagger read as a verb or a pronoun (AI, the US): the sentence is read again with a name the tagger does not know in their place. */
334
+ const withNamesReread = (tagger: Tagger, text: string, entries: readonly Tagged[]): readonly Tagged[] => {
335
+ const positions = namesReadAsVerbs(entries, NAME_READINGS);
336
+ if (positions.length === 0) return entries;
337
+ return rereadAsNames(entries, tagged(tagger, withUnknownNamesAt(text, entries, positions)), positions, UNKNOWN_NAME);
338
+ };
339
+
309
340
  export const tokenize = (text: string): Token[] | undefined => {
310
341
  const tagger = state.ready;
311
342
  if (tagger === undefined) return undefined;
312
343
  // 語は解析させた字(don't)で返す。語彙表の語と同じ字で比べられる。span は本文を指したまま。
313
344
  const words = straightApostrophes(blankLongRuns(text, RUN_LIMIT));
314
- return locate(words, capitalisedGuessedVerbsAsNames(withSentenceInitialCase(tagger, words, tagged(tagger, words)), state.vocabulary, state.senses));
345
+ const read = withNamesReread(tagger, words, tagged(tagger, words));
346
+ return locate(words, capitalisedGuessedVerbsAsNames(withSentenceInitialCase(tagger, words, read), state.vocabulary, state.senses));
315
347
  };
package/src/structure.ts CHANGED
@@ -29,9 +29,23 @@ const numberOf = (text: string | undefined): string | undefined => {
29
29
  const titleOf = (rest: string): string | undefined => {
30
30
  const trimmed = rest.replace(/^[.:\-–—\s]+/u, "").trim();
31
31
  if (trimmed !== "" && /^\p{Ll}/u.test(trimmed)) return undefined;
32
+ if (citesInSentence(rest)) return undefined;
32
33
  return trimmed;
33
34
  };
34
35
 
36
+ /** "(c) is brief", "(a) of the Act, 50 U.S.C.", "(15 USCA § 46), among other things": subdivisions or a bracketed citation. */
37
+ const LEADING_BRACKETS = /^(?:\s*\([^()]{1,40}\))+/u;
38
+
39
+ /**
40
+ * An opinion's paragraph or footnote that opens with a citation ("§ 331, 7 U.S.C. § 1331.", "Section 9(c) is brief…")
41
+ * cites another Act's section: a comma, or a lowercase word, right after the number and its bracketed parts. A heading's
42
+ * title follows its number without one ("Section 5 (Reserved).", "§ 3(a) Definitions").
43
+ */
44
+ const citesInSentence = (rest: string): boolean => {
45
+ const after = rest.replace(LEADING_BRACKETS, "").trimStart();
46
+ return after.startsWith(",") || (after !== rest.trimStart() && /^\p{Ll}/u.test(after));
47
+ };
48
+
35
49
  const ARTICLE = /^\s{0,3}(?:ARTICLE|Article)\s+(?<n>\d{1,3}|[IVXLC]{1,7})\b(?<rest>.*)$/u;
36
50
  const SECTION = /^\s{0,3}(?:SECTION|Section|§)\s*(?<n>\d{1,3}(?:\.\d{1,3}){0,5})\b(?<rest>.*)$/u;
37
51
  /**