@chaffjs/lang-en 0.25.0 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/definitions.d.ts.map +1 -1
- package/dist/definitions.js +15 -5
- package/dist/definitions.js.map +1 -1
- package/dist/emphasis.d.ts +14 -1
- package/dist/emphasis.d.ts.map +1 -1
- package/dist/emphasis.js +41 -1
- package/dist/emphasis.js.map +1 -1
- package/dist/example-name.d.ts +7 -0
- package/dist/example-name.d.ts.map +1 -0
- package/dist/example-name.js +33 -0
- package/dist/example-name.js.map +1 -0
- package/dist/pos.d.ts.map +1 -1
- package/dist/pos.js +13 -4
- package/dist/pos.js.map +1 -1
- package/dist/structure.d.ts.map +1 -1
- package/dist/structure.js +19 -1
- package/dist/structure.js.map +1 -1
- package/lexicons/action-due-label.yaml +8 -0
- package/lexicons/action-section-heading.yaml +9 -0
- package/lexicons/amount-bound-label.yaml +14 -0
- package/lexicons/amount-change-word.yaml +33 -0
- package/lexicons/amount-range-word.yaml +6 -0
- package/lexicons/amount-short-scale.yaml +9 -0
- package/lexicons/article-sound.yaml +3 -0
- package/lexicons/clock-range-joiner.yaml +12 -0
- package/lexicons/code-global.yaml +27 -0
- package/lexicons/common-acronym.yaml +19 -0
- package/lexicons/company-form.yaml +14 -11
- package/lexicons/count-counter.yaml +5 -0
- package/lexicons/count-hedge.yaml +32 -1
- package/lexicons/count-member.yaml +25 -0
- package/lexicons/date-change-word.yaml +5 -0
- package/lexicons/day-shift-mark.yaml +12 -0
- package/lexicons/deadline-passed-word.yaml +10 -0
- package/lexicons/default-header.yaml +7 -0
- package/lexicons/default-phrase.yaml +8 -0
- package/lexicons/deprecation-negated.yaml +10 -0
- package/lexicons/deprecation-word.yaml +8 -0
- package/lexicons/doubled-nado.yaml +15 -0
- package/lexicons/due-date-label.yaml +10 -0
- package/lexicons/duty-marker.yaml +9 -0
- package/lexicons/duty-negation.yaml +7 -0
- package/lexicons/duty-terms-elsewhere.yaml +10 -0
- package/lexicons/earlier-section-word.yaml +8 -0
- package/lexicons/extra-word.yaml +5 -0
- package/lexicons/fact-change-from.yaml +6 -0
- package/lexicons/fact-change-not.yaml +13 -0
- package/lexicons/fact-change-to.yaml +6 -0
- package/lexicons/fact-condition-from.yaml +10 -0
- package/lexicons/fact-condition-to.yaml +9 -0
- package/lexicons/fact-event.yaml +14 -0
- package/lexicons/fact-label-lead.yaml +9 -0
- package/lexicons/fact-name-joiner.yaml +6 -0
- package/lexicons/fact-separator.yaml +5 -0
- package/lexicons/install-command.yaml +20 -0
- package/lexicons/issue-date-label.yaml +9 -0
- package/lexicons/item-done-word.yaml +9 -0
- package/lexicons/leg-column.yaml +15 -0
- package/lexicons/leg-mark.yaml +25 -0
- package/lexicons/line-amount-column.yaml +8 -0
- package/lexicons/local-time-mark.yaml +7 -0
- package/lexicons/meeting-date-label.yaml +8 -0
- package/lexicons/minutes-title-word.yaml +7 -0
- package/lexicons/nado-closed-list.yaml +22 -0
- package/lexicons/name-joiner.yaml +9 -0
- package/lexicons/name-label.yaml +10 -0
- package/lexicons/name-quote.yaml +7 -0
- package/lexicons/name-stem-suffix.yaml +12 -0
- package/lexicons/name-title-joiner.yaml +9 -0
- package/lexicons/number-word.yaml +45 -0
- package/lexicons/ordinal-frame.yaml +15 -0
- package/lexicons/parameter-header.yaml +11 -0
- package/lexicons/partial-listing-marker.yaml +15 -0
- package/lexicons/per-unit-mark.yaml +25 -0
- package/lexicons/period-aside.yaml +46 -0
- package/lexicons/period-deadline.yaml +19 -0
- package/lexicons/period-label.yaml +19 -0
- package/lexicons/period-overview-heading.yaml +10 -0
- package/lexicons/period-term-label.yaml +12 -0
- package/lexicons/person-not-after.yaml +42 -1
- package/lexicons/phone-label.yaml +14 -0
- package/lexicons/place-name-joiner.yaml +10 -0
- package/lexicons/place-word.yaml +29 -0
- package/lexicons/prerequisite-opener.yaml +10 -0
- package/lexicons/proper-name-word.yaml +55 -0
- package/lexicons/quantity-column.yaml +9 -0
- package/lexicons/rate-unit.yaml +18 -0
- package/lexicons/requirement-heading.yaml +22 -0
- package/lexicons/requirement-item-ending.yaml +6 -0
- package/lexicons/requirement-length-unit.yaml +10 -0
- package/lexicons/requirement-length-word.yaml +14 -0
- package/lexicons/return-clause.yaml +16 -0
- package/lexicons/return-item.yaml +7 -0
- package/lexicons/return-label.yaml +10 -0
- package/lexicons/return-type-word.yaml +16 -0
- package/lexicons/return-verb.yaml +6 -0
- package/lexicons/session-count-unit.yaml +10 -0
- package/lexicons/session-total.yaml +11 -0
- package/lexicons/share-column-rate.yaml +15 -0
- package/lexicons/share-column.yaml +12 -0
- package/lexicons/share-rest.yaml +8 -0
- package/lexicons/stated-deadline.yaml +17 -0
- package/lexicons/stay-check-in-column.yaml +10 -0
- package/lexicons/stay-check-out-column.yaml +10 -0
- package/lexicons/stay-night.yaml +6 -0
- package/lexicons/stay-nights-column.yaml +7 -0
- package/lexicons/stay-pair-joiner.yaml +9 -0
- package/lexicons/summed-counter.yaml +17 -0
- package/lexicons/tax-included-word.yaml +8 -0
- package/lexicons/tax-label.yaml +9 -0
- package/lexicons/timed-duty.yaml +14 -0
- package/lexicons/travel-time-word.yaml +38 -0
- package/lexicons/unit-price-column.yaml +9 -0
- package/lexicons/unnamed-speaker.yaml +58 -0
- package/lexicons/us-state-code.yaml +1 -1
- package/lexicons/very-copula.yaml +5 -0
- package/lexicons/warning-action.yaml +20 -0
- package/lexicons/warning-label.yaml +8 -0
- package/lexicons/word-suffix.yaml +3 -0
- package/lexicons/working-hours-break.yaml +10 -0
- package/lexicons/working-hours-total.yaml +11 -0
- package/package.json +1 -1
- package/src/definitions.ts +13 -5
- package/src/emphasis.ts +53 -2
- package/src/example-name.ts +36 -0
- package/src/pos.ts +11 -5
- package/src/structure.ts +24 -1
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# Counters that a total line may add up ("16 credits", "4 sessions"). total-mismatch reads a number directly followed by one
|
|
2
|
+
# of these as a quantity in that unit. The singular and the plural share a group, so "1 credit" and "3 credits" add up.
|
|
3
|
+
id: summed-counter
|
|
4
|
+
language: en
|
|
5
|
+
entries:
|
|
6
|
+
- pattern: credits
|
|
7
|
+
group: credit
|
|
8
|
+
- pattern: credit
|
|
9
|
+
group: credit
|
|
10
|
+
- pattern: units
|
|
11
|
+
group: unit
|
|
12
|
+
- pattern: unit
|
|
13
|
+
group: unit
|
|
14
|
+
- pattern: sessions
|
|
15
|
+
group: session
|
|
16
|
+
- pattern: session
|
|
17
|
+
group: session
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# Words that mark a tax line. A list or table line starting with one and giving a rate (10%) is compared with that rate of the subtotal above it (tax-mismatch).
|
|
2
|
+
id: tax-label
|
|
3
|
+
language: en
|
|
4
|
+
entries:
|
|
5
|
+
- pattern: Sales tax
|
|
6
|
+
- pattern: Consumption tax
|
|
7
|
+
- pattern: Tax
|
|
8
|
+
- pattern: VAT
|
|
9
|
+
- pattern: GST
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# Acts a duty binds to a point in time (pay, deliver, return, notify). After a duty marker (shall, will) in a sentence that
|
|
2
|
+
# states no limit, nobody can say by when the act is due (obligation-without-deadline). "Indemnify" and "keep" are not here:
|
|
3
|
+
# they bind for as long as the contract runs, not at a time.
|
|
4
|
+
id: timed-duty
|
|
5
|
+
language: en
|
|
6
|
+
entries:
|
|
7
|
+
- pattern: pay
|
|
8
|
+
- pattern: remit
|
|
9
|
+
- pattern: refund
|
|
10
|
+
- pattern: invoice
|
|
11
|
+
- pattern: deliver
|
|
12
|
+
- pattern: submit
|
|
13
|
+
- pattern: return
|
|
14
|
+
- pattern: notify
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# The words that make a length of time a travel time (connection-time-short). Group travel names a move: a length in
|
|
2
|
+
# the same sentence ("25 minutes by Metro", "allow 45 minutes") is read as the time it takes. Group bound makes a length
|
|
3
|
+
# an upper limit ("within 10 minutes"), which is not used. position says on which side of the length a bound is written.
|
|
4
|
+
# Matched as whole words, case aside.
|
|
5
|
+
id: travel-time-word
|
|
6
|
+
language: en
|
|
7
|
+
entries:
|
|
8
|
+
- { pattern: "travel", group: travel }
|
|
9
|
+
- { pattern: "travel time", group: travel }
|
|
10
|
+
- { pattern: "transfer", group: travel }
|
|
11
|
+
- { pattern: "connection", group: travel }
|
|
12
|
+
- { pattern: "minimum connection time", group: travel }
|
|
13
|
+
- { pattern: "allow", group: travel }
|
|
14
|
+
- { pattern: "walk", group: travel }
|
|
15
|
+
- { pattern: "walking", group: travel }
|
|
16
|
+
- { pattern: "on foot", group: travel }
|
|
17
|
+
- { pattern: "drive", group: travel }
|
|
18
|
+
- { pattern: "driving", group: travel }
|
|
19
|
+
- { pattern: "by car", group: travel }
|
|
20
|
+
- { pattern: "by Metro", group: travel }
|
|
21
|
+
- { pattern: "by subway", group: travel }
|
|
22
|
+
- { pattern: "by underground", group: travel }
|
|
23
|
+
- { pattern: "by Tube", group: travel }
|
|
24
|
+
- { pattern: "by rail", group: travel }
|
|
25
|
+
- { pattern: "by train", group: travel }
|
|
26
|
+
- { pattern: "train", group: travel }
|
|
27
|
+
- { pattern: "coach", group: travel }
|
|
28
|
+
- { pattern: "bus", group: travel }
|
|
29
|
+
- { pattern: "shuttle", group: travel }
|
|
30
|
+
- { pattern: "taxi", group: travel }
|
|
31
|
+
- { pattern: "cab", group: travel }
|
|
32
|
+
- { pattern: "within", position: before, group: bound }
|
|
33
|
+
- { pattern: "up to", position: before, group: bound }
|
|
34
|
+
- { pattern: "under", position: before, group: bound }
|
|
35
|
+
- { pattern: "less than", position: before, group: bound }
|
|
36
|
+
- { pattern: "no more than", position: before, group: bound }
|
|
37
|
+
- { pattern: "at most", position: before, group: bound }
|
|
38
|
+
- { pattern: "or less", position: after, group: bound }
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# Headings of the unit price column of an itemised table (line-amount-mismatch).
|
|
2
|
+
id: unit-price-column
|
|
3
|
+
language: en
|
|
4
|
+
entries:
|
|
5
|
+
- pattern: Unit price
|
|
6
|
+
- pattern: Unit cost
|
|
7
|
+
- pattern: Price
|
|
8
|
+
# A rate is a price per a period of time, and the header does not say which (group: a unit no quantity word counts).
|
|
9
|
+
- { pattern: Rate, group: unstated }
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# Words that quote someone without a name (quote-unnamed-speaker). group says what each word does:
|
|
2
|
+
# role: a role written in place of a speaker's name. A role that is one person (the CEO of Acme) is not one.
|
|
3
|
+
# after-role: a word that may follow a role at the head of the speaker's phrase (an analyst at Acme, a source close to).
|
|
4
|
+
# A role followed by anything else (an official statement) is not the speaker.
|
|
5
|
+
# honorific: a title before a name (Ms Mori): the phrase names its speaker.
|
|
6
|
+
# cue: words that give a quotation to its speaker, besides quote-attribution's (told reporters, noted an analyst).
|
|
7
|
+
# anaphor: a word right before the role that points back at someone (the analyst, our spokesperson): not reported.
|
|
8
|
+
id: unnamed-speaker
|
|
9
|
+
language: en
|
|
10
|
+
entries:
|
|
11
|
+
- { pattern: analyst, group: role }
|
|
12
|
+
- { pattern: analysts, group: role }
|
|
13
|
+
- { pattern: expert, group: role }
|
|
14
|
+
- { pattern: experts, group: role }
|
|
15
|
+
- { pattern: source, group: role }
|
|
16
|
+
- { pattern: sources, group: role }
|
|
17
|
+
- { pattern: official, group: role }
|
|
18
|
+
- { pattern: officials, group: role }
|
|
19
|
+
- { pattern: insider, group: role }
|
|
20
|
+
- { pattern: insiders, group: role }
|
|
21
|
+
- { pattern: spokesperson, group: role }
|
|
22
|
+
- { pattern: spokesman, group: role }
|
|
23
|
+
- { pattern: spokeswoman, group: role }
|
|
24
|
+
- { pattern: observer, group: role }
|
|
25
|
+
- { pattern: observers, group: role }
|
|
26
|
+
- { pattern: at, group: after-role }
|
|
27
|
+
- { pattern: with, group: after-role }
|
|
28
|
+
- { pattern: from, group: after-role }
|
|
29
|
+
- { pattern: for, group: after-role }
|
|
30
|
+
- { pattern: in, group: after-role }
|
|
31
|
+
- { pattern: of, group: after-role }
|
|
32
|
+
- { pattern: on, group: after-role }
|
|
33
|
+
- { pattern: close, group: after-role }
|
|
34
|
+
- { pattern: familiar, group: after-role }
|
|
35
|
+
- { pattern: who, group: after-role }
|
|
36
|
+
- { pattern: Mr, group: honorific }
|
|
37
|
+
- { pattern: Mrs, group: honorific }
|
|
38
|
+
- { pattern: Ms, group: honorific }
|
|
39
|
+
- { pattern: Dr, group: honorific }
|
|
40
|
+
- { pattern: Prof, group: honorific }
|
|
41
|
+
- { pattern: say, group: cue }
|
|
42
|
+
- { pattern: told, group: cue }
|
|
43
|
+
- { pattern: tells, group: cue }
|
|
44
|
+
- { pattern: noted, group: cue }
|
|
45
|
+
- { pattern: notes, group: cue }
|
|
46
|
+
- { pattern: added, group: cue }
|
|
47
|
+
- { pattern: adds, group: cue }
|
|
48
|
+
- { pattern: explained, group: cue }
|
|
49
|
+
- { pattern: the, group: anaphor }
|
|
50
|
+
- { pattern: this, group: anaphor }
|
|
51
|
+
- { pattern: that, group: anaphor }
|
|
52
|
+
- { pattern: these, group: anaphor }
|
|
53
|
+
- { pattern: those, group: anaphor }
|
|
54
|
+
- { pattern: our, group: anaphor }
|
|
55
|
+
- { pattern: its, group: anaphor }
|
|
56
|
+
- { pattern: their, group: anaphor }
|
|
57
|
+
- { pattern: his, group: anaphor }
|
|
58
|
+
- { pattern: her, group: anaphor }
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# Words that make a sentence of a warning tell the reader what to do, matched as whole words ("Do not unplug the drive",
|
|
2
|
+
# "Make sure the export finished"). A sentence that opens with its verb ("Export today's orders first") is an action too,
|
|
3
|
+
# read from the parts of speech.
|
|
4
|
+
id: warning-action
|
|
5
|
+
language: en
|
|
6
|
+
entries:
|
|
7
|
+
- pattern: do not
|
|
8
|
+
- pattern: don't
|
|
9
|
+
- pattern: never
|
|
10
|
+
- pattern: always
|
|
11
|
+
- pattern: make sure
|
|
12
|
+
- pattern: be sure
|
|
13
|
+
- pattern: ensure
|
|
14
|
+
- pattern: avoid
|
|
15
|
+
- pattern: must
|
|
16
|
+
- pattern: should
|
|
17
|
+
- pattern: need to
|
|
18
|
+
- pattern: have to
|
|
19
|
+
- pattern: please
|
|
20
|
+
- pattern: before you
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# Words a warning opens with, standing alone ("Warning:", "**Caution**", "> [!WARNING]"). warning-without-action reads the
|
|
2
|
+
# paragraph after one. "Note" and "Tip" are left out: they give information, not a risk.
|
|
3
|
+
id: warning-label
|
|
4
|
+
language: en
|
|
5
|
+
entries:
|
|
6
|
+
- pattern: warning
|
|
7
|
+
- pattern: caution
|
|
8
|
+
- pattern: danger
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# Words beside the length of a break (duration-mismatch, working hours). before: written before the length ("break of 1 hour"); after: behind it ("a 1-hour break").
|
|
2
|
+
id: working-hours-break
|
|
3
|
+
language: en
|
|
4
|
+
entries:
|
|
5
|
+
- { pattern: "break", position: before }
|
|
6
|
+
- { pattern: "break of", position: before }
|
|
7
|
+
- { pattern: "lunch break", position: before }
|
|
8
|
+
- { pattern: "lunch break of", position: before }
|
|
9
|
+
- { pattern: "break", position: after }
|
|
10
|
+
- { pattern: "lunch break", position: after }
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# Words beside the hours worked once the break is taken out (duration-mismatch, working hours). before: written before the length; after: behind it ("8 hours of work", "8 hours a day").
|
|
2
|
+
id: working-hours-total
|
|
3
|
+
language: en
|
|
4
|
+
entries:
|
|
5
|
+
- { pattern: "net", position: before }
|
|
6
|
+
- { pattern: "of work", position: after }
|
|
7
|
+
- { pattern: "of working time", position: after }
|
|
8
|
+
- { pattern: "work", position: after }
|
|
9
|
+
- { pattern: "a day", position: after }
|
|
10
|
+
- { pattern: "per day", position: after }
|
|
11
|
+
- { pattern: "net", position: after }
|
package/package.json
CHANGED
package/src/definitions.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { Mention } from "chaffjs/plugin";
|
|
2
2
|
import { CHAPTER_DEPTH, PART_DEPTH } from "./depth.ts";
|
|
3
|
+
import { readsAsExample } from "./example-name.ts";
|
|
3
4
|
|
|
4
5
|
// Definitions in English documents, and how far a definition holds.
|
|
5
6
|
|
|
@@ -69,12 +70,18 @@ const namesInList = (inside: string): QuotedName[] => {
|
|
|
69
70
|
const nameListDefinitions = (text: string): Mention[] =>
|
|
70
71
|
[...text.matchAll(BRACKET)].flatMap((match) => {
|
|
71
72
|
const inside = match.groups?.["inside"] ?? "";
|
|
72
|
-
const
|
|
73
|
+
const isBare = BARE.test(match[0]);
|
|
74
|
+
const bare = isBare ? { form: "bare" } : {};
|
|
73
75
|
const scope = APPLIES.test(text) ? { scope: "local" } : {};
|
|
74
|
-
|
|
76
|
+
const names = namesInList(inside);
|
|
77
|
+
// A phrase first makes the bracket a list of examples; a phrase later is an example beside a name (("Cheers", "I hope
|
|
78
|
+
// this helps")), or an alias in sentence case (("Provider", "service provider")). The name is kept, the phrase is not.
|
|
79
|
+
if (isBare && readsAsExample(names[0]?.term ?? "")) return [];
|
|
80
|
+
return names.flatMap((name, index) => {
|
|
81
|
+
if (isBare && readsAsExample(name.term)) return [];
|
|
75
82
|
const start = index === 0 ? match.index : match.index + 1 + name.start;
|
|
76
83
|
const end = index === 0 ? match.index + match[0].length : match.index + 1 + name.end;
|
|
77
|
-
return { start, end, attrs: { term: name.term, ...scope, ...(index === 0 ? { placement: "inline" } : {}), ...bare } };
|
|
84
|
+
return [{ start, end, attrs: { term: name.term, ...scope, ...(index === 0 ? { placement: "inline" } : {}), ...bare } }];
|
|
78
85
|
});
|
|
79
86
|
});
|
|
80
87
|
|
|
@@ -82,12 +89,13 @@ const patternDefinitions = (text: string): Mention[] =>
|
|
|
82
89
|
DEFINITIONS.flatMap((pattern) =>
|
|
83
90
|
[...text.matchAll(pattern)].flatMap((match) => {
|
|
84
91
|
const term = match.groups?.["term"];
|
|
85
|
-
|
|
92
|
+
const isBare = BARE.test(match[0]);
|
|
93
|
+
if (term === undefined || (isBare && readsAsExample(term))) return [];
|
|
86
94
|
const end = match.index + match[0].length;
|
|
87
95
|
const inline = match[0].startsWith("(");
|
|
88
96
|
const namesAParty = inline && APPLIES.test(text);
|
|
89
97
|
const scope = isPointer(text, end) || namesAParty ? { scope: "local" } : {};
|
|
90
|
-
const bare =
|
|
98
|
+
const bare = isBare ? { form: "bare" } : {};
|
|
91
99
|
return [{ start: match.index, end, attrs: { term, ...scope, ...(inline ? { placement: "inline" } : {}), ...bare } }];
|
|
92
100
|
}),
|
|
93
101
|
);
|
package/src/emphasis.ts
CHANGED
|
@@ -1,16 +1,67 @@
|
|
|
1
|
-
// A word written in capitals for emphasis (the office will NEVER call you), told from an acronym by the dictionary
|
|
1
|
+
// A word written in capitals for emphasis (the office will NEVER call you), told from an acronym by the dictionary and by how
|
|
2
|
+
// the tagger read it in its sentence.
|
|
2
3
|
|
|
3
4
|
const CAPITALS = /^\p{Lu}{2,}$/u;
|
|
5
|
+
const LOWERCASE = /\p{Ll}/u;
|
|
4
6
|
|
|
5
7
|
const ADVERB_TAGS: ReadonlySet<string> = new Set(["RB", "RBR", "RBS"]);
|
|
6
8
|
|
|
9
|
+
/**
|
|
10
|
+
* Penn tags of closed word classes: preposition, conjunction, determiner, numeral, pronoun, particle, modal. An acronym names a
|
|
11
|
+
* thing, so it stands where a noun stands, never where one of these does.
|
|
12
|
+
*/
|
|
13
|
+
const CLOSED_CLASS_TAGS: ReadonlySet<string> = new Set(["IN", "CC", "DT", "PDT", "CD", "PRP", "PRP$", "TO", "RP", "MD", "WDT", "WP", "EX"]);
|
|
14
|
+
|
|
15
|
+
/** A determiner or possessive opens a noun phrase: what follows it (the IT team, the US office) is a name, though a numeral may come between (the ONE storm). */
|
|
16
|
+
const NOUN_PHRASE_OPENERS: ReadonlySet<string> = new Set(["DT", "PDT", "PRP$", "POS"]);
|
|
17
|
+
|
|
18
|
+
/** What can stand right before a finite verb in a sentence: a common noun or pronoun subject, or a modal (a hurricane DID form). */
|
|
19
|
+
const SUBJECT_TAGS: ReadonlySet<string> = new Set(["NN", "NNS", "PRP", "MD"]);
|
|
20
|
+
|
|
21
|
+
type TaggedWord = { readonly value: string; readonly pos: string };
|
|
22
|
+
|
|
23
|
+
type TagsOf = (word: string) => readonly string[] | undefined;
|
|
24
|
+
|
|
7
25
|
/**
|
|
8
26
|
* The dictionary knows the lower-case word, and only as an adverb (never, always, also). An acronym names a thing, so a word
|
|
9
27
|
* with no noun or adjective reading cannot be one. A word that is also a noun or an adjective (fast, eagle, cheese) is kept:
|
|
10
28
|
* FAST and EAGLE are a telescope and a simulation, and the context tagger cannot tell them from emphasis.
|
|
11
29
|
*/
|
|
12
|
-
export const isEmphasisedAdverb = (surface: string, tagsOf:
|
|
30
|
+
export const isEmphasisedAdverb = (surface: string, tagsOf: TagsOf): boolean => {
|
|
13
31
|
if (!CAPITALS.test(surface)) return false;
|
|
14
32
|
const tags = tagsOf(surface.toLowerCase()) ?? [];
|
|
15
33
|
return tags.length > 0 && tags.every((tag) => ADVERB_TAGS.has(tag));
|
|
16
34
|
};
|
|
35
|
+
|
|
36
|
+
/** After a verb, a word the tagger read as a noun must also be a verb in the dictionary (DID form), or it is what the capitals name (DID token). */
|
|
37
|
+
const canFollowVerb = (next: TaggedWord, tagsOf: TagsOf): boolean => !next.pos.startsWith("NN") || (tagsOf(next.value.toLowerCase()) ?? []).includes("VB");
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* The tagger read the capitals word, in its sentence, as one of the dictionary's readings of the lower-case word, and that
|
|
41
|
+
* reading is a closed class (AFTER the departure, NOR a remote position), or a verb right after its subject (a hurricane DID
|
|
42
|
+
* form). The word sits inside lower-case prose, with a lower-case word on each side: at the start of a sentence (ON
|
|
43
|
+
* Semiconductor), at its end (via IT.) or on a line alone (AT.) there is nothing to read it in. A verb after anything but its
|
|
44
|
+
* subject, or before a noun, stays a name: on the GOES satellites, the states are: LISTEN, the user DID token. Adjectives
|
|
45
|
+
* stay too: the WHOLE month reads like the SAFE framework, and only meaning tells them apart.
|
|
46
|
+
*/
|
|
47
|
+
const isReadAsWord = (tagged: readonly TaggedWord[], at: number, tagsOf: TagsOf): boolean => {
|
|
48
|
+
const word = tagged[at];
|
|
49
|
+
const [previous, next] = [tagged[at - 1], tagged[at + 1]];
|
|
50
|
+
if (word === undefined || previous === undefined || next === undefined) return false;
|
|
51
|
+
if (!(tagsOf(word.value.toLowerCase()) ?? []).includes(word.pos) || !LOWERCASE.test(previous.value) || !LOWERCASE.test(next.value)) return false;
|
|
52
|
+
if (CLOSED_CLASS_TAGS.has(word.pos)) return word.pos === "CD" || !NOUN_PHRASE_OPENERS.has(previous.pos);
|
|
53
|
+
return word.pos.startsWith("VB") && SUBJECT_TAGS.has(previous.pos) && canFollowVerb(next, tagsOf);
|
|
54
|
+
};
|
|
55
|
+
|
|
56
|
+
/** A word joined to its neighbour with no space (US-CERT, SOME/2, IT's) is part of a compound, a label or a name, not a word on its own. */
|
|
57
|
+
const JOINED = /[\p{L}\p{N}_&/'’-]/u;
|
|
58
|
+
|
|
59
|
+
type WordSpan = { readonly start: number; readonly end: number };
|
|
60
|
+
|
|
61
|
+
export const standsAlone = (text: string, span: WordSpan): boolean => !JOINED.test(text.charAt(span.start - 1)) && !JOINED.test(text.charAt(span.end));
|
|
62
|
+
|
|
63
|
+
/** A capitals word the sentence reads as an English word (not the adverb-only case above), so emphasis and not an acronym. */
|
|
64
|
+
export const isEmphasisedWord = (tagged: readonly TaggedWord[], at: number, tagsOf: TagsOf): boolean => {
|
|
65
|
+
const surface = tagged[at]?.value ?? "";
|
|
66
|
+
return CAPITALS.test(surface) && isReadAsWord(tagged, at, tagsOf);
|
|
67
|
+
};
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
// Whether a name quoted in bare brackets (("The key point is")) reads as a quoted example rather than a defined term.
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Words a capitalised name keeps in lower case: articles, conjunctions, prepositions, determiners and the particles of
|
|
5
|
+
* personal and company names (Terms of Use, Fees under this Agreement, Banco de Chile, Ludwig van Beethoven).
|
|
6
|
+
*/
|
|
7
|
+
const MINOR_WORDS: ReadonlySet<string> = new Set(
|
|
8
|
+
[
|
|
9
|
+
"a an the and or but nor so yet",
|
|
10
|
+
"of in on at to for with as by from per via vs into onto upon under over between among within without about after",
|
|
11
|
+
"before against along across around through throughout during until toward towards beneath beside besides beyond",
|
|
12
|
+
"near off out up down than like unless whether arising",
|
|
13
|
+
"this that these those each every any all such its their his her our your",
|
|
14
|
+
"de del della di da du des la le van von der den ter ten y e",
|
|
15
|
+
]
|
|
16
|
+
.join(" ")
|
|
17
|
+
.split(" "),
|
|
18
|
+
);
|
|
19
|
+
|
|
20
|
+
const WORD = /[\p{L}\p{N}][\p{L}\p{N}'’-]*/gu;
|
|
21
|
+
const LOWER_START = /^\p{Ll}/u;
|
|
22
|
+
/** A capital inside the word (iOS, eBay) makes it a name, whatever its first letter. */
|
|
23
|
+
const HAS_UPPER = /\p{Lu}/u;
|
|
24
|
+
/** A statute names a party in lower case after an article: ("the seller"), ("a sub-processor"). */
|
|
25
|
+
const ARTICLE_LEAD = /^(?:the|a|an)\s/u;
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* A defined term is one word ("Seller"), a capitalised name ("Use of Customer Data"), or a statute's lower-case party
|
|
29
|
+
* after an article ("the annual period"). Several words with one in lower case that a name would capitalise ("The key
|
|
30
|
+
* point is", "silently fails", "sections 44 and 45") quote an example instead.
|
|
31
|
+
*/
|
|
32
|
+
export const readsAsExample = (name: string): boolean => {
|
|
33
|
+
const words = [...name.matchAll(WORD)].map((match) => match[0]);
|
|
34
|
+
if (words.length < 2 || ARTICLE_LEAD.test(name)) return false;
|
|
35
|
+
return words.some((word) => LOWER_START.test(word) && !HAS_UPPER.test(word) && !MINOR_WORDS.has(word));
|
|
36
|
+
};
|
package/src/pos.ts
CHANGED
|
@@ -4,7 +4,7 @@ import { loadLexicons } from "./lexicons.ts";
|
|
|
4
4
|
import { blankLongRuns } from "./long-runs.ts";
|
|
5
5
|
import { lowercasedAt, properNounChecked, rereadAt, sentenceInitialCommonWord } from "./proper-noun.ts";
|
|
6
6
|
import { isStativeParticiple, stativeVocabulary } from "./stative-participle.ts";
|
|
7
|
-
import { isEmphasisedAdverb } from "./emphasis.ts";
|
|
7
|
+
import { isEmphasisedAdverb, isEmphasisedWord, standsAlone } from "./emphasis.ts";
|
|
8
8
|
import { gerundFeatures } from "./gerund.ts";
|
|
9
9
|
import { winkLexiconWords, winkPosTagger } from "./wink-modules.ts";
|
|
10
10
|
|
|
@@ -234,9 +234,15 @@ export const prepare = (): void => {
|
|
|
234
234
|
|
|
235
235
|
export const isReady = (): boolean => state.ready !== undefined;
|
|
236
236
|
|
|
237
|
-
/**
|
|
238
|
-
|
|
239
|
-
|
|
237
|
+
/**
|
|
238
|
+
* 大文字で強調した副詞(NEVER)は、解析器が名前と付けても副詞。文の中で前置詞や動詞として読めた語(AFTER the departure、
|
|
239
|
+
* a hurricane DID form)は、解析器の読みのまま。どちらも Emph=Yes で、略語ではないと detector に伝える。
|
|
240
|
+
*/
|
|
241
|
+
const withEmphasis = (token: Token, text: string, tagged: readonly Tagged[], at: number): Token => {
|
|
242
|
+
if (isEmphasisedAdverb(token.surface, state.vocabulary)) return { ...token, pos: "ADV", lemma: token.surface.toLowerCase(), features: { Emph: "Yes" } };
|
|
243
|
+
if (!standsAlone(text, token.span) || !isEmphasisedWord(tagged, at, state.vocabulary)) return token;
|
|
244
|
+
return { ...token, lemma: (token.lemma ?? token.surface).toLowerCase(), features: { ...token.features, Emph: "Yes" } };
|
|
245
|
+
};
|
|
240
246
|
|
|
241
247
|
/**
|
|
242
248
|
* wink は位置を返さないので、表層を順に照合して復元する。
|
|
@@ -257,7 +263,7 @@ const locate = (text: string, tagged: readonly Tagged[]): Token[] => {
|
|
|
257
263
|
...(entry.lemma === undefined ? {} : { lemma: entry.lemma }),
|
|
258
264
|
...featuresOf(tagged, at),
|
|
259
265
|
};
|
|
260
|
-
tokens.push(withEmphasis(token));
|
|
266
|
+
tokens.push(withEmphasis(token, text, tagged, at));
|
|
261
267
|
cursor = end;
|
|
262
268
|
});
|
|
263
269
|
return tokens;
|
package/src/structure.ts
CHANGED
|
@@ -274,12 +274,35 @@ const currencyBefore = (text: string, at: number): string | undefined => {
|
|
|
274
274
|
return CURRENCIES.find((currency) => text.startsWith(currency, end - currency.length));
|
|
275
275
|
};
|
|
276
276
|
|
|
277
|
+
/** A count of days or months is written "six (6)" or "one thousand (1,000)": never "zero (0)", never "two million (2,000,000)". */
|
|
278
|
+
const FEWEST_COUNTED = 1;
|
|
279
|
+
const MOST_COUNTED = 1000;
|
|
280
|
+
|
|
281
|
+
const countsPeriods = (weight: number | undefined): boolean => weight !== undefined && weight >= FEWEST_COUNTED && weight <= MOST_COUNTED;
|
|
282
|
+
|
|
283
|
+
const FIGURE_WORDS: ReadonlySet<string> = new Set((LEXICONS["number-word"] ?? []).filter((entry) => countsPeriods(entry.weight)).map((entry) => entry.pattern));
|
|
284
|
+
const OPEN_BRACKET = " (";
|
|
285
|
+
const NOT_WORD_LETTER = /[^\p{L}-]/u;
|
|
286
|
+
const BRACKET_REACH = 30;
|
|
287
|
+
|
|
288
|
+
/** The word right before " (" is a number word, or ends with one: "six (", "forty-five (". */
|
|
289
|
+
const followsNumberWord = (text: string, start: number): boolean => {
|
|
290
|
+
const before = text.slice(Math.max(0, start - BRACKET_REACH), start);
|
|
291
|
+
if (!before.endsWith(OPEN_BRACKET)) return false;
|
|
292
|
+
const word = before.slice(0, -OPEN_BRACKET.length).split(NOT_WORD_LETTER).at(-1) ?? "";
|
|
293
|
+
return FIGURE_WORDS.has(word.toLowerCase().split("-").at(-1) ?? "");
|
|
294
|
+
};
|
|
295
|
+
|
|
296
|
+
/** "six (6) months": a figure in brackets after a number in words, with the unit after the bracket. */
|
|
297
|
+
const unitAfterBracket = (text: string, start: number, end: number): string | undefined =>
|
|
298
|
+
text[end] === ")" && followsNumberWord(text, start) ? unitAfter(text, end + 1) : undefined;
|
|
299
|
+
|
|
277
300
|
const quantities = (text: string): Mention[] =>
|
|
278
301
|
[...text.matchAll(NUMBER_RUN)].flatMap((match) => {
|
|
279
302
|
const digits = withoutTrailingPunctuation(match[0]);
|
|
280
303
|
const value = Number(digits.replace(/,/gu, ""));
|
|
281
304
|
const end = match.index + digits.length;
|
|
282
|
-
const unit = unitAfter(text, end) ?? currencyBefore(text, match.index);
|
|
305
|
+
const unit = unitAfter(text, end) ?? unitAfterBracket(text, match.index, end) ?? currencyBefore(text, match.index);
|
|
283
306
|
return unit === undefined || Number.isNaN(value) || isWordChar(text[match.index - 1]) ? [] : [{ start: match.index, end, attrs: { value, unit } }];
|
|
284
307
|
});
|
|
285
308
|
|