chaffjs 0.24.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/detectors/connective-hierarchy.d.ts +16 -0
- package/dist/detectors/connective-hierarchy.d.ts.map +1 -0
- package/dist/detectors/connective-hierarchy.js +68 -0
- package/dist/detectors/connective-hierarchy.js.map +1 -0
- package/dist/detectors/missing-final-period.d.ts +17 -0
- package/dist/detectors/missing-final-period.d.ts.map +1 -0
- package/dist/detectors/missing-final-period.js +62 -0
- package/dist/detectors/missing-final-period.js.map +1 -0
- package/dist/detectors/missing-space-after-comma.d.ts +10 -0
- package/dist/detectors/missing-space-after-comma.d.ts.map +1 -0
- package/dist/detectors/missing-space-after-comma.js +42 -0
- package/dist/detectors/missing-space-after-comma.js.map +1 -0
- package/dist/detectors/registry/connective-hierarchy.d.ts +3 -0
- package/dist/detectors/registry/connective-hierarchy.d.ts.map +1 -0
- package/dist/detectors/registry/connective-hierarchy.js +3 -0
- package/dist/detectors/registry/connective-hierarchy.js.map +1 -0
- package/dist/detectors/registry/missing-final-period.d.ts +3 -0
- package/dist/detectors/registry/missing-final-period.d.ts.map +1 -0
- package/dist/detectors/registry/missing-final-period.js +3 -0
- package/dist/detectors/registry/missing-final-period.js.map +1 -0
- package/dist/detectors/registry/missing-space-after-comma.d.ts +3 -0
- package/dist/detectors/registry/missing-space-after-comma.d.ts.map +1 -0
- package/dist/detectors/registry/missing-space-after-comma.js +3 -0
- package/dist/detectors/registry/missing-space-after-comma.js.map +1 -0
- package/package.json +2 -2
- package/rules/connective-hierarchy.yaml +62 -0
- package/rules/missing-final-period.yaml +76 -0
- package/rules/missing-space-after-comma.yaml +57 -0
- package/skills/chaff/SKILL.md +20 -3
- package/src/detectors/connective-hierarchy.ts +99 -0
- package/src/detectors/missing-final-period.ts +79 -0
- package/src/detectors/missing-space-after-comma.ts +54 -0
- package/src/detectors/registry/connective-hierarchy.ts +4 -0
- package/src/detectors/registry/missing-final-period.ts +4 -0
- package/src/detectors/registry/missing-space-after-comma.ts +4 -0
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { Detector, LexiconEntry, Sentence } from "../plugin.ts";
|
|
2
|
+
/** One outer-level connective with nothing to stand above, and where it is in the document. */
|
|
3
|
+
export type LoneConnective = {
|
|
4
|
+
readonly offset: number;
|
|
5
|
+
readonly written: string;
|
|
6
|
+
readonly usual: string;
|
|
7
|
+
};
|
|
8
|
+
/** The word lists the detector reads: the connectives, and the phrases that hold one's letters without being one (歯並びに). */
|
|
9
|
+
export type ConnectiveLists = {
|
|
10
|
+
readonly connectives: readonly LexiconEntry[];
|
|
11
|
+
readonly lookalikes: readonly LexiconEntry[];
|
|
12
|
+
};
|
|
13
|
+
/** The outer-level connectives of a sentence that stand over no inner level. */
|
|
14
|
+
export declare const loneConnectivesIn: (sentence: Sentence, lists: ConnectiveLists) => LoneConnective[];
|
|
15
|
+
export declare const connectiveHierarchy: Detector;
|
|
16
|
+
//# sourceMappingURL=connective-hierarchy.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"connective-hierarchy.d.ts","sourceRoot":"","sources":["../../src/detectors/connective-hierarchy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAW,YAAY,EAAE,QAAQ,EAAe,MAAM,cAAc,CAAC;AAQ3F,+FAA+F;AAC/F,MAAM,MAAM,cAAc,GAAG;IAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAI3G,4HAA4H;AAC5H,MAAM,MAAM,eAAe,GAAG;IAAE,QAAQ,CAAC,WAAW,EAAE,SAAS,YAAY,EAAE,CAAC;IAAC,QAAQ,CAAC,UAAU,EAAE,SAAS,YAAY,EAAE,CAAA;CAAE,CAAC;AA4D9H,gFAAgF;AAChF,eAAO,MAAM,iBAAiB,GAAI,UAAU,QAAQ,EAAE,OAAO,eAAe,KAAG,cAAc,EAS5F,CAAC;AAEF,eAAO,MAAM,mBAAmB,EAAE,QAYjC,CAAC"}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { escapeRegExp } from "../orthography.js";
|
|
2
|
+
import { quotedSpans } from "../quoted-span.js";
|
|
3
|
+
const CONJUNCTION = "CCONJ";
|
|
4
|
+
/**
|
|
5
|
+
* The tagger does not always tell a connective from a noun and a particle: it reads 申請書並びに as 並び + に and
|
|
6
|
+
* 文字の並びに as the conjunction. After の, a determiner or an adjective (の並びに, この並びに, 正しい並びに) the word is
|
|
7
|
+
* a noun of its own. Where the tagger splits it, it is a connective only between two nouns: a noun, a number or a mark
|
|
8
|
+
* before it, and no verb right after it (一列並びに座る).
|
|
9
|
+
*/
|
|
10
|
+
const MODIFIES_NOUN = new Set(["DET", "ADJ"]);
|
|
11
|
+
const GENITIVE = "の";
|
|
12
|
+
const JOINS_AFTER = new Set(["NOUN", "PROPN", "NUM", "PUNCT", "SYM"]);
|
|
13
|
+
const ENDS_CLAUSE = new Set(["VERB", "AUX"]);
|
|
14
|
+
const modifiesNoun = (token) => token !== undefined && (MODIFIES_NOUN.has(token.pos) || (token.pos === "ADP" && token.surface === GENITIVE));
|
|
15
|
+
const joinsNouns = (previous, next) => previous !== undefined && JOINS_AFTER.has(previous.pos) && next !== undefined && !ENDS_CLAUSE.has(next.pos);
|
|
16
|
+
const placesOf = (text, word) => [...text.matchAll(new RegExp(escapeRegExp(word), "gu"))].map((match) => match.index);
|
|
17
|
+
/** Inside 「」 or 『』 (a quoted phrase holds half a list) or inside a lookalike (歯並びに), by the sentence's own offsets. */
|
|
18
|
+
const hiddenLetters = (text, lookalikes) => {
|
|
19
|
+
const hidden = Array.from({ length: text.length }, () => false);
|
|
20
|
+
const hide = (start, end) => hidden.fill(true, start, end);
|
|
21
|
+
quotedSpans(text).forEach((span) => hide(span.start, span.end));
|
|
22
|
+
lookalikes.forEach(({ pattern }) => placesOf(text, pattern).forEach((at) => hide(at, at + pattern.length)));
|
|
23
|
+
return hidden;
|
|
24
|
+
};
|
|
25
|
+
const readingOf = (sentence, lookalikes) => {
|
|
26
|
+
const tokens = sentence.tokens ?? [];
|
|
27
|
+
return {
|
|
28
|
+
byStart: new Map(tokens.map((token) => [token.span.start, token])),
|
|
29
|
+
byEnd: new Map(tokens.map((token) => [token.span.end, token])),
|
|
30
|
+
hidden: hiddenLetters(sentence.text, lookalikes),
|
|
31
|
+
};
|
|
32
|
+
};
|
|
33
|
+
const standsAsConnective = (reading, span) => {
|
|
34
|
+
const previous = reading.byEnd.get(span.start);
|
|
35
|
+
if (modifiesNoun(previous))
|
|
36
|
+
return false;
|
|
37
|
+
const own = reading.byStart.get(span.start);
|
|
38
|
+
const tagged = own !== undefined && own.pos === CONJUNCTION && own.span.end === span.end;
|
|
39
|
+
return tagged || joinsNouns(previous, reading.byStart.get(span.end));
|
|
40
|
+
};
|
|
41
|
+
const hasInnerLevel = (sentence, outer, connectives) => connectives.some((entry) => entry.instead_of === outer.pattern && sentence.text.includes(entry.pattern));
|
|
42
|
+
/** Where one outer-level word stands as a connective in the sentence. */
|
|
43
|
+
const connectiveSpans = (sentence, outer, reading) => placesOf(sentence.text, outer.pattern)
|
|
44
|
+
.filter((at) => reading.hidden[at] !== true)
|
|
45
|
+
.map((at) => ({ start: sentence.span.start + at, end: sentence.span.start + at + outer.pattern.length }))
|
|
46
|
+
.filter((span) => standsAsConnective(reading, span));
|
|
47
|
+
/** The outer-level connectives of a sentence that stand over no inner level. */
|
|
48
|
+
export const loneConnectivesIn = (sentence, lists) => {
|
|
49
|
+
const lone = lists.connectives.filter((entry) => entry.rewrite !== undefined && sentence.text.includes(entry.pattern) && !hasInnerLevel(sentence, entry, lists.connectives));
|
|
50
|
+
if (lone.length === 0)
|
|
51
|
+
return [];
|
|
52
|
+
const reading = readingOf(sentence, lists.lookalikes);
|
|
53
|
+
return lone
|
|
54
|
+
.flatMap((entry) => connectiveSpans(sentence, entry, reading).map((span) => ({ offset: span.start, written: entry.pattern, usual: entry.rewrite ?? "" })))
|
|
55
|
+
.sort((a, b) => a.offset - b.offset);
|
|
56
|
+
};
|
|
57
|
+
export const connectiveHierarchy = (doc, options) => {
|
|
58
|
+
const lists = { connectives: options.lexicon ?? [], lookalikes: doc.lexicons["connective-lookalike"] ?? [] };
|
|
59
|
+
return doc.sentences.flatMap((sentence) => loneConnectivesIn(sentence, lists).map((lone) => ({
|
|
60
|
+
rule: "",
|
|
61
|
+
severity: "info",
|
|
62
|
+
line: 0,
|
|
63
|
+
column: 0,
|
|
64
|
+
quote: sentence.text.trim(),
|
|
65
|
+
values: { written: lone.written, usual: lone.usual, offset: lone.offset },
|
|
66
|
+
})));
|
|
67
|
+
};
|
|
68
|
+
//# sourceMappingURL=connective-hierarchy.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"connective-hierarchy.js","sourceRoot":"","sources":["../../src/detectors/connective-hierarchy.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAShD,MAAM,WAAW,GAAG,OAAO,CAAC;AAK5B;;;;;GAKG;AACH,MAAM,aAAa,GAAwB,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;AACnE,MAAM,QAAQ,GAAG,GAAG,CAAC;AACrB,MAAM,WAAW,GAAwB,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;AAC3F,MAAM,WAAW,GAAwB,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;AAElE,MAAM,YAAY,GAAG,CAAC,KAAwB,EAAW,EAAE,CACzD,KAAK,KAAK,SAAS,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,KAAK,KAAK,IAAI,KAAK,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC;AAE/G,MAAM,UAAU,GAAG,CAAC,QAA2B,EAAE,IAAuB,EAAW,EAAE,CACnF,QAAQ,KAAK,SAAS,IAAI,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAK9G,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAE,IAAY,EAAY,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAEhJ,uHAAuH;AACvH,MAAM,aAAa,GAAG,CAAC,IAAY,EAAE,UAAmC,EAAa,EAAE;IACrF,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAChE,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,GAAW,EAAa,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;IACtF,WAAW,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAChE,UAAU,CAAC,OAAO,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC5G,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,CAAC,QAAkB,EAAE,UAAmC,EAAW,EAAE;IACrF,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC;IACrC,OAAO;QACL,OAAO,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;QAClE,KAAK,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;QAC9D,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;KACjD,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CAAC,OAAgB,EAAE,IAAU,EAAW,EAAE;IACnE,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC/C,IAAI,YAAY,CAAC,QAAQ,CAAC;QAAE,OAAO,KAAK,CAAC;IACzC,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,GAAG,KAAK,SAAS,IAAI,GAAG,CAAC,GAAG,KAAK,WAAW,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,GAAG,CAAC;IACzF,OAAO,MAAM,IAAI,UAAU,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AACvE,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,QAAkB,EAAE,KAAmB,EAAE,WAAoC,EAAW,EAAE,CAC/G,WAAW,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,KAAK,KAAK,CAAC,OAAO,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;AAE3G,yEAAyE;AACzE,MAAM,eAAe,GAAG,CAAC,QAAkB,EAAE,KAAmB,EAAE,OAAgB,EAAU,EAAE,CAC5F,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC;KACnC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,IAAI,CAAC;KAC3C,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,EAAE,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;KACxG,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;AAEzD,gFAAgF;AAChF,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,QAAkB,EAAE,KAAsB,EAAoB,EAAE;IAChG,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,CACnC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,WAAW,CAAC,CACtI,CAAC;IACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,MAAM,OAAO,GAAG,SAAS,CAAC,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;IACtD,OAAO,IAAI;SACR,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,eAAe,CAAC,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;SACzJ,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;AACzC,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,mBAAmB,GAAa,CAAC,GAAG,EAAE,OAAO,EAAa,EAAE;IACvE,MAAM,KAAK,GAAG,EAAE,WAAW,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,QAAQ,CAAC,sBAAsB,CAAC,IAAI,EAAE,EAAE,CAAC;IAC7G,OAAO,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE,CACxC,iBAAiB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAChD,IAAI,EAAE,EAAE;QACR,QAAQ,EAAE,MAAM;QAChB,IAAI,EAAE,CAAC;QACP,MAAM,EAAE,CAAC;QACT,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE;QAC3B,MAAM,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE;KAC1E,CAAC,CAAC,CACJ,CAAC;AACJ,CAAC,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { Detector, Paragraph } from "../plugin.ts";
|
|
2
|
+
/** A paragraph's end, and whether it closes with a sentence-ending mark. form is "closed" or "open". */
|
|
3
|
+
export type ParagraphEnd = {
|
|
4
|
+
readonly offset: number;
|
|
5
|
+
readonly form: string;
|
|
6
|
+
readonly text: string;
|
|
7
|
+
};
|
|
8
|
+
/** The marks the detector reads, from the lexicon. */
|
|
9
|
+
export type PeriodMarks = {
|
|
10
|
+
readonly ends: readonly string[];
|
|
11
|
+
readonly otherEnds: readonly string[];
|
|
12
|
+
readonly notes: readonly string[];
|
|
13
|
+
};
|
|
14
|
+
/** How a paragraph ends, or undefined when its last line is not a sentence (a label, a lead-in, a note, a laid-out row). */
|
|
15
|
+
export declare const paragraphEnd: (text: string, paragraph: Paragraph, marks: PeriodMarks) => ParagraphEnd | undefined;
|
|
16
|
+
export declare const missingFinalPeriod: Detector;
|
|
17
|
+
//# sourceMappingURL=missing-final-period.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"missing-final-period.d.ts","sourceRoot":"","sources":["../../src/detectors/missing-final-period.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAyB,SAAS,EAAwB,MAAM,cAAc,CAAC;AASrG,wGAAwG;AACxG,MAAM,MAAM,YAAY,GAAG;IAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAErG,sDAAsD;AACtD,MAAM,MAAM,WAAW,GAAG;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;IAAC,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAA;CAAE,CAAC;AAiCzI,4HAA4H;AAC5H,eAAO,MAAM,YAAY,GAAI,MAAM,MAAM,EAAE,WAAW,SAAS,EAAE,OAAO,WAAW,KAAG,YAAY,GAAG,SASpG,CAAC;AAQF,eAAO,MAAM,kBAAkB,EAAE,QAchC,CAAC"}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { formMinority } from "./form-minority.js";
|
|
2
|
+
/** Brackets and quotes that may follow a closing mark (〜です。」). */
|
|
3
|
+
const CLOSERS = '」』))"”’';
|
|
4
|
+
/** A line that is itself a quotation closes like a sentence. */
|
|
5
|
+
const QUOTE_CLOSERS = "」』";
|
|
6
|
+
/** A paragraph ending in a colon leads into a list or a block; it is not a sentence left open. */
|
|
7
|
+
const LEAD_IN = /[::]$/u;
|
|
8
|
+
/** Two spaces in a row, half- or full-width, lay a line out like a table (an ingredient, then its amount), so it is not a sentence. */
|
|
9
|
+
const LAYOUT = /[ \u3000]{2}/u;
|
|
10
|
+
/** Fewer characters than this is a label or a caption, not a sentence. */
|
|
11
|
+
const MIN_SENTENCE_CHARS = 15;
|
|
12
|
+
/** Words that end a clause: a verb, an adjective or an auxiliary in its dictionary form ends the sentence. */
|
|
13
|
+
const PREDICATE = new Set(["VERB", "ADJ", "AUX"]);
|
|
14
|
+
/** The line's last mark before any closing brackets and quotes is one of these. */
|
|
15
|
+
const endsWithMark = (line, marks) => {
|
|
16
|
+
const chars = Array.from(line);
|
|
17
|
+
return marks.includes(chars[chars.findLastIndex((char) => !CLOSERS.includes(char))] ?? "");
|
|
18
|
+
};
|
|
19
|
+
const isClosed = (line, ends) => endsWithMark(line, ends) || QUOTE_CLOSERS.includes(line.at(-1) ?? "\0");
|
|
20
|
+
/** A note, a laid-out row, or a question or exclamation: not a sentence that could take the period. */
|
|
21
|
+
const isAside = (line, marks) => marks.notes.some((note) => line.startsWith(note)) || LAYOUT.test(line) || endsWithMark(line, marks.otherEnds);
|
|
22
|
+
/** The paragraph's last word ends a sentence: a predicate written in its dictionary form (〜します, 〜である, 望ましい). */
|
|
23
|
+
const endsInPredicate = (paragraph) => {
|
|
24
|
+
const last = (paragraph.sentences.at(-1)?.tokens ?? []).filter((token) => token.surface.trim() !== "").at(-1);
|
|
25
|
+
return last !== undefined && PREDICATE.has(last.pos) && last.lemma === last.surface;
|
|
26
|
+
};
|
|
27
|
+
/** How a paragraph ends, or undefined when its last line is not a sentence (a label, a lead-in, a note, a laid-out row). */
|
|
28
|
+
export const paragraphEnd = (text, paragraph, marks) => {
|
|
29
|
+
const body = text.slice(paragraph.span.start, paragraph.span.end).trimEnd();
|
|
30
|
+
const last = body.split("\n").at(-1)?.trim() ?? "";
|
|
31
|
+
if (Array.from(last).length < MIN_SENTENCE_CHARS || LEAD_IN.test(last))
|
|
32
|
+
return undefined;
|
|
33
|
+
const offset = paragraph.span.start + body.length - 1;
|
|
34
|
+
if (isAside(last, marks))
|
|
35
|
+
return undefined;
|
|
36
|
+
if (isClosed(last, marks.ends))
|
|
37
|
+
return { offset, form: "closed", text: last };
|
|
38
|
+
// A closing parenthetical after the sentence (〜します(別紙を参照する)) is not where the period goes.
|
|
39
|
+
return !CLOSERS.includes(last.at(-1) ?? "\0") && endsInPredicate(paragraph) ? { offset, form: "open", text: last } : undefined;
|
|
40
|
+
};
|
|
41
|
+
const marksOf = (lexicon) => ({
|
|
42
|
+
ends: lexicon.filter((entry) => entry.group === "end").map((entry) => entry.pattern),
|
|
43
|
+
otherEnds: lexicon.filter((entry) => entry.group === "other-end").map((entry) => entry.pattern),
|
|
44
|
+
notes: lexicon.filter((entry) => entry.group === "note").map((entry) => entry.pattern),
|
|
45
|
+
});
|
|
46
|
+
export const missingFinalPeriod = (doc, options) => {
|
|
47
|
+
const text = doc.prose ?? doc.source;
|
|
48
|
+
const marks = marksOf(doc.lexicons["final-period"] ?? []);
|
|
49
|
+
const ends = doc.paragraphs.flatMap((paragraph) => paragraphEnd(text, paragraph, marks) ?? []);
|
|
50
|
+
const minority = formMinority(ends, options.limit);
|
|
51
|
+
if (minority === undefined || minority.usual.form !== "closed")
|
|
52
|
+
return [];
|
|
53
|
+
return minority.odd.map((end) => ({
|
|
54
|
+
rule: "",
|
|
55
|
+
severity: "info",
|
|
56
|
+
line: 0,
|
|
57
|
+
column: 0,
|
|
58
|
+
quote: end.text,
|
|
59
|
+
values: { count: minority.odd.length, of: ends.length, offset: end.offset },
|
|
60
|
+
}));
|
|
61
|
+
};
|
|
62
|
+
//# sourceMappingURL=missing-final-period.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"missing-final-period.js","sourceRoot":"","sources":["../../src/detectors/missing-final-period.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAclD,kEAAkE;AAClE,MAAM,OAAO,GAAG,SAAS,CAAC;AAC1B,gEAAgE;AAChE,MAAM,aAAa,GAAG,IAAI,CAAC;AAC3B,kGAAkG;AAClG,MAAM,OAAO,GAAG,QAAQ,CAAC;AACzB,uIAAuI;AACvI,MAAM,MAAM,GAAG,eAAe,CAAC;AAC/B,0EAA0E;AAC1E,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,8GAA8G;AAC9G,MAAM,SAAS,GAAwB,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;AAEvE,mFAAmF;AACnF,MAAM,YAAY,GAAG,CAAC,IAAY,EAAE,KAAwB,EAAW,EAAE;IACvE,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/B,OAAO,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;AAC7F,CAAC,CAAC;AAEF,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAE,IAAuB,EAAW,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;AAE7I,uGAAuG;AACvG,MAAM,OAAO,GAAG,CAAC,IAAY,EAAE,KAAkB,EAAW,EAAE,CAC5D,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;AAEhH,gHAAgH;AAChH,MAAM,eAAe,GAAG,CAAC,SAAoB,EAAW,EAAE;IACxD,MAAM,IAAI,GAAG,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,KAAY,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACrH,OAAO,IAAI,KAAK,SAAS,IAAI,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,OAAO,CAAC;AACtF,CAAC,CAAC;AAEF,4HAA4H;AAC5H,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,IAAY,EAAE,SAAoB,EAAE,KAAkB,EAA4B,EAAE;IAC/G,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC;IAC5E,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACnD,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACzF,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IACtD,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC3C,IAAI,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC9E,2FAA2F;IAC3F,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,eAAe,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AACjI,CAAC,CAAC;AAEF,MAAM,OAAO,GAAG,CAAC,OAAgC,EAAe,EAAE,CAAC,CAAC;IAClE,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC;IACpF,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC;IAC/F,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC;CACvF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,kBAAkB,GAAa,CAAC,GAAkB,EAAE,OAAO,EAAa,EAAE;IACrF,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,CAAC;IACrC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC;IAC1D,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IAC/F,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACnD,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAC1E,OAAO,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAChC,IAAI,EAAE,EAAE;QACR,QAAQ,EAAE,MAAM;QAChB,IAAI,EAAE,CAAC;QACP,MAAM,EAAE,CAAC;QACT,KAAK,EAAE,GAAG,CAAC,IAAI;QACf,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE;KAC5E,CAAC,CAAC,CAAC;AACN,CAAC,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { Detector } from "../plugin.ts";
|
|
2
|
+
/** One comma with no space after it, where it is, and the two words it joins as written. */
|
|
3
|
+
export type TightComma = {
|
|
4
|
+
readonly offset: number;
|
|
5
|
+
readonly written: string;
|
|
6
|
+
};
|
|
7
|
+
/** Each run is read once, so a long line of values costs one pass. */
|
|
8
|
+
export declare const tightCommasIn: (text: string) => TightComma[];
|
|
9
|
+
export declare const missingSpaceAfterComma: Detector;
|
|
10
|
+
//# sourceMappingURL=missing-space-after-comma.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"missing-space-after-comma.d.ts","sourceRoot":"","sources":["../../src/detectors/missing-space-after-comma.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAW,MAAM,cAAc,CAAC;AAMtD,4FAA4F;AAC5F,MAAM,MAAM,UAAU,GAAG;IAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AA6B/E,sEAAsE;AACtE,eAAO,MAAM,aAAa,GAAI,MAAM,MAAM,KAAG,UAAU,EAInD,CAAC;AAEL,eAAO,MAAM,sBAAsB,EAAE,QAUpC,CAAC"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { quoteAround } from "./quote-around.js";
|
|
2
|
+
/** A word of three letters or more (a possessive 's may follow), a comma, and a lower-case word of three letters or more. */
|
|
3
|
+
const JOINED = /(?<![\p{L}\p{N}])[A-Za-z]{3,}(?:['’]s)?,[a-z]{3,}(?![\p{L}\p{N}])/u;
|
|
4
|
+
/** Characters that make a run of text an address, a path, a parameter or data (a=b, a/b, a.b, @a, #a, a_b, 3,abc). */
|
|
5
|
+
const DATA_MARK = /[/\\=@#:.&?_|+*~$%^<>{}[\]\p{N}]/u;
|
|
6
|
+
/** Opening and closing marks a word of prose may carry ("(and,then)"), and the sentence's own end ("and,then."). */
|
|
7
|
+
const OPENERS = "(\"'“‘[";
|
|
8
|
+
const CLOSERS = ")\"'”’].,;!?";
|
|
9
|
+
/** A run quoted whole, inside brackets or not ("true,false", ("true,false")): a value written out, not two words of the sentence. */
|
|
10
|
+
const QUOTED_WHOLE = /^[([]*["'“‘].*["'”’][)\].,;!?]*$/u;
|
|
11
|
+
/** A run of text between whitespace. */
|
|
12
|
+
const RUN = /\S+/gu;
|
|
13
|
+
const COMMA = ",";
|
|
14
|
+
/** The run without the marks around it, trimmed by scanning rather than by a pattern that could backtrack. */
|
|
15
|
+
const bareOf = (run) => {
|
|
16
|
+
const chars = run.split("");
|
|
17
|
+
const start = chars.findIndex((char) => !OPENERS.includes(char));
|
|
18
|
+
const end = chars.findLastIndex((char) => !CLOSERS.includes(char));
|
|
19
|
+
return start === -1 || end < start ? "" : run.slice(start, end + 1);
|
|
20
|
+
};
|
|
21
|
+
/** A run holding one comma and nothing that marks it as data. */
|
|
22
|
+
const isProseRun = (run) => {
|
|
23
|
+
const bare = bareOf(run);
|
|
24
|
+
return bare.split(COMMA).length === 2 && !DATA_MARK.test(bare) && !QUOTED_WHOLE.test(run);
|
|
25
|
+
};
|
|
26
|
+
/** Each run is read once, so a long line of values costs one pass. */
|
|
27
|
+
export const tightCommasIn = (text) => [...text.matchAll(RUN)].flatMap((run) => {
|
|
28
|
+
const joined = isProseRun(run[0]) ? JOINED.exec(run[0]) : null;
|
|
29
|
+
return joined === null ? [] : [{ offset: run.index + joined.index + joined[0].indexOf(COMMA), written: joined[0] }];
|
|
30
|
+
});
|
|
31
|
+
export const missingSpaceAfterComma = (doc) => {
|
|
32
|
+
const text = doc.prose ?? doc.source;
|
|
33
|
+
return tightCommasIn(text).map((comma) => ({
|
|
34
|
+
rule: "",
|
|
35
|
+
severity: "warning",
|
|
36
|
+
line: 0,
|
|
37
|
+
column: 0,
|
|
38
|
+
quote: quoteAround(text, comma.offset, comma.offset + 1),
|
|
39
|
+
values: { written: comma.written, offset: comma.offset },
|
|
40
|
+
}));
|
|
41
|
+
};
|
|
42
|
+
//# sourceMappingURL=missing-space-after-comma.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"missing-space-after-comma.js","sourceRoot":"","sources":["../../src/detectors/missing-space-after-comma.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAQhD,6HAA6H;AAC7H,MAAM,MAAM,GAAG,oEAAoE,CAAC;AACpF,sHAAsH;AACtH,MAAM,SAAS,GAAG,mCAAmC,CAAC;AACtD,oHAAoH;AACpH,MAAM,OAAO,GAAG,SAAS,CAAC;AAC1B,MAAM,OAAO,GAAG,cAAc,CAAC;AAC/B,qIAAqI;AACrI,MAAM,YAAY,GAAG,mCAAmC,CAAC;AACzD,wCAAwC;AACxC,MAAM,GAAG,GAAG,OAAO,CAAC;AACpB,MAAM,KAAK,GAAG,GAAG,CAAC;AAElB,8GAA8G;AAC9G,MAAM,MAAM,GAAG,CAAC,GAAW,EAAU,EAAE;IACrC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC5B,MAAM,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IACjE,MAAM,GAAG,GAAG,KAAK,CAAC,aAAa,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IACnE,OAAO,KAAK,KAAK,CAAC,CAAC,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;AACtE,CAAC,CAAC;AAEF,iEAAiE;AACjE,MAAM,UAAU,GAAG,CAAC,GAAW,EAAW,EAAE;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IACzB,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC5F,CAAC,CAAC;AAEF,sEAAsE;AACtE,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,IAAY,EAAgB,EAAE,CAC1D,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;IACtC,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/D,OAAO,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AACtH,CAAC,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,sBAAsB,GAAa,CAAC,GAAG,EAAa,EAAE;IACjE,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,CAAC;IACrC,OAAO,aAAa,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACzC,IAAI,EAAE,EAAE;QACR,QAAQ,EAAE,SAAS;QACnB,IAAI,EAAE,CAAC;QACP,MAAM,EAAE,CAAC;QACT,KAAK,EAAE,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;QACxD,MAAM,EAAE,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE;KACzD,CAAC,CAAC,CAAC;AACN,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"connective-hierarchy.d.ts","sourceRoot":"","sources":["../../../src/detectors/registry/connective-hierarchy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAGhD,eAAO,MAAM,QAAQ,EAAE,QAA8B,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"connective-hierarchy.js","sourceRoot":"","sources":["../../../src/detectors/registry/connective-hierarchy.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAEjE,MAAM,CAAC,MAAM,QAAQ,GAAa,mBAAmB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"missing-final-period.d.ts","sourceRoot":"","sources":["../../../src/detectors/registry/missing-final-period.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAGhD,eAAO,MAAM,QAAQ,EAAE,QAA6B,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"missing-final-period.js","sourceRoot":"","sources":["../../../src/detectors/registry/missing-final-period.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAEhE,MAAM,CAAC,MAAM,QAAQ,GAAa,kBAAkB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"missing-space-after-comma.d.ts","sourceRoot":"","sources":["../../../src/detectors/registry/missing-space-after-comma.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAGhD,eAAO,MAAM,QAAQ,EAAE,QAAiC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"missing-space-after-comma.js","sourceRoot":"","sources":["../../../src/detectors/registry/missing-space-after-comma.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,sBAAsB,EAAE,MAAM,iCAAiC,CAAC;AAEzE,MAAM,CAAC,MAAM,QAAQ,GAAa,sBAAsB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "chaffjs",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.25.0",
|
|
4
4
|
"description": "Prose validation harness. Finds what is hard to read, and never rewrites it for you.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"chaff",
|
|
@@ -67,7 +67,7 @@
|
|
|
67
67
|
"dependencies": {
|
|
68
68
|
"@anthropic-ai/sdk": "^0.125.0",
|
|
69
69
|
"@chaffjs/lang-en": "0.22.0",
|
|
70
|
-
"@chaffjs/lang-ja": "0.
|
|
70
|
+
"@chaffjs/lang-ja": "0.24.0",
|
|
71
71
|
"mdast-util-from-markdown": "^2.0.3",
|
|
72
72
|
"mdast-util-frontmatter": "^2.0.1",
|
|
73
73
|
"mdast-util-gfm-table": "^2.0.0",
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
id: connective-hierarchy
|
|
2
|
+
layer: L2
|
|
3
|
+
status: stable
|
|
4
|
+
how_to_find: connective-hierarchy
|
|
5
|
+
word_list: connective-hierarchy
|
|
6
|
+
extra_word_lists: [connective-lookalike]
|
|
7
|
+
requires: [pos]
|
|
8
|
+
languages: [ja]
|
|
9
|
+
severity: warning
|
|
10
|
+
|
|
11
|
+
name:
|
|
12
|
+
ja: 内の段の無い「並びに」「若しくは」
|
|
13
|
+
en: 並びに or 若しくは with no inner level
|
|
14
|
+
|
|
15
|
+
why:
|
|
16
|
+
ja: 法令と公用文では、並べる語を段で使い分けます。一段の並べ方は「及び」「又は」で書き、二段に重ねたときだけ、外の段を「並びに」、内の段を「若しくは」で書きます(A及びB並びにC、A若しくはB又はC)。「及び」の無い文の「並びに」と、「又は」の無い文の「若しくは」は、段の無い並べ方に外の段の語を使っていて、読み手は内の段を探して止まります。
|
|
17
|
+
en: Japanese statutes and official writing join lists by level. A one-level list uses 及び (and) or 又は (or); only a two-level list uses 並びに for the outer "and" (A及びB並びにC) and 若しくは for the inner "or" (A若しくはB又はC). 並びに in a sentence with no 及び, or 若しくは in one with no 又は, uses the word for a level the sentence does not have, and the reader looks for the other level.
|
|
18
|
+
|
|
19
|
+
message:
|
|
20
|
+
ja: 「{written}」の文に、段の組になる語がありません(一段なら「{usual}」)
|
|
21
|
+
en: '"{written}" with no other level of the list in the sentence (a one-level list uses "{usual}")'
|
|
22
|
+
|
|
23
|
+
how_to_fix:
|
|
24
|
+
ja: 一段の並べ方なら「並びに」を「及び」に、「若しくは」を「又は」に替えてください。二段に重ねるつもりなら、内の段の語を書き足してください。
|
|
25
|
+
en: For a one-level list, write 及び for 並びに and 又は for 若しくは. If the list has two levels, write the other level's word too.
|
|
26
|
+
|
|
27
|
+
levels: { strict: error, normal: warning, relaxed: info }
|
|
28
|
+
|
|
29
|
+
use_for: [business, technical, blog, legal, docs, academic, literature, speech]
|
|
30
|
+
off_for:
|
|
31
|
+
literature: Older prose uses 並びに and 若しくは as plain literary connectives, not by the official levels.
|
|
32
|
+
speech: Spoken Japanese, as in Diet and press transcripts, says 並びに and 若しくは for a single "and" and "or".
|
|
33
|
+
|
|
34
|
+
group: wording
|
|
35
|
+
summary:
|
|
36
|
+
ja: 「及び」の無い文の「並びに」、「又は」の無い文の「若しくは」
|
|
37
|
+
en: 並びに in a sentence with no 及び, or 若しくは in one with no 又は
|
|
38
|
+
example:
|
|
39
|
+
ja:
|
|
40
|
+
before: |
|
|
41
|
+
申請書並びに添付書類を提出する。
|
|
42
|
+
after: |
|
|
43
|
+
申請書及び添付書類を提出する。
|
|
44
|
+
|
|
45
|
+
not_flagged:
|
|
46
|
+
ja: 日本語の文書だけで動きます。仮名の「ならびに」「もしくは」は、普通の文で一段の並べ方にも使われるので見ません。「及び」「および」「又は」「または」が同じ文にあれば、段の組を作ったと読みます。「」『』の中は、引いた語句の片方だけなので見ません。名詞の後ろで名詞をつなぐときだけ見ます(文字の並びに、正しい並びに、縦並びには見ません)。
|
|
47
|
+
en: Only in Japanese documents. The kana forms ならびに and もしくは, which plain writing also uses for a one-level list, are not read. A sentence holding 及び, および, 又は or または is read as having the other level. A phrase inside 「」 or 『』 quotes half of a list and is not read. Only a word joining two nouns is read (文字の並びに, 正しい並びに and 縦並びに, the noun 並び with に, are not).
|
|
48
|
+
sources: [koyobun-2022]
|
|
49
|
+
rewrite:
|
|
50
|
+
depth: light
|
|
51
|
+
ja:
|
|
52
|
+
direction: 一段の並べ方なら、「並びに」を「及び」に、「若しくは」を「又は」に替えます。並べた語と順は変えません。
|
|
53
|
+
pairs:
|
|
54
|
+
- before: 申請書並びに添付書類を提出する。
|
|
55
|
+
after: 申請書及び添付書類を提出する。
|
|
56
|
+
- before: 書面若しくは電磁的方法により通知する。
|
|
57
|
+
after: 書面又は電磁的方法により通知する。
|
|
58
|
+
keep:
|
|
59
|
+
- 並べた語とその順
|
|
60
|
+
avoid:
|
|
61
|
+
- 文に無い内の段を書き足す
|
|
62
|
+
- 仮名の「および」「または」に書き換える
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
id: missing-final-period
|
|
2
|
+
layer: L2
|
|
3
|
+
status: stable
|
|
4
|
+
how_to_find: missing-final-period
|
|
5
|
+
requires: [pos]
|
|
6
|
+
languages: [ja]
|
|
7
|
+
severity: info
|
|
8
|
+
|
|
9
|
+
name:
|
|
10
|
+
ja: 「。」で終わらない段落
|
|
11
|
+
en: A paragraph ending without 。
|
|
12
|
+
|
|
13
|
+
why:
|
|
14
|
+
ja: ほかの段落が「。」で終わる文書で、一つの段落だけが「〜しました」のように句点無しで終わると、書きかけで止まったか、貼り合わせた所に見えます。どちらの書き方が正しいかは決めず、文書の中で少ないほうを指します。見出しのような短い行、「:」で次へ続く行、注の行、表のように空白で並べた行、名詞で終わる行(体言止めの項目)は、文ではないので数えません。
|
|
15
|
+
en: In a Japanese document whose paragraphs end with 。, one paragraph that stops without it (〜しました) reads as unfinished, or as pasted in from elsewhere. chaff does not pick a side; it points at the paragraphs that differ from most of the document. Short lines such as captions, lines leading into a list with :, notes, lines laid out with runs of spaces, and lines ending in a noun (an item in noun style) are not sentences and are not counted.
|
|
16
|
+
|
|
17
|
+
message:
|
|
18
|
+
ja: この段落は「。」で終わっていません(この文書の段落はふつう「。」で終わります。{of} 段落のうち {count} 段落)
|
|
19
|
+
en: "This paragraph ends without 。, where the document's paragraphs usually end with it ({count} of {of})"
|
|
20
|
+
|
|
21
|
+
how_to_fix:
|
|
22
|
+
ja: 段落の終わりに「。」を付けてください。
|
|
23
|
+
en: Add 。 at the end of the paragraph.
|
|
24
|
+
|
|
25
|
+
# 少ないほうが、数えた段落の何パーセントまでなら「混ざっている」と見るか。これを超えたら、文書が使い分けている。
|
|
26
|
+
levels: { strict: 34, normal: 25, relaxed: 15 }
|
|
27
|
+
|
|
28
|
+
use_for: [business, technical, blog, legal, docs, academic, literature, speech]
|
|
29
|
+
off_for:
|
|
30
|
+
literature: Verse and dialogue end lines without 。 on purpose.
|
|
31
|
+
speech: A transcript breaks where the speaker stops, not where a written sentence would end.
|
|
32
|
+
|
|
33
|
+
group: consistency
|
|
34
|
+
summary:
|
|
35
|
+
ja: 「。」で終わる段落の多い文書で、「。」無しで終わった段落
|
|
36
|
+
en: A paragraph ending without 。 in a document whose other paragraphs end with it
|
|
37
|
+
example:
|
|
38
|
+
ja:
|
|
39
|
+
before: |
|
|
40
|
+
申請は市役所の窓口で受け付けます。
|
|
41
|
+
|
|
42
|
+
受付の時間は平日の午前九時から午後五時までです。
|
|
43
|
+
|
|
44
|
+
提出した書類は二週間ほどで審査が終わります。
|
|
45
|
+
|
|
46
|
+
審査の結果は郵送でお知らせしました
|
|
47
|
+
after: |
|
|
48
|
+
申請は市役所の窓口で受け付けます。
|
|
49
|
+
|
|
50
|
+
受付の時間は平日の午前九時から午後五時までです。
|
|
51
|
+
|
|
52
|
+
提出した書類は二週間ほどで審査が終わります。
|
|
53
|
+
|
|
54
|
+
審査の結果は郵送でお知らせしました。
|
|
55
|
+
|
|
56
|
+
not_flagged:
|
|
57
|
+
ja: 日本語の文書だけで動きます。十五字に満たない行、「:」で終わる行、※ や * で始まる注の行、空白を二つ続けて並べた行、名詞や連用形で終わる行(受付時間の案内、可能性あり)は文として数えません。閉じ括弧で終わる行(〜をご覧ください)と、「?」「!」で終わる行も数えません。段落の多くが「。」で終わらない文書(スライド、一覧)は指しません。
|
|
58
|
+
en: Only in Japanese documents. Lines under fifteen characters, lines ending in :, note lines starting with ※ or *, lines laid out with two spaces in a row, and lines ending in a noun or a continuative form are not counted as sentences, nor are lines ending in a closing bracket, ? or !. A document whose paragraphs mostly end without 。 (slides, lists) is not reported.
|
|
59
|
+
level_meaning:
|
|
60
|
+
ja: 「。」の無い段落が {limit}% までなら指摘
|
|
61
|
+
en: reported while paragraphs without 。 are at most {limit}% of the counted paragraphs
|
|
62
|
+
sources: [textlint-ja-presets]
|
|
63
|
+
rewrite:
|
|
64
|
+
depth: light
|
|
65
|
+
ja:
|
|
66
|
+
direction: 指した段落の終わりに「。」を一つ付けます。文は変えません。
|
|
67
|
+
pairs:
|
|
68
|
+
- before: 審査の結果は郵送でお知らせしました
|
|
69
|
+
after: 審査の結果は郵送でお知らせしました。
|
|
70
|
+
- before: 詳しい手順は担当者から説明します
|
|
71
|
+
after: 詳しい手順は担当者から説明します。
|
|
72
|
+
keep:
|
|
73
|
+
- 段落の言葉と順
|
|
74
|
+
avoid:
|
|
75
|
+
- 文を書き換える
|
|
76
|
+
- 名詞で終わる項目にまで「。」を付ける
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
id: missing-space-after-comma
|
|
2
|
+
layer: L1
|
|
3
|
+
status: stable
|
|
4
|
+
how_to_find: missing-space-after-comma
|
|
5
|
+
languages: [en]
|
|
6
|
+
severity: warning
|
|
7
|
+
|
|
8
|
+
name:
|
|
9
|
+
ja: カンマの後ろに空白が無い
|
|
10
|
+
en: No space after a comma
|
|
11
|
+
|
|
12
|
+
why:
|
|
13
|
+
ja: 英文では、語をつなぐカンマの後ろに空白を一つ置きます。「address,and」のように空白が無いと、二つの語が一語に見え、検索でも別の語として当たりません。急いで打ったときや、書き換えの途中で空白が落ちた跡です。どちらの書き方を選ぶかの話ではなく、英語の句読点の決まりです。
|
|
14
|
+
en: 'English puts one space after a comma between words. Without it ("address,and") the two words read as one, and a search does not find either. It is a space dropped while typing or editing, not a matter of style.'
|
|
15
|
+
|
|
16
|
+
message:
|
|
17
|
+
ja: 「{written}」のカンマの後ろに空白がありません
|
|
18
|
+
en: 'No space after the comma in "{written}"'
|
|
19
|
+
|
|
20
|
+
how_to_fix:
|
|
21
|
+
ja: カンマの後ろに空白を一つ入れてください。
|
|
22
|
+
en: Put one space after the comma.
|
|
23
|
+
|
|
24
|
+
levels: { strict: error, normal: warning, relaxed: info }
|
|
25
|
+
|
|
26
|
+
use_for: [business, technical, blog, legal, docs, academic, literature, speech]
|
|
27
|
+
off_for:
|
|
28
|
+
literature: Older printed texts and poetry set commas tight on purpose, and a transcribed edition keeps the printer's spacing.
|
|
29
|
+
|
|
30
|
+
group: slips
|
|
31
|
+
summary:
|
|
32
|
+
ja: 英文の「address,and」のように、語をつなぐカンマの後ろに空白が無い所
|
|
33
|
+
en: A comma between two words with no space after it, as in "address,and"
|
|
34
|
+
example:
|
|
35
|
+
en:
|
|
36
|
+
before: |
|
|
37
|
+
Send the form to the office address,and keep a copy.
|
|
38
|
+
after: |
|
|
39
|
+
Send the form to the office address, and keep a copy.
|
|
40
|
+
|
|
41
|
+
not_flagged:
|
|
42
|
+
ja: 英語の文書だけで動きます。値を並べたもの(blue,black,brown のようにカンマが二つ以上)、数字や記号を含むもの(3,000、a,b=c、path/a,b、user@x,y、file_name,size)、引用符で囲んだ値("true,false")、二字以下の語(id,x)、コードと URL は見ません。
|
|
43
|
+
en: 'Only in English documents. A run of values (two or more commas, as in blue,black,brown), a run holding a digit or a mark (3,000, a,b=c, path/a,b, user@x,y, file_name,size), a quoted value ("true,false"), words of one or two letters (id,x), code and URLs are not read.'
|
|
44
|
+
sources: [vale-styles]
|
|
45
|
+
rewrite:
|
|
46
|
+
depth: light
|
|
47
|
+
en:
|
|
48
|
+
direction: Put one space after the marked comma. Change nothing else.
|
|
49
|
+
pairs:
|
|
50
|
+
- before: Send it to the office address,and keep a copy.
|
|
51
|
+
after: Send it to the office address, and keep a copy.
|
|
52
|
+
- before: If the box is damaged,call us first.
|
|
53
|
+
after: If the box is damaged, call us first.
|
|
54
|
+
keep:
|
|
55
|
+
- the words and their order
|
|
56
|
+
avoid:
|
|
57
|
+
- adding spaces inside lists of values or code
|
package/skills/chaff/SKILL.md
CHANGED
|
@@ -96,9 +96,26 @@ Pick the easiest way that can say the requirement:
|
|
|
96
96
|
| Requirement | Use |
|
|
97
97
|
| --- | --- |
|
|
98
98
|
| An existing rule covers it | A level, a number, `prefer`, `jargon` or `required_sections` in `chaff.yaml` |
|
|
99
|
-
| A fixed phrase or pattern, with the team's own message | `custom_rules` of type `words` or `pattern`
|
|
100
|
-
| Decided by part of speech or inflection | `custom_rules` of type `tokens`
|
|
101
|
-
| Counting or comparing that the above cannot express | a Node function, `type: module`
|
|
99
|
+
| A fixed phrase or pattern, with the team's own message | `custom_rules` of type `words` or `pattern` |
|
|
100
|
+
| Decided by part of speech or inflection | `custom_rules` of type `tokens` |
|
|
101
|
+
| Counting or comparing that the above cannot express | a Node function, `type: module` |
|
|
102
|
+
| The same rules shared across repositories | a rule pack of YAML files: `npx chaffjs init --plugin <name>`, then `plugins:` in `chaff.yaml` |
|
|
103
|
+
|
|
104
|
+
A `words` rule names the words, the team's text for each field `explain` shows, and one example:
|
|
105
|
+
|
|
106
|
+
```yaml
|
|
107
|
+
custom_rules:
|
|
108
|
+
- id: banned-word
|
|
109
|
+
type: words
|
|
110
|
+
words: [禁止ワード]
|
|
111
|
+
severity: warning
|
|
112
|
+
name: { ja: 禁止ワード }
|
|
113
|
+
why: { ja: この文書で避ける語です }
|
|
114
|
+
message: { ja: "「{matched}」は使いません" }
|
|
115
|
+
how_to_fix: { ja: 別の言い方にします }
|
|
116
|
+
example:
|
|
117
|
+
ja: { before: 禁止ワードを使った文, after: 別の言い方の文 }
|
|
118
|
+
```
|
|
102
119
|
|
|
103
120
|
Test a new team rule the same way: `explain`, then a sample it must report and a sample it must not, before
|
|
104
121
|
committing `chaff.yaml`. The guide page "Adding a rule" covers each way and how to add a rule to chaff itself.
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import type { Detector, Finding, LexiconEntry, Sentence, Span, Token } from "../plugin.ts";
|
|
2
|
+
import { escapeRegExp } from "../orthography.ts";
|
|
3
|
+
import { quotedSpans } from "../quoted-span.ts";
|
|
4
|
+
|
|
5
|
+
// A connective that joins only the outer level of a two-level list, in a sentence with no inner level under it. The
|
|
6
|
+
// lexicon holds both levels: an entry with a rewrite is an outer-level word (rewrite is the word a one-level list uses),
|
|
7
|
+
// and an entry with instead_of is an inner-level word that stands under that outer-level word.
|
|
8
|
+
|
|
9
|
+
/** One outer-level connective with nothing to stand above, and where it is in the document. */
|
|
10
|
+
export type LoneConnective = { readonly offset: number; readonly written: string; readonly usual: string };
|
|
11
|
+
|
|
12
|
+
const CONJUNCTION = "CCONJ";
|
|
13
|
+
|
|
14
|
+
/** The word lists the detector reads: the connectives, and the phrases that hold one's letters without being one (歯並びに). */
|
|
15
|
+
export type ConnectiveLists = { readonly connectives: readonly LexiconEntry[]; readonly lookalikes: readonly LexiconEntry[] };
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* The tagger does not always tell a connective from a noun and a particle: it reads 申請書並びに as 並び + に and
|
|
19
|
+
* 文字の並びに as the conjunction. After の, a determiner or an adjective (の並びに, この並びに, 正しい並びに) the word is
|
|
20
|
+
* a noun of its own. Where the tagger splits it, it is a connective only between two nouns: a noun, a number or a mark
|
|
21
|
+
* before it, and no verb right after it (一列並びに座る).
|
|
22
|
+
*/
|
|
23
|
+
const MODIFIES_NOUN: ReadonlySet<string> = new Set(["DET", "ADJ"]);
|
|
24
|
+
const GENITIVE = "の";
|
|
25
|
+
const JOINS_AFTER: ReadonlySet<string> = new Set(["NOUN", "PROPN", "NUM", "PUNCT", "SYM"]);
|
|
26
|
+
const ENDS_CLAUSE: ReadonlySet<string> = new Set(["VERB", "AUX"]);
|
|
27
|
+
|
|
28
|
+
const modifiesNoun = (token: Token | undefined): boolean =>
|
|
29
|
+
token !== undefined && (MODIFIES_NOUN.has(token.pos) || (token.pos === "ADP" && token.surface === GENITIVE));
|
|
30
|
+
|
|
31
|
+
const joinsNouns = (previous: Token | undefined, next: Token | undefined): boolean =>
|
|
32
|
+
previous !== undefined && JOINS_AFTER.has(previous.pos) && next !== undefined && !ENDS_CLAUSE.has(next.pos);
|
|
33
|
+
|
|
34
|
+
/** A sentence's tokens by where they start and end, and the letters a connective there would not be read in. */
|
|
35
|
+
type Reading = { readonly byStart: ReadonlyMap<number, Token>; readonly byEnd: ReadonlyMap<number, Token>; readonly hidden: readonly boolean[] };
|
|
36
|
+
|
|
37
|
+
const placesOf = (text: string, word: string): number[] => [...text.matchAll(new RegExp(escapeRegExp(word), "gu"))].map((match) => match.index);
|
|
38
|
+
|
|
39
|
+
/** Inside 「」 or 『』 (a quoted phrase holds half a list) or inside a lookalike (歯並びに), by the sentence's own offsets. */
|
|
40
|
+
const hiddenLetters = (text: string, lookalikes: readonly LexiconEntry[]): boolean[] => {
|
|
41
|
+
const hidden = Array.from({ length: text.length }, () => false);
|
|
42
|
+
const hide = (start: number, end: number): boolean[] => hidden.fill(true, start, end);
|
|
43
|
+
quotedSpans(text).forEach((span) => hide(span.start, span.end));
|
|
44
|
+
lookalikes.forEach(({ pattern }) => placesOf(text, pattern).forEach((at) => hide(at, at + pattern.length)));
|
|
45
|
+
return hidden;
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
const readingOf = (sentence: Sentence, lookalikes: readonly LexiconEntry[]): Reading => {
|
|
49
|
+
const tokens = sentence.tokens ?? [];
|
|
50
|
+
return {
|
|
51
|
+
byStart: new Map(tokens.map((token) => [token.span.start, token])),
|
|
52
|
+
byEnd: new Map(tokens.map((token) => [token.span.end, token])),
|
|
53
|
+
hidden: hiddenLetters(sentence.text, lookalikes),
|
|
54
|
+
};
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
const standsAsConnective = (reading: Reading, span: Span): boolean => {
|
|
58
|
+
const previous = reading.byEnd.get(span.start);
|
|
59
|
+
if (modifiesNoun(previous)) return false;
|
|
60
|
+
const own = reading.byStart.get(span.start);
|
|
61
|
+
const tagged = own !== undefined && own.pos === CONJUNCTION && own.span.end === span.end;
|
|
62
|
+
return tagged || joinsNouns(previous, reading.byStart.get(span.end));
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
const hasInnerLevel = (sentence: Sentence, outer: LexiconEntry, connectives: readonly LexiconEntry[]): boolean =>
|
|
66
|
+
connectives.some((entry) => entry.instead_of === outer.pattern && sentence.text.includes(entry.pattern));
|
|
67
|
+
|
|
68
|
+
/** Where one outer-level word stands as a connective in the sentence. */
|
|
69
|
+
const connectiveSpans = (sentence: Sentence, outer: LexiconEntry, reading: Reading): Span[] =>
|
|
70
|
+
placesOf(sentence.text, outer.pattern)
|
|
71
|
+
.filter((at) => reading.hidden[at] !== true)
|
|
72
|
+
.map((at) => ({ start: sentence.span.start + at, end: sentence.span.start + at + outer.pattern.length }))
|
|
73
|
+
.filter((span) => standsAsConnective(reading, span));
|
|
74
|
+
|
|
75
|
+
/** The outer-level connectives of a sentence that stand over no inner level. */
|
|
76
|
+
export const loneConnectivesIn = (sentence: Sentence, lists: ConnectiveLists): LoneConnective[] => {
|
|
77
|
+
const lone = lists.connectives.filter(
|
|
78
|
+
(entry) => entry.rewrite !== undefined && sentence.text.includes(entry.pattern) && !hasInnerLevel(sentence, entry, lists.connectives),
|
|
79
|
+
);
|
|
80
|
+
if (lone.length === 0) return [];
|
|
81
|
+
const reading = readingOf(sentence, lists.lookalikes);
|
|
82
|
+
return lone
|
|
83
|
+
.flatMap((entry) => connectiveSpans(sentence, entry, reading).map((span) => ({ offset: span.start, written: entry.pattern, usual: entry.rewrite ?? "" })))
|
|
84
|
+
.sort((a, b) => a.offset - b.offset);
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
export const connectiveHierarchy: Detector = (doc, options): Finding[] => {
|
|
88
|
+
const lists = { connectives: options.lexicon ?? [], lookalikes: doc.lexicons["connective-lookalike"] ?? [] };
|
|
89
|
+
return doc.sentences.flatMap((sentence) =>
|
|
90
|
+
loneConnectivesIn(sentence, lists).map((lone) => ({
|
|
91
|
+
rule: "",
|
|
92
|
+
severity: "info",
|
|
93
|
+
line: 0,
|
|
94
|
+
column: 0,
|
|
95
|
+
quote: sentence.text.trim(),
|
|
96
|
+
values: { written: lone.written, usual: lone.usual, offset: lone.offset },
|
|
97
|
+
})),
|
|
98
|
+
);
|
|
99
|
+
};
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import type { Detector, Finding, LexiconEntry, Paragraph, ProseDocument, Token } from "../plugin.ts";
|
|
2
|
+
import { formMinority } from "./form-minority.ts";
|
|
3
|
+
|
|
4
|
+
// A paragraph whose last sentence stops without its closing mark, in a document whose other paragraphs end with one. It
|
|
5
|
+
// takes no side: a document that ends none of its paragraphs with the mark (slides, labels) is not reported. The marks
|
|
6
|
+
// come from the language's lexicon "final-period": group "end" is the period, group "other-end" closes a sentence another
|
|
7
|
+
// way (?, !) so the paragraph is not counted, and group "note" opens a note line. The
|
|
8
|
+
// lexicon is read directly rather than as the rule's word list, since its marks are what a fixed paragraph ends with.
|
|
9
|
+
|
|
10
|
+
/** A paragraph's end, and whether it closes with a sentence-ending mark. form is "closed" or "open". */
|
|
11
|
+
export type ParagraphEnd = { readonly offset: number; readonly form: string; readonly text: string };
|
|
12
|
+
|
|
13
|
+
/** The marks the detector reads, from the lexicon. */
|
|
14
|
+
export type PeriodMarks = { readonly ends: readonly string[]; readonly otherEnds: readonly string[]; readonly notes: readonly string[] };
|
|
15
|
+
|
|
16
|
+
/** Brackets and quotes that may follow a closing mark (〜です。」). */
|
|
17
|
+
const CLOSERS = '」』))"”’';
|
|
18
|
+
/** A line that is itself a quotation closes like a sentence. */
|
|
19
|
+
const QUOTE_CLOSERS = "」』";
|
|
20
|
+
/** A paragraph ending in a colon leads into a list or a block; it is not a sentence left open. */
|
|
21
|
+
const LEAD_IN = /[::]$/u;
|
|
22
|
+
/** Two spaces in a row, half- or full-width, lay a line out like a table (an ingredient, then its amount), so it is not a sentence. */
|
|
23
|
+
const LAYOUT = /[ \u3000]{2}/u;
|
|
24
|
+
/** Fewer characters than this is a label or a caption, not a sentence. */
|
|
25
|
+
const MIN_SENTENCE_CHARS = 15;
|
|
26
|
+
/** Words that end a clause: a verb, an adjective or an auxiliary in its dictionary form ends the sentence. */
|
|
27
|
+
const PREDICATE: ReadonlySet<string> = new Set(["VERB", "ADJ", "AUX"]);
|
|
28
|
+
|
|
29
|
+
/** The line's last mark before any closing brackets and quotes is one of these. */
|
|
30
|
+
const endsWithMark = (line: string, marks: readonly string[]): boolean => {
|
|
31
|
+
const chars = Array.from(line);
|
|
32
|
+
return marks.includes(chars[chars.findLastIndex((char) => !CLOSERS.includes(char))] ?? "");
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
const isClosed = (line: string, ends: readonly string[]): boolean => endsWithMark(line, ends) || QUOTE_CLOSERS.includes(line.at(-1) ?? "\0");
|
|
36
|
+
|
|
37
|
+
/** A note, a laid-out row, or a question or exclamation: not a sentence that could take the period. */
|
|
38
|
+
const isAside = (line: string, marks: PeriodMarks): boolean =>
|
|
39
|
+
marks.notes.some((note) => line.startsWith(note)) || LAYOUT.test(line) || endsWithMark(line, marks.otherEnds);
|
|
40
|
+
|
|
41
|
+
/** The paragraph's last word ends a sentence: a predicate written in its dictionary form (〜します, 〜である, 望ましい). */
|
|
42
|
+
const endsInPredicate = (paragraph: Paragraph): boolean => {
|
|
43
|
+
const last = (paragraph.sentences.at(-1)?.tokens ?? []).filter((token: Token) => token.surface.trim() !== "").at(-1);
|
|
44
|
+
return last !== undefined && PREDICATE.has(last.pos) && last.lemma === last.surface;
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
/** How a paragraph ends, or undefined when its last line is not a sentence (a label, a lead-in, a note, a laid-out row). */
|
|
48
|
+
export const paragraphEnd = (text: string, paragraph: Paragraph, marks: PeriodMarks): ParagraphEnd | undefined => {
|
|
49
|
+
const body = text.slice(paragraph.span.start, paragraph.span.end).trimEnd();
|
|
50
|
+
const last = body.split("\n").at(-1)?.trim() ?? "";
|
|
51
|
+
if (Array.from(last).length < MIN_SENTENCE_CHARS || LEAD_IN.test(last)) return undefined;
|
|
52
|
+
const offset = paragraph.span.start + body.length - 1;
|
|
53
|
+
if (isAside(last, marks)) return undefined;
|
|
54
|
+
if (isClosed(last, marks.ends)) return { offset, form: "closed", text: last };
|
|
55
|
+
// A closing parenthetical after the sentence (〜します(別紙を参照する)) is not where the period goes.
|
|
56
|
+
return !CLOSERS.includes(last.at(-1) ?? "\0") && endsInPredicate(paragraph) ? { offset, form: "open", text: last } : undefined;
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
const marksOf = (lexicon: readonly LexiconEntry[]): PeriodMarks => ({
|
|
60
|
+
ends: lexicon.filter((entry) => entry.group === "end").map((entry) => entry.pattern),
|
|
61
|
+
otherEnds: lexicon.filter((entry) => entry.group === "other-end").map((entry) => entry.pattern),
|
|
62
|
+
notes: lexicon.filter((entry) => entry.group === "note").map((entry) => entry.pattern),
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
export const missingFinalPeriod: Detector = (doc: ProseDocument, options): Finding[] => {
|
|
66
|
+
const text = doc.prose ?? doc.source;
|
|
67
|
+
const marks = marksOf(doc.lexicons["final-period"] ?? []);
|
|
68
|
+
const ends = doc.paragraphs.flatMap((paragraph) => paragraphEnd(text, paragraph, marks) ?? []);
|
|
69
|
+
const minority = formMinority(ends, options.limit);
|
|
70
|
+
if (minority === undefined || minority.usual.form !== "closed") return [];
|
|
71
|
+
return minority.odd.map((end) => ({
|
|
72
|
+
rule: "",
|
|
73
|
+
severity: "info",
|
|
74
|
+
line: 0,
|
|
75
|
+
column: 0,
|
|
76
|
+
quote: end.text,
|
|
77
|
+
values: { count: minority.odd.length, of: ends.length, offset: end.offset },
|
|
78
|
+
}));
|
|
79
|
+
};
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import type { Detector, Finding } from "../plugin.ts";
|
|
2
|
+
import { quoteAround } from "./quote-around.ts";
|
|
3
|
+
|
|
4
|
+
// A comma between two words with no space after it ("address,and"): a slip in English prose. A run of values joined by
|
|
5
|
+
// commas ("blue,black,brown"), and anything inside an address, a path or an assignment, is data, not prose.
|
|
6
|
+
|
|
7
|
+
/** One comma with no space after it, where it is, and the two words it joins as written. */
|
|
8
|
+
export type TightComma = { readonly offset: number; readonly written: string };
|
|
9
|
+
|
|
10
|
+
/** A word of three letters or more (a possessive 's may follow), a comma, and a lower-case word of three letters or more. */
|
|
11
|
+
const JOINED = /(?<![\p{L}\p{N}])[A-Za-z]{3,}(?:['’]s)?,[a-z]{3,}(?![\p{L}\p{N}])/u;
|
|
12
|
+
/** Characters that make a run of text an address, a path, a parameter or data (a=b, a/b, a.b, @a, #a, a_b, 3,abc). */
|
|
13
|
+
const DATA_MARK = /[/\\=@#:.&?_|+*~$%^<>{}[\]\p{N}]/u;
|
|
14
|
+
/** Opening and closing marks a word of prose may carry ("(and,then)"), and the sentence's own end ("and,then."). */
|
|
15
|
+
const OPENERS = "(\"'“‘[";
|
|
16
|
+
const CLOSERS = ")\"'”’].,;!?";
|
|
17
|
+
/** A run quoted whole, inside brackets or not ("true,false", ("true,false")): a value written out, not two words of the sentence. */
|
|
18
|
+
const QUOTED_WHOLE = /^[([]*["'“‘].*["'”’][)\].,;!?]*$/u;
|
|
19
|
+
/** A run of text between whitespace. */
|
|
20
|
+
const RUN = /\S+/gu;
|
|
21
|
+
const COMMA = ",";
|
|
22
|
+
|
|
23
|
+
/** The run without the marks around it, trimmed by scanning rather than by a pattern that could backtrack. */
|
|
24
|
+
const bareOf = (run: string): string => {
|
|
25
|
+
const chars = run.split("");
|
|
26
|
+
const start = chars.findIndex((char) => !OPENERS.includes(char));
|
|
27
|
+
const end = chars.findLastIndex((char) => !CLOSERS.includes(char));
|
|
28
|
+
return start === -1 || end < start ? "" : run.slice(start, end + 1);
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
/** A run holding one comma and nothing that marks it as data. */
|
|
32
|
+
const isProseRun = (run: string): boolean => {
|
|
33
|
+
const bare = bareOf(run);
|
|
34
|
+
return bare.split(COMMA).length === 2 && !DATA_MARK.test(bare) && !QUOTED_WHOLE.test(run);
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
/** Each run is read once, so a long line of values costs one pass. */
|
|
38
|
+
export const tightCommasIn = (text: string): TightComma[] =>
|
|
39
|
+
[...text.matchAll(RUN)].flatMap((run) => {
|
|
40
|
+
const joined = isProseRun(run[0]) ? JOINED.exec(run[0]) : null;
|
|
41
|
+
return joined === null ? [] : [{ offset: run.index + joined.index + joined[0].indexOf(COMMA), written: joined[0] }];
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
export const missingSpaceAfterComma: Detector = (doc): Finding[] => {
|
|
45
|
+
const text = doc.prose ?? doc.source;
|
|
46
|
+
return tightCommasIn(text).map((comma) => ({
|
|
47
|
+
rule: "",
|
|
48
|
+
severity: "warning",
|
|
49
|
+
line: 0,
|
|
50
|
+
column: 0,
|
|
51
|
+
quote: quoteAround(text, comma.offset, comma.offset + 1),
|
|
52
|
+
values: { written: comma.written, offset: comma.offset },
|
|
53
|
+
}));
|
|
54
|
+
};
|