chaffjs 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter-load.d.ts.map +1 -1
- package/dist/adapter-load.js +18 -2
- package/dist/adapter-load.js.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +6 -0
- package/dist/cli.js.map +1 -1
- package/dist/commands/cite.d.ts +16 -0
- package/dist/commands/cite.d.ts.map +1 -0
- package/dist/commands/cite.js +69 -0
- package/dist/commands/cite.js.map +1 -0
- package/dist/commands/tree.d.ts +33 -0
- package/dist/commands/tree.d.ts.map +1 -0
- package/dist/commands/tree.js +76 -0
- package/dist/commands/tree.js.map +1 -0
- package/dist/detectors/index.d.ts.map +1 -1
- package/dist/detectors/index.js +4 -0
- package/dist/detectors/index.js.map +1 -1
- package/dist/detectors/signals.d.ts.map +1 -1
- package/dist/detectors/signals.js +6 -1
- package/dist/detectors/signals.js.map +1 -1
- package/dist/detectors/structure-tree.d.ts +5 -0
- package/dist/detectors/structure-tree.d.ts.map +1 -0
- package/dist/detectors/structure-tree.js +25 -0
- package/dist/detectors/structure-tree.js.map +1 -0
- package/dist/document.d.ts +9 -0
- package/dist/document.d.ts.map +1 -1
- package/dist/document.js +21 -0
- package/dist/document.js.map +1 -1
- package/dist/eval.d.ts.map +1 -1
- package/dist/eval.js +13 -2
- package/dist/eval.js.map +1 -1
- package/dist/plugin.d.ts +71 -0
- package/dist/plugin.d.ts.map +1 -1
- package/dist/position.d.ts.map +1 -1
- package/dist/position.js +21 -1
- package/dist/position.js.map +1 -1
- package/dist/run.d.ts.map +1 -1
- package/dist/run.js +7 -2
- package/dist/run.js.map +1 -1
- package/dist/sentence-shape.d.ts.map +1 -1
- package/dist/sentence-shape.js +18 -1
- package/dist/sentence-shape.js.map +1 -1
- package/dist/structure/build.d.ts +20 -0
- package/dist/structure/build.d.ts.map +1 -0
- package/dist/structure/build.js +162 -0
- package/dist/structure/build.js.map +1 -0
- package/dist/structure/cite.d.ts +21 -0
- package/dist/structure/cite.d.ts.map +1 -0
- package/dist/structure/cite.js +91 -0
- package/dist/structure/cite.js.map +1 -0
- package/dist/structure/issues.d.ts +15 -0
- package/dist/structure/issues.d.ts.map +1 -0
- package/dist/structure/issues.js +53 -0
- package/dist/structure/issues.js.map +1 -0
- package/dist/structure/lines.d.ts +15 -0
- package/dist/structure/lines.d.ts.map +1 -0
- package/dist/structure/lines.js +33 -0
- package/dist/structure/lines.js.map +1 -0
- package/dist/structure/markdown-path.d.ts +3 -0
- package/dist/structure/markdown-path.d.ts.map +1 -0
- package/dist/structure/markdown-path.js +5 -0
- package/dist/structure/markdown-path.js.map +1 -0
- package/dist/structure/numbering.d.ts +9 -0
- package/dist/structure/numbering.d.ts.map +1 -0
- package/dist/structure/numbering.js +38 -0
- package/dist/structure/numbering.js.map +1 -0
- package/dist/structure/of.d.ts +13 -0
- package/dist/structure/of.d.ts.map +1 -0
- package/dist/structure/of.js +8 -0
- package/dist/structure/of.js.map +1 -0
- package/dist/structure/sexp.d.ts +7 -0
- package/dist/structure/sexp.d.ts.map +1 -0
- package/dist/structure/sexp.js +25 -0
- package/dist/structure/sexp.js.map +1 -0
- package/dist/structure/universal.d.ts +7 -0
- package/dist/structure/universal.d.ts.map +1 -0
- package/dist/structure/universal.js +21 -0
- package/dist/structure/universal.js.map +1 -0
- package/package.json +3 -3
- package/rules/concrete-evidence-density.yaml +2 -2
- package/rules/dangling-reference.yaml +27 -0
- package/rules/duplicate-definition.yaml +26 -0
- package/rules/numbering-gap.yaml +26 -0
- package/src/adapter-load.ts +19 -2
- package/src/cli.ts +6 -0
- package/src/commands/cite.ts +74 -0
- package/src/commands/tree.ts +94 -0
- package/src/detectors/index.ts +4 -0
- package/src/detectors/signals.ts +6 -1
- package/src/detectors/structure-tree.ts +32 -0
- package/src/document.ts +26 -2
- package/src/eval.ts +11 -2
- package/src/plugin.ts +75 -0
- package/src/position.ts +21 -1
- package/src/run.ts +7 -2
- package/src/sentence-shape.ts +19 -1
- package/src/structure/build.ts +203 -0
- package/src/structure/cite.ts +115 -0
- package/src/structure/issues.ts +63 -0
- package/src/structure/lines.ts +32 -0
- package/src/structure/markdown-path.ts +6 -0
- package/src/structure/numbering.ts +44 -0
- package/src/structure/of.ts +15 -0
- package/src/structure/sexp.ts +28 -0
- package/src/structure/universal.ts +21 -0
package/src/adapter-load.ts
CHANGED
|
@@ -50,7 +50,24 @@ const pickExport = (module: object): unknown => {
|
|
|
50
50
|
return isAdapter(record["adapter"]) ? record["adapter"] : record["default"];
|
|
51
51
|
};
|
|
52
52
|
|
|
53
|
-
|
|
53
|
+
/**
|
|
54
|
+
* 同梱の言語は表から、それ以外は `@chaffjs/lang-<言語>` を探す。
|
|
55
|
+
* 利用者が lang-zh のようなパッケージを書いて入れれば、core を変えずにその言語で動く。
|
|
56
|
+
*/
|
|
57
|
+
const LANGUAGE_ID = /^[a-z]{2,3}$/u;
|
|
58
|
+
|
|
59
|
+
export const packageFor = (language: string): string | undefined =>
|
|
60
|
+
ADAPTER_PACKAGE[language] ?? (LANGUAGE_ID.test(language) ? `@chaffjs/lang-${language}` : undefined);
|
|
61
|
+
|
|
62
|
+
const importAdapter = async (specifier: string, language: string): Promise<unknown> => {
|
|
63
|
+
try {
|
|
64
|
+
return await import(specifier);
|
|
65
|
+
} catch (err) {
|
|
66
|
+
const missing = err instanceof Error && "code" in err && err.code === "ERR_MODULE_NOT_FOUND";
|
|
67
|
+
if (missing) throw new Error(`言語 ${language} のパッケージ ${specifier} が入っていません(npm i -D ${specifier})`, { cause: err });
|
|
68
|
+
throw err;
|
|
69
|
+
}
|
|
70
|
+
};
|
|
54
71
|
|
|
55
72
|
/**
|
|
56
73
|
* アダプタは「必要になったものだけ」を実行時に解決する。spec §17.2。
|
|
@@ -60,7 +77,7 @@ export const packageFor = (language: string): string | undefined => ADAPTER_PACK
|
|
|
60
77
|
export const loadAdapter = async (language: string): Promise<LanguageAdapter> => {
|
|
61
78
|
const specifier = packageFor(language);
|
|
62
79
|
if (specifier === undefined) throw new Error(`no adapter for language "${language}"`);
|
|
63
|
-
const module
|
|
80
|
+
const module = await importAdapter(specifier, language);
|
|
64
81
|
if (typeof module !== "object" || module === null) throw new Error(`${specifier} did not export a module`);
|
|
65
82
|
const candidate = pickExport(module);
|
|
66
83
|
const broken = checkAdapter(candidate);
|
package/src/cli.ts
CHANGED
|
@@ -23,6 +23,8 @@ import { renderFriendly } from "./render/friendly.ts";
|
|
|
23
23
|
import { rulesJson } from "./render/rules-json.ts";
|
|
24
24
|
import { renderSarif } from "./render/sarif.ts";
|
|
25
25
|
import { VERSION } from "./version.ts";
|
|
26
|
+
import { runTree, treeTargets } from "./commands/tree.ts";
|
|
27
|
+
import { citeTargets, runCite } from "./commands/cite.ts";
|
|
26
28
|
import { renderSummary, type FileOutcome } from "./render/summary.ts";
|
|
27
29
|
import { neededBy, runRules } from "./run.ts";
|
|
28
30
|
import type { Level, RuleDefinition } from "./plugin.ts";
|
|
@@ -37,6 +39,8 @@ const USAGE = `chaff — 文章の読みにくいところを見つけます。
|
|
|
37
39
|
chaff eval <dir> 手元の文書で閾値を測り直す
|
|
38
40
|
chaff explain <rule> そのルールの意図と根拠を読む
|
|
39
41
|
chaff genres ジャンルの一覧
|
|
42
|
+
chaff tree <file> [--format sexp|json] 文書を番地の付いた木にする(条・項・定義・参照)
|
|
43
|
+
chaff cite <原文> <引用.json> 回答の引用(番地と引用文)が原文にあるかを確かめる
|
|
40
44
|
chaff rules --json いまの設定を JSON で出す(AI に渡す用)
|
|
41
45
|
chaff baseline <dir> いまある指摘を棚上げする(既存の repo に入れるとき)
|
|
42
46
|
chaff suppressions <dir> stet で黙らせている指摘を数える
|
|
@@ -283,6 +287,8 @@ const HANDLERS: Readonly<Record<string, Handler>> = {
|
|
|
283
287
|
rules: showRules,
|
|
284
288
|
explain: (argv) => explain(argv[1]),
|
|
285
289
|
eval: (argv) => runEval(positional(argv), argv, { config: readConfig(), resolveGenre, flag }),
|
|
290
|
+
tree: (argv) => runTree(treeTargets(argv), argv, { config: readConfig(), flag }),
|
|
291
|
+
cite: (argv) => runCite(citeTargets(argv), argv, { config: readConfig(), flag }),
|
|
286
292
|
test: (argv) => runTest(positional(argv), argv, { config: readConfig(), resolveGenre, inspect }),
|
|
287
293
|
baseline: (argv) => runBaseline(positional(argv), argv),
|
|
288
294
|
suppressions: (argv) => runSuppressions(positional(argv), argv),
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { checkCitations, type Citation, type CitationResult } from "../structure/cite.ts";
|
|
2
|
+
import { readSource, readTree, type TreeContext } from "./tree.ts";
|
|
3
|
+
|
|
4
|
+
const USAGE = "使い方: chaff cite <原文> <引用.json> [--format text|json] [--language ja|en|…]";
|
|
5
|
+
const FORMATS = ["text", "json"];
|
|
6
|
+
const QUOTE_WIDTH = 40;
|
|
7
|
+
const VALUED = ["--format", "--language"];
|
|
8
|
+
|
|
9
|
+
export const citeTargets = (argv: readonly string[]): string[] =>
|
|
10
|
+
argv.slice(1).filter((arg, index, all) => !arg.startsWith("--") && !VALUED.includes(all[index - 1] ?? ""));
|
|
11
|
+
|
|
12
|
+
const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
13
|
+
|
|
14
|
+
const isCitation = (value: unknown): value is Citation => isRecord(value) && typeof value["address"] === "string" && typeof value["quote"] === "string";
|
|
15
|
+
|
|
16
|
+
export type ParsedCitations = { readonly citations: readonly Citation[] } | { readonly error: string };
|
|
17
|
+
|
|
18
|
+
const parseJson = (text: string): { readonly value: unknown } | { readonly error: string } => {
|
|
19
|
+
try {
|
|
20
|
+
return { value: JSON.parse(text) };
|
|
21
|
+
} catch (err) {
|
|
22
|
+
return { error: `JSON として読めません: ${err instanceof Error ? err.message : String(err)}` };
|
|
23
|
+
}
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
/** 引用の一覧。[{ address, quote }] の配列だけを受け取る。形が違えば理由を返す。 */
|
|
27
|
+
export const parseCitations = (text: string): ParsedCitations => {
|
|
28
|
+
const parsed = parseJson(text);
|
|
29
|
+
if ("error" in parsed) return parsed;
|
|
30
|
+
if (!Array.isArray(parsed.value)) return { error: '引用は配列で渡してください: [{ "address": "3.2", "quote": "…" }]' };
|
|
31
|
+
const entries: readonly unknown[] = parsed.value;
|
|
32
|
+
const bad = entries.findIndex((entry) => !isCitation(entry));
|
|
33
|
+
if (bad !== -1) return { error: `${String(bad + 1)} 件目に address(文字列)と quote(文字列)がありません` };
|
|
34
|
+
return { citations: entries.filter(isCitation).map((entry) => ({ address: entry.address, quote: entry.quote })) };
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
const MESSAGES: Readonly<Record<CitationResult["status"], (result: CitationResult) => string>> = {
|
|
38
|
+
ok: () => "一致",
|
|
39
|
+
"missing-address": (result) => `番地 ${result.citation.address} は原文にありません`,
|
|
40
|
+
"quote-elsewhere": (result) => `引用文は ${result.citation.address} ではなく ${result.foundAt ?? "番地の外"}(${String(result.line ?? "?")} 行目)にあります`,
|
|
41
|
+
"quote-not-found": () => "引用文が原文のどこにもありません",
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
const describe = (result: CitationResult): string => {
|
|
45
|
+
const mark = result.status === "ok" ? "✓" : "✗";
|
|
46
|
+
const oneLine = result.citation.quote.replace(/\s+/gu, " ").trim();
|
|
47
|
+
const quote = oneLine.length > QUOTE_WIDTH ? `${oneLine.slice(0, QUOTE_WIDTH)}…` : oneLine;
|
|
48
|
+
return `${mark} ${result.citation.address}「${quote}」: ${MESSAGES[result.status](result)}`;
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* 回答の引用が原文にあるかを確かめる。書き換えはしない。
|
|
53
|
+
* 一つでも外れていれば 1 で終わるので、AI の回答を単体試験のように検査できる。
|
|
54
|
+
*/
|
|
55
|
+
export const runCite = async (targets: readonly string[], argv: readonly string[], context: TreeContext): Promise<number> => {
|
|
56
|
+
const [sourcePath, citationsPath] = targets;
|
|
57
|
+
const format = context.flag(argv, "--format") ?? "text";
|
|
58
|
+
if (sourcePath === undefined || citationsPath === undefined || targets.length !== 2 || !FORMATS.includes(format)) {
|
|
59
|
+
console.error(USAGE);
|
|
60
|
+
return 1;
|
|
61
|
+
}
|
|
62
|
+
const citationsText = await readSource(citationsPath);
|
|
63
|
+
if (citationsText === undefined) return 1;
|
|
64
|
+
const parsed = parseCitations(citationsText);
|
|
65
|
+
if ("error" in parsed) {
|
|
66
|
+
console.error(`${citationsPath}: ${parsed.error}`);
|
|
67
|
+
return 1;
|
|
68
|
+
}
|
|
69
|
+
const read = await readTree(sourcePath, argv, context);
|
|
70
|
+
if (read === undefined) return 1;
|
|
71
|
+
const results = checkCitations(read.source, read.tree, parsed.citations);
|
|
72
|
+
console.log(format === "json" ? JSON.stringify(results, null, 2) : results.map(describe).join("\n"));
|
|
73
|
+
return results.every((result) => result.status === "ok") ? 0 : 1;
|
|
74
|
+
};
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { loadAdapter } from "../adapter-load.ts";
|
|
3
|
+
import { applyByPath } from "../config/by-path.ts";
|
|
4
|
+
import { guessLanguage } from "../detect.ts";
|
|
5
|
+
import { isMarkdownPath } from "../structure/markdown-path.ts";
|
|
6
|
+
import { buildStructure } from "../structure/of.ts";
|
|
7
|
+
import { toSexp } from "../structure/sexp.ts";
|
|
8
|
+
import type { Config } from "../config/load.ts";
|
|
9
|
+
import type { StructureNode } from "../plugin.ts";
|
|
10
|
+
|
|
11
|
+
export type TreeContext = {
|
|
12
|
+
readonly config: Config;
|
|
13
|
+
readonly flag: (argv: readonly string[], name: string) => string | undefined;
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
const FORMATS = ["sexp", "json"];
|
|
17
|
+
|
|
18
|
+
const USAGE = "使い方: chaff tree <file>... [--format sexp|json] [--language ja|en|…]";
|
|
19
|
+
|
|
20
|
+
/** 値を取るフラグ。その次の引数は値で、対象のファイルではない。 */
|
|
21
|
+
const VALUED = ["--format", "--language"];
|
|
22
|
+
|
|
23
|
+
export const treeTargets = (argv: readonly string[]): string[] =>
|
|
24
|
+
argv.slice(1).filter((arg, index, all) => !arg.startsWith("--") && !VALUED.includes(all[index - 1] ?? ""));
|
|
25
|
+
|
|
26
|
+
export const readSource = async (path: string): Promise<string | undefined> => {
|
|
27
|
+
try {
|
|
28
|
+
return await readFile(path, "utf8");
|
|
29
|
+
} catch (err) {
|
|
30
|
+
console.error(`${path} を読めませんでした: ${err instanceof Error ? err.message : String(err)}`);
|
|
31
|
+
return undefined;
|
|
32
|
+
}
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* 言語の決め方は lint / test / eval と同じ。そのファイルの by_path、全体の language、中身からの推定の順で、
|
|
37
|
+
* その前に --language を置く。混在するリポジトリで、英語の契約書を日本語として読まないため。
|
|
38
|
+
*/
|
|
39
|
+
export const treeLanguage = (path: string, source: string, argv: readonly string[], context: TreeContext): string =>
|
|
40
|
+
context.flag(argv, "--language") ??
|
|
41
|
+
applyByPath(context.config.byPath, context.config.baseDir, path).language ??
|
|
42
|
+
context.config.language ??
|
|
43
|
+
guessLanguage(source).language;
|
|
44
|
+
|
|
45
|
+
export type SourceTree = { readonly source: string; readonly tree: StructureNode };
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* 1 ファイルを読んで木にする。読めない・言語パッケージが構造を読めないときは、黙らずに言って undefined を返す。
|
|
49
|
+
* chaff tree と chaff cite が同じ木を見るように、ここを一つにしておく。
|
|
50
|
+
*/
|
|
51
|
+
export const readTree = async (path: string, argv: readonly string[], context: TreeContext): Promise<SourceTree | undefined> => {
|
|
52
|
+
const source = await readSource(path);
|
|
53
|
+
if (source === undefined) return undefined;
|
|
54
|
+
const language = treeLanguage(path, source, argv, context);
|
|
55
|
+
const adapter = await loadAdapter(language);
|
|
56
|
+
if (adapter.structure === undefined) {
|
|
57
|
+
console.error(`${path}: 言語 ${language} のパッケージは文書の構造を読めません(structure がありません)`);
|
|
58
|
+
return undefined;
|
|
59
|
+
}
|
|
60
|
+
// 日本語は形態素で数量と日付を読む。解析器が無ければ単位の表で読むので、木は作れる。
|
|
61
|
+
await adapter.prepare?.({ pos: true });
|
|
62
|
+
return { source, tree: buildStructure({ path, source, language, markdown: isMarkdownPath(path) }, adapter.structure) };
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
const printTree = async (path: string, argv: readonly string[], context: TreeContext): Promise<boolean> => {
|
|
66
|
+
const read = await readTree(path, argv, context);
|
|
67
|
+
if (read === undefined) return false;
|
|
68
|
+
console.log(context.flag(argv, "--format") === "json" ? JSON.stringify(read.tree, null, 2) : toSexp(read.tree));
|
|
69
|
+
return true;
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* 1 つずつ、渡された順に処理する。前のファイルが終わってから次を始めるので、出力の順が入れ替わらない。
|
|
74
|
+
* 失敗したファイルがあっても残りは処理し、1 つでも失敗していれば false を返す。
|
|
75
|
+
*/
|
|
76
|
+
export const inOrder = async (paths: readonly string[], each: (path: string) => Promise<boolean>): Promise<boolean> =>
|
|
77
|
+
paths.reduce<Promise<boolean>>(async (previous, path) => {
|
|
78
|
+
const earlier = await previous;
|
|
79
|
+
return (await each(path)) && earlier;
|
|
80
|
+
}, Promise.resolve(true));
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* 文書を番地の付いた木にして出す。S 式は人と AI が読むため、JSON は機械が読むため。
|
|
84
|
+
* 書き換えはしない。Markdown でない .txt(見出しの無い契約書)も、行頭の番号で木にする。
|
|
85
|
+
*/
|
|
86
|
+
export const runTree = async (paths: readonly string[], argv: readonly string[], context: TreeContext): Promise<number> => {
|
|
87
|
+
const format = context.flag(argv, "--format") ?? "sexp";
|
|
88
|
+
if (paths.length === 0 || !FORMATS.includes(format)) {
|
|
89
|
+
console.error(USAGE);
|
|
90
|
+
return 1;
|
|
91
|
+
}
|
|
92
|
+
const printed = await inOrder(paths, (path) => printTree(path, argv, context));
|
|
93
|
+
return printed ? 0 : 1;
|
|
94
|
+
};
|
package/src/detectors/index.ts
CHANGED
|
@@ -14,6 +14,7 @@ import { paragraphLength, paragraphVariance, preambleLength, ruleOfThree, sectio
|
|
|
14
14
|
import { concreteEvidence, dashDensity, emojiDensity, ngramRepetition, undefinedAcronym } from "./signals.ts";
|
|
15
15
|
import { aiTell, contractionMix, cushionDensity, hedgingDensity, repeatedConjunction, unqualifiedSuperlative } from "./lexicon.ts";
|
|
16
16
|
import { internalJargon, properNounDensity, requiredSections } from "./team.ts";
|
|
17
|
+
import { danglingReference, duplicateDefinition, numberingGap } from "./structure-tree.ts";
|
|
17
18
|
|
|
18
19
|
/** rule 定義の how_to_find がここを引く。rule 側は実装を知らない。 */
|
|
19
20
|
export const DETECTORS: Readonly<Record<string, Detector>> = {
|
|
@@ -53,4 +54,7 @@ export const DETECTORS: Readonly<Record<string, Detector>> = {
|
|
|
53
54
|
"internal-jargon": internalJargon,
|
|
54
55
|
"required-sections": requiredSections,
|
|
55
56
|
"proper-noun-density": properNounDensity,
|
|
57
|
+
"dangling-reference": danglingReference,
|
|
58
|
+
"numbering-gap": numberingGap,
|
|
59
|
+
"duplicate-definition": duplicateDefinition,
|
|
56
60
|
};
|
package/src/detectors/signals.ts
CHANGED
|
@@ -72,7 +72,12 @@ const countGrams = (text: string, width: number): Map<string, number> => {
|
|
|
72
72
|
* どちらを見るかは adapter が宣言する単位で決める。ひらがなを持たない char 単位の言語が
|
|
73
73
|
* 来たら、この見分けは効かなくなる(そのときは adapter 側に判定を移す)。
|
|
74
74
|
*/
|
|
75
|
-
|
|
75
|
+
/**
|
|
76
|
+
* つなぎが 1 つでは足りない。「detectorは」は用語に助詞が 1 つ付いただけで、
|
|
77
|
+
* 繰り返して当たり前の**用語**であって言い回しではない。
|
|
78
|
+
* 実文書で測ると、本物の言い回しはひらがなを 7 つ前後含み、用語+助詞は 1〜2 だった。
|
|
79
|
+
*/
|
|
80
|
+
const CONNECTIVE = { char: /[ぁ-ゖ].*[ぁ-ゖ].*[ぁ-ゖ]/u, word: /\S \S.*\S \S/u };
|
|
76
81
|
|
|
77
82
|
const isPhrasing = (gram: string, unit: ProseDocument["lengthUnit"]): boolean => CONNECTIVE[unit].test(gram);
|
|
78
83
|
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { Detector, Finding, ProseDocument } from "../plugin.ts";
|
|
2
|
+
import { danglingReferences, duplicateDefinitions, type StructureIssue } from "../structure/issues.ts";
|
|
3
|
+
import { numberingBreaks } from "../structure/numbering.ts";
|
|
4
|
+
|
|
5
|
+
const QUOTE_LENGTH = 80;
|
|
6
|
+
|
|
7
|
+
/** 指摘の位置から行末までを引用する。条の見出しや参照を含む一行が、読み手の探す手がかりになる。 */
|
|
8
|
+
const quoteAt = (source: string, offset: number): string => {
|
|
9
|
+
const end = source.indexOf("\n", offset);
|
|
10
|
+
return source
|
|
11
|
+
.slice(offset, end === -1 ? source.length : end)
|
|
12
|
+
.slice(0, QUOTE_LENGTH)
|
|
13
|
+
.trim();
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
const findingsOf =
|
|
17
|
+
(rule: string, issuesOf: (tree: NonNullable<ProseDocument["structure"]>) => StructureIssue[]): Detector =>
|
|
18
|
+
(doc): Finding[] =>
|
|
19
|
+
doc.structure === undefined
|
|
20
|
+
? []
|
|
21
|
+
: issuesOf(doc.structure).map((issue) => ({
|
|
22
|
+
rule,
|
|
23
|
+
severity: "error",
|
|
24
|
+
line: 0,
|
|
25
|
+
column: 0,
|
|
26
|
+
quote: quoteAt(doc.source, issue.offset),
|
|
27
|
+
values: { ...issue.values, offset: issue.offset },
|
|
28
|
+
}));
|
|
29
|
+
|
|
30
|
+
export const danglingReference: Detector = findingsOf("dangling-reference", danglingReferences);
|
|
31
|
+
export const numberingGap: Detector = findingsOf("numbering-gap", numberingBreaks);
|
|
32
|
+
export const duplicateDefinition: Detector = findingsOf("duplicate-definition", duplicateDefinitions);
|
package/src/document.ts
CHANGED
|
@@ -4,7 +4,9 @@ import { gfmTableFromMarkdown } from "mdast-util-gfm-table";
|
|
|
4
4
|
import { frontmatter } from "micromark-extension-frontmatter";
|
|
5
5
|
import { frontmatterFromMarkdown } from "mdast-util-frontmatter";
|
|
6
6
|
import { maskSpans } from "./mask.ts";
|
|
7
|
-
import
|
|
7
|
+
import { buildTree, NO_OUTLINE, type Outline } from "./structure/build.ts";
|
|
8
|
+
import { isMarkdownPath } from "./structure/markdown-path.ts";
|
|
9
|
+
import type { BulletList, LanguageAdapter, Paragraph, ProseDocument, Section, Sentence, Span, StructureNode } from "./plugin.ts";
|
|
8
10
|
|
|
9
11
|
type Place = { readonly offset?: number | undefined };
|
|
10
12
|
type Node = {
|
|
@@ -120,7 +122,7 @@ const textOf = (node: Node, source: string): string => {
|
|
|
120
122
|
return parts.join("");
|
|
121
123
|
};
|
|
122
124
|
|
|
123
|
-
type Heading = { readonly depth: number; readonly text: string; readonly start: number; readonly end: number };
|
|
125
|
+
export type Heading = { readonly depth: number; readonly text: string; readonly start: number; readonly end: number };
|
|
124
126
|
|
|
125
127
|
const headingsOf = (root: Node, source: string): Heading[] => {
|
|
126
128
|
const found: Heading[] = [];
|
|
@@ -247,6 +249,9 @@ export const buildDocument = (path: string, source: string, adapter: LanguageAda
|
|
|
247
249
|
const paragraphSpans = spansOfType(root, "paragraph");
|
|
248
250
|
const listItems = spansOfType(root, "listItem");
|
|
249
251
|
const sentences = sentencesOf(prose, paragraphSpans, adapter);
|
|
252
|
+
const patterns = adapter.structure;
|
|
253
|
+
// null は「作ったが構造を読めない言語だった」、undefined は「まだ作っていない」。
|
|
254
|
+
const tree: { value: StructureNode | null | undefined } = { value: undefined };
|
|
250
255
|
return {
|
|
251
256
|
path,
|
|
252
257
|
source,
|
|
@@ -260,5 +265,24 @@ export const buildDocument = (path: string, source: string, adapter: LanguageAda
|
|
|
260
265
|
lists: listsOf(root, source),
|
|
261
266
|
lexicons: { ...adapter.lexicons, "internal-jargon": team.jargon.map((pattern) => ({ pattern })) },
|
|
262
267
|
requiredSections: team.requiredSections,
|
|
268
|
+
// 構造の rule(参照先が無い・番号の抜け)が読む木。どの rule も読まなければ作らない。何万行の契約書で、他の rule の lint に代金を払わせない。
|
|
269
|
+
get structure(): StructureNode | undefined {
|
|
270
|
+
tree.value ??=
|
|
271
|
+
patterns === undefined
|
|
272
|
+
? null
|
|
273
|
+
: buildTree({ path, source, language: adapter.id, outline: isMarkdownPath(path) ? outlineOf(root, source) : NO_OUTLINE }, patterns);
|
|
274
|
+
return tree.value ?? undefined;
|
|
275
|
+
},
|
|
263
276
|
};
|
|
264
277
|
};
|
|
278
|
+
|
|
279
|
+
/** 番号を探してはいけない範囲。コードの中の「第3条」は条ではなく、参照でもない。 */
|
|
280
|
+
const OPAQUE = ["code", "inlineCode", "html", "yaml", "toml"];
|
|
281
|
+
|
|
282
|
+
const outlineOf = (root: Node, source: string): Outline => ({
|
|
283
|
+
headings: headingsOf(root, source),
|
|
284
|
+
opaque: OPAQUE.flatMap((type) => spansOfType(root, type)),
|
|
285
|
+
});
|
|
286
|
+
|
|
287
|
+
/** 構造を読むための Markdown の手がかり。見出しと、中を読まない範囲。 */
|
|
288
|
+
export const markdownOutline = (source: string): Outline => outlineOf(parse(source), source);
|
package/src/eval.ts
CHANGED
|
@@ -63,6 +63,16 @@ const recommend = (rule: RuleDefinition, sweep: readonly Point[], total: number)
|
|
|
63
63
|
return strictest;
|
|
64
64
|
};
|
|
65
65
|
|
|
66
|
+
/** lint と同じ要求の読み方。知らない要求は満たさないものとする。pos に読み替えると、動かない rule を測ってしまう。 */
|
|
67
|
+
const meets = (need: string, docs: readonly ProseDocument[]): boolean => {
|
|
68
|
+
const capabilities = docs[0]?.capabilities;
|
|
69
|
+
if (need === "pos") return capabilities?.pos === true;
|
|
70
|
+
if (need === "lemma") return capabilities?.lemma === true;
|
|
71
|
+
// 木は adapter が structure を持つときだけ作れる。無いまま測ると、どの閾値でも 0 件になる。
|
|
72
|
+
if (need === "structure") return docs.length > 0 && docs.every((doc) => doc.structure !== undefined);
|
|
73
|
+
return false;
|
|
74
|
+
};
|
|
75
|
+
|
|
66
76
|
/**
|
|
67
77
|
* lint で動かない rule は掃引しない。
|
|
68
78
|
* 動かないものの閾値を測ると、「どの閾値でも 0 件」が「よく校正されている」に見える。
|
|
@@ -71,8 +81,7 @@ const measurable = (rule: RuleDefinition, docs: readonly ProseDocument[], genre:
|
|
|
71
81
|
if (rule.layer === "L4" || !rule.use_for.some((target) => genre.startsWith(target))) return false;
|
|
72
82
|
if (rule.languages !== undefined && !rule.languages.includes(language)) return false;
|
|
73
83
|
if (rule.from.length > 0) return false;
|
|
74
|
-
|
|
75
|
-
return !rule.requires.some((need) => (need === "lemma" ? capabilities?.lemma : capabilities?.pos) !== true);
|
|
84
|
+
return rule.requires.every((need) => meets(need, docs));
|
|
76
85
|
};
|
|
77
86
|
|
|
78
87
|
export const evaluate = (docs: readonly ProseDocument[], rules: readonly RuleDefinition[], genre: string, language: string): RuleReport[] =>
|
package/src/plugin.ts
CHANGED
|
@@ -71,6 +71,79 @@ export type LanguageAdapter = {
|
|
|
71
71
|
readonly segment: (text: string) => Segmentation;
|
|
72
72
|
/** L2 rule が word_list で引く。アダプタが自分の言語のぶんだけを持つ。 */
|
|
73
73
|
readonly lexicons: Readonly<Record<string, Lexicon>>;
|
|
74
|
+
/** 文書の構造を読む型。無い言語では `chaff tree` がそう言って止まる。 */
|
|
75
|
+
readonly structure?: StructurePatterns;
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
// ───────── 文書の構造 ─────────
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* 行頭の番号 1 つ。「第3条」「2」「(a)」を読むのは言語パッケージで、
|
|
82
|
+
* 入れ子と番地を決めるのは core。core は番号の書き方を知らない。
|
|
83
|
+
*/
|
|
84
|
+
export type NumberedLine = {
|
|
85
|
+
readonly kind: "chapter" | "article" | "item";
|
|
86
|
+
/** 入れ子の深さ。条が 1、項が 2、号が 3 のように、言語パッケージが決める。条より外の編・章・節は 0 以下。 */
|
|
87
|
+
readonly depth: number;
|
|
88
|
+
/** 番地の部品。"3"、"4.2"、"a"。 */
|
|
89
|
+
readonly number: string;
|
|
90
|
+
/** true なら number だけで番地が決まる(条番号、4.2 のような通し番号)。false なら親の番地に続ける。 */
|
|
91
|
+
readonly absolute: boolean;
|
|
92
|
+
/** 書かれたままの番号。「第3条」「Section 4.2」。 */
|
|
93
|
+
readonly label: string;
|
|
94
|
+
/** 番号に続く見出し。「第3条(支払)」の「支払」。本文しか無ければ空。 */
|
|
95
|
+
readonly heading: string;
|
|
96
|
+
/** 番号の後ろの文字列全部。定義や参照はここから探す。 */
|
|
97
|
+
readonly rest: string;
|
|
98
|
+
/**
|
|
99
|
+
* 並びの中の位置。第三条は 3、(ii) は 2、(b) は 2、4.2 は 2。番号の抜けを見る rule が比べる。
|
|
100
|
+
* 枝番号(第3条の2)は並びの外なので付けない。読み方を知っているのは言語パッケージだけなので、core は計算しない。
|
|
101
|
+
*/
|
|
102
|
+
readonly ordinal?: number | undefined;
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
/** 番号を読むときに見える周り。「(i)」がローマ数字か英字かは、開いている番号で決まる。 */
|
|
106
|
+
export type NumberingContext = {
|
|
107
|
+
readonly open: readonly NumberedLine[];
|
|
108
|
+
/** Markdown の見出しの行か。本文の「1. 」は箇条書きで、見出しの「1. 」は章番号。 */
|
|
109
|
+
readonly isHeading: boolean;
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
/** 行の中で見つけたもの。start / end は渡した文字列の中の位置。 */
|
|
113
|
+
export type Mention = { readonly start: number; readonly end: number; readonly attrs: Readonly<Record<string, string | number>> };
|
|
114
|
+
|
|
115
|
+
export type StructurePatterns = {
|
|
116
|
+
readonly numbered: (line: string, context: NumberingContext) => NumberedLine | undefined;
|
|
117
|
+
/** 定義。attrs.term に定義された語。 */
|
|
118
|
+
readonly definitions: (text: string) => readonly Mention[];
|
|
119
|
+
/** 参照。attrs.target に正規化した番地("12.1")、attrs.label に書かれたまま。 */
|
|
120
|
+
readonly references: (text: string) => readonly Mention[];
|
|
121
|
+
/** 義務・禁止・許可。attrs.marker に語、attrs.type に must / must-not / may。 */
|
|
122
|
+
readonly obligations: (text: string) => readonly Mention[];
|
|
123
|
+
/** 数量。attrs.value に数、attrs.unit に単位。 */
|
|
124
|
+
readonly quantities: (text: string) => readonly Mention[];
|
|
125
|
+
/** 日付。attrs.value に "2024-04-01"・"2024-04"・"04-01"・"2024" のどれか。無い言語は日付を読まない。 */
|
|
126
|
+
readonly dates?: (text: string) => readonly Mention[];
|
|
127
|
+
/**
|
|
128
|
+
* 「1.5 倍」「2.5 days」の 1.5 は通し番号ではない。番号と後ろの文字列を渡し、後ろが単位なら true。
|
|
129
|
+
* 数字と点だけの通し番号は core が言語を問わず読むので、それを数量と見分けられるのは言語パッケージだけ。
|
|
130
|
+
*/
|
|
131
|
+
readonly countedAfter?: (number: string, rest: string) => boolean;
|
|
132
|
+
};
|
|
133
|
+
|
|
134
|
+
export type StructureKind = "doc" | "section" | "chapter" | "article" | "item" | "definition" | "reference" | "obligation" | "quantity" | "date";
|
|
135
|
+
|
|
136
|
+
/** 番地の付いた木の節点。すべて元の文書の位置(UTF-16)と行を持つ。 */
|
|
137
|
+
export type StructureNode = {
|
|
138
|
+
readonly kind: StructureKind;
|
|
139
|
+
/** 節・条・項の番地。"3.2"、見出しは "h2.1"。定義や参照のような葉は空。 */
|
|
140
|
+
readonly address: string;
|
|
141
|
+
readonly span: Span;
|
|
142
|
+
readonly line: number;
|
|
143
|
+
readonly attrs: Readonly<Record<string, string | number>>;
|
|
144
|
+
readonly children: readonly StructureNode[];
|
|
145
|
+
/** 番号付きのまとまりの、並びの中の位置(NumberedLine.ordinal)。S 式には出さない。 */
|
|
146
|
+
readonly ordinal?: number;
|
|
74
147
|
};
|
|
75
148
|
|
|
76
149
|
// ───────── 文書モデルと rule ─────────
|
|
@@ -115,6 +188,8 @@ export type ProseDocument = {
|
|
|
115
188
|
readonly lexicons: Readonly<Record<string, Lexicon>>;
|
|
116
189
|
/** この種類の文書に無いと困る見出し。チームが chaff.yaml で決める。 */
|
|
117
190
|
readonly requiredSections: readonly string[];
|
|
191
|
+
/** 番地の付いた木(§27)。adapter が structure を持たない言語では無い。 */
|
|
192
|
+
readonly structure: StructureNode | undefined;
|
|
118
193
|
};
|
|
119
194
|
|
|
120
195
|
export type Severity = "error" | "warning" | "info";
|
package/src/position.ts
CHANGED
|
@@ -12,7 +12,27 @@ export const lineStarts = (source: string): number[] => {
|
|
|
12
12
|
return starts;
|
|
13
13
|
};
|
|
14
14
|
|
|
15
|
+
/**
|
|
16
|
+
* offset を含む行。starts は昇順なので二分探索で引く。指摘が何千もある長い文書で、指摘ごとに全行をなめない。
|
|
17
|
+
* offset より前に行頭が無ければ(負の値、NaN)先頭の行とする。
|
|
18
|
+
*/
|
|
19
|
+
const lineIndexOf = (starts: readonly number[], offset: number): number => {
|
|
20
|
+
let low = 0;
|
|
21
|
+
let high = starts.length - 1;
|
|
22
|
+
let found = 0;
|
|
23
|
+
while (low <= high) {
|
|
24
|
+
const middle = (low + high) >> 1;
|
|
25
|
+
if ((starts[middle] ?? Number.POSITIVE_INFINITY) <= offset) {
|
|
26
|
+
found = middle;
|
|
27
|
+
low = middle + 1;
|
|
28
|
+
} else {
|
|
29
|
+
high = middle - 1;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
return found;
|
|
33
|
+
};
|
|
34
|
+
|
|
15
35
|
export const placeOf = (starts: readonly number[], offset: number): Place => {
|
|
16
|
-
const index = starts
|
|
36
|
+
const index = lineIndexOf(starts, offset);
|
|
17
37
|
return { line: index + 1, column: offset - (starts[index] ?? 0) + 1 };
|
|
18
38
|
};
|
package/src/run.ts
CHANGED
|
@@ -43,9 +43,9 @@ const hasTokens = (doc: ProseDocument): boolean => doc.sentences.length === 0 ||
|
|
|
43
43
|
/** 要求を満たさない rule は動かせない。満たさないまま動かすと「指摘 0 件」が保証に見える。 */
|
|
44
44
|
const unmet = (rule: RuleDefinition, doc: ProseDocument): string | undefined => {
|
|
45
45
|
if (rule.languages !== undefined && !rule.languages.includes(doc.language)) return `${doc.language} 向けの rule ではないため`;
|
|
46
|
-
const missing = rule.requires.find((need) => !has(doc.capabilities, need));
|
|
46
|
+
const missing = rule.requires.filter((need) => need !== "structure").find((need) => !has(doc.capabilities, need));
|
|
47
47
|
if (missing !== undefined) return `この言語では${CAPABILITY_NAME[missing] ?? missing}が使えないため`;
|
|
48
|
-
if (rule.requires.
|
|
48
|
+
if (rule.requires.some((need) => need === "pos" || need === "lemma") && !hasTokens(doc)) return "アダプタが品詞を返さなかったため";
|
|
49
49
|
return undefined;
|
|
50
50
|
};
|
|
51
51
|
|
|
@@ -111,6 +111,11 @@ export const runRules = (doc: ProseDocument, rules: readonly RuleDefinition[], s
|
|
|
111
111
|
const why = rule.status === "experimental" && settings[rule.id] === undefined ? "まだ試験中のため" : "設定で止めているため";
|
|
112
112
|
return { findings: acc.findings, skipped: [...acc.skipped, { rule: rule.id, why }] };
|
|
113
113
|
}
|
|
114
|
+
// 木は capability ではなく、adapter が structure を持つかで決まる。持たない言語で動かすと「参照先が無い」が 0 件に見える。
|
|
115
|
+
// 段階を見た後で聞く。doc.structure は触れたときに木を作るので、止めている rule のために作らない。
|
|
116
|
+
if (rule.requires.includes("structure") && doc.structure === undefined) {
|
|
117
|
+
return { findings: acc.findings, skipped: [...acc.skipped, { rule: rule.id, why: `${doc.language} のパッケージは文書の構造を読めないため` }] };
|
|
118
|
+
}
|
|
114
119
|
// 複合シグナルは二段目で扱う。一段目では「検出器が無い」と言わせない。
|
|
115
120
|
if (rule.from.length > 0) return acc;
|
|
116
121
|
const detector = DETECTORS[rule.how_to_find];
|
package/src/sentence-shape.ts
CHANGED
|
@@ -19,4 +19,22 @@ export const hasParticle = (sentence: Sentence): boolean => (sentence.tokens ??
|
|
|
19
19
|
|
|
20
20
|
const SKIP = new Set(["PUNCT", "PART", "SYM"]);
|
|
21
21
|
|
|
22
|
-
|
|
22
|
+
/**
|
|
23
|
+
* 文末の括弧は、文の終わりではなく添え物。
|
|
24
|
+
*
|
|
25
|
+
* 「これを最優先制約とする(§17)。」の述語は「とする」であって「17」ではない。
|
|
26
|
+
* 仕様書は相互参照を括弧で添えるので、これを数えると文末が全部そこになる。
|
|
27
|
+
*/
|
|
28
|
+
const TRAILING = /[((][^((]*[))][\s。.!?!?]*$/u;
|
|
29
|
+
|
|
30
|
+
const beforeTrailing = (sentence: Sentence): readonly Token[] => {
|
|
31
|
+
const tokens = sentence.tokens ?? [];
|
|
32
|
+
const match = TRAILING.exec(sentence.text);
|
|
33
|
+
if (match?.index === undefined) return tokens;
|
|
34
|
+
const cut = sentence.span.start + match.index;
|
|
35
|
+
const kept = tokens.filter((token) => token.span.end <= cut);
|
|
36
|
+
// 括弧を外したら何も残らない文は、括弧そのものが中身。そのまま見る。
|
|
37
|
+
return kept.length === 0 ? tokens : kept;
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
export const lastContent = (sentence: Sentence): Token | undefined => [...beforeTrailing(sentence)].reverse().find((token) => !SKIP.has(token.pos));
|