chaffjs 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter-load.d.ts +5 -0
- package/dist/adapter-load.d.ts.map +1 -1
- package/dist/adapter-load.js +39 -3
- package/dist/adapter-load.js.map +1 -1
- package/dist/backends/anthropic.d.ts +2 -2
- package/dist/backends/anthropic.d.ts.map +1 -1
- package/dist/backends/anthropic.js +1 -1
- package/dist/backends/anthropic.js.map +1 -1
- package/dist/backends/openai.d.ts +2 -1
- package/dist/backends/openai.d.ts.map +1 -1
- package/dist/backends/types.d.ts +0 -1
- package/dist/backends/types.d.ts.map +1 -1
- package/dist/backends/types.js +1 -1
- package/dist/backends/types.js.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +22 -2
- package/dist/cli.js.map +1 -1
- package/dist/config/load.d.ts +0 -3
- package/dist/config/load.d.ts.map +1 -1
- package/dist/config/load.js +1 -1
- package/dist/config/load.js.map +1 -1
- package/dist/detectors/en-shape.d.ts.map +1 -1
- package/dist/detectors/en-shape.js +4 -2
- package/dist/detectors/en-shape.js.map +1 -1
- package/dist/detectors/heading-echo.d.ts.map +1 -1
- package/dist/detectors/heading-echo.js +25 -2
- package/dist/detectors/heading-echo.js.map +1 -1
- package/dist/detectors/index.d.ts.map +1 -1
- package/dist/detectors/index.js +4 -2
- package/dist/detectors/index.js.map +1 -1
- package/dist/detectors/lexicon.d.ts +1 -0
- package/dist/detectors/lexicon.d.ts.map +1 -1
- package/dist/detectors/lexicon.js +41 -0
- package/dist/detectors/lexicon.js.map +1 -1
- package/dist/detectors/signals.d.ts +1 -0
- package/dist/detectors/signals.d.ts.map +1 -1
- package/dist/detectors/signals.js +42 -10
- package/dist/detectors/signals.js.map +1 -1
- package/dist/document.d.ts.map +1 -1
- package/dist/document.js +8 -1
- package/dist/document.js.map +1 -1
- package/dist/eval.d.ts.map +1 -1
- package/dist/eval.js +15 -1
- package/dist/eval.js.map +1 -1
- package/dist/files.d.ts +0 -10
- package/dist/files.d.ts.map +1 -1
- package/dist/files.js +1 -1
- package/dist/files.js.map +1 -1
- package/dist/levels.d.ts +0 -1
- package/dist/levels.d.ts.map +1 -1
- package/dist/levels.js +1 -1
- package/dist/levels.js.map +1 -1
- package/dist/mask.d.ts.map +1 -1
- package/dist/mask.js +8 -1
- package/dist/mask.js.map +1 -1
- package/dist/measure.d.ts +0 -1
- package/dist/measure.d.ts.map +1 -1
- package/dist/measure.js +1 -1
- package/dist/measure.js.map +1 -1
- package/dist/plugin.d.ts +7 -0
- package/dist/plugin.d.ts.map +1 -1
- package/dist/position.d.ts +6 -1
- package/dist/position.d.ts.map +1 -1
- package/dist/position.js +12 -2
- package/dist/position.js.map +1 -1
- package/dist/render/rules-json.d.ts +1 -2
- package/dist/render/rules-json.d.ts.map +1 -1
- package/dist/render/rules-json.js +0 -1
- package/dist/render/rules-json.js.map +1 -1
- package/dist/render/sarif.d.ts +10 -0
- package/dist/render/sarif.d.ts.map +1 -0
- package/dist/render/sarif.js +65 -0
- package/dist/render/sarif.js.map +1 -0
- package/dist/render/semantic.d.ts +0 -1
- package/dist/render/semantic.d.ts.map +1 -1
- package/dist/render/semantic.js +1 -1
- package/dist/render/semantic.js.map +1 -1
- package/dist/render/text.d.ts +0 -1
- package/dist/render/text.d.ts.map +1 -1
- package/dist/render/text.js +1 -1
- package/dist/render/text.js.map +1 -1
- package/dist/rule-load.d.ts.map +1 -1
- package/dist/rule-load.js +15 -2
- package/dist/rule-load.js.map +1 -1
- package/dist/run-semantic.d.ts.map +1 -1
- package/dist/run-semantic.js +15 -4
- package/dist/run-semantic.js.map +1 -1
- package/dist/run.d.ts.map +1 -1
- package/dist/run.js +44 -4
- package/dist/run.js.map +1 -1
- package/dist/version.d.ts +2 -0
- package/dist/version.d.ts.map +1 -0
- package/dist/version.js +11 -0
- package/dist/version.js.map +1 -0
- package/package.json +3 -3
- package/rules/ai-generated-composite.yaml +36 -0
- package/rules/concrete-evidence-density.yaml +1 -1
- package/rules/contraction-consistency.yaml +28 -0
- package/rules/max-paragraph-length.yaml +1 -1
- package/rules/ngram-repetition.yaml +1 -1
- package/rules/no-doubled-joshi.yaml +1 -1
- package/rules/no-em-dash.yaml +32 -0
- package/rules/no-nakaguro-parallel.yaml +1 -1
- package/rules/preamble-length.yaml +1 -1
- package/rules/taigen-dome-in-prose.yaml +1 -1
- package/src/adapter-load.ts +40 -2
- package/src/backends/anthropic.ts +2 -2
- package/src/backends/openai.ts +1 -1
- package/src/backends/types.ts +1 -1
- package/src/cli.ts +33 -3
- package/src/config/load.ts +1 -1
- package/src/detectors/en-shape.ts +4 -2
- package/src/detectors/heading-echo.ts +29 -3
- package/src/detectors/index.ts +4 -2
- package/src/detectors/lexicon.ts +41 -0
- package/src/detectors/signals.ts +43 -10
- package/src/document.ts +11 -1
- package/src/eval.ts +13 -1
- package/src/files.ts +1 -1
- package/src/levels.ts +1 -1
- package/src/mask.ts +8 -1
- package/src/measure.ts +1 -1
- package/src/plugin.ts +7 -1
- package/src/position.ts +11 -2
- package/src/render/rules-json.ts +1 -3
- package/src/render/sarif.ts +103 -0
- package/src/render/semantic.ts +1 -1
- package/src/render/text.ts +1 -1
- package/src/rule-load.ts +14 -2
- package/src/run-semantic.ts +14 -4
- package/src/run.ts +42 -3
- package/src/version.ts +14 -0
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
id: contraction-consistency
|
|
2
|
+
layer: L2
|
|
3
|
+
status: experimental
|
|
4
|
+
how_to_find: contraction-mix
|
|
5
|
+
word_list: contraction
|
|
6
|
+
languages: [en]
|
|
7
|
+
severity: info
|
|
8
|
+
|
|
9
|
+
name:
|
|
10
|
+
ja: 短縮形の使いかたが揃っていない
|
|
11
|
+
en: Contraction use is inconsistent
|
|
12
|
+
|
|
13
|
+
why:
|
|
14
|
+
ja: 短縮形を使うかどうかは文体の選択で、どちらが正しいかは決めません。1 つの文書で混ざっていると、読み手は文ごとに距離感を測り直すことになります。
|
|
15
|
+
en: Whether to use contractions is a choice of register, and neither is wrong. Mixed within one document, the reader re-measures the distance sentence by sentence.
|
|
16
|
+
|
|
17
|
+
message:
|
|
18
|
+
ja: 「{matched}」だけ他と書きかたが違います(そういう書きかたが {count} 種)
|
|
19
|
+
en: '"{matched}" is written differently from the rest of the document'
|
|
20
|
+
|
|
21
|
+
how_to_fix:
|
|
22
|
+
ja: 多いほうに揃えてください。
|
|
23
|
+
en: Match the majority.
|
|
24
|
+
|
|
25
|
+
# 少数派が何種までなら「混在」と見るか。これを超えたら混在ではなく、そういう文体。
|
|
26
|
+
levels: { strict: 4, normal: 2, relaxed: 1 }
|
|
27
|
+
|
|
28
|
+
use_for: [blog, business, technical]
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
id: no-em-dash
|
|
2
|
+
layer: L1
|
|
3
|
+
status: experimental
|
|
4
|
+
how_to_find: dash-density
|
|
5
|
+
|
|
6
|
+
# 記号の許容度は言語で違う。英語では正当な用法が多く、日本語の組版では扱いが難しい。spec §12.4。
|
|
7
|
+
severity:
|
|
8
|
+
ja: warning
|
|
9
|
+
en: info
|
|
10
|
+
|
|
11
|
+
name:
|
|
12
|
+
ja: ダッシュが多い
|
|
13
|
+
en: Too many dashes
|
|
14
|
+
|
|
15
|
+
why:
|
|
16
|
+
ja: 日本語の組版でダッシュは扱いが難しく、環境によって長さも見えかたも変わります。読点や括弧で書けることが多いです。
|
|
17
|
+
en: In English a dash is a legitimate tool, and a heavy hand with it is also one of the better-known marks of generated text. On its own it says little; it counts toward the composite signal.
|
|
18
|
+
|
|
19
|
+
message:
|
|
20
|
+
ja: 1000 字あたり {density} 個のダッシュがあります({limit} 個まで)
|
|
21
|
+
en: "{density} dashes per 1000 words (limit {limit})"
|
|
22
|
+
|
|
23
|
+
how_to_fix:
|
|
24
|
+
ja: 読点か括弧に置き換えてください。挟み込みなら括弧、言い換えなら読点が読みやすいです。
|
|
25
|
+
en: A comma or a pair of brackets usually reads better.
|
|
26
|
+
|
|
27
|
+
# 1000 あたりの個数。英語は許容度が高いので緩い。
|
|
28
|
+
levels:
|
|
29
|
+
ja: { strict: 1, normal: 3, relaxed: 8 }
|
|
30
|
+
en: { strict: 3, normal: 8, relaxed: 20 }
|
|
31
|
+
|
|
32
|
+
use_for: [blog, business, technical]
|
package/src/adapter-load.ts
CHANGED
|
@@ -2,10 +2,47 @@ import type { LanguageAdapter } from "./plugin.ts";
|
|
|
2
2
|
|
|
3
3
|
const ADAPTER_PACKAGE: Readonly<Record<string, string>> = { ja: "@chaffjs/lang-ja", en: "@chaffjs/lang-en" };
|
|
4
4
|
|
|
5
|
+
/** core が読める契約の版。合わないアダプタは、動かしてから壊れるより先に断る。 */
|
|
6
|
+
const API_VERSION = 1;
|
|
7
|
+
|
|
8
|
+
const isCapabilities = (value: unknown): boolean => {
|
|
9
|
+
if (typeof value !== "object" || value === null) return false;
|
|
10
|
+
const record: Record<string, unknown> = { ...value };
|
|
11
|
+
const unit = record["lengthUnit"];
|
|
12
|
+
return (
|
|
13
|
+
record["sentenceSplit"] === true && (unit === "char" || unit === "word") && ["wordSplit", "pos", "lemma"].every((name) => typeof record[name] === "boolean")
|
|
14
|
+
);
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* 形だけを見る。中身が正しいかは動かさないと分からないが、**形が違うものを通すと
|
|
19
|
+
* 指摘 0 件で終わる**。0 件は「問題なし」と見分けがつかない。
|
|
20
|
+
*/
|
|
5
21
|
const isAdapter = (value: unknown): value is LanguageAdapter => {
|
|
6
22
|
if (typeof value !== "object" || value === null) return false;
|
|
7
23
|
const record: Record<string, unknown> = { ...value };
|
|
8
|
-
return
|
|
24
|
+
return (
|
|
25
|
+
record["kind"] === "language" &&
|
|
26
|
+
typeof record["id"] === "string" &&
|
|
27
|
+
record["id"] !== "" &&
|
|
28
|
+
record["apiVersion"] === API_VERSION &&
|
|
29
|
+
typeof record["segment"] === "function" &&
|
|
30
|
+
typeof record["detect"] === "function" &&
|
|
31
|
+
isCapabilities(record["capabilities"])
|
|
32
|
+
);
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* 形が違うときに、どこが違うのかを返す。満たしていれば undefined。
|
|
37
|
+
* 「読めません」だけでは、アダプタを書いた人が直せない。
|
|
38
|
+
*/
|
|
39
|
+
export const checkAdapter = (value: unknown): string | undefined => {
|
|
40
|
+
if (isAdapter(value)) return undefined;
|
|
41
|
+
if (typeof value !== "object" || value === null) return "object ではありません";
|
|
42
|
+
const record: Record<string, unknown> = { ...value };
|
|
43
|
+
if (record["apiVersion"] !== API_VERSION) return `apiVersion が ${String(record["apiVersion"])} です(core は ${String(API_VERSION)})`;
|
|
44
|
+
if (!isCapabilities(record["capabilities"])) return "capabilities の形が違います";
|
|
45
|
+
return "kind / id / segment / detect のどれかがありません";
|
|
9
46
|
};
|
|
10
47
|
|
|
11
48
|
const pickExport = (module: object): unknown => {
|
|
@@ -26,6 +63,7 @@ export const loadAdapter = async (language: string): Promise<LanguageAdapter> =>
|
|
|
26
63
|
const module: unknown = await import(specifier);
|
|
27
64
|
if (typeof module !== "object" || module === null) throw new Error(`${specifier} did not export a module`);
|
|
28
65
|
const candidate = pickExport(module);
|
|
29
|
-
|
|
66
|
+
const broken = checkAdapter(candidate);
|
|
67
|
+
if (broken !== undefined || !isAdapter(candidate)) throw new Error(`${specifier} が LanguageAdapter を export していません: ${broken ?? ""}`);
|
|
30
68
|
return candidate;
|
|
31
69
|
};
|
|
@@ -5,7 +5,7 @@ import Anthropic from "@anthropic-ai/sdk";
|
|
|
5
5
|
import { toFailure } from "./types.ts";
|
|
6
6
|
import type { Failure, Judge, Prompt } from "./types.ts";
|
|
7
7
|
|
|
8
|
-
|
|
8
|
+
type AnthropicResponse = { readonly content: readonly { readonly type: string; readonly text?: string | undefined }[] };
|
|
9
9
|
|
|
10
10
|
/** judge が使うのは messages.create だけ。最小の面だけを要求する。テストで差し替えられる。 */
|
|
11
11
|
export type AnthropicClient = { readonly messages: { readonly create: (params: Anthropic.MessageCreateParamsNonStreaming) => Promise<AnthropicResponse> } };
|
|
@@ -26,7 +26,7 @@ export const DEFAULT_MODEL = "claude-opus-5";
|
|
|
26
26
|
*/
|
|
27
27
|
const ENV_KEYS = ["ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_IDENTITY_TOKEN", "ANTHROPIC_IDENTITY_TOKEN_FILE"];
|
|
28
28
|
|
|
29
|
-
|
|
29
|
+
const configDir = (): string => process.env["ANTHROPIC_CONFIG_DIR"] ?? join(homedir(), ".config", "anthropic");
|
|
30
30
|
|
|
31
31
|
export const hasCredentials = (profileDir: string = configDir()): boolean =>
|
|
32
32
|
ENV_KEYS.some((name) => (process.env[name] ?? "").length > 0) || existsSync(profileDir);
|
package/src/backends/openai.ts
CHANGED
|
@@ -2,7 +2,7 @@ import OpenAI from "openai";
|
|
|
2
2
|
import { toFailure } from "./types.ts";
|
|
3
3
|
import type { Failure, Judge, Prompt } from "./types.ts";
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
type OpenAIResponse = { readonly choices: readonly { readonly message: { readonly content: string | null } }[] };
|
|
6
6
|
|
|
7
7
|
/** judge が使うのは chat.completions.create だけ。最小の面だけを要求する。 */
|
|
8
8
|
export type OpenAIClient = {
|
package/src/backends/types.ts
CHANGED
|
@@ -12,7 +12,7 @@ export type Judge = (prompt: Prompt) => Promise<string>;
|
|
|
12
12
|
|
|
13
13
|
export type BackendName = "anthropic" | "openai";
|
|
14
14
|
|
|
15
|
-
|
|
15
|
+
const BACKENDS: readonly BackendName[] = ["anthropic", "openai"];
|
|
16
16
|
|
|
17
17
|
export const isBackend = (value: unknown): value is BackendName => BACKENDS.some((name) => name === value);
|
|
18
18
|
|
package/src/cli.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { readFile } from "node:fs/promises";
|
|
2
|
-
import { existsSync } from "node:fs";
|
|
3
|
-
import { join } from "node:path";
|
|
2
|
+
import { existsSync, mkdirSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { dirname, join } from "node:path";
|
|
4
4
|
import { loadAdapter, packageFor } from "./adapter-load.ts";
|
|
5
5
|
import { CONFIG_FILE, EMPTY, loadConfig, type Config } from "./config/load.ts";
|
|
6
6
|
import { applyByPath } from "./config/by-path.ts";
|
|
@@ -21,6 +21,8 @@ import { renderCompact } from "./render/compact.ts";
|
|
|
21
21
|
import { renderExplain } from "./render/explain.ts";
|
|
22
22
|
import { renderFriendly } from "./render/friendly.ts";
|
|
23
23
|
import { rulesJson } from "./render/rules-json.ts";
|
|
24
|
+
import { renderSarif } from "./render/sarif.ts";
|
|
25
|
+
import { VERSION } from "./version.ts";
|
|
24
26
|
import { renderSummary, type FileOutcome } from "./render/summary.ts";
|
|
25
27
|
import { neededBy, runRules } from "./run.ts";
|
|
26
28
|
import type { Level, RuleDefinition } from "./plugin.ts";
|
|
@@ -45,6 +47,7 @@ const USAGE = `chaff — 文章の読みにくいところを見つけます。
|
|
|
45
47
|
--show-baseline 棚上げした分も含めて全部見る
|
|
46
48
|
--watch 保存のたびに見直し、変わったところだけ出す
|
|
47
49
|
--dry-run test で、何を AI に送るかだけを見る(API を呼びません)
|
|
50
|
+
--sarif <path> 指摘を SARIF で書き出す(GitHub の PR の行に出すため)
|
|
48
51
|
|
|
49
52
|
この箇所だけ黙らせる: <!-- stet: rule-id — 理由 -->
|
|
50
53
|
|
|
@@ -81,7 +84,15 @@ const flag = (argv: readonly string[], name: string): string | undefined => {
|
|
|
81
84
|
return at === -1 ? undefined : argv[at + 1];
|
|
82
85
|
};
|
|
83
86
|
|
|
84
|
-
type Inspected = {
|
|
87
|
+
type Inspected = {
|
|
88
|
+
readonly text: string;
|
|
89
|
+
readonly rules: readonly RuleDefinition[];
|
|
90
|
+
/** ファイルごとの言語。by_path で 1 つの repo に 2 言語が混ざる。 */
|
|
91
|
+
readonly language: string;
|
|
92
|
+
readonly outcome: FileOutcome;
|
|
93
|
+
readonly perFile: PerFile;
|
|
94
|
+
readonly all: readonly string[];
|
|
95
|
+
};
|
|
85
96
|
|
|
86
97
|
const headerFor = (path: string, genre: string, from: string, language: string, shelved: number, hushed: number): string => {
|
|
87
98
|
const shelf = shelved > 0 ? ` 棚上げ ${shelved} 件` : "";
|
|
@@ -112,12 +123,30 @@ const inspect = async (path: string, config: Config, argv: readonly string[]): P
|
|
|
112
123
|
const text = argv.includes("--compact") ? renderCompact(header, result, rules, language) : renderFriendly(header, result, rules, language);
|
|
113
124
|
return {
|
|
114
125
|
text,
|
|
126
|
+
rules,
|
|
127
|
+
language,
|
|
115
128
|
outcome: { path, findings: split.fresh, notRun: raw.skipped.length },
|
|
116
129
|
perFile: { path, suppressed: applied.suppressed, reasonless: applied.unusedReasonless },
|
|
117
130
|
all: applied.kept.map((finding) => fingerprint(path, finding)),
|
|
118
131
|
};
|
|
119
132
|
};
|
|
120
133
|
|
|
134
|
+
/**
|
|
135
|
+
* 指摘を PR の変更行に出すための出口。--sarif <path> を書いたときだけ作る。
|
|
136
|
+
* 端末の出力は変えない。CI で上げるためのファイルが増えるだけ。
|
|
137
|
+
*/
|
|
138
|
+
const writeSarif = (results: readonly Inspected[], argv: readonly string[]): void => {
|
|
139
|
+
const path = flag(argv, "--sarif");
|
|
140
|
+
if (path === undefined) return;
|
|
141
|
+
// 文言はファイルの言語で描く。by_path で 1 つの repo に 2 言語が混ざるため。
|
|
142
|
+
const located = results.flatMap((result) =>
|
|
143
|
+
result.outcome.findings.map((finding) => ({ path: result.outcome.path, finding, language: result.language, rules: result.rules })),
|
|
144
|
+
);
|
|
145
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
146
|
+
writeFileSync(path, renderSarif(located, VERSION), "utf8");
|
|
147
|
+
console.log(`\n SARIF を書きました: ${path}(${located.length} 件)`);
|
|
148
|
+
};
|
|
149
|
+
|
|
121
150
|
const lint = async (targets: readonly string[], argv: readonly string[]): Promise<number> => {
|
|
122
151
|
const paths = collectTargets(targets);
|
|
123
152
|
if (paths.length === 0) {
|
|
@@ -132,6 +161,7 @@ const lint = async (targets: readonly string[], argv: readonly string[]): Promis
|
|
|
132
161
|
return 1;
|
|
133
162
|
}
|
|
134
163
|
const results = await Promise.all(paths.map((path) => inspect(path, config, argv)));
|
|
164
|
+
writeSarif(results, argv);
|
|
135
165
|
results.filter((result) => result.outcome.findings.length > 0 || paths.length === 1).forEach((result) => console.log(result.text));
|
|
136
166
|
renderSummary(results.map((result) => result.outcome)).forEach((line) => console.log(line));
|
|
137
167
|
return results.some((result) => result.outcome.findings.some((finding) => finding.severity === "error")) ? 1 : 0;
|
package/src/config/load.ts
CHANGED
|
@@ -29,7 +29,7 @@ export type Config = {
|
|
|
29
29
|
|
|
30
30
|
/** 判定の質が誤検知に直結するので、既定は最上位のモデル。cost は絞り込みで削る。spec §14。 */
|
|
31
31
|
/** 既定は Anthropic。openai にすると判定役だけが替わり、rule も判定の形も変わらない。 */
|
|
32
|
-
|
|
32
|
+
const DEFAULT_BACKEND: BackendName = "anthropic";
|
|
33
33
|
|
|
34
34
|
export const EMPTY: Config = {
|
|
35
35
|
genre: undefined,
|
|
@@ -134,7 +134,8 @@ export const titleCaseMix: Detector = (doc, options): Finding[] => {
|
|
|
134
134
|
const title = judged.filter((entry) => entry.title).length;
|
|
135
135
|
const minorityIsTitle = title <= judged.length - title;
|
|
136
136
|
const few = minorityIsTitle ? title : judged.length - title;
|
|
137
|
-
|
|
137
|
+
// 同数なら少数派は無い。どちらかを「他と違う」と呼ぶのは、選びかたが恣意的になる。
|
|
138
|
+
if (few === 0 || few * 2 === judged.length || few > options.limit) return [];
|
|
138
139
|
return judged
|
|
139
140
|
.filter((entry) => entry.title === minorityIsTitle)
|
|
140
141
|
.map(({ section }) => ({
|
|
@@ -173,7 +174,8 @@ export const oxfordComma: Detector = (doc, options): Finding[] => {
|
|
|
173
174
|
const withComma = judged.filter((entry) => entry.oxford).length;
|
|
174
175
|
const minorityUsesComma = withComma <= judged.length - withComma;
|
|
175
176
|
const few = minorityUsesComma ? withComma : judged.length - withComma;
|
|
176
|
-
|
|
177
|
+
// 同数なら少数派は無い。
|
|
178
|
+
if (few === 0 || few * 2 === judged.length || few > options.limit) return [];
|
|
177
179
|
return judged
|
|
178
180
|
.filter((entry) => entry.oxford === minorityUsesComma)
|
|
179
181
|
.map(({ sentence }) => ({
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { lengthOf } from "../measure.ts";
|
|
2
|
+
import type { Detector, Finding, LengthUnit, Section } from "../plugin.ts";
|
|
2
3
|
|
|
3
4
|
/** 見出しが短すぎると、偶然の一致で 100% になる。これ未満の見出しは見ない。 */
|
|
4
5
|
const MIN_GRAMS = 4;
|
|
@@ -8,7 +9,9 @@ const MIN_GRAMS = 4;
|
|
|
8
9
|
* 語 n-gram にすると wordSplit capability が要り、L1(言語を問わず動く)から外れる。
|
|
9
10
|
*/
|
|
10
11
|
const trigrams = (text: string): Set<string> => {
|
|
11
|
-
|
|
12
|
+
// 大文字小文字を畳む。英語では見出しが Title Case、本文が小文字になり、
|
|
13
|
+
// 同じ語でも一致しなくなる(Generating Output → generated output)。
|
|
14
|
+
const clean = [...text.toLowerCase().replace(/\s+/gu, "")];
|
|
12
15
|
return new Set(clean.slice(0, Math.max(0, clean.length - 2)).map((__char, index) => clean.slice(index, index + 3).join("")));
|
|
13
16
|
};
|
|
14
17
|
|
|
@@ -25,9 +28,32 @@ const containment = (heading: Set<string>, sentence: Set<string>): number => {
|
|
|
25
28
|
return [...heading].filter((gram) => sentence.has(gram)).length / heading.size;
|
|
26
29
|
};
|
|
27
30
|
|
|
31
|
+
/**
|
|
32
|
+
* 見出しの語を含んでいても、文がそのぶん以上に中身を足していれば「何も受け取れない」ではない。
|
|
33
|
+
*
|
|
34
|
+
* 実文書(英語 11 本)で測ったら、この条件なしでは 72.7% の文書が該当した。
|
|
35
|
+
* 「## ToolsAgent」に対する「GraphAI provides ToolsAgent components that use LLMs to…」は
|
|
36
|
+
* 見出しの語を含むが、読み進める価値がある。見出し以外の中身の量で分ける。
|
|
37
|
+
*/
|
|
38
|
+
const NEW_MATERIAL = { word: 6, char: 20 };
|
|
39
|
+
|
|
40
|
+
const headingUnits = (heading: string, unit: LengthUnit): number =>
|
|
41
|
+
unit === "word"
|
|
42
|
+
? heading
|
|
43
|
+
.trim()
|
|
44
|
+
.split(/\s+/u)
|
|
45
|
+
.filter((word) => word.length > 0).length
|
|
46
|
+
: heading.replace(/\s+/gu, "").length;
|
|
47
|
+
|
|
48
|
+
const addsLittle = (section: Section, unit: LengthUnit): boolean => {
|
|
49
|
+
const first = section.firstSentence;
|
|
50
|
+
if (first === undefined) return false;
|
|
51
|
+
return lengthOf(first, unit) - headingUnits(section.heading, unit) <= NEW_MATERIAL[unit];
|
|
52
|
+
};
|
|
53
|
+
|
|
28
54
|
export const headingEcho: Detector = (doc, options): Finding[] =>
|
|
29
55
|
doc.sections
|
|
30
|
-
.filter((section) => section.heading.length > 0 && section.firstSentence !== undefined)
|
|
56
|
+
.filter((section) => section.heading.length > 0 && section.firstSentence !== undefined && addsLittle(section, doc.lengthUnit))
|
|
31
57
|
.map((section) => ({ section, overlap: Math.round(containment(trigrams(section.heading), trigrams(section.firstSentence?.text ?? "")) * 100) }))
|
|
32
58
|
.filter(({ overlap }) => overlap >= options.limit)
|
|
33
59
|
.map(({ section, overlap }) => ({
|
package/src/detectors/index.ts
CHANGED
|
@@ -11,8 +11,8 @@ import { doubledParticle, nounEnding } from "./token-shape.ts";
|
|
|
11
11
|
import { kanjiRun, middleDot } from "./char-shape.ts";
|
|
12
12
|
import { adverbDensity, conjunctionRun, expletive, oxfordComma, titleCaseMix } from "./en-shape.ts";
|
|
13
13
|
import { paragraphLength, paragraphVariance, preambleLength, ruleOfThree, sectionUniformity } from "./structure.ts";
|
|
14
|
-
import { concreteEvidence, emojiDensity, ngramRepetition, undefinedAcronym } from "./signals.ts";
|
|
15
|
-
import { aiTell, cushionDensity, hedgingDensity, repeatedConjunction, unqualifiedSuperlative } from "./lexicon.ts";
|
|
14
|
+
import { concreteEvidence, dashDensity, emojiDensity, ngramRepetition, undefinedAcronym } from "./signals.ts";
|
|
15
|
+
import { aiTell, contractionMix, cushionDensity, hedgingDensity, repeatedConjunction, unqualifiedSuperlative } from "./lexicon.ts";
|
|
16
16
|
import { internalJargon, properNounDensity, requiredSections } from "./team.ts";
|
|
17
17
|
|
|
18
18
|
/** rule 定義の how_to_find がここを引く。rule 側は実装を知らない。 */
|
|
@@ -40,6 +40,7 @@ export const DETECTORS: Readonly<Record<string, Detector>> = {
|
|
|
40
40
|
"rule-of-three": ruleOfThree,
|
|
41
41
|
"preamble-length": preambleLength,
|
|
42
42
|
"emoji-density": emojiDensity,
|
|
43
|
+
"dash-density": dashDensity,
|
|
43
44
|
"ngram-repetition": ngramRepetition,
|
|
44
45
|
"undefined-acronym": undefinedAcronym,
|
|
45
46
|
"concrete-evidence": concreteEvidence,
|
|
@@ -48,6 +49,7 @@ export const DETECTORS: Readonly<Record<string, Detector>> = {
|
|
|
48
49
|
"unqualified-superlative": unqualifiedSuperlative,
|
|
49
50
|
"repeated-conjunction": repeatedConjunction,
|
|
50
51
|
"ai-tell": aiTell,
|
|
52
|
+
"contraction-mix": contractionMix,
|
|
51
53
|
"internal-jargon": internalJargon,
|
|
52
54
|
"required-sections": requiredSections,
|
|
53
55
|
"proper-noun-density": properNounDensity,
|
package/src/detectors/lexicon.ts
CHANGED
|
@@ -122,3 +122,44 @@ export const aiTell: Detector = (doc, options): Finding[] => {
|
|
|
122
122
|
},
|
|
123
123
|
];
|
|
124
124
|
};
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* 同じことを言う 2 つの書きかたが、1 つの文書で混ざっているか。
|
|
128
|
+
*
|
|
129
|
+
* **どちらが正しいかは決めない。** 短縮形を使うかどうかは文体の選択で、
|
|
130
|
+
* 立場を取ると方針の違う書き手に rule ごと無視される。spec §12.3。
|
|
131
|
+
*
|
|
132
|
+
* 語彙表は対を持つ(`pattern` と `instead_of`)。片方だけを数えても、
|
|
133
|
+
* 意図した硬い文体なのか不統一なのかが分からない。
|
|
134
|
+
*/
|
|
135
|
+
/**
|
|
136
|
+
* 語の境界で照合する。部分一致だと「it isn't」が「it is」を含んでしまい、
|
|
137
|
+
* 短縮形を使っている文が「使っていない」側に数えられる。
|
|
138
|
+
*/
|
|
139
|
+
const containsWord = (text: string, word: string): boolean => new RegExp(`\\b${word.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&")}\\b`, "u").test(text);
|
|
140
|
+
|
|
141
|
+
export const contractionMix: Detector = (doc, options): Finding[] => {
|
|
142
|
+
const pairs = (options.lexicon ?? []).flatMap((entry) => (entry.instead_of === undefined ? [] : [{ short: entry.pattern, long: entry.instead_of }]));
|
|
143
|
+
const body = doc.sentences.map((sentence) => sentence.text.toLowerCase()).join(" ");
|
|
144
|
+
const used = pairs.filter((pair) => containsWord(body, pair.short.toLowerCase()));
|
|
145
|
+
const spelled = pairs.filter((pair) => containsWord(body, pair.long.toLowerCase()));
|
|
146
|
+
// 片方しか無ければ一貫している。両方あるときだけ、少数派を指摘する。
|
|
147
|
+
if (used.length === 0 || spelled.length === 0) return [];
|
|
148
|
+
const minorityIsShort = used.length <= spelled.length;
|
|
149
|
+
const few = minorityIsShort ? used : spelled;
|
|
150
|
+
if (few.length > options.limit) return [];
|
|
151
|
+
const wanted = few.map((pair) => (minorityIsShort ? pair.short : pair.long).toLowerCase());
|
|
152
|
+
const hits = doc.sentences.flatMap((sentence) => {
|
|
153
|
+
const text = sentence.text.toLowerCase();
|
|
154
|
+
const matched = wanted.find((word) => containsWord(text, word));
|
|
155
|
+
return matched === undefined ? [] : [{ sentence, matched }];
|
|
156
|
+
});
|
|
157
|
+
return hits.map((hit) => ({
|
|
158
|
+
rule: "contraction-consistency",
|
|
159
|
+
severity: "info",
|
|
160
|
+
line: 0,
|
|
161
|
+
column: 0,
|
|
162
|
+
quote: hit.sentence.text.trim(),
|
|
163
|
+
values: { matched: hit.matched, count: few.length, limit: options.limit, offset: hit.sentence.span.start },
|
|
164
|
+
}));
|
|
165
|
+
};
|
package/src/detectors/signals.ts
CHANGED
|
@@ -49,13 +49,16 @@ export const emojiDensity: Detector = (doc, options): Finding[] => {
|
|
|
49
49
|
* 同じ言い回しの繰り返し。character n-gram で見るので、語の区切りが要らない。
|
|
50
50
|
* spec §10 が n-gram を選んだのはこのためで、新しい言語でそのまま動く。
|
|
51
51
|
*/
|
|
52
|
-
|
|
52
|
+
/**
|
|
53
|
+
* 窓の幅は単位で分ける。同じ 8 文字でも、日本語では 4〜5 形態素だが英語では 1 語半にしかならない。
|
|
54
|
+
* 実文書(英語 11 本)で 8 文字にしたら、上位は " generat"(generate)のような**語の断片**だった。
|
|
55
|
+
*/
|
|
56
|
+
const GRAM = { char: 8, word: 20 };
|
|
53
57
|
|
|
54
|
-
|
|
55
|
-
const countGrams = (text: string): Map<string, number> => {
|
|
58
|
+
const countGrams = (text: string, width: number): Map<string, number> => {
|
|
56
59
|
const counts = new Map<string, number>();
|
|
57
|
-
Array.from({ length: Math.max(0, text.length -
|
|
58
|
-
const gram = text.slice(index, index +
|
|
60
|
+
Array.from({ length: Math.max(0, text.length - width + 1) }).forEach((_, index) => {
|
|
61
|
+
const gram = text.slice(index, index + width);
|
|
59
62
|
counts.set(gram, (counts.get(gram) ?? 0) + 1);
|
|
60
63
|
});
|
|
61
64
|
return counts;
|
|
@@ -69,7 +72,7 @@ const countGrams = (text: string): Map<string, number> => {
|
|
|
69
72
|
* どちらを見るかは adapter が宣言する単位で決める。ひらがなを持たない char 単位の言語が
|
|
70
73
|
* 来たら、この見分けは効かなくなる(そのときは adapter 側に判定を移す)。
|
|
71
74
|
*/
|
|
72
|
-
const CONNECTIVE = { char: /[ぁ-ゖ]/u, word:
|
|
75
|
+
const CONNECTIVE = { char: /[ぁ-ゖ]/u, word: /\S \S.*\S \S/u };
|
|
73
76
|
|
|
74
77
|
const isPhrasing = (gram: string, unit: ProseDocument["lengthUnit"]): boolean => CONNECTIVE[unit].test(gram);
|
|
75
78
|
|
|
@@ -78,7 +81,7 @@ const gramsOf = (doc: ProseDocument): Map<string, number> => {
|
|
|
78
81
|
const counts = new Map<string, number>();
|
|
79
82
|
doc.sentences.forEach((sentence) => {
|
|
80
83
|
const text = doc.lengthUnit === "char" ? proseText(sentence).replace(/\s+/gu, "") : proseText(sentence);
|
|
81
|
-
countGrams(text).forEach((count, gram) => counts.set(gram, (counts.get(gram) ?? 0) + count));
|
|
84
|
+
countGrams(text, GRAM[doc.lengthUnit]).forEach((count, gram) => counts.set(gram, (counts.get(gram) ?? 0) + count));
|
|
82
85
|
});
|
|
83
86
|
return counts;
|
|
84
87
|
};
|
|
@@ -143,14 +146,21 @@ const COMMON = new Set([
|
|
|
143
146
|
/**
|
|
144
147
|
* 展開は略語の**すぐ隣**にあるときだけ認める。
|
|
145
148
|
* 60 文字も見ると、同じ文のどこかに括弧があるだけで「説明済み」になり、1 件も出なくなる。
|
|
149
|
+
*
|
|
150
|
+
* 認めるのは 2 つの形。どちらも実際によく書かれる。
|
|
151
|
+
* CI(継続的インテグレーション) 略語のあとに括弧
|
|
152
|
+
* Continuous Integration (CI) 括弧の中が略語
|
|
146
153
|
*/
|
|
147
|
-
const
|
|
148
|
-
const
|
|
154
|
+
const OPENS = /^\s*[((]/u;
|
|
155
|
+
const CLOSES = /^\s*[))]/u;
|
|
156
|
+
const OPENED = /[((]\s*$/u;
|
|
149
157
|
|
|
150
158
|
const isExpanded = (body: string, acronym: string): boolean => {
|
|
151
159
|
const at = body.indexOf(acronym);
|
|
152
160
|
if (at === -1) return false;
|
|
153
|
-
|
|
161
|
+
const after = body.slice(at + acronym.length, at + acronym.length + 3);
|
|
162
|
+
const before = body.slice(Math.max(0, at - 3), at);
|
|
163
|
+
return OPENS.test(after) || (OPENED.test(before) && CLOSES.test(after));
|
|
154
164
|
};
|
|
155
165
|
|
|
156
166
|
export const undefinedAcronym: Detector = (doc, options): Finding[] => {
|
|
@@ -197,3 +207,26 @@ export const concreteEvidence: Detector = (doc, options): Finding[] => {
|
|
|
197
207
|
},
|
|
198
208
|
}));
|
|
199
209
|
};
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* ダッシュの多用。英語では正当な用法が多いが、2026 年時点で最も知られた生成文のシグナルでもある。
|
|
213
|
+
* 日本語の組版ではそもそも扱いが難しい。どちらが正しいかは severity の言語別指定で分ける。spec §12.4。
|
|
214
|
+
*/
|
|
215
|
+
const DASH = /[\u2014\u2015\u2013]/gu;
|
|
216
|
+
|
|
217
|
+
export const dashDensity: Detector = (doc, options): Finding[] => {
|
|
218
|
+
const hits = findIn(doc, DASH);
|
|
219
|
+
const first = hits[0];
|
|
220
|
+
const rate = density(doc, hits.length);
|
|
221
|
+
if (wordsOf(doc) < FLOOR[doc.lengthUnit] || first === undefined || rate <= options.limit) return [];
|
|
222
|
+
return [
|
|
223
|
+
{
|
|
224
|
+
rule: "no-em-dash",
|
|
225
|
+
severity: "info",
|
|
226
|
+
line: 0,
|
|
227
|
+
column: 0,
|
|
228
|
+
quote: first.sentence.text.trim(),
|
|
229
|
+
values: { count: hits.length, density: rate, limit: options.limit, offset: first.offset },
|
|
230
|
+
},
|
|
231
|
+
];
|
|
232
|
+
};
|
package/src/document.ts
CHANGED
|
@@ -76,7 +76,17 @@ const EMPHASIS = new Set(["strong", "emphasis", "delete"]);
|
|
|
76
76
|
*/
|
|
77
77
|
const DIRECTIVE = /^:::[^\n]*/gmu;
|
|
78
78
|
|
|
79
|
-
|
|
79
|
+
/**
|
|
80
|
+
* `https://…` をそのまま書いた URL。GFM の autolink 拡張を入れていないので mdast では
|
|
81
|
+
* ただのテキストになり、本文として残る。残すと、見出しと URL の中の識別子が一致して
|
|
82
|
+
* 「見出しの繰り返し」と読まれる。表示される文字も本文ではない。
|
|
83
|
+
*/
|
|
84
|
+
const BARE_URL = /https?:\/\/[^\s)<>"'\]]+/gu;
|
|
85
|
+
|
|
86
|
+
const matchSpans = (source: string, pattern: RegExp): Span[] =>
|
|
87
|
+
[...source.matchAll(pattern)].map((match) => ({ start: match.index, end: match.index + match[0].length }));
|
|
88
|
+
|
|
89
|
+
const directiveSpans = (source: string): Span[] => [...matchSpans(source, DIRECTIVE), ...matchSpans(source, BARE_URL)];
|
|
80
90
|
|
|
81
91
|
const collectMasks = (root: Node, source: string): Span[] => {
|
|
82
92
|
const spans: Span[] = [...directiveSpans(source)];
|
package/src/eval.ts
CHANGED
|
@@ -63,9 +63,21 @@ const recommend = (rule: RuleDefinition, sweep: readonly Point[], total: number)
|
|
|
63
63
|
return strictest;
|
|
64
64
|
};
|
|
65
65
|
|
|
66
|
+
/**
|
|
67
|
+
* lint で動かない rule は掃引しない。
|
|
68
|
+
* 動かないものの閾値を測ると、「どの閾値でも 0 件」が「よく校正されている」に見える。
|
|
69
|
+
*/
|
|
70
|
+
const measurable = (rule: RuleDefinition, docs: readonly ProseDocument[], genre: string, language: string): boolean => {
|
|
71
|
+
if (rule.layer === "L4" || !rule.use_for.some((target) => genre.startsWith(target))) return false;
|
|
72
|
+
if (rule.languages !== undefined && !rule.languages.includes(language)) return false;
|
|
73
|
+
if (rule.from.length > 0) return false;
|
|
74
|
+
const capabilities = docs[0]?.capabilities;
|
|
75
|
+
return !rule.requires.some((need) => (need === "lemma" ? capabilities?.lemma : capabilities?.pos) !== true);
|
|
76
|
+
};
|
|
77
|
+
|
|
66
78
|
export const evaluate = (docs: readonly ProseDocument[], rules: readonly RuleDefinition[], genre: string, language: string): RuleReport[] =>
|
|
67
79
|
rules
|
|
68
|
-
.filter((rule) => rule
|
|
80
|
+
.filter((rule) => measurable(rule, docs, genre, language))
|
|
69
81
|
.map((rule) => {
|
|
70
82
|
const sweep = candidates(rule).map((limit) => countAt(docs, rule, limit));
|
|
71
83
|
const current = rule.levels.normal ?? 1;
|
package/src/files.ts
CHANGED
|
@@ -26,7 +26,7 @@ const expand = (target: string): string[] => {
|
|
|
26
26
|
* ロケールを明示する。既定のロケールに任せると、同じ入力でも機械によって順序が変わる。
|
|
27
27
|
* 指摘の並びは CI のログにも baseline にも入るので、順序が揺れてはいけない。
|
|
28
28
|
*/
|
|
29
|
-
|
|
29
|
+
const byPath = (left: string, right: string): number => left.localeCompare(right, "en");
|
|
30
30
|
|
|
31
31
|
export const collectTargets = (targets: readonly string[]): string[] => {
|
|
32
32
|
const found = targets.flatMap(expand).filter((path) => !isSkipped(path));
|
package/src/levels.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Level, LevelTable, RuleDefinition } from "./plugin.ts";
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
const LEVELS: readonly Level[] = ["strict", "normal", "relaxed", "off"];
|
|
4
4
|
|
|
5
5
|
export const isLevel = (value: unknown): value is Level => typeof value === "string" && LEVELS.some((level) => level === value);
|
|
6
6
|
|
package/src/mask.ts
CHANGED
|
@@ -7,7 +7,14 @@ import type { Span } from "./plugin.ts";
|
|
|
7
7
|
* 切り出して繋ぐのではなく覆うのは、オフセットを元文字列と一致させ続けるため。
|
|
8
8
|
* 行・列の計算も、引用して見せる範囲も、これで元の位置のまま使える。
|
|
9
9
|
*/
|
|
10
|
-
|
|
10
|
+
/**
|
|
11
|
+
* 空白は**元の文字と同じ UTF-16 長**にする。
|
|
12
|
+
*
|
|
13
|
+
* `u` 付きの正規表現は 1 文字(コードポイント)ずつ当たるので、絵文字のような
|
|
14
|
+
* サロゲートペアを空白 1 つに置き換えると文字列が 1 だけ縮む。mdast のオフセットは
|
|
15
|
+
* UTF-16 単位なので、そこから先の指摘がすべて 1 ずれる。
|
|
16
|
+
*/
|
|
17
|
+
const blankOut = (text: string): string => text.replace(/[^\n]/gu, (char) => " ".repeat(char.length));
|
|
11
18
|
|
|
12
19
|
const merge = (spans: readonly Span[]): Span[] =>
|
|
13
20
|
[...spans]
|
package/src/measure.ts
CHANGED
|
@@ -13,7 +13,7 @@ export const proseText = (sentence: Sentence): string => sentence.text.replace(/
|
|
|
13
13
|
|
|
14
14
|
export const charLength = (sentence: Sentence): number => sentence.text.replace(/\s+/gu, "").length;
|
|
15
15
|
|
|
16
|
-
|
|
16
|
+
const wordLength = (sentence: Sentence): number =>
|
|
17
17
|
proseText(sentence)
|
|
18
18
|
.split(" ")
|
|
19
19
|
.filter((word) => word.length > 0).length;
|