jupytermind 0.3.0

package/.github/skills/japanese-prose/scripts/lint.py

@@ -0,0 +1,378 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#   "ginza>=5.2.0,<5.3",
+#   "ja-ginza>=5.2.0,<5.3",
+#   "spacy>=3.8.0,<4",
+# ]
+# ///
+"""GiNZA-based Japanese prose diagnostics developed for kotonoha."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import statistics
+import sys
+from collections import Counter
+from pathlib import Path
+
+from core import (
+    Finding,
+    content_tokens,
+    dependency_depth,
+    excerpt,
+    finding_key,
+    iter_prose_blocks,
+    load_ginza,
+    read_text,
+    score_findings,
+    sentence_line,
+)
+
+
+PHRASE_RULES = {
+    "と言えるでしょう": ("formulaic_conclusion", "断定を弱める定型句です。根拠か具体的な判断に置き換えます。"),
+    "と言えるだろう": ("formulaic_conclusion", "断定を弱める定型句です。根拠か具体的な判断に置き換えます。"),
+    "いかがでしょうか": ("reader_prompt_cliche", "読者への定型的な呼びかけです。必要な行動を直接示します。"),
+    "結論として": ("formulaic_transition", "結論の内容を見出しまたは先頭文で直接示します。"),
+    "以下のとおりです": ("empty_preview", "後続内容の要点を先に示します。"),
+    "重要なポイント": ("abstract_emphasis", "何が重要かを具体的な条件や数値で示します。"),
+}
+
+TRANSLATIONESE_RULES = (
+    (
+        re.compile(r"することが(?:可能|できます|できる)"),
+        "indirect_ability",
+        "「できる」または具体的な動作へ簡潔にします。",
+    ),
+    (
+        re.compile(r"において(?:は|、)?"),
+        "heavy_location_phrase",
+        "場所・場面を表すなら「で」で意味が保てるか確認します。",
+    ),
+    (
+        re.compile(r"を通じて(?:、)?"),
+        "through_translation",
+        "手段・期間・経験のどれを表すか具体化します。",
+    ),
+)
+
+CONNECTIVES = {
+    "また",
+    "さらに",
+    "一方",
+    "そのため",
+    "したがって",
+    "しかし",
+    "なお",
+}
+
+DOUBLE_NEGATIVE_RULES = (
+    re.compile(r"ないとは言えない"),
+    re.compile(r"ないわけではない"),
+    re.compile(r"ないことはない"),
+    re.compile(r"なくはない"),
+    re.compile(r"なくてはならない"),
+    re.compile(r"ざるを得ない"),
+)
+
+GENRE_LIMITS = {
+    "general": {"length": 90, "critical_length": 150, "commas": 5, "depth": 8},
+    "tech": {"length": 110, "critical_length": 180, "commas": 6, "depth": 9},
+    "business": {"length": 80, "critical_length": 140, "commas": 5, "depth": 8},
+}
+
+
+def add(
+    findings: list[Finding],
+    line: int,
+    category: str,
+    severity: str,
+    message: str,
+    text: str,
+    evidence: dict | None = None,
+) -> None:
+    findings.append(
+        Finding(line, category, severity, message, excerpt(text), evidence)
+    )
+
+
+def analyze(path: Path, genre: str, reading_load: bool) -> dict:
+    text = read_text(path)
+    blocks = iter_prose_blocks(text)
+    nlp = load_ginza(enable_ner=False)
+    limits = GENRE_LIMITS[genre]
+    findings: list[Finding] = []
+    sentence_records: list[dict] = []
+    paragraph_leads: list[tuple[int, str]] = []
+
+    for block, doc in zip(blocks, nlp.pipe([item.text for item in blocks])):
+        paragraph_tokens = content_tokens(doc)
+        if paragraph_tokens:
+            paragraph_leads.append((block.start_line, paragraph_tokens[0].lemma_))
+
+        for phrase, (category, message) in PHRASE_RULES.items():
+            if not reading_load and phrase in block.text:
+                add(findings, block.start_line, category, "warning", message, block.text)
+
+        if not reading_load:
+            for pattern, category, message in TRANSLATIONESE_RULES:
+                if pattern.search(block.text):
+                    add(findings, block.start_line, category, "info", message, block.text)
+
+        for sent in doc.sents:
+            tokens = content_tokens(sent)
+            if not tokens:
+                continue
+            sent_text = sent.text.strip()
+            line = sentence_line(block, sent.start_char)
+            length = len(re.sub(r"\s+", "", sent_text))
+            comma_count = sent_text.count("、")
+            max_depth = max((dependency_depth(token) for token in tokens), default=0)
+
+            noun_run = 0
+            max_noun_run = 0
+            genitive_count = 0
+            max_genitive_count = 0
+            for token in sent:
+                if token.is_space or token.is_punct or token.pos_ == "SYM":
+                    noun_run = 0
+                    genitive_count = 0
+                    continue
+                if token.pos_ in {"NOUN", "PROPN"}:
+                    noun_run += 1
+                    max_noun_run = max(max_noun_run, noun_run)
+                else:
+                    noun_run = 0
+                if token.lemma_ == "の" and token.pos_ == "ADP":
+                    genitive_count += 1
+                    max_genitive_count = max(max_genitive_count, genitive_count)
+                elif token.pos_ in {"VERB", "AUX"}:
+                    genitive_count = 0
+
+            first_lemmas = tuple(token.lemma_ for token in tokens[:2])
+            final_pos = tokens[-1].pos_
+            verb_count = sum(
+                token.pos_ in {"VERB", "AUX"} for token in tokens
+            )
+            enumeration = comma_count > limits["commas"] and verb_count <= 1
+            sentence_records.append(
+                {
+                    "line": line,
+                    "text": sent_text,
+                    "length": length,
+                    "first": first_lemmas,
+                    "final_pos": final_pos,
+                }
+            )
+
+            if length > limits["length"] and not enumeration:
+                severity = (
+                    "critical"
+                    if length > limits["critical_length"]
+                    else "warning"
+                )
+                add(
+                    findings,
+                    line,
+                    "long_sentence",
+                    severity,
+                    "一文が長く、係り受けを保持しにくい状態です。意味単位で分割できるか確認します。",
+                    sent_text,
+                    {"characters": length, "threshold": limits["length"]},
+                )
+            if comma_count > limits["commas"] and not enumeration:
+                add(
+                    findings,
+                    line,
+                    "comma_overload",
+                    "warning",
+                    "読点が多く、複数の論点が一文に入っている可能性があります。",
+                    sent_text,
+                    {"commas": comma_count, "threshold": limits["commas"]},
+                )
+            if max_depth > limits["depth"] and not enumeration:
+                add(
+                    findings,
+                    line,
+                    "deep_dependency",
+                    "warning",
+                    "係り受けが深く、文の骨格を追う負荷が高い状態です。",
+                    sent_text,
+                    {"depth": max_depth, "threshold": limits["depth"]},
+                )
+            if max_noun_run >= 6:
+                add(
+                    findings,
+                    line,
+                    "noun_chain",
+                    "warning",
+                    "名詞が連続しています。助詞や述語を補って関係を明示します。",
+                    sent_text,
+                    {"consecutive_nouns": max_noun_run},
+                )
+            if max_genitive_count >= 3:
+                add(
+                    findings,
+                    line,
+                    "genitive_chain",
+                    "info",
+                    "「の」が重なっています。所有・対象・所属の関係を分けられるか確認します。",
+                    sent_text,
+                    {"consecutive_genitives": max_genitive_count},
+                )
+            if any(pattern.search(sent_text) for pattern in DOUBLE_NEGATIVE_RULES):
+                add(
+                    findings,
+                    line,
+                    "double_negative",
+                    "warning",
+                    "否定が重なっています。肯定文で同じ意味を表せるか確認します。",
+                    sent_text,
+                )
+
+    if not reading_load:
+        for previous, current in zip(sentence_records, sentence_records[1:]):
+            if current["first"] and current["first"] == previous["first"]:
+                add(
+                    findings,
+                    current["line"],
+                    "repeated_sentence_lead",
+                    "info",
+                    "連続する文が同じ語順で始まっています。意図した反復か確認します。",
+                    current["text"],
+                    {"lemmas": list(current["first"])},
+                )
+
+        if len(sentence_records) >= 5:
+            lengths = [record["length"] for record in sentence_records]
+            mean = statistics.mean(lengths)
+            if mean and statistics.pstdev(lengths) / mean < 0.16:
+                add(
+                    findings,
+                    0,
+                    "uniform_sentence_rhythm",
+                    "info",
+                    "文長が均一です。重要度に応じた長短があるか通読します。",
+                    "文書全体の文長分布",
+                    {"coefficient_of_variation": round(statistics.pstdev(lengths) / mean, 3)},
+                )
+
+            nominal_count = sum(
+                record["final_pos"] in {"NOUN", "PROPN"}
+                for record in sentence_records
+            )
+            if nominal_count / len(sentence_records) >= 0.55:
+                add(
+                    findings,
+                    0,
+                    "nominal_ending_density",
+                    "info",
+                    "体言止めが多く、説明の関係が省略されている可能性があります。",
+                    "文書全体の文末分布",
+                    {
+                        "ratio": round(nominal_count / len(sentence_records), 3),
+                        "sentences": len(sentence_records),
+                    },
+                )
+
+        lead_counts = Counter(lemma for _, lemma in paragraph_leads if lemma in CONNECTIVES)
+        for lemma, count in lead_counts.items():
+            if count >= 3:
+                line = next(line for line, value in paragraph_leads if value == lemma)
+                add(
+                    findings,
+                    line,
+                    "repeated_paragraph_connective",
+                    "info",
+                    "複数段落が同じ接続表現で始まっています。",
+                    lemma,
+                    {"connective": lemma, "count": count},
+                )
+
+    findings.sort(key=lambda item: (item.line, item.category))
+    return {
+        "file": str(path),
+        "engine": "GiNZA",
+        "model": nlp.meta.get("name", "ja_ginza"),
+        "genre": genre,
+        "mode": "reading-load" if reading_load else "prose",
+        "score": score_findings(findings),
+        "finding_count": len(findings),
+        "findings": [item.to_dict() for item in findings],
+    }
+
+
+def compare_baseline(result: dict, baseline_path: Path) -> None:
+    baseline = json.loads(read_text(baseline_path))
+    previous = {
+        finding_key(
+            Finding(
+                item.get("line", 0),
+                item["category"],
+                item.get("severity", "warning"),
+                item.get("message", ""),
+                item.get("excerpt", ""),
+            )
+        )
+        for item in baseline.get("findings", [])
+    }
+    current = set()
+    for item in result["findings"]:
+        finding = Finding(
+            item["line"],
+            item["category"],
+            item["severity"],
+            item["message"],
+            item["excerpt"],
+            item.get("evidence"),
+        )
+        key = finding_key(finding)
+        current.add(key)
+        item["status"] = "persisting" if key in previous else "new"
+    result["resolved_count"] = len(previous - current)
+    result["new_count"] = len(current - previous)
+    result["persisting_count"] = len(current & previous)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="GiNZA-based Japanese prose lint")
+    parser.add_argument("file")
+    parser.add_argument("--genre", choices=GENRE_LIMITS, default="general")
+    parser.add_argument("--reading-load", action="store_true")
+    parser.add_argument("--baseline")
+    parser.add_argument("--json", action="store_true")
+    args = parser.parse_args()
+
+    try:
+        result = analyze(Path(args.file), args.genre, args.reading_load)
+        if args.baseline:
+            compare_baseline(result, Path(args.baseline))
+    except (OSError, ValueError) as error:
+        print(f"error: {error}", file=sys.stderr)
+        return 1
+    except Exception as error:
+        print(f"error: GiNZA analysis failed: {error}", file=sys.stderr)
+        return 1
+
+    if args.json:
+        print(json.dumps(result, ensure_ascii=False, indent=2))
+    else:
+        print(
+            f"{result['file']}: score {result['score']}/100, "
+            f"{result['finding_count']} finding(s)"
+        )
+        for finding in result["findings"]:
+            print(
+                f"  L{finding['line']} [{finding['severity']}] "
+                f"{finding['category']}: {finding['message']}"
+            )
+            print(f"    > {finding['excerpt']}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/.github/skills/japanese-prose/scripts/outline.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+"""Extract headings and paragraph openings from a Markdown document."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+from core import excerpt, iter_content_lines, iter_prose_blocks, read_text
+
+
+HEADING_CAPTURE_RE = re.compile(r"^\s{0,3}(#{1,6})\s+(.+?)\s*$")
+
+
+def extract(path: Path) -> dict:
+    text = read_text(path)
+    headings = []
+    for line_number, line in iter_content_lines(text):
+        match = HEADING_CAPTURE_RE.match(line)
+        if match:
+            headings.append(
+                {
+                    "line": line_number,
+                    "level": len(match.group(1)),
+                    "text": match.group(2),
+                }
+            )
+    paragraphs = [
+        {"line": block.start_line, "opening": excerpt(block.text, 140)}
+        for block in iter_prose_blocks(text)
+    ]
+    return {
+        "file": str(path),
+        "heading_count": len(headings),
+        "paragraph_count": len(paragraphs),
+        "headings": headings,
+        "paragraphs": paragraphs,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Extract a Markdown skeleton")
+    parser.add_argument("file")
+    parser.add_argument("--json", action="store_true")
+    args = parser.parse_args()
+    try:
+        result = extract(Path(args.file))
+    except OSError as error:
+        print(f"error: {error}", file=sys.stderr)
+        return 1
+
+    if args.json:
+        print(json.dumps(result, ensure_ascii=False, indent=2))
+    else:
+        for heading in result["headings"]:
+            print(
+                f"L{heading['line']} H{heading['level']} {heading['text']}"
+            )
+        for paragraph in result["paragraphs"]:
+            print(f"L{paragraph['line']} P {paragraph['opening']}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/.github/skills/japanese-prose/scripts/terms.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#   "ginza>=5.2.0,<5.3",
+#   "ja-ginza>=5.2.0,<5.3",
+#   "spacy>=3.8.0,<4",
+# ]
+# ///
+"""Extract terminology candidates with GiNZA POS and entity analysis."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+from core import iter_prose_blocks, load_ginza, read_text, sentence_line
+
+
+ASCII_TERM_RE = re.compile(r"^[A-Za-z][A-Za-z0-9.+#/_-]{1,}$")
+EXPLANATION_MARKERS = ("とは", "を指す", "と呼ぶ", "（", "(")
+
+
+def is_candidate(token) -> bool:
+    if token.ent_type_:
+        return True
+    if token.pos_ == "PROPN":
+        return True
+    return bool(ASCII_TERM_RE.match(token.text))
+
+
+def extract(path: Path) -> dict:
+    text = read_text(path)
+    blocks = iter_prose_blocks(text)
+    nlp = load_ginza(enable_ner=True)
+    terms: dict[str, dict] = {}
+    occurrences: dict[str, list[int]] = defaultdict(list)
+
+    for block, doc in zip(blocks, nlp.pipe([item.text for item in blocks])):
+        for token in doc:
+            if not is_candidate(token):
+                continue
+            value = token.text.strip()
+            if len(value) < 2:
+                continue
+            line = sentence_line(block, token.idx)
+            occurrences[value].append(line)
+            if value not in terms:
+                context = block.text[
+                    max(0, token.idx - 30) : token.idx + len(value) + 45
+                ]
+                explained = any(
+                    marker in block.text[token.idx + len(value) : token.idx + len(value) + 24]
+                    for marker in EXPLANATION_MARKERS
+                )
+                terms[value] = {
+                    "term": value,
+                    "lemma": token.lemma_,
+                    "pos": token.pos_,
+                    "entity": token.ent_type_ or None,
+                    "first_line": line,
+                    "explained_near_first_use": explained,
+                    "context": re.sub(r"\s+", " ", context).strip(),
+                }
+
+    result_terms = []
+    for value, item in terms.items():
+        item["count"] = len(occurrences[value])
+        item["lines"] = sorted(set(occurrences[value]))
+        result_terms.append(item)
+    result_terms.sort(key=lambda item: (item["first_line"], item["term"]))
+    return {
+        "file": str(path),
+        "engine": "GiNZA",
+        "term_count": len(result_terms),
+        "terms": result_terms,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Extract terminology candidates")
+    parser.add_argument("file")
+    parser.add_argument("--json", action="store_true")
+    args = parser.parse_args()
+    try:
+        result = extract(Path(args.file))
+    except OSError as error:
+        print(f"error: {error}", file=sys.stderr)
+        return 1
+    except Exception as error:
+        print(f"error: GiNZA analysis failed: {error}", file=sys.stderr)
+        return 1
+
+    if args.json:
+        print(json.dumps(result, ensure_ascii=False, indent=2))
+    else:
+        for item in result["terms"]:
+            marker = "explained" if item["explained_near_first_use"] else "review"
+            entity = f" {item['entity']}" if item["entity"] else ""
+            print(
+                f"L{item['first_line']} {item['term']} "
+                f"({item['pos']}{entity}, {item['count']} occurrence(s), {marker})"
+            )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/.github/skills/japanese-prose/scripts/test_engine.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#   "ginza>=5.2.0,<5.3",
+#   "ja-ginza>=5.2.0,<5.3",
+#   "spacy>=3.8.0,<4",
+# ]
+# ///
+"""Behavioral tests for kotonoha's original GiNZA diagnostics."""
+
+from __future__ import annotations
+
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+from core import strip_markdown_inline
+from lint import analyze, compare_baseline
+from outline import extract as extract_outline
+from terms import extract as extract_terms
+
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+FIXTURES = SCRIPT_DIR / "fixtures"
+
+
+class DiagnosticTests(unittest.TestCase):
+    def test_unnatural_fixture_scores_below_natural_fixture(self) -> None:
+        unnatural = analyze(FIXTURES / "unnatural.md", "tech", False)
+        natural = analyze(FIXTURES / "natural.md", "tech", False)
+
+        self.assertLess(unnatural["score"], natural["score"])
+        self.assertEqual(natural["finding_count"], 0)
+        categories = {item["category"] for item in unnatural["findings"]}
+        self.assertIn("formulaic_conclusion", categories)
+        self.assertIn("double_negative", categories)
+        self.assertIn("deep_dependency", categories)
+
+    def test_markdown_code_is_not_analyzed_as_prose(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "code.md"
+            path.write_text(
+                "# Test\n\n自然な説明です。\n\n"
+                "```text\n結論として、利用することが可能です。\n```\n",
+                encoding="utf-8",
+            )
+            result = analyze(path, "tech", False)
+            categories = {item["category"] for item in result["findings"]}
+            self.assertNotIn("formulaic_transition", categories)
+            self.assertNotIn("indirect_ability", categories)
+
+    def test_inline_code_is_masked_without_corrupting_plain_identifiers(self) -> None:
+        self.assertEqual(strip_markdown_inline("`MAX_RETRY_COUNT`"), "コード")
+        self.assertEqual(
+            strip_markdown_inline("MAX_RETRY_COUNTを確認する"),
+            "MAX_RETRY_COUNTを確認する",
+        )
+
+    def test_ordinary_conditional_negation_is_not_double_negative(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "condition.md"
+            path.write_text(
+                "# Test\n\n設定が存在しない場合、既定値は適用されません。\n",
+                encoding="utf-8",
+            )
+            result = analyze(path, "tech", False)
+        categories = {item["category"] for item in result["findings"]}
+        self.assertNotIn("double_negative", categories)
+
+    def test_baseline_reports_resolved_findings(self) -> None:
+        initial = analyze(FIXTURES / "unnatural.md", "tech", False)
+        final = analyze(FIXTURES / "natural.md", "tech", False)
+        with tempfile.TemporaryDirectory() as directory:
+            baseline = Path(directory) / "baseline.json"
+            baseline.write_text(
+                json.dumps(initial, ensure_ascii=False),
+                encoding="utf-8",
+            )
+            compare_baseline(final, baseline)
+        self.assertGreater(final["resolved_count"], 0)
+        self.assertEqual(final["new_count"], 0)
+
+    def test_outline_and_terms_return_structured_results(self) -> None:
+        outline = extract_outline(FIXTURES / "natural.md")
+        terms = extract_terms(FIXTURES / "natural.md")
+        self.assertEqual(outline["heading_count"], 1)
+        self.assertGreater(outline["paragraph_count"], 0)
+        self.assertIn("terms", terms)
+
+    def test_outline_ignores_headings_inside_code_fences(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "outline.md"
+            path.write_text(
+                "# Real heading\n\n説明です。\n\n"
+                "```bash\n# コマンド内のコメント\n```\n",
+                encoding="utf-8",
+            )
+            outline = extract_outline(path)
+        self.assertEqual(outline["heading_count"], 1)
+        self.assertEqual(outline["headings"][0]["text"], "Real heading")
+
+    def test_terms_ignore_inline_commands(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "terms.md"
+            path.write_text(
+                "# Test\n\n`npm run build --workspace=packages/core`を実行します。\n",
+                encoding="utf-8",
+            )
+            terms = extract_terms(path)
+        values = {item["term"] for item in terms["terms"]}
+        self.assertTrue(values.isdisjoint({"npm", "run", "build", "workspace", "core"}))
+
+
+if __name__ == "__main__":
+    unittest.main()