asciidoc-pubkit 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1c17eb36b6d72e3d59299a979d643ff36206c881f3515e33d4ed88a3fb37a711
-  data.tar.gz: 7ecfa1dc6e556d15b35af4e942919f6700d7c5fae46fc8355fc43467ef31f2e3
+  metadata.gz: de07af1758fdba08a7099b9e66f63b14084d0dad6cba8437498f0ddd0577b425
+  data.tar.gz: 8474d883c10c23f1221ff85cbd023d781c84f93dd55436a542c0d304c0a8f424
 SHA512:
-  metadata.gz: 2955fc8990b297e8280080d1cd1e8f3d8f72e976e1d0e2c23845d9543328115f7164111ae1bb9323b24c72c31762e36429e399049ac666d10014aef629b803e1
-  data.tar.gz: 412ffe4c96f1d00c5f00a8ffc47e0f6f9fa92d5e5fd3fa7af88c911fb51169cfc7e206543fdaac6caaabf41cf7c0c000fcc587912e60c58358b134a1ce1b47c7
+  metadata.gz: 6983524a7967131b469360bebd417125d7eea14cdf8054d14c928940528d685ee50fd0c402ca354073a55e643217ed931ddda3ca3e402d252d02e4030b7b3a9e
+  data.tar.gz: 8e8111fea8d32dfb1449c48cfc8d8f2df22e58dec14a44a55c6dcb6596a6bd3e87aa86fbd596cfc332a6ca49c9d1452f5dd34b23126137d9359a081585e1ae56

data/CHANGELOG.md

@@ -1,6 +1,26 @@
 # Changelog
 
-## 0.1.0 — Unreleased
+## 0.1.2 — Released
+
+- Package default Japanese review rules as a validated UTF-8 YAML file.
+- Support custom rule files through `review scan --rules` and `review.rules`.
+- Save resolved rules in sessions for reproducible prompts and verification.
+- Add concept, distinction, and negative meaning review candidates.
+- Add a Make target for running a local checkout with arbitrary CLI arguments.
+- Require a new review session when upgrading from earlier tool versions.
+
+## 0.1.1 — Released
+
+- Add contextual phrases, compound nouns, and potential and causative verb forms.
+- Add requested abstract nouns, degree adjectives, and vague predicates.
+- Use MeCab with UTF-8 IPADIC by default to match inflected predicates and adjectives.
+- Preserve original surfaces, dictionary forms, and negative auxiliary information.
+- Record analyzer and dictionary fingerprints in review sessions.
+- Add an explicit literal mode for environments without MeCab.
+- Reject incompatible dictionaries and unavailable analyzers with setup guidance.
+- Require a new review session when upgrading from 0.1.0.
+
+## 0.1.0 — Released
 
 - Add AsciiDoc running-prose extraction with checked source locations.
 - Add Japanese review candidates and configurable terminology checks.

data/README.md

@@ -1,3 +1,8 @@
+![](https://img.shields.io/badge/status-Work%20In%20Progress-8A2BE2)
+![GitHub tag (latest SemVer)](https://img.shields.io/github/v/tag/cybergarage/asciidoc-pubkit)
+[![Gem Version](https://img.shields.io/gem/v/asciidoc-pubkit.svg)](https://rubygems.org/gems/asciidoc-pubkit)
+[![Build Status](https://github.com/cybergarage/asciidoc-pubkit/actions/workflows/test.yml/badge.svg)](https://github.com/cybergarage/asciidoc-pubkit/actions/workflows/test.yml)
+
 # asciidoc-pubkit
 
 A toolkit for authoring, reviewing, and publishing AsciiDoc books.
@@ -14,12 +19,15 @@ rewrite manuscripts, or publish books. EPUB, image, and book scaffolding command
 are planned extensions, not available features.
 
 Ruby 3.2 or later is required. Asciidoctor is installed as a gem dependency.
-Node.js, textlint, and a morphological analyzer are not required in this release.
-The built-in rules use literal phrase matching and simple sentence heuristics.
+Starting with version 0.1.1, the default review backend requires the external MeCab
+command and a UTF-8 IPADIC dictionary. Node.js and textlint are not required.
+Explicit `--tokenizer literal` mode provides limited phrase matching without MeCab.
+
+Version 0.1.1 includes morphological analysis. Version 0.1.0 uses literal matching.
 
 ## Install from RubyGems
 
-After a release is published on RubyGems, install the CLI with:
+Install the CLI from [RubyGems](https://rubygems.org/gems/asciidoc-pubkit):
 
 ```sh
 gem install asciidoc-pubkit
@@ -28,8 +36,8 @@ asciidoc-pubkit --help
 ```
 
 Ruby 3.2 or later is required. RubyGems installs the required Ruby dependencies;
-no repository clone or Node.js installation is needed. If no release is available
-on RubyGems yet, use the source installation instructions below.
+no repository clone or Node.js installation is needed. MeCab and IPADIC must be
+installed separately when using version 0.1.1 or later in the default mode.
 
 Run the review workflow from your manuscript directory:
 
@@ -52,7 +60,7 @@ Add the gem to your project's `Gemfile` to manage its version with Bundler:
 
 ```ruby
 source 'https://rubygems.org'
-gem 'asciidoc-pubkit', '~> 0.1.0'
+gem 'asciidoc-pubkit', '~> 0.1.2'
 ```
 
 Then install dependencies and run the CLI through Bundler:
@@ -74,7 +82,7 @@ git clone https://github.com/cybergarage/asciidoc-pubkit.git
 cd asciidoc-pubkit
 bundle install
 gem build asciidoc-pubkit.gemspec
-gem install ./asciidoc-pubkit-0.1.0.gem
+gem install ./asciidoc-pubkit-0.1.2.gem
 asciidoc-pubkit --version
 ```
 
@@ -85,6 +93,64 @@ For a small trial, use `examples/book.adoc` as the scan input. Its Japanese
 paragraphs deliberately contain review candidates; its code block must remain
 unchanged.
 
+## Run from a local checkout
+
+Use the `run` Make target to execute the checkout without installing the
+asciidoc-pubkit gem. Ruby dependencies and, for the default tokenizer, MeCab and
+UTF-8 IPADIC must already be installed.
+
+```sh
+make run ARGS="--version"
+make run ARGS="review scan examples/book.adoc"
+make run ARGS='review scan "manuscripts/my book.adoc"'
+```
+
+With no `ARGS`, `make run` displays CLI help. Set `RUBY` to choose another Ruby
+executable. `ARGS` is shell command-line text; quote paths containing spaces and
+use only trusted arguments.
+
+To run from a manuscript directory, select the checkout's Makefile with `-f`.
+Relative manuscript paths and output paths remain relative to your current
+working directory:
+
+```sh
+make -f "$HOME/Src/asciidoc-pubkit/Makefile" run ARGS="review scan book.adoc"
+```
+
+Use `ARGS` instead of `make review scan book.adoc`: Make interprets positional
+words as build targets, not CLI arguments. Avoid `make -C` when manuscript paths
+should remain relative to the current directory, because it changes directories.
+
+## Install the morphological analyzer (0.1.1 and later)
+
+On macOS with Homebrew:
+
+```sh
+brew install mecab mecab-ipadic
+mecab -D
+```
+
+On Ubuntu or Debian:
+
+```sh
+sudo apt-get update
+sudo apt-get install mecab mecab-ipadic-utf8
+mecab -D
+```
+
+Use a UTF-8 IPADIC dictionary. If the default dictionary is different, configure
+`review.mecab_dictionary` with the IPADIC directory reported by your package
+manager. UniDic and other feature layouts are rejected explicitly. MeCab and
+IPADIC are separately installed dependencies, not bundled inside this gem.
+
+The tool checks MeCab availability, dictionary encoding, and feature layout.
+It never silently falls back to literal matching. To deliberately run without
+morphological analysis:
+
+```sh
+asciidoc-pubkit review scan book.adoc --tokenizer literal --output .pubkit/literal-review
+```
+
 ## Review workflow
 
 Run the following commands from the manuscript project directory:
@@ -111,7 +177,7 @@ asciidoc-pubkit review scan book.adoc --attribute edition=print --base-dir .
 
 An entrypoint or a standalone chapter can be scanned. Includes and conditionals
 are processed by Asciidoctor. Match the publishing build's attributes and base
-directory to review the intended edition. `--only` selects one included source
+ directory to review the intended edition. `--only` selects one included source
 file while retaining the book's attributes and heading hierarchy. Its path is
 relative to the current working directory, as are other CLI path arguments.
 
@@ -190,6 +256,10 @@ The initial release loads one configuration file, not merged book/repository fil
 review:
   language: ja
   style: desu-masu
+  tokenizer: mecab
+  # Optional overrides (dictionary paths are relative to this file):
+  # mecab_command: /opt/homebrew/bin/mecab
+  # mecab_dictionary: /opt/homebrew/lib/mecab/dic/ipadic
   base_dir: .
   glossary: glossary.yml
   exclude:
@@ -213,10 +283,13 @@ JavaScript:
   - Java Script
 ```
 
-Variants are review candidates, not automatic replacement instructions. `allows`
-suppresses exact terms from the built-in phrase rules; it does not disable glossary
-checks. Regex patterns are not interpreted in glossary or allow-list entries.
-Unknown configuration keys are rejected.
+Variants are review candidates, not automatic replacement instructions. In MeCab
+mode, `allows` can suppress a canonical dictionary form (all its inflections) or
+an exact matched surface (that occurrence's form only). In literal mode it
+suppresses exact dictionary entries. It does not disable glossary checks.
+Regex patterns are not interpreted in glossary or allow-list entries. Unknown
+configuration keys are rejected. A bare `mecab_command` is resolved through PATH;
+use an absolute path for an explicit executable override.
 
 ## Rules and coverage
 
@@ -224,11 +297,44 @@ Unknown configuration keys are rejected.
 | --- | --- | --- |
 | `abstract-reference` | hint | Ask what an abstract noun refers to |
 | `weak-predicate` | hint | Ask whether an operation's purpose or result is clear |
+| `contextual-phrase` | hint | Review referents, assumptions, and qualifications while preserving negation |
 | `generic-framing` | hint | Review generic introductions and emphasis |
+| `vague-degree` | hint | Ask what depth, level, scope, or comparison is intended |
 | `repeated-ending` | info | Identify three consecutive sentences with the same detected ending |
 | `glossary-variant` | warning | Identify project-specific terminology variants |
 | `style-candidate` | hint | Check selected polite/plain endings against an explicit style |
 
+MeCab mode matches noun and adjective tokens and dictionary forms of verbs.
+Sahen predicates are matched as a noun followed by the verb for "do"; standalone
+sahen nouns are not treated as verbal predicates. Auxiliary sequences retain
+negation, past tense, passive forms, and progressive forms in the reported surface.
+The added reach predicate is negative-only; the existing handling predicate is
+reviewed in both affirmative and negative forms. Glossary variants and generic
+framing phrases continue to use literal matching. Contextual phrases also use
+literal matching and suppress overlapping morphological candidates. Compound
+nouns are matched across adjacent noun tokens. Selection and narrowing verbs
+include potential forms; predicate surfaces also preserve causative auxiliaries.
+
+Morphological candidates include `lemma`, `part_of_speech`, `negative`, and
+`detector` alongside the original `match`, line, and column. Negation detection
+covers common IPADIC negative auxiliaries; it is not full semantic analysis of
+negation scope or double negatives. Unknown tokens are not guessed. Kana/kanji
+variants of the alignment and gathering verbs have explicit canonical mappings;
+other spelling variants are not automatically normalized.
+
+The session records the MeCab version and dictionary file hashes. Verification
+reports analyzer changes instead of treating results from different dictionaries
+as directly comparable. Prompt generation uses saved evidence and does not need
+MeCab. Changed rules or dictionary settings require a new scan.
+
+Sessions from earlier tool versions are not compatible with 0.1.2. Keep the original baseline for
+an ongoing review and finish it with the original version, or start a new review
+pass in a different directory:
+
+```sh
+asciidoc-pubkit review scan book.adoc --output .pubkit/review-0.1.2
+```
+
 Severity describes review priority, not proof of an error. There is no AI-authorship
 score and no requirement to eliminate every match.
 
@@ -254,7 +360,9 @@ gem build asciidoc-pubkit.gemspec
 ```
 
 The tests exercise include-boundary mapping, inline masking, configuration,
-contextual prompts, stale inputs, and protected-content verification.
+contextual prompts, stale inputs, protected-content verification, and real MeCab
+analysis of inflections, negative predicates, Unicode positions, and long lines.
+Install MeCab and UTF-8 IPADIC before running the complete test suite.
 GitHub Actions is configured for Ruby 3.2, 3.3, 3.4, and 4.0 on Linux.
 
 ## License
@@ -262,3 +370,69 @@ GitHub Actions is configured for Ruby 3.2, 3.3, 3.4, and 4.0 on Linux.
 Copyright 2026 CyberGarage.
 
 Licensed under the Apache License, Version 2.0. See [LICENSE](LICENSE).
+
+## Customize review rules
+
+The UTF-8 YAML file [`data/review-rules.ja.yml`](data/review-rules.ja.yml) is
+included in the gem and loaded by default. Edit that file when running a local
+checkout, or copy it to a project-owned file for custom rules. No Ruby changes
+are required to update candidate terms.
+
+```sh
+asciidoc-pubkit review scan book.adoc --rules ./review-rules.yml
+make run ARGS="review scan book.adoc --rules ./review-rules.yml"
+```
+
+For an installed gem, copy the default file with:
+
+```sh
+ruby -rasciidoc_pubkit -e 'puts File.read(AsciidocPubkit::RuleSet::DEFAULT_PATH)' > review-rules.yml
+```
+
+Alternatively, configure a path in `.asciidoc-pubkit.yml`:
+
+```yaml
+review:
+  rules: review-rules.yml
+```
+
+Precedence is `--rules`, then `review.rules`, then the packaged default.
+CLI paths are relative to the current working directory; configuration paths
+are relative to the configuration file. A custom file replaces the entire rule
+set; it is not merged with defaults. Start by copying the standard file.
+
+### Rule file format (schema version 1)
+
+All top-level fields below are required. Unknown keys and invalid types are
+rejected. YAML aliases and object tags are not supported.
+
+| Field | Format and behavior |
+| --- | --- |
+| `schema_version` | Integer `1` |
+| `terms` | Mapping containing all five categories listed below |
+| `verbs` | Mapping from canonical verb forms to nonempty arrays of MeCab/IPADIC dictionary forms; each dictionary form belongs to only one canonical form |
+| `sahen` | Array of nouns matched with a following `する` verb |
+| `negative_only` | Array of canonical predicates restricted to negative forms; use the noun plus `する` for sahen predicates |
+| `compound_nouns` | Array of terms matched across contiguous noun tokens; each must also appear in `abstract-reference.terms` |
+
+Each `terms` category must contain `terms` (an array of unique nonempty strings)
+and `question` (a nonempty review instruction string). The required categories
+are `abstract-reference`, `weak-predicate`, `vague-degree`, `contextual-phrase`,
+and `generic-framing`. Empty term arrays disable that category's literal
+candidates. Empty `verbs`, `sahen`, and `compound_nouns` collections disable their
+respective morphological matchers. Keep `negative_only` consistent with the
+configured predicates.
+
+In literal mode, all category term lists use exact phrase matching. In MeCab
+mode, abstract nouns and degree adjectives use dictionary forms, while predicates
+use `verbs` and `sahen`; add a predicate's desired literal surface to
+`weak-predicate.terms` as well if literal mode should detect it. Contextual and
+generic framing phrases use literal matching in both modes. Questions apply to
+both detectors. Glossary, style, repeated-ending checks, inline exclusions, and
+morphological suffix handling remain implemented in Ruby.
+
+A scan saves the resolved rule contents and source path in `manifest.json`.
+Prompt generation and verification use the saved contents, even if the original
+YAML file is subsequently edited or removed. Start a new session to apply rule
+changes. Older sessions without a rule snapshot fall back to the currently
+installed default file; start a new session for reproducible custom-rule reviews.

data/data/review-rules.ja.yml

@@ -0,0 +1,123 @@
+---
+schema_version: 1
+terms:
+  abstract-reference:
+    terms:
+    - コスト
+    - 境界
+    - 契約
+    - 観点
+    - 土台
+    - 橋渡し
+    - 入口
+    - 記述
+    - 場所
+    - 意図
+    - 役割
+    - 一続き
+    - 根拠
+    - 部品
+    - 開発者
+    - 概念
+    question: Identify the concrete referent, components, or measurable work. Keep
+      established technical meanings.
+  weak-predicate:
+    terms:
+    - 利用します
+    - 整理します
+    - 扱います
+    - 示します
+    - 変わります
+    - 把握します
+    - 分けられます
+    - そろえます
+    - まとまっています
+    - 加えます
+    - 探します
+    - 到達しません
+    - 扱いません
+    - そろいます
+    - 選べます
+    - 成り立たせています
+    - 確かめます
+    - 書き換える
+    - 絞れます
+    - 渡します
+    - あります
+    - 意味しません
+    question: Check whether the purpose, operation, or result is clear from the surrounding
+      paragraph. Preserve negation and conditions.
+  vague-degree:
+    terms:
+    - 浅い
+    - 深い
+    question: Identify the concrete depth, level, scope, or comparison. Keep literal
+      measurements and established technical meanings.
+  contextual-phrase:
+    terms:
+    - これらを
+    - であることだけでは
+    - あるものとします
+    - わけではありません
+    - 別です
+    question: Check the referent, assumption, or qualification against the surrounding
+      explanation. Preserve conditions and negation.
+  generic-framing:
+    terms:
+    - 重要なのは
+    - ポイントは
+    - 本章では
+    - ここでは
+    - まとめると
+    question: Check whether this framing adds useful scope or information instead
+      of repeating the explanation.
+verbs:
+  扱う:
+  - 扱う
+  示す:
+  - 示す
+  変わる:
+  - 変わる
+  分ける:
+  - 分ける
+  そろえる:
+  - そろえる
+  - 揃える
+  まとまる:
+  - まとまる
+  - 纏まる
+  加える:
+  - 加える
+  探す:
+  - 探す
+  そろう:
+  - そろう
+  - 揃う
+  選ぶ:
+  - 選ぶ
+  - 選べる
+  成り立つ:
+  - 成り立つ
+  確かめる:
+  - 確かめる
+  書き換える:
+  - 書き換える
+  絞る:
+  - 絞る
+  - 絞れる
+  渡す:
+  - 渡す
+  ある:
+  - ある
+sahen:
+- 利用
+- 整理
+- 把握
+- 到達
+- 意味
+negative_only:
+- 到達する
+- 意味する
+compound_nouns:
+- 一続き
+- 開発者

data/lib/asciidoc_pubkit/cli.rb

@@ -36,8 +36,10 @@ module AsciidocPubkit
         if command == 'scan'
           opts.on('--only FILE', 'Review one included file in the book context') { |v| options[:only] = v }
           opts.on('--config FILE', 'Use an explicit YAML configuration') { |v| options[:config] = v }
+          opts.on('--rules FILE', 'Replace default review rules with a YAML rule set') { |v| options[:rules] = v }
           opts.on('--base-dir DIR', 'Set the Asciidoctor base directory') { |v| options[:base_dir] = v }
           opts.on('--lang LANG', 'Prose language (ja only)') { |v| options[:language] = v }
+          opts.on('--tokenizer NAME', 'mecab (default) or literal (limited phrase matching)') { |v| options[:tokenizer] = v }
           opts.on('--style STYLE', 'preserve (default), desu-masu, or dearu') { |v| options[:style] = v }
           opts.on('-a', '--attribute NAME=VALUE', 'Set an Asciidoctor attribute; repeat as needed') do |v|
             key, value = v.split('=', 2)
@@ -58,6 +60,7 @@ module AsciidocPubkit
       if command == 'scan'
         result = Session.scan(args.first, options)
         out.puts "Scanned #{result['paragraphs']} paragraphs; found #{result['findings']} review candidates."
+        out.puts "Tokenizer: #{result['tokenizer']}#{result['tokenizer'] == 'literal' ? ' (limited phrase matching; no morphological analysis)' : ' (UTF-8 IPADIC)'}"
         out.puts "Coverage notices: #{result['coverage_notices']}. See document.json for limitations."
         out.puts "Review session: #{result['session']}"
         return 0

data/lib/asciidoc_pubkit/morphology.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require 'open3'
+
+module AsciidocPubkit
+  # An adapter for the MeCab CLI with UTF-8 IPADIC features. No shell is used.
+  class Morphology
+    attr_reader :identity
+
+    def initialize(settings)
+      @command = settings.fetch('mecab_command', 'mecab')
+      @args = []
+      dictionary = settings['mecab_dictionary']
+      @args += ['--dicdir', dictionary] if dictionary
+      version = execute(['--version']).strip
+      info = execute(@args + ['--dictionary-info'])
+      charsets = info.scan(/^charset:\s*(\S+)/).flatten
+      unless !charsets.empty? && charsets.all? { |charset| %w[utf8 utf-8].include?(charset.downcase) }
+        raise Error, 'MeCab requires a UTF-8 IPADIC dictionary. Set review.mecab_dictionary to its directory.'
+      end
+      files = info.scan(/^filename:\s*(.+)$/).flatten.map(&:strip)
+      raise Error, 'MeCab did not report a dictionary filename.' if files.empty?
+      fingerprints = files.flat_map do |file|
+        [file, *%w[unk.dic matrix.bin char.bin dicrc].map { |name| File.join(File.dirname(file), name) }]
+      end.uniq.to_h do |file|
+        raise Error, "MeCab dictionary component is missing: #{file}" unless File.file?(file)
+        [File.realpath(file), Digest::SHA256.file(file).hexdigest]
+      end
+      @identity = { 'engine' => 'mecab', 'version' => version, 'feature_schema' => 'ipadic', 'dictionary_files' => fingerprints }
+      probe = tokenize('食べました。浅かった。')
+      unless probe.any? { |t| t['lemma'] == '食べる' && t['pos'] == '動詞' } &&
+             probe.any? { |t| t['lemma'] == '浅い' && t['pos'] == '形容詞' }
+        raise Error, 'Unsupported MeCab dictionary. Use UTF-8 IPADIC; UniDic and other feature schemas are not supported.'
+      end
+    end
+
+    def tokenize(text)
+      return [] if text.empty?
+      raise Error, 'MeCab input contains a NUL character.' if text.include?("\0")
+      format = '%m\t%H\t%s\n'
+      buffer = [8192, text.lines.map(&:bytesize).max.to_i + 1].max
+      output = execute(@args + ['--node-format', format, '--unk-format', format,
+                                '--bos-format', '', '--eos-format', '', '--input-buffer-size', buffer.to_s], text)
+      cursor = 0
+      tokens = output.lines.filter_map do |line|
+        surface, feature, kind = line.chomp.split("\t", 3)
+        raise Error, 'MeCab returned an invalid token record.' unless surface && feature && kind && !surface.empty?
+        offset = text.index(surface, cursor)
+        unless offset && text[cursor...offset].match?(/\A[[:space:]]*\z/)
+          raise Error, 'MeCab token positions could not be aligned with the source text.'
+        end
+        cursor = offset + surface.length
+        fields = feature.split(',', -1)
+        if kind == '0' && fields.length != 9
+          raise Error, 'Unsupported MeCab feature schema. A UTF-8 IPADIC dictionary is required.'
+        end
+        {
+          'surface' => surface, 'offset' => offset, 'end_offset' => cursor,
+          'pos' => fields[0], 'pos_detail' => fields[1],
+          'lemma' => fields[6] == '*' ? surface : fields[6], 'unknown' => kind != '0'
+        }
+      end
+      unless text[cursor..].match?(/\A[[:space:]]*\z/)
+        raise Error, 'MeCab did not analyze the entire source text.'
+      end
+      tokens
+    end
+
+    private
+
+    def execute(args, input = '')
+      stdout, stderr, status = Open3.capture3(@command, *args, stdin_data: input)
+      # MeCab 0.996 returns 1 after successfully printing dictionary information.
+      dictionary_info = args.include?('--dictionary-info') && status.exitstatus == 1 && stderr.empty? && stdout.start_with?('filename:')
+      raise Error, "MeCab failed (exit #{status.exitstatus}): #{(stderr.empty? ? stdout : stderr).strip[0, 1000]}" unless status.success? || dictionary_info
+      stdout.force_encoding(Encoding::UTF_8)
+      raise Error, 'MeCab output is not UTF-8.' unless stdout.valid_encoding?
+      stdout
+    rescue Errno::ENOENT
+      raise Error, 'MeCab is not installed. Install MeCab and UTF-8 IPADIC (macOS: brew install mecab mecab-ipadic), or explicitly use --tokenizer literal for limited phrase matching.'
+    end
+  end
+end

data/lib/asciidoc_pubkit/rule_set.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module AsciidocPubkit
+  class RuleSet
+    DEFAULT_PATH = File.expand_path('../../data/review-rules.ja.yml', __dir__)
+    CATEGORIES = %w[abstract-reference weak-predicate vague-degree contextual-phrase generic-framing].freeze
+    KEYS = %w[schema_version terms verbs sahen negative_only compound_nouns].freeze
+
+    def self.load(path = DEFAULT_PATH)
+      validate(YAML.safe_load(AsciidocPubkit.read_text(path), permitted_classes: [], aliases: false))
+    rescue Psych::Exception => e
+      raise Error, "Invalid rule YAML in #{path}: #{e.message}"
+    end
+
+    def self.mapping(value, keys, label)
+      unless value.is_a?(Hash) && (value.keys - keys).empty? && (keys - value.keys).empty?
+        raise Error, "#{label} must contain exactly these keys: #{keys.join(', ')}."
+      end
+    end
+
+    def self.strings(value, label)
+      unless value.is_a?(Array) && value.all? { |s| s.is_a?(String) && !s.strip.empty? } && value.uniq == value
+        raise Error, "#{label} must be an array of unique nonempty strings."
+      end
+    end
+
+    def self.validate(data)
+      mapping(data, KEYS, 'Rule set')
+      raise Error, 'Rule schema_version must be integer 1.' unless data['schema_version'].is_a?(Integer) && data['schema_version'] == 1
+      mapping(data['terms'], CATEGORIES, 'Rule terms')
+      data['terms'].each do |category, entry|
+        mapping(entry, %w[terms question], category)
+        strings(entry['terms'], "#{category}.terms")
+        raise Error, "#{category}.question must be a nonempty string." unless entry['question'].is_a?(String) && !entry['question'].strip.empty?
+      end
+      verbs = data['verbs']
+      raise Error, 'verbs must be a mapping of canonical forms to lemma arrays.' unless verbs.is_a?(Hash)
+      verbs.each do |canonical, forms|
+        raise Error, 'Verb canonical forms must be nonempty strings.' unless canonical.is_a?(String) && !canonical.strip.empty?
+        strings(forms, "verbs.#{canonical}")
+        raise Error, "verbs.#{canonical} must contain at least one lemma." if forms.empty?
+      end
+      aliases = verbs.values.flatten
+      raise Error, 'Verb lemmas must have only one canonical form.' unless aliases.uniq == aliases
+      %w[sahen negative_only compound_nouns].each { |key| strings(data[key], key) }
+      canonical_forms = verbs.keys + data['sahen'].map { |noun| noun + 'する' }
+      raise Error, 'negative_only must refer to configured canonical predicates.' unless (data['negative_only'] - canonical_forms).empty?
+      raise Error, 'compound_nouns must also appear in abstract-reference terms.' unless (data['compound_nouns'] - data['terms']['abstract-reference']['terms']).empty?
+      data
+    end
+  end
+end

data/lib/asciidoc_pubkit/rules.rb

@@ -3,22 +3,24 @@
 module AsciidocPubkit
   class Rules
     INLINE = /`[^`\n]*`|\+\+\+.*?\+\+\+|\+\+[^\n]*?\+\+|(?<!\w)\+[^+\n]+\+|「[^」\n]*」|『[^』\n]*』|\{[^}\n]+\}|<<[^>\n]+>>|\[\[[^\]\n]+\]\]|(?:link|xref|image|footnote|pass):[^\s\[]*\[[^\]\n]*\]|https?:\/\/[^\s\[\]<>]+(?:\[[^\]\n]*\])?/m
-    TERMS = {
-      'abstract-reference' => [%w[コスト 境界 契約 観点 土台 橋渡し], 'Identify the concrete referent, components, or measurable work. Keep established technical meanings.'],
-      'weak-predicate' => [%w[利用します 整理します 扱います 示します], 'Check whether the purpose, operation, or result is clear from the surrounding paragraph.'],
-      'generic-framing' => [%w[重要なのは ポイントは 本章では ここでは まとめると], 'Check whether this framing adds useful scope or information instead of repeating the explanation.']
-    }.freeze
-
     def self.mask(text)
       # Keep character offsets stable while excluding common inline constructs.
       text.gsub(INLINE) { |match| match.gsub(/[^\n]/, ' ') }
     end
 
-    def self.scan(paragraphs, settings)
+    def self.scan(paragraphs, settings, tokenizer: nil)
+      rules = RuleSet.validate(settings.fetch('rules') { RuleSet.load })
+      terms = rules.fetch('terms').transform_values { |entry| entry.values_at('terms', 'question') }
       findings = []
-      paragraphs.each do |paragraph|
+      if settings.fetch('tokenizer', 'mecab') == 'mecab'
+        tokenizer ||= Morphology.new(settings)
+      end
+      token_groups = tokenizer ? tokenize_paragraphs(paragraphs, tokenizer) : []
+      paragraphs.each_with_index do |paragraph, index|
         text = mask(paragraph.fetch('text'))
-        TERMS.each do |rule, (terms, question)|
+        scan_morphemes(findings, paragraph, text, token_groups[index], settings, rules, terms) if tokenizer
+        terms.each do |rule, (terms, question)|
+          next if tokenizer && !%w[generic-framing contextual-phrase].include?(rule)
           terms.each do |term|
             next if settings.fetch('allows').include?(term)
             find_term(findings, paragraph, text, term, rule, 'hint', question)
@@ -49,6 +51,104 @@ module AsciidocPubkit
       findings
     end
 
+    def self.tokenize_paragraphs(paragraphs, tokenizer)
+      text = +''
+      spans = paragraphs.map do |paragraph|
+        start = text.length
+        text << mask(paragraph['text'])
+        finish = text.length
+        text << "\n"
+        [start, finish]
+      end
+      tokens = tokenizer.tokenize(text)
+      cursor = 0
+      spans.map do |start, finish|
+        group = []
+        while cursor < tokens.length && tokens[cursor]['offset'] < finish
+          token = tokens[cursor]
+          raise Error, 'A morphological token crossed a paragraph boundary.' if token['offset'] < start || token['end_offset'] > finish
+          group << token.merge('offset' => token['offset'] - start, 'end_offset' => token['end_offset'] - start)
+          cursor += 1
+        end
+        group
+      end
+    end
+
+    def self.scan_morphemes(findings, paragraph, text, tokens, settings, rules, terms)
+      phrase_ranges = terms['contextual-phrase'][0].flat_map do |phrase|
+        text.to_enum(:scan, Regexp.new(Regexp.escape(phrase))).map do
+          match = Regexp.last_match
+          match.begin(0)...match.end(0)
+        end
+      end
+      tokens.each_with_index do |token, index|
+        next if token['unknown'] || phrase_ranges.any? { |range| range.cover?(token['offset']) }
+        lemma = token['lemma']
+        rule = nil
+        finish_index = index
+        compound = rules.fetch('compound_nouns').find do |term|
+          surface = +''
+          cursor = index
+          while (member = tokens[cursor]) && member['pos'] == '名詞' && !member['unknown']
+            break if cursor > index && tokens[cursor - 1]['end_offset'] != member['offset']
+            surface << member['surface']
+            break unless term.start_with?(surface)
+            if surface == term
+              finish_index = cursor
+              break
+            end
+            cursor += 1
+          end
+          surface == term
+        end
+        if compound
+          lemma = compound
+          rule = 'abstract-reference'
+        elsif token['pos'] == '名詞' && terms['abstract-reference'][0].include?(lemma)
+          rule = 'abstract-reference'
+        elsif token['pos'] == '形容詞' && terms['vague-degree'][0].include?(lemma)
+          rule = 'vague-degree'
+          finish_index = predicate_end(tokens, index, text)
+        elsif token['pos'] == '動詞' && (entry = rules.fetch('verbs').find { |_canonical, forms| forms.include?(lemma) })
+          lemma = entry[0]
+          rule = 'weak-predicate'
+          finish_index = predicate_end(tokens, index, text)
+        elsif token['pos'] == '名詞' && token['pos_detail'] == 'サ変接続' && rules.fetch('sahen').include?(lemma)
+          following = tokens[index + 1]
+          next unless following && following['pos'] == '動詞' && following['lemma'] == 'する' && adjacent?(token, following, text)
+          lemma += 'する'
+          rule = 'weak-predicate'
+          finish_index = predicate_end(tokens, index + 1, text)
+        end
+        next unless rule
+        members = tokens[index..finish_index]
+        negative = members.any? { |member| member['pos'] == '助動詞' && %w[ない ぬ ん].include?(member['lemma']) }
+        next if rules.fetch('negative_only').include?(lemma) && !negative
+        surface = text[token['offset']...tokens[finish_index]['end_offset']]
+        allows = settings.fetch('allows')
+        next if [lemma, token['lemma'], surface, surface + '。'].any? { |form| allows.include?(form) }
+        add(findings, paragraph, text, token['offset'], surface, rule, 'hint', terms.fetch(rule)[1])
+        findings.last.merge!('lemma' => lemma, 'part_of_speech' => token['pos'], 'negative' => negative,
+                             'detector' => 'mecab-ipadic')
+      end
+    end
+
+    def self.adjacent?(left, right, text)
+      text[left['end_offset']...right['offset']].match?(/\A\n?\z/)
+    end
+
+    def self.predicate_end(tokens, index, text)
+      finish = index
+      while (following = tokens[finish + 1]) && adjacent?(tokens[finish], following, text)
+        auxiliary = following['pos'] == '助動詞'
+        dependent_verb = following['pos'] == '動詞' && %w[非自立 接尾].include?(following['pos_detail'])
+        connector = following['pos'] == '助詞' && following['pos_detail'] == '接続助詞' && %w[て で].include?(following['lemma'])
+        break unless auxiliary || dependent_verb || connector
+        finish += 1
+      end
+      finish
+    end
+
     def self.find_term(findings, paragraph, text, term, rule, severity, question)
       text.to_enum(:scan, Regexp.new(Regexp.escape(term))).each do
         match = Regexp.last_match

data/lib/asciidoc_pubkit/session.rb

@@ -14,7 +14,9 @@ module AsciidocPubkit
       end
       destination = File.expand_path(options.fetch(:output, '.pubkit/review'))
       raise Error, "Output already exists: #{destination}" if File.exist?(destination)
-      findings = Rules.scan(document.paragraphs, settings.data)
+      tokenizer = settings.data['tokenizer'] == 'mecab' ? Morphology.new(settings.data) : nil
+      analysis = tokenizer ? tokenizer.identity : { 'engine' => 'literal' }
+      findings = Rules.scan(document.paragraphs, settings.data, tokenizer: tokenizer)
       parent = File.dirname(destination)
       FileUtils.mkdir_p(parent)
       staging = Dir.mktmpdir('.pubkit-', parent)
@@ -29,6 +31,7 @@ module AsciidocPubkit
           'schema_version' => SCHEMA, 'tool_version' => VERSION,
           'entry' => File.realpath(entry), 'only' => options[:only] && File.realpath(options[:only]),
           'settings' => settings.data, 'sources' => sources,
+          'analysis' => analysis,
           'protected' => protected_content(document),
           'numeric_tokens' => numeric_tokens(document)
         }
@@ -43,7 +46,8 @@ module AsciidocPubkit
         FileUtils.remove_entry(staging) if File.exist?(staging)
       end
       { 'session' => destination, 'paragraphs' => document.paragraphs.length,
-        'findings' => findings.length, 'coverage_notices' => document.coverage.length }
+        'findings' => findings.length, 'coverage_notices' => document.coverage.length,
+        'tokenizer' => settings.data['tokenizer'] }
     end
 
     def self.write_json(path, value)
@@ -126,6 +130,13 @@ module AsciidocPubkit
 
         #{JSON.pretty_generate(@manifest['settings'])}
 
+        ## Analysis backend
+
+        #{JSON.pretty_generate(@manifest['analysis'])}
+
+        Morphological findings include a dictionary form and the original inflected surface.
+        A negative form must not be rewritten as an affirmative assertion. Keep the original polarity and uncertainty.
+
         ## Coverage
 
         Only source-mapped running-prose paragraphs are reviewed. Inline macros and literal spans are masked by a conservative heuristic.
@@ -176,10 +187,15 @@ module AsciidocPubkit
       changed = @manifest['sources'].filter_map do |source|
         source['path'] if document.sources[source['path']] && AsciidocPubkit.hash_text(document.sources[source['path']]) != source['sha256']
       end
+      tokenizer = @manifest['settings']['tokenizer'] == 'mecab' ? Morphology.new(@manifest['settings']) : nil
+      analysis = tokenizer ? tokenizer.identity : { 'engine' => 'literal' }
+      if analysis != @manifest['analysis']
+        issues << { 'kind' => 'analyzer-changed', 'message' => 'The MeCab version or dictionary changed. Finish verification with the original analyzer or start a new review pass.' }
+      end
       {
         'passed' => issues.empty?, 'meaning_verified' => false, 'changed_files' => changed,
         'issues' => issues, 'notices' => notices, 'coverage' => document.coverage,
-        'findings' => Rules.scan(document.paragraphs, @manifest['settings'])
+        'analysis' => analysis, 'findings' => Rules.scan(document.paragraphs, @manifest['settings'], tokenizer: tokenizer)
       }
     rescue Error, Errno::ENOENT => e
       { 'passed' => false, 'meaning_verified' => false, 'issues' => [{ 'kind' => 'verification-error', 'message' => e.message }],

data/lib/asciidoc_pubkit/settings.rb

@@ -23,22 +23,29 @@ module AsciidocPubkit
       reject_keys(config, ['review'], 'configuration')
       review = config.fetch('review', {})
       raise Error, 'review must be a mapping.' unless review.is_a?(Hash)
-      reject_keys(review, %w[language style glossary exclude allows attributes base_dir], 'review')
+      reject_keys(review, %w[language style glossary exclude allows attributes base_dir tokenizer mecab_command mecab_dictionary rules], 'review')
       raise Error, 'attributes must be a mapping.' unless review.fetch('attributes', {}).is_a?(Hash)
-      %w[base_dir glossary].each do |key|
+      %w[base_dir glossary mecab_command mecab_dictionary rules].each do |key|
         raise Error, "#{key} must be a nonempty path string." if review.key?(key) && (!review[key].is_a?(String) || review[key].empty?)
       end
       base = @path ? File.dirname(@path) : File.dirname(File.expand_path(entry))
+      rules_path = options[:rules] ? File.expand_path(options[:rules]) : (review['rules'] ? File.expand_path(review['rules'], base) : RuleSet::DEFAULT_PATH)
       @data = {
+        'rules' => RuleSet.load(rules_path),
+        'rules_path' => rules_path,
         'language' => options[:language] || review.fetch('language', 'ja'),
         'style' => options[:style] || review.fetch('style', 'preserve'),
         'exclude' => review.fetch('exclude', []),
         'allows' => review.fetch('allows', []),
+        'tokenizer' => options[:tokenizer] || review.fetch('tokenizer', 'mecab'),
+        'mecab_command' => review.fetch('mecab_command', 'mecab'),
+        'mecab_dictionary' => review['mecab_dictionary'] && File.expand_path(review['mecab_dictionary'], base),
         'attributes' => review.fetch('attributes', {}).merge(options.fetch(:attributes, {})),
         'base_dir' => File.expand_path(options[:base_dir] || review.fetch('base_dir', base), options[:base_dir] ? Dir.pwd : base),
         'glossary' => {}
       }
       raise Error, 'Only Japanese (ja) is supported in this release.' unless @data['language'] == 'ja'
+      raise Error, 'tokenizer must be mecab or literal.' unless %w[mecab literal].include?(@data['tokenizer'])
       raise Error, 'style must be preserve, desu-masu, or dearu.' unless %w[preserve desu-masu dearu].include?(@data['style'])
       %w[exclude allows].each do |key|
         raise Error, "#{key} must be an array of strings." unless @data[key].is_a?(Array) && @data[key].all? { |v| v.is_a?(String) }

data/lib/asciidoc_pubkit.rb

@@ -9,7 +9,7 @@ require 'pathname'
 require 'optparse'
 
 module AsciidocPubkit
-  VERSION = '0.1.0'
+  VERSION = '0.1.2'
   class Error < StandardError; end
 
   def self.hash_text(text)
@@ -23,8 +23,10 @@ module AsciidocPubkit
   end
 end
 
+require_relative 'asciidoc_pubkit/rule_set'
 require_relative 'asciidoc_pubkit/settings'
 require_relative 'asciidoc_pubkit/document'
+require_relative 'asciidoc_pubkit/morphology'
 require_relative 'asciidoc_pubkit/rules'
 require_relative 'asciidoc_pubkit/session'
 require_relative 'asciidoc_pubkit/cli'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: asciidoc-pubkit
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - CyberGarage
@@ -47,11 +47,14 @@ files:
 - CHANGELOG.md
 - LICENSE
 - README.md
+- data/review-rules.ja.yml
 - examples/book.adoc
 - exe/asciidoc-pubkit
 - lib/asciidoc_pubkit.rb
 - lib/asciidoc_pubkit/cli.rb
 - lib/asciidoc_pubkit/document.rb
+- lib/asciidoc_pubkit/morphology.rb
+- lib/asciidoc_pubkit/rule_set.rb
 - lib/asciidoc_pubkit/rules.rb
 - lib/asciidoc_pubkit/session.rb
 - lib/asciidoc_pubkit/settings.rb