asciidoc-pubkit 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1c17eb36b6d72e3d59299a979d643ff36206c881f3515e33d4ed88a3fb37a711
-  data.tar.gz: 7ecfa1dc6e556d15b35af4e942919f6700d7c5fae46fc8355fc43467ef31f2e3
+  metadata.gz: b5e89872635e44464421539669ea3fb4a76c886a3adae9f5ca1df98fd864d05c
+  data.tar.gz: bf081702caf7eca34423eea571fd23ad9dfffc89ed11c9b1d013dd84fb25d57c
 SHA512:
-  metadata.gz: 2955fc8990b297e8280080d1cd1e8f3d8f72e976e1d0e2c23845d9543328115f7164111ae1bb9323b24c72c31762e36429e399049ac666d10014aef629b803e1
-  data.tar.gz: 412ffe4c96f1d00c5f00a8ffc47e0f6f9fa92d5e5fd3fa7af88c911fb51169cfc7e206543fdaac6caaabf41cf7c0c000fcc587912e60c58358b134a1ce1b47c7
+  metadata.gz: 93bae7f5ef964afa60b0d6eee95b332270907df107dedae820c09fbbc1829d0d88077df96c20eaff8d9654f31ff6e47390517d1922b6486186dfa04ff17d107a
+  data.tar.gz: 5ab456dedf2c322836c827e00a0e2443b3bb9709202d88a0e82203561af98c6953debc601c799355f4df45b744d645f7c4cb670b812c0d7240c374973de09f7c

data/CHANGELOG.md

@@ -1,6 +1,17 @@
 # Changelog
 
-## 0.1.0 — Unreleased
+## 0.1.1 — Released
+
+- Add contextual phrases, compound nouns, and potential and causative verb forms.
+- Add requested abstract nouns, degree adjectives, and vague predicates.
+- Use MeCab with UTF-8 IPADIC by default to match inflected predicates and adjectives.
+- Preserve original surfaces, dictionary forms, and negative auxiliary information.
+- Record analyzer and dictionary fingerprints in review sessions.
+- Add an explicit literal mode for environments without MeCab.
+- Reject incompatible dictionaries and unavailable analyzers with setup guidance.
+- Require a new review session when upgrading from 0.1.0.
+
+## 0.1.0 — Released
 
 - Add AsciiDoc running-prose extraction with checked source locations.
 - Add Japanese review candidates and configurable terminology checks.

data/README.md

@@ -1,3 +1,7 @@
+![](https://img.shields.io/badge/status-Work%20In%20Progress-8A2BE2)
+![GitHub tag (latest SemVer)](https://img.shields.io/github/v/tag/cybergarage/asciidoc-pubkit)
+[![Gem Version](https://img.shields.io/gem/v/asciidoc-pubkit.svg)](https://rubygems.org/gems/asciidoc-pubkit)
+
 # asciidoc-pubkit
 
 A toolkit for authoring, reviewing, and publishing AsciiDoc books.
@@ -14,12 +18,15 @@ rewrite manuscripts, or publish books. EPUB, image, and book scaffolding command
 are planned extensions, not available features.
 
 Ruby 3.2 or later is required. Asciidoctor is installed as a gem dependency.
-Node.js, textlint, and a morphological analyzer are not required in this release.
-The built-in rules use literal phrase matching and simple sentence heuristics.
+Starting with version 0.1.1, the default review backend requires the external MeCab
+command and a UTF-8 IPADIC dictionary. Node.js and textlint are not required.
+Explicit `--tokenizer literal` mode provides limited phrase matching without MeCab.
+
+Version 0.1.1 includes morphological analysis. Version 0.1.0 uses literal matching.
 
 ## Install from RubyGems
 
-After a release is published on RubyGems, install the CLI with:
+Install the CLI from [RubyGems](https://rubygems.org/gems/asciidoc-pubkit):
 
 ```sh
 gem install asciidoc-pubkit
@@ -28,8 +35,8 @@ asciidoc-pubkit --help
 ```
 
 Ruby 3.2 or later is required. RubyGems installs the required Ruby dependencies;
-no repository clone or Node.js installation is needed. If no release is available
-on RubyGems yet, use the source installation instructions below.
+no repository clone or Node.js installation is needed. MeCab and IPADIC must be
+installed separately when using version 0.1.1 or later in the default mode.
 
 Run the review workflow from your manuscript directory:
 
@@ -52,7 +59,7 @@ Add the gem to your project's `Gemfile` to manage its version with Bundler:
 
 ```ruby
 source 'https://rubygems.org'
-gem 'asciidoc-pubkit', '~> 0.1.0'
+gem 'asciidoc-pubkit', '~> 0.1.1'
 ```
 
 Then install dependencies and run the CLI through Bundler:
@@ -74,7 +81,7 @@ git clone https://github.com/cybergarage/asciidoc-pubkit.git
 cd asciidoc-pubkit
 bundle install
 gem build asciidoc-pubkit.gemspec
-gem install ./asciidoc-pubkit-0.1.0.gem
+gem install ./asciidoc-pubkit-0.1.1.gem
 asciidoc-pubkit --version
 ```
 
@@ -85,6 +92,36 @@ For a small trial, use `examples/book.adoc` as the scan input. Its Japanese
 paragraphs deliberately contain review candidates; its code block must remain
 unchanged.
 
+## Install the morphological analyzer (0.1.1 and later)
+
+On macOS with Homebrew:
+
+```sh
+brew install mecab mecab-ipadic
+mecab -D
+```
+
+On Ubuntu or Debian:
+
+```sh
+sudo apt-get update
+sudo apt-get install mecab mecab-ipadic-utf8
+mecab -D
+```
+
+Use a UTF-8 IPADIC dictionary. If the default dictionary is different, configure
+`review.mecab_dictionary` with the IPADIC directory reported by your package
+manager. UniDic and other feature layouts are rejected explicitly. MeCab and
+IPADIC are separately installed dependencies, not bundled inside this gem.
+
+The tool checks MeCab availability, dictionary encoding, and feature layout.
+It never silently falls back to literal matching. To deliberately run without
+morphological analysis:
+
+```sh
+asciidoc-pubkit review scan book.adoc --tokenizer literal --output .pubkit/literal-review
+```
+
 ## Review workflow
 
 Run the following commands from the manuscript project directory:
@@ -111,7 +148,7 @@ asciidoc-pubkit review scan book.adoc --attribute edition=print --base-dir .
 
 An entrypoint or a standalone chapter can be scanned. Includes and conditionals
 are processed by Asciidoctor. Match the publishing build's attributes and base
-directory to review the intended edition. `--only` selects one included source
+ directory to review the intended edition. `--only` selects one included source
 file while retaining the book's attributes and heading hierarchy. Its path is
 relative to the current working directory, as are other CLI path arguments.
 
@@ -190,6 +227,10 @@ The initial release loads one configuration file, not merged book/repository fil
 review:
   language: ja
   style: desu-masu
+  tokenizer: mecab
+  # Optional overrides (dictionary paths are relative to this file):
+  # mecab_command: /opt/homebrew/bin/mecab
+  # mecab_dictionary: /opt/homebrew/lib/mecab/dic/ipadic
   base_dir: .
   glossary: glossary.yml
   exclude:
@@ -213,10 +254,13 @@ JavaScript:
   - Java Script
 ```
 
-Variants are review candidates, not automatic replacement instructions. `allows`
-suppresses exact terms from the built-in phrase rules; it does not disable glossary
-checks. Regex patterns are not interpreted in glossary or allow-list entries.
-Unknown configuration keys are rejected.
+Variants are review candidates, not automatic replacement instructions. In MeCab
+mode, `allows` can suppress a canonical dictionary form (all its inflections) or
+an exact matched surface (that occurrence's form only). In literal mode it
+suppresses exact dictionary entries. It does not disable glossary checks.
+Regex patterns are not interpreted in glossary or allow-list entries. Unknown
+configuration keys are rejected. A bare `mecab_command` is resolved through PATH;
+use an absolute path for an explicit executable override.
 
 ## Rules and coverage
 
@@ -224,11 +268,44 @@ Unknown configuration keys are rejected.
 | --- | --- | --- |
 | `abstract-reference` | hint | Ask what an abstract noun refers to |
 | `weak-predicate` | hint | Ask whether an operation's purpose or result is clear |
+| `contextual-phrase` | hint | Review referents, assumptions, and qualifications while preserving negation |
 | `generic-framing` | hint | Review generic introductions and emphasis |
+| `vague-degree` | hint | Ask what depth, level, scope, or comparison is intended |
 | `repeated-ending` | info | Identify three consecutive sentences with the same detected ending |
 | `glossary-variant` | warning | Identify project-specific terminology variants |
 | `style-candidate` | hint | Check selected polite/plain endings against an explicit style |
 
+MeCab mode matches noun and adjective tokens and dictionary forms of verbs.
+Sahen predicates are matched as a noun followed by the verb for "do"; standalone
+sahen nouns are not treated as verbal predicates. Auxiliary sequences retain
+negation, past tense, passive forms, and progressive forms in the reported surface.
+The added reach predicate is negative-only; the existing handling predicate is
+reviewed in both affirmative and negative forms. Glossary variants and generic
+framing phrases continue to use literal matching. Contextual phrases also use
+literal matching and suppress overlapping morphological candidates. Compound
+nouns are matched across adjacent noun tokens. Selection and narrowing verbs
+include potential forms; predicate surfaces also preserve causative auxiliaries.
+
+Morphological candidates include `lemma`, `part_of_speech`, `negative`, and
+`detector` alongside the original `match`, line, and column. Negation detection
+covers common IPADIC negative auxiliaries; it is not full semantic analysis of
+negation scope or double negatives. Unknown tokens are not guessed. Kana/kanji
+variants of the alignment and gathering verbs have explicit canonical mappings;
+other spelling variants are not automatically normalized.
+
+The session records the MeCab version and dictionary file hashes. Verification
+reports analyzer changes instead of treating results from different dictionaries
+as directly comparable. Prompt generation uses saved evidence and does not need
+MeCab. Changed rules or dictionary settings require a new scan.
+
+Sessions from 0.1.0 are not compatible with 0.1.1. Keep the original baseline for
+an ongoing review and finish it with the original version, or start a new review
+pass in a different directory:
+
+```sh
+asciidoc-pubkit review scan book.adoc --output .pubkit/review-0.1.1
+```
+
 Severity describes review priority, not proof of an error. There is no AI-authorship
 score and no requirement to eliminate every match.
 
@@ -254,7 +331,9 @@ gem build asciidoc-pubkit.gemspec
 ```
 
 The tests exercise include-boundary mapping, inline masking, configuration,
-contextual prompts, stale inputs, and protected-content verification.
+contextual prompts, stale inputs, protected-content verification, and real MeCab
+analysis of inflections, negative predicates, Unicode positions, and long lines.
+Install MeCab and UTF-8 IPADIC before running the complete test suite.
 GitHub Actions is configured for Ruby 3.2, 3.3, 3.4, and 4.0 on Linux.
 
 ## License

data/lib/asciidoc_pubkit/cli.rb

@@ -38,6 +38,7 @@ module AsciidocPubkit
           opts.on('--config FILE', 'Use an explicit YAML configuration') { |v| options[:config] = v }
           opts.on('--base-dir DIR', 'Set the Asciidoctor base directory') { |v| options[:base_dir] = v }
           opts.on('--lang LANG', 'Prose language (ja only)') { |v| options[:language] = v }
+          opts.on('--tokenizer NAME', 'mecab (default) or literal (limited phrase matching)') { |v| options[:tokenizer] = v }
           opts.on('--style STYLE', 'preserve (default), desu-masu, or dearu') { |v| options[:style] = v }
           opts.on('-a', '--attribute NAME=VALUE', 'Set an Asciidoctor attribute; repeat as needed') do |v|
             key, value = v.split('=', 2)
@@ -58,6 +59,7 @@ module AsciidocPubkit
       if command == 'scan'
         result = Session.scan(args.first, options)
         out.puts "Scanned #{result['paragraphs']} paragraphs; found #{result['findings']} review candidates."
+        out.puts "Tokenizer: #{result['tokenizer']}#{result['tokenizer'] == 'literal' ? ' (limited phrase matching; no morphological analysis)' : ' (UTF-8 IPADIC)'}"
         out.puts "Coverage notices: #{result['coverage_notices']}. See document.json for limitations."
         out.puts "Review session: #{result['session']}"
         return 0

data/lib/asciidoc_pubkit/morphology.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require 'open3'
+
+module AsciidocPubkit
+  # An adapter for the MeCab CLI with UTF-8 IPADIC features. No shell is used.
+  class Morphology
+    attr_reader :identity
+
+    def initialize(settings)
+      @command = settings.fetch('mecab_command', 'mecab')
+      @args = []
+      dictionary = settings['mecab_dictionary']
+      @args += ['--dicdir', dictionary] if dictionary
+      version = execute(['--version']).strip
+      info = execute(@args + ['--dictionary-info'])
+      charsets = info.scan(/^charset:\s*(\S+)/).flatten
+      unless !charsets.empty? && charsets.all? { |charset| %w[utf8 utf-8].include?(charset.downcase) }
+        raise Error, 'MeCab requires a UTF-8 IPADIC dictionary. Set review.mecab_dictionary to its directory.'
+      end
+      files = info.scan(/^filename:\s*(.+)$/).flatten.map(&:strip)
+      raise Error, 'MeCab did not report a dictionary filename.' if files.empty?
+      fingerprints = files.flat_map do |file|
+        [file, *%w[unk.dic matrix.bin char.bin dicrc].map { |name| File.join(File.dirname(file), name) }]
+      end.uniq.to_h do |file|
+        raise Error, "MeCab dictionary component is missing: #{file}" unless File.file?(file)
+        [File.realpath(file), Digest::SHA256.file(file).hexdigest]
+      end
+      @identity = { 'engine' => 'mecab', 'version' => version, 'feature_schema' => 'ipadic', 'dictionary_files' => fingerprints }
+      probe = tokenize('食べました。浅かった。')
+      unless probe.any? { |t| t['lemma'] == '食べる' && t['pos'] == '動詞' } &&
+             probe.any? { |t| t['lemma'] == '浅い' && t['pos'] == '形容詞' }
+        raise Error, 'Unsupported MeCab dictionary. Use UTF-8 IPADIC; UniDic and other feature schemas are not supported.'
+      end
+    end
+
+    def tokenize(text)
+      return [] if text.empty?
+      raise Error, 'MeCab input contains a NUL character.' if text.include?("\0")
+      format = '%m\t%H\t%s\n'
+      buffer = [8192, text.lines.map(&:bytesize).max.to_i + 1].max
+      output = execute(@args + ['--node-format', format, '--unk-format', format,
+                                '--bos-format', '', '--eos-format', '', '--input-buffer-size', buffer.to_s], text)
+      cursor = 0
+      tokens = output.lines.filter_map do |line|
+        surface, feature, kind = line.chomp.split("\t", 3)
+        raise Error, 'MeCab returned an invalid token record.' unless surface && feature && kind && !surface.empty?
+        offset = text.index(surface, cursor)
+        unless offset && text[cursor...offset].match?(/\A[[:space:]]*\z/)
+          raise Error, 'MeCab token positions could not be aligned with the source text.'
+        end
+        cursor = offset + surface.length
+        fields = feature.split(',', -1)
+        if kind == '0' && fields.length != 9
+          raise Error, 'Unsupported MeCab feature schema. A UTF-8 IPADIC dictionary is required.'
+        end
+        {
+          'surface' => surface, 'offset' => offset, 'end_offset' => cursor,
+          'pos' => fields[0], 'pos_detail' => fields[1],
+          'lemma' => fields[6] == '*' ? surface : fields[6], 'unknown' => kind != '0'
+        }
+      end
+      unless text[cursor..].match?(/\A[[:space:]]*\z/)
+        raise Error, 'MeCab did not analyze the entire source text.'
+      end
+      tokens
+    end
+
+    private
+
+    def execute(args, input = '')
+      stdout, stderr, status = Open3.capture3(@command, *args, stdin_data: input)
+      # MeCab 0.996 returns 1 after successfully printing dictionary information.
+      dictionary_info = args.include?('--dictionary-info') && status.exitstatus == 1 && stderr.empty? && stdout.start_with?('filename:')
+      raise Error, "MeCab failed (exit #{status.exitstatus}): #{(stderr.empty? ? stdout : stderr).strip[0, 1000]}" unless status.success? || dictionary_info
+      stdout.force_encoding(Encoding::UTF_8)
+      raise Error, 'MeCab output is not UTF-8.' unless stdout.valid_encoding?
+      stdout
+    rescue Errno::ENOENT
+      raise Error, 'MeCab is not installed. Install MeCab and UTF-8 IPADIC (macOS: brew install mecab mecab-ipadic), or explicitly use --tokenizer literal for limited phrase matching.'
+    end
+  end
+end

data/lib/asciidoc_pubkit/rules.rb

@@ -4,21 +4,39 @@ module AsciidocPubkit
   class Rules
     INLINE = /`[^`\n]*`|\+\+\+.*?\+\+\+|\+\+[^\n]*?\+\+|(?<!\w)\+[^+\n]+\+|「[^」\n]*」|『[^』\n]*』|\{[^}\n]+\}|<<[^>\n]+>>|\[\[[^\]\n]+\]\]|(?:link|xref|image|footnote|pass):[^\s\[]*\[[^\]\n]*\]|https?:\/\/[^\s\[\]<>]+(?:\[[^\]\n]*\])?/m
     TERMS = {
-      'abstract-reference' => [%w[コスト 境界 契約 観点 土台 橋渡し], 'Identify the concrete referent, components, or measurable work. Keep established technical meanings.'],
-      'weak-predicate' => [%w[利用します 整理します 扱います 示します], 'Check whether the purpose, operation, or result is clear from the surrounding paragraph.'],
+      'abstract-reference' => [%w[コスト 境界 契約 観点 土台 橋渡し 入口 記述 場所 意図 役割 一続き 根拠 部品 開発者], 'Identify the concrete referent, components, or measurable work. Keep established technical meanings.'],
+      'weak-predicate' => [%w[利用します 整理します 扱います 示します 変わります 把握します 分けられます そろえます まとまっています 加えます 探します 到達しません 扱いません そろいます 選べます 成り立たせています 確かめます 書き換える 絞れます 渡します あります], 'Check whether the purpose, operation, or result is clear from the surrounding paragraph. Preserve negation and conditions.'],
+      'vague-degree' => [%w[浅い 深い], 'Identify the concrete depth, level, scope, or comparison. Keep literal measurements and established technical meanings.'],
+      'contextual-phrase' => [%w[これらを であることだけでは あるものとします わけではありません], 'Check the referent, assumption, or qualification against the surrounding explanation. Preserve conditions and negation.'],
       'generic-framing' => [%w[重要なのは ポイントは 本章では ここでは まとめると], 'Check whether this framing adds useful scope or information instead of repeating the explanation.']
     }.freeze
+    VERBS = {
+      '扱う' => %w[扱う], '示す' => %w[示す], '変わる' => %w[変わる],
+      '分ける' => %w[分ける], 'そろえる' => %w[そろえる 揃える],
+      'まとまる' => %w[まとまる 纏まる], '加える' => %w[加える],
+      '探す' => %w[探す], 'そろう' => %w[そろう 揃う],
+      '選ぶ' => %w[選ぶ 選べる], '成り立つ' => %w[成り立つ],
+      '確かめる' => %w[確かめる], '書き換える' => %w[書き換える],
+      '絞る' => %w[絞る 絞れる], '渡す' => %w[渡す], 'ある' => %w[ある]
+    }.freeze
+    SAHEN = %w[利用 整理 把握 到達].freeze
 
     def self.mask(text)
       # Keep character offsets stable while excluding common inline constructs.
       text.gsub(INLINE) { |match| match.gsub(/[^\n]/, ' ') }
     end
 
-    def self.scan(paragraphs, settings)
+    def self.scan(paragraphs, settings, tokenizer: nil)
       findings = []
-      paragraphs.each do |paragraph|
+      if settings.fetch('tokenizer', 'mecab') == 'mecab'
+        tokenizer ||= Morphology.new(settings)
+      end
+      token_groups = tokenizer ? tokenize_paragraphs(paragraphs, tokenizer) : []
+      paragraphs.each_with_index do |paragraph, index|
         text = mask(paragraph.fetch('text'))
+        scan_morphemes(findings, paragraph, text, token_groups[index], settings) if tokenizer
         TERMS.each do |rule, (terms, question)|
+          next if tokenizer && !%w[generic-framing contextual-phrase].include?(rule)
           terms.each do |term|
             next if settings.fetch('allows').include?(term)
             find_term(findings, paragraph, text, term, rule, 'hint', question)
@@ -49,6 +67,96 @@ module AsciidocPubkit
       findings
     end
 
+    def self.tokenize_paragraphs(paragraphs, tokenizer)
+      text = +''
+      spans = paragraphs.map do |paragraph|
+        start = text.length
+        text << mask(paragraph['text'])
+        finish = text.length
+        text << "\n"
+        [start, finish]
+      end
+      tokens = tokenizer.tokenize(text)
+      cursor = 0
+      spans.map do |start, finish|
+        group = []
+        while cursor < tokens.length && tokens[cursor]['offset'] < finish
+          token = tokens[cursor]
+          raise Error, 'A morphological token crossed a paragraph boundary.' if token['offset'] < start || token['end_offset'] > finish
+          group << token.merge('offset' => token['offset'] - start, 'end_offset' => token['end_offset'] - start)
+          cursor += 1
+        end
+        group
+      end
+    end
+
+    def self.scan_morphemes(findings, paragraph, text, tokens, settings)
+      phrase_ranges = TERMS['contextual-phrase'][0].flat_map do |phrase|
+        text.to_enum(:scan, Regexp.new(Regexp.escape(phrase))).map do
+          match = Regexp.last_match
+          match.begin(0)...match.end(0)
+        end
+      end
+      tokens.each_with_index do |token, index|
+        next if token['unknown'] || phrase_ranges.any? { |range| range.cover?(token['offset']) }
+        lemma = token['lemma']
+        rule = nil
+        finish_index = index
+        compound = %w[一続き 開発者].find do |term|
+          following = tokens[index + 1]
+          following && token['pos'] == '名詞' && following['pos'] == '名詞' &&
+            !following['unknown'] && token['end_offset'] == following['offset'] &&
+            token['surface'] + following['surface'] == term
+        end
+        if compound
+          lemma = compound
+          finish_index = index + 1
+          rule = 'abstract-reference'
+        elsif token['pos'] == '名詞' && TERMS['abstract-reference'][0].include?(lemma)
+          rule = 'abstract-reference'
+        elsif token['pos'] == '形容詞' && TERMS['vague-degree'][0].include?(lemma)
+          rule = 'vague-degree'
+          finish_index = predicate_end(tokens, index, text)
+        elsif token['pos'] == '動詞' && (entry = VERBS.find { |_canonical, forms| forms.include?(lemma) })
+          lemma = entry[0]
+          rule = 'weak-predicate'
+          finish_index = predicate_end(tokens, index, text)
+        elsif token['pos'] == '名詞' && token['pos_detail'] == 'サ変接続' && SAHEN.include?(lemma)
+          following = tokens[index + 1]
+          next unless following && following['pos'] == '動詞' && following['lemma'] == 'する' && adjacent?(token, following, text)
+          lemma += 'する'
+          rule = 'weak-predicate'
+          finish_index = predicate_end(tokens, index + 1, text)
+        end
+        next unless rule
+        members = tokens[index..finish_index]
+        negative = members.any? { |member| member['pos'] == '助動詞' && %w[ない ぬ ん].include?(member['lemma']) }
+        next if lemma == '到達する' && !negative
+        surface = text[token['offset']...tokens[finish_index]['end_offset']]
+        allows = settings.fetch('allows')
+        next if [lemma, token['lemma'], surface, surface + '。'].any? { |form| allows.include?(form) }
+        add(findings, paragraph, text, token['offset'], surface, rule, 'hint', TERMS.fetch(rule)[1])
+        findings.last.merge!('lemma' => lemma, 'part_of_speech' => token['pos'], 'negative' => negative,
+                             'detector' => 'mecab-ipadic')
+      end
+    end
+
+    def self.adjacent?(left, right, text)
+      text[left['end_offset']...right['offset']].match?(/\A\n?\z/)
+    end
+
+    def self.predicate_end(tokens, index, text)
+      finish = index
+      while (following = tokens[finish + 1]) && adjacent?(tokens[finish], following, text)
+        auxiliary = following['pos'] == '助動詞'
+        dependent_verb = following['pos'] == '動詞' && %w[非自立 接尾].include?(following['pos_detail'])
+        connector = following['pos'] == '助詞' && following['pos_detail'] == '接続助詞' && %w[て で].include?(following['lemma'])
+        break unless auxiliary || dependent_verb || connector
+        finish += 1
+      end
+      finish
+    end
+
     def self.find_term(findings, paragraph, text, term, rule, severity, question)
       text.to_enum(:scan, Regexp.new(Regexp.escape(term))).each do
         match = Regexp.last_match

data/lib/asciidoc_pubkit/session.rb

@@ -14,7 +14,9 @@ module AsciidocPubkit
       end
       destination = File.expand_path(options.fetch(:output, '.pubkit/review'))
       raise Error, "Output already exists: #{destination}" if File.exist?(destination)
-      findings = Rules.scan(document.paragraphs, settings.data)
+      tokenizer = settings.data['tokenizer'] == 'mecab' ? Morphology.new(settings.data) : nil
+      analysis = tokenizer ? tokenizer.identity : { 'engine' => 'literal' }
+      findings = Rules.scan(document.paragraphs, settings.data, tokenizer: tokenizer)
       parent = File.dirname(destination)
       FileUtils.mkdir_p(parent)
       staging = Dir.mktmpdir('.pubkit-', parent)
@@ -29,6 +31,7 @@ module AsciidocPubkit
           'schema_version' => SCHEMA, 'tool_version' => VERSION,
           'entry' => File.realpath(entry), 'only' => options[:only] && File.realpath(options[:only]),
           'settings' => settings.data, 'sources' => sources,
+          'analysis' => analysis,
           'protected' => protected_content(document),
           'numeric_tokens' => numeric_tokens(document)
         }
@@ -43,7 +46,8 @@ module AsciidocPubkit
         FileUtils.remove_entry(staging) if File.exist?(staging)
       end
       { 'session' => destination, 'paragraphs' => document.paragraphs.length,
-        'findings' => findings.length, 'coverage_notices' => document.coverage.length }
+        'findings' => findings.length, 'coverage_notices' => document.coverage.length,
+        'tokenizer' => settings.data['tokenizer'] }
     end
 
     def self.write_json(path, value)
@@ -126,6 +130,13 @@ module AsciidocPubkit
 
         #{JSON.pretty_generate(@manifest['settings'])}
 
+        ## Analysis backend
+
+        #{JSON.pretty_generate(@manifest['analysis'])}
+
+        Morphological findings include a dictionary form and the original inflected surface.
+        A negative form must not be rewritten as an affirmative assertion. Keep the original polarity and uncertainty.
+
         ## Coverage
 
         Only source-mapped running-prose paragraphs are reviewed. Inline macros and literal spans are masked by a conservative heuristic.
@@ -176,10 +187,15 @@ module AsciidocPubkit
       changed = @manifest['sources'].filter_map do |source|
         source['path'] if document.sources[source['path']] && AsciidocPubkit.hash_text(document.sources[source['path']]) != source['sha256']
       end
+      tokenizer = @manifest['settings']['tokenizer'] == 'mecab' ? Morphology.new(@manifest['settings']) : nil
+      analysis = tokenizer ? tokenizer.identity : { 'engine' => 'literal' }
+      if analysis != @manifest['analysis']
+        issues << { 'kind' => 'analyzer-changed', 'message' => 'The MeCab version or dictionary changed. Finish verification with the original analyzer or start a new review pass.' }
+      end
       {
         'passed' => issues.empty?, 'meaning_verified' => false, 'changed_files' => changed,
         'issues' => issues, 'notices' => notices, 'coverage' => document.coverage,
-        'findings' => Rules.scan(document.paragraphs, @manifest['settings'])
+        'analysis' => analysis, 'findings' => Rules.scan(document.paragraphs, @manifest['settings'], tokenizer: tokenizer)
       }
     rescue Error, Errno::ENOENT => e
       { 'passed' => false, 'meaning_verified' => false, 'issues' => [{ 'kind' => 'verification-error', 'message' => e.message }],

data/lib/asciidoc_pubkit/settings.rb

@@ -23,9 +23,9 @@ module AsciidocPubkit
       reject_keys(config, ['review'], 'configuration')
       review = config.fetch('review', {})
       raise Error, 'review must be a mapping.' unless review.is_a?(Hash)
-      reject_keys(review, %w[language style glossary exclude allows attributes base_dir], 'review')
+      reject_keys(review, %w[language style glossary exclude allows attributes base_dir tokenizer mecab_command mecab_dictionary], 'review')
       raise Error, 'attributes must be a mapping.' unless review.fetch('attributes', {}).is_a?(Hash)
-      %w[base_dir glossary].each do |key|
+      %w[base_dir glossary mecab_command mecab_dictionary].each do |key|
         raise Error, "#{key} must be a nonempty path string." if review.key?(key) && (!review[key].is_a?(String) || review[key].empty?)
       end
       base = @path ? File.dirname(@path) : File.dirname(File.expand_path(entry))
@@ -34,11 +34,15 @@ module AsciidocPubkit
         'style' => options[:style] || review.fetch('style', 'preserve'),
         'exclude' => review.fetch('exclude', []),
         'allows' => review.fetch('allows', []),
+        'tokenizer' => options[:tokenizer] || review.fetch('tokenizer', 'mecab'),
+        'mecab_command' => review.fetch('mecab_command', 'mecab'),
+        'mecab_dictionary' => review['mecab_dictionary'] && File.expand_path(review['mecab_dictionary'], base),
         'attributes' => review.fetch('attributes', {}).merge(options.fetch(:attributes, {})),
         'base_dir' => File.expand_path(options[:base_dir] || review.fetch('base_dir', base), options[:base_dir] ? Dir.pwd : base),
         'glossary' => {}
       }
       raise Error, 'Only Japanese (ja) is supported in this release.' unless @data['language'] == 'ja'
+      raise Error, 'tokenizer must be mecab or literal.' unless %w[mecab literal].include?(@data['tokenizer'])
       raise Error, 'style must be preserve, desu-masu, or dearu.' unless %w[preserve desu-masu dearu].include?(@data['style'])
       %w[exclude allows].each do |key|
         raise Error, "#{key} must be an array of strings." unless @data[key].is_a?(Array) && @data[key].all? { |v| v.is_a?(String) }

data/lib/asciidoc_pubkit.rb

@@ -9,7 +9,7 @@ require 'pathname'
 require 'optparse'
 
 module AsciidocPubkit
-  VERSION = '0.1.0'
+  VERSION = '0.1.1'
   class Error < StandardError; end
 
   def self.hash_text(text)
@@ -25,6 +25,7 @@ end
 
 require_relative 'asciidoc_pubkit/settings'
 require_relative 'asciidoc_pubkit/document'
+require_relative 'asciidoc_pubkit/morphology'
 require_relative 'asciidoc_pubkit/rules'
 require_relative 'asciidoc_pubkit/session'
 require_relative 'asciidoc_pubkit/cli'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: asciidoc-pubkit
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - CyberGarage
@@ -52,6 +52,7 @@ files:
 - lib/asciidoc_pubkit.rb
 - lib/asciidoc_pubkit/cli.rb
 - lib/asciidoc_pubkit/document.rb
+- lib/asciidoc_pubkit/morphology.rb
 - lib/asciidoc_pubkit/rules.rb
 - lib/asciidoc_pubkit/session.rb
 - lib/asciidoc_pubkit/settings.rb