bitclust-dev 1.3.0 → 1.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a17aae0f1153193b6c5f295680331f6661bb8ac0fd3916f46a9d143820af535c
-  data.tar.gz: 5a9bce05ca0b5b2053a6c62f6c047d258d01743b88deb6034ae3c6386ac38583
+  metadata.gz: b6f8d4dcefaa581b519600e255c2e9fba43bcdc3610f808de7fc101126fe1961
+  data.tar.gz: 3a86cd213fb1cb1be92d1d1f71c8cf3da91ece0cbd9e24ac05f11639f7a47aee
 SHA512:
-  metadata.gz: f879ecf2eab0add265f0e8807999cb57f366e755f6d378de6da3db6803c793f1231e88d6c24e03ca565fe101c4b5aaf08a42eca33b0c9a92ce74f5c11f1fb72f
-  data.tar.gz: 036f33247f1b23c469861caae060d47f1a8921e9f271e85c826e1b4e626c17d6fe8243c9f561990986bd9916209d39950546af3c6588fafc56029638d9ec5f10
+  metadata.gz: d28e9f59a0dda51014c904e290b636cf3e6eef1516b04c6a22d3280cced4cf598dba82475e7aec4a3ab2694f7811b9a4bec29197ab7fd2f2027f23c0a45ab2ba
+  data.tar.gz: fe4fa1117faa16fd0f4b6cf7f0aa4c09e8cbb8f244b1ba7eb3e20be9be62f4c5e91a79f6adaddc783f5f0c227a34a006c9b6cdc1c691044185f0331db7658504

data/tools/md-bridge-check.rb

@@ -0,0 +1,94 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+#
+# md ブリッジのパーサレベル等価検証。
+# 各ライブラリを「旧ソース（doctree の .rd）」と「md ツリー → MarkdownBridge で
+# 生成した rd ツリー」の両方から RRDParser でパースし、クラス・エントリ構造が
+# 一致することを確認する。
+#
+# usage: ruby tools/md-bridge-check.rb <doctree-src-root> <md-tree-root> [--version V] [-v]
+
+$LOAD_PATH.unshift File.expand_path('../lib', __dir__)
+require 'tmpdir'
+require 'bitclust'
+require 'bitclust/markdown_bridge'
+
+version = '3.4'
+verbose = false
+paths = []
+args = ARGV.dup
+until args.empty?
+  case (a = args.shift)
+  when '--version' then version = args.shift
+  when '-v' then verbose = true
+  else paths << a
+  end
+end
+src_root, md_root = paths
+abort "usage: #{$0} <doctree-src-root> <md-tree-root> [--version V]" unless src_root && md_root
+
+params = { 'version' => version }
+
+def library_names(root, params)
+  # 旧 LIBRARIES には重複エントリがある（webrick/httputils）ため uniq
+  BitClust::Preprocessor.read(File.join(root, 'LIBRARIES'), params)
+                        .lines.map(&:strip).reject(&:empty?).uniq
+end
+
+# ライブラリを単体パースし、{[type, class名] => 整列済みエントリ名} を返す
+def structure(root, lib, params)
+  db = BitClust::MethodDatabase.dummy(params)
+  library = BitClust::RRDParser.new(db).parse_file(File.join(root, "#{lib}.rd"), lib, params)
+  library.classes.to_h do |c|
+    [[c.type.to_s, c.name], c.entries.flat_map(&:names).sort]
+  end
+rescue StandardError => e
+  { error: "#{e.class}: #{e.message}" }
+end
+
+Dir.mktmpdir do |bridge|
+  t0 = Time.now
+  BitClust::MarkdownBridge.build(md_root, bridge)
+  puts "bridge built in #{(Time.now - t0).round(1)}s"
+
+  old_libs = library_names(src_root, params)
+  new_libs = library_names(bridge, params)
+  puts "library set @#{params['version']}: " \
+       "#{old_libs.sort == new_libs.sort ? 'OK' : 'MISMATCH'} " \
+       "(old #{old_libs.size}, new #{new_libs.size})"
+  (old_libs - new_libs).each { |l| puts "  missing in bridge: #{l}" }
+  (new_libs - old_libs).each { |l| puts "  extra in bridge: #{l}" }
+
+  ok = 0
+  diffs = []
+  errors = []
+  (old_libs & new_libs).sort.each do |lib|
+    old_s = structure(src_root, lib, params)
+    new_s = structure(bridge, lib, params)
+    if old_s[:error] || new_s[:error]
+      if old_s[:error].to_s == new_s[:error].to_s
+        ok += 1   # 両側同一条件で失敗（dummy DB 都合等）は等価とみなす
+      else
+        errors << [lib, old_s[:error], new_s[:error]]
+      end
+      next
+    end
+    if old_s == new_s
+      ok += 1
+      puts "OK   #{lib}" if verbose
+    else
+      diffs << lib
+      puts "DIFF #{lib}"
+      (old_s.keys - new_s.keys).each { |k| puts "  only in old: #{k.inspect}" }
+      (new_s.keys - old_s.keys).each { |k| puts "  only in new: #{k.inspect}" }
+      (old_s.keys & new_s.keys).each do |k|
+        next if old_s[k] == new_s[k]
+        puts "  #{k.inspect}: entries differ"
+        puts "    only old: #{(old_s[k] - new_s[k]).first(5).inspect}"
+        puts "    only new: #{(new_s[k] - old_s[k]).first(5).inspect}"
+      end
+    end
+  end
+  puts "#{ok}/#{(old_libs & new_libs).size} libraries structurally identical"
+  errors.each { |lib, o, n| puts "  ERROR #{lib}: old=#{o.inspect} new=#{n.inspect}" }
+end

data/tools/md-compile-check.rb

@@ -0,0 +1,217 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+#
+# MDCompiler の HTML 等価検証（ネイティブ MD 描画 M1）。
+# データベースの全エントリについて、
+#   rd source → RDCompiler → HTML（リファレンス）
+#   rd source → RRDToMarkdown → md source → MDCompiler → HTML
+# が一致することを確認する。fragment の md→rd ラウンドトリップも同時に検証。
+#
+# usage: ruby tools/md-compile-check.rb <db-path> [--limit N] [-v]
+#          [--only methods|docs|libs|functions] [--shard K/N] [--gfm]
+#
+# メモリの少ないマシンでは 1 プロセスで全件を回さず、--only/--shard で
+# 分割して直列に実行する（例: --only methods --shard 0/4 ... 3/4）。
+#
+# --gfm: M2 GFM モードの整合も検証する。GFM 出力と M1 出力の差が
+# <code>/<strong>/GNU 引用（`x'）の正規化で消えることを確認する
+# （= GFM モードが追加するのは表現マークアップだけで内容は同一）。
+
+$stdout.sync = true
+$LOAD_PATH.unshift File.expand_path('../lib', __dir__)
+require 'bitclust'
+require 'bitclust/rdcompiler'
+require 'bitclust/mdcompiler'
+require 'bitclust/rrd_to_markdown'
+require 'bitclust/markdown_to_rrd'
+require 'bitclust/capi_converter'
+
+limit = nil
+verbose = false
+only = nil
+shard_k = nil
+shard_n = nil
+gfm_mode = false
+paths = []
+args = ARGV.dup
+until args.empty?
+  case (a = args.shift)
+  when '--limit' then limit = args.shift.to_i
+  when '--only' then only = args.shift
+  when '--shard'
+    shard_k, shard_n = (args.shift || '').split('/').map(&:to_i)
+  when '--gfm' then gfm_mode = true
+  when '-v' then verbose = true
+  else paths << a
+  end
+end
+db_path = paths.shift or abort "usage: #{$0} <db-path> [--limit N]"
+
+run = ->(kind) { only.nil? || only == kind }
+in_shard = ->(i) { shard_n.nil? || i % shard_n == shard_k }
+
+db = BitClust::MethodDatabase.new(db_path)
+urlmapper = BitClust::URLMapper.new(Hash.new { 'dummy' })
+rd = BitClust::RDCompiler.new(urlmapper, 1, { database: db })
+md = BitClust::MDCompiler.new(urlmapper, 1, { database: db })
+mdg = BitClust::MDCompiler.new(urlmapper, 1, { database: db, gfm: true })
+
+stats = Hash.new(0)
+diffs = []
+
+# GFM が追加する表現マークアップを落として M1 と比較するための正規化
+normalize_gfm = lambda do |html|
+  html.gsub(%r{</?code>|</?strong>|<(th|td) align="[a-z]+">}, '')
+      .gsub(%r{</?(?:table|thead|tbody|tr|th|td)>}, '')
+      .gsub(/`([^`'\s]+)'/) { $1 }
+end
+
+check = lambda do |label, kind, rd_src, ref_html|
+  md_src = kind == :function ? BitClust::CapiConverter.convert(rd_src)
+                             : BitClust::RRDToMarkdown.convert(rd_src)
+  back = BitClust::MarkdownToRRD.convert(md_src, capi: kind == :function)
+  stats[:rt_diff] += 1 if back != rd_src
+
+  html =
+    case kind
+    when :method
+      # 実エントリの source を一時差し替え（save しないので永続化されない）
+      entry = stats[:current_entry]
+      original = entry.source
+      begin
+        entry.source = md_src
+        md.compile_method(entry, nil)
+      ensure
+        entry.source = original
+      end
+    when :function
+      entry = stats[:current_entry]
+      original = entry.source
+      begin
+        entry.source = md_src
+        md.compile_function(entry, nil)
+      ensure
+        entry.source = original
+      end
+    else
+      md.compile(md_src)
+    end
+  if gfm_mode
+    gfm_html =
+      case kind
+      when :method, :function
+        entry = stats[:current_entry]
+        original = entry.source
+        begin
+          entry.source = md_src
+          kind == :method ? mdg.compile_method(entry, nil) : mdg.compile_function(entry, nil)
+        ensure
+          entry.source = original
+        end
+      else
+        mdg.compile(md_src)
+      end
+    if normalize_gfm.call(gfm_html) == normalize_gfm.call(html)
+      stats[:gfm_ok] += 1
+    else
+      stats[:gfm_diff] += 1
+      if stats[:gfm_diff] <= 10
+        puts "GFM-DIFF #{label}"
+        normalize_gfm.call(html).lines.zip(normalize_gfm.call(gfm_html).lines).each_with_index do |(a, b), i|
+          next if a == b
+          puts "  line #{i + 1}: M1 =#{a.inspect}"
+          puts "            GFM=#{b.inspect}"
+          break
+        end
+      end
+    end
+  end
+
+  if html == ref_html
+    stats[:ok] += 1
+    puts "OK   #{label}" if verbose
+  else
+    stats[:diff] += 1
+    diffs << label
+    if diffs.size <= 10
+      puts "DIFF #{label}"
+      ref_html.lines.zip(html.lines).each_with_index do |(a, b), i|
+        next if a == b
+        puts "  line #{i + 1}: RD=#{a.inspect}"
+        puts "           MD=#{b.inspect}"
+        break
+      end
+    end
+  end
+rescue StandardError => e
+  stats[:error] += 1
+  diffs << label
+  puts "ERROR #{label}: #{e.class}: #{e.message}" if stats[:error] <= 10
+end
+
+count = 0
+db.classes.sort_by(&:name).each_with_index do |c, ci|
+  break unless run.call('methods')
+  next unless in_shard.call(ci)
+  c.entries.sort_by { |e| e.names.join }.each do |m|
+    break if limit && count >= limit
+    count += 1
+    ref = begin
+      rd.compile_method(m, nil)
+    rescue StandardError => e
+      stats[:ref_error] += 1
+      puts "REF-ERROR #{c.name}##{m.names.first}: #{e.class}: #{e.message}" if stats[:ref_error] <= 5
+      next
+    end
+    stats[:current_entry] = m
+    check.call("#{c.name}##{m.names.first}", :method, m.source, ref)
+  end
+end
+
+db.docs.sort_by(&:name).each do |d|
+  break unless run.call('docs')
+  ref = begin
+    rd.compile(d.source)
+  rescue StandardError
+    stats[:ref_error] += 1
+    next
+  end
+  check.call("doc:#{d.name}", :doc, d.source, ref)
+end
+
+db.libraries.sort_by(&:name).each do |l|
+  break unless run.call('libs')
+  next if l.source.strip.empty?
+  ref = begin
+    rd.compile(l.source)
+  rescue StandardError
+    stats[:ref_error] += 1
+    next
+  end
+  check.call("lib:#{l.name}", :doc, l.source, ref)
+end
+
+begin
+  fdb = BitClust::FunctionDatabase.new(db_path) if run.call('functions')
+  fdb ||= nil
+  (fdb ? fdb.functions.sort_by(&:name) : []).each do |f|
+    ref = begin
+      rd.compile_function(f, nil)
+    rescue StandardError
+      stats[:ref_error] += 1
+      next
+    end
+    stats[:current_entry] = f
+    check.call("capi:#{f.name}", :function, f.source, ref)
+  end
+rescue StandardError
+  puts '(no function database)'
+end
+
+puts "compared: #{stats[:ok] + stats[:diff]}, identical: #{stats[:ok]}, " \
+     "diffs: #{stats[:diff]}, errors: #{stats[:error]}, " \
+     "ref-errors(skipped): #{stats[:ref_error]}, fragment-roundtrip diffs: #{stats[:rt_diff]}"
+puts "gfm: consistent: #{stats[:gfm_ok]}, diffs: #{stats[:gfm_diff]}" if gfm_mode
+diffs.first(30).each { |d| puts "  DIFF #{d}" } if stats[:diff] > 10
+ok = stats[:diff].zero? && stats[:error].zero? && stats[:gfm_diff].zero?
+puts ok ? 'HTML EQUIVALENT' : 'NOT EQUIVALENT'

data/tools/md-db-check.rb

@@ -0,0 +1,112 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+#
+# 2つの BitClust データベースを比較する（md 移行の DB レベル等価検証）。
+# ライブラリ・クラス・メソッドの集合と、全エントリの本文（source）まで突き合わせる。
+#
+# usage: ruby tools/md-db-check.rb <db-old> <db-new>
+
+$LOAD_PATH.unshift File.expand_path('../lib', __dir__)
+require 'bitclust'
+
+old_path, new_path = ARGV
+abort "usage: #{$0} <db-old> <db-new>" unless old_path && new_path
+
+old_db = BitClust::MethodDatabase.new(old_path)
+new_db = BitClust::MethodDatabase.new(new_path)
+
+diffs = 0
+report = lambda do |msg|
+  diffs += 1
+  puts "  DIFF #{msg}"
+end
+
+compare_sets = lambda do |label, old_set, new_set|
+  only_old = old_set - new_set
+  only_new = new_set - old_set
+  only_old.each { |x| report.call("#{label} only in old: #{x}") }
+  only_new.each { |x| report.call("#{label} only in new: #{x}") }
+  old_set & new_set
+end
+
+# ライブラリ
+old_libs = old_db.libraries.to_h { |l| [l.name, l] }
+new_libs = new_db.libraries.to_h { |l| [l.name, l] }
+common_libs = compare_sets.call('library', old_libs.keys.sort, new_libs.keys.sort)
+puts "libraries: #{common_libs.size} common"
+
+lib_field_diffs = 0
+common_libs.each do |name|
+  o, n = old_libs[name], new_libs[name]
+  lib_field_diffs += 1 if o.requires.map(&:name).sort != n.requires.map(&:name).sort
+  lib_field_diffs += 1 if o.sublibraries.map(&:name).sort != n.sublibraries.map(&:name).sort
+  report.call("library #{name}: source differs") if o.source.strip != n.source.strip
+end
+puts "library require/sublibrary diffs: #{lib_field_diffs}"
+
+# クラスとエントリ
+old_classes = old_db.classes.to_h { |c| [c.name, c] }
+new_classes = new_db.classes.to_h { |c| [c.name, c] }
+common = compare_sets.call('class', old_classes.keys.sort, new_classes.keys.sort)
+puts "classes: #{common.size} common"
+
+entry_count = 0
+ws_only = 0
+sig_spacing_only = 0
+source_diffs = []
+# 「---name」（スペース無しシグネチャ）は正規形「--- name」へ寄せる
+# （RRDParser/RDCompiler はどちらも受理。openssl の8箇所）
+normalize_sig = ->(src) { src.gsub(/^---(?=[^\s-])/, '--- ') }
+common.each do |name|
+  o, n = old_classes[name], new_classes[name]
+  report.call("class #{name}: type #{o.type} vs #{n.type}") if o.type != n.type
+  report.call("class #{name}: superclass differs") if o.superclass&.name != n.superclass&.name
+  report.call("class #{name}: source differs") if o.source.strip != n.source.strip
+
+  o_entries = o.entries.to_h { |e| [e.names.sort.join(','), e] }
+  n_entries = n.entries.to_h { |e| [e.names.sort.join(','), e] }
+  common_entries = compare_sets.call("#{name} entry", o_entries.keys.sort, n_entries.keys.sort)
+  common_entries.each do |key|
+    entry_count += 1
+    next if o_entries[key].source == n_entries[key].source
+    if o_entries[key].source.rstrip == n_entries[key].source.rstrip
+      ws_only += 1   # 末尾空白のみ（描画に影響しない）
+    elsif normalize_sig.call(o_entries[key].source) == n_entries[key].source
+      sig_spacing_only += 1
+    else
+      source_diffs << "#{name}##{key}"
+    end
+  end
+end
+puts "entries compared: #{entry_count}, real source diffs: #{source_diffs.size}, " \
+     "trailing-whitespace-only: #{ws_only}, signature-spacing-only: #{sig_spacing_only}"
+source_diffs.first(10).each { |s| puts "  SOURCE DIFF #{s}" }
+
+# doc（散文ページ）。md 経路は DocConverter.reduce の正規化（末尾スペース・
+# タブ・定義リスト空白等、意味を変えない表記ゆれ）を通るため、
+# 旧側も reduce に通した上での一致を「正規化後一致」として別集計する
+require 'bitclust/doc_converter'
+old_docs = old_db.docs.to_h { |d| [d.name, d] }
+new_docs = new_db.docs.to_h { |d| [d.name, d] }
+common_docs = compare_sets.call('doc', old_docs.keys.sort, new_docs.keys.sort)
+doc_normalized = 0
+doc_diffs = common_docs.count do |k|
+  next false if old_docs[k].source == new_docs[k].source
+  if BitClust::DocConverter.reduce(old_docs[k].source) == new_docs[k].source
+    doc_normalized += 1
+    false
+  else
+    true
+  end
+end
+puts "docs: #{common_docs.size} common, real source diffs: #{doc_diffs}, " \
+     "normalized-only: #{doc_normalized}"
+
+if diffs.zero? && source_diffs.empty? && doc_diffs.zero?
+  notes = []
+  notes << "#{ws_only} trailing-whitespace-only" if ws_only.positive?
+  notes << "#{sig_spacing_only} signature-spacing-only" if sig_spacing_only.positive?
+  puts "DATABASES EQUIVALENT#{notes.empty? ? '' : " (#{notes.join(', ')} diffs)"}"
+else
+  puts "TOTAL STRUCTURAL DIFFS: #{diffs + source_diffs.size + doc_diffs}"
+end

data/tools/md-parse-check.rb

@@ -0,0 +1,185 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+#
+# MDParser のネイティブ md パース検証（M3）。
+# manual/api をネイティブパースした DB と、ブリッジ経由で構築した DB を比較する:
+#   (1) library/class/entry の集合と属性が一致
+#   (2) native の entry.source（md）を md→rd 変換するとブリッジの source（rd）に一致
+#
+# usage: ruby tools/md-parse-check.rb <manual/api> <bridge-db> [--keep DIR]
+#        ruby tools/md-parse-check.rb --capi <manual/capi> <bridge-db>
+
+$stdout.sync = true
+$LOAD_PATH.unshift File.expand_path('../lib', __dir__)
+require 'bitclust'
+require 'bitclust/markdown_to_rrd'
+require 'tmpdir'
+
+capi = ARGV.delete('--capi')
+md_root = ARGV[0] or abort "usage: #{$0} <manual/api> <bridge-db>"
+bridge_path = ARGV[1] or abort "usage: #{$0} <manual/api> <bridge-db>"
+keep_dir = (i = ARGV.index('--keep')) ? ARGV[i + 1] : nil
+
+if capi
+  require 'bitclust/capi_converter'
+  bridge = BitClust::FunctionDatabase.new(bridge_path)
+  version = bridge.properties['version']
+  native_dir = Dir.mktmpdir('md-parse-check-capi')
+  # init は MethodDatabase 側にのみある（properties 作成のため一時利用）
+  seed = BitClust::MethodDatabase.new(native_dir)
+  seed.init
+  seed.transaction do
+    seed.propset 'version', version
+    seed.propset 'encoding', bridge.properties['encoding']
+  end
+  native = BitClust::FunctionDatabase.new(native_dir)
+  puts "native capi parse (version #{version})..."
+  native.transaction { native.update_by_markdowntree(md_root) }
+  native = BitClust::FunctionDatabase.new(native_dir)
+
+  bf = bridge.functions.to_h { |f| [f.name, f] }
+  nf = native.functions.to_h { |f| [f.name, f] }
+  diffs = 0
+  puts "DIFF function set: missing=#{(bf.keys - nf.keys).first(5)} extra=#{(nf.keys - bf.keys).first(5)}" or diffs += 1 if bf.keys.sort != nf.keys.sort
+  src_diffs = 0
+  (bf.keys & nf.keys).each do |name|
+    b, n = bf[name], nf[name]
+    %i[filename macro private type params].each do |attr|
+      if b.send(attr) != n.send(attr)
+        diffs += 1
+        puts "DIFF #{name}: #{attr} #{b.send(attr).inspect} vs #{n.send(attr).inspect}" if diffs <= 10
+      end
+    end
+    converted = BitClust::MarkdownToRRD.convert(n.source.to_s, capi: true)
+                  .gsub(/^\#[@%]samplecode(?: (.*))?$/) { "//emlist[#{$1&.strip}][ruby]{" }
+                  .gsub(/^\#[@%]end[ \t]*$/, '//}')
+    if converted.rstrip != b.source.to_s.rstrip
+      src_diffs += 1
+      puts "SRC-DIFF #{name}" if src_diffs <= 5
+    end
+  end
+  puts "functions: #{(bf.keys & nf.keys).size}, attribute diffs: #{diffs}, source diffs: #{src_diffs}"
+  puts diffs.zero? && src_diffs.zero? ? 'NATIVE CAPI PARSE EQUIVALENT' : 'NOT EQUIVALENT'
+  FileUtils.remove_entry(native_dir)
+  exit(diffs.zero? && src_diffs.zero? ? 0 : 1)
+end
+
+bridge = BitClust::MethodDatabase.new(bridge_path)
+version = bridge.properties['version']
+
+native_dir = keep_dir || Dir.mktmpdir('md-parse-check')
+native = BitClust::MethodDatabase.new(native_dir)
+unless File.exist?(File.join(native_dir, 'properties'))
+  native.init
+  native.transaction do
+    native.propset 'version', version
+    native.propset 'encoding', bridge.properties['encoding']
+  end
+end
+puts "native parse (version #{version})..."
+native.transaction do
+  native.update_by_markdowntree(md_root)
+end
+# ブリッジ側（ディスクから読む）と条件を揃える。require が作る未保存の
+# スタブライブラリ等、メモリ上にしか無いものを比較に混ぜない
+native = BitClust::MethodDatabase.new(native_dir)
+
+diffs = 0
+report = ->(msg) { diffs += 1; puts "DIFF #{msg}" if diffs <= 30 }
+
+# ブリッジ DB の source は Preprocessor 通過後（#@samplecode → //emlist 済み）。
+# native md → rd 変換の生形式を同じ形へ正規化して比較する
+to_rd = lambda do |md_src|
+  BitClust::MarkdownToRRD.convert(md_src)
+    .gsub(/^\#[@%]samplecode(?: (.*))?$/) { "//emlist[#{$1&.strip}][ruby]{" }
+    .gsub(/^\#[@%]end[ \t]*$/, '//}')
+    .rstrip
+end
+
+# (1) ライブラリ
+bl = bridge.libraries.to_h { |l| [l.name, l] }
+nl = native.libraries.to_h { |l| [l.name, l] }
+report.call("library set: missing=#{(bl.keys - nl.keys).size} extra=#{(nl.keys - bl.keys).size}: #{((bl.keys - nl.keys) + (nl.keys - bl.keys)).first(5)}") if bl.keys.sort != nl.keys.sort
+common_libs = bl.keys & nl.keys
+common_libs.each do |name|
+  b, n = bl[name], nl[name]
+  report.call("lib #{name}: category #{b.category.inspect} vs #{n.category.inspect}") if b.category != n.category
+  report.call("lib #{name}: requires #{b.requires.map(&:name)} vs #{n.requires.map(&:name)}") if b.requires.map(&:name).sort != n.requires.map(&:name).sort
+  report.call("lib #{name}: sublibraries") if b.sublibraries.map(&:name).sort != n.sublibraries.map(&:name).sort
+  report.call("lib #{name}: classes #{(b.classes.map(&:name) - n.classes.map(&:name)).first(3)} / #{(n.classes.map(&:name) - b.classes.map(&:name)).first(3)}") if b.classes.map(&:name).sort != n.classes.map(&:name).sort
+  if to_rd.call(n.source.to_s + "\n") != b.source.to_s.rstrip
+    report.call("lib #{name}: source differs")
+  end
+end
+
+# (2) クラスとエントリ
+bc = bridge.classes.to_h { |c| [c.name, c] }
+nc = native.classes.to_h { |c| [c.name, c] }
+report.call("class set: missing=#{(bc.keys - nc.keys).first(5)} extra=#{(nc.keys - bc.keys).first(5)}") if bc.keys.sort != nc.keys.sort
+entry_count = 0
+src_diffs = 0
+(bc.keys & nc.keys).each do |name|
+  b, n = bc[name], nc[name]
+  report.call("class #{name}: type #{b.type} vs #{n.type}") if b.type != n.type
+  report.call("class #{name}: superclass") if b.superclass&.name != n.superclass&.name
+  report.call("class #{name}: included #{b.included.map(&:name)} vs #{n.included.map(&:name)}") if b.included.map(&:name) != n.included.map(&:name)
+  report.call("class #{name}: extended") if b.extended.map(&:name) != n.extended.map(&:name)
+  report.call("class #{name}: dyn-included") if b.dynamically_included.map(&:name).sort != n.dynamically_included.map(&:name).sort
+  if to_rd.call(n.source.to_s + "\n") != b.source.to_s.rstrip
+    report.call("class #{name}: source differs")
+  end
+
+  be = b.entries.to_h { |e| [e.names.sort.join(','), e] }
+  ne = n.entries.to_h { |e| [e.names.sort.join(','), e] }
+  report.call("#{name} entries: missing=#{(be.keys - ne.keys).first(3)} extra=#{(ne.keys - be.keys).first(3)}") if be.keys.sort != ne.keys.sort
+  (be.keys & ne.keys).each do |key|
+    entry_count += 1
+    x, y = be[key], ne[key]
+    report.call("#{name}##{key}: type #{x.type} vs #{y.type}") if x.type != y.type
+    report.call("#{name}##{key}: visibility") if x.visibility != y.visibility
+    if to_rd.call(y.source.to_s) != x.source.to_s.rstrip
+      src_diffs += 1
+      if src_diffs <= 5
+        puts "SRC-DIFF #{name}##{key}:"
+        a = x.source.to_s.rstrip.lines
+        c = to_rd.call(y.source.to_s).lines
+        a.zip(c).each_with_index do |(p, q), i|
+          next if p == q
+          puts "  line #{i + 1}: BRIDGE #{p.inspect}"
+          puts "            NATIVE #{q.inspect}"
+          break
+        end
+      end
+    end
+  end
+end
+
+# (3) doc ページ
+bd = bridge.docs.to_h { |d| [d.name, d] }
+nd = native.docs.to_h { |d| [d.name, d] }
+report.call("doc set: missing=#{(bd.keys - nd.keys).first(5)} extra=#{(nd.keys - bd.keys).first(5)}") if bd.keys.sort != nd.keys.sort
+doc_src_diffs = 0
+(bd.keys & nd.keys).each do |name|
+  b, n = bd[name], nd[name]
+  report.call("doc #{name}: title #{b.title.inspect} vs #{n.title.inspect}") if b.title != n.title
+  if to_rd.call(n.source.to_s) != b.source.to_s.rstrip
+    doc_src_diffs += 1
+    if doc_src_diffs <= 3
+      puts "DOC-SRC-DIFF #{name}:"
+      a = b.source.to_s.rstrip.lines
+      c = to_rd.call(n.source.to_s).lines
+      a.zip(c).each_with_index do |(p, q), i|
+        next if p == q
+        puts "  line #{i + 1}: BRIDGE #{p.inspect}"
+        puts "            NATIVE #{q.inspect}"
+        break
+      end
+    end
+  end
+end
+
+puts "libraries: #{common_libs.size}, classes: #{(bc.keys & nc.keys).size}, " \
+     "entries: #{entry_count}, docs: #{(bd.keys & nd.keys).size}"
+puts "attribute diffs: #{diffs}, source diffs (md→rd 変換後): #{src_diffs}, doc source diffs: #{doc_src_diffs}"
+puts diffs.zero? && src_diffs.zero? && doc_src_diffs.zero? ? 'NATIVE PARSE EQUIVALENT' : 'NOT EQUIVALENT'
+FileUtils.remove_entry(native_dir) unless keep_dir

data/tools/md-roundtrip-check.rb

@@ -0,0 +1,145 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+#
+# Markdown 変換のラウンドトリップ検証。
+# refm/api/src の全ファイルについて rd → md → rd がバイト一致するかを確認する。
+#
+# usage: ruby tools/md-roundtrip-check.rb [options] <doctree-root>
+#   --with-doc   refm/doc/**/*.rd も検証する（DocConverter の reduce 基準）
+#   --with-capi  refm/capi/src/*.rd も検証する（CapiConverter）
+#   --inject     MarkdownOrchestrator の変換（prune・全体ゲート解除・front matter 注入）で
+#                検証する: md → rd が reduce 後の rd を復元すること、および
+#                md に grouping include が残っていないことを確認する
+#   --diff       差分のあったファイルの先頭差分行を表示する
+#   -v           全ファイルの結果を表示する
+
+$LOAD_PATH.unshift File.expand_path('../lib', __dir__)
+require 'bitclust/rrd_to_markdown'
+require 'bitclust/markdown_to_rrd'
+require 'bitclust/markdown_orchestrator'
+require 'bitclust/doc_converter'
+require 'bitclust/capi_converter'
+
+with_doc = ARGV.delete('--with-doc')
+with_capi = ARGV.delete('--with-capi')
+inject = ARGV.delete('--inject')
+show_diff = ARGV.delete('--diff')
+verbose = ARGV.delete('-v')
+scope_arg = nil
+if (i = ARGV.index('--scope'))
+  ARGV.delete_at(i)
+  scope_arg = ARGV.delete_at(i) or abort '--scope requires LO,HI'
+end
+doctree = ARGV.shift or abort "usage: #{$0} [--with-doc] [--inject] [--scope LO,HI] [--diff] [-v] <doctree-root>"
+
+src_root = File.join(doctree, 'refm/api/src')
+files = Dir.glob('**/*', base: src_root).select { |f| File.file?(File.join(src_root, f)) }
+files -= ['LIBRARIES']
+
+orchestrator = nil
+prune_sites = {}
+if inject
+  opts = {}
+  if scope_arg
+    opts[:scope] = BitClust::IncludeGraph::Scope.new(*scope_arg.split(','))
+  end
+  orchestrator = BitClust::MarkdownOrchestrator.new(src_root, **opts)
+  orchestrator.warnings.each { |w| warn "W: #{w}" }
+  prune_sites = orchestrator.graph.grouping_include_sites
+  puts "prune: #{prune_sites.size} files"
+end
+
+targets = files.map { |f| [File.join(src_root, f), f] }
+if with_doc
+  doc_root = File.join(doctree, 'refm/doc')
+  # *.rd と拡張子なし（spec/regexp18 等）。news/1.8.0.rd-2 系の
+  # 旧分割ファイルは未参照のため対象外
+  doc_files = Dir.glob('**/*', base: doc_root).select { |f|
+    File.file?(File.join(doc_root, f)) &&
+      (f.end_with?('.rd') || !File.basename(f).include?('.'))
+  }
+  targets += doc_files.map { |f| [File.join(doc_root, f), "doc:#{f}"] }
+end
+if with_capi
+  capi_root = File.join(doctree, 'refm/capi/src')
+  targets += Dir.glob('*.rd', base: capi_root).map { |f| [File.join(capi_root, f), "capi:#{f}"] }
+end
+
+# md 内のエンティティ H1 直後のヘッダ領域に関係行が残っていないか
+# （O3 の不変条件: 関係は front matter のみ）
+def body_relations?(md)
+  in_header = false
+  md.each_line do |line|
+    if line =~ /\A#(?!#)\s*(?:class|module|object|reopen|redefine)\b/
+      in_header = true
+    elsif in_header
+      case line
+      when /\A(?:include|extend|alias)\s+\S/ then return true
+      when /\A\#[@%]/, /\A\s*\z/ then nil
+      else in_header = false
+      end
+    end
+  end
+  false
+end
+
+ok = 0
+units = 0
+failed = []
+leftover = []
+body_rels = []
+targets.sort_by(&:last).each do |full, label|
+  rrd = File.read(full)
+  outs =
+    if orchestrator
+      orchestrator.units(label, rrd)
+                  .map { |u| [u.path, u.rrd, orchestrator.convert_unit(u), u.front_matter] }
+    elsif label.start_with?('doc:')
+      reduced = BitClust::DocConverter.reduce(rrd)
+      [[label, reduced, BitClust::DocConverter.convert(rrd), nil]]
+    elsif label.start_with?('capi:')
+      [[label, rrd, BitClust::CapiConverter.convert(rrd), nil]]
+    else
+      [[label, rrd, BitClust::RRDToMarkdown.convert(rrd), nil]]
+    end
+  outs.each do |path, reduced, md, front_matter|
+    units += 1
+    ulabel = outs.size > 1 ? "#{label} -> #{path}" : label
+    if (sites = prune_sites[label])
+      remaining = md.lines.filter_map { |l| $1 if l =~ /\A\#[@%]include\s*\((.*?)\)/ }
+      leftover << ulabel unless (remaining & sites).empty?
+    end
+    # body 関係の不変条件は in-scope（front matter 注入あり）のみ。
+    # スコープ外ファイルは凍結形のままなので対象外（サルベージで扱う）
+    body_rels << ulabel if front_matter && !front_matter.empty? && body_relations?(md)
+    back = BitClust::MarkdownToRRD.convert(md, capi: label.start_with?('capi:'))
+    if back == reduced
+      ok += 1
+      puts "OK   #{ulabel}" if verbose
+    else
+      failed << ulabel
+      puts "DIFF #{ulabel}" if verbose || show_diff
+      if show_diff
+        reduced.lines.zip(back.lines).each_with_index do |(a, b), i|
+          next if a == b
+          puts "  line #{i + 1}: #{a.inspect} -> #{b.inspect}"
+          break
+        end
+      end
+    end
+  end
+end
+
+puts "#{ok}/#{units} byte-exact roundtrip#{inject ? " (reduced base, #{targets.size} sources)" : ''}"
+unless failed.empty?
+  puts "failed: #{failed.size}"
+  failed.each { |f| puts "  #{f}" } unless verbose || show_diff
+end
+unless leftover.empty?
+  puts "grouping includes left unpruned: #{leftover.size}"
+  leftover.each { |f| puts "  #{f}" }
+end
+unless body_rels.empty?
+  puts "header relations left in body: #{body_rels.size}"
+  body_rels.each { |f| puts "  #{f}" }
+end

data/tools/md-tree-check.rb

@@ -0,0 +1,107 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+#
+# 変換後の Markdown ツリーの発見・検証（MARKUP_SPEC §1.1）。
+# LIBRARIES を使わず glob + front matter + H1 だけで構成を組み立て、
+# 孤児・関係リント等の警告を報告する。
+#
+# usage: ruby tools/md-tree-check.rb <md-tree-root> [--src <doctree-src-root>]
+#   --src を与えると、ソース側 include グラフと突き合わせて
+#   - 発見したライブラリ集合が in-scope ライブラリと一致するか
+#   - 「library の無いエンティティ」がスコープ外（サルベージ待ち）由来だけか
+#   - in-scope エンティティ名の重複が無いか
+#   を確認する
+
+$LOAD_PATH.unshift File.expand_path('../lib', __dir__)
+require 'bitclust/markdown_tree'
+require 'bitclust/include_graph'
+
+md_root = nil
+src_root = nil
+scope_arg = nil
+args = ARGV.dup
+until args.empty?
+  a = args.shift
+  if a == '--src'
+    src_root = args.shift
+  elsif a == '--scope'
+    scope_arg = args.shift
+  else
+    md_root = a
+  end
+end
+abort "usage: #{$0} <md-tree-root> [--src <doctree-src-root>] [--scope LO,HI]" unless md_root
+
+tree = BitClust::MarkdownTree.scan(md_root)
+entity_count = tree.entities.values.sum { |e| e[:names].size }
+puts "libraries: #{tree.libraries.size}, entity files: #{tree.entities.size} " \
+     "(#{entity_count} entities), fragments: #{tree.fragments.size}"
+
+no_lib = tree.entities.select { |path, e| e[:library].nil? && !tree.libraries.key?(path.sub(/\.md\z/, '')) }
+puts "entities without library: #{no_lib.size}"
+
+if src_root
+  graph = BitClust::IncludeGraph.analyze(src_root)
+  scope = BitClust::IncludeGraph::Scope.new(*(scope_arg || '3.0,4.2').split(','))
+  fm = graph.front_matter_map(scope)
+  lib_fm = graph.library_front_matter_map(scope)
+
+  md_path = ->(rel) { rel.end_with?('.rd') ? rel.sub(/\.rd\z/, '.md') : "#{rel}.md" }
+
+  # ライブラリ集合のパリティ
+  expected_libs = lib_fm.keys.map { |r| r.sub(/\.rd\z/, '') }.sort
+  actual_libs = tree.libraries.keys.sort
+  puts "library set parity: #{actual_libs == expected_libs ? 'OK' : 'MISMATCH'} " \
+       "(expected #{expected_libs.size}, found #{actual_libs.size})"
+  (expected_libs - actual_libs).each { |l| puts "  missing library: #{l}" }
+  (actual_libs - expected_libs).each { |l| puts "  unexpected library: #{l}" }
+
+  # library なしエンティティの期待バケット:
+  # (1) スコープ外メンバー（グラフ到達だがスコープ外。サルベージ待ち）
+  # (2) スコープ外ライブラリのルート（LIBRARIES ゲートで除外、インライン・エンティティ持ち）
+  # (3) ソース孤児（グラフ到達不能の旧世代ファイル）
+  out_members = (graph.groupings.keys - fm.keys).map { |r| md_path.call(r) }
+  all_lib_roots = File.foreach(File.join(src_root, 'LIBRARIES'))
+                      .map(&:chomp).reject { |l| l.empty? || l.start_with?('#@', '#%') }
+                      .map { |n| "#{n}.rd" }.uniq
+  out_lib_mds = (all_lib_roots - lib_fm.keys).map { |r| md_path.call(r) }
+  reachable = graph.groupings.keys + graph.fragments + all_lib_roots
+  src_files = Dir.glob('**/*', base: src_root)
+                 .select { |f| File.file?(File.join(src_root, f)) } - ['LIBRARIES']
+  orphan_mds = (src_files - reachable).map { |r| md_path.call(r) }
+
+  buckets = { 'out-of-scope member' => out_members,
+              'out-of-scope library' => out_lib_mds,
+              'source orphan' => orphan_mds }
+  rest = no_lib.keys
+  buckets.each do |label, set|
+    hit = rest & set
+    rest -= hit
+    puts "  #{label}: #{hit.size}"
+  end
+  puts "  UNEXPECTED: #{rest.size}"
+  rest.each { |p| puts "    #{p}" }
+
+  # in-scope エンティティ名の重複（reopen/redefine は各 lib からの寄与なので除外）
+  names = Hash.new { |h, k| h[k] = [] }
+  tree.entities.each do |path, e|
+    next if e[:library].nil? && !tree.libraries.key?(path.sub(/\.md\z/, ''))
+    e[:kinds].each { |kind, n| names[n] << path if %w[class module object].include?(kind) }
+  end
+  dups = names.select { |_, paths| paths.uniq.size > 1 }
+  puts "in-scope entity definition duplicates (reopen/redefine 除外): #{dups.size}"
+  dups.each { |n, paths| puts "  #{n}: #{paths.uniq.join(', ')}" }
+
+  # 警告をスコープ外・ソース孤児（期待=サルベージ待ち）とそれ以外に分類
+  expected_paths = (out_members + out_lib_mds + orphan_mds).to_h { |p| [p, true] }
+  expected, real = tree.warnings.partition do |w|
+    path = w[/\S+\.md\z/]
+    path && expected_paths[path]
+  end
+  puts "warnings: #{tree.warnings.size} " \
+       "(expected out-of-scope/salvage: #{expected.size}, needs attention: #{real.size})"
+  real.each { |w| puts "  W: #{w}" }
+else
+  puts "warnings: #{tree.warnings.size}"
+  tree.warnings.each { |w| puts "  W: #{w}" }
+end

data/tools/stattodo.rb

@@ -85,7 +85,7 @@ def count_todo_in_file(path)
   n = 0
   File.open(path) {|f|
     pp = BitClust::LineCollector.wrap(f)
-    pp.grep(/\A\#@todo/) { n += 1 }
+    pp.grep(/\A\#[@%]todo/) { n += 1 }
   }
   n
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: bitclust-dev
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.6.1
 platform: ruby
 authors:
 - https://github.com/rurema
-autorequire: 
 bindir: tools
 cert_chain: []
-date: 2020-12-29 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bitclust-core
@@ -16,14 +15,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.6.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: 1.6.1
 description: |
   Rurema is a Japanese ruby documentation project, and
   bitclust is a rurema document processor.
@@ -31,15 +30,21 @@ description: |
 email:
 - ''
 executables:
-- gencatalog.rb
-- insert-canonical.rb
+- bc-checkparams.rb
 - bc-convert.rb
+- bc-rdoc.rb
 - forall-ruby.rb
-- bc-checkparams.rb
+- gencatalog.rb
+- insert-canonical.rb
+- md-bridge-check.rb
+- md-compile-check.rb
+- md-db-check.rb
+- md-parse-check.rb
+- md-roundtrip-check.rb
+- md-tree-check.rb
 - statrefm.rb
 - stattodo.rb
 - update-database.rb
-- bc-rdoc.rb
 extensions: []
 extra_rdoc_files: []
 files:
@@ -50,6 +55,12 @@ files:
 - tools/forall-ruby.rb
 - tools/gencatalog.rb
 - tools/insert-canonical.rb
+- tools/md-bridge-check.rb
+- tools/md-compile-check.rb
+- tools/md-db-check.rb
+- tools/md-parse-check.rb
+- tools/md-roundtrip-check.rb
+- tools/md-tree-check.rb
 - tools/statrefm.rb
 - tools/stattodo.rb
 - tools/update-database.rb
@@ -62,7 +73,6 @@ metadata:
   source_code_uri: https://github.com/rurema/bitclust
   github_repo: https://github.com/rurema/bitclust
   wiki_uri: https://github.com/rurema/doctree/wiki
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -77,8 +87,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key: 
+rubygems_version: 3.6.9
 specification_version: 4
 summary: BitClust is a rurema document processor.
 test_files: []