ucode 0.2.2 → 0.3.0

data/lib/ucode/commands/glyphs.rb

@@ -1,94 +0,0 @@
-# frozen_string_literal: true
-
-require "pathname"
-
-require "ucode/glyphs"
-
-module Ucode
-  module Commands
-    # `ucode glyphs` — extract per-codepoint SVGs from Code Charts PDFs.
-    # Thin Thor-facing wrapper around {Ucode::Glyphs::Pipeline}:
-    # opt-in gate + experimental warning live here; the pipeline
-    # assembly (block loading, fetcher, per-block specs) lives in
-    # {Ucode::Glyphs::Pipeline}.
-    #
-    # **Status (v0.1): EXPERIMENTAL.** The cell-extraction pipeline
-    # currently includes cell-border decorations alongside the actual
-    # character outline because the Code Charts PDFs composite the two
-    # into a single glyph definition. The output is therefore not yet
-    # suitable for end-user display. The command is retained so the
-    # pipeline can be iterated on without churning the CLI surface, but
-    # callers MUST opt in via `include_glyphs: true` (CLI: `--include-glyphs`)
-    # and will receive a printed warning. Tracked for v0.2.
-    #
-    # Takes a resolved version string; CLI callers resolve via
-    # {VersionResolver.resolve} once and thread it through. See
-    # Candidate 4 of the 2026-06-29 architecture review.
-    class GlyphsCommand
-      ExperimentalWarning = "ucode glyphs is experimental in v0.1: " \
-                            "extracted SVGs include cell-border decorations " \
-                            "alongside the character outline."
-      private_constant :ExperimentalWarning
-
-      class << self
-        # @return [String] the experimental-status banner. Exposed so the
-        #   CLI and BuildCommand surface the same message verbatim.
-        def experimental_warning
-          ExperimentalWarning
-        end
-      end
-
-      # @param version [String] resolved UCD version
-      # @param output_root [String, Pathname]
-      # @param block_filter [Array<String>, nil] block ids to limit to;
-      #   nil = every block
-      # @param force [Boolean] re-fetch PDFs even when cached
-      # @param monolith_path [String, Pathname, nil] path to CodeCharts.pdf
-      #   for fallback slicing; defaults to ./CodeCharts.pdf
-      # @param include_glyphs [Boolean] opt-in for the experimental v0.1
-      #   pipeline. When false (default), the command returns a `skipped`
-      #   payload without touching disk.
-      # @param warn [IO, nil] when provided, the experimental warning is
-      #   written here exactly once before work begins.
-      # @return [Hash] aggregated Writer tally + version, or a `skipped`
-      #   payload when opt-in is false.
-      def call(version, output_root:,
-               block_filter: nil, force: false,
-               monolith_path: Glyphs::Pipeline::DEFAULT_MONOLITH_PATH,
-               include_glyphs: false, warn: nil)
-        return skipped(version) unless include_glyphs
-
-        warn&.puts(ExperimentalWarning)
-
-        pipeline = Glyphs::Pipeline.new(
-          version: version,
-          block_filter: block_filter,
-          monolith_path: monolith_path,
-        )
-        specs = pipeline.build_specs(force: force)
-
-        writer = Glyphs::Writer.new(
-          output_root: Pathname.new(output_root),
-          parallel_workers: workers,
-        )
-        tally = writer.write_all(specs)
-        tally.merge(version: version, block_count: specs.size)
-      end
-
-      private
-
-      def workers
-        Ucode.configuration.parallel_workers
-      end
-
-      def skipped(version)
-        {
-          version: version,
-          skipped: true,
-          reason: :experimental_v0_1,
-          warning: ExperimentalWarning,
-        }
-      end
-    end
-  end
-end

data/lib/ucode/glyphs/cell_extractor.rb

@@ -1,130 +0,0 @@
-# frozen_string_literal: true
-
-require "nokogiri"
-
-require "ucode/glyphs/path_bbox"
-
-module Ucode
-  module Glyphs
-    # Extracts a single character cell from a Code Charts SVG page and
-    # returns a normalized standalone SVG containing only that cell's
-    # vector paths.
-    #
-    # The cell is identified by codepoint. The extractor asks the Grid
-    # for the cell's anchor position, finds the `<use>` element placed
-    # at that position, resolves its glyph definition from `<defs>`,
-    # and emits a fresh `<svg>` whose viewBox is `0 0 1000 1000` and
-    # whose body is the glyph's `<path>` data translated and scaled to
-    # fit that viewBox with a small margin.
-    #
-    # Vector-only. Never rasterizes, never OCRs. If the cell is empty
-    # (no character glyph placed there, e.g. unassigned codepoint or
-    # control character), the extractor returns nil.
-    class CellExtractor
-      ViewBoxSize = 1000.0
-      MarginRatio = 0.1
-      private_constant :ViewBoxSize, :MarginRatio
-
-      # @param doc [Nokogiri::XML::Document] the rendered Code Charts page
-      def initialize(doc)
-        @doc = doc
-        @glyph_cache = {}
-      end
-
-      # @param grid [Ucode::Glyphs::Grid]
-      # @param codepoint [Integer]
-      # @return [Nokogiri::XML::Document, nil] a standalone `<svg>` doc
-      #   with viewBox `0 0 1000 1000`, or nil if the cell is empty.
-      def extract(grid, codepoint)
-        anchor = grid.cell_position(codepoint)
-        return nil unless anchor
-
-        use_node = find_use_at(anchor, grid)
-        return nil unless use_node
-
-        path_data = collect_paths(use_node["xlink:href"] || use_node["href"])
-        return nil if path_data.empty?
-
-        bbox = PathBbox.estimate(path_data.join(" "))
-        return nil if bbox.empty?
-
-        build_svg(path_data, bbox, use_node["x"].to_f, use_node["y"].to_f)
-      end
-
-      private
-
-      def find_use_at(anchor, grid)
-        tolerance_x = grid.column_pitch / 2
-        tolerance_y = grid.row_pitch / 2
-
-        candidates = @doc.css("use").select do |node|
-          href = node["xlink:href"] || node["href"] || ""
-          href.start_with?("#glyph-") &&
-            (node["x"].to_f - anchor[0]).abs <= tolerance_x &&
-            (node["y"].to_f - anchor[1]).abs <= tolerance_y
-        end
-
-        candidates.min_by do |node|
-          dx = node["x"].to_f - anchor[0]
-          dy = node["y"].to_f - anchor[1]
-          (dx * dx) + (dy * dy)
-        end
-      end
-
-      def collect_paths(href)
-        return [] unless href
-
-        glyph_id = href.sub(/\A#/, "")
-        node = glyph_definition(glyph_id)
-        return [] unless node
-
-        node.css("path").map { |p| p["d"] }.compact
-      end
-
-      def glyph_definition(glyph_id)
-        return @glyph_cache[glyph_id] if @glyph_cache.key?(glyph_id)
-
-        @glyph_cache[glyph_id] = @doc.at_css("defs ##{glyph_id}")
-      end
-
-      def build_svg(path_data, glyph_bbox, place_x, place_y)
-        placed = PathBbox::Result.new(
-          min_x: place_x + glyph_bbox.min_x,
-          min_y: place_y + glyph_bbox.min_y,
-          max_x: place_x + glyph_bbox.max_x,
-          max_y: place_y + glyph_bbox.max_y,
-        )
-
-        width = placed.width
-        height = placed.height
-        return nil if width <= 0 || height <= 0
-
-        content_size = ViewBoxSize * (1.0 - (2.0 * MarginRatio))
-        scale = [content_size / width, content_size / height].min
-        offset_x = (ViewBoxSize - (width * scale)) / 2.0
-        offset_y = (ViewBoxSize - (height * scale)) / 2.0
-        translate_x = offset_x - (placed.min_x * scale)
-        translate_y = offset_y - (placed.min_y * scale)
-
-        builder = Nokogiri::XML::Document.new
-        root = builder.create_element(
-          "svg",
-          xmlns: "http://www.w3.org/2000/svg",
-          viewBox: "0 0 #{ViewBoxSize.to_i} #{ViewBoxSize.to_i}",
-          width: ViewBoxSize.to_i,
-          height: ViewBoxSize.to_i,
-        )
-        group = builder.create_element(
-          "g",
-          transform: "scale(#{format('%.6f', scale)}) translate(#{format('%.6f', translate_x)}, #{format('%.6f', translate_y)})",
-        )
-        path_data.each do |d|
-          group.add_child(builder.create_element("path", d: d, fill: "black"))
-        end
-        root.add_child(group)
-        builder.add_child(root)
-        builder
-      end
-    end
-  end
-end

data/lib/ucode/glyphs/dvisvgm_renderer.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-require "ucode/glyphs/page_renderer"
-
-module Ucode
-  module Glyphs
-    # `dvisvgm` — originally a DVI-to-SVG converter, also handles PDF.
-    # The `--no-fonts` flag forces outline-only output (no font subsetting
-    # artifacts), which is what we want for vector glyph extraction.
-    #
-    # Command: `dvisvgm --pdf --no-fonts --page=<n> <in.pdf> -o <out.svg>`
-    class DvisvgmRenderer < PageRenderer
-      class << self
-        def renderer_name
-          :dvisvgm
-        end
-
-        def binary_name
-          :dvisvgm
-        end
-
-        def build_command(pdf_path, page_num, out_path)
-          ["dvisvgm", "--pdf", "--no-fonts", "--page=#{page_num}",
-           pdf_path.to_s, "-o", out_path.to_s]
-        end
-      end
-    end
-  end
-end

data/lib/ucode/glyphs/grid.rb

@@ -1,30 +0,0 @@
-# frozen_string_literal: true
-
-module Ucode
-  module Glyphs
-    Grid = Struct.new(
-      :origin_x, :origin_y,
-      :column_pitch, :row_pitch,
-      :columns, :rows,
-      :block_first_cp,
-      keyword_init: true,
-    ) do
-      def cell_position(codepoint)
-        offset = codepoint - block_first_cp
-        return nil if offset.negative?
-
-        row, col = offset.divmod(columns)
-        return nil if row >= rows
-
-        [origin_x + (col * column_pitch), origin_y + (row * row_pitch)]
-      end
-
-      def codepoint_at(row, col)
-        return nil if row.negative? || row >= rows
-        return nil if col.negative? || col >= columns
-
-        block_first_cp + (row * columns) + col
-      end
-    end
-  end
-end

data/lib/ucode/glyphs/grid_detector.rb

@@ -1,165 +0,0 @@
-# frozen_string_literal: true
-
-require "nokogiri"
-
-require "ucode/glyphs/grid"
-require "ucode/glyphs/path_bbox"
-
-module Ucode
-  module Glyphs
-    # Detects the chart grid in a Code Charts PDF page rendered to SVG.
-    #
-    # The PDF page produced by pdftocairo / pdf2svg / dvisvgm contains
-    # every visible element (title, block name, row labels, codepoint
-    # digits, and the actual character glyphs) as positioned `<use>`
-    # references into a `<defs>` block of named glyph outlines. The
-    # character cells we want to extract correspond to glyphs whose
-    # bounding box is larger than every label or digit font on the
-    # page — the chart's character samples are drawn at a larger size
-    # than any of the surrounding text.
-    #
-    # Algorithm:
-    #   1. Walk `<defs>`, estimate each glyph's bbox via `PathBbox`.
-    #   2. Classify a glyph as "character-sized" when its width and
-    #      height both exceed `CharSizeThreshold` (default 8 pt).
-    #      This excludes title, row-label, and digit glyphs while
-    #      keeping every actual character sample — including pages
-    #      where the chart mixes multiple character fonts (e.g. the
-    #      Basic Latin page uses one font for punctuation/digits and
-    #      another for letters).
-    #   3. Collect every `<use>` that references a character-sized
-    #      glyph; these are the cell origins.
-    #   4. Cluster the Y values of those uses into rows, and within
-    #      each row cluster the X values into columns.
-    #   5. Drop rows whose column count diverges from the modal value
-    #      (these are footer/header artifacts, not chart rows).
-    #   6. Return a `Grid` value object anchored at the top-left cell
-    #      with uniform column/row pitches derived from the median
-    #      spacing between adjacent clusters.
-    #
-    # This is pure (no I/O). The detector takes a parsed Nokogiri
-    # document and returns a `Grid`.
-    class GridDetector
-      CharSizeThreshold = 8.0
-      ClusterEpsilon = 15.0
-      private_constant :CharSizeThreshold, :ClusterEpsilon
-
-      class << self
-        # @param doc [Nokogiri::XML::Document]
-        # @param block_first_cp [Integer] first codepoint of the block;
-        #   stored on the Grid so callers can map codepoint ↔ cell.
-        # @return [Ucode::Glyphs::Grid, nil] nil if no character grid
-        #   could be detected.
-        def detect(doc, block_first_cp:)
-          uses = collect_uses(doc)
-          return nil if uses.empty?
-
-          char_glyph_ids = char_sized_glyph_ids(doc)
-          return nil if char_glyph_ids.empty?
-
-          cell_uses = uses.select { |u| char_glyph_ids.include?(u.glyph_id) }
-          return nil if cell_uses.empty?
-
-          build_grid(cell_uses, block_first_cp)
-        end
-
-        private
-
-        UsePosition = Struct.new(:x, :y, :glyph_id, :set_id, keyword_init: true)
-
-        def collect_uses(doc)
-          doc.css("use").map do |node|
-            href = node["xlink:href"] || node["href"] || ""
-            glyph_id = href.sub(/\A#/, "")
-            match = glyph_id.match(/\Aglyph-(\d+)-(\d+)\z/)
-            next nil unless match
-
-            UsePosition.new(
-              x: node["x"].to_f,
-              y: node["y"].to_f,
-              glyph_id: glyph_id,
-              set_id: match[1].to_i,
-            )
-          end.compact
-        end
-
-        def char_sized_glyph_ids(doc)
-          doc.css("defs g[id^='glyph-']").each_with_object({}) do |g, acc|
-            id = g["id"]
-            next unless id =~ /\Aglyph-\d+-\d+\z/
-
-            paths = g.css("path")
-            next if paths.empty?
-
-            bbox = paths.map { |p| PathBbox.estimate(p["d"]) }.reject(&:empty?).reduce do |a, b|
-              PathBbox::Result.new(
-                min_x: [a.min_x, b.min_x].min,
-                min_y: [a.min_y, b.min_y].min,
-                max_x: [a.max_x, b.max_x].max,
-                max_y: [a.max_y, b.max_y].max,
-              )
-            end
-            next unless bbox
-
-            acc[id] = true if char_sized?(bbox)
-          end
-        end
-
-        def char_sized?(bbox)
-          bbox.width >= CharSizeThreshold && bbox.height >= CharSizeThreshold
-        end
-
-        def median(values)
-          return 0.0 if values.empty?
-
-          sorted = values.sort
-          mid = sorted.size / 2
-          sorted.size.even? ? (sorted[mid - 1] + sorted[mid]) / 2.0 : sorted[mid]
-        end
-
-        def build_grid(cell_uses, block_first_cp)
-          row_clusters = cluster_by_value(cell_uses, :y)
-          return nil if row_clusters.empty?
-
-          column_clusters = cluster_by_value(cell_uses, :x)
-          return nil if column_clusters.empty?
-
-          column_starts = column_clusters.map { |c| c.map(&:x).min }.sort
-          row_starts = row_clusters.map { |c| c.map(&:y).min }.sort
-
-          Grid.new(
-            origin_x: column_starts.first,
-            origin_y: row_starts.first,
-            column_pitch: median_pitch(column_starts),
-            row_pitch: median_pitch(row_starts),
-            columns: column_starts.size,
-            rows: row_starts.size,
-            block_first_cp: block_first_cp,
-          )
-        end
-
-        def cluster_by_value(items, attr)
-          sorted = items.sort_by { |i| i.public_send(attr) }
-          clusters = []
-          sorted.each do |item|
-            value = item.public_send(attr)
-            if clusters.empty? || (value - clusters.last[:max]).abs > ClusterEpsilon
-              clusters << { max: value, items: [item] }
-            else
-              clusters.last[:max] = value
-              clusters.last[:items] << item
-            end
-          end
-          clusters.map { |c| c[:items] }
-        end
-
-        def median_pitch(sorted_values)
-          return 0.0 if sorted_values.size < 2
-
-          pitches = sorted_values.each_cons(2).map { |a, b| b - a }
-          median(pitches)
-        end
-      end
-    end
-  end
-end

data/lib/ucode/glyphs/monolith_page_map.rb

@@ -1,181 +0,0 @@
-# frozen_string_literal: true
-
-require "pathname"
-require "json"
-require "open3"
-
-module Ucode
-  module Glyphs
-    # Maps a Unicode block's first codepoint to its page range inside the
-    # monolith `CodeCharts.pdf` by parsing the PDF's bookmark outline and
-    # matching each bookmark title to a Block.name from `Blocks.txt`.
-    #
-    # Each chart cluster printed by the Unicode Consortium is a single
-    # bookmark entry:
-    #
-    #   BookmarkTitle: Greek and Coptic
-    #   BookmarkLevel: 1
-    #   BookmarkPageNumber: 415
-    #
-    # The cluster title usually equals a Block.name verbatim, but a few
-    # clusters carry a heading that prepends "C0 Controls and " /
-    # "C1 Controls and " to the block name. We resolve both forms.
-    #
-    # End-page of a cluster is one page before the next cluster's start
-    # page (last cluster's end-page is the PDF's last page).
-    #
-    # The map is cached as JSON at `data/codecharts_page_map.json` so
-    # we don't re-scan the 3,156-page monolith on every run.
-    class MonolithPageMap
-      BookmarkTitleRegex = /BookmarkTitle:\s*(.+)/.freeze
-      BookmarkPageRegex  = /BookmarkPageNumber:\s*(\d+)/.freeze
-      private_constant :BookmarkTitleRegex, :BookmarkPageRegex
-
-      # The Unicode charts print these multi-block clusters as a single
-      # chart page (the C0/C1 control chars are drawn alongside their
-      # block's other characters). Each cluster title maps to the single
-      # block it belongs to.
-      ClusterPrefixes = [
-        "C0 Controls and ",
-        "C1 Controls and ",
-      ].freeze
-      private_constant :ClusterPrefixes
-
-      MapEntry = Struct.new(:first_cp, :start_page, :end_page, keyword_init: true)
-
-      class << self
-        # Build the map by parsing the monolith's outline and matching
-        # each bookmark title to a Block.
-        #
-        # @param monolith_path [String, Pathname]
-        # @param blocks [Array<Ucode::Models::Block>] the parsed Blocks table
-        # @return [Hash{Integer => MapEntry}] keyed by block.range_first
-        def build(monolith_path:, blocks:)
-          name_to_first_cp = blocks.each_with_object({}) do |b, h|
-            h[b.name] = b.range_first
-          end
-          total_pages = page_count(monolith_path)
-          entries = parse_bookmarks(dump_bookmarks(monolith_path), name_to_first_cp)
-          attach_end_pages(entries, total_pages)
-          entries.each_with_object({}) do |e, h|
-            h[e.first_cp] = e
-          end
-        end
-
-        # Pure: parse a `pdftk dump_data` string into a list of
-        # MapEntry rows (without end_pages). Exposed for unit tests
-        # and any caller that already has the dump cached.
-        #
-        # @param dump [String] the raw `pdftk dump_data` output
-        # @param name_to_first_cp [Hash{String => Integer}]
-        # @return [Array<MapEntry>]
-        def parse_bookmarks(dump, name_to_first_cp)
-          entries = []
-          current_title = nil
-          dump.each_line do |line|
-            case line
-            when BookmarkTitleRegex
-              current_title = Regexp.last_match(1).strip
-            when BookmarkPageRegex
-              page = Regexp.last_match(1).to_i
-              cp = resolve_first_cp(current_title, name_to_first_cp)
-              entries << MapEntry.new(first_cp: cp, start_page: page) if cp
-              current_title = nil
-            end
-          end
-          entries.sort_by(&:start_page)
-        end
-
-        # Pure: attach end_pages by sorting entries and assigning each
-        # entry's end to one page before the next entry's start.
-        #
-        # @param entries [Array<MapEntry>]
-        # @param total_pages [Integer, nil] page count of the source PDF;
-        #   the last entry's end_page falls back to this when present.
-        # @return [Array<MapEntry>] the same entries, mutated with end_pages.
-        def attach_end_pages(entries, total_pages = nil)
-          sorted = entries.sort_by(&:start_page)
-          sorted.each_with_index do |entry, i|
-            next_entry = sorted[i + 1]
-            entry.end_page = next_entry ? next_entry.start_page - 1 : total_pages
-          end
-          sorted
-        end
-
-        # Load from cache, or build and cache.
-        # @param monolith_path [String, Pathname]
-        # @param blocks [Array<Ucode::Models::Block>]
-        # @param cache_path [String, Pathname, nil]
-        # @return [Hash{Integer => MapEntry}]
-        def load(monolith_path:, blocks:, cache_path: nil)
-          cache = cache_path && Pathname.new(cache_path)
-          if cache&.exist?
-            return load_from_json(cache.read)
-          end
-
-          map = build(monolith_path: monolith_path, blocks: blocks)
-          write_cache(map, cache) if cache
-          map
-        end
-
-        # Look up a block's page range by its first cp.
-        # @param map [Hash{Integer => MapEntry}]
-        # @param block_first_cp [Integer]
-        # @return [MapEntry, nil]
-        def range_for(map, block_first_cp)
-          map[block_first_cp]
-        end
-
-        # ---- I/O helpers (impure) --------------------------------------
-
-        def dump_bookmarks(monolith_path)
-          out, status = Open3.capture2e("pdftk", monolith_path.to_s, "dump_data")
-          return "" unless status.success?
-
-          out
-        end
-
-        def page_count(monolith_path)
-          out, status = Open3.capture2e("pdfinfo", monolith_path.to_s)
-          return nil unless status.success?
-
-          match = out.match(/^Pages:\s+(\d+)/)
-          match ? match[1].to_i : nil
-        end
-
-        private
-
-        def resolve_first_cp(title, name_to_first_cp)
-          return nil unless title
-
-          return name_to_first_cp[title] if name_to_first_cp.key?(title)
-
-          ClusterPrefixes.each do |prefix|
-            stripped = title.sub(/\A#{Regexp.escape(prefix)}/, "")
-            return name_to_first_cp[stripped] if name_to_first_cp.key?(stripped)
-          end
-
-          nil
-        end
-
-        def write_cache(map, cache_path)
-          payload = map.values.map { |e| { "first_cp" => e.first_cp,
-                                            "start_page" => e.start_page,
-                                            "end_page" => e.end_page } }
-          cache_path.dirname.mkpath
-          cache_path.write(JSON.pretty_generate(payload))
-        end
-
-        def load_from_json(json)
-          payload = JSON.parse(json)
-          payload.each_with_object({}) do |row, h|
-            entry = MapEntry.new(first_cp: row["first_cp"],
-                                 start_page: row["start_page"],
-                                 end_page: row["end_page"])
-            h[entry.first_cp] = entry
-          end
-        end
-      end
-    end
-  end
-end

data/lib/ucode/glyphs/mutool_renderer.rb

@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-
-require "ucode/glyphs/page_renderer"
-
-module Ucode
-  module Glyphs
-    # `mutool draw` from MuPDF — typically the fastest and cleanest.
-    # Emits one `<svg>` per page with `<path>` vector data.
-    #
-    # Command: `mutool draw -F svg -o <out.svg> <in.pdf> <page>`
-    class MutoolRenderer < PageRenderer
-      class << self
-        def renderer_name
-          :mutool
-        end
-
-        def binary_name
-          :mutool
-        end
-
-        def build_command(pdf_path, page_num, out_path)
-          ["mutool", "draw", "-F", "svg", "-o", out_path.to_s,
-           pdf_path.to_s, page_num.to_s]
-        end
-      end
-    end
-  end
-end