ucode 0.1.0

data/lib/ucode/db_builder.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "sqlite3"
+require "time"
+
+require "ucode/cache"
+require "ucode/coordinator"
+require "ucode/database"
+require "ucode/error"
+require "ucode/index_builder"
+
+module Ucode
+  # Builds the SQLite cache for one UCD version.
+  #
+  # Single entry point: `DbBuilder.build(version)`. Streams the
+  # Coordinator output through an IndexBuilder, then persists the
+  # coalesced block + script ranges into a SQLite DB at
+  # `Cache.sqlite_path(version)`.
+  #
+  # **Streaming**: the Coordinator yields one CodePoint at a time; the
+  # IndexBuilder folds it into per-property accumulators. Peak memory
+  # is the in-progress accumulators (~10 MB for the full UCD) plus one
+  # CodePoint — never all 160k CodePoints at once.
+  module DbBuilder
+    SCHEMA_SQL = <<~SQL
+      PRAGMA journal_mode = DELETE;
+      PRAGMA synchronous = NORMAL;
+
+      CREATE TABLE schema_meta (
+        key TEXT PRIMARY KEY,
+        value TEXT NOT NULL
+      );
+
+      CREATE TABLE blocks (
+        first_cp INTEGER NOT NULL,
+        last_cp INTEGER NOT NULL,
+        name TEXT NOT NULL
+      );
+      CREATE INDEX idx_blocks_first_cp ON blocks(first_cp);
+      CREATE INDEX idx_blocks_name ON blocks(name);
+
+      CREATE TABLE scripts (
+        first_cp INTEGER NOT NULL,
+        last_cp INTEGER NOT NULL,
+        name TEXT NOT NULL
+      );
+      CREATE INDEX idx_scripts_first_cp ON scripts(first_cp);
+      CREATE INDEX idx_scripts_name ON scripts(name);
+    SQL
+    private_constant :SCHEMA_SQL
+
+    class << self
+      # @param version [String]
+      # @return [Pathname] path to the built SQLite file
+      def build(version)
+        Ucode::VersionResolver.validate!(version)
+
+        ucd_dir = Cache.ucd_dir(version)
+        unihan_dir = Cache.unihan_dir(version)
+        db_path = Cache.sqlite_path(version)
+
+        Cache.ensure_version_dir!(version)
+
+        builder = IndexBuilder.new
+        Coordinator.new.each_codepoint(ucd_dir: ucd_dir, unihan_dir: unihan_dir) do |cp|
+          builder.add(cp)
+        end
+
+        write_db(db_path, version, builder.blocks_index, builder.scripts_index)
+        db_path
+      end
+
+      private
+
+      def write_db(db_path, version, blocks_index, scripts_index)
+        SQLite3::Database.new(db_path.to_s) do |db|
+          db.execute_batch(SCHEMA_SQL)
+          insert_meta(db, "schema_version", Database::SCHEMA_VERSION)
+          insert_meta(db, "ucd_version", version)
+          insert_meta(db, "built_at", Time.now.utc.iso8601)
+
+          db.transaction do
+            insert_rows(db, "blocks", blocks_index.entries)
+            insert_rows(db, "scripts", scripts_index.entries)
+          end
+        end
+      end
+
+      def insert_meta(db, key, value)
+        db.execute(
+          "INSERT INTO schema_meta (key, value) VALUES (?, ?)",
+          [key.to_s, value.to_s],
+        )
+      end
+
+      def insert_rows(db, table, entries)
+        stmt = db.prepare("INSERT INTO #{table} (first_cp, last_cp, name) VALUES (?, ?, ?)")
+        entries.each do |entry|
+          stmt.execute(entry.first_cp, entry.last_cp, entry.name)
+        end
+      ensure
+        stmt&.close
+      end
+    end
+  end
+end

data/lib/ucode/error.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Ucode
+  # Base error class for all ucode failures.
+  #
+  # Every error raised anywhere in the codebase is_a?(Ucode::Error). Errors
+  # carry structured context (file:, line:, codepoint:, version:, etc.) so
+  # CLI formatters can render useful diagnostics without re-parsing strings.
+  #
+  # Hierarchy:
+  #
+  #   Ucode::Error
+  #   ├── Ucode::FetchError
+  #   │   ├── Ucode::NetworkError
+  #   │   └── Ucode::ChecksumError
+  #   ├── Ucode::ParseError
+  #   │   ├── Ucode::MalformedLineError
+  #   │   └── Ucode::UnknownPropertyError
+  #   ├── Ucode::LookupError
+  #   │   ├── Ucode::DatabaseMissingError
+  #   │   ├── Ucode::DatabaseSchemaError
+  #   │   └── Ucode::UnknownVersionError
+  #   └── Ucode::GlyphError
+  #       ├── Ucode::PdfRenderError
+  #       ├── Ucode::GridDetectionError
+  #       ├── Ucode::LastResortMissingError
+  #       └── Ucode::EmbeddedFontsMissingError
+  class Error < StandardError
+    attr_reader :context
+
+    # @param message [String, nil]
+    # @param context [Hash{Symbol=>Object}] structured diagnostic context
+    def initialize(message = nil, context: {})
+      @context = context
+      super(build_message(message))
+    end
+
+    private
+
+    def build_message(message)
+      return self.class.to_s if message.nil? && context.empty?
+
+      parts = []
+      parts << message if message
+      parts << context.map { |k, v| "#{k}=#{v.inspect}" }.join(" ") unless context.empty?
+      parts.join(" | ")
+    end
+  end
+
+  # Fetch-time failures.
+  class FetchError < Error; end
+
+  # Network failures during fetch.
+  class NetworkError < FetchError; end
+
+  # Checksum or integrity failure.
+  class ChecksumError < FetchError; end
+
+  # Parse-time failures.
+  class ParseError < Error; end
+
+  # A UCD text file line that does not match the expected column layout.
+  class MalformedLineError < ParseError; end
+
+  # A property short code we don't have in PropertyAliases/PropertyValueAliases.
+  class UnknownPropertyError < ParseError; end
+
+  # Lookup-time failures.
+  class LookupError < Error; end
+
+  # Cache missing for a requested version.
+  class DatabaseMissingError < LookupError; end
+
+  # On-disk schema version mismatch.
+  class DatabaseSchemaError < LookupError; end
+
+  # Version string not in Config.known_versions.
+  class UnknownVersionError < LookupError; end
+
+  # Glyph pipeline failures.
+  class GlyphError < Error; end
+
+  # PDF → SVG rendering failure.
+  class PdfRenderError < GlyphError; end
+
+  # Grid detection couldn't anchor on codepoint labels.
+  class GridDetectionError < GlyphError; end
+
+  # The Last Resort Font UFO source cannot be located or is missing a
+  # required artifact (cmap-f13.ttx, font.ufo/glyphs/, contents.plist).
+  class LastResortMissingError < GlyphError; end
+
+  # The Code Charts PDF (per-block or monolith) cannot be located, or
+  # `mutool` is not installed on the PATH.
+  class EmbeddedFontsMissingError < GlyphError; end
+end

data/lib/ucode/fetch/code_charts.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Ucode
+  module Fetch
+    # Downloads per-block Code Charts PDFs from unicode.org/charts/PDF/.
+    #
+    # URL pattern: `https://www.unicode.org/charts/PDF/U<XXXX>.pdf`
+    # where `XXXX` is the block's first codepoint zero-padded to 4 digits
+    # (5–6 digits for planes > 0).
+    module CodeCharts
+      class << self
+        # @param version [String] used as the on-disk path namespace; PDFs
+        #   are not versioned on unicode.org so the argument is mostly a
+        #   convention.
+        # @param block_first_cps [Array<Integer>] first codepoint of each
+        #   block to download. If nil, caller is expected to derive the
+        #   list from `Parsers::Blocks` (the PDF URL is `U<hex>.pdf`).
+        # @param force [Boolean] re-download even if cached.
+        # @return [Integer] number of PDFs downloaded.
+        def call(version, block_first_cps:, force: false)
+          Cache.ensure_version_dir!(version)
+          pdfs_dir = Cache.pdfs_dir(version)
+          pdfs_dir.mkpath
+
+          downloaded = 0
+          block_first_cps.each do |first_cp|
+            filename = "U#{hex_pad(first_cp)}.pdf"
+            dest = pdfs_dir.join(filename)
+            next if dest.exist? && !force
+
+            url = "#{Ucode.configuration.charts_base_url}/#{filename}"
+            Http.get(url, dest: dest)
+            downloaded += 1
+          end
+          downloaded
+        end
+
+        # Build the block→first-cp list from a parsed Blocks index. The
+        # caller passes the output of `Ucode::Parsers::Blocks.each_record`
+        # collapsed into `block_id => first_cp`.
+        #
+        # @param blocks [Array<Ucode::Models::Block>] sorted by first_cp
+        # @return [Array<Integer>] first-cp values
+        def first_cps_from(blocks)
+          blocks.map(&:range_first)
+        end
+
+        private
+
+        def hex_pad(codepoint)
+          width = codepoint > 0xFFFF ? 6 : 4
+          codepoint.to_s(16).upcase.rjust(width, "0")
+        end
+      end
+    end
+  end
+end

data/lib/ucode/fetch/http.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+require "fileutils"
+require "pathname"
+
+module Ucode
+  module Fetch
+    # Shared HTTP wrapper. Single network boundary for the whole project.
+    #
+    # Streaming download with retries and exponential backoff. Raises
+    # Ucode::NetworkError on final failure (after `http_retries` attempts).
+    module Http
+      DEFAULT_BACKOFF = [1, 2, 4, 8, 16].freeze
+      private_constant :DEFAULT_BACKOFF
+
+      class << self
+        # Stream `url` to `dest` (a Pathname or String path).
+        #
+        # @param url [String, URI] full URL.
+        # @param dest [Pathname, String] destination file path. Parent
+        #   directory is created if absent.
+        # @param retries [Integer, nil] override Config.http_retries.
+        # @param timeout [Integer, nil] override Config.http_timeout.
+        # @return [Pathname] destination path on success.
+        # @raise [Ucode::NetworkError] if all retries fail.
+        def get(url, dest:, retries: nil, timeout: nil)
+          uri = url.is_a?(URI) ? url : URI(url)
+          destination = Pathname.new(dest)
+          destination.dirname.mkpath
+
+          attempts = retries || Ucode.configuration.http_retries
+          read_timeout = timeout || Ucode.configuration.http_timeout
+          backoff_sequence = DEFAULT_BACKOFF.take(attempts + 1)
+
+          last_error = nil
+          (attempts + 1).times do |attempt|
+            return stream_to(uri, destination, read_timeout)
+          rescue StandardError => e
+            last_error = e
+            sleep_for = backoff_sequence[attempt] || backoff_sequence.last
+            Ucode.configuration.logger&.warn do
+              "Http GET #{uri} failed (attempt #{attempt + 1}/#{attempts + 1}): " \
+                "#{e.class}: #{e.message}; retrying in #{sleep_for}s"
+            end
+            sleep(sleep_for)
+          end
+
+          raise Ucode::NetworkError.new(
+            "GET #{uri} failed after #{attempts + 1} attempts",
+            context: { url: uri.to_s, last_error: last_error&.message },
+          )
+        end
+
+        private
+
+        def stream_to(uri, destination, read_timeout)
+          Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
+                                              read_timeout: read_timeout) do |http|
+            request = Net::HTTP::Get.new(uri)
+            http.request(request) do |response|
+              unless response.is_a?(Net::HTTPSuccess)
+                raise "HTTP #{response.code} #{response.message}"
+              end
+
+              write_body(response, destination)
+            end
+          end
+          destination
+        end
+
+        def write_body(response, destination)
+          partial = destination.sub_ext("#{destination.extname}.part")
+          File.open(partial, "wb") do |file|
+            response.read_body { |chunk| file.write(chunk) }
+          end
+          File.rename(partial.to_s, destination.to_s)
+        end
+      end
+    end
+  end
+end

data/lib/ucode/fetch/ucd_zip.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "zip"
+
+module Ucode
+  module Fetch
+    # Downloads UCD.zip from unicode.org and unpacks it into
+    # `Cache.ucd_dir(version)`.
+    module UcdZip
+      URL_SUFFIX = "/ucd/UCD.zip"
+      private_constant :URL_SUFFIX
+
+      class << self
+        # @param version [String] e.g. "17.0.0"
+        # @param force [Boolean] re-download even if cached.
+        # @return [Pathname] the ucd_dir after extraction.
+        def call(version, force: false)
+          Cache.ensure_version_dir!(version)
+          target_dir = Cache.ucd_dir(version)
+
+          marker = target_dir.join("UnicodeData.txt")
+          return target_dir if marker.exist? && !force
+
+          url = "#{Ucode.configuration.ucd_base_url}/#{version}#{URL_SUFFIX}"
+          zip_path = Cache.version_dir(version).join("ucd.zip")
+          Http.get(url, dest: zip_path)
+          extract(zip_path, target_dir)
+          zip_path.delete if zip_path.exist?
+          target_dir
+        end
+
+        private
+
+        def extract(zip_path, target_dir)
+          target_dir.mkpath
+          Zip::File.open(zip_path.to_s) do |zip|
+            zip.each do |entry|
+              next if entry.directory?
+              next if entry.name.start_with?("__MACOSX/") || entry.name.include?("/._")
+
+              relative = entry.name.sub(%r{^/+}, "")
+              dest = target_dir.join(relative)
+              dest.dirname.mkpath
+              next if dest.exist?
+
+              entry.get_input_stream do |input|
+                File.open(dest, "wb") do |output|
+                  IO.copy_stream(input, output)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ucode/fetch/unihan_zip.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "zip"
+
+module Ucode
+  module Fetch
+    # Downloads Unihan.zip from unicode.org and unpacks it into
+    # `Cache.unihan_dir(version)`.
+    module UnihanZip
+      URL_SUFFIX = "/ucd/Unihan.zip"
+      private_constant :URL_SUFFIX
+
+      class << self
+        # @param version [String]
+        # @param force [Boolean]
+        # @return [Pathname]
+        def call(version, force: false)
+          Cache.ensure_version_dir!(version)
+          target_dir = Cache.unihan_dir(version)
+
+          marker = target_dir.join("Unihan_Readings.txt")
+          return target_dir if marker.exist? && !force
+
+          url = "#{Ucode.configuration.unihan_base_url}/#{version}#{URL_SUFFIX}"
+          zip_path = Cache.version_dir(version).join("unihan.zip")
+          Http.get(url, dest: zip_path)
+          extract(zip_path, target_dir)
+          zip_path.delete if zip_path.exist?
+          target_dir
+        end
+
+        private
+
+        def extract(zip_path, target_dir)
+          target_dir.mkpath
+          Zip::File.open(zip_path.to_s) do |zip|
+            zip.each do |entry|
+              next if entry.directory?
+              next if entry.name.start_with?("__MACOSX/") || entry.name.include?("/._")
+
+              relative = entry.name.sub(%r{^/+}, "")
+              dest = target_dir.join(relative)
+              dest.dirname.mkpath
+              next if dest.exist?
+
+              entry.get_input_stream do |input|
+                File.open(dest, "wb") do |output|
+                  IO.copy_stream(input, output)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ucode/fetch.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Ucode
+  # Fetchers — download UCD.zip, Unihan.zip, and per-block Code Charts PDFs.
+  #
+  # OCP: Http is the single network boundary. New source types add a new
+  # Fetcher class that calls Http.get; no new HTTP stack.
+  module Fetch
+    autoload :Http, "ucode/fetch/http"
+    autoload :UcdZip, "ucode/fetch/ucd_zip"
+    autoload :UnihanZip, "ucode/fetch/unihan_zip"
+    autoload :CodeCharts, "ucode/fetch/code_charts"
+  end
+end

data/lib/ucode/glyphs/cell_extractor.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+
+require "ucode/glyphs/path_bbox"
+
+module Ucode
+  module Glyphs
+    # Extracts a single character cell from a Code Charts SVG page and
+    # returns a normalized standalone SVG containing only that cell's
+    # vector paths.
+    #
+    # The cell is identified by codepoint. The extractor asks the Grid
+    # for the cell's anchor position, finds the `<use>` element placed
+    # at that position, resolves its glyph definition from `<defs>`,
+    # and emits a fresh `<svg>` whose viewBox is `0 0 1000 1000` and
+    # whose body is the glyph's `<path>` data translated and scaled to
+    # fit that viewBox with a small margin.
+    #
+    # Vector-only. Never rasterizes, never OCRs. If the cell is empty
+    # (no character glyph placed there, e.g. unassigned codepoint or
+    # control character), the extractor returns nil.
+    class CellExtractor
+      ViewBoxSize = 1000.0
+      MarginRatio = 0.1
+      private_constant :ViewBoxSize, :MarginRatio
+
+      # @param doc [Nokogiri::XML::Document] the rendered Code Charts page
+      def initialize(doc)
+        @doc = doc
+        @glyph_cache = {}
+      end
+
+      # @param grid [Ucode::Glyphs::Grid]
+      # @param codepoint [Integer]
+      # @return [Nokogiri::XML::Document, nil] a standalone `<svg>` doc
+      #   with viewBox `0 0 1000 1000`, or nil if the cell is empty.
+      def extract(grid, codepoint)
+        anchor = grid.cell_position(codepoint)
+        return nil unless anchor
+
+        use_node = find_use_at(anchor, grid)
+        return nil unless use_node
+
+        path_data = collect_paths(use_node["xlink:href"] || use_node["href"])
+        return nil if path_data.empty?
+
+        bbox = PathBbox.estimate(path_data.join(" "))
+        return nil if bbox.empty?
+
+        build_svg(path_data, bbox, use_node["x"].to_f, use_node["y"].to_f)
+      end
+
+      private
+
+      def find_use_at(anchor, grid)
+        tolerance_x = grid.column_pitch / 2
+        tolerance_y = grid.row_pitch / 2
+
+        candidates = @doc.css("use").select do |node|
+          href = node["xlink:href"] || node["href"] || ""
+          href.start_with?("#glyph-") &&
+            (node["x"].to_f - anchor[0]).abs <= tolerance_x &&
+            (node["y"].to_f - anchor[1]).abs <= tolerance_y
+        end
+
+        candidates.min_by do |node|
+          dx = node["x"].to_f - anchor[0]
+          dy = node["y"].to_f - anchor[1]
+          (dx * dx) + (dy * dy)
+        end
+      end
+
+      def collect_paths(href)
+        return [] unless href
+
+        glyph_id = href.sub(/\A#/, "")
+        node = glyph_definition(glyph_id)
+        return [] unless node
+
+        node.css("path").map { |p| p["d"] }.compact
+      end
+
+      def glyph_definition(glyph_id)
+        return @glyph_cache[glyph_id] if @glyph_cache.key?(glyph_id)
+
+        @glyph_cache[glyph_id] = @doc.at_css("defs ##{glyph_id}")
+      end
+
+      def build_svg(path_data, glyph_bbox, place_x, place_y)
+        placed = PathBbox::Result.new(
+          min_x: place_x + glyph_bbox.min_x,
+          min_y: place_y + glyph_bbox.min_y,
+          max_x: place_x + glyph_bbox.max_x,
+          max_y: place_y + glyph_bbox.max_y,
+        )
+
+        width = placed.width
+        height = placed.height
+        return nil if width <= 0 || height <= 0
+
+        content_size = ViewBoxSize * (1.0 - (2.0 * MarginRatio))
+        scale = [content_size / width, content_size / height].min
+        offset_x = (ViewBoxSize - (width * scale)) / 2.0
+        offset_y = (ViewBoxSize - (height * scale)) / 2.0
+        translate_x = offset_x - (placed.min_x * scale)
+        translate_y = offset_y - (placed.min_y * scale)
+
+        builder = Nokogiri::XML::Document.new
+        root = builder.create_element(
+          "svg",
+          xmlns: "http://www.w3.org/2000/svg",
+          viewBox: "0 0 #{ViewBoxSize.to_i} #{ViewBoxSize.to_i}",
+          width: ViewBoxSize.to_i,
+          height: ViewBoxSize.to_i,
+        )
+        group = builder.create_element(
+          "g",
+          transform: "scale(#{format('%.6f', scale)}) translate(#{format('%.6f', translate_x)}, #{format('%.6f', translate_y)})",
+        )
+        path_data.each do |d|
+          group.add_child(builder.create_element("path", d: d, fill: "black"))
+        end
+        root.add_child(group)
+        builder.add_child(root)
+        builder
+      end
+    end
+  end
+end

data/lib/ucode/glyphs/dvisvgm_renderer.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "ucode/glyphs/page_renderer"
+
+module Ucode
+  module Glyphs
+    # `dvisvgm` — originally a DVI-to-SVG converter, also handles PDF.
+    # The `--no-fonts` flag forces outline-only output (no font subsetting
+    # artifacts), which is what we want for vector glyph extraction.
+    #
+    # Command: `dvisvgm --pdf --no-fonts --page=<n> <in.pdf> -o <out.svg>`
+    class DvisvgmRenderer < PageRenderer
+      class << self
+        def renderer_name
+          :dvisvgm
+        end
+
+        def binary_name
+          :dvisvgm
+        end
+
+        def build_command(pdf_path, page_num, out_path)
+          ["dvisvgm", "--pdf", "--no-fonts", "--page=#{page_num}",
+           pdf_path.to_s, "-o", out_path.to_s]
+        end
+      end
+    end
+  end
+end