ucode 0.1.0

data/lib/ucode/cli.rb

@@ -0,0 +1,272 @@
+# frozen_string_literal: true
+
+require "thor"
+
+require "ucode/commands"
+
+module Ucode
+  # Top-level CLI entry.
+  #
+  # **Thin Thor**: every method delegates to a `Commands::*Command`
+  # class and only formats the result. The Command classes are pure
+  # and testable in-process — Thor never holds business logic.
+  class Cli < Thor
+    package_name "ucode"
+
+    def self.exit_on_failure?
+      true
+    end
+
+    # ─────────────── version ───────────────
+    desc "version", "Print ucode version"
+    def version
+      puts "ucode #{Ucode::VERSION}"
+    end
+
+    # ─────────────── fetch ───────────────
+    class Fetch < Thor
+      desc "ucd [VERSION]", "Download UCD.zip"
+      option :force, type: :boolean, default: false, desc: "Re-download even if cached"
+      def ucd(version = nil)
+        puts format_result Commands::FetchCommand.new.fetch_ucd(version, force: options[:force])
+      end
+
+      desc "unihan [VERSION]", "Download Unihan.zip"
+      option :force, type: :boolean, default: false
+      def unihan(version = nil)
+        puts format_result Commands::FetchCommand.new.fetch_unihan(version, force: options[:force])
+      end
+
+      desc "charts [VERSION]", "Download per-block Code Charts PDFs"
+      option :force, type: :boolean, default: false
+      option :block, type: :array, desc: "Limit to these block ids"
+      def charts(version = nil)
+        cps = options[:block]&.map { |id| block_id_to_first_cp(id) }&.compact
+        puts format_result Commands::FetchCommand.new
+          .fetch_charts(version, block_first_cps: cps, force: options[:force])
+      end
+
+      private
+
+      def block_id_to_first_cp(id)
+        return Integer(id) if id.match?(/\A\d+\z/)
+
+        warn "Warning: --block=#{id.inspect} cannot be resolved to a first codepoint; skipping"
+        nil
+      end
+
+      def format_result(result)
+        JSON.pretty_generate(result)
+      end
+    end
+
+    desc "fetch", "Download UCD sources"
+    subcommand "fetch", Fetch
+
+    # ─────────────── parse ───────────────
+    desc "parse [VERSION]", "Stream UCD → output/"
+    option :to, type: :string, default: "./output", desc: "Output directory"
+    def parse(version = nil)
+      result = Commands::ParseCommand.new.call(version, output_root: options[:to])
+      puts JSON.pretty_generate(result)
+    end
+
+    # ─────────────── glyphs ───────────────
+    desc "glyphs [VERSION]", "Extract per-codepoint SVGs from Code Charts PDFs (experimental)"
+    long_desc <<~LONG
+      EXPERIMENTAL in v0.1. The cell extractor currently includes cell-border
+      decorations alongside the actual character outline, so the output is not
+      yet suitable for end-user display. Opt in with --include-glyphs to run
+      the pipeline anyway; otherwise it returns a skipped payload.
+    LONG
+    option :to, type: :string, default: "./output"
+    option :block, type: :array, desc: "Limit to these block ids"
+    option :force, type: :boolean, default: false
+    option :monolith, type: :string, default: "CodeCharts.pdf",
+                      desc: "Path to CodeCharts.pdf for fallback slicing"
+    option :include_glyphs, type: :boolean, default: false,
+                            desc: "Opt into the experimental v0.1 pipeline"
+    def glyphs(version = nil)
+      result = Commands::GlyphsCommand.new.call(
+        version,
+        output_root: options[:to],
+        block_filter: options[:block],
+        force: options[:force],
+        monolith_path: options[:monolith],
+        include_glyphs: options[:include_glyphs],
+        warn: $stderr,
+      )
+      puts JSON.pretty_generate(result)
+    end
+
+    # ─────────────── site ───────────────
+    class Site < Thor
+      desc "init", "Copy the Vitepress scaffold into site/"
+      option :to, type: :string, default: "./site"
+      def init
+        puts JSON.pretty_generate(Commands::SiteCommand.new.init(site_root: options[:to]))
+      end
+
+      desc "build", "Regenerate site/.vitepress/config.ts + pages from output/"
+      option :from, type: :string, default: "./output", desc: "Dataset root"
+      option :to, type: :string, default: "./site", desc: "Site root"
+      def build
+        puts JSON.pretty_generate(
+          Commands::SiteCommand.new.build(output_root: options[:from], site_root: options[:to]),
+        )
+      end
+    end
+
+    desc "site", "Generate the Vitepress site"
+    subcommand "site", Site
+
+    # ─────────────── lookup ───────────────
+    class Lookup < Thor
+      desc "block CODEPOINT", "Block name covering CODEPOINT (integer or 0xNNNN)"
+      option :version, type: :string, default: nil
+      def block(codepoint)
+        cp = parse_cp(codepoint)
+        with_db_handling do
+          result = Commands::LookupCommand.new.lookup_block(options[:version], codepoint: cp)
+          puts "#{format("U+%04X", cp)} → #{result.block || "(unassigned)"}"
+        end
+      end
+
+      desc "script CODEPOINT", "Script name covering CODEPOINT"
+      option :version, type: :string, default: nil
+      def script(codepoint)
+        cp = parse_cp(codepoint)
+        with_db_handling do
+          result = Commands::LookupCommand.new.lookup_script(options[:version], codepoint: cp)
+          puts "#{format("U+%04X", cp)} → #{result.script || "(none)"}"
+        end
+      end
+
+      desc "char CODEPOINT", "Block + glyph path for CODEPOINT"
+      option :version, type: :string, default: nil
+      option :from, type: :string, default: "./output"
+      def char(codepoint)
+        cp = parse_cp(codepoint)
+        with_db_handling do
+          result = Commands::LookupCommand.new
+            .lookup_char(options[:version], codepoint: cp, output_root: options[:from])
+          puts "#{format("U+%04X", cp)} block=#{result.block_id} glyph=#{result.glyph_path}"
+        end
+      end
+
+      private
+
+      def parse_cp(s)
+        Integer(s.gsub(/^U\+/i, ""), 16)
+      rescue ArgumentError
+        raise Thor::Error, "Invalid codepoint: #{s.inspect} (try '0x0041' or 'U+0041')"
+      end
+
+      def with_db_handling
+        yield
+      rescue Ucode::DatabaseMissingError => e
+        version = e.context[:version]
+        raise Thor::Error, "No SQLite cache for version #{version.inspect}. " \
+                           "Run: ucode build #{version} --to ./output"
+      end
+    end
+
+    desc "lookup", "Read-only lookups against the SQLite cache"
+    subcommand "lookup", Lookup
+
+    # ─────────────── cache ───────────────
+    class Cache < Thor
+      desc "list", "List cached UCD versions"
+      def list
+        Commands::CacheCommand.new.list.each { |v| puts v }
+      end
+
+      desc "info VERSION", "Show what's cached for VERSION"
+      def info(version)
+        result = Commands::CacheCommand.new.info(version)
+        if result.nil?
+          puts "Nothing cached for #{version}"
+        else
+          puts JSON.pretty_generate(result.to_h)
+        end
+      end
+
+      desc "remove VERSION", "Remove VERSION from the cache"
+      def remove(version)
+        ok = Commands::CacheCommand.new.remove(version)
+        puts(ok ? "Removed #{version}" : "#{version} not in cache")
+      end
+    end
+
+    desc "cache", "Inspect and manage the cache"
+    subcommand "cache", Cache
+
+    # ─────────────── build ───────────────
+    desc "build [VERSION]", "Full pipeline: fetch + parse + (optional) glyphs + site"
+    option :to, type: :string, default: "./output"
+    option :site, type: :string, default: nil, desc: "Build the site here (skipped if nil)"
+    option :monolith, type: :string, default: "CodeCharts.pdf"
+    option :force_fetch, type: :boolean, default: false
+    option :include_glyphs, type: :boolean, default: false,
+                            desc: "Opt into the experimental v0.1 glyph step"
+    def build(version = nil)
+      result = Commands::BuildCommand.new.call(
+        version,
+        output_root: options[:to],
+        site_root: options[:site],
+        monolith_path: options[:monolith],
+        force_fetch: options[:force_fetch],
+        include_glyphs: options[:include_glyphs],
+        warn: $stderr,
+      )
+      puts JSON.pretty_generate(result)
+    end
+
+    # ─────────────── font-coverage ───────────────
+    desc "font-coverage FONT [FONT...]", "Audit Unicode 17 block coverage for one or more fonts"
+    long_desc <<~LONG
+      Each FONT argument is either a fontist formula name (resolved via
+      `Fontist::Font.find` then `install`) or `label=/path/to/font.ttf`
+      (uses the local file directly). For every font, walks the cmap via
+      fontisan and emits per-Unicode-17-block coverage to
+      `<to>/font_coverage/<label>.json`.
+
+      Examples:
+
+        ucode font-coverage Lentariso=/tmp/lentariso/TTFs/Lentariso-Re.ttf \\
+                             Kedebideri=/tmp/kedebideri/Kedebideri-3.001/Kedebideri-Regular.ttf
+
+        ucode font-coverage Kedebideri  # resolves + installs via fontist
+    LONG
+    option :to, type: :string, default: "./output"
+    option :no_install, type: :boolean, default: false,
+                        desc: "Don't auto-install missing fonts via fontist"
+    def font_coverage(*fonts)
+      raise Thor::Error, "Provide at least one font" if fonts.empty?
+
+      results = Commands::FontCoverageCommand.new.call(
+        fonts,
+        output_root: options[:to],
+        install: !options[:no_install],
+      )
+      puts JSON.pretty_generate(results.map { |r| result_to_h(r) })
+    end
+
+    private
+
+    def result_to_h(result)
+      if result.error
+        { spec: result.spec, error: result.error }
+      else
+        {
+          spec: result.spec,
+          label: result.located.name,
+          source: result.located.path.to_s,
+          via: result.located.via,
+          output_path: result.output_path.to_s,
+          complete_blocks: result.complete_blocks,
+        }
+      end
+    end
+  end
+end

data/lib/ucode/commands/build.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+require "ucode/commands"
+require "ucode/version_resolver"
+
+module Ucode
+  module Commands
+    # `ucode build` — full pipeline: fetch (ucd + unihan + charts) →
+    # parse → (optional) glyphs → (optional) site. Resumable: each step
+    # is idempotent and safe to re-run.
+    #
+    # **Glyph step is opt-in as of v0.1** because the SVG cell extractor
+    # is still experimental. Pass `include_glyphs: true` to enable it;
+    # otherwise the glyphs step is recorded as skipped.
+    class BuildCommand
+      # @param version_intent [nil, :default, :latest, String]
+      # @param output_root [String, Pathname]
+      # @param site_root [String, Pathname, nil] if nil, skip site build
+      # @param monolith_path [String, Pathname, nil] CodeCharts.pdf fallback
+      # @param force_fetch [Boolean] re-download sources
+      # @param include_glyphs [Boolean] opt into the experimental glyph
+      #   step (default false)
+      # @param warn [IO, nil] forwarded to GlyphsCommand when enabled
+      # @return [Hash] aggregated step results
+      def call(version_intent, output_root:, site_root: nil,
+               monolith_path: nil, force_fetch: false,
+               include_glyphs: false, warn: nil)
+        version = VersionResolver.resolve(version_intent)
+        steps = {}
+
+        steps[:fetch] = run_fetch(version, force: force_fetch)
+        steps[:parse] = ParseCommand.new.call(version, output_root: output_root)
+        steps[:glyphs] = run_glyphs(version, output_root, monolith_path,
+                                     include_glyphs: include_glyphs, warn: warn)
+        steps[:site] = run_site(output_root, site_root) if site_root
+
+        { version: version, steps: steps }
+      end
+
+      private
+
+      def run_fetch(version, force:)
+        fetch = FetchCommand.new
+        {
+          ucd: fetch.fetch_ucd(version, force: force),
+          unihan: fetch.fetch_unihan(version, force: force),
+          charts: fetch.fetch_charts(version, force: force),
+        }
+      end
+
+      def run_glyphs(version, output_root, monolith_path, include_glyphs:, warn:)
+        GlyphsCommand.new.call(
+          version,
+          output_root: output_root,
+          monolith_path: monolith_path || "CodeCharts.pdf",
+          include_glyphs: include_glyphs,
+          warn: warn,
+        )
+      end
+
+      def run_site(output_root, site_root)
+        SiteCommand.new.build(output_root: output_root, site_root: site_root)
+      end
+    end
+  end
+end

data/lib/ucode/commands/cache.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+require "ucode/cache"
+
+module Ucode
+  module Commands
+    # `ucode cache` — inspect and manage the on-disk cache.
+    # Three subactions: list, info, remove.
+    class CacheCommand
+      VersionInfo = Struct.new(:version, :path, :has_ucd, :has_unihan,
+                               :has_pdfs, :has_sqlite, keyword_init: true)
+      private_constant :VersionInfo
+
+      # @return [Array<String>] sorted versions present in the cache
+      def list
+        Cache.cached_versions
+      end
+
+      # @param version [String]
+      # @return [VersionInfo, nil] nil if version not in cache
+      def info(version)
+        return nil unless Cache.cached?(version)
+
+        VersionInfo.new(
+          version: version,
+          path: Cache.version_dir(version),
+          has_ucd: Cache.ucd_dir(version).join("UnicodeData.txt").exist?,
+          has_unihan: Cache.unihan_dir(version).children.any?,
+          has_pdfs: Cache.pdfs_dir(version).children.any?,
+          has_sqlite: Cache.sqlite_path(version).exist?,
+        )
+      end
+
+      # @param version [String]
+      # @return [Boolean] true if a directory was removed
+      def remove(version)
+        return false unless Cache.cached?(version)
+
+        Cache.remove_version(version)
+        true
+      end
+    end
+  end
+end

data/lib/ucode/commands/fetch.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+require "ucode/cache"
+require "ucode/fetch"
+require "ucode/version_resolver"
+
+module Ucode
+  module Commands
+    # `ucode fetch` — downloads UCD/Unihan/Code-Charts sources into the
+    # per-version cache. Three subactions: ucd, unihan, charts.
+    #
+    # Thin shell over `Ucode::Fetch::*`. The command layer's job is to
+    # resolve the version intent and format the result; the fetcher does
+    # the network I/O.
+    class FetchCommand
+      # @param version_intent [nil, :default, :latest, String]
+      # @param force [Boolean]
+      # @return [Hash] { version:, ucd_dir: }
+      def fetch_ucd(version_intent, force: false)
+        version = VersionResolver.resolve(version_intent)
+        Cache.ensure_version_dir!(version)
+        path = Fetch::UcdZip.call(version, force: force)
+        { version: version, ucd_dir: path }
+      end
+
+      # @param version_intent [nil, :default, :latest, String]
+      # @param force [Boolean]
+      # @return [Hash] { version:, unihan_dir: }
+      def fetch_unihan(version_intent, force: false)
+        version = VersionResolver.resolve(version_intent)
+        Cache.ensure_version_dir!(version)
+        path = Fetch::UnihanZip.call(version, force: force)
+        { version: version, unihan_dir: path }
+      end
+
+      # @param version_intent [nil, :default, :latest, String]
+      # @param block_first_cps [Array<Integer>, nil] nil = all known blocks
+      # @param force [Boolean]
+      # @return [Hash] { version:, downloaded: }
+      def fetch_charts(version_intent, block_first_cps: nil, force: false)
+        version = VersionResolver.resolve(version_intent)
+        Cache.ensure_version_dir!(version)
+
+        cps = block_first_cps || default_block_first_cps(version)
+        count = Fetch::CodeCharts.call(version, block_first_cps: cps, force: force)
+        { version: version, downloaded: count }
+      end
+
+      private
+
+      def default_block_first_cps(version)
+        ucd_dir = Cache.ucd_dir(version)
+        blocks_file = ucd_dir.join("Blocks.txt")
+        return [] unless blocks_file.exist?
+
+        Parsers::Blocks.each_record(blocks_file).map(&:range_first)
+      end
+    end
+  end
+end

data/lib/ucode/commands/font_coverage.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "json"
+
+require "ucode/glyphs/real_fonts"
+
+module Ucode
+  module Commands
+    # `ucode font-coverage` — audit named fonts against the Unicode
+    # 17 new-blocks table and emit per-font JSON coverage reports
+    # under `output/font_coverage/`.
+    #
+    # Each font spec on the CLI is either a bare name (resolved via
+    # `Fontist::Font.find` then `install`) or `label=/path/to/font.ttf`
+    # (uses the local file directly). The label is what shows up in
+    # the audit JSON; the path is what gets audited.
+    class FontCoverageCommand
+      Result = Struct.new(:spec, :located, :output_path, :covered_blocks,
+                          :complete_blocks, :error, keyword_init: true)
+      private_constant :Result
+
+      # @param specs [Array<String>] font specs (see file docs).
+      # @param output_root [Pathname, String] parent directory.
+      # @param install [Boolean] allow fontist install on miss.
+      # @return [Array<Result>]
+      def call(specs, output_root:, install: true)
+        locator = Ucode::Glyphs::RealFonts::FontLocator.new
+        auditor = Ucode::Glyphs::RealFonts::CoverageAuditor.new
+        writer = Ucode::Glyphs::RealFonts::Writer.new(output_root)
+
+        specs.map do |spec|
+          audit_one(spec, locator, auditor, writer, install: install)
+        end
+      end
+
+      private
+
+      def audit_one(spec, locator, auditor, writer, install:)
+        located = locator.locate(spec, install: install)
+        report = auditor.audit(located.path)
+        path = writer.write(report)
+        Result.new(spec: spec, located: located, output_path: path,
+                   **summary_kwargs(report))
+      rescue StandardError => e
+        Result.new(spec: spec, error: "#{e.class}: #{e.message}")
+      end
+
+      def summary_kwargs(report)
+        {
+          covered_blocks: report.blocks.count { |b| b.covered.positive? },
+          complete_blocks: report.blocks.count(&:complete?),
+        }
+      end
+    end
+  end
+end

data/lib/ucode/commands/glyphs.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "set"
+
+require "ucode/cache"
+require "ucode/glyphs"
+require "ucode/parsers"
+require "ucode/version_resolver"
+
+module Ucode
+  module Commands
+    # `ucode glyphs` — extract per-codepoint SVGs from Code Charts PDFs.
+    # Builds block specs from the cached Blocks.txt + per-block PDFs (or
+    # monolith fallback), then drains them through the Glyphs::Writer
+    # worker pool.
+    #
+    # **Status (v0.1): EXPERIMENTAL.** The cell-extraction pipeline
+    # currently includes cell-border decorations alongside the actual
+    # character outline because the Code Charts PDFs composite the two
+    # into a single glyph definition. The output is therefore not yet
+    # suitable for end-user display. The command is retained so the
+    # pipeline can be iterated on without churning the CLI surface, but
+    # callers MUST opt in via `include_glyphs: true` (CLI: `--include-glyphs`)
+    # and will receive a printed warning. Tracked for v0.2.
+    class GlyphsCommand
+      ExperimentalWarning = "ucode glyphs is experimental in v0.1: " \
+                            "extracted SVGs include cell-border decorations " \
+                            "alongside the character outline."
+      private_constant :ExperimentalWarning
+
+      MonolithPath = "CodeCharts.pdf"
+      PageMapCache = "data/codecharts_page_map.json"
+      private_constant :MonolithPath, :PageMapCache
+
+      class << self
+        # @return [String] the experimental-status banner. Exposed so the
+        #   CLI and BuildCommand surface the same message verbatim.
+        def experimental_warning
+          ExperimentalWarning
+        end
+      end
+
+      # @param version_intent [nil, :default, :latest, String]
+      # @param output_root [String, Pathname]
+      # @param block_filter [Array<String>, nil] block ids to limit to;
+      #   nil = every block
+      # @param force [Boolean] re-fetch PDFs even when cached
+      # @param monolith_path [String, Pathname, nil] path to CodeCharts.pdf
+      #   for fallback slicing; defaults to ./CodeCharts.pdf
+      # @param include_glyphs [Boolean] opt-in for the experimental v0.1
+      #   pipeline. When false (default), the command returns a `skipped`
+      #   payload without touching disk.
+      # @param warn [IO, nil] when provided, the experimental warning is
+      #   written here exactly once before work begins.
+      # @return [Hash] aggregated Writer tally + version, or a `skipped`
+      #   payload when opt-in is false.
+      def call(version_intent, output_root:,
+               block_filter: nil, force: false, monolith_path: MonolithPath,
+               include_glyphs: false, warn: nil)
+        return skipped(version_intent) unless include_glyphs
+
+        warn&.puts(ExperimentalWarning)
+        version = VersionResolver.resolve(version_intent)
+        root = Pathname.new(output_root)
+
+        blocks = load_blocks(version, block_filter)
+        fetcher = build_fetcher(version, monolith_path, blocks)
+        specs = blocks.map { |block| spec_for(block, fetcher, force) }.compact
+
+        writer = Glyphs::Writer.new(output_root: root,
+                                     parallel_workers: workers)
+        tally = writer.write_all(specs)
+        tally.merge(version: version, block_count: specs.size)
+      end
+
+      private
+
+      def load_blocks(version, block_filter)
+        ucd_dir = Cache.ucd_dir(version)
+        path = ucd_dir.join("Blocks.txt")
+        return [] unless path.exist?
+
+        all = Parsers::Blocks.each_record(path).to_a
+        return all unless block_filter && !block_filter.empty?
+
+        filter_set = block_filter.to_set
+        all.select { |block| filter_set.include?(block.id) }
+      end
+
+      def build_fetcher(version, monolith_path, blocks)
+        monolith = Pathname.new(monolith_path)
+        monolith = monolith.exist? ? monolith : nil
+        Glyphs::PdfFetcher.new(
+          version,
+          monolith_path: monolith,
+          blocks: blocks,
+          page_map_cache: PageMapCache,
+        )
+      end
+
+      def spec_for(block, fetcher, force)
+        pdf_path = fetcher.fetch(block_first_cp: block.range_first, force: force)
+        return nil unless pdf_path
+
+        { block: block, pdf_path: pdf_path, page_map: page_map_for(block) }
+      end
+
+      # Heuristic page map: per-block PDFs are page 1 = title, page 2 =
+      # first chart page starting at the block's first codepoint. True for
+      # most BMP blocks; multi-page blocks (CJK) need a richer resolver.
+      # Mismatches yield placeholder SVGs only — never wrong glyphs.
+      def page_map_for(block)
+        { 2 => block.range_first }
+      end
+
+      def workers
+        Ucode.configuration.parallel_workers
+      end
+
+      def skipped(version_intent)
+        version = begin
+          VersionResolver.resolve(version_intent)
+        rescue UnknownVersionError
+          version_intent
+        end
+        {
+          version: version,
+          skipped: true,
+          reason: :experimental_v0_1,
+          warning: ExperimentalWarning,
+        }
+      end
+    end
+  end
+end