fontisan 0.2.17 → 0.2.22

data/lib/fontisan/cldr/downloader.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+require "tempfile"
+require "zip"
+
+module Fontisan
+  module Cldr
+    # Fetches CLDR JSON archives from unicode-org/cldr-json GitHub releases
+    # and unpacks them into the cache.
+    #
+    # Single entry point: `Downloader.download(version, force:)`.
+    # Idempotent unless `force: true`. Returns the path to the extracted
+    # `main/` characters directory.
+    module Downloader
+      class << self
+        # Download and unpack CLDR JSON for `version`.
+        #
+        # @param version [String] e.g. "46.0.0"
+        # @param force [Boolean] if false and cache already has the
+        #   extracted files, return without re-fetching.
+        # @return [Pathname] path to the extracted main/ characters dir
+        # @raise [DownloadError] on HTTP failure or zip extraction failure
+        def download(version, force: false)
+          target = CacheManager.characters_main_dir(version)
+          return target if target.exist? && !force
+
+          CacheManager.ensure_version_dir!(version)
+          zip_data = fetch_zip(version)
+          extract_archive(zip_data, CacheManager.json_dir(version))
+          target
+        end
+
+        private
+
+        def fetch_zip(version)
+          uri = URI(Config.archive_url_for(version))
+          response = Net::HTTP.get_response(uri)
+          unless response.is_a?(Net::HTTPSuccess)
+            raise DownloadError,
+                  "GET #{uri} returned HTTP #{response.code}: #{response.message}"
+          end
+
+          body = response.body
+          if body.nil? || body.empty?
+            raise DownloadError, "GET #{uri} returned an empty body"
+          end
+
+          body
+        rescue StandardError => e
+          raise e if e.is_a?(DownloadError)
+
+          raise DownloadError, "Failed to fetch #{uri}: #{e.message}"
+        end
+
+        def extract_archive(zip_data, target_dir)
+          Tempfile.create(["fontisan-cldr", ".zip"]) do |tmp|
+            tmp.binmode
+            tmp.write(zip_data)
+            tmp.flush
+            tmp.rewind
+
+            target_dir.mkpath
+            Zip::File.open(tmp.path) do |zip|
+              zip.each do |entry|
+                next unless entry.file?
+
+                out = target_dir.join(entry.name)
+                out.dirname.mkpath
+                entry.extract(out.to_s) { true } # overwrite
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fontisan/cldr/error.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Fontisan
+  module Cldr
+    # Base error class for all CLDR-related failures.
+    class Error < StandardError; end
+  end
+end

data/lib/fontisan/cldr/index.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Fontisan
+  module Cldr
+    # In-memory per-language codepoint lookup.
+    #
+    # Loads a YAML index of `{language: [codepoint, ...]}`. Each language's
+    # codepoints are stored as a Set<Integer> for O(1) intersection checks.
+    #
+    # Used by {Cldr::Aggregator} to compute per-language coverage %.
+    class Index
+      include Enumerable
+
+      # @param entries [Hash{String=>Set<Integer>, Array<Integer>}]
+      def initialize(entries = {})
+        @entries = entries.transform_values do |cps|
+          cps.is_a?(Set) ? cps : Set.new(cps)
+        end
+      end
+
+      # @return [Hash{String=>Set<Integer>}]
+      attr_reader :entries
+
+      def each(&)
+        @entries.each(&)
+      end
+
+      def size
+        @entries.size
+      end
+
+      def languages
+        @entries.keys.sort
+      end
+
+      # @param language [String]
+      # @return [Set<Integer>, nil]
+      def lookup(language)
+        @entries[language]
+      end
+
+      def include?(language)
+        @entries.key?(language)
+      end
+
+      # Serialize to a YAML file.
+      # @param path [String, Pathname]
+      # @return [void]
+      def save(path)
+        File.open(path, "w") do |file|
+          YAML.dump(@entries.transform_values(&:sort), file)
+        end
+      end
+
+      # Load from a YAML file previously written by #save.
+      # @param path [String, Pathname]
+      # @return [Index]
+      def self.load(path)
+        hash = YAML.load_file(path)
+        new(hash)
+      end
+    end
+  end
+end

data/lib/fontisan/cldr/index_builder.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Fontisan
+  module Cldr
+    # Builds the per-language codepoint index from a cached CLDR JSON
+    # archive and persists it as a YAML file for future Index loads.
+    #
+    # Walks every `main/<lang>/characters.json` file under the cached
+    # archive, extracts `exemplarCharacters` (plus auxiliary and index
+    # sets when present), parses each via {UnicodeSetParser}, and unions
+    # the result into a single codepoint set per language.
+    module IndexBuilder
+      class << self
+        # Build + persist the languages index for a cached version.
+        # @param version [String]
+        # @return [Index]
+        def build(version)
+          entries = collect_from_cache(version)
+          CacheManager.index_dir(version).mkpath
+          index = Index.new(entries)
+          index.save(CacheManager.languages_index_path(version))
+          index
+        end
+
+        # Pure: build an Index from a hash of `language => exemplar_string`.
+        # @param exemplars_by_lang [Hash{String=>String}]
+        # @return [Index]
+        def build_from_exemplars(exemplars_by_lang)
+          entries = exemplars_by_lang.transform_values do |set_str|
+            set_str.nil? ? Set.new : Set.new(UnicodeSetParser.call(set_str))
+          end
+          Index.new(entries)
+        end
+
+        private
+
+        def collect_from_cache(version)
+          main_dir = CacheManager.characters_main_dir(version)
+          return {} unless main_dir.exist?
+
+          main_dir.children.select(&:directory?).each_with_object({}) do |lang_dir, hash|
+            file = lang_dir.join("characters.json")
+            next unless file.exist?
+
+            lang = lang_dir.basename.to_s
+            hash[lang] = parse_language_file(file)
+          end
+        end
+
+        def parse_language_file(file)
+          data = JSON.parse(file.read)
+          lang_key = data.dig("main", "locale") ||
+            data["main"]&.keys&.first
+          return Set.new unless lang_key
+
+          chars_node = data.dig("main", lang_key, "characters") || {}
+          sets = %w[exemplarCharacters auxiliary exemplarCharactersIndex
+                    exemplarCharactersPunctuation].filter_map do |field|
+            chars_node[field]
+          end
+          sets.inject(Set.new) do |acc, set_str|
+            acc | Set.new(UnicodeSetParser.call(set_str))
+          rescue UnicodeSetParser::ParseError
+            acc
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fontisan/cldr/unicode_set_parser.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+module Fontisan
+  module Cldr
+    # Parses ICU UnicodeSet bracket notation as used in CLDR
+    # exemplarCharacters fields.
+    #
+    # Supported syntax (sufficient for exemplar sets):
+    #   - Single chars: `a`, `à`, any BMP or supplementary codepoint
+    #   - Ranges: `a-z`, `A-Z`
+    #   - Escapes: `\uXXXX`, `\UXXXXXXXX`, `\u{XXXX...}`
+    #   - Negation: `[^...]` (inverts against 0..0x10FFFF)
+    #
+    # Unsupported (CLDR exemplars do not use these; raise ParseError):
+    #   - Property syntax `[:script=Latin:]`
+    #   - Set operations `[a-z & [b-c]]`
+    #   - Nested sets `[a[b-c]]`
+    #   - Named sequences `{a b c}`
+    #
+    # Output: sorted, deduplicated Array<Integer> of codepoints.
+    module UnicodeSetParser
+      class ParseError < Cldr::Error; end
+      MAX_CODEPOINT = 0x10FFFF
+      private_constant :MAX_CODEPOINT
+
+      module_function
+
+      # @param set_string [String] bracketed ICU UnicodeSet, e.g. "[a-zà]"
+      # @return [Array<Integer>] sorted, deduplicated codepoints
+      def call(set_string)
+        raise ParseError, "input must be bracketed" unless set_string.start_with?("[") && set_string.end_with?("]")
+
+        body = set_string[1..-2]
+        negate = body.start_with?("^")
+        body = body[1..] if negate
+
+        cps = parse_body(body)
+        cps = invert(cps) if negate
+        cps.sort.uniq
+      end
+
+      # Walk the body char by char, emitting codepoints and ranges.
+      def parse_body(body)
+        cps = []
+        chars = body.chars.to_a
+        i = 0
+        prev_cp = nil
+
+        while i < chars.length
+          ch = chars[i]
+
+          case ch
+          when "\\"
+            cp, advance = parse_escape(chars, i)
+            cps << cp
+            prev_cp = cp
+            i += advance
+          when "-"
+            raise ParseError, "dangling '-' at start" if prev_cp.nil?
+            raise ParseError, "dangling '-' at end" if i + 1 >= chars.length
+
+            next_cp, advance = read_next_codepoint(chars, i + 1)
+            raise ParseError, "range with no upper bound" if next_cp.nil?
+
+            cps.concat(((prev_cp + 1)..next_cp).to_a)
+            prev_cp = next_cp
+            i += 1 + advance
+          when "[", "]"
+            raise ParseError, "nested set syntax is not supported"
+          when "{"
+            raise ParseError, "named sequences ({...}) are not supported"
+          when ":"
+            raise ParseError, "property syntax ([:...:]) is not supported"
+          else
+            cps << ch.ord
+            prev_cp = ch.ord
+            i += 1
+          end
+        end
+
+        cps
+      end
+      private_class_method :parse_body
+
+      # Read the next codepoint starting at index `start`. Handles escapes.
+      # @return [Array(Integer, Integer)] codepoint + chars consumed, or
+      #   [nil, 0] if no codepoint is available.
+      def read_next_codepoint(chars, start)
+        return [nil, 0] if start >= chars.length
+
+        ch = chars[start]
+        if ch == "\\"
+          parse_escape(chars, start)
+        else
+          [ch.ord, 1]
+        end
+      end
+      private_class_method :read_next_codepoint
+
+      # Parse a backslash escape sequence.
+      # Supports \uXXXX, \UXXXXXXXX, \u{XXXX...}, and standard backslash
+      # escapes (\\, \[, \], \-, \^).
+      # @return [Array(Integer, Integer)] codepoint + chars consumed
+      def parse_escape(chars, start)
+        # chars[start] is "\\"
+        return [0, 1] if start + 1 >= chars.length
+
+        marker = chars[start + 1]
+        case marker
+        when "u"
+          brace_form(chars, start) || four_hex(chars, start, "u")
+        when "U"
+          eight_hex(chars, start)
+        when "\\"
+          [0x5C, 2]
+        when "[", "]", "-", "^"
+          [marker.ord, 2]
+        else
+          raise ParseError, "unknown escape sequence \\#{marker}"
+        end
+      end
+      private_class_method :parse_escape
+
+      def brace_form(chars, start)
+        return nil unless chars[start + 2] == "{"
+
+        # \u{XXX...} variable hex
+        end_idx = (start + 3..).find { |j| j >= chars.length || chars[j] == "}" }
+        raise ParseError, "unclosed \\u{ escape" if end_idx.nil? || chars[end_idx] != "}"
+
+        hex = chars[(start + 3)...end_idx].join
+        cp = hex.to_i(16)
+        raise ParseError, "\\u{ escape with no digits" if cp.zero? && hex.empty?
+
+        [cp, (end_idx - start) + 1]
+      end
+      private_class_method :brace_form
+
+      def four_hex(chars, start, marker)
+        # \uXXXX — exactly 4 hex digits
+        hex = chars[(start + 2), 4]&.join
+        raise ParseError, "truncated \\#{marker} escape" if hex.nil? || hex.length < 4
+
+        cp = hex.to_i(16)
+        raise ParseError, "\\#{marker} escape with non-hex digits" if cp.zero? && !hex.match?(/\A0+\z/)
+
+        [cp, 6]
+      end
+      private_class_method :four_hex
+
+      def eight_hex(chars, start)
+        # \UXXXXXXXX — exactly 8 hex digits
+        hex = chars[(start + 2), 8]&.join
+        raise ParseError, "truncated \\U escape" if hex.nil? || hex.length < 8
+
+        cp = hex.to_i(16)
+        raise ParseError, "\\U escape with non-hex digits" if cp.zero? && !hex.match?(/\A0+\z/)
+
+        [cp, 10]
+      end
+      private_class_method :eight_hex
+
+      def invert(cps)
+        set = cps.to_set
+        (0..MAX_CODEPOINT).each_with_object([]) do |cp, arr|
+          arr << cp unless set.include?(cp)
+        end
+      end
+      private_class_method :invert
+    end
+  end
+end

data/lib/fontisan/cldr/unknown_version_error.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Fontisan
+  module Cldr
+    # Raised by Cldr::VersionResolver when a user-supplied version string
+    # is not in Cldr::Config.known_versions.
+    class UnknownVersionError < Cldr::Error; end
+  end
+end

data/lib/fontisan/cldr/version_resolver.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+require "json"
+require "rubygems"
+
+module Fontisan
+  module Cldr
+    # Resolves a user-supplied version intent to a concrete CLDR version.
+    #
+    # Mirrors {Ucd::VersionResolver}. Three input modes:
+    #
+    #   resolve(nil)           # default_version from config
+    #   resolve(:default)      # default_version from config
+    #   resolve("46.0.0")      # explicit; validated against known_versions
+    #   resolve(:latest)       # probes GitHub releases, picks highest;
+    #                          # falls back to default on failure
+    module VersionResolver
+      GITHUB_RELEASE_TAG = %r{ref/tags/(\d+(?:\.\d+)+)}
+      private_constant :GITHUB_RELEASE_TAG
+
+      class << self
+        # @param intent [nil, :default, :latest, String]
+        # @return [String] a concrete version string
+        def resolve(intent)
+          case intent
+          when nil, :default
+            Config.default_version
+          when :latest
+            probe_latest
+          else
+            validate!(intent)
+            intent
+          end
+        end
+
+        # Raise UnknownVersionError unless `version` is in known_versions.
+        # @param version [String]
+        # @return [void]
+        def validate!(version)
+          return if Config.known?(version)
+
+          raise UnknownVersionError,
+                "CLDR version #{version.inspect} is not recognized. " \
+                "Known versions: #{Config.known_versions.join(', ')}"
+        end
+
+        private
+
+        # Best-effort probe of the GitHub releases API for cldr-json.
+        # Returns the highest semver found among tagged releases, or
+        # Config.default_version on any failure.
+        def probe_latest
+          versions = fetch_release_versions
+          return fallback_latest("releases listing was empty") if versions.empty?
+
+          highest = versions.max_by { |v| Gem::Version.new(v) }
+          if Config.known?(highest)
+            highest
+          else
+            fallback_latest("#{highest.inspect} is not in known_versions; using default")
+          end
+        rescue StandardError => e
+          fallback_latest(e.message)
+        end
+
+        def fallback_latest(reason)
+          warn "Cldr::VersionResolver: --latest probe failed (#{reason}); " \
+               "falling back to default #{Config.default_version.inspect}"
+          Config.default_version
+        end
+
+        def fetch_release_versions
+          uri = URI(Config.listing_url)
+          response = Net::HTTP.get_response(uri)
+          return [] unless response.is_a?(Net::HTTPSuccess)
+
+          releases = JSON.parse(response.body || "[]")
+          releases.filter_map do |release|
+            tag = release["tag_name"]
+            next unless tag
+
+            match = tag.match(/\A(\d+(?:\.\d+)+)\z/)
+            match && match[1]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fontisan/cldr.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# Namespace hub for CLDR (Common Locale Data Repository) support.
+#
+# Provides per-language exemplar character sets so the audit can
+# compute "this font covers X% of language Y". All Cldr::* constants
+# are autoloaded from here.
+
+module Fontisan
+  module Cldr
+    autoload :Config,              "fontisan/cldr/config"
+    autoload :CacheManager,        "fontisan/cldr/cache_manager"
+    autoload :Downloader,          "fontisan/cldr/downloader"
+    autoload :VersionResolver,     "fontisan/cldr/version_resolver"
+    autoload :IndexBuilder,        "fontisan/cldr/index_builder"
+    autoload :Index,               "fontisan/cldr/index"
+    autoload :Aggregator,          "fontisan/cldr/aggregator"
+    autoload :UnicodeSetParser,    "fontisan/cldr/unicode_set_parser"
+    autoload :Error,               "fontisan/cldr/error"
+    autoload :DownloadError,       "fontisan/cldr/download_error"
+    autoload :UnknownVersionError, "fontisan/cldr/unknown_version_error"
+  end
+end

data/lib/fontisan/cli/cldr_cli.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "thor"
+
+module Fontisan
+  # Thor subcommand for managing the local CLDR (Common Locale Data
+  # Repository) cache used by `fontisan audit` for per-language coverage.
+  #
+  #   fontisan cldr download [VERSION]   fetch + index CLDR exemplars
+  #   fontisan cldr status               show what's cached
+  #   fontisan cldr path [VERSION]       print local cache path
+  #   fontisan cldr list                 list known versions
+  #   fontisan cldr remove VERSION       delete a cached version
+  #
+  # With no arguments, `download` resolves the configured default version
+  # (see lib/fontisan/config/cldr.yml).
+  class CldrCli < Thor
+    desc "download [VERSION]",
+         "Download and index CLDR exemplar characters (default: configured default version)"
+    option :force, type: :boolean, default: false,
+                   desc: "Re-download even if already cached"
+    option :latest, type: :boolean, default: false,
+                    desc: "Probe GitHub releases for the latest version"
+    def download(version = nil)
+      intent = resolve_intent(version, options[:latest])
+      actual = Cldr::VersionResolver.resolve(intent)
+
+      Cldr::Downloader.download(actual, force: options[:force])
+      Cldr::IndexBuilder.build(actual) unless index_present?(actual)
+      puts "CLDR #{actual} ready at: #{Cldr::CacheManager.version_dir(actual)}"
+    rescue Cldr::Error => e
+      warn "ERROR: #{e.message}"
+      exit 1
+    end
+
+    desc "status", "Show cached CLDR versions and default version"
+    def status
+      cached = Cldr::CacheManager.cached_versions
+      puts "Default version: #{Cldr::Config.default_version}"
+      puts "Cache root:      #{Cldr::CacheManager.root}"
+      puts "Cached versions: #{cached.empty? ? '(none)' : cached.join(', ')}"
+    end
+
+    desc "path [VERSION]", "Print local cache directory for a version"
+    def path(version = nil)
+      actual = Cldr::VersionResolver.resolve(version)
+      puts Cldr::CacheManager.version_dir(actual)
+    rescue Cldr::UnknownVersionError => e
+      warn "ERROR: #{e.message}"
+      exit 1
+    end
+
+    desc "list", "List CLDR versions known to this Fontisan release"
+    def list
+      Cldr::Config.known_versions.each { |v| puts v }
+    end
+
+    desc "remove VERSION", "Remove a cached CLDR version"
+    def remove(version)
+      Cldr::VersionResolver.validate!(version)
+      unless Cldr::CacheManager.cached?(version)
+        warn "Version #{version} is not cached; nothing to remove."
+        return
+      end
+
+      Cldr::CacheManager.remove_version(version)
+      puts "Removed CLDR #{version}."
+    rescue Cldr::UnknownVersionError => e
+      warn "ERROR: #{e.message}"
+      exit 1
+    end
+
+    private
+
+    def resolve_intent(version, latest)
+      return :latest if latest && version.nil?
+
+      version
+    end
+
+    def index_present?(version)
+      Cldr::CacheManager.languages_index_path(version).exist?
+    end
+  end
+end