ucode 0.1.0

data/lib/ucode/audit/block_aggregator.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Ucode
+  module Audit
+    # Produces one {Models::Audit::BlockSummary} per touched Unicode block
+    # for a font's cmap codepoint set, compared against a ucode UCD
+    # baseline.
+    #
+    # Pure transformation: takes the resolved baseline Database + the
+    # font's codepoint list, returns BlockSummary[]. No I/O beyond the
+    # database lookups, no mutation of inputs.
+    #
+    # The "assigned" set for a block is derived from the Database's
+    # ranges-with-that-name. The Database stores coalesced runs of
+    # consecutive assigned codepoints grouped by block name, so the
+    # union of those ranges IS the assigned set for that block.
+    class BlockAggregator
+      # @param database [Ucode::Database, nil] resolved baseline. When
+      #   nil, #call returns an empty array — caller should treat that
+      #   as "no UCD baseline available" and surface a warning.
+      def initialize(database)
+        @database = database
+      end
+
+      # @param codepoints [Enumerable<Integer>]
+      # @return [Array<Models::Audit::BlockSummary>] sorted by first_cp
+      def call(codepoints)
+        return [] if @database.nil? || codepoints.empty?
+
+        grouped = group_by_block(codepoints)
+        grouped.map { |name, covered| build_summary(name, covered) }
+          .sort_by(&:first_cp)
+      end
+
+      private
+
+      def group_by_block(codepoints)
+        codepoints.each_with_object(Hash.new { |h, k| h[k] = [] }) do |cp, acc|
+          name = @database.lookup_block(cp)
+          acc[name] << cp if name
+        end
+      end
+
+      def build_summary(name, covered_cps)
+        ranges = @database.block_ranges_by_name(name)
+        # ranges is non-empty here: the name came from lookup_block,
+        # which only returns names present in the blocks table.
+        first_cp = ranges.map(&:first_cp).min
+        last_cp = ranges.map(&:last_cp).max
+        assigned_set = expand_assigned(ranges)
+        covered_set = covered_cps.to_set & assigned_set
+        missing_set = assigned_set - covered_set
+
+        Models::Audit::BlockSummary.new(
+          name: name,
+          first_cp: first_cp,
+          last_cp: last_cp,
+          range: format_range(first_cp, last_cp),
+          plane: first_cp >> 16,
+          total_assigned: assigned_set.size,
+          covered_count: covered_set.size,
+          missing_count: missing_set.size,
+          coverage_percent: percent(covered_set.size, assigned_set.size),
+          status: Models::Audit::BlockSummary.derive_status(
+            covered_count: covered_set.size,
+            total_assigned: assigned_set.size,
+          ),
+          missing_codepoints: missing_set.sort,
+          covered_codepoints: covered_set.sort,
+        )
+      end
+
+      def expand_assigned(ranges)
+        ranges.each_with_object(Set.new) do |r, acc|
+          (r.first_cp..r.last_cp).each { |cp| acc << cp }
+        end
+      end
+
+      def percent(covered, total)
+        return 0.0 if total.zero?
+
+        (covered.to_f / total * 100).round(2)
+      end
+
+      def format_range(first, last)
+        format("U+%<first>04X–U+%<last>04X", first: first, last: last)
+      end
+    end
+  end
+end

data/lib/ucode/audit/codepoint_range_coalescer.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Ucode
+  module Audit
+    # Coalesces a flat codepoint list into contiguous
+    # {Models::Audit::CodepointRange} instances.
+    #
+    # Pure function: input is any Enumerable<Integer>, output is a sorted
+    # array of contiguous ranges. Used by {Extractors::Coverage} to produce
+    # the compact range view that is the default AuditReport shape.
+    module CodepointRangeCoalescer
+      module_function
+
+      # @param codepoints [Enumerable<Integer>]
+      # @return [Array<Models::Audit::CodepointRange>] contiguous, sorted
+      def call(codepoints)
+        return [] if codepoints.nil? || codepoints.empty?
+
+        sorted = codepoints.sort.uniq
+        ranges = []
+        range_start = sorted[0]
+        prev = sorted[0]
+
+        sorted[1..].each do |cp|
+          next if cp == prev
+
+          if cp == prev + 1
+            prev = cp
+          else
+            ranges << Models::Audit::CodepointRange.new(first_cp: range_start,
+                                                        last_cp: prev)
+            range_start = cp
+            prev = cp
+          end
+        end
+        ranges << Models::Audit::CodepointRange.new(first_cp: range_start,
+                                                    last_cp: prev)
+        ranges
+      end
+    end
+  end
+end

data/lib/ucode/audit/context.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require "time"
+
+require "fontisan"
+
+module Ucode
+  module Audit
+    # Value object carrying everything an extractor needs to do its job.
+    #
+    # Extractors never reach back into AuditCommand state — they read
+    # exclusively from the Context. Shared derived data (codepoints,
+    # UCD baseline, source format) is memoized here so multiple
+    # extractors don't recompute it.
+    #
+    # ucode deltas vs fontisan's Context:
+    #
+    # - Drops `cldr` and the entire CLDR resolution path (out of scope).
+    # - Replaces fontisan's `ucd` memoizer with `baseline`, a struct
+    #   carrying version + database + metadata.
+    # - Adds optional `renderer` for `--with-glyphs` mode (nil otherwise).
+    class Context
+      Baseline = Struct.new(:version, :database, :metadata, :warning, keyword_init: true) do
+        # True when the baseline is usable (database present and no warning).
+        def available?
+          !database.nil? && warning.nil?
+        end
+      end
+
+      private_constant :Baseline
+
+      attr_reader :font, :font_path, :font_index, :num_fonts_in_source,
+                  :options, :renderer
+
+      # @param font [Fontisan::Font] parsed font handle (has_table?, table).
+      # @param font_path [Pathname, String] source path for format detection.
+      # @param font_index [Integer] 0-based face index within a collection.
+      # @param num_fonts_in_source [Integer] total faces in the source file.
+      # @param options [Hash{Symbol=>Object}] audit options (ucd_version,
+      #   all_codepoints, with_glyphs, etc.).
+      # @param renderer [Object, nil] glyph renderer for --with-glyphs mode.
+      def initialize(font:, font_path:, font_index:, num_fonts_in_source:,
+                     options:, renderer: nil)
+        @font = font
+        @font_path = font_path
+        @font_index = font_index
+        @num_fonts_in_source = num_fonts_in_source
+        @options = options
+        @renderer = renderer
+      end
+
+      # Codepoints the font's cmap actually maps. Memoized.
+      # @return [Array<Integer>]
+      def codepoints
+        @codepoints ||= extract_codepoints
+      end
+
+      # Pre-resolved baseline (UCD version + database + metadata).
+      # Memoized. When resolution fails, returns a Baseline with a
+      # `warning` and nil database so extractors can degrade gracefully.
+      # @return [Baseline]
+      def baseline
+        @baseline ||= resolve_baseline
+      end
+
+      # Detected source format string ("ttf", "otf", "ttc", ...). Memoized.
+      # @return [String, nil]
+      def source_format
+        @source_format ||= Fontisan::FontLoader.detect_format(@font_path)&.to_s
+      end
+
+      # True when the user asked for every codepoint (including unassigned)
+      # in the report's `codepoints` field.
+      # @return [Boolean]
+      def all_codepoints?
+        @options[:all_codepoints] == true
+      end
+
+      # True when glyph rendering is requested (--with-glyphs).
+      # @return [Boolean]
+      def with_glyphs?
+        @options[:with_glyphs] == true && !@renderer.nil?
+      end
+
+      private
+
+      def extract_codepoints
+        return [] unless @font.has_table?("cmap")
+
+        @font.table("cmap").unicode_mappings.keys
+      end
+
+      def resolve_baseline
+        version = Ucode::VersionResolver.resolve(@options[:ucd_version])
+        database = open_or_build_database(version)
+        Baseline.new(
+          version: version,
+          database: database,
+          metadata: build_metadata(version),
+          warning: nil,
+        )
+      rescue Ucode::UnknownVersionError => e
+        Baseline.new(version: nil, database: nil, metadata: nil,
+                     warning: "UCD version rejected: #{e.message}")
+      rescue Ucode::DatabaseMissingError => e
+        Baseline.new(version: version, database: nil, metadata: nil,
+                     warning: "UCD unavailable for version #{version}: #{e.message}")
+      rescue StandardError => e
+        Baseline.new(version: nil, database: nil, metadata: nil,
+                     warning: "UCD resolution failed: #{e.message}")
+      end
+
+      def open_or_build_database(version)
+        return Ucode::Database.open(version) if Ucode::Database.cached?(version)
+
+        ensure_ucdzip(version)
+        Ucode::Database.build(version)
+      end
+
+      def ensure_ucdzip(version)
+        return if Ucode::Cache.cached?(version)
+
+        Ucode::Fetch::UcdZip.call(version)
+      end
+
+      def build_metadata(version)
+        Models::Audit::Baseline.new(
+          unicode_version: version,
+          ucode_version: Ucode::VERSION,
+          fontisan_version: Fontisan::VERSION,
+          source: "ucode SQLite index (blocks + scripts tables)",
+          generated_at: Time.now.utc.iso8601,
+        )
+      end
+    end
+  end
+end

data/lib/ucode/audit/discrepancy_detector.rb

@@ -0,0 +1,213 @@
+# frozen_string_literal: true
+
+module Ucode
+  module Audit
+    # Detects cheap audit signals — currently OS/2 ulUnicodeRange bit
+    # claims that disagree with the font's cmap coverage.
+    #
+    # Pure transformation: takes the four OS/2 ulUnicodeRange 32-bit
+    # words + the font's codepoint set, returns Discrepancy[]. No I/O,
+    # no font handle.
+    #
+    # OCP: a new discrepancy kind = one constant on
+    # {Models::Audit::Discrepancy} + one method here. The detector
+    # never enumerates kinds directly.
+    class DiscrepancyDetector
+      # Map of OS/2 ulUnicodeRange bit position => [first_cp, last_cp]
+      # per the OpenType spec (OS/2.ulUnicodeRange). Bits without a
+      # well-defined contiguous range (e.g. PUA, reserved) are omitted
+      # — they cannot be cross-checked against the cmap by this
+      # detector.
+      #
+      # Spec reference:
+      # https://learn.microsoft.com/en-us/typography/opentype/spec/os2#ur
+      BIT_RANGES = {
+        0 => [0x0000, 0x007F], # Basic Latin
+        1 => [0x0080, 0x00FF], # Latin-1 Supplement
+        2 => [0x0100, 0x017F], # Latin Extended-A
+        3 => [0x0180, 0x024F], # Latin Extended-B
+        4 => [0x0250, 0x02AF], # IPA Extension
+        5 => [0x02B0, 0x02FF], # Spacing Modifier Letters
+        6 => [0x0300, 0x036F], # Combining Diacritical Marks
+        7 => [0x0370, 0x03FF], # Greek and Coptic
+        8 => [0x2C80, 0x2CFF], # Coptic
+        9 => [0x0400, 0x04FF], # Cyrillic
+        10 => [0x0530, 0x058F], # Armenian
+        11 => [0x0590, 0x05FF], # Hebrew
+        13 => [0x0600, 0x06FF], # Arabic
+        14 => [0x07C0, 0x07FF], # NKo
+        15 => [0x0900, 0x097F], # Devanagari
+        16 => [0x0980, 0x09FF], # Bengali
+        17 => [0x0A00, 0x0A7F], # Gurmukhi
+        18 => [0x0A80, 0x0AFF], # Gujarati
+        19 => [0x0B00, 0x0B7F], # Oriya
+        20 => [0x0B80, 0x0BFF], # Tamil
+        21 => [0x0C00, 0x0C7F], # Telugu
+        22 => [0x0C80, 0x0CFF], # Kannada
+        23 => [0x0D00, 0x0D7F], # Malayalam
+        24 => [0x0E00, 0x0E7F], # Thai
+        25 => [0x0E80, 0x0EFF], # Lao
+        26 => [0x10A0, 0x10FF], # Georgian
+        27 => [0x1B00, 0x1B7F], # Balinese
+        29 => [0x1E00, 0x1EFF], # Latin Extended Additional
+        30 => [0x1F00, 0x1FFF], # Greek Extended
+        31 => [0x2000, 0x206F], # General Punctuation
+        32 => [0x2070, 0x209F], # Superscripts And Subscripts
+        33 => [0x20A0, 0x20CF], # Currency Symbols
+        34 => [0x20D0, 0x20FF], # Combining Marks Symbols
+        35 => [0x2100, 0x214F], # Letterlike Symbols
+        36 => [0x2150, 0x218F], # Number Forms
+        37 => [0x2190, 0x21FF], # Arrows
+        38 => [0x2200, 0x22FF], # Mathematical Operators
+        39 => [0x2300, 0x23FF], # Miscellaneous Technical
+        40 => [0x2400, 0x243F], # Control Pictures
+        41 => [0x2440, 0x245F], # Optical Character Recognition
+        42 => [0x2460, 0x24FF], # Enclosed Alphanumerics
+        43 => [0x2500, 0x257F], # Box Drawing
+        44 => [0x2580, 0x259F], # Block Elements
+        45 => [0x25A0, 0x25FF], # Geometric Shapes
+        46 => [0x2600, 0x26FF], # Miscellaneous Symbols
+        47 => [0x2700, 0x27BF], # Dingbats
+        48 => [0x3000, 0x303F], # CJK Symbols and Punctuation
+        49 => [0x3040, 0x309F], # Hiragana
+        50 => [0x30A0, 0x30FF], # Katakana
+        51 => [0x3100, 0x312F], # Bopomofo
+        52 => [0x3130, 0x318F], # Hangul Compatibility Jamo
+        53 => [0xA840, 0xA87F], # Phags-pa
+        54 => [0x3200, 0x32FF], # Enclosed CJK Letters and Months
+        55 => [0x3300, 0x33FF], # CJK Compatibility
+        56 => [0xAC00, 0xD7AF], # Hangul Syllables
+        57 => [0x10000, 0x10FFFF], # Surrogate / Non-BMP fallback
+        58 => [0x10900, 0x1091F], # Phoenician
+        59 => [0x4E00, 0x9FFF], # CJK Unified Ideographs (incl. Ext A)
+        60 => [0xE000, 0xF8FF], # Private Use Area
+        61 => [0xF900, 0xFAFF], # CJK Compatibility Ideographs
+        62 => [0xFB00, 0xFB4F], # Alphabetic Presentation Forms
+        63 => [0xFB50, 0xFDFF], # Arabic Presentation Forms-A
+        64 => [0xFE20, 0xFE2F], # Combining Half Marks
+        65 => [0xFE10, 0xFE1F], # Vertical Forms
+        66 => [0xFE50, 0xFE6F], # Small Form Variants
+        67 => [0xFE70, 0xFEFF], # Arabic Presentation Forms-B
+        68 => [0xFF00, 0xFFEF], # Halfwidth And Fullwidth Forms
+        69 => [0xFFF0, 0xFFFF], # Specials
+        70 => [0x0F00, 0x0FFF], # Tibetan
+        71 => [0x0700, 0x074F], # Syriac
+        72 => [0x0780, 0x07BF], # Thaana
+        73 => [0x0D80, 0x0DFF], # Sinhala
+        74 => [0x1000, 0x109F], # Myanmar
+        75 => [0x1200, 0x137F], # Ethiopic
+        76 => [0x13A0, 0x13FF], # Cherokee
+        77 => [0x1400, 0x167F], # Unified Canadian Aboriginal Syllabics
+        78 => [0x1680, 0x169F], # Ogham
+        79 => [0x16A0, 0x16FF], # Runic
+        80 => [0x1780, 0x17FF], # Khmer
+        81 => [0x1800, 0x18AF], # Mongolian
+        82 => [0x2800, 0x28FF], # Braille Patterns
+        83 => [0xA000, 0xA48F], # Yi Syllables
+        84 => [0x1700, 0x171F], # Tagalog
+        85 => [0x10300, 0x1032F], # Old Italic
+        86 => [0x10330, 0x1034F], # Gothic
+        87 => [0x10400, 0x1044F], # Deseret
+        88 => [0x1D000, 0x1D0FF], # Byzantine Musical Symbols
+        89 => [0x1D400, 0x1D7FF], # Mathematical Alphanumeric Symbols
+        90 => [0xFF000, 0xFFFFD], # Private Use (Plane 15)
+        91 => [0xFE00, 0xFE0F], # Variation Selectors
+        92 => [0xE0000, 0xE007F], # Tags
+        93 => [0x1900, 0x194F], # Limbu
+        94 => [0x1950, 0x197F], # Tai Le
+        95 => [0x1980, 0x19DF], # New Tai Lue
+        96 => [0x1A00, 0x1A1F], # Buginese
+        97 => [0x2C00, 0x2C5F], # Glagolitic
+        98 => [0x2D30, 0x2D7F], # Tifinagh
+        99 => [0x4DC0, 0x4DFF], # Yijing Hexagram Symbols
+        100 => [0xA800, 0xA82F], # Syloti Nagri
+        101 => [0xA500, 0xA63F], # Vai
+        102 => [0xA640, 0xA69F], # Cyrillic Extended-B
+        103 => [0xA700, 0xA71F], # Modifier Tone Letters
+        104 => [0xA720, 0xA7FF], # Latin Extended-D
+        105 => [0xA800, 0xA82F], # Syloti Nagri (duplicate of 100; spec)
+        106 => [0xA840, 0xA87F], # Phags-pa (duplicate of 53; spec)
+        107 => [0x100000, 0x10FFFF], # Supplementary PUA-A fallback
+        108 => [0xA4D0, 0xA4FF], # Lisu
+        109 => [0xA490, 0xA4CF], # Bamum
+        110 => [0x10800, 0x1083F], # Cypriot Syllabary
+        111 => [0x10A00, 0x10A5F], # Kharoshthi
+        112 => [0x1B80, 0x1BBF], # Sundanese
+        113 => [0x1BC0, 0x1BFF], # Batak
+        114 => [0x11000, 0x1107F], # Brahmi
+        115 => [0xA8E0, 0xA8FF], # Devanagari Extended
+        116 => [0x11100, 0x1114F], # Kaithi
+        117 => [0x1D360, 0x1D37F], # Counting Rod Numerals
+        118 => [0x12000, 0x1247F], # Cuneiform
+        119 => [0x1F000, 0x1F09F], # Mahjong Tiles
+        120 => [0xA930, 0xA95F], # Rejang
+        121 => [0xA960, 0xA97F], # Hangul Jamo Extended-A
+        122 => [0xAA00, 0xAA5F], # Cham
+        123 => [0xA980, 0xA9DF], # Javanese
+        124 => [0x11600, 0x1165F], # Modi
+        125 => [0x1E900, 0x1E95F], # Adlam
+        126 => [0x1EE00, 0x1EEFF], # Arabic Mathematical Alphabetic Symbols
+      }.freeze
+      private_constant :BIT_RANGES
+
+      # @param ul_unicode_range1 [Integer]
+      # @param ul_unicode_range2 [Integer]
+      # @param ul_unicode_range3 [Integer]
+      # @param ul_unicode_range4 [Integer]
+      # @param codepoints [Enumerable<Integer>] font cmap codepoint set
+      def initialize(ul_unicode_range1:, ul_unicode_range2:,
+                     ul_unicode_range3:, ul_unicode_range4:,
+                     codepoints:)
+        @bits = bits_from_words([
+          ul_unicode_range1 || 0,
+          ul_unicode_range2 || 0,
+          ul_unicode_range3 || 0,
+          ul_unicode_range4 || 0,
+        ])
+        @codepoint_set = codepoints.to_set
+      end
+
+      # @return [Array<Models::Audit::Discrepancy>]
+      def call
+        @bits.sort.map do |bit|
+          first, last = BIT_RANGES.fetch(bit, [nil, nil])
+          next nil if first.nil? # bit set but range unknown — skip
+
+          next nil if range_has_codepoints?(first, last)
+
+          Models::Audit::Discrepancy.new(
+            kind: Models::Audit::Discrepancy::KIND_OS2_UNICODE_RANGE_BIT_WITHOUT_CMAP_CODEPOINTS,
+            detail: format(
+              "OS/2 ulUnicodeRange bit %<bit>d claims %<first>s–%<last>s " \
+              "but cmap has 0 codepoints in that range",
+              bit: bit,
+              first: format("U+%04X", first),
+              last: format("U+%04X", last),
+            ),
+            bit_position: bit,
+          )
+        end.compact
+      end
+
+      private
+
+      def bits_from_words(words)
+        words.each_with_index.flat_map do |word, word_index|
+          bits_in_word(word).map { |bit| word_index * 32 + bit }
+        end
+      end
+
+      # Yields bit positions (0-31) that are set in a 32-bit word.
+      def bits_in_word(word)
+        (0..31).reject { |i| (word & (1 << i)).zero? }
+      end
+
+      def range_has_codepoints?(first, last)
+        # Linear scan; codepoint_set is typically small relative to
+        # the OS/2 range set. For very large fonts (CJK), this is O(N)
+        # per bit — acceptable for one-shot audit cost.
+        @codepoint_set.any? { |cp| cp >= first && cp <= last }
+      end
+    end
+  end
+end

data/lib/ucode/audit/extractors/aggregations.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Ucode
+  module Audit
+    module Extractors
+      # Aggregations: UCD block/script coverage driven by ucode's own
+      # parsed baseline (not ucd.all.flat.zip), plus OS/2 ulUnicodeRange
+      # discrepancies.
+      #
+      # Returned fields:
+      #   baseline, blocks, scripts, plane_summaries, discrepancies
+      #
+      # MECE: this extractor owns UCD-driven aggregations + the OS/2
+      # bit-vs-cmap cross-check. SFNT-driven GSUB/GPOS script/feature
+      # coverage lives in {OpenTypeLayout}.
+      #
+      # ucode delta vs fontisan: replaces UCDXML flat-zip lookup with
+      # ucode's own SQLite-backed Database. The Database exposes
+      # `lookup_block`, `lookup_script`, `block_ranges_by_name`, and
+      # `script_ranges_by_name` — those power every aggregation here.
+      class Aggregations < Base
+        # @param context [Ucode::Audit::Context]
+        # @return [Hash{Symbol=>Object}]
+        def extract(context)
+          baseline = context.baseline
+          return empty_with_warning(baseline) unless baseline.available?
+
+          codepoints = context.codepoints
+          blocks = BlockAggregator.new(baseline.database).call(codepoints)
+          scripts = ScriptAggregator.new(baseline.database).call(codepoints)
+          planes = PlaneAggregator.new.call(blocks)
+          discrepancies = DiscrepancyDetector.new(**os2_args(context))
+            .call
+
+          {
+            baseline: baseline.metadata,
+            blocks: blocks,
+            scripts: scripts,
+            plane_summaries: planes,
+            discrepancies: discrepancies,
+          }
+        end
+
+        private
+
+        def empty_with_warning(baseline)
+          {
+            baseline: baseline.metadata,
+            blocks: [],
+            scripts: [],
+            plane_summaries: [],
+            discrepancies: [],
+          }
+        end
+
+        def os2_args(context)
+          font = context.font
+          os2 = font.has_table?("OS/2") ? font.table("OS/2") : nil
+          {
+            ul_unicode_range1: os2&.ul_unicode_range1,
+            ul_unicode_range2: os2&.ul_unicode_range2,
+            ul_unicode_range3: os2&.ul_unicode_range3,
+            ul_unicode_range4: os2&.ul_unicode_range4,
+            codepoints: context.codepoints,
+          }
+        end
+      end
+    end
+  end
+end

data/lib/ucode/audit/extractors/base.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Ucode
+  module Audit
+    module Extractors
+      # Abstract extractor interface. Subclasses implement `#extract`.
+      #
+      # An extractor reads from a {Context} and returns a hash of fields
+      # suitable for `Models::Audit::AuditReport.new(**fields)`.
+      # Returning an empty hash is valid (no-op).
+      class Base
+        # @param context [Ucode::Audit::Context]
+        # @return [Hash{Symbol=>Object}] fields merged into the AuditReport
+        def extract(context)
+          raise NotImplementedError,
+                "#{self.class} must implement #extract"
+        end
+      end
+    end
+  end
+end