ucode 0.1.0 → 0.1.1

data/config/unicode17_universal_glyph_set.yml

@@ -0,0 +1,293 @@
+---
+unicode_version: 17.0.0
+ucode_version: 0.1.1
+generated_at: '2026-06-28T00:00:00Z'
+default_sources:
+- kind: fontist
+  label: noto-sans
+  priority: 1
+  license: OFL
+  provenance: Universal fallback for Latin/Cyrillic/Greek/symbols and any block without
+    a specialist entry below
+map:
+  CJK_Unified_Ideographs_Extension_A:
+    sources:
+    - kind: path
+      label: FSung-2
+      priority: 1
+      license: OFL
+      path: "~/Downloads/全宋體/FSung-2.ttf"
+    - kind: fontist
+      label: noto-sans-cjk-jp
+      priority: 99
+  CJK_Unified_Ideographs:
+    sources:
+    - kind: path
+      label: FSung-1
+      priority: 1
+      license: OFL
+      provenance: Taiwan MOE 全宋體; covers U+4E00..U+9FFF core
+      path: "~/Downloads/全宋體/FSung-1.ttf"
+    - kind: fontist
+      label: noto-sans-cjk-jp
+      priority: 99
+      provenance: Catch-all fallback for CJK codepoints FSung misses
+  Sidetic:
+    sources:
+    - kind: fontist
+      label: lentariso
+      priority: 1
+      license: OFL
+      provenance: Lentariso >=1.029 (github.com/Bry10022/Lentariso); 26/26 cmap-verified
+    - kind: fontist
+      label: noto-sans-sidetic
+      priority: 2
+  Sharada_Supplement:
+    sources:
+    - kind: fontist
+      label: noto-sans-sharada
+      priority: 1
+      license: OFL
+  Tolong_Siki:
+    sources:
+    - kind: fontist
+      label: noto-sans-tolong-siki
+      priority: 1
+      license: OFL
+  Egyptian_Hieroglyphs:
+    sources:
+    - kind: path
+      label: UniHieroglyphica
+      priority: 1
+      license: OFL
+      provenance: suignard.com; authoritative for Egyptian Hieroglyphs
+      path: data/fonts/UniHieroglyphica.ttf
+  Egyptian_Hieroglyph_Format_Controls:
+    sources:
+    - kind: path
+      label: Egyptian-Text
+      priority: 1
+      license: OFL
+      provenance: microsoft/font-tools; OFL
+      path: data/fonts/EgyptianText-Regular.ttf
+  Egyptian_Hieroglyphs_Extended-A:
+    sources:
+    - kind: path
+      label: UniHieroglyphica
+      priority: 1
+      license: OFL
+      path: data/fonts/UniHieroglyphica.ttf
+  Beria_Erfe:
+    sources:
+    - kind: fontist
+      label: kedebideri
+      priority: 1
+      license: OFL
+      provenance: Kedebideri 3.001 (software.sil.org/kedebideri); 50/50 cmap-verified
+  Tangut:
+    sources:
+    - kind: fontist
+      label: noto-sans-tangut
+      priority: 1
+      license: OFL
+  Tangut_Components:
+    sources:
+    - kind: fontist
+      label: noto-sans-tangut
+      priority: 1
+      license: OFL
+  Tangut_Supplement:
+    sources:
+    - kind: fontist
+      label: noto-sans-tangut
+      priority: 1
+      license: OFL
+  Symbols_for_Legacy_Computing_Supplement:
+    sources:
+    - kind: fontist
+      label: babelstone-pseudographica
+      priority: 1
+      license: OFL
+      provenance: BabelStone; partial Unicode 17 coverage
+  Miscellaneous_Symbols_Supplement:
+    sources:
+    - kind: fontist
+      label: noto-sans-symbols-2
+      priority: 1
+      license: OFL
+  Musical_Symbols:
+    sources:
+    - kind: fontist
+      label: noto-music
+      priority: 1
+      license: OFL
+  Tai_Yo:
+    sources:
+    - kind: path
+      label: NotoSerifTaiYo
+      priority: 1
+      license: OFL
+      provenance: translationcommons.org; proven via correlate-v4 Pillar 2 extraction
+      path: data/fonts/NotoSerifTaiYo.ttf
+  Adlam:
+    sources:
+    - kind: path
+      label: NotoSansAdlam
+      priority: 1
+      license: OFL
+      provenance: Noto Sans Adlam fixture; 88/88 cmap-verified at Unicode 16 baseline
+      path: spec/fixtures/fonts/NotoSansAdlam-Regular.ttf
+  Alchemical_Symbols:
+    sources:
+    - kind: fontist
+      label: noto-sans-symbols
+      priority: 1
+      license: OFL
+    - kind: fontist
+      label: symbola
+      priority: 2
+  Supplemental_Arrows-C:
+    sources:
+    - kind: fontist
+      label: symbola
+      priority: 1
+  CJK_Unified_Ideographs_Extension_B:
+    sources:
+    - kind: path
+      label: FSung-2
+      priority: 1
+      license: OFL
+      path: "~/Downloads/全宋體/FSung-2.ttf"
+    - kind: fontist
+      label: noto-sans-cjk-jp
+      priority: 99
+  CJK_Unified_Ideographs_Extension_C:
+    sources:
+    - kind: path
+      label: FSung-2
+      priority: 1
+      license: OFL
+      path: "~/Downloads/全宋體/FSung-2.ttf"
+    - kind: fontist
+      label: noto-sans-cjk-jp
+      priority: 99
+  CJK_Unified_Ideographs_Extension_D:
+    sources:
+    - kind: path
+      label: FSung-2
+      priority: 1
+      license: OFL
+      path: "~/Downloads/全宋體/FSung-2.ttf"
+    - kind: fontist
+      label: noto-sans-cjk-jp
+      priority: 99
+  CJK_Unified_Ideographs_Extension_E:
+    sources:
+    - kind: path
+      label: FSung-3
+      priority: 1
+      license: OFL
+      path: "~/Downloads/全宋體/FSung-3.ttf"
+    - kind: fontist
+      label: noto-sans-cjk-jp
+      priority: 99
+  CJK_Unified_Ideographs_Extension_F:
+    sources:
+    - kind: path
+      label: FSung-3
+      priority: 1
+      license: OFL
+      path: "~/Downloads/全宋體/FSung-3.ttf"
+    - kind: fontist
+      label: noto-sans-cjk-jp
+      priority: 99
+  CJK_Unified_Ideographs_Extension_I:
+    sources:
+    - kind: path
+      label: FSung-3
+      priority: 1
+      license: OFL
+      path: "~/Downloads/全宋體/FSung-3.ttf"
+  CJK_Unified_Ideographs_Extension_G:
+    sources:
+    - kind: path
+      label: FSung-3
+      priority: 1
+      license: OFL
+      path: "~/Downloads/全宋體/FSung-3.ttf"
+    - kind: fontist
+      label: noto-sans-cjk-jp
+      priority: 99
+  CJK_Unified_Ideographs_Extension_H:
+    sources:
+    - kind: path
+      label: FSung-3
+      priority: 1
+      license: OFL
+      provenance: FSung-3 cmap-verified 4192/4192 for Ext H
+      path: "~/Downloads/全宋體/FSung-3.ttf"
+  CJK_Unified_Ideographs_Extension_J:
+    sources:
+    - kind: path
+      label: FSung-3
+      priority: 1
+      license: OFL
+      provenance: FSung-3 cmap-verified 4298/4298 for Ext J (Unicode 17 new block)
+      path: "~/Downloads/全宋體/FSung-3.ttf"
+  Egyptian_Hieroglyphs_Extended-B:
+    sources:
+    - kind: fontist
+      label: UniHieroglyphica
+      priority: 1
+      license: OFL
+      provenance: suignard.com/UniHieroglyphica v16; authoritative for Egyptian Hieroglyphs
+        Extended-B (UC17 new block)
+  Arabic_Extended-B:
+    sources:
+    - kind: fontist
+      label: noto-sans-arabic
+      priority: 1
+      license: OFL
+      provenance: UC17 additions to Arabic Extended-B
+  Arabic_Extended-C:
+    sources:
+    - kind: fontist
+      label: noto-sans-arabic
+      priority: 1
+      license: OFL
+      provenance: Arabic Extended-C (UC17 new block)
+  Telugu:
+    sources:
+    - kind: fontist
+      label: noto-sans-telugu
+      priority: 1
+      license: OFL
+      provenance: UC17 Telugu addition
+  Kannada:
+    sources:
+    - kind: fontist
+      label: noto-sans-kannada
+      priority: 1
+      license: OFL
+      provenance: UC17 Kannada addition
+  Chess_Symbols:
+    sources:
+    - kind: fontist
+      label: noto-sans-symbols-2
+      priority: 1
+      license: OFL
+      provenance: UC17 additions to Chess Symbols
+  Transport_and_Map_Symbols:
+    sources:
+    - kind: fontist
+      label: noto-sans-symbols-2
+      priority: 1
+      license: OFL
+      provenance: UC17 additions to Transport and Map Symbols
+  Symbols_and_Pictographs_Extended-A:
+    sources:
+    - kind: fontist
+      label: noto-sans-symbols-2
+      priority: 1
+      license: OFL
+      provenance: UC17 additions to Symbols and Pictographs Extended-A

data/lib/ucode/audit/block_aggregator.rb

@@ -3,55 +3,72 @@
 module Ucode
   module Audit
     # Produces one {Models::Audit::BlockSummary} per touched Unicode block
-    # for a font's cmap codepoint set, compared against a ucode UCD
-    # baseline.
+    # for a font's cmap codepoint set, compared against a
+    # {CoverageReference}.
     #
-    # Pure transformation: takes the resolved baseline Database + the
-    # font's codepoint list, returns BlockSummary[]. No I/O beyond the
-    # database lookups, no mutation of inputs.
+    # Pure transformation: takes a reference + the font's codepoint list,
+    # returns BlockSummary[]. No I/O beyond the reference's lookups, no
+    # mutation of inputs.
     #
-    # The "assigned" set for a block is derived from the Database's
-    # ranges-with-that-name. The Database stores coalesced runs of
-    # consecutive assigned codepoints grouped by block name, so the
-    # union of those ranges IS the assigned set for that block.
+    # The "assigned" set for a block comes from
+    # `reference.entries_for_block(name)`. For a {UcdOnlyReference}
+    # that's every codepoint in the block's UCD ranges. For a
+    # {UniversalSetReference} it's every codepoint the universal glyph
+    # set built a glyph for in that block — each entry carries tier +
+    # source provenance that gets attached to the missing-codepoint
+    # list (TODO 25).
     class BlockAggregator
-      # @param database [Ucode::Database, nil] resolved baseline. When
-      #   nil, #call returns an empty array — caller should treat that
-      #   as "no UCD baseline available" and surface a warning.
-      def initialize(database)
-        @database = database
+      # @param reference [CoverageReference, Ucode::Database, nil]
+      #   pluggable baseline. For backwards compatibility a raw
+      #   Ucode::Database is still accepted and wrapped in a
+      #   {UcdOnlyReference} at construction time. When nil, #call
+      #   returns an empty array.
+      def initialize(reference)
+        @reference = coerce_reference(reference)
       end
 
       # @param codepoints [Enumerable<Integer>]
       # @return [Array<Models::Audit::BlockSummary>] sorted by first_cp
       def call(codepoints)
-        return [] if @database.nil? || codepoints.empty?
+        return [] if @reference.nil? || codepoints.empty?
 
         grouped = group_by_block(codepoints)
-        grouped.map { |name, covered| build_summary(name, covered) }
+        grouped.filter_map { |name, covered| build_summary(name, covered) }
           .sort_by(&:first_cp)
       end
 
       private
 
+      def coerce_reference(input)
+        return nil if input.nil?
+        return input if input.is_a?(CoverageReference)
+
+        UcdOnlyReference.new(database: input)
+      end
+
       def group_by_block(codepoints)
         codepoints.each_with_object(Hash.new { |h, k| h[k] = [] }) do |cp, acc|
-          name = @database.lookup_block(cp)
+          name = block_name_for(cp)
           acc[name] << cp if name
         end
       end
 
+      def block_name_for(codepoint)
+        @reference.block_name_for(codepoint)
+      end
+
       def build_summary(name, covered_cps)
-        ranges = @database.block_ranges_by_name(name)
-        # ranges is non-empty here: the name came from lookup_block,
-        # which only returns names present in the blocks table.
-        first_cp = ranges.map(&:first_cp).min
-        last_cp = ranges.map(&:last_cp).max
-        assigned_set = expand_assigned(ranges)
+        entries = @reference.entries_for_block(name)
+        return nil if entries.empty?
+
+        first_cp = entries.first.codepoint
+        last_cp = entries.last.codepoint
+        assigned_set = entries.to_set(&:codepoint)
         covered_set = covered_cps.to_set & assigned_set
         missing_set = assigned_set - covered_set
+        missing_sorted = missing_set.sort
 
-        Models::Audit::BlockSummary.new(
+        kwargs = {
           name: name,
           first_cp: first_cp,
           last_cp: last_cp,
@@ -65,14 +82,25 @@ module Ucode
             covered_count: covered_set.size,
             total_assigned: assigned_set.size,
           ),
-          missing_codepoints: missing_set.sort,
+          missing_codepoints: missing_sorted,
           covered_codepoints: covered_set.sort,
-        )
+        }
+
+        provenance = @reference.provenance_for(missing_sorted)
+        kwargs[:missing_codepoint_provenance] = provenance_rows(missing_sorted, provenance) if provenance
+
+        Models::Audit::BlockSummary.new(**kwargs)
       end
 
-      def expand_assigned(ranges)
-        ranges.each_with_object(Set.new) do |r, acc|
-          (r.first_cp..r.last_cp).each { |cp| acc << cp }
+      def provenance_rows(codepoints, rows)
+        return [] if rows.nil? || rows.empty?
+
+        codepoints.zip(rows).map do |cp, row|
+          Models::Audit::CodepointProvenance.new(
+            codepoint: cp,
+            tier: row[:tier],
+            source: row[:source],
+          )
         end
       end
 

data/lib/ucode/audit/browser/face_page.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "json"
+
+require "ucode/repo/atomic_writes"
+require "ucode/audit/browser"
+require "ucode/audit/browser/template"
+require "ucode/audit/emitter/paths"
+require "ucode/audit/emitter/index_emitter"
+
+module Ucode
+  module Audit
+    module Browser
+      # Renders one face's `index.html` — a fully self-contained
+      # browser page for one audited font.
+      #
+      # The page inlines the same JSON shape that {Emitter::IndexEmitter}
+      # writes to `index.json`, plus inlined CSS and JS. Opening the
+      # file via `file://` renders the overview immediately; lazy
+      # fetches of the per-block chunks (`blocks/<NAME>.json`,
+      # `codepoints/<NAME>.json`, `glyphs/U+XXXX.svg`) work when the
+      # directory is served over HTTP.
+      #
+      # Self-contained: no external CSS, no external JS, no CDN.
+      # Portable: the entire `<label>/` directory can be moved/served
+      # whole and the page still works.
+      #
+      # Two construction modes — pass exactly one of:
+      #   - `report:` — a live {Models::Audit::AuditReport}. The JSON
+      #     shape is derived via {Emitter::IndexEmitter#build_index}.
+      #     Used by {Emitter::FaceDirectory} when emitting alongside
+      #     the audit.
+      #   - `overview_json:` — a pre-built JSON string of the overview
+      #     shape. Used by {Commands::AuditBrowserCommand} when
+      #     regenerating HTML from an existing `index.json`.
+      class FacePage
+        include Ucode::Repo::AtomicWrites
+
+        # @param report [Models::Audit::AuditReport, nil]
+        # @param overview_json [String, nil] pre-built overview JSON
+        # @param verbose [Boolean] when true, the rendered page
+        #   advertises per-block codepoint detail chunks
+        # @param with_glyphs [Boolean] when true, the rendered page
+        #   advertises that `glyphs/U+XXXX.svg` chunks exist
+        # @param universal_set_root [String, Pathname, nil] when both
+        #   this and `face_dir:` are present and the root exists, the
+        #   inlined overview JSON carries a `universal_set` section
+        #   with relative paths. The face browser JS uses these to
+        #   fetch universal-set glyphs for missing-codepoint chips.
+        # @param face_dir [String, Pathname, nil] destination face
+        #   directory. Required when `universal_set_root:` is set and
+        #   the caller wants relative paths resolved; otherwise
+        #   optional. `write(face_dir)` overrides this.
+        def initialize(report: nil, overview_json: nil, verbose: false,
+                       with_glyphs: false, universal_set_root: nil, face_dir: nil)
+          raise ArgumentError, "pass exactly one of report: / overview_json:" \
+            unless report.nil? ^ overview_json.nil?
+
+          @report = report
+          @overview_json = overview_json
+          @verbose = verbose
+          @with_glyphs = with_glyphs
+          @universal_set_root = universal_set_root
+          @face_dir = face_dir
+        end
+
+        # Write the rendered page to `<face_dir>/index.html`.
+        # @param face_dir [String, Pathname]
+        # @return [Boolean] true if written, false if skipped
+        def write(face_dir)
+          @face_dir = Pathname.new(face_dir)
+          write_atomic(@face_dir.join("index.html"), render)
+        end
+
+        # Render the page as a string. Useful in tests.
+        # @return [String]
+        def render
+          Template.new(:face).render(
+            overview_json: overview_json,
+            page_title: page_title,
+            verbose: @verbose,
+            with_glyphs: @with_glyphs,
+            universal_set: universal_set_section,
+          )
+        end
+
+        private
+
+        def overview_hash
+          return @overview_hash if @overview_hash
+          return JSON.parse(@overview_json) if @overview_json
+
+          @overview_hash = Ucode::Audit::Emitter::IndexEmitter.new.build_index(
+            @report,
+            universal_set_root: @face_dir ? @universal_set_root : nil,
+            face_dir: @face_dir,
+          )
+        end
+
+        def overview_json
+          return @overview_json if @overview_json
+
+          JSON.generate(overview_hash)
+        end
+
+        def universal_set_section
+          overview_hash["universal_set"] || { "available" => false }
+        end
+
+        def page_title
+          @report ? report_title : derived_title
+        end
+
+        def report_title
+          [@report.family_name, @report.subfamily_name].compact.join(" ")
+        end
+
+        def derived_title
+          font = JSON.parse(@overview_json)["font"] || {}
+          [font["family_name"], font["subfamily_name"]].compact.join(" ")
+        rescue JSON::ParserError
+          "ucode audit"
+        end
+      end
+    end
+  end
+end

data/lib/ucode/audit/browser/glyph_panel.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "json"
+
+require "ucode/audit/browser"
+
+module Ucode
+  module Audit
+    module Browser
+      # Service that builds per-codepoint glyph panel data for the
+      # missing-glyph reporter (TODO 26).
+      #
+      # Two consumers:
+      #
+      #   - {MissingGlyphPage} — standalone per-block gallery; calls
+      #     {#to_hash} once per missing codepoint to inline SVG markup.
+      #   - The face browser JS — via the `universal_set` section that
+      #     {Emitter::IndexEmitter} embeds in `index.json`. The JS
+      #     fetches `<glyphs_dir>/U+XXXX.svg` at runtime using those
+      #     paths; this class doesn't need to be loaded client-side.
+      #
+      # Reads the universal-set manifest once and builds a
+      # codepoint → {Models::UniversalSetEntry} index so per-codepoint
+      # lookups are O(1). SVG markup is read on demand from
+      # `<universal_set_root>/glyphs/U+XXXX.svg`.
+      #
+      # When the universal-set root is `nil` or unreachable on disk,
+      # {#available?} returns false and {#to_hash} returns a minimal
+      # stub with `available: false`, `svg: nil`. Consumers render a
+      # text-only fallback in that case — the surrounding page still
+      # works.
+      class GlyphPanel
+        GLYPHS_DIRNAME = "glyphs"
+        MANIFEST_FILENAME = "manifest.json"
+        private_constant :GLYPHS_DIRNAME, :MANIFEST_FILENAME
+
+        # @param universal_set_root [String, Pathname, nil] root of the
+        #   universal-set build (e.g. "output/universal_glyph_set").
+        #   nil when no set is co-located.
+        def initialize(universal_set_root:)
+          @root = universal_set_root.nil? ? nil : Pathname.new(universal_set_root)
+          @available = set_available?
+          @entries_by_cp = @available ? build_entries_index : {}
+        end
+
+        # @return [Boolean] true when the universal-set root, manifest,
+        #   and glyphs directory are all reachable on disk
+        def available?
+          @available
+        end
+
+        # @param codepoint [Integer]
+        # @return [Hash] panel payload:
+        #   - "codepoint" => Integer
+        #   - "id"        => "U+XXXX"
+        #   - "available" => Boolean (per-codepoint glyph file exists)
+        #   - "svg"       => String markup, or nil when the SVG file
+        #     is missing or the universal set is unavailable
+        #   - "tier"      => String (e.g. "tier-1"), or nil
+        #   - "source"    => String (e.g. "noto-sans"), or nil
+        def to_hash(codepoint)
+          {
+            "codepoint" => codepoint,
+            "id" => cp_id(codepoint),
+            "available" => glyph_available?(codepoint),
+            "svg" => read_svg(codepoint),
+            "tier" => entry_for(codepoint)&.tier,
+            "source" => entry_for(codepoint)&.source,
+          }
+        end
+
+        private
+
+        attr_reader :root
+
+        def set_available?
+          return false if root.nil?
+
+          root.directory? && manifest_path.exist? && glyphs_dir.directory?
+        end
+
+        def manifest_path
+          root.join(MANIFEST_FILENAME)
+        end
+
+        def glyphs_dir
+          root.join(GLYPHS_DIRNAME)
+        end
+
+        def glyph_path(codepoint)
+          glyphs_dir.join("#{cp_id(codepoint)}.svg")
+        end
+
+        def glyph_available?(codepoint)
+          return false unless @available
+
+          glyph_path(codepoint).exist?
+        end
+
+        def read_svg(codepoint)
+          return nil unless @available
+
+          path = glyph_path(codepoint)
+          path.exist? ? path.read : nil
+        end
+
+        def entry_for(codepoint)
+          @entries_by_cp[codepoint]
+        end
+
+        def build_entries_index
+          hash = JSON.parse(manifest_path.read)
+          manifest = Ucode::Models::UniversalSetManifest.from_hash(hash)
+          manifest.entries.to_h { |e| [e.codepoint, e] }
+        end
+
+        def cp_id(codepoint)
+          format("U+%04X", codepoint.to_i)
+        end
+      end
+    end
+  end
+end