ucode 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e2e60f073662cc78885f8ee6a0333a2307a8b894ddb9789b1e206f6ae63d25e3
-  data.tar.gz: 50b3984de26589d0aab193250e9c8dd3f56fd0d10a4a77a21ffdcb236c38c737
+  metadata.gz: a6674c0e1a8cab0ef2ea7782878be4571e68603843742de78a43da6a6499220c
+  data.tar.gz: 416b0965b6fa4e1e22ef9c431a7c27a16093ba90b708224b7e9c9e1f15ac2c49
 SHA512:
-  metadata.gz: 88d9ef3df0f99af9b3897cd429666092d48a373d9c60d3c83a6e06a59404e0523c736a8d971fbb6d6d29c9b4b80d9610cad54a64cda439326e19a80670d67ba9
-  data.tar.gz: 8146f6984defddb5b45204fe7a8852cd3051bf1474584eddc7489e5ec9fd4900d48452ac5462a1c6c2ac9b8213823ace68cebe6c5b5176dd0c25c922f8f1d4ba
+  metadata.gz: 361d385dd8b5bb04d3ce528ddda0bed64b8c5db46a1ef7e2f77c79ba5c4cf41f67235e8e55d5a38ec2957e1650f7fa4610b50722a299f2dfd67f900bd25e66c0
+  data.tar.gz: 1049d2b20c925da363339af6bcf1b98fd1d7fcf93a50adf9bde2240314eb272f89af6981a6604d8e7b5fa8b075b6c36deb68d42e691425f18fa70fca94a4286e

data/Rakefile

@@ -26,4 +26,4 @@ begin
 rescue LoadError
 end
 
-task default: %i[spec rubocop]
+task default: %i[spec rubocop]

data/config/unicode17_universal_glyph_set.yml

@@ -1,6 +1,6 @@
 ---
 unicode_version: 17.0.0
-ucode_version: 0.2.1
+ucode_version: 0.2.3
 generated_at: '2026-06-28T00:00:00Z'
 default_sources:
 - kind: fontist

data/lib/ucode/cli.rb

@@ -109,34 +109,6 @@ module Ucode
       puts JSON.pretty_generate(result)
     end
 
-    # ─────────────── glyphs ───────────────
-    desc "glyphs [VERSION]", "Extract per-codepoint SVGs from Code Charts PDFs (experimental)"
-    long_desc <<~LONG
-      EXPERIMENTAL in v0.1. The cell extractor currently includes cell-border
-      decorations alongside the actual character outline, so the output is not
-      yet suitable for end-user display. Opt in with --include-glyphs to run
-      the pipeline anyway; otherwise it returns a skipped payload.
-    LONG
-    option :to, type: :string, default: "./output"
-    option :block, type: :array, desc: "Limit to these block ids"
-    option :force, type: :boolean, default: false
-    option :monolith, type: :string, default: "CodeCharts.pdf",
-                      desc: "Path to CodeCharts.pdf for fallback slicing"
-    option :include_glyphs, type: :boolean, default: false,
-                            desc: "Opt into the experimental v0.1 pipeline"
-    def glyphs(version = nil)
-      result = Commands::GlyphsCommand.new.call(
-        VersionResolver.resolve(version),
-        output_root: options[:to],
-        block_filter: options[:block],
-        force: options[:force],
-        monolith_path: options[:monolith],
-        include_glyphs: options[:include_glyphs],
-        warn: $stderr,
-      )
-      puts JSON.pretty_generate(result)
-    end
-
     # ─────────────── site ───────────────
     class Site < Thor
       desc "init", "Copy the Vitepress scaffold into site/"
@@ -345,22 +317,16 @@ module Ucode
     subcommand "cache", Cache
 
     # ─────────────── build ───────────────
-    desc "build [VERSION]", "Full pipeline: fetch + parse + (optional) glyphs + site"
+    desc "build [VERSION]", "Full pipeline: fetch + parse + site"
     option :to, type: :string, default: "./output"
     option :site, type: :string, default: nil, desc: "Build the site here (skipped if nil)"
-    option :monolith, type: :string, default: "CodeCharts.pdf"
     option :force_fetch, type: :boolean, default: false
-    option :include_glyphs, type: :boolean, default: false,
-                            desc: "Opt into the experimental v0.1 glyph step"
     def build(version = nil)
       result = Commands::BuildCommand.new.call(
         version,
         output_root: options[:to],
         site_root: options[:site],
-        monolith_path: options[:monolith],
         force_fetch: options[:force_fetch],
-        include_glyphs: options[:include_glyphs],
-        warn: $stderr,
       )
       puts JSON.pretty_generate(result)
     end

data/lib/ucode/code_chart/extractor.rb

@@ -2,14 +2,6 @@
 
 require "pathname"
 
-require "ucode/error"
-require "ucode/glyphs/embedded_fonts/catalog"
-require "ucode/glyphs/embedded_fonts/renderer"
-require "ucode/glyphs/embedded_fonts/source"
-require "ucode/glyphs/resolver"
-require "ucode/glyphs/sources/pillar1_embedded_tounicode"
-require "ucode/glyphs/sources/tier1_real_font"
-
 module Ucode
   module CodeChart
     # Walks every assigned codepoint in a block and returns one
@@ -108,7 +100,7 @@ module Ucode
       end
 
       def embedded_pillar_sources
-        embedded_source = Glyphs::EmbeddedFonts::Source.new(
+        embedded_source = Glyphs::EmbeddedFonts::PdfLocation.new(
           pdf: @pdf_path, cache_dir: @cache_dir,
         )
         catalog = Glyphs::EmbeddedFonts::Catalog.new(embedded_source)

data/lib/ucode/code_chart/writer.rb

@@ -47,7 +47,7 @@ module Ucode
       # @param ucd_version [String, nil] UCD version to stamp on
       #   provenance. nil = resolved via {VersionResolver.resolve(nil)}.
       # @param cache_dir [Pathname, String, nil] font-stream cache
-      #   directory for the EmbeddedFonts::Source.
+      #   directory for the EmbeddedFonts::PdfLocation.
       # @param now [Time, nil] timestamp override (for tests).
       # @param pillar3_source, tier1_sources: forwarded to the Extractor.
       def initialize(output_root:, pdf_path:, ucd_version: nil,

data/lib/ucode/commands/build.rb

@@ -8,36 +8,23 @@ require "ucode/version_resolver"
 module Ucode
   module Commands
     # `ucode build` — full pipeline: fetch (ucd + unihan + charts) →
-    # parse → (optional) glyphs → (optional) site. Resumable: each step
-    # is idempotent and safe to re-run.
+    # parse → site. Resumable: each step is idempotent and safe to re-run.
     #
     # Resolves the version intent once at the top and threads the
-    # resolved string through every sub-command. See Candidate 4 of the
-    # 2026-06-29 architecture review.
-    #
-    # **Glyph step is opt-in as of v0.1** because the SVG cell extractor
-    # is still experimental. Pass `include_glyphs: true` to enable it;
-    # otherwise the glyphs step is recorded as skipped.
+    # resolved string through every sub-command.
     class BuildCommand
       # @param version_intent [nil, :default, :latest, String]
       # @param output_root [String, Pathname]
       # @param site_root [String, Pathname, nil] if nil, skip site build
-      # @param monolith_path [String, Pathname, nil] CodeCharts.pdf fallback
       # @param force_fetch [Boolean] re-download sources
-      # @param include_glyphs [Boolean] opt into the experimental glyph
-      #   step (default false)
-      # @param warn [IO, nil] forwarded to GlyphsCommand when enabled
       # @return [Hash] aggregated step results
       def call(version_intent, output_root:, site_root: nil,
-               monolith_path: nil, force_fetch: false,
-               include_glyphs: false, warn: nil)
+               force_fetch: false)
         version = VersionResolver.resolve(version_intent)
         steps = {}
 
         steps[:fetch] = run_fetch(version, force: force_fetch)
         steps[:parse] = ParseCommand.new.call(version, output_root: output_root)
-        steps[:glyphs] = run_glyphs(version, output_root, monolith_path,
-                                     include_glyphs: include_glyphs, warn: warn)
         steps[:site] = run_site(output_root, site_root) if site_root
 
         { version: version, steps: steps }
@@ -54,16 +41,6 @@ module Ucode
         }
       end
 
-      def run_glyphs(version, output_root, monolith_path, include_glyphs:, warn:)
-        GlyphsCommand.new.call(
-          version,
-          output_root: output_root,
-          monolith_path: monolith_path || "CodeCharts.pdf",
-          include_glyphs: include_glyphs,
-          warn: warn,
-        )
-      end
-
       def run_site(output_root, site_root)
         SiteCommand.new.build(output_root: output_root, site_root: site_root)
       end

data/lib/ucode/commands/canonical_build.rb

@@ -20,10 +20,7 @@ module Ucode
     # `index.json` + `glyph.svg` atomically, accumulate per-tier +
     # per-block stats, and emit `output/build-report.json`.
     #
-    # This is the v0.2 replacement for the v0.1 cell-extractor pipeline
-    # in {GlyphsCommand}. The two coexist until the v0.1 pipeline is
-    # removed (TODOs 17-19); CanonicalBuildCommand is the path forward
-    # for production dataset runs.
+    # This is the production path for dataset runs.
     #
     # == Pre-conditions (per TODO 21)
     #
@@ -121,10 +118,10 @@ module Ucode
       end
 
       def build_resolver(version, source_config_path)
-        database = Database.open(version)
-        config = Glyphs::SourceConfig.new(path: source_config_path_or_default(source_config_path))
-        builder = Glyphs::SourceBuilder.new(config: config, database: database)
-        Glyphs::Resolver.new(sources: builder.tier1_sources(install: false))
+        Glyphs::ResolverFactory.build(
+          version: version,
+          source_config_path: source_config_path,
+        )
       end
 
       def source_config_path_or_default(path)

data/lib/ucode/commands/universal_set.rb

@@ -126,9 +126,11 @@ module Ucode
         end
 
         def build_resolver(_version, config_path, database)
-          config = Glyphs::SourceConfig.new(path: config_path)
-          builder = Glyphs::SourceBuilder.new(config: config, database: database)
-          Glyphs::Resolver.new(sources: builder.tier1_sources(install: false))
+          Glyphs::ResolverFactory.build(
+            version: _version,
+            source_config_path: config_path,
+            database: database,
+          )
         end
 
         def codepoint_enum(version)

data/lib/ucode/commands.rb

@@ -9,7 +9,6 @@ module Ucode
   module Commands
     autoload :FetchCommand, "ucode/commands/fetch"
     autoload :ParseCommand, "ucode/commands/parse"
-    autoload :GlyphsCommand, "ucode/commands/glyphs"
     autoload :SiteCommand, "ucode/commands/site"
     autoload :LookupCommand, "ucode/commands/lookup"
     autoload :CacheCommand, "ucode/commands/cache"

data/lib/ucode/coordinator/enrichment/bidi.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Ucode
+  class Coordinator
+    module Enrichment
+      # Bidirectional behaviour: mirroring glyph and paired bracket info.
+      module Bidi
+        class << self
+          # @param cp [Ucode::Models::CodePoint]
+          # @param indices [Ucode::Coordinator::Indices]
+          def enrich(cp, indices)
+            mirroring = indices.bidi_mirroring[cp.cp]
+            brackets = indices.bidi_brackets[cp.cp]
+            return unless mirroring || brackets
+
+            cp.bidi ||= Ucode::Models::CodePoint::Bidi.new
+            apply_mirroring(cp, mirroring) if mirroring
+            apply_brackets(cp, brackets) if brackets
+          end
+
+          private
+
+          def apply_mirroring(cp, mirroring)
+            cp.bidi.mirroring_glyph_id = mirroring.mirrored_id
+          end
+
+          def apply_brackets(cp, brackets)
+            cp.bidi.paired_bracket_type = brackets.type
+            cp.bidi.paired_bracket_id = brackets.paired_id
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ucode/coordinator/enrichment/binary.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Ucode
+  class Coordinator
+    module Enrichment
+      # Binary properties: DerivedCoreProperties (core) + PropList (extra).
+      # Both contribute to the same `cp.binary_properties` array.
+      module Binary
+        class << self
+          def enrich(cp, indices)
+            assign_core(cp, indices)
+            assign_extra(cp, indices)
+          end
+
+          private
+
+          def assign_core(cp, indices)
+            records = indices.binary_properties[cp.cp]
+            return unless records && !records.empty?
+
+            cp.binary_properties = records.map(&:property_short)
+          end
+
+          # PropList carries binary properties beyond DerivedCoreProperties
+          # (White_Space, Hyphen, Variation_Selector, etc.). Merge into the
+          # same binary_properties list, deduped.
+          def assign_extra(cp, indices)
+            extras = RangeLookup.all_range_values(cp.cp, indices.extra_binary_properties)
+            return if extras.empty?
+
+            cp.binary_properties.concat(extras)
+            cp.binary_properties.uniq!
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ucode/coordinator/enrichment/casing.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Ucode
+  class Coordinator
+    module Enrichment
+      # Context-sensitive case mappings and case folding for comparison.
+      module Casing
+        class << self
+          def enrich(cp, indices)
+            assign_special_casing(cp, indices)
+            assign_case_folding(cp, indices)
+          end
+
+          private
+
+          # NOTE: do not uniq the *_ids arrays — a mapping like U+00DF → "SS"
+          # legitimately contains two U+0053 entries and they must be
+          # preserved in order. Conditions, by contrast, are categorical
+          # tags (Final_Sigma, tr, After_I) and deduping them is correct.
+          def assign_special_casing(cp, indices)
+            rules = indices.special_casing[cp.cp]
+            return unless rules && !rules.empty?
+
+            cp.casing ||= Ucode::Models::CodePoint::Casing.new
+            apply_casing_rules(cp.casing, rules)
+          end
+
+          def apply_casing_rules(casing, rules)
+            casing.full_upper_ids = rules.flat_map(&:upper_ids)
+            casing.full_lower_ids = rules.flat_map(&:lower_ids)
+            casing.full_title_ids = rules.flat_map(&:title_ids)
+            casing.conditions = rules.flat_map(&:conditions).uniq
+          end
+
+          def assign_case_folding(cp, indices)
+            rules = indices.case_folding[cp.cp]
+            return unless rules && !rules.empty?
+
+            cp.case_folding ||= Ucode::Models::CodePoint::CaseFolding.new
+            rules.each { |rule| apply_folding_rule(cp, rule) }
+          end
+
+          def apply_folding_rule(cp, rule)
+            case rule.status
+            when "C" then cp.case_folding.common_id = rule.mapping_ids.first
+            when "S" then cp.case_folding.simple_id = rule.mapping_ids.first
+            when "T" then cp.case_folding.turkic_id = rule.mapping_ids.first
+            when "F" then cp.case_folding.full_ids = rule.mapping_ids
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ucode/coordinator/enrichment/cjk.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Ucode
+  class Coordinator
+    module Enrichment
+      # CJK-specific data: Unihan readings, KangXi radical mapping,
+      # Hangul syllable type.
+      module CJK
+        class << self
+          def enrich(cp, indices)
+            assign_unihan(cp, indices)
+            assign_cjk_radical(cp, indices)
+            assign_hangul(cp, indices)
+          end
+
+          private
+
+          def assign_unihan(cp, indices)
+            entry = indices.unihan[cp.cp]
+            return unless entry
+
+            cp.unihan = entry
+          end
+
+          def assign_cjk_radical(cp, indices)
+            radicals = indices.cjk_radicals[cp.id]
+            return unless radicals && !radicals.empty?
+
+            radicals.each do |radical|
+              cp.relationships << Ucode::Models::Relationship::CrossReference.new(
+                target_ids: [radical.cjk_radical_id],
+                description: "KangXi radical ##{radical.radical_number}",
+                source: "cjk_radicals",
+              )
+            end
+          end
+
+          def assign_hangul(cp, indices)
+            tuple = RangeLookup.find_in_range(cp.cp, indices.hangul_syllable_type)
+            return unless tuple
+
+            cp.hangul ||= Ucode::Models::CodePoint::HangulSyllable.new
+            cp.hangul.type = tuple.value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ucode/coordinator/enrichment/display.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Ucode
+  class Coordinator
+    module Enrichment
+      # Display layout properties: Line Break class, East Asian Width,
+      # Vertical Orientation. All three are range+value files, looked up
+      # via bsearch on sorted arrays of ExtractedProperties::Tuple.
+      module Display
+        class << self
+          def enrich(cp, indices)
+            lb = lookup_value(cp, indices.line_break)
+            eaw = lookup_value(cp, indices.east_asian_width)
+            vo = lookup_value(cp, indices.vertical_orientation)
+            return if lb.nil? && eaw.nil? && vo.nil?
+
+            cp.display ||= Ucode::Models::CodePoint::Display.new
+            apply_values(cp.display, lb, eaw, vo)
+          end
+
+          private
+
+          def lookup_value(cp, ranges)
+            RangeLookup.find_in_range(cp.cp, ranges)&.value
+          end
+
+          def apply_values(display, lb, eaw, vo)
+            display.line_break_class = lb if lb
+            display.east_asian_width = eaw if eaw
+            display.vertical_orientation = vo if vo
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ucode/coordinator/enrichment/emoji.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Ucode
+  class Coordinator
+    module Enrichment
+      # Emoji property bundle. Each Emoji_* property from emoji-data.txt
+      # flips the matching boolean on the Emoji sub-model.
+      module Emoji
+        class << self
+          def enrich(cp, indices)
+            return unless RangeLookup.find_in_range(cp.cp, indices.emoji_properties)
+
+            props = RangeLookup.all_range_values(cp.cp, indices.emoji_properties)
+            return if props.empty?
+
+            cp.emoji ||= Ucode::Models::CodePoint::Emoji.new
+            props.each { |prop| apply_property(cp, prop) }
+          end
+
+          private
+
+          def apply_property(cp, prop)
+            case prop
+            when "Emoji"                 then cp.emoji.is_emoji = true
+            when "Emoji_Presentation"    then cp.emoji.is_presentation_default = true
+            when "Emoji_Modifier"        then cp.emoji.is_modifier = true
+            when "Emoji_Modifier_Base"   then cp.emoji.is_base = true
+            when "Emoji_Component"       then cp.emoji.is_component = true
+            when "Extended_Pictographic" then cp.emoji.is_extended_pictographic = true
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ucode/coordinator/enrichment/identity.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Ucode
+  class Coordinator
+    module Enrichment
+      # Basic identity properties: primary script, script extensions,
+      # and the Unicode version when the codepoint was introduced.
+      module Identity
+        class << self
+          def enrich(cp, indices)
+            assign_script(cp, indices)
+            assign_script_extensions(cp, indices)
+            assign_age(cp, indices)
+          end
+
+          private
+
+          def assign_script(cp, indices)
+            script = RangeLookup.find_in_range(cp.cp, indices.scripts)
+            return unless script
+
+            cp.script_code = script.code || script.name
+          end
+
+          def assign_script_extensions(cp, indices)
+            tuples = indices.script_extensions[cp.cp]
+            return unless tuples && !tuples.empty?
+
+            tuples.each { |tuple| cp.script_extensions << tuple.script_code }
+          end
+
+          def assign_age(cp, indices)
+            record = indices.derived_age[cp.cp]
+            return unless record
+
+            cp.age = record.age
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ucode/coordinator/enrichment/indic.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Ucode
+  class Coordinator
+    module Enrichment
+      # Indic script shaping categories: positional and syllabic.
+      module Indic
+        class << self
+          def enrich(cp, indices)
+            positional = lookup_value(cp, indices.indic_positional)
+            syllabic = lookup_value(cp, indices.indic_syllabic)
+            return if positional.nil? && syllabic.nil?
+
+            cp.indic ||= Ucode::Models::CodePoint::Indic.new
+            apply_values(cp.indic, positional, syllabic)
+          end
+
+          private
+
+          def lookup_value(cp, ranges)
+            RangeLookup.find_in_range(cp.cp, ranges)&.value
+          end
+
+          def apply_values(indic, positional, syllabic)
+            indic.positional_category = positional if positional
+            indic.syllabic_category = syllabic if syllabic
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ucode/coordinator/enrichment/names.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Ucode
+  class Coordinator
+    module Enrichment
+      # Human-curated name annotations: cross-references, informal aliases,
+      # footnotes from NamesList.txt; formal name aliases from
+      # NameAliases.txt; standardized variation sequences.
+      module Names
+        class << self
+          def enrich(cp, indices)
+            assign_names_list(cp, indices)
+            assign_name_aliases(cp, indices)
+            assign_standardized_variants(cp, indices)
+          end
+
+          private
+
+          def assign_names_list(cp, indices)
+            entry = indices.names_list[cp.cp]
+            return unless entry
+
+            cp.names_list = entry
+            cp.relationships.concat(entry.cross_references)
+            cp.relationships.concat(entry.sample_sequences)
+            cp.relationships.concat(entry.compatibility_equivalents)
+            cp.relationships.concat(entry.informal_aliases)
+            cp.relationships.concat(entry.footnotes)
+          end
+
+          def assign_name_aliases(cp, indices)
+            aliases = indices.name_aliases[cp.cp]
+            return unless aliases && !aliases.empty?
+
+            aliases.each do |alias_record|
+              cp.relationships << Ucode::Models::Relationship::InformalAlias.new(
+                description: alias_record.text,
+                source: "name_aliases",
+              )
+            end
+          end
+
+          def assign_standardized_variants(cp, indices)
+            variants = indices.standardized_variants[cp.id]
+            return unless variants && !variants.empty?
+
+            cp.standardized_variants = variants
+            variants.each { |v| add_variant_relationship(cp, v) }
+          end
+
+          def add_variant_relationship(cp, variant)
+            cp.relationships << Ucode::Models::Relationship::VariationSequence.new(
+              target_ids: [variant.base_id, variant.variation_selector_id],
+              description: variant.description,
+              contexts: variant.contexts,
+              source: "standardized_variants",
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ucode/coordinator/enrichment/segmentation.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Ucode
+  class Coordinator
+    module Enrichment
+      # UAX #29 text segmentation: Grapheme / Word / Sentence break class.
+      module Segmentation
+        class << self
+          def enrich(cp, indices)
+            grapheme = lookup_value(cp, indices.grapheme_break)
+            word = lookup_value(cp, indices.word_break)
+            sentence = lookup_value(cp, indices.sentence_break)
+            return if grapheme.nil? && word.nil? && sentence.nil?
+
+            cp.break_segmentation ||= Ucode::Models::CodePoint::BreakSegmentation.new
+            apply_values(cp.break_segmentation, grapheme, word, sentence)
+          end
+
+          private
+
+          def lookup_value(cp, ranges)
+            RangeLookup.find_in_range(cp.cp, ranges)&.value
+          end
+
+          def apply_values(seg, grapheme, word, sentence)
+            seg.grapheme = grapheme if grapheme
+            seg.word = word if word
+            seg.sentence = sentence if sentence
+          end
+        end
+      end
+    end
+  end
+end