ucode 0.1.0 → 0.1.1

data/lib/ucode/commands/canonical_build.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+require "ucode/cache"
+require "ucode/commands"
+require "ucode/coordinator"
+require "ucode/database"
+require "ucode/glyphs"
+require "ucode/models"
+require "ucode/repo"
+require "ucode/version"
+require "ucode/version_resolver"
+
+module Ucode
+  module Commands
+    # `ucode canonical-build` — Mode 1's canonical Unicode dataset build
+    # (TODO 21). Single pass: enrich each codepoint via {Coordinator},
+    # resolve its glyph via the 4-tier {Ucode::Glyphs::Resolver}, write
+    # `index.json` + `glyph.svg` atomically, accumulate per-tier +
+    # per-block stats, and emit `output/build-report.json`.
+    #
+    # This is the v0.2 replacement for the v0.1 cell-extractor pipeline
+    # in {GlyphsCommand}. The two coexist until the v0.1 pipeline is
+    # removed (TODOs 17-19); CanonicalBuildCommand is the path forward
+    # for production dataset runs.
+    #
+    # == Pre-conditions (per TODO 21)
+    #
+    # 1. UCD + Unihan fetched for `version` (`ucode fetch ucd`,
+    #    `ucode fetch unihan`).
+    # 2. Ucode::Database built for `version` (`ucode db build`).
+    # 3. Tier 1 fonts resolvable via the configured SourceConfig YAML.
+    # 4. Code Charts PDFs cached (for Pillar 1) — optional, only if
+    #    pillar-1 sources are configured.
+    # 5. Last Resort UFO cloned (for Pillar 3) — optional, only if
+    #    pillar-3 fallback is configured.
+    #
+    # Missing pre-conditions cause silent fallthrough to lower tiers;
+    # the build report's `by_tier` totals surface what ran.
+    class CanonicalBuildCommand
+      # @param version_intent [nil, :default, :latest, String]
+      # @param output_root [String, Pathname]
+      # @param source_config_path [String, Pathname, nil] override the
+      #   Tier 1 font config YAML; nil uses the default
+      #   (`config/unicode17_tier1_fonts.yml`).
+      # @param resolver [Ucode::Glyphs::Resolver, nil] inject a
+      #   pre-built resolver (skips SourceBuilder); used by tests.
+      # @param validate [Boolean] run {Ucode::Repo::BuildValidator}
+      #   after the build and emit `validation-report.json`. Default
+      #   true; tests that don't care about validation pass false.
+      # @param baseline [Hash{String=>Integer}, nil] per-block expected
+      #   built counts forwarded to the validator when `validate:` is
+      #   true. nil skips the block_coverage check.
+      # @return [Hash] { version:, codepoint_count:, report_path:,
+      #   validation_report_path:, validation_passed: }
+      def call(version_intent, output_root:, source_config_path: nil,
+               resolver: nil, validate: true, baseline: nil)
+        version = VersionResolver.resolve(version_intent)
+        root = Pathname.new(output_root)
+
+        resolved_resolver = resolver || build_resolver(version, source_config_path)
+        accumulator = Repo::BuildReportAccumulator.new(
+          unicode_version: version,
+          ucode_version: Ucode::VERSION,
+        )
+
+        coordinator = Coordinator.new
+        writer = Repo::CodepointWriter.new(
+          root,
+          parallel_workers: workers,
+          resolver: resolved_resolver,
+          observer: accumulator,
+        )
+
+        ucd_dir = Cache.ucd_dir(version)
+        unihan_dir = Cache.unihan_dir(version)
+        codepoint_count = iterate(coordinator, ucd_dir, unihan_dir, writer,
+                                  accumulator)
+
+        report = accumulator.to_report
+        report_path = Repo::BuildReportWriter.new(root).write(report)
+
+        result = {
+          version: version,
+          codepoint_count: codepoint_count,
+          report_path: report_path,
+          totals: report.totals.to_hash,
+        }
+        return result unless validate
+
+        merge_validation_result(result, root, version, baseline)
+      end
+
+      private
+
+      def merge_validation_result(result, root, version, baseline)
+        outcome = Repo::BuildValidator.new(
+          root, unicode_version: version, baseline: baseline,
+        ).validate
+        result.merge(
+          validation_report_path: outcome[:report_path],
+          validation_passed: outcome[:passed],
+        )
+      end
+
+      def workers
+        Ucode.configuration.parallel_workers
+      end
+
+      def iterate(coordinator, ucd_dir, unihan_dir, writer, accumulator)
+        count = 0
+        coordinator.each_codepoint(ucd_dir: ucd_dir, unihan_dir: unihan_dir) do |cp|
+          begin
+            writer.write(cp)
+          rescue StandardError => e
+            accumulator.record_failure(cp, e)
+          end
+          count += 1
+        end
+        count
+      end
+
+      def build_resolver(version, source_config_path)
+        database = Database.open(version)
+        config = Glyphs::SourceConfig.new(path: source_config_path_or_default(source_config_path))
+        builder = Glyphs::SourceBuilder.new(config: config, database: database)
+        Glyphs::Resolver.new(sources: builder.tier1_sources(install: false))
+      end
+
+      def source_config_path_or_default(path)
+        return Glyphs::SourceConfig::DEFAULT_PATH if path.nil?
+
+        Pathname.new(path)
+      end
+    end
+  end
+end

data/lib/ucode/commands/fetch.rb

@@ -4,17 +4,23 @@ require "pathname"
 
 require "ucode/cache"
 require "ucode/fetch"
+require "ucode/glyphs/source_config"
 require "ucode/version_resolver"
 
 module Ucode
   module Commands
     # `ucode fetch` — downloads UCD/Unihan/Code-Charts sources into the
-    # per-version cache. Three subactions: ucd, unihan, charts.
+    # per-version cache, plus the specialist Tier 1 fonts referenced by
+    # the curated source config.
     #
     # Thin shell over `Ucode::Fetch::*`. The command layer's job is to
     # resolve the version intent and format the result; the fetcher does
     # the network I/O.
     class FetchCommand
+      DEFAULT_SPECIALIST_FONTS_MANIFEST =
+        Ucode::Glyphs::SourceConfig::DEFAULT_PATH.dirname.join("specialist_fonts.yml")
+      private_constant :DEFAULT_SPECIALIST_FONTS_MANIFEST
+
       # @param version_intent [nil, :default, :latest, String]
       # @param force [Boolean]
       # @return [Hash] { version:, ucd_dir: }
@@ -48,6 +54,36 @@ module Ucode
         { version: version, downloaded: count }
       end
 
+      # Fetch specialist Tier 1 fonts listed in the manifest. Returns
+      # a structured summary; per-font detail lives on the returned
+      # results array (one {Fetch::FontFetcher::Result} per entry).
+      #
+      # @param manifest_path [String, Pathname, nil] defaults to
+      #   `config/specialist_fonts.yml`.
+      # @param only_label [String, nil] restrict to one label.
+      # @param allow_proprietary [Boolean] required for non-OFL entries.
+      # @param dry_run [Boolean] plan only; no network or disk writes.
+      # @return [Hash] { manifest:, total:, downloaded:, skipped:,
+      #   failed:, local:, planned:, results: }
+      def fetch_fonts(manifest_path: nil, only_label: nil, allow_proprietary: false,
+                      dry_run: false)
+        path = Pathname.new(manifest_path || DEFAULT_SPECIALIST_FONTS_MANIFEST)
+        results = Fetch::SpecialistFontFetcher.new(
+          manifest_path: path,
+          allow_proprietary: allow_proprietary,
+          dry_run: dry_run,
+        ).call(only_label: only_label)
+
+        { manifest: path.to_s,
+          total: results.size,
+          downloaded: results.count(&:downloaded?),
+          skipped: results.count(&:skipped?),
+          failed: results.count(&:failed?),
+          local: results.count(&:local?),
+          planned: results.count(&:planned?),
+          results: results }
+      end
+
       private
 
       def default_block_first_cps(version)

data/lib/ucode/commands/release.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "time"
+
+require "ucode/audit"
+require "ucode/audit/library_auditor"
+require "ucode/audit/release"
+require "ucode/audit/emitter/paths"
+require "ucode/commands/audit/reference_builder"
+
+module Ucode
+  module Commands
+    # `ucode release` — assemble the fontist.org-consumable release
+    # tree (TODO 27).
+    #
+    # Walks a directory of per-formula font subdirectories, audits each
+    # via {Audit::LibraryAuditor}, and passes the resulting
+    # {Audit::Release::FormulaAudits} list to {Audit::Release::Emitter}.
+    #
+    # The release tree lives at `<output_root>/font_audit_release/`.
+    # The CI collector job invokes this after matrix-auditing every
+    # formula and pre-staging the universal-set directory.
+    class ReleaseCommand
+      FormulaSource = Struct.new(:slug, :path, keyword_init: true)
+
+      Result = Struct.new(:release_root, :formulas_total, :faces_total,
+                          :formulas, :universal_set_available,
+                          :library_index_written, :manifest_written,
+                          :error, keyword_init: true)
+
+      # @param from [String, Pathname] directory containing one
+      #   subdirectory per formula. Each subdirectory's name becomes
+      #   the formula slug; its contents are audited via
+      #   {Audit::LibraryAuditor} (recursive walk).
+      # @param output_root [String, Pathname] parent of the release
+      #   root. Release tree lives at
+      #   `<output_root>/font_audit_release/`.
+      # @param universal_set_root [String, Pathname, nil] location of
+      #   the universal_glyph_set directory. Defaults to
+      #   `<release_root>/universal_glyph_set` inside the release tree.
+      # @param unicode_version [String, nil] baseline UCD version.
+      # @param recursive [Boolean] recursively walk each formula
+      #   subdirectory. Default true.
+      # @param brief [Boolean] cheap-extractor-only audit mode.
+      # @param browse [Boolean] also emit per-face HTML browsers.
+      # @param source_config_sha256 [String, nil] sha256 of the Tier 1
+      #   source-config YAML (TODO 23). Recorded in the manifest for
+      #   curation provenance.
+      # @param reference [Audit::CoverageReference, nil] baseline
+      #   forwarded to every per-face audit (TODO 25).
+      # @param generated_at [String] ISO8601 timestamp. Default: now.
+      # @return [Result]
+      def call(from:, output_root:, universal_set_root: nil, unicode_version: nil,
+               recursive: true, brief: false, browse: true,
+               source_config_sha256: nil, reference: nil,
+               generated_at: Time.now.utc.iso8601)
+        formula_sources = discover_formulas(from)
+        formulas = formula_sources.map do |src|
+          summary = audit_formula(src.path, recursive: recursive,
+                                            unicode_version: unicode_version,
+                                            brief: brief, reference: reference)
+          Ucode::Audit::Release::FormulaAudits.new(slug: src.slug, summary: summary)
+        end
+
+        emitter = Ucode::Audit::Release::Emitter.new(
+          output_root: output_root,
+          universal_set_root: universal_set_root,
+          with_missing_glyph_pages: browse,
+        )
+        emit_result = emitter.emit(
+          formulas: formulas,
+          unicode_version: unicode_version,
+          generated_at: generated_at,
+          source_config_sha256: source_config_sha256,
+        )
+
+        Result.new(
+          release_root: emit_result.release_root,
+          formulas_total: emit_result.formulas_total,
+          faces_total: emit_result.faces_total,
+          formulas: formula_sources,
+          universal_set_available: emit_result.universal_set_available,
+          library_index_written: emit_result.library_index_written,
+          manifest_written: emit_result.manifest_written,
+        )
+      rescue StandardError => e
+        Result.new(error: "#{e.class}: #{e.message}")
+      end
+
+      private
+
+      def discover_formulas(from)
+        Pathname.new(from).children.select(&:directory?).sort.map do |d|
+          FormulaSource.new(slug: d.basename.to_s, path: d.to_s)
+        end
+      end
+
+      def audit_formula(path, recursive:, unicode_version:, brief:, reference:)
+        options = audit_options(unicode_version: unicode_version, brief: brief)
+        auditor = Ucode::Audit::LibraryAuditor.new(
+          path, recursive: recursive, options: options, reference: reference
+        )
+        auditor.audit
+      end
+
+      def audit_options(unicode_version:, brief:)
+        opts = {}
+        opts[:ucd_version] = unicode_version if unicode_version
+        opts[:audit_brief] = true if brief
+        opts
+      end
+    end
+  end
+end

data/lib/ucode/commands/universal_set.rb

@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+
+require "digest"
+require "json"
+require "pathname"
+
+require "ucode/cache"
+require "ucode/commands"
+require "ucode/coordinator"
+require "ucode/database"
+require "ucode/glyphs"
+require "ucode/models"
+require "ucode/version"
+require "ucode/version_resolver"
+
+module Ucode
+  module Commands
+    # `ucode universal-set` subcommands (TODOs 24 + 31):
+    #
+    # - `BuildCommand` — drain the codepoint stream through the
+    #   4-tier resolver and write glyphs + manifest. Runs
+    #   {PreBuildCheck} first; emits {CoverageReport} after.
+    # - `PreCheckCommand` — standalone pre-build validation
+    #   (config + fonts + coverage assertion).
+    # - `ReportCommand` — re-emit coverage reports from an
+    #   existing manifest. Useful for iterating on curation without
+    #   re-running the build.
+    # - `ValidateCommand` — post-build structural validation
+    #   (manifest parses, every entry has a glyph, totals reconcile,
+    #   provenance recorded).
+    #
+    # The set is the canonical reference for "what Unicode 17 looks
+    # like" — every assigned codepoint has exactly one glyph, with
+    # documented provenance. Audits (TODO 25) and the fontist.org
+    # consumer (TODO 27) read the manifest to answer "is this
+    # codepoint in the universal set?" without re-reading every SVG.
+    module UniversalSet
+      # `ucode universal-set build` action class. Pure Ruby — Thor
+      # (in `lib/ucode/cli.rb`) is responsible only for argument
+      # parsing and dispatch.
+      class BuildCommand
+        # @param version_intent [nil, :default, :latest, String]
+        # @param output_root [String, Pathname] directory that will
+        #   hold `manifest.json`, `glyphs/`, `reports/`.
+        # @param source_config_path [String, Pathname, nil] override
+        #   the Tier 1 font config YAML; nil uses the default at
+        #   `Ucode::Glyphs::SourceConfig::DEFAULT_PATH`.
+        # @param resolver [Ucode::Glyphs::Resolver, nil] inject a
+        #   pre-built resolver (skips SourceBuilder + PreBuildCheck);
+        #   used by tests.
+        # @param block_filter [String, nil] limit the build to one
+        #   block (canonical underscore form). Useful for partial
+        #   rebuilds when iterating on Tier 1 curation.
+        # @param parallel_workers [Integer] forwarded to the Builder.
+        #   Defaults to {Ucode::Configuration#parallel_workers}.
+        # @param skip_pre_check [Boolean] when true, skip the
+        #   {PreBuildCheck} step. Used by tests that inject a custom
+        #   resolver and don't have a real source config on disk.
+        # @return [Hash] { version:, manifest_path:, totals:,
+        #   by_tier:, coverage:, validation: }
+        def call(version_intent, output_root:, source_config_path: nil,
+                 resolver: nil, block_filter: nil,
+                 parallel_workers: default_workers, skip_pre_check: false)
+          version = VersionResolver.resolve(version_intent)
+          root = Pathname.new(output_root)
+
+          config_path = source_config_path_or_default(source_config_path)
+          sha = source_config_sha256(config_path)
+          database = Database.open(version)
+
+          run_pre_check(config_path, database) unless skip_pre_check
+
+          resolved_resolver = resolver || build_resolver(version, config_path, database)
+
+          builder = Glyphs::UniversalSet::Builder.new(
+            output_root: root,
+            resolver: resolved_resolver,
+            unicode_version: version,
+            ucode_version: Ucode::VERSION,
+            source_config_sha256: sha,
+            parallel_workers: parallel_workers,
+            block_filter: block_filter,
+          )
+
+          manifest_path = builder.build(codepoint_enum(version))
+
+          manifest = Ucode::Models::UniversalSetManifest.from_hash(
+            JSON.parse(manifest_path.read),
+          )
+          coverage = Glyphs::UniversalSet::CoverageReport
+            .new(root, database: database).emit(manifest)
+          validation = Glyphs::UniversalSet::Validator
+            .new(root, unicode_version: version).validate
+          {
+            version: version,
+            manifest_path: manifest_path,
+            totals: manifest.totals.to_hash,
+            by_tier: manifest.by_tier,
+            coverage: coverage,
+            validation: validation,
+          }
+        end
+
+        private
+
+        def default_workers
+          Ucode.configuration.parallel_workers
+        end
+
+        def source_config_path_or_default(path)
+          return Glyphs::SourceConfig::DEFAULT_PATH if path.nil?
+
+          Pathname.new(path)
+        end
+
+        def source_config_sha256(path)
+          return "" unless path.exist?
+
+          Digest::SHA256.file(path).hexdigest
+        end
+
+        def run_pre_check(config_path, database)
+          Glyphs::UniversalSet::PreBuildCheck.new(
+            source_config_path: config_path,
+            database: database,
+          ).call
+        end
+
+        def build_resolver(_version, config_path, database)
+          config = Glyphs::SourceConfig.new(path: config_path)
+          builder = Glyphs::SourceBuilder.new(config: config, database: database)
+          Glyphs::Resolver.new(sources: builder.tier1_sources(install: false))
+        end
+
+        def codepoint_enum(version)
+          ucd_dir = Cache.ucd_dir(version)
+          unihan_dir = Cache.unihan_dir(version)
+          Coordinator.new.each_codepoint(ucd_dir: ucd_dir, unihan_dir: unihan_dir)
+        end
+      end
+
+      # `ucode universal-set pre-check` — standalone pre-build
+      # validation. Runs the three TODO 31 §Pre-build validation
+      # checks (config loads, fonts present, coverage assertion runs)
+      # without starting the 4-hour build.
+      class PreCheckCommand
+        # @param version_intent [nil, :default, :latest, String]
+        # @param source_config_path [String, Pathname, nil]
+        # @param cmaps [#covers?] injectable; defaults to
+        #   RealFonts::CmapCache.
+        # @param font_locator [#locate] injectable; defaults to a
+        #   fresh FontLocator.
+        # @return [Ucode::Glyphs::UniversalSet::PreBuildReport]
+        # @raise [Ucode::UniversalSetPreBuildError] when missing_fonts
+        #   is non-empty or the config fails to load.
+        def call(version_intent, source_config_path: nil, cmaps: nil,
+                 font_locator: nil)
+          version = VersionResolver.resolve(version_intent)
+          database = Database.open(version)
+          config_path = source_config_path || Glyphs::SourceConfig::DEFAULT_PATH
+
+          kwargs = { source_config_path: config_path, database: database }
+          kwargs[:cmaps] = cmaps if cmaps
+          kwargs[:font_locator] = font_locator if font_locator
+          Glyphs::UniversalSet::PreBuildCheck.new(**kwargs).call
+        end
+      end
+
+      # `ucode universal-set report` — re-emit coverage reports from
+      # an existing manifest. Useful when iterating on the manifest
+      # shape (or regenerating reports after a model change) without
+      # re-running the build.
+      class ReportCommand
+        # @param version_intent [nil, :default, :latest, String]
+        # @param output_root [String, Pathname] directory holding
+        #   `manifest.json`.
+        # @return [Hash] the {CoverageReport#emit} payload.
+        def call(version_intent, output_root:)
+          version = VersionResolver.resolve(version_intent)
+          root = Pathname.new(output_root)
+          manifest_path = root.join("manifest.json")
+          raise Ucode::Error, "manifest not found at #{manifest_path}" unless manifest_path.exist?
+
+          manifest = Ucode::Models::UniversalSetManifest.from_hash(
+            JSON.parse(manifest_path.read),
+          )
+          database = Database.open(version)
+          Glyphs::UniversalSet::CoverageReport.new(root, database: database)
+            .emit(manifest)
+        end
+      end
+
+      # `ucode universal-set validate` — post-build structural
+      # validation. Reads `manifest.json` + `glyphs/` and runs the
+      # four checks (manifest_loadable, glyph_files_present,
+      # totals_reconcile, provenance_complete).
+      class ValidateCommand
+        # @param output_root [String, Pathname]
+        # @param version_intent [nil, :default, :latest, String]
+        #   used only to stamp the report's unicode_version when the
+        #   manifest's recorded value is missing.
+        # @return [Hash] the {Validator#validate} payload.
+        def call(output_root, version_intent: nil)
+          version = version_intent && VersionResolver.resolve(version_intent)
+          Glyphs::UniversalSet::Validator
+            .new(output_root, unicode_version: version).validate
+        end
+      end
+    end
+  end
+end

data/lib/ucode/commands.rb

@@ -14,6 +14,11 @@ module Ucode
     autoload :LookupCommand, "ucode/commands/lookup"
     autoload :CacheCommand, "ucode/commands/cache"
     autoload :BuildCommand, "ucode/commands/build"
+    autoload :CanonicalBuildCommand, "ucode/commands/canonical_build"
     autoload :FontCoverageCommand, "ucode/commands/font_coverage"
+    autoload :UniversalSet, "ucode/commands/universal_set"
+    autoload :Audit, "ucode/commands/audit"
+    autoload :ReleaseCommand, "ucode/commands/release"
+    autoload :BlockFeedCommand, "ucode/commands/block_feed"
   end
 end

data/lib/ucode/coordinator/indices.rb

@@ -28,6 +28,17 @@ module Ucode
       :standardized_variants,
       :names_list,
       :unihan,
+      :line_break,
+      :east_asian_width,
+      :vertical_orientation,
+      :grapheme_break,
+      :word_break,
+      :sentence_break,
+      :indic_positional,
+      :indic_syllabic,
+      :hangul_syllable_type,
+      :emoji_properties,
+      :extra_binary_properties,
       keyword_init: true,
     )
   end