ucode 0.1.0 → 0.1.1

data/lib/ucode/repo/block_feed_emitter.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+require "json"
+require "pathname"
+require "time"
+
+require "ucode/repo/atomic_writes"
+
+module Ucode
+  module Repo
+    # Emits a flat, per-block Unicode data feed from ucode's canonical
+    # output tree. The feed is a denormalized shape: each block file
+    # inlines all its codepoints (no joins needed at read time).
+    #
+    # Three files are emitted under `output_root`:
+    #
+    #   unicode-blocks.json
+    #     [{ start, end, name, unicode_version }, ...]
+    #
+    #   unicode/blocks/<slug>.json
+    #     { chars: [{ cp, n, c, s, cc?, bc?, mir? }, ...] }
+    #
+    #   unicode-version.json
+    #     { version, blockCount, charCount, generatedAt }
+    #
+    # This emitter reads ucode's canonical output (blocks/index.json,
+    # blocks/<ID>/index.json, index/labels.json) and translates shapes.
+    # ucode stays canonical; the feed is one-way derived.
+    #
+    # Block slug algorithm (matches common practice; no consumer
+    # assumptions baked in):
+    #
+    #   name.downcase.gsub(/[^a-z0-9]+/, "-").gsub(/^-|-$/, "")
+    #
+    # Block display name uses Unicode's verbatim spacing (e.g.
+    # "Basic Latin", "Greek and Coptic") from ucode's canonical name.
+    #
+    # The shape of this feed is documented in
+    # schema/block-feed.output.schema.yml — that YAML is the canonical
+    # contract for any consumer of the feed.
+    class BlockFeedEmitter
+      include AtomicWrites
+
+      # @param ucode_output_root [String, Pathname] ucode's `output/`
+      # @param output_root [String, Pathname] target directory;
+      #   `unicode-blocks.json`, `unicode-version.json`, and `unicode/`
+      #   are written here.
+      def initialize(ucode_output_root, output_root)
+        @ucode_root = Pathname.new(ucode_output_root)
+        @output_root = Pathname.new(output_root)
+      end
+
+      # @param ucd_version [String] e.g. "17.0.0"
+      # @return [Hash] { blocks_written:, codepoints_written:,
+      #   unicode_blocks_path:, unicode_version_path: }
+      def emit(ucd_version:)
+        labels = load_json(ucode_path("index", "labels.json"))
+        blocks_index = load_json(ucode_path("blocks", "index.json"))
+
+        per_block = blocks_index.map do |entry|
+          emit_block(entry, labels)
+        end
+
+        write_unicode_blocks(per_block)
+        version_payload = write_unicode_version(ucd_version, per_block)
+
+        {
+          blocks_written: per_block.length,
+          codepoints_written: per_block.sum { |b| b[:char_count] },
+          unicode_blocks_path: @output_root.join("unicode-blocks.json"),
+          unicode_version_path: @output_root.join("unicode-version.json"),
+          version: version_payload,
+        }
+      end
+
+      private
+
+      def emit_block(entry, labels)
+        block_id = entry["id"]
+        block_file = load_json(ucode_path("blocks", block_id, "index.json"))
+        chars = chars_for(block_file["codepoint_ids"] || [], labels)
+        slug = block_slug(entry["name"])
+
+        write_block_file(slug, chars)
+
+        {
+          slug: slug,
+          char_count: chars.length,
+          summary: {
+            "start" => entry["first_cp"],
+            "end" => entry["last_cp"],
+            "name" => entry["name"],
+            "unicode_version" => entry["age"] || block_file["age"] || "1.1",
+          },
+        }
+      end
+
+      def chars_for(codepoint_ids, labels)
+        codepoint_ids.map do |cp_id|
+          label = labels[cp_id] || {}
+          {
+            "cp" => cp_id_to_i(cp_id),
+            "n" => label["name"],
+            "c" => label["gc"],
+            "s" => label["sc"],
+            "cc" => label["cc"],
+            "bc" => label["bc"],
+            "mir" => label["mir"],
+          }.reject { |_, v| v.nil? || v == "" }
+        end
+      end
+
+      def write_block_file(slug, chars)
+        path = @output_root.join("unicode", "blocks", "#{slug}.json")
+        write_atomic(path, to_pretty_json("chars" => chars))
+      end
+
+      def write_unicode_blocks(per_block)
+        path = @output_root.join("unicode-blocks.json")
+        summaries = per_block.map { |b| b[:summary] }
+        write_atomic(path, to_pretty_json(summaries))
+      end
+
+      def write_unicode_version(ucd_version, per_block)
+        payload = {
+          "version" => ucd_version,
+          "blockCount" => per_block.length,
+          "charCount" => per_block.sum { |b| b[:char_count] },
+          "generatedAt" => Time.now.utc.iso8601,
+        }
+        path = @output_root.join("unicode-version.json")
+        write_atomic(path, to_pretty_json(payload))
+        payload
+      end
+
+      def block_slug(name)
+        name.downcase.gsub(/[^a-z0-9]+/, "-").gsub(/^-|-$/, "")
+      end
+
+      def cp_id_to_i(cp_id)
+        cp_id.to_s.sub(/^U\+/i, "").to_i(16)
+      end
+
+      def ucode_path(*parts)
+        @ucode_root.join(*parts)
+      end
+
+      def load_json(path)
+        JSON.parse(path.read)
+      end
+    end
+  end
+end

data/lib/ucode/repo/build_report_accumulator.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+require "time"
+
+require "ucode/models/build_report"
+
+module Ucode
+  module Repo
+    # Observes {CodepointWriter} and tallies per-tier + per-block
+    # statistics for the canonical build report (TODO 21).
+    #
+    # Wire as the `observer:` kwarg on {CodepointWriter}:
+    #
+    #   accumulator = BuildReportAccumulator.new(unicode_version: "17.0.0")
+    #   writer = CodepointWriter.new(root, resolver: resolver, observer: accumulator)
+    #   coordinator.each_codepoint { |cp| writer.write(cp) }
+    #   report = accumulator.to_report
+    #
+    # The accumulator is thread-safe — the writer's worker pool calls
+    # `#call` from multiple threads.
+    #
+    # == Semantics
+    #
+    # `assigned` counts every codepoint the writer attempted (passed
+    # the block_id guard). `built` counts codepoints whose resolver
+    # returned a glyph. `skipped` counts codepoints that resolved to
+    # nil (no tier produced a glyph). `failed` counts exceptions
+    # recorded via {#record_failure} (the writer rescues nothing;
+    # the orchestrating command decides what to surface).
+    #
+    # `by_tier` counts ONLY the winning tier per codepoint (not the
+    # overlap semantics mentioned in TODO 21's example). TODO 21
+    # notes the overlap counts as descriptive; the per-codepoint
+    # winning tier is the load-bearing number for validation.
+    class BuildReportAccumulator
+      TIER_TO_WIRE = {
+        tier1: "tier-1",
+        pillar1: "pillar-1",
+        pillar2: "pillar-2",
+        pillar3: "pillar-3",
+      }.freeze
+      private_constant :TIER_TO_WIRE
+
+      # @param unicode_version [String]
+      # @param ucode_version [String]
+      def initialize(unicode_version:, ucode_version:)
+        @unicode_version = unicode_version
+        @ucode_version = ucode_version
+        @totals = { assigned: 0, built: 0, skipped: 0, failed: 0 }
+        @by_tier = Hash.new(0)
+        @by_block = Hash.new do |h, name|
+          h[name] = { assigned: 0, built: 0, tier_breakdown: Hash.new(0) }
+        end
+        @failures = []
+        @mutex = Mutex.new
+      end
+
+      # Observer entry point — invoked by {CodepointWriter#write} as
+      # `observer.call(codepoint, result)`. Records one attempt.
+      #
+      # @param codepoint [Ucode::Models::CodePoint]
+      # @param result [Ucode::Glyphs::Source::Result, nil]
+      # @return [void]
+      def call(codepoint, result)
+        synchronize do
+          @totals[:assigned] += 1
+          block_stats = @by_block[codepoint.block_id]
+          block_stats[:assigned] += 1
+
+          if result
+            @totals[:built] += 1
+            tier = wire_tier(result.tier)
+            @by_tier[tier] += 1
+            block_stats[:built] += 1
+            block_stats[:tier_breakdown][tier] += 1
+          else
+            @totals[:skipped] += 1
+          end
+        end
+      end
+
+      # Record an exception encountered while building a codepoint.
+      # The orchestrating command calls this when rescuing around
+      # writer.write; the writer itself does not rescue.
+      #
+      # @param codepoint [Ucode::Models::CodePoint, nil]
+      # @param error [StandardError]
+      # @param tier [Symbol, nil] resolver tier that raised, if known
+      # @return [void]
+      def record_failure(codepoint, error, tier: nil)
+        synchronize do
+          @totals[:failed] += 1
+          @failures << Ucode::Models::BuildReport::Failure.new(
+            codepoint: codepoint&.cp,
+            block_name: codepoint&.block_id,
+            tier: tier&.to_s,
+            error_class: error.class.name,
+            message: error.message,
+            backtrace: Array(error.backtrace).first(10),
+          )
+        end
+      end
+
+      # Build the immutable {Ucode::Models::BuildReport} snapshot.
+      # @return [Ucode::Models::BuildReport]
+      def to_report
+        synchronize do
+          Ucode::Models::BuildReport.new(
+            unicode_version: @unicode_version,
+            ucode_version: @ucode_version,
+            generated_at: Time.now.utc.iso8601,
+            totals: Ucode::Models::BuildReport::Totals.new(@totals),
+            by_tier: @by_tier.dup,
+            by_block: @by_block.map do |name, stats|
+              Ucode::Models::BuildReport::BlockSummary.new(
+                name: name,
+                assigned: stats[:assigned],
+                built: stats[:built],
+                tier_breakdown: stats[:tier_breakdown].dup,
+              )
+            end,
+            failures: @failures.dup,
+          )
+        end
+      end
+
+      private
+
+      def wire_tier(symbol)
+        TIER_TO_WIRE.fetch(symbol, symbol.to_s)
+      end
+
+      def synchronize(&)
+        @mutex.synchronize(&)
+      end
+    end
+  end
+end

data/lib/ucode/repo/build_report_writer.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+require "ucode/repo/atomic_writes"
+require "ucode/repo/paths"
+
+module Ucode
+  module Repo
+    # Writes the canonical build report (TODO 21) to
+    # `output/build-report.json` atomically and idempotently.
+    #
+    # Re-running a build with no changed stats produces zero file
+    # writes — the existing build-report.json is byte-compared to the
+    # new payload before writing.
+    #
+    # The `generated_at` field is the only non-deterministic part of
+    # the report; callers wanting strict idempotency can override the
+    # accumulator's `to_report` to use a fixed timestamp.
+    class BuildReportWriter
+      include AtomicWrites
+
+      # @param output_root [String, Pathname]
+      def initialize(output_root)
+        @output_root = Pathname.new(output_root)
+      end
+
+      # @param report [Ucode::Models::BuildReport]
+      # @return [Pathname, nil] the path written, or nil if the
+      #   existing file was byte-identical (no-op).
+      def write(report)
+        path = @output_root.join("build-report.json")
+        payload = serialize(report)
+        return nil unless write_atomic(path, payload)
+
+        path
+      end
+
+      private
+
+      def serialize(report)
+        report.to_json(pretty: true)
+      end
+    end
+  end
+end

data/lib/ucode/repo/build_validator.rb

@@ -0,0 +1,229 @@
+# frozen_string_literal: true
+
+require "json"
+require "pathname"
+require "time"
+
+require "ucode/models"
+require "ucode/repo/atomic_writes"
+
+module Ucode
+  module Repo
+    # Walks an output tree produced by {CanonicalBuildCommand} and
+    # runs the four automated validation checks from TODO 21
+    # §Validation:
+    #
+    #   1. `completeness` — every codepoint folder has both
+    #      `index.json` and `glyph.svg`.
+    #   2. `schema` — every `index.json` deserializes via
+    #      `Ucode::Models::CodePoint.from_hash`.
+    #   3. `provenance_sanity` — every deserialized CodePoint carries
+    #      a non-nil `glyph.source.tier`.
+    #   4. `block_coverage` — per-block built count matches the
+    #      supplied baseline (status is `skipped` when no baseline).
+    #
+    # Sample inspection (check 5 in TODO 21) is manual and out of
+    # scope.
+    #
+    # The validator is stateless from the outside: one call to
+    # {#validate} walks the tree, builds a {ValidationReport}, and
+    # writes it atomically to `output/validation-report.json`. Safe
+    # to re-run on the same tree — idempotent via {AtomicWrites}.
+    #
+    # == Baseline shape
+    #
+    # `baseline:` is a `Hash{String block_name => Integer expected}`
+    # — the per-block built count expected from TODO 05's audit.
+    # Missing blocks in the baseline are ignored; blocks present in
+    # the output but absent from the baseline are not flagged (the
+    # baseline is authoritative only for what it covers).
+    class BuildValidator
+      include AtomicWrites
+
+      CHECK_COMPLETENESS = "completeness"
+      CHECK_SCHEMA = "schema"
+      CHECK_PROVENANCE = "provenance_sanity"
+      CHECK_BLOCK_COVERAGE = "block_coverage"
+      ALL_CHECKS = [
+        CHECK_COMPLETENESS, CHECK_SCHEMA, CHECK_PROVENANCE, CHECK_BLOCK_COVERAGE
+      ].freeze
+      private_constant :ALL_CHECKS
+
+      # @param output_root [String, Pathname]
+      # @param unicode_version [String, nil] stamped onto the report;
+      #   nil leaves the field blank (callers usually know the version).
+      # @param baseline [Hash{String=>Integer}, nil] per-block expected
+      #   built counts; when nil, the block_coverage check is skipped.
+      def initialize(output_root, unicode_version: nil, baseline: nil)
+        @output_root = Pathname.new(output_root)
+        @unicode_version = unicode_version
+        @baseline = baseline
+      end
+
+      # Walk the tree, run all checks, emit validation-report.json.
+      # @return [Hash] { report:, report_path:, passed: }
+      def validate
+        failures = []
+        per_block_counts = Hash.new(0)
+
+        each_codepoint_dir do |block_name, cp_id, cp_dir|
+          per_block_counts[block_name] += 1
+          validate_codepoint(block_name, cp_id, cp_dir, failures)
+        end
+
+        validate_block_coverage(per_block_counts, failures)
+
+        report = build_report(failures, per_block_counts)
+        report_path = write_report(report)
+        {
+          report: report,
+          report_path: report_path,
+          passed: report.totals.failures.zero?,
+        }
+      end
+
+      private
+
+      def each_codepoint_dir
+        blocks_path = @output_root.join("blocks")
+        return unless blocks_path.exist?
+
+        blocks_path.children.select(&:directory?).each do |block_dir|
+          block_name = block_dir.basename.to_s
+          block_dir.children.select(&:directory?).each do |cp_dir|
+            yield block_name, cp_dir.basename.to_s, cp_dir
+          end
+        end
+      end
+
+      def validate_codepoint(block_name, cp_id, cp_dir, failures)
+        index_path = cp_dir.join("index.json")
+        glyph_path = cp_dir.join(Paths.glyph_filename)
+        cp_int = parse_cp_int(cp_id)
+
+        unless index_path.exist?
+          failures << make_failure(cp_int, block_name, CHECK_COMPLETENESS,
+                                   "missing index.json")
+          return
+        end
+        unless glyph_path.exist?
+          failures << make_failure(cp_int, block_name, CHECK_COMPLETENESS,
+                                   "missing glyph.svg")
+        end
+
+        parsed = parse_index(index_path, cp_int, block_name, failures)
+        return unless parsed
+
+        check_provenance(parsed, cp_int, block_name, failures)
+      end
+
+      def parse_index(index_path, cp_int, block_name, failures)
+        hash = parse_json(index_path.read, cp_int, block_name, failures)
+        return nil unless hash
+
+        begin
+          Ucode::Models::CodePoint.from_hash(hash)
+        rescue StandardError => e
+          failures << make_failure(cp_int, block_name, CHECK_SCHEMA,
+                                   "deserialization failed: #{e.class}: #{e.message}")
+          nil
+        end
+      end
+
+      def parse_json(body, cp_int, block_name, failures)
+        JSON.parse(body)
+      rescue JSON::ParserError => e
+        failures << make_failure(cp_int, block_name, CHECK_SCHEMA,
+                                 "JSON parse failed: #{e.message}")
+        nil
+      end
+
+      def check_provenance(model, cp_int, block_name, failures)
+        return if model.glyph&.source&.tier
+
+        failures << make_failure(cp_int, block_name, CHECK_PROVENANCE,
+                                 "glyph.source.tier is missing")
+      end
+
+      def validate_block_coverage(per_block_counts, failures)
+        return if @baseline.nil?
+
+        @baseline.each do |block_name, expected|
+          actual = per_block_counts[block_name]
+          next if actual == expected
+
+          failures << make_failure(nil, block_name, CHECK_BLOCK_COVERAGE,
+                                   "expected #{expected} built, found #{actual}")
+        end
+      end
+
+      def build_report(failures, per_block_counts)
+        checks = ALL_CHECKS.map do |name|
+          build_check_summary(name, failures, per_block_counts)
+        end
+
+        Ucode::Models::ValidationReport.new(
+          unicode_version: @unicode_version.to_s,
+          generated_at: Time.now.utc.iso8601,
+          totals: Ucode::Models::ValidationReport::Totals.new(
+            codepoints_checked: per_block_counts.values.sum,
+            failures: failures.length,
+            checks_run: checks.count { |c| c.status != "skipped" },
+            checks_passed: checks.count { |c| c.status == "passed" },
+          ),
+          checks: checks,
+          failures: failures,
+        )
+      end
+
+      def build_check_summary(name, failures, per_block_counts)
+        count = failures.count { |f| f.check == name }
+        total = total_for_check(name, per_block_counts)
+
+        status = if name == CHECK_BLOCK_COVERAGE && @baseline.nil?
+                   "skipped"
+                 elsif count.zero?
+                   "passed"
+                 else
+                   "failed"
+                 end
+
+        Ucode::Models::ValidationReport::CheckSummary.new(
+          name: name,
+          status: status,
+          total: total,
+          failures: count,
+        )
+      end
+
+      def total_for_check(name, per_block_counts)
+        return @baseline&.length || 0 if name == CHECK_BLOCK_COVERAGE
+
+        per_block_counts.values.sum
+      end
+
+      def write_report(report)
+        path = @output_root.join("validation-report.json")
+        write_atomic(path, report.to_json(pretty: true))
+        path
+      end
+
+      def make_failure(cp_int, block_name, check, message)
+        Ucode::Models::ValidationReport::Failure.new(
+          codepoint: cp_int,
+          block: block_name,
+          check: check,
+          message: message,
+        )
+      end
+
+      def parse_cp_int(cp_id)
+        return nil unless cp_id.start_with?("U+")
+
+        Integer("0x#{cp_id[2..]}")
+      rescue ArgumentError
+        nil
+      end
+    end
+  end
+end

data/lib/ucode/repo/codepoint_writer.rb

@@ -22,15 +22,38 @@ module Ucode
     #   - **Atomic**: writes go to `<path>.tmp`, then rename. A crash
     #     mid-write leaves either the old file or no file, never a
     #     truncated one.
+    #
+    # When a {Ucode::Glyphs::Resolver} is supplied via `resolver:`, each
+    # write also resolves the codepoint's glyph, writes `glyph.svg`
+    # alongside `index.json` (same atomic + idempotent semantics), and
+    # records the resolver tier + provenance on the codepoint's `glyph`
+    # attribute so it lands in the serialized JSON. When `resolver:` is
+    # nil (default), the writer is glyph-agnostic and only writes
+    # `index.json` — preserving backward compatibility.
     class CodepointWriter
       include AtomicWrites
 
       # @param output_root [String, Pathname]
       # @param parallel_workers [Integer] size of the worker pool. Set to
       #   1 (or less) to run synchronously — useful in tests.
-      def initialize(output_root, parallel_workers: 8)
+      # @param resolver [Ucode::Glyphs::Resolver, nil] when non-nil, each
+      #   write resolves the codepoint's glyph via this resolver and
+      #   writes `glyph.svg` next to `index.json`. Sources inside the
+      #   resolver must be safe for concurrent access — the worker pool
+      #   calls into them from multiple threads.
+      # @param observer [#call, nil] when non-nil, invoked as
+      #   `observer.call(codepoint, result)` after each resolve attempt
+      #   (and before the JSON write). `result` is the
+      #   {Ucode::Glyphs::Source::Result} when a tier produced a glyph,
+      #   or nil when no resolver is configured / no tier matched. Used
+      #   by {Ucode::Repo::BuildReportAccumulator} to tally per-tier
+      #   stats. The observer must be thread-safe.
+      def initialize(output_root, parallel_workers: 8, resolver: nil,
+                     observer: nil)
         @output_root = Pathname.new(output_root)
         @parallel_workers = parallel_workers
+        @resolver = resolver
+        @observer = observer
       end
 
       # Write one codepoint synchronously.
@@ -38,6 +61,8 @@ module Ucode
       # @return [Pathname, nil] the path written, or nil if skipped
       #   (missing block_id or content-identical to existing file)
       def write(codepoint)
+        result = codepoint.block_id.nil? ? nil : resolve_glyph(codepoint)
+        @observer&.call(codepoint, result)
         return nil if codepoint.block_id.nil?
 
         path = Paths.codepoint_json_path(@output_root, codepoint.block_id, codepoint.id)
@@ -91,6 +116,30 @@ module Ucode
       def serialize(codepoint)
         codepoint.to_json(pretty: true)
       end
+
+      def resolve_glyph(codepoint)
+        return nil unless @resolver
+
+        result = @resolver.resolve(codepoint.cp)
+        codepoint.glyph = build_glyph_bundle(result)
+        return nil unless result
+
+        path = Paths.codepoint_glyph_path(@output_root, codepoint.block_id, codepoint.id)
+        write_atomic(path, result.svg)
+        result
+      end
+
+      def build_glyph_bundle(result)
+        return nil unless result
+
+        Ucode::Models::CodePoint::Glyph.new(
+          svg_path: Paths.glyph_filename,
+          source: Ucode::Models::CodePoint::Glyph::Source.new(
+            tier: result.tier.to_s,
+            provenance: result.provenance,
+          ),
+        )
+      end
     end
   end
 end

data/lib/ucode/repo/paths.rb

@@ -24,6 +24,14 @@ module Ucode
                        :PLANE_FILENAME_PREFIX
 
       class << self
+        # The fixed filename every codepoint's SVG glyph is written to
+        # (relative to the codepoint's own directory). Exposed so the
+        # Glyph model bundle records the same string the layout uses.
+        # @return [String]
+        def glyph_filename
+          GLYPH_FILENAME
+        end
+
         # Format an integer codepoint as the canonical "U+XXXX" id used
         # everywhere (paths, JSON, cross-references). Always at least
         # 4 hex digits, uppercase, no extra padding.

data/lib/ucode/repo.rb

@@ -18,5 +18,9 @@ module Ucode
     autoload :AtomicWrites, "ucode/repo/atomic_writes"
     autoload :CodepointWriter, "ucode/repo/codepoint_writer"
     autoload :AggregateWriter, "ucode/repo/aggregate_writer"
+    autoload :BuildReportAccumulator, "ucode/repo/build_report_accumulator"
+    autoload :BuildReportWriter, "ucode/repo/build_report_writer"
+    autoload :BuildValidator, "ucode/repo/build_validator"
+    autoload :BlockFeedEmitter, "ucode/repo/block_feed_emitter"
   end
 end

data/lib/ucode/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ucode
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end