ucode 0.2.0 → 0.2.1

data/lib/ucode/code_chart/writer.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require "digest"
+require "pathname"
+
+require "ucode/code_chart/extractor"
+require "ucode/code_chart/provenance"
+require "ucode/code_chart/sidecar"
+require "ucode/error"
+require "ucode/version_resolver"
+
+module Ucode
+  module CodeChart
+    # Orchestrates extraction + provenance sidecar writing for one
+    # block. The Writer is the **only thing that touches disk** in the
+    # CodeChart namespace; everything else is composition.
+    #
+    # Output layout (per block):
+    #
+    #     <output_root>/<block_id>/<U+XXXX>.svg
+    #     <output_root>/<block_id>/<U+XXXX>.json   # provenance sidecar
+    #
+    # One folder per block keeps each block's output self-contained
+    # and discoverable — a downstream consumer (fontisan) can iterate
+    # a block's folder without scanning the whole tree.
+    #
+    # Idempotent: re-running `write` on the same inputs produces
+    # byte-identical files (SVGs via content check; sidecars via
+    # {Ucode::Repo::AtomicWrites#write_atomic}'s canonical-JSON
+    # byte-equality). The {Summary} tally distinguishes "first run"
+    # writes from no-op re-writes.
+    class Writer
+      # Per-block run summary. Returned from {#write}.
+      Summary = Struct.new(
+        :block,
+        :codepoints_extracted,
+        :svgs_written,
+        :sidecars_written,
+        :pdf_sha256,
+        keyword_init: true,
+      )
+
+      # @param output_root [Pathname, String] parent directory. The
+      #   `<block_id>/` subdirectory is created inside it.
+      # @param pdf_path [Pathname, String] Code Charts PDF (already
+      #   downloaded by the caller; Writer doesn't fetch).
+      # @param ucd_version [String, nil] UCD version to stamp on
+      #   provenance. nil = resolved via {VersionResolver.resolve(nil)}.
+      # @param cache_dir [Pathname, String, nil] font-stream cache
+      #   directory for the EmbeddedFonts::Source.
+      # @param now [Time, nil] timestamp override (for tests).
+      # @param pillar3_source, tier1_sources: forwarded to the Extractor.
+      def initialize(output_root:, pdf_path:, ucd_version: nil,
+                     cache_dir: nil, now: nil,
+                     pillar3_source: nil, tier1_sources: nil)
+        @output_root = Pathname.new(output_root)
+        @pdf_path = Pathname.new(pdf_path)
+        @ucd_version = ucd_version || VersionResolver.resolve(nil)
+        @cache_dir = cache_dir && Pathname.new(cache_dir)
+        @now = now
+        @pillar3_source = pillar3_source
+        @tier1_sources = tier1_sources
+      end
+
+      # Extracts every codepoint in `block` and writes `<block_id>/<cp>.svg`
+      # + `<block_id>/<cp>.json` under `@output_root`. Returns a
+      # {Summary} tally.
+      #
+      # @param block [Ucode::Models::Block]
+      # @return [Summary]
+      def write(block)
+        block_dir = @output_root.join(block.id)
+        block_dir.mkpath
+
+        pdf_sha = CodeChart.sha256_of(@pdf_path)
+
+        sidecar = Sidecar.new(output_root: block_dir)
+        extractor = Extractor.new(
+          block: block,
+          pdf_path: @pdf_path,
+          cache_dir: @cache_dir,
+          pillar3_source: @pillar3_source,
+          tier1_sources: @tier1_sources,
+        )
+
+        results = extractor.extract
+        svgs = 0
+        sidecars = 0
+        results.each do |result|
+          write_svg(block_dir, result)
+          svgs += 1
+          provenance = CodeChart.build(
+            block: block, codepoint: result.codepoint,
+            ucd_version: @ucd_version, pdf_path: @pdf_path,
+            now: @now,
+          )
+          sidecar.write(provenance)
+          sidecars += 1
+        end
+
+        Summary.new(
+          block: block.id,
+          codepoints_extracted: results.size,
+          svgs_written: svgs,
+          sidecars_written: sidecars,
+          pdf_sha256: pdf_sha,
+        )
+      end
+
+      private
+
+      # Writes one SVG, skipping the write when the existing content
+      # is byte-identical (so mtime is preserved on idempotent
+      # re-runs — the Sidecar uses `Repo::AtomicWrites` for the same
+      # reason but at a different layer).
+      def write_svg(block_dir, result)
+        path = block_dir.join("#{format_cp(result.codepoint)}.svg")
+        return if path.exist? && path.read == result.svg
+
+        path.write(result.svg)
+      end
+
+      def format_cp(codepoint)
+        "U+#{codepoint.to_s(16).upcase.rjust(4, '0')}"
+      end
+    end
+  end
+end

data/lib/ucode/code_chart.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Ucode
+  # CodeChart — per-codepoint SVG glyph extraction from Unicode Code
+  # Charts PDFs.
+  #
+  # The "Code Chart donor" use case (essenfont consumer): for blocks
+  # where no OFL real-font covers the glyphs (Sidetic in Unicode 17,
+  # Egyptian Hieroglyphs Extended-B), the only canonical source is
+  # the Unicode Consortium's Code Chart PDF. This namespace turns one
+  # such PDF into a tree of standalone SVG files plus provenance
+  # sidecar JSON.
+  #
+  # ## Architecture (MECE)
+  #
+  # Every concern has exactly one home:
+  #
+  #   * **Block metadata** (range + assigned codepoints) — Parsers::Blocks
+  #   * **PDF download + cache** — Fetch::CodeCharts + Glyphs::PdfFetcher
+  #   * **PDF object-graph walk + font extraction** — Glyphs::EmbeddedFonts::*
+  #   * **Tier selection (Pillar 1 / 2 / 3)** — Glyphs::Resolver
+  #   * **SVG conversion + y-flip + viewBox** — Glyphs::EmbeddedFonts::Svg
+  #   * **Provenance schema** — CodeChart::Provenance (this namespace)
+  #   * **Sidecar JSON write** — CodeChart::Sidecar (this namespace)
+  #   * **Per-block orchestration + idempotent disk write** — CodeChart::Writer
+  #   * **CLI dispatch** — Cli::CodeChartCmd
+  #
+  # CodeChart::* is the feature-facing namespace. It does not
+  # implement extraction, font parsing, or PDF I/O — it composes
+  # the existing infrastructure. Replacing the implementation
+  # (e.g. a future pure-Ruby PDF parser per ADR-0001) does not
+  # change the public API.
+  module CodeChart
+    autoload :Extractor, "ucode/code_chart/extractor"
+    autoload :Provenance, "ucode/code_chart/provenance"
+    autoload :Sidecar, "ucode/code_chart/sidecar"
+    autoload :Writer, "ucode/code_chart/writer"
+  end
+end

data/lib/ucode/error.rb

@@ -97,6 +97,10 @@ module Ucode
   # Version string not in Config.known_versions.
   class UnknownVersionError < LookupError; end
 
+  # Block identifier not present in the cached Blocks.txt. Carries the
+  # offending id and the path searched in `context:`.
+  class UnknownBlockError < LookupError; end
+
   # Glyph pipeline failures.
   class GlyphError < Error; end
 
@@ -114,6 +118,13 @@ module Ucode
   # `mutool` is not installed on the PATH.
   class EmbeddedFontsMissingError < GlyphError; end
 
+  # The Code Charts PDF for a requested block cannot be obtained: the
+  # network returned 4xx/5xx, the response wasn't application/pdf, or
+  # the body didn't start with the `%PDF` magic. Distinct from
+  # {EmbeddedFontsMissingError} (which fires when the file is already
+  # on disk and we just can't open it): this fires at fetch time.
+  class CodeChartNotFoundError < GlyphError; end
+
   # Pre-build validation failed for a universal-set build. The
   # context carries the failing checks so the CLI can render a
   # useful diagnostic without re-running them. Distinct from

data/lib/ucode/fetch/code_charts.rb

@@ -29,7 +29,7 @@ module Ucode
             next if dest.exist? && !force
 
             url = "#{Ucode.configuration.charts_base_url}/#{filename}"
-            Http.get(url, dest: dest)
+            Http.get(url, dest: dest, validate: :pdf)
             downloaded += 1
           end
           downloaded

data/lib/ucode/fetch/http.rb

@@ -23,9 +23,17 @@ module Ucode
         #   directory is created if absent.
         # @param retries [Integer, nil] override Config.http_retries.
         # @param timeout [Integer, nil] override Config.http_timeout.
+        # @param validate [Symbol, nil] when `:pdf`, after a successful
+        #   download verify (a) Content-Type starts with `application/pdf`
+        #   and (b) the first 4 bytes of the body are `%PDF`. Raises
+        #   {Ucode::CodeChartNotFoundError} with the offending header
+        #   value in `context:` on failure. nil = no validation (the
+        #   default for non-PDF callers like UcdZip and UnihanZip).
         # @return [Pathname] destination path on success.
         # @raise [Ucode::NetworkError] if all retries fail.
-        def get(url, dest:, retries: nil, timeout: nil)
+        # @raise [Ucode::CodeChartNotFoundError] when `validate: :pdf`
+        #   and the response fails content validation.
+        def get(url, dest:, retries: nil, timeout: nil, validate: nil)
           uri = url.is_a?(URI) ? url : URI(url)
           destination = Pathname.new(dest)
           destination.dirname.mkpath
@@ -36,15 +44,21 @@ module Ucode
 
           last_error = nil
           (attempts + 1).times do |attempt|
-            return stream_to(uri, destination, read_timeout)
-          rescue StandardError => e
-            last_error = e
-            sleep_for = backoff_sequence[attempt] || backoff_sequence.last
-            Ucode.configuration.logger&.warn do
-              "Http GET #{uri} failed (attempt #{attempt + 1}/#{attempts + 1}): " \
-                "#{e.class}: #{e.message}; retrying in #{sleep_for}s"
+            begin
+              response = stream_to(uri, destination, read_timeout)
+              validate_response!(validate, response, destination) if validate
+              return destination
+            rescue ValidationFailure => e
+              raise e.cause
+            rescue StandardError => e
+              last_error = e
+              sleep_for = backoff_sequence[attempt] || backoff_sequence.last
+              Ucode.configuration.logger&.warn do
+                "Http GET #{uri} failed (attempt #{attempt + 1}/#{attempts + 1}): " \
+                  "#{e.class}: #{e.message}; retrying in #{sleep_for}s"
+              end
+              sleep(sleep_for)
             end
-            sleep(sleep_for)
           end
 
           raise Ucode::NetworkError.new(
@@ -55,19 +69,34 @@ module Ucode
 
         private
 
+        # Internal carrier for a validation failure inside a retry
+        # attempt. Re-raised from the loop so the response body (which
+        # is partial on retries) isn't double-validated against
+        # truncated bytes.
+        class ValidationFailure < StandardError
+          attr_reader :cause
+
+          def initialize(cause)
+            @cause = cause
+            super(cause.message)
+          end
+        end
+
         def stream_to(uri, destination, read_timeout)
+          response = nil
           Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
                                               read_timeout: read_timeout) do |http|
             request = Net::HTTP::Get.new(uri)
-            http.request(request) do |response|
-              unless response.is_a?(Net::HTTPSuccess)
-                raise "HTTP #{response.code} #{response.message}"
+            http.request(request) do |r|
+              unless r.is_a?(Net::HTTPSuccess)
+                raise "HTTP #{r.code} #{r.message}"
               end
 
-              write_body(response, destination)
+              write_body(r, destination)
+              response = r
             end
           end
-          destination
+          response or raise "no response received"
         end
 
         def write_body(response, destination)
@@ -77,6 +106,47 @@ module Ucode
           end
           File.rename(partial.to_s, destination.to_s)
         end
+
+        # Verifies Content-Type and magic bytes for a downloaded file.
+        # Raises ValidationFailure carrying a CodeChartNotFoundError so
+        # the retry loop in `get` doesn't re-attempt a download that's
+        # structurally invalid (only the transport is retriable).
+        def validate_response!(mode, response, destination)
+          case mode
+          when :pdf then validate_pdf!(response, destination)
+          else raise ArgumentError, "unknown validate mode: #{mode.inspect}"
+          end
+        end
+
+        PDF_CONTENT_TYPE_PREFIX = "application/pdf"
+        PDF_MAGIC = "%PDF"
+        private_constant :PDF_CONTENT_TYPE_PREFIX, :PDF_MAGIC
+
+        def validate_pdf!(response, destination)
+          content_type = response["Content-Type"].to_s
+          unless content_type.start_with?(PDF_CONTENT_TYPE_PREFIX)
+            raise ValidationFailure.new(
+              Ucode::CodeChartNotFoundError.new(
+                "expected Content-Type application/pdf, got #{content_type.inspect}",
+                context: { url: response.uri.to_s, content_type: content_type },
+              ),
+            )
+          end
+
+          # Re-open the destination file and peek at the first 4 bytes.
+          # The response body has already been written to disk by
+          # `stream_to`; we don't re-read from the response (which is
+          # consumed by then).
+          magic = File.open(destination, "rb") { |f| f.read(4) }
+          unless magic == PDF_MAGIC
+            raise ValidationFailure.new(
+              Ucode::CodeChartNotFoundError.new(
+                "expected %PDF magic bytes, got #{magic.inspect}",
+                context: { url: response.uri.to_s, magic: magic },
+              ),
+            )
+          end
+        end
       end
     end
   end

data/lib/ucode/parsers/blocks.rb

@@ -37,6 +37,40 @@ module Ucode
           nil
         end
 
+        # Resolves a block by its identifier (the underscored form of
+        # the block name, e.g. "Basic_Latin", "Egyptian_Hieroglyphs_Extended-B").
+        # Streams `Blocks.txt` once and short-circuits on first match —
+        # callers don't need to walk the whole ~340-block file.
+        #
+        # @param path [Pathname, String] path to a Blocks.txt
+        # @param id [String] block identifier (matches `Models::Block#id`)
+        # @return [Models::Block, nil] the block, or nil when no block
+        #   has the given id
+        def find_by_id(path, id)
+          return nil if id.nil? || id.empty?
+
+          each_record(path) do |block|
+            return block if block.id == id
+          end
+          nil
+        end
+
+        # Same as {find_by_id} but raises {Ucode::UnknownBlockError} on
+        # miss. Use this in callers that can't recover from a missing
+        # block (CLI commands, extractors that need a block to proceed).
+        #
+        # @param path [Pathname, String] path to a Blocks.txt
+        # @param id [String] block identifier
+        # @return [Models::Block]
+        # @raise [Ucode::UnknownBlockError] when no block matches
+        def find_by_id!(path, id)
+          find_by_id(path, id) or
+            raise Ucode::UnknownBlockError.new(
+              "unknown Unicode block: #{id.inspect}",
+              context: { block_id: id, blocks_txt: path.to_s },
+            )
+        end
+
         private
 
         def build_block(range, name)

data/lib/ucode/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ucode
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

data/lib/ucode.rb

@@ -30,11 +30,13 @@ module Ucode
   autoload :DatabaseMissingError, "ucode/error"
   autoload :DatabaseSchemaError, "ucode/error"
   autoload :UnknownVersionError, "ucode/error"
+  autoload :UnknownBlockError, "ucode/error"
   autoload :GlyphError, "ucode/error"
   autoload :PdfRenderError, "ucode/error"
   autoload :GridDetectionError, "ucode/error"
   autoload :LastResortMissingError, "ucode/error"
   autoload :EmbeddedFontsMissingError, "ucode/error"
+  autoload :CodeChartNotFoundError, "ucode/error"
 
   # Infrastructure
   autoload :Cache, "ucode/cache"
@@ -54,6 +56,7 @@ module Ucode
   autoload :Repo, "ucode/repo"
   autoload :Glyphs, "ucode/glyphs"
   autoload :Audit, "ucode/audit"
+  autoload :CodeChart, "ucode/code_chart"
   autoload :Site, "ucode/site"
   autoload :Commands, "ucode/commands"
   autoload :Cli, "ucode/cli"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ucode
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-06-29 00:00:00.000000000 Z
+date: 2026-06-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64
@@ -156,6 +156,14 @@ files:
 - Gemfile
 - README.md
 - Rakefile
+- TODO.extract-code-chart/01-pdf-fetch-validation.md
+- TODO.extract-code-chart/02-block-name-resolver.md
+- TODO.extract-code-chart/03-codechart-namespace.md
+- TODO.extract-code-chart/04-codechart-extractor.md
+- TODO.extract-code-chart/05-provenance-and-sidecar.md
+- TODO.extract-code-chart/06-codechart-writer.md
+- TODO.extract-code-chart/07-codechart-cli.md
+- TODO.extract-code-chart/08-specs.md
 - TODO.full/00-README.md
 - TODO.full/01-panglyph-vision.md
 - TODO.full/02-panglyph-repo-bootstrap.md
@@ -295,6 +303,11 @@ files:
 - lib/ucode/audit/universal_set_reference.rb
 - lib/ucode/cache.rb
 - lib/ucode/cli.rb
+- lib/ucode/code_chart.rb
+- lib/ucode/code_chart/extractor.rb
+- lib/ucode/code_chart/provenance.rb
+- lib/ucode/code_chart/sidecar.rb
+- lib/ucode/code_chart/writer.rb
 - lib/ucode/commands.rb
 - lib/ucode/commands/audit.rb
 - lib/ucode/commands/audit/browser_command.rb