ucode 0.1.0

data/lib/ucode/repo/aggregate_writer.rb

@@ -0,0 +1,364 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "json"
+require "time"
+
+require "ucode/models"
+require "ucode/repo/atomic_writes"
+require "ucode/repo/paths"
+
+module Ucode
+  module Repo
+    # Writes every aggregate JSON file under `output/`:
+    #
+    #   output/planes/<n>.json
+    #   output/blocks/<ID>.json
+    #   output/blocks/index.json              (block index)
+    #   output/scripts/<code>.json
+    #   output/index/names.json               (cp_id → name)
+    #   output/index/labels.json              (cp_id → {name, gc, sc})
+    #   output/index/codepoint_to_block.json  (cp_id → block_id)
+    #   output/relationships/*.json           (per-property tables)
+    #   output/enums.json                     (property aliases + value aliases)
+    #   output/named_sequences/<slug>.json
+    #   output/manifest.json
+    #
+    # **Single pass**: callers feed one CodePoint at a time via `#add`;
+    # `#flush` writes all derived files using the Coordinator's indices
+    # for the static tables (relationships, enums, named sequences).
+    #
+    # **MECE**:
+    #   - paths: `Repo::Paths`
+    #   - atomic writes: `Repo::AtomicWrites`
+    #   - stream aggregation: this class
+    #   - serialization: lutaml-model `to_yaml_hash` / `to_json`
+    class AggregateWriter
+      include AtomicWrites
+
+      # Static metadata for the 17 Unicode planes. Planes 4–13 are
+      # unassigned in Unicode 17; their entries use placeholder names.
+      PLANE_TABLE = {
+        0  => ["Basic Multilingual Plane",            "BMP"],
+        1  => ["Supplementary Multilingual Plane",    "SMP"],
+        2  => ["Supplementary Ideographic Plane",     "SIP"],
+        3  => ["Tertiary Ideographic Plane",          "TIP"],
+        4  => ["Unassigned Plane 4",                  "—"],
+        5  => ["Unassigned Plane 5",                  "—"],
+        6  => ["Unassigned Plane 6",                  "—"],
+        7  => ["Unassigned Plane 7",                  "—"],
+        8  => ["Unassigned Plane 8",                  "—"],
+        9  => ["Unassigned Plane 9",                  "—"],
+        10 => ["Unassigned Plane 10",                 "—"],
+        11 => ["Unassigned Plane 11",                 "—"],
+        12 => ["Unassigned Plane 12",                 "—"],
+        13 => ["Unassigned Plane 13",                 "—"],
+        14 => ["Supplementary Special-purpose Plane", "SSP"],
+        15 => ["Supplementary Private Use Area-A",    "SPUA-A"],
+        16 => ["Supplementary Private Use Area-B",    "SPUA-B"],
+      }.freeze
+      private_constant :PLANE_TABLE
+
+      # Coordinator::Indices fields paired with the file slug used
+      # under `output/relationships/`. Each field is a Hash<Integer,
+      # Record> or Hash<Integer, Array<Record>>.
+      RELATIONSHIP_SOURCES = {
+        special_casing:        "special_casing",
+        case_folding:          "case_folding",
+        bidi_mirroring:        "bidi_mirroring",
+        bidi_brackets:         "bidi_brackets",
+        cjk_radicals:          "cjk_radicals",
+        standardized_variants: "standardized_variants",
+        name_aliases:          "name_aliases",
+      }.freeze
+      private_constant :RELATIONSHIP_SOURCES
+
+      attr_reader :codepoint_count
+
+      # @param output_root [String, Pathname]
+      def initialize(output_root)
+        @output_root = Pathname.new(output_root)
+        @block_codepoint_ids = Hash.new { |h, k| h[k] = [] }
+        @script_codepoint_ids = Hash.new { |h, k| h[k] = [] }
+        @names_index = {}
+        @labels_index = {}
+        @cp_to_block = {}
+        @codepoint_count = 0
+      end
+
+      # Fold one CodePoint into the stream accumulators. No-ops if the
+      # cp has no block_id (it has no home in the output tree).
+      # @param cp [Ucode::Models::CodePoint]
+      # @return [void]
+      def add(cp)
+        return if cp.block_id.nil?
+
+        @block_codepoint_ids[cp.block_id] << cp.id
+        if cp.script_code
+          @script_codepoint_ids[cp.script_code] << cp.id
+        end
+        if cp.name && !cp.name.empty?
+          @names_index[cp.id] = cp.name
+        end
+        @labels_index[cp.id] = build_label(cp)
+        @cp_to_block[cp.id] = cp.block_id
+        @codepoint_count += 1
+      end
+
+      # Write every aggregate file. Optional params supply data that is
+      # not in `Coordinator::Indices` (the Coordinator only resolves the
+      # `sc` subset of PropertyValueAliases; the full alias tables and
+      # the named sequences are passed through from the CLI/parsers).
+      #
+      # @param ucd_version [String]
+      # @param indices [Ucode::Coordinator::Indices]
+      # @param property_aliases [Array<Ucode::Models::PropertyAlias>]
+      # @param property_value_aliases [Array<Ucode::Models::PropertyValueAlias>]
+      # @param named_sequences [Array<Ucode::Models::NamedSequence>]
+      # @param glyph_count [Integer]
+      # @return [Integer] number of files written
+      def flush(ucd_version:, indices:, property_aliases: [],
+                property_value_aliases: [], named_sequences: [], glyph_count: 0)
+        writes = 0
+        writes += write_planes(indices.blocks)
+        writes += write_blocks(indices.blocks)
+        writes += write_scripts(indices.scripts)
+        writes += write_indexes
+        writes += write_relationships(indices)
+        writes += write_enums(property_aliases, property_value_aliases)
+        writes += write_named_sequences(named_sequences)
+        writes += write_manifest(ucd_version: ucd_version, glyph_count: glyph_count)
+        writes
+      end
+
+      private
+
+      # ---- Per-codepoint accumulator helpers ---------------------------
+
+      def build_label(cp)
+        label = { "name" => cp.name, "gc" => cp.general_category, "sc" => cp.script_code }
+        label.reject { |_, v| v.nil? }
+      end
+
+      # ---- Plane files -------------------------------------------------
+
+      def write_planes(blocks)
+        plane_block_ids = group_block_ids_by_plane(blocks)
+        count = 0
+        (0..16).each do |n|
+          path = Paths.plane_metadata_path(@output_root, n)
+          count += 1 if write_atomic(path, plane_payload(n, plane_block_ids[n] || []))
+        end
+        count
+      end
+
+      def group_block_ids_by_plane(blocks)
+        blocks.each_with_object(Hash.new { |h, k| h[k] = [] }) do |block, h|
+          h[block.plane_number] << block.id
+        end
+      end
+
+      def plane_payload(plane_number, block_ids)
+        name, abbrev = PLANE_TABLE.fetch(plane_number)
+        range_first = plane_number * 0x10000
+        range_last  = range_first + 0xFFFF
+        to_pretty_json(
+          "number"       => plane_number,
+          "name"         => name,
+          "abbrev"       => abbrev,
+          "range_first"  => range_first,
+          "range_last"   => range_last,
+          "block_ids"    => block_ids,
+        )
+      end
+
+      # ---- Block files -------------------------------------------------
+
+      def write_blocks(blocks)
+        count = blocks.sum do |block|
+          path = Paths.block_metadata_path(@output_root, block.id)
+          write_atomic(path, block_payload(block)) ? 1 : 0
+        end
+        count + write_blocks_index(blocks)
+      end
+
+      def write_blocks_index(blocks)
+        path = Paths.blocks_index_path(@output_root)
+        summary = blocks.map do |block|
+          {
+            "id"           => block.id,
+            "name"         => block.name,
+            "first_cp"     => block.range_first,
+            "last_cp"      => block.range_last,
+            "plane_number" => block.plane_number,
+          }
+        end
+        write_atomic(path, to_pretty_json(summary)) ? 1 : 0
+      end
+
+      def block_payload(block)
+        to_pretty_json(
+          "id"             => block.id,
+          "name"           => block.name,
+          "range_first"    => block.range_first,
+          "range_last"     => block.range_last,
+          "plane_number"   => block.plane_number,
+          "codepoint_ids"  => (@block_codepoint_ids[block.id] || []),
+        )
+      end
+
+      # ---- Script files ------------------------------------------------
+
+      def write_scripts(scripts)
+        count = 0
+        scripts.group_by(&:code).each do |code, ranges|
+          next if code.nil? || code.empty?
+
+          path = Paths.script_metadata_path(@output_root, code)
+          count += 1 if write_atomic(path, script_payload(code, ranges))
+        end
+        count
+      end
+
+      def script_payload(code, ranges)
+        to_pretty_json(
+          "code"           => code,
+          "name"           => ranges.first&.name,
+          "range_first"    => ranges.map(&:range_first).min,
+          "range_last"     => ranges.map(&:range_last).max,
+          "codepoint_ids"  => (@script_codepoint_ids[code] || []),
+        )
+      end
+
+      # ---- Lookup indexes ---------------------------------------------
+
+      def write_indexes
+        count = 0
+        count += 1 if write_atomic(Paths.names_index_path(@output_root), to_pretty_json(@names_index))
+        count += 1 if write_atomic(Paths.labels_index_path(@output_root), to_pretty_json(@labels_index))
+        count += 1 if write_atomic(codepoint_to_block_path, to_pretty_json(@cp_to_block))
+        count
+      end
+
+      def codepoint_to_block_path
+        Pathname(@output_root).join("index", "codepoint_to_block.json")
+      end
+
+      # ---- Relationships ----------------------------------------------
+
+      def write_relationships(indices)
+        RELATIONSHIP_SOURCES.sum do |field, slug|
+          records = indices.public_send(field)
+          write_relationship_file(slug, records)
+        end
+      end
+
+      def write_relationship_file(slug, records)
+        return 0 if records.nil? || records.empty?
+
+        path = Pathname(@output_root).join("relationships", "#{slug}.json")
+        write_atomic(path, relationship_payload(records)) ? 1 : 0
+      end
+
+      # records is Hash<Integer, Record>, Hash<Integer, Array<Record>>,
+      # Hash<String, Record>, or Hash<String, Array<Record>>.
+      # Output: { "U+XXXX" => record.to_yaml_hash, ... } or
+      # { "U+XXXX" => [record.to_yaml_hash, ...], ... }
+      def relationship_payload(records)
+        payload = records.each_with_object({}) do |(key, value), h|
+          h[key_to_cp_id(key)] = serialize_value(value)
+        end
+        to_pretty_json(payload)
+      end
+
+      # Indices that are keyed by Integer codepoint (most of them) get
+      # formatted into "U+XXXX". Indices keyed by string ids already
+      # (cjk_radicals by ideograph_id, standardized_variants by base_id)
+      # are passed through verbatim.
+      def key_to_cp_id(key)
+        key.is_a?(Integer) ? Paths.cp_id(key) : key
+      end
+
+      def serialize_value(value)
+        return value.map { |v| serialize_one(v) } if value.is_a?(Array)
+
+        serialize_one(value)
+      end
+
+      def serialize_one(record)
+        record.to_yaml_hash
+      end
+
+      # ---- Enums -------------------------------------------------------
+
+      def write_enums(property_aliases, property_value_aliases)
+        path = Pathname(@output_root).join("enums.json")
+        payload = {
+          "properties"      => property_aliases.map(&:to_yaml_hash),
+          "property_values" => property_value_aliases.map(&:to_yaml_hash),
+        }
+        write_atomic(path, to_pretty_json(payload)) ? 1 : 0
+      end
+
+      # ---- Named sequences --------------------------------------------
+
+      def write_named_sequences(named_sequences)
+        return 0 if named_sequences.nil? || named_sequences.empty?
+
+        dir = Pathname(@output_root).join("named_sequences")
+        named_sequences.sum do |ns|
+          path = dir.join("#{slug_for(ns)}.json")
+          write_atomic(path, ns.to_json(pretty: true)) ? 1 : 0
+        end
+      end
+
+      # Slug derived from the name: downcase, non-alphanumerics → "_".
+      def slug_for(named_sequence)
+        named_sequence.name
+          .downcase
+          .gsub(/[^a-z0-9]+/, "_")
+          .gsub(/^_+|_+$/, "")
+      end
+
+      # ---- Manifest ---------------------------------------------------
+
+      # Fields that define the manifest's semantic content. When these
+      # match the existing manifest on disk, we preserve the old
+      # `generated_at` so that re-runs are byte-idempotent (no rewrite
+      # unless something actually changed).
+      MANIFEST_CONTENT_KEYS = %w[
+        ucd_version codepoint_count glyph_count schema_version
+      ].freeze
+      private_constant :MANIFEST_CONTENT_KEYS
+
+      def write_manifest(ucd_version:, glyph_count:)
+        path = Paths.manifest_path(@output_root)
+        content = {
+          "ucd_version"     => ucd_version,
+          "codepoint_count" => @codepoint_count,
+          "glyph_count"     => glyph_count,
+          "schema_version"  => "1",
+        }
+        ts = preserved_or_new_timestamp(path, content)
+        payload = content.merge("generated_at" => ts)
+        write_atomic(path, to_pretty_json(payload)) ? 1 : 0
+      end
+
+      def preserved_or_new_timestamp(path, content)
+        existing = read_manifest(path)
+        return Time.now.utc.iso8601 unless existing
+
+        unchanged = MANIFEST_CONTENT_KEYS.all? { |k| existing[k] == content[k] }
+        unchanged ? existing["generated_at"] : Time.now.utc.iso8601
+      end
+
+      def read_manifest(path)
+        return nil unless path.exist?
+
+        JSON.parse(path.read)
+      rescue JSON::ParserError
+        nil
+      end
+    end
+  end
+end

data/lib/ucode/repo/atomic_writes.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "json"
+
+require "ucode/repo/paths"
+
+module Ucode
+  module Repo
+    # Atomic, idempotent file-write helpers shared by CodepointWriter
+    # and AggregateWriter.
+    #
+    # - Atomic: write to a sibling `.tmp` file, then rename. A crash
+    #   mid-write leaves either the old file or no file, never a
+    #   truncated one.
+    # - Idempotent: byte-compare the existing file before writing;
+    #   identical content is a no-op. Safe to re-run on the full
+    #   dataset.
+    module AtomicWrites
+      # @param path [Pathname]
+      # @param payload [String] the exact bytes to write
+      # @return [Boolean] true if the file was written, false if skipped
+      def write_atomic(path, payload)
+        return false if same_content?(path, payload)
+
+        path.dirname.mkpath
+        tmp = Paths.tmp_path(path)
+        tmp.write(payload)
+        tmp.rename(path.to_s)
+        true
+      end
+
+      # @param path [Pathname]
+      # @param payload [String]
+      # @return [Boolean]
+      def same_content?(path, payload)
+        path.exist? && path.read == payload
+      end
+
+      # Pretty JSON for any Hash/Array value.
+      # @param value [Hash, Array]
+      # @return [String]
+      def to_pretty_json(value)
+        JSON.pretty_generate(value)
+      end
+    end
+  end
+end

data/lib/ucode/repo/codepoint_writer.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "thread"
+
+require "ucode/repo/atomic_writes"
+require "ucode/repo/paths"
+
+module Ucode
+  module Repo
+    # Writes one `index.json` per codepoint under `output/blocks/<id>/<cp>/`.
+    #
+    # Streaming + threaded + idempotent:
+    #
+    #   - **Streaming**: callers pass an Enumerator; the writer pulls one
+    #     codepoint at a time, never the full 160k set in memory.
+    #   - **Threaded**: a fixed-size worker pool drains a shared queue.
+    #     Each codepoint maps to a unique path → no per-file contention.
+    #   - **Idempotent**: existing files are byte-compared to the new
+    #     payload before writing; identical content is a no-op. Safe to
+    #     re-run on the full dataset.
+    #   - **Atomic**: writes go to `<path>.tmp`, then rename. A crash
+    #     mid-write leaves either the old file or no file, never a
+    #     truncated one.
+    class CodepointWriter
+      include AtomicWrites
+
+      # @param output_root [String, Pathname]
+      # @param parallel_workers [Integer] size of the worker pool. Set to
+      #   1 (or less) to run synchronously — useful in tests.
+      def initialize(output_root, parallel_workers: 8)
+        @output_root = Pathname.new(output_root)
+        @parallel_workers = parallel_workers
+      end
+
+      # Write one codepoint synchronously.
+      # @param codepoint [Ucode::Models::CodePoint]
+      # @return [Pathname, nil] the path written, or nil if skipped
+      #   (missing block_id or content-identical to existing file)
+      def write(codepoint)
+        return nil if codepoint.block_id.nil?
+
+        path = Paths.codepoint_json_path(@output_root, codepoint.block_id, codepoint.id)
+        payload = serialize(codepoint)
+        return nil unless write_atomic(path, payload)
+
+        path
+      end
+
+      # Drain an Enumerator through the worker pool. Returns the total
+      # count of codepoints seen (whether or not each one was written).
+      # @param enum [Enumerator<Ucode::Models::CodePoint>, Enumerable]
+      # @return [Integer]
+      def write_each(enum)
+        return drain_inline(enum) if @parallel_workers <= 1
+
+        drain_threaded(enum)
+      end
+
+      private
+
+      def drain_inline(enum)
+        count = 0
+        enum.each { |cp| write(cp); count += 1 }
+        count
+      end
+
+      def drain_threaded(enum)
+        queue = Queue.new
+        mutex = Mutex.new
+        count = 0
+
+        workers = Array.new(@parallel_workers) do
+          Thread.new do
+            loop do
+              cp = queue.pop
+              break if cp.nil?
+
+              write(cp)
+              mutex.synchronize { count += 1 }
+            end
+          end
+        end
+
+        enum.each { |cp| queue << cp }
+        @parallel_workers.times { queue << nil }
+        workers.each(&:join)
+        count
+      end
+
+      def serialize(codepoint)
+        codepoint.to_json(pretty: true)
+      end
+    end
+  end
+end

data/lib/ucode/repo/paths.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module Ucode
+  module Repo
+    # Pure functions describing the on-disk layout of the output tree.
+    #
+    # The only code that knows the path conventions. Site generator,
+    # CLI, glyph writer, and fontisan adapter all go through here.
+    #
+    # All methods are pure: no I/O, no global state, no side effects.
+    # Returns Pathname instances so callers can compose further.
+    module Paths
+      BLOCKS_DIR = "blocks"
+      PLANES_DIR = "planes"
+      SCRIPTS_DIR = "scripts"
+      INDEX_DIR = "index"
+      INDEX_FILENAME = "index.json"
+      GLYPH_FILENAME = "glyph.svg"
+      PLANE_FILENAME_PREFIX = "" # plane files are <n>.json
+      private_constant :BLOCKS_DIR, :PLANES_DIR, :SCRIPTS_DIR, :INDEX_DIR,
+                       :INDEX_FILENAME, :GLYPH_FILENAME,
+                       :PLANE_FILENAME_PREFIX
+
+      class << self
+        # Format an integer codepoint as the canonical "U+XXXX" id used
+        # everywhere (paths, JSON, cross-references). Always at least
+        # 4 hex digits, uppercase, no extra padding.
+        # @param cp [Integer]
+        # @return [String]
+        def cp_id(cp)
+          format("U+%04X", cp)
+        end
+
+        # @param output_root [String, Pathname]
+        # @param block_id [String] verbatim block id (e.g. "ASCII", "CJK_Ext_A")
+        # @return [Pathname]
+        def block_dir(output_root, block_id)
+          Pathname(output_root).join(BLOCKS_DIR, block_id)
+        end
+
+        # @param output_root [String, Pathname]
+        # @param block_id [String]
+        # @param cp_id [String] e.g. "U+0041"
+        # @return [Pathname]
+        def codepoint_dir(output_root, block_id, cp_id)
+          block_dir(output_root, block_id).join(cp_id)
+        end
+
+        # @param output_root [String, Pathname]
+        # @param block_id [String]
+        # @param cp_id [String]
+        # @return [Pathname]
+        def codepoint_json_path(output_root, block_id, cp_id)
+          codepoint_dir(output_root, block_id, cp_id).join(INDEX_FILENAME)
+        end
+
+        # @param output_root [String, Pathname]
+        # @param block_id [String]
+        # @param cp_id [String]
+        # @return [Pathname]
+        def codepoint_glyph_path(output_root, block_id, cp_id)
+          codepoint_dir(output_root, block_id, cp_id).join(GLYPH_FILENAME)
+        end
+
+        # @param output_root [String, Pathname]
+        # @param block_id [String]
+        # @return [Pathname]
+        def block_metadata_path(output_root, block_id)
+          block_dir(output_root, block_id).join(INDEX_FILENAME)
+        end
+
+        # @param output_root [String, Pathname]
+        # @return [Pathname]
+        def blocks_index_path(output_root)
+          Pathname(output_root).join(BLOCKS_DIR, INDEX_FILENAME)
+        end
+
+        # @param output_root [String, Pathname]
+        # @param plane_number [Integer]
+        # @return [Pathname]
+        def plane_metadata_path(output_root, plane_number)
+          Pathname(output_root).join(PLANES_DIR, "#{plane_number}.json")
+        end
+
+        # @param output_root [String, Pathname]
+        # @param script_code [String]
+        # @return [Pathname]
+        def script_metadata_path(output_root, script_code)
+          Pathname(output_root).join(SCRIPTS_DIR, "#{script_code}.json")
+        end
+
+        # @param output_root [String, Pathname]
+        # @return [Pathname]
+        def names_index_path(output_root)
+          Pathname(output_root).join(INDEX_DIR, "names.json")
+        end
+
+        # @param output_root [String, Pathname]
+        # @return [Pathname]
+        def labels_index_path(output_root)
+          Pathname(output_root).join(INDEX_DIR, "labels.json")
+        end
+
+        # @param output_root [String, Pathname]
+        # @return [Pathname]
+        def manifest_path(output_root)
+          Pathname(output_root).join("manifest.json")
+        end
+
+        # Temporary path for atomic writes — same directory as `path`,
+        # so rename stays within one filesystem.
+        # @param path [Pathname]
+        # @return [Pathname]
+        def tmp_path(path)
+          path.parent.join("#{path.basename}.tmp")
+        end
+      end
+    end
+  end
+end

data/lib/ucode/repo.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Ucode
+  # Repo — writes the output tree under output/.
+  #
+  # One folder per codepoint (CJK included), no exceptions:
+  #
+  #   output/planes/<n>.json
+  #   output/blocks/<ORIGINAL_NAME>.json
+  #   output/blocks/<ORIGINAL_NAME>/<U+XXXX>/index.json
+  #   output/blocks/<ORIGINAL_NAME>/<U+XXXX>/glyph.svg
+  #   output/scripts/<ScriptCode>.json
+  #   output/index/names.json
+  #   output/index/labels.json
+  #   output/manifest.json
+  module Repo
+    autoload :Paths, "ucode/repo/paths"
+    autoload :AtomicWrites, "ucode/repo/atomic_writes"
+    autoload :CodepointWriter, "ucode/repo/codepoint_writer"
+    autoload :AggregateWriter, "ucode/repo/aggregate_writer"
+  end
+end