relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.1

data/lib/relaton/un/wasm/memory.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Un
+    module Wasm
+      class Memory
+        PAGE = 65_536
+
+        attr_reader :max_pages
+
+        def initialize(initial_pages:, max_pages: nil)
+          @pages = initial_pages
+          @max_pages = max_pages
+          @buf = String.new("\x00".b * (initial_pages * PAGE), encoding: Encoding::ASCII_8BIT)
+        end
+
+        def size_pages
+          @pages
+        end
+
+        def grow(delta)
+          new_pages = @pages + delta
+          return -1 if @max_pages && new_pages > @max_pages
+
+          old = @pages
+          @pages = new_pages
+          @buf << ("\x00".b * (delta * PAGE))
+          old
+        end
+
+        def read(addr, len)
+          bounds!(addr, len)
+          @buf.byteslice(addr, len)
+        end
+
+        def write(addr, bytes)
+          bytes = bytes.b
+          bounds!(addr, bytes.bytesize)
+          @buf.bytesplice(addr, bytes.bytesize, bytes)
+        end
+
+        def load_u8(addr)
+          bounds!(addr, 1)
+          @buf.getbyte(addr)
+        end
+
+        def load_i8(addr)
+          v = load_u8(addr)
+          v >= 0x80 ? v - 0x100 : v
+        end
+
+        def load_u16(addr)
+          bounds!(addr, 2)
+          @buf.byteslice(addr, 2).unpack1("v")
+        end
+
+        def load_i16(addr)
+          v = load_u16(addr)
+          v >= 0x8000 ? v - 0x10000 : v
+        end
+
+        def load_u32(addr)
+          bounds!(addr, 4)
+          @buf.byteslice(addr, 4).unpack1("V")
+        end
+
+        def load_i32(addr)
+          v = load_u32(addr)
+          v >= 0x8000_0000 ? v - 0x1_0000_0000 : v
+        end
+
+        def load_u64(addr)
+          bounds!(addr, 8)
+          @buf.byteslice(addr, 8).unpack1("Q<")
+        end
+
+        def load_i64(addr)
+          v = load_u64(addr)
+          v >= 0x8000_0000_0000_0000 ? v - 0x1_0000_0000_0000_0000 : v
+        end
+
+        def store_u8(addr, val)
+          bounds!(addr, 1)
+          @buf.setbyte(addr, val & 0xff)
+        end
+
+        def store_u16(addr, val)
+          bounds!(addr, 2)
+          @buf.bytesplice(addr, 2, [val & 0xffff].pack("v"))
+        end
+
+        def store_u32(addr, val)
+          bounds!(addr, 4)
+          @buf.bytesplice(addr, 4, [val & 0xffff_ffff].pack("V"))
+        end
+
+        def store_u64(addr, val)
+          bounds!(addr, 8)
+          @buf.bytesplice(addr, 8, [val & 0xffff_ffff_ffff_ffff].pack("Q<"))
+        end
+
+        private
+
+        def bounds!(addr, len)
+          return if addr >= 0 && addr + len <= @buf.bytesize
+
+          raise Trap, "out of bounds memory access (addr=#{addr}, len=#{len}, size=#{@buf.bytesize})"
+        end
+      end
+    end
+  end
+end

data/lib/relaton/un/wasm/module.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Un
+    module Wasm
+      FuncType = Struct.new(:params, :results)
+      Import   = Struct.new(:mod, :field, :kind, :desc)
+      Export   = Struct.new(:name, :kind, :index)
+      Global   = Struct.new(:type, :mutable, :init_expr)
+      Element  = Struct.new(:table_idx, :offset_expr, :func_indices)
+      Data     = Struct.new(:mem_idx, :offset_expr, :bytes)
+      Code     = Struct.new(:locals, :body)
+
+      MemoryType = Struct.new(:initial, :max)
+      TableType  = Struct.new(:elem_type, :initial, :max)
+
+      class Module
+        attr_reader :types, :imports, :function_type_indices, :tables, :memories,
+                    :globals, :exports, :elements, :code, :data, :start
+
+        def initialize
+          @types = []
+          @imports = []
+          @function_type_indices = []
+          @tables = []
+          @memories = []
+          @globals = []
+          @exports = []
+          @elements = []
+          @code = []
+          @data = []
+          @start = nil
+        end
+
+        def self.parse(bytes)
+          Decoder.new(bytes).decode
+        end
+
+        # Number of imported functions (they precede module-defined functions
+        # in the funcidx space).
+        def imported_function_count
+          @imports.count { |i| i.kind == :func }
+        end
+      end
+    end
+  end
+end

data/lib/relaton/un/wasm.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Un
+    module Wasm
+      class Trap < StandardError; end
+      class DecodeError < StandardError; end
+      class LinkError < StandardError; end
+    end
+  end
+end
+
+require_relative "wasm/memory"
+require_relative "wasm/decoder"
+require_relative "wasm/module"
+require_relative "wasm/interpreter"
+require_relative "wasm/instance"

data/lib/relaton/un.rb

@@ -0,0 +1,28 @@
+require "addressable"
+require "relaton/bib"
+require_relative "version"
+require_relative "un/util"
+require_relative "un/item_data"
+require_relative "un/item"
+require_relative "un/bibitem"
+require_relative "un/bibdata"
+require_relative "un/token_generator"
+require_relative "un/parser"
+require_relative "un/hit"
+require_relative "un/hit_collection"
+require_relative "un/bibliography"
+
+module Relaton
+  module Un
+    class Error < StandardError; end
+
+    # Returns hash of XML reammar
+    # @return [String]
+    def self.grammar_hash
+      # gem_path = File.expand_path "..", __dir__
+      # grammars_path = File.join gem_path, "grammars", "*"
+      # grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end

data/lib/relaton/version.rb

@@ -1,3 +1,3 @@
 module Relaton
-  VERSION = "2.2.0.pre.alpha.1".freeze
+  VERSION = "3.0.0.pre.alpha.1".freeze
 end

data/lib/relaton/w3c/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module W3c
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/w3c/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module W3c
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/w3c/bibliography.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "relaton/bib/hash_parser_v1"
+require_relative "pubid"
+
+module Relaton
+  module W3c
+    # Class methods for search W3C standards.
+    class Bibliography
+      SOURCE = "https://raw.githubusercontent.com/relaton/relaton-data-w3c/v2/"
+
+      class << self
+        # @param text [String]
+        # @return [Relaton::W3c::ItemData]
+        def search(text) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+          pubid = PubId.parse text.sub(/^W3C\s/, "")
+          index = Relaton::Index.find_or_create(
+            :W3C, url: "#{SOURCE}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml", id_keys: PubId::PARTS,
+          )
+          row = index.search { |r| pubid == r[:id] }.sort_by { |r| (r[:id][:date] || r[:id][:year]).to_i }.last
+          return unless row
+
+          url = "#{SOURCE}#{row[:file]}"
+          resp = Net::HTTP.get_response(URI.parse(url))
+          return unless resp.code == "200"
+
+          Item.from_yaml(resp.body).tap { |i| i.fetched = Date.today.to_s }
+        rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
+               EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
+               Net::ProtocolError, Errno::ETIMEDOUT => e
+          raise Relaton::RequestError, "Could not access #{url}: #{e.message}"
+        end
+
+        # @param ref [String] the W3C standard Code to look up
+        # @param year [String, NilClass] not used
+        # @param opts [Hash] options
+        # @return [Relaton::W3c::ItemData]
+        def get(ref, _year = nil, _opts = {})
+          Util.info "Fetching from Relaton repository ...", key: ref
+          result = search(ref)
+          unless result
+            Util.info "Not found.", key: ref
+            return
+          end
+
+          found = result.docidentifier.first.content
+          Util.info "Found: `#{found}`", key: ref
+          result
+        end
+      end
+    end
+  end
+end

data/lib/relaton/w3c/data_fetcher.rb

@@ -0,0 +1,309 @@
+require "relaton/core"
+require "w3c_api"
+require_relative "../w3c"
+require_relative "safe_realize"
+require_relative "data_parser"
+require_relative "pubid"
+
+module Relaton
+  module W3c
+    class DataFetcher < Core::DataFetcher
+      include Relaton::W3c::SafeRealize
+
+      # Raised when pagination over the specifications index stops before the
+      # last page (e.g. a page fetch fails after retries, or the API reports
+      # more pages than were reached). It aborts the whole crawl so a truncated
+      # dataset is never saved or committed — see #fetch and #enqueue_specs.
+      class CrawlIncompleteError < StandardError; end
+
+      # Conservative default: too many parallel workers burst the per-spec
+      # version-history requests fast enough to trip the W3C API rate limiter
+      # (429s), which is what silently truncated the dataset before the crawl
+      # learned to abort on incomplete pagination. Raise it via the env var on
+      # a faster/shallower run; lower it further if 429s still appear.
+      DEFAULT_CONCURRENCY = 4
+
+      # How many times #fetch_specifications_page retries a transient failure
+      # (rate-limit/connection) before giving up and aborting the crawl.
+      PAGE_FETCH_ATTEMPTS = 3
+
+      # Number of fetch_spec worker threads. Tunable via env var so CI or
+      # local runs can dial it up for speed or down to lighten load on
+      # api.w3.org (or for debugging).
+      def self.concurrency
+        (ENV["RELATON_W3C_FETCH_CONCURRENCY"] || DEFAULT_CONCURRENCY).to_i
+      end
+
+      # Whether to crawl each specification's version history (version_history,
+      # predecessor_versions, successor_versions). Enabled by default for a
+      # complete dataset. Set RELATON_W3C_FETCH_VERSIONS=false for a faster,
+      # shallower crawl that emits only the top-level specifications and skips
+      # the per-spec version fan-out (the bulk of the API requests).
+      def self.fetch_versions?
+        val = ENV["RELATON_W3C_FETCH_VERSIONS"]
+        return true if val.nil? || val.empty?
+
+        !%w[0 false no off].include?(val.strip.downcase)
+      end
+
+      def initialize(*args)
+        super
+        @mutex = Mutex.new
+        @interrupted = false
+      end
+
+      def index
+        @index ||= Relaton::Index.find_or_create(:W3C, file: "#{INDEXFILE}.yaml")
+      end
+
+      def log_error(msg)
+        Util.error msg
+      end
+
+      def client
+        @client ||= W3cApi::Client.new
+      end
+
+      #
+      # Parse documents in parallel. The crawler is heavily I/O-bound on
+      # api.w3.org round-trips (~30-50k requests per run), so a small thread
+      # pool gives a near-linear speedup. Pagination still happens serially:
+      # each page's `next?` flag gates whether the next page is requested.
+      #
+      # A SIGINT (Ctrl-C) is handled gracefully: the producer stops queuing and
+      # the workers stop processing after their in-flight spec, then the index
+      # of everything fetched so far is saved rather than the run being lost.
+      #
+      def fetch(_source = nil)
+        n_workers = self.class.concurrency
+        queue = SizedQueue.new(n_workers * 4)
+        workers = Array.new(n_workers) { spawn_worker(queue) }
+
+        with_interrupt_handler do
+          # The poison pills + join run in `ensure` so an exception raised while
+          # enqueuing (e.g. CrawlIncompleteError) still unblocks the producer
+          # and drains the workers instead of deadlocking on queue.pop.
+          begin
+            enqueue_specs(queue)
+          ensure
+            n_workers.times { queue << nil } # poison pills
+            workers.each(&:join)
+          end
+          Util.warn "Crawl interrupted — saving progress collected so far." if @interrupted
+          index.save
+        end
+
+        report_errors
+      end
+
+      #
+      # Page through the specifications index, feeding each spec (paired with
+      # its embedded page) to the worker queue. Returns early when interrupted.
+      #
+      # embed: true inlines each specification's full payload into the index
+      # page's `_embedded` block, so a spec link realizes from that page in
+      # memory instead of making its own HTTP request — one request per page
+      # rather than one per specification. The page is queued alongside each
+      # link so the worker can hand it back to realize as the parent_resource.
+      #
+      def enqueue_specs(queue)
+        specs = client.specifications(embed: true)
+        expected_pages = specs.pages
+        last_page = nil
+        loop do
+          page = specs
+          page.links.specifications.each do |spec|
+            break if @interrupted
+
+            queue << [spec, page]
+          end
+          break if @interrupted
+
+          last_page = page.page
+          break unless page.next?
+
+          # Fetch the next page through the client's fetch path rather than
+          # realizing the `next` link: only fetch populates the page's
+          # embedded_data, so this keeps embed working past page 1. Realizing
+          # the `next` link drops `_embedded` and forces a per-spec HTTP
+          # request for every specification on every later page.
+          next_page = fetch_specifications_page(page.page + 1)
+          # A nil here means the page fetch failed after retries (not the end
+          # of the list — that is `!page.next?` above). Aborting rather than
+          # `break`ing prevents a rate-limit blip from silently truncating the
+          # dataset: a partial crawl must never be saved/committed.
+          unless next_page
+            raise CrawlIncompleteError,
+                  "specifications pagination stopped at page #{page.page}: " \
+                  "failed to fetch page #{page.page + 1}"
+          end
+
+          specs = next_page
+        end
+
+        return if @interrupted
+
+        guard_complete_pagination(last_page, expected_pages)
+      end
+
+      # Defense in depth: even when no page fetch raised, make sure pagination
+      # actually reached the last page the API advertised. Catches truncation
+      # modes other than a failed fetch (e.g. a `next` link that goes missing).
+      # Only enforced when the index reported a positive page count.
+      def guard_complete_pagination(last_page, expected_pages)
+        return unless expected_pages.is_a?(Integer) && expected_pages.positive?
+        return unless last_page.is_a?(Integer) && last_page < expected_pages
+
+        raise CrawlIncompleteError,
+              "specifications pagination ended at page #{last_page} of " \
+              "#{expected_pages}; refusing to save a partial dataset"
+      end
+
+      def fetch_spec(unrealized_spec, page = nil)
+        # When `page` came from an embed:true fetch, realizing against it as the
+        # parent_resource serves the spec from embedded data (no HTTP request).
+        spec = realize(unrealized_spec, parent_resource: page)
+        return unless spec
+
+        local_errors = Hash.new(true)
+        save_doc DataParser.parse(spec, local_errors)
+
+        fetch_versions(spec) if self.class.fetch_versions?
+
+        @mutex.synchronize { local_errors.each { |k, v| @errors[k] &&= v } }
+      end
+
+      #
+      # Crawl a specification's version history: its dated editions plus the
+      # predecessor/successor version chains. Each entry is a separate HTTP
+      # request, so this is the bulk of a run and can be skipped via
+      # RELATON_W3C_FETCH_VERSIONS=false (see .fetch_versions?).
+      #
+      def fetch_versions(spec)
+        if spec.links.respond_to?(:version_history) && spec.links.version_history
+          version_history = realize spec.links.version_history
+          version_history&.links&.spec_versions&.each { |version| parse_and_save version }
+        end
+
+        if spec.links.respond_to?(:predecessor_versions) && spec.links.predecessor_versions
+          predecessor_versions = realize spec.links.predecessor_versions
+          predecessor_versions&.links&.predecessor_versions&.each { |version| parse_and_save version }
+        end
+
+        return unless spec.links.respond_to?(:successor_versions) && spec.links.successor_versions
+
+        successor_versions = realize spec.links.successor_versions
+        successor_versions&.links&.successor_versions&.each { |version| parse_and_save version }
+      end
+
+      #
+      # Save document to file
+      #
+      # @param [Relaton::W3c::ItemData, nil] bib bibliographic item
+      #
+      def save_doc(bib, warn_duplicate: true)
+        return unless bib
+
+        file = file_name(bib.docnumber)
+        @mutex.synchronize do
+          if @files.include?(file)
+            Util.warn "File #{file} already exists. Document: #{bib.docnumber}" if warn_duplicate
+          else
+            pubid = PubId.parse bib.docnumber
+            index.add_or_update pubid.to_hash, file
+            @files << file
+          end
+          File.write file, serialize(bib), encoding: "UTF-8"
+        end
+      end
+
+      def to_xml(bib)
+        bib.to_xml(bibdata: true)
+      end
+
+      def to_yaml(bib)
+        bib.to_yaml
+      end
+
+      def to_bibxml(bib)
+        bib.to_xml
+      end
+
+      #
+      # Generate file name
+      #
+      # @param [String] id document id
+      #
+      # @return [String] file name
+      #
+      def file_name(id)
+        name = id.sub(/^W3C\s/, "").gsub(/[\s,:\/+]/, "_").squeeze("_").downcase
+        File.join @output, "#{name}.#{@ext}"
+      end
+
+      private
+
+      # Install a SIGINT handler for the duration of the crawl so Ctrl-C sets
+      # the @interrupted flag (observed by the producer loop and the workers)
+      # instead of killing the process mid-write. The trap body is kept minimal
+      # (no I/O or locking) because trap context is restricted; the user-facing
+      # notice is printed from the main thread once the crawl winds down. The
+      # previous handler is restored on the way out so the trap doesn't leak
+      # into the host process.
+      def with_interrupt_handler
+        previous = Signal.trap("INT") { @interrupted = true }
+        yield
+      ensure
+        Signal.trap("INT", previous || "DEFAULT")
+      end
+
+      # Fetch one page of the specifications index with embed enabled. Goes
+      # through the client (the register's fetch path) so the page's
+      # embedded_data is populated. Transient 403/5xx/connection failures are
+      # already retried upstream (w3c_api/lutaml-hal), but losing an index page
+      # drops every spec on it, so retry a few more times here with backoff to
+      # ride out a brief rate-limit window. Returns nil only once the attempts
+      # are exhausted; the caller turns that into a CrawlIncompleteError so the
+      # crawl aborts instead of committing a truncated dataset.
+      def fetch_specifications_page(number)
+        attempt = 0
+        begin
+          attempt += 1
+          client.specifications(embed: true, page: number)
+        rescue Lutaml::Hal::Error, Faraday::Error => e
+          log_error "Failed to fetch specifications page #{number} " \
+                    "(attempt #{attempt}/#{PAGE_FETCH_ATTEMPTS}): " \
+                    "#{e.class}: #{e.message}"
+          if attempt < PAGE_FETCH_ATTEMPTS
+            sleep(2**attempt)
+            retry
+          end
+          nil
+        end
+      end
+
+      def spawn_worker(queue)
+        Thread.new do
+          while (item = queue.pop)
+            # Once interrupted, drain the queue without processing so the
+            # producer unblocks and the pool reaches its poison pills quickly.
+            next if @interrupted
+
+            spec, page = item
+            begin
+              fetch_spec spec, page
+            rescue StandardError => e
+              log_error "fetch_spec failed: #{e.class}: #{e.message}\n" \
+                        "#{e.backtrace.first(5).join("\n")}"
+            end
+          end
+        end
+      end
+
+      def parse_and_save(version)
+        realized = realize version
+        save_doc DataParser.parse(realized) if realized
+      end
+    end
+  end
+end