obp-access 0.1.1

data/lib/obp/access/fetcher.rb

@@ -0,0 +1,63 @@
+module Obp
+  class Access
+    class Fetcher
+      USER_AGENT_PROFILES = [
+        {
+          user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
+                      "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                      "Chrome/131.0.0.0 Safari/537.36",
+          platform: '"macOS"',
+          chrome_version: "131",
+        },
+        {
+          user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \
+                      "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                      "Chrome/130.0.0.0 Safari/537.36",
+          platform: '"Windows"',
+          chrome_version: "130",
+        },
+        {
+          user_agent: "Mozilla/5.0 (X11; Linux x86_64) " \
+                      "AppleWebKit/537.36 (KHTML, like Gecko) " \
+                      "Chrome/131.0.0.0 Safari/537.36",
+          platform: '"Linux"',
+          chrome_version: "131",
+        },
+      ].freeze
+
+      def initialize(urn:)
+        @urn = urn
+      end
+
+      def fetch_state
+        response = post_ui_request
+        parse_state(response)
+      end
+
+      private
+
+      def post_ui_request
+        uri = URI(API_URL)
+        request = Net::HTTP::Post.new(uri)
+        profile = USER_AGENT_PROFILES.sample
+        request["User-Agent"] = profile[:user_agent]
+        request["Accept"] = "application/json"
+        request.set_form_data(
+          "v-browserDetails" => 1,
+          "theme" => "iso-red",
+          "v-loc" => "#{API_URL}##{@urn}",
+        )
+
+        Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
+          http.request(request)
+        end
+      end
+
+      def parse_state(response)
+        json = JSON.parse(response.body)
+        state_json = JSON.parse(json["uidl"])
+        state_json["state"].values
+      end
+    end
+  end
+end

data/lib/obp/access/grammar_parser.rb

@@ -0,0 +1,135 @@
+module Obp
+  class Access
+    class GrammarParser
+      Result = Struct.new(:term, :pos, :genders, keyword_init: true)
+
+      POS_MAP = {
+        "adj." => "adjective",
+        "Adj." => "adjective",
+        "verb" => "verb",
+      }.freeze
+
+      GENDER_VALUES = %w[m f n].freeze
+
+      BOLD_PATTERNS = [
+        [->(t) { POS_MAP.key?(t) }, :handle_pos_marker],
+        [->(t) { GENDER_VALUES.include?(t) }, :handle_gender_marker],
+        [->(t) { t.match?(/\A[mfn],\z/) }, :handle_gender_with_comma],
+        [->(t) { t.match?(/\A[mfn][,\s]+[mfn]([,\s]+[mfn])*\z/) }, :handle_multi_gender],
+        [->(t) { t == "," }, :handle_comma],
+        [->(t) { t == "〈" },                     :handle_enter_bracket],
+        [->(t) { t == "〉" },                     :handle_exit_bracket],
+        [->(t) { t.match?(/\A[mfn]\s+/) },       :handle_gender_qualifier],
+        [->(t) { t.match?(/,.+[mfn]\z/) },       :handle_term_with_gender],
+      ].freeze
+
+      def self.parse(inner_html)
+        state = { pos: "noun", genders: [], term_parts: [], in_bracket: false }
+        segments = parse_segments(inner_html)
+
+        segments.each do |seg|
+          handler = find_handler(seg, state[:in_bracket])
+          handler.call(seg[:text], state)
+        end
+
+        Result.new(term: clean_term(state[:term_parts]), pos: state[:pos], genders: state[:genders].uniq)
+      end
+
+      class << self
+        private
+
+        def find_handler(seg, in_bracket)
+          if seg[:bold]
+            bold_handler(seg[:text].strip, in_bracket)
+          elsif in_bracket
+            method(:handle_skip)
+          else
+            method(:handle_text)
+          end
+        end
+
+        def bold_handler(text, in_bracket)
+          _pattern, handler = BOLD_PATTERNS.find { |pred, _| pred.call(text) }
+          return method(handler) if handler
+
+          in_bracket ? method(:handle_skip) : method(:handle_term_text)
+        end
+
+        def handle_pos_marker(text, state)
+          state[:pos] = POS_MAP[text.strip]
+        end
+
+        def handle_gender_marker(text, state)
+          state[:genders] << text.strip
+        end
+
+        def handle_gender_with_comma(text, state)
+          state[:genders] << text.strip[0]
+        end
+
+        def handle_multi_gender(text, state)
+          text.strip.scan(/[mfn]/).each { |g| state[:genders] << g }
+        end
+
+        def handle_enter_bracket(_text, state)
+          state[:in_bracket] = true
+        end
+
+        def handle_exit_bracket(_text, state)
+          state[:in_bracket] = false
+        end
+
+        def handle_gender_qualifier(text, state)
+          state[:genders] << text.strip[0]
+        end
+
+        def handle_term_with_gender(text, state)
+          stripped = text.strip
+          if stripped =~ /\A(.+),\s*([mfn])\z/
+            state[:term_parts] << $1.strip
+            state[:genders] << $2
+          else
+            state[:term_parts] << stripped
+          end
+        end
+
+        def handle_comma(_text, _state); end
+
+        def handle_term_text(text, state)
+          state[:term_parts] << text
+        end
+
+        def handle_text(text, state)
+          state[:term_parts] << text
+        end
+
+        def handle_skip(_text, _state); end
+
+        def parse_segments(html)
+          segments = []
+          remaining = html.dup
+
+          while remaining.length.positive?
+            match = remaining.match(/\A(.*?)(<b>(.*?)<\/b>)(.*)/m)
+            if match
+              segments << { text: match[1], bold: false } if match[1].length.positive?
+              segments << { text: match[3], bold: true }
+              remaining = match[4]
+            else
+              segments << { text: remaining, bold: false }
+              break
+            end
+          end
+
+          segments
+        end
+
+        def clean_term(parts)
+          combined = parts.join
+          combined = combined.gsub(/,\s*\z/, "")
+          combined.gsub(/\s+/, " ").strip
+        end
+      end
+    end
+  end
+end

data/lib/obp/access/imager.rb

@@ -0,0 +1,39 @@
+module Obp
+  class Access
+    class Imager
+      attr_reader :html, :directory
+
+      def initialize(html:, directory:)
+        @html = html
+        @directory = directory
+      end
+
+      def images
+        doc = Nokogiri::HTML(html)
+        images = doc.search("div.sts-fig > img").to_h do |img|
+          key = img.attr("src")
+          path = File.join(imgdir, key.split("/").last)
+          [key, path]
+        end
+        download_images(images)
+        images
+      end
+
+      private
+
+      def imgdir
+        @imgdir ||= FileUtils.mkdir(File.join(directory, "images")).first
+      end
+
+      def download_images(images)
+        Parallel.each(images) { |key, path| download_image(key, path) }
+      end
+
+      def download_image(key, path)
+        url = "#{BASE_URL}#{key}"
+        blob = Net::HTTP.get_response(URI(url)).body
+        File.write(path, blob, mode: "wb")
+      end
+    end
+  end
+end

data/lib/obp/access/inline_renderer.rb

@@ -0,0 +1,97 @@
+module Obp
+  class Access
+    module InlineRenderer
+      CLASS_TYPES = {
+        %w[sts-tbx-entailedTerm] => :entailed_term,
+        %w[sts-xref] => :xref,
+        %w[sts-std-ref] => :std_ref,
+        %w[sts-label] => :label,
+      }.freeze
+
+      def render_inline(xml, node)
+        return xml.text(node.content) if node.is_a?(Nokogiri::XML::Text)
+
+        render_node_by_type(xml, node, inline_type(node))
+      end
+
+      CONTAINER_TYPES = { italic: :italic, bold: :bold }.freeze
+
+      def render_node_by_type(xml, node, type)
+        if CONTAINER_TYPES.key?(type)
+          render_container(xml, node, CONTAINER_TYPES[type])
+        elsif type == :label
+          nil
+        elsif type == :element
+          render_children(xml, node)
+        else
+          render_named_type(xml, node, type)
+        end
+      end
+
+      def render_named_type(xml, node, type)
+        case type
+        when :entailed_term then render_entailed_term(xml, node)
+        when :xref then render_xref(xml, node)
+        when :std_ref then render_std_ref(xml, node)
+        when :ext_link then render_ext_link(xml, node)
+        end
+      end
+
+      def render_container(xml, node, tag)
+        xml.public_send(tag) { node.children.each { |c| render_inline(xml, c) } }
+      end
+
+      def render_children(xml, node)
+        node.children.each { |c| render_inline(xml, c) }
+      end
+
+      def inline_type(node)
+        return :text if node.is_a?(Nokogiri::XML::Text)
+
+        CLASS_TYPES.fetch(node.classes) do
+          case node.name
+          when "i" then :italic
+          when "a" then :ext_link
+          when "b" then :bold
+          else :element
+          end
+        end
+      end
+
+      def xref_ref_type(text)
+        case text
+        when /\AFigure/ then "fig"
+        when /\ATable/  then "table"
+        when /\ANote/   then "fn"
+        else "sec"
+        end
+      end
+
+      private
+
+      def render_entailed_term(xml, node)
+        target = node.at_css("a").attr("href").split(":").last
+        xml.public_send(:"tbx:entailedTerm", target: "term_#{target}") do
+          xml << node.text.strip
+        end
+      end
+
+      def render_xref(xml, node)
+        rid = node.attr("href").split(":").last
+        ref_type = xref_ref_type(node.text)
+        xml.xref("ref-type": ref_type, rid: "#{ref_type}_#{rid}") { xml << node.text.strip }
+      end
+
+      def render_std_ref(xml, node)
+        rid = node.attr("href").split(":").last
+        xml.xref("ref-type": "bibr", rid: "ref_#{rid}") { xml << node.text.strip }
+      end
+
+      def render_ext_link(xml, node)
+        xml.public_send(:"ext-link",
+                        "ext-link-type" => "uri",
+                        "xlink:href" => node.attr("href")) { xml << node.text.strip }
+      end
+    end
+  end
+end

data/lib/obp/access/parser.rb

@@ -0,0 +1,82 @@
+module Obp
+  class Access
+    class Parser
+      attr_reader :urn, :directory
+
+      def initialize(urn:, directory:)
+        @urn = urn
+        @directory = directory
+      end
+
+      def to_xml
+        xml
+      end
+
+      def title
+        tab_data["description"]
+      end
+
+      def html
+        @html ||= begin
+          content = state.filter_map { |attr| attr["htmlContent"] }.first
+          raise "OBP content not found for URN #{urn}" unless content
+
+          content
+        end
+      end
+
+      def available_languages
+        state
+          .select { |attr| !attr["caption"]&.empty? && attr["styles"]&.include?("toggle") }
+          .filter_map { |attr| attr["caption"] }
+          .uniq
+      end
+
+      private
+
+      def fetcher
+        @fetcher ||= Fetcher.new(urn:)
+      end
+
+      def state
+        @state ||= fetcher.fetch_state
+      end
+
+      def xml
+        @xml ||= begin
+          metas = {
+            "titles" => titles,
+            "images" => images,
+            "language" => urn.language,
+          }.merge(tab_data)
+
+          Converter.new(urn:, metas:, source: html).to_xml
+        end
+      end
+
+      def tab_data
+        @tab_data ||= state.filter_map { |attr| attr["tabs"] }.first.last
+      end
+
+      def titles
+        languages = available_languages
+        languages = [urn.language] if languages.empty?
+
+        Parallel.map(languages) { |lang| fetch_title(lang) }.to_h
+      end
+
+      def fetch_title(lang)
+        if lang == urn.language
+          [lang, title]
+        else
+          other_urn = Urn.new("#{urn.base}:#{lang}")
+          [lang, Parser.new(urn: other_urn, directory:).title]
+        end
+      end
+
+      def images
+        Imager.new(html:, directory:).images
+      end
+    end
+  end
+end

data/lib/obp/access/renderer.rb

@@ -0,0 +1,43 @@
+module Obp
+  class Access
+    class Renderer
+      attr_reader :urn, :metas, :nodes, :document
+
+      def initialize(urn:, metas:, nodes:)
+        @urn = urn
+        @metas = metas
+        @nodes = nodes
+        @document = Elements::Root.new(urn:, metas:).to_document
+      end
+
+      def to_xml
+        @nodes.each { |node| render(node:) }
+        @document.to_xml
+      end
+
+      private
+
+      def render(node:, target: nil)
+        return unless css_classes_match?(node)
+
+        ElementRegistry.elements.each do |element_class|
+          element = element_class.new(document:, metas:, node:)
+          next unless element.match_node?
+
+          xml = element.render(target:)
+          section_path = xml.first.path
+
+          node.children.each do |child|
+            render(node: child, target: section_path)
+          end
+
+          xml
+        end
+      end
+
+      def css_classes_match?(node)
+        ElementRegistry.css_classes.any?(node.classes)
+      end
+    end
+  end
+end

data/lib/obp/access/retriever.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+
+module Obp
+  class Access
+    class Retriever
+      MANIFEST_FILE = "manifest.json"
+
+      attr_reader :output_dir, :catalog, :concurrency
+
+      def initialize(output_dir:, catalog:, concurrency: 4)
+        @output_dir = output_dir
+        @catalog = catalog
+        @concurrency = concurrency
+      end
+
+      def run
+        FileUtils.mkdir_p(output_dir)
+        pending = pending_deliverables
+        total = pending.size
+
+        if total.zero?
+          puts "Nothing to retrieve — all #{catalog.retrievable.size} deliverables already fetched."
+          return
+        end
+
+        puts "Retrieving #{total} deliverables to #{output_dir} (concurrency: #{concurrency})..."
+        process_all(pending, total)
+        puts "Done. Fetched #{total} documents."
+      end
+
+      private
+
+      def pending_deliverables
+        catalog.retrievable.reject { |d| manifest.key?(d.id.to_s) }
+      end
+
+      def process_all(pending, total)
+        Parallel.each_with_index(pending, in_threads: concurrency) do |deliverable, i|
+          process_one(deliverable, i + 1, total)
+        end
+      end
+
+      def process_one(deliverable, index, total)
+        deliverable.languages.each { |lang| fetch_and_save(deliverable, lang) }
+        record_success(deliverable)
+        puts "[#{index}/#{total}] #{deliverable.reference} — OK"
+      rescue StandardError => e
+        record_failure(deliverable, e)
+        puts "[#{index}/#{total}] #{deliverable.reference} — FAILED: #{e.message}"
+      end
+
+      def fetch_and_save(deliverable, language)
+        urn = deliverable.to_urn(language:)
+        access = Access.fetch(urn.to_s)
+        xml = access.to_xml(pretty: true)
+
+        dir = File.join(output_dir, deliverable.reference.gsub(%r{[/:\s]}, "-"))
+        FileUtils.mkdir_p(dir)
+        File.write(File.join(dir, "#{language}.xml"), xml)
+      end
+
+      def record_success(deliverable)
+        manifest[deliverable.id.to_s] = {
+          "reference" => deliverable.reference,
+          "status" => "success",
+          "timestamp" => Time.now.utc.iso8601,
+        }
+        save_manifest
+      end
+
+      def record_failure(deliverable, error)
+        manifest[deliverable.id.to_s] = {
+          "reference" => deliverable.reference,
+          "status" => "failed",
+          "error" => error.message,
+          "timestamp" => Time.now.utc.iso8601,
+        }
+        save_manifest
+      end
+
+      def manifest
+        @manifest ||= begin
+          path = File.join(output_dir, MANIFEST_FILE)
+          File.exist?(path) ? JSON.parse(File.read(path)) : {}
+        end
+      end
+
+      def save_manifest
+        path = File.join(output_dir, MANIFEST_FILE)
+        File.write(path, JSON.pretty_generate(manifest))
+      end
+    end
+  end
+end

data/lib/obp/access/urn.rb

@@ -0,0 +1,31 @@
+module Obp
+  class Access
+    class Urn
+      attr_reader :raw, :language, :base
+
+      def initialize(raw)
+        @raw = raw
+        parts = raw.split(":")
+        @language = parts.last
+        @base = parts[0...-1].join(":")
+      end
+
+      def safe
+        @safe ||= raw.tr(":", "-")
+      end
+
+      def to_s
+        raw
+      end
+
+      def ==(other)
+        other.is_a?(self.class) && raw == other.raw
+      end
+      alias_method :eql?, :==
+
+      def hash
+        raw.hash
+      end
+    end
+  end
+end

data/lib/obp/access/version.rb

@@ -0,0 +1,5 @@
+module Obp
+  class Access
+    VERSION = "0.1.1".freeze
+  end
+end