obp-access 0.1.4 → 0.1.6

data/lib/obp/access/elements/table_wrap.rb

@@ -4,7 +4,7 @@ module Obp
       class Elements
         class TableWrap < Base
           def self.classes
-            %w[sts-table-wrap fig-index]
+            %w[sts-table-wrap]
           end
 
           def match_node?
@@ -26,17 +26,31 @@ module Obp
                     xml.title caption_text
                   end
                 end
-                xml.table { xml << node.at_css("table").inner_html }
+                xml.table { xml << table_markup }
               end
             end
           end
 
+          # Serialize the children as XML so void elements (e.g. <col>)
+          # self-close; inner_html emits unclosed HTML voids that swallow
+          # following rows when the builder re-parses the fragment. OBP's
+          # "sts-unknown-element" placeholders are stripped first.
+          def table_markup
+            table = node.at_css("table")
+            return "" unless table
+
+            table = table.dup
+            table.css(".sts-unknown-element").each(&:remove)
+            table.children.map(&:to_xml).join
+          end
+
           def caption_label
             @caption_label ||= node.at_css(".sts-caption-label")&.content
           end
 
           def caption_text
-            @caption_text ||= node.at_css(".sts-caption")&.content
+            @caption_text ||= node.at_css(".sts-caption-title")&.content ||
+              node.at_css(".sts-caption")&.content
           end
         end
       end

data/lib/obp/access/fetcher.rb

@@ -48,7 +48,10 @@ module Obp
           "v-loc" => "#{API_URL}##{@urn}",
         )
 
-        Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
+        Net::HTTP.start(
+          uri.hostname, uri.port, use_ssl: true,
+                                  open_timeout: 30, read_timeout: 60
+        ) do |http|
           http.request(request)
         end
       end

data/lib/obp/access/inline_renderer.rb

@@ -6,6 +6,9 @@ module Obp
         %w[sts-xref] => :xref,
         %w[sts-std-ref] => :std_ref,
         %w[sts-label] => :label,
+        # OBP's marker for content it could not map; drop its placeholder
+        # text ("[no rendering defined for element: ...]").
+        %w[sts-unknown-element] => :skip,
       }.freeze
 
       def render_inline(xml, node)
@@ -19,7 +22,7 @@ module Obp
       def render_node_by_type(xml, node, type)
         if CONTAINER_TYPES.key?(type)
           render_container(xml, node, CONTAINER_TYPES[type])
-        elsif type == :label
+        elsif %i[label skip].include?(type)
           nil
         elsif type == :element
           render_children(xml, node)

data/lib/obp/access/parser.rb

@@ -55,7 +55,7 @@ module Obp
       end
 
       def tab_data
-        @tab_data ||= state.filter_map { |attr| attr["tabs"] }.first.last
+        @tab_data ||= state.filter_map { |attr| attr["tabs"] }.first&.last || {}
       end
 
       def titles

data/lib/obp/access/raw_html_parser.rb

@@ -0,0 +1,41 @@
+module Obp
+  class Access
+    # Parser that converts an already-downloaded OBP HTML fragment (e.g.
+    # captured via a browser / waffle-punch) without touching the network.
+    # Duck-types with Parser for the Access pipeline.
+    class RawHtmlParser < Parser
+      def initialize(urn:, directory:, html:, caption: nil, titles: nil)
+        super(urn:, directory:)
+        @raw_html = html
+        @caption = caption
+        @raw_titles = titles
+      end
+
+      def html
+        @raw_html
+      end
+
+      def available_languages
+        [urn.language]
+      end
+
+      private
+
+      def state
+        raise "RawHtmlParser does not fetch OBP state"
+      end
+
+      def tab_data
+        { "caption" => @caption, "description" => title }
+      end
+
+      def title
+        @caption
+      end
+
+      def titles
+        @raw_titles || { urn.language => title }
+      end
+    end
+  end
+end

data/lib/obp/access/renderer.rb

@@ -20,23 +20,35 @@ module Obp
       def render(node:, target: nil)
         return unless css_classes_match?(node)
 
-        ElementRegistry.elements.each do |element_class|
-          element = element_class.new(document:, metas:, node:)
-          next unless element.match_node?
-
-          xml = element.render(target:)
-          section_path = xml.first.path
+        element = matching_element(node)
+        # A nil xml means the element rendered nothing (e.g. an explicit
+        # skip element); recursion into its children halts with it.
+        xml = element&.render(target:)
+        render_children(node, xml) if xml
+      end
 
-          node.children.each do |child|
-            render(node: child, target: section_path)
-          end
+      def render_children(node, xml)
+        section_path = xml.first&.path
+        return unless section_path
 
-          xml
+        node.children.each do |child|
+          render(node: child, target: section_path)
         end
       end
 
+      # First match wins: the matching element with the most registered
+      # classes is the most specific (e.g. Terminology beats Section for
+      # "sts-section sts-tbx-sec"); ties resolve in registration order
+      # because max_by keeps the first maximum.
+      def matching_element(node)
+        ElementRegistry.elements.filter_map do |element_class|
+          element = element_class.new(document:, metas:, node:)
+          element if element.match_node?
+        end.max_by { |element| element.class.classes.size }
+      end
+
       def css_classes_match?(node)
-        ElementRegistry.css_classes.any?(node.classes)
+        ElementRegistry.css_classes.intersect?(node.classes)
       end
     end
   end

data/lib/obp/access/table_mapper.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+require "yaml"
+
+module Obp
+  class Access
+    # Maps the table-carrying nodes of an OBP preview HTML fragment —
+    # div.sts-table-wrap and div.sts-array, in document order — to a
+    # generic table model built from plain hashes, arrays and strings,
+    # ready for YAML serialization. This is an additive read of the same
+    # HTML source the Converter renders to STS XML; TableTermExtractor
+    # consumes this model.
+    class TableMapper
+      # Both wrapper kinds carry a single <table>.
+      WRAP_SELECTOR = "div.sts-table-wrap, div.sts-array"
+
+      # Section ids look like "toc_iso_std_iso_80000-12_ed-2_v2_en_sec_3";
+      # the clause key is whatever follows "_sec_" ("3", "1.1", "index").
+      SECTION_ID_PATTERN = /_sec_(.+)\z/
+
+      attr_reader :source
+
+      def initialize(source:)
+        @source = source
+      end
+
+      def tables
+        @tables ||= wraps.map { |node| map_table(node) }
+      end
+
+      def to_yaml
+        tables.to_yaml
+      end
+
+      private
+
+      def document
+        @document ||= Nokogiri::HTML(source.gsub(/[[:space:]]/, " "))
+      end
+
+      def wraps
+        @wraps ||= begin
+          nodes = document.css("body > div.sts-standard").css(WRAP_SELECTOR)
+          nodes.each { |node| node.css(".sts-unknown-element").each(&:remove) }
+          nodes
+        end
+      end
+
+      def map_table(node)
+        {
+          "id" => node["id"],
+          "section" => section_of(node),
+          "label" => text_of(node.at_css(".sts-caption-label")),
+          "caption" => caption_of(node),
+          "header" => map_rows(node.css("thead tr")),
+          "rows" => map_rows(body_rows(node)),
+        }.compact
+      end
+
+      def section_of(node)
+        section = node.ancestors("div").find do |ancestor|
+          ancestor.classes.include?("sts-section")
+        end
+        section&.[]("id")&.[](SECTION_ID_PATTERN, 1)
+      end
+
+      def caption_of(node)
+        text_of(node.at_css(".sts-caption-title")) ||
+          text_of(node.at_css(".sts-caption"))
+      end
+
+      # Body rows come from <tbody> when the source splits head/body;
+      # otherwise every <tr> outside a <thead> is a body row.
+      def body_rows(node)
+        rows = node.css("tbody tr")
+        return rows unless rows.empty?
+
+        node.css("table tr").reject { |tr| tr.ancestors("thead").any? }
+      end
+
+      def map_rows(rows)
+        rows.map do |tr|
+          tr.element_children.map { |cell| map_cell(cell) }
+        end.reject(&:empty?)
+      end
+
+      def map_cell(cell)
+        mapped = { "text" => cell_text(cell) }
+        %w[colspan rowspan].each do |attribute|
+          span = cell[attribute].to_i
+          mapped[attribute] = span if span > 1
+        end
+        mapped
+      end
+
+      # Plain stripped cell text. OBP's unknown-element placeholders are
+      # removed at the wrapper level; <br> and block-level div.sts-p
+      # boundaries become spaces so multi-paragraph cells do not glue
+      # words together.
+      def cell_text(cell)
+        copy = cell.dup
+        copy.css("br, div.sts-p").each do |node|
+          node.add_previous_sibling(" ")
+        end
+        copy.content.gsub(/[[:space:]]+/, " ").strip
+      end
+
+      def text_of(node)
+        return unless node
+
+        text = node.content.gsub(/[[:space:]]+/, " ").strip
+        text unless text.empty?
+      end
+    end
+  end
+end

data/lib/obp/access/table_term_extractor.rb

@@ -0,0 +1,248 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module Obp
+  class Access
+    # Extracts term entries from TableMapper's generic table model.
+    #
+    # Recognition rules, first match wins per table:
+    #
+    # 1. Quantity/item tables (the ISO 80000 family): at least 4 columns,
+    #    placed in a numbered clause (the terms clause), and a majority of
+    #    body rows whose first cell is an item number (e.g. "12-1.1").
+    #    Column roles come from header text when possible ("Quantity
+    #    Symbol", "Grandeur Symbole", ...), falling back to the ISO 80000
+    #    column order [item, designation, symbol, definition, unit,
+    #    remarks].
+    # 2. Equivalent-terms tables: a header row naming languages in at
+    #    least 2 columns; one entry per row with designations keyed by
+    #    the downcased language name.
+    #
+    # Anything else is not a term table and yields no entries.
+    class TableTermExtractor
+      # "12-1.1" or "4-1"; OBP sometimes prints U+2011 NON-BREAKING
+      # HYPHEN, normalized away by normalize_item.
+      ITEM_PATTERN = /\A\d+-\d+(?:\.\d+)?\z/
+
+      MIN_QUANTITY_COLUMNS = 4
+      MIN_LANGUAGE_COLUMNS = 2
+
+      # Terms clauses are numbered ("3", "3.2"); foreword, intro, bibl
+      # and index sections are not.
+      NUMERIC_SECTION = /\A\d+(\.\d+)*\z/
+
+      # Checked in order against each column's joined header text; the
+      # first unassigned column matching wins, so "Quantity Symbol"
+      # becomes "symbol" and not "designation".
+      COLUMN_ROLE_PATTERNS = [
+        ["symbol", /symbol|symbole/i],
+        ["definition", /definition|définition/i],
+        ["unit", /unit|unité/i],
+        ["remarks", /remark|remarque/i],
+        ["item", /item\s+no|n[°º]|numéro|no\./i],
+        ["designation", /quantity|grandeur|term|terme|name|nom/i],
+      ].freeze
+
+      # Fallback when header-text matching cannot place item+designation.
+      POSITIONAL_ROLES = %w[
+        item designation symbol definition unit remarks
+      ].freeze
+
+      # Language names recognized in equivalent-terms table headers.
+      LANGUAGES = %w[
+        english french français francais anglais russian russe русский
+        german deutsch allemand spanish español espagnol italian italiano
+        italien chinese chinois japanese japonais arabic arabe portuguese
+        portugais korean coréen
+      ].freeze
+
+      attr_reader :tables
+
+      def initialize(tables:)
+        @tables = tables
+      end
+
+      def entries
+        @entries ||= tables.flat_map { |table| table_entries(table) }
+      end
+
+      def to_yaml
+        entries.to_yaml
+      end
+
+      private
+
+      def table_entries(table)
+        if quantity_table?(table)
+          quantity_entries(table)
+        elsif equivalent_terms_table?(table)
+          equivalent_entries(table)
+        else
+          []
+        end
+      end
+
+      def quantity_table?(table)
+        return false unless NUMERIC_SECTION.match?(table["section"].to_s)
+        return false unless column_count(table) >= MIN_QUANTITY_COLUMNS
+
+        rows = table["rows"]
+        return false if rows.empty?
+
+        rows.count { |row| item_number?(cell_text(row, 0)) } * 2 > rows.size
+      end
+
+      def quantity_entries(table)
+        roles = column_roles(table)
+        seen = {}
+        table["rows"].filter_map do |row|
+          id = normalize_item(cell_text(row, roles["item"]))
+          next unless ITEM_PATTERN.match?(id)
+          next if seen[id]
+
+          seen[id] = true
+          quantity_entry(table, row, roles, id)
+        end
+      end
+
+      def quantity_entry(table, row, roles, id)
+        entry = { "id" => id,
+                  "designation" => cell_text(row, roles["designation"]).to_s }
+        %w[definition symbol unit remarks].each do |role|
+          text = cell_text(row, roles[role])
+          entry[role] = text unless text.nil? || text.empty?
+        end
+        entry["source"] = source_of(table)
+        entry
+      end
+
+      def equivalent_terms_table?(table)
+        !language_header_row(table).nil?
+      end
+
+      def equivalent_entries(table)
+        languages = language_columns(language_header_row(table))
+        seen = {}
+        table["rows"].each_with_index.filter_map do |row, index|
+          entry = equivalent_entry(table, row, languages, index)
+          next if entry.nil? || seen[entry["id"]]
+
+          seen[entry["id"]] = true
+          entry
+        end
+      end
+
+      def equivalent_entry(table, row, languages, index)
+        designations = languages.each_with_object({}) do |(column, lang), hash|
+          text = cell_text(row, column)
+          hash[lang] = text unless text.nil? || text.empty?
+        end
+        # Rows with a single designation are letter dividers ("A", "B",
+        # ...), not equivalences.
+        return if designations.size < 2
+
+        { "id" => entry_id(row, index),
+          "designations" => designations,
+          "source" => source_of(table) }
+      end
+
+      def entry_id(row, index)
+        first = normalize_item(cell_text(row, 0))
+        ITEM_PATTERN.match?(first) ? first : (index + 1).to_s
+      end
+
+      def language_header_row(table)
+        table["header"].find do |row|
+          row.count { |cell| language?(cell["text"]) } >= MIN_LANGUAGE_COLUMNS
+        end
+      end
+
+      def language_columns(header_row)
+        columns = {}
+        position = 0
+        header_row.each do |cell|
+          columns[position] = cell["text"].downcase if language?(cell["text"])
+          position += cell["colspan"] || 1
+        end
+        columns
+      end
+
+      def language?(text)
+        LANGUAGES.include?(text.to_s.downcase)
+      end
+
+      # First unassigned column matching each role pattern wins; matched
+      # columns are blanked so later roles cannot claim them again.
+      def column_roles(table)
+        headers = column_header_texts(table)
+        roles = {}
+        COLUMN_ROLE_PATTERNS.each do |role, pattern|
+          column = headers.index { |text| text.match?(pattern) }
+          next unless column
+
+          headers[column] = ""
+          roles[role] = column
+        end
+        return roles if roles["item"] && roles["designation"]
+
+        positional_roles(table)
+      end
+
+      def positional_roles(table)
+        POSITIONAL_ROLES.first(column_count(table)).each_with_index.to_h
+      end
+
+      # Each column's header text joined across header rows, colspans
+      # repeated ("Quantity" + "Symbol" => "Quantity Symbol").
+      def column_header_texts(table)
+        grid = table["header"].map { |row| expand_row(row) }
+        (0...grid_width(grid)).map do |column|
+          grid.filter_map { |row| row[column] }.reject(&:empty?).join(" ")
+        end
+      end
+
+      def grid_width(grid)
+        grid.map(&:size).max || 0
+      end
+
+      def expand_row(row)
+        row.flat_map { |cell| [cell["text"]] * (cell["colspan"] || 1) }
+      end
+
+      def column_count(table)
+        (table["header"] + table["rows"])
+          .map { |row| row.sum { |cell| cell["colspan"] || 1 } }.max || 0
+      end
+
+      def cell_text(row, column)
+        return if column.nil?
+
+        cell_at(row, column)&.fetch("text")
+      end
+
+      def cell_at(row, column)
+        position = 0
+        row.each do |cell|
+          span = cell["colspan"] || 1
+          return cell if column >= position && column < position + span
+
+          position += span
+        end
+        nil
+      end
+
+      def item_number?(text)
+        ITEM_PATTERN.match?(normalize_item(text))
+      end
+
+      def normalize_item(text)
+        text.to_s.tr("\u2011", "-")
+      end
+
+      def source_of(table)
+        { "table" => table["id"], "section" => table["section"] }.compact
+      end
+    end
+  end
+end

data/lib/obp/access/urn.rb

@@ -1,6 +1,8 @@
 module Obp
   class Access
     class Urn
+      DOC_TYPE_SEGMENTS = %w[ts tr pas guide iwa].freeze
+
       attr_reader :raw, :language, :base
 
       def initialize(raw)
@@ -26,6 +28,59 @@ module Obp
       def hash
         raw.hash
       end
+
+      def parts
+        @parts ||= raw.split(":")
+      end
+
+      # Originator segment: "iso", "iec", "itu", ...
+      def originator
+        parts[2]
+      end
+
+      # Document type segment if present ("ts", "tr", ...), else nil
+      def doc_type_segment
+        segments = identifier_segments
+        segments.find { |s| DOC_TYPE_SEGMENTS.include?(s) }
+      end
+
+      # Document number, including part number when present.
+      # "iso:std:iso:80000:-12:ed-2:v1:en" → "80000-12"
+      def doc_number
+        identifier_segments
+          .reject { |s| DOC_TYPE_SEGMENTS.include?(s) }
+          .join
+      end
+
+      def edition
+        segment = parts.find { |p| p.start_with?("ed-") }
+        segment&.delete_prefix("ed-")
+      end
+
+      def version
+        segment = parts.find { |p| p.match?(/\Av\d+\z/) }
+        segment&.delete_prefix("v")
+      end
+
+      def doc_type
+        case doc_type_segment
+        when "ts" then "TS"
+        when "tr" then "TR"
+        when "pas" then "PAS"
+        when "guide" then "Guide"
+        when "iwa" then "IWA"
+        else "IS"
+        end
+      end
+
+      private
+
+      # Segments between the originator and the edition segment.
+      def identifier_segments
+        start_index = 3
+        end_index = parts.index { |p| p.start_with?("ed-") } || parts.size
+        parts[start_index...end_index]
+      end
     end
   end
 end

data/lib/obp/access/version.rb

@@ -1,5 +1,5 @@
 module Obp
   class Access
-    VERSION = "0.1.4".freeze
+    VERSION = "0.1.6".freeze
   end
 end