relaton 2.1.0 → 3.0.0.pre.alpha.1

data/lib/relaton/bsi/scraper.rb

@@ -0,0 +1,288 @@
+# frozen_string_literal: true
+
+require "graphql/client"
+require "graphql/client/http"
+
+Encoding.default_internal = Encoding::UTF_8
+Encoding.default_external = Encoding::UTF_8
+
+module Relaton::Bsi
+  # Scraper.
+  module Scraper
+    HTTP = GraphQL::Client::HTTP.new "https://shop-bsi.myshopify.com/api/2021-04/graphql.json" do
+      def headers(_context)
+        { "x-shopify-storefront-access-token": "c935c196c0b7d1d86bfb5139006cfd46" }
+      end
+    end
+
+    Schema = GraphQL::Client.load_schema File.join(__dir__, "schema.json")
+
+    Client = GraphQL::Client.new(schema: Schema, execute: HTTP)
+
+    Product = Client.parse <<~'GRAPHQL'
+      fragment ProductFragment on Product {
+        createdAt
+        publishedAt
+        updatedAt
+        productType
+        committee: metafield(namespace: "global", key: "committee") {
+          value
+        }
+        designated: metafield(namespace: "global", key: "designatedStandard") {
+          value
+        }
+        packContents: metafield(namespace: "global", key: "packContents") {
+          value
+        }
+        summary: metafield(namespace: "global", key: "summary") {
+          value
+        }
+        corrigendumHandle: metafield(namespace: "global", key: "corrigendumHandle") {
+          value
+        }
+        variants(first: 250) {
+          edges {
+            node {
+              version: metafield(namespace: "global", key: "version") {
+                value
+              }
+              isbn: metafield(namespace: "global", key: "isbn") {
+                value
+              }
+            }
+          }
+        }
+        description
+      }
+    GRAPHQL
+
+    Query = Client.parse <<~GRAPHQL
+      query GetProducts($h0: String!) {
+        productByHandle(handle: $h0) {
+          ...Relaton::Bsi::Scraper::Product::ProductFragment
+        }
+      }
+    GRAPHQL
+
+    class << self
+      # Parse page.
+      # @param hit [Relaton::Bsi::Hit]
+      # @return [Relaton::Bsi::IetemData]
+      def parse_page(hit) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        # doc = hit.hit_collection.agent.get hit.hit[:url]
+        result = Client.query(Query::GetProducts, variables: { h0: hit.hit[:url] })
+        data = result.data.product_by_handle.to_h
+        item = Relaton::Bsi::ItemData.new(
+          fetched: Date.today.to_s,
+          type: "standard",
+          docidentifier: fetch_docid(hit.hit[:code], data),
+          docnumber: hit.hit[:code].match(/\d+/).to_s,
+          language: ["en"],
+          script: ["Latn"],
+          title: fetch_titles(hit.hit[:title]),
+          status: fetch_status(hit.hit[:status]),
+          date: fetch_dates(hit),
+          contributor: fetch_contributors(hit, data),
+          abstract: fetch_abstract(data),
+          copyright: fetch_copyright(hit),
+          source: fetch_link(hit.hit[:url]),
+          # relation: fetch_relations(doc),
+          place: [Relaton::Bib::Place.new(city: "London")],
+          ext: fetch_ext(hit, data),
+        )
+        item.create_id
+        item
+      end
+
+      private
+
+      def fetch_ext(hit, data)
+        Ext.new(
+          doctype: fetch_doctype(hit),
+          flavor: "bsi",
+          ics: fetch_ics(hit.hit[:ics]),
+          structuredidentifier: fetch_structuredid(hit),
+        )
+      end
+
+      # @param ics [Array<String>]
+      # @return [Array<Relaton::Bib::ICS>]
+      def fetch_ics(ics)
+        ics.map do |s|
+          code, = s.split
+          Relaton::Bib::ICS.new(code: code, text: Isoics.fetch(code).description)
+        end
+      end
+
+      # Fetch abstracts.
+      # @param data [Hash]
+      # @return [Array<Relaton::Bib::Abstract>]
+      def fetch_abstract(data)
+        return [] unless data["description"]
+
+        [Relaton::Bib::Abstract.new(content: data["description"], language: "en", script: "Latn")]
+      end
+
+      # Fetch docid.
+      # @param docid [String]
+      # @param data [Hash]
+      # @return [Array<Relaton::Bib::Docidentifier>]
+      def fetch_docid(docid, data) # rubocop:disable Metrics/AbcSize
+        ids = [{ type: "BSI", content: docid, primary: true }]
+        if data.any? && data["variants"]["edges"][0]["node"]["isbn"]
+          isbn = data["variants"]["edges"][0]["node"]["isbn"]["value"]
+          ids << { type: "ISBN", content: isbn }
+        end
+        ids.map do |did|
+          Relaton::Bsi::Docidentifier.new(**did)
+        end
+      end
+
+      # Fetch status.
+      # @param status [String]
+      # @return [Relaton::Bib::Status, nil]
+      def fetch_status(status)
+        return unless status
+
+        stage = Relaton::Bib::Status::Stage.new(content: status)
+        Relaton::Bib::Status.new(stage: stage)
+      end
+
+      # @param hit [Relaton::Bsi::Hit]
+      # @return [Relaton::Iso::StructuredIdentifier]
+      def fetch_structuredid(hit)
+        content, origyr = hit.hit[:code].split(":")
+        project_number = Relaton::Iso::ProjectNumber.new(content: content, origyr: origyr)
+        Relaton::Iso::StructuredIdentifier.new project_number: project_number
+      end
+
+      # Fetch relations.
+      # @param doc [Mechanize::Page]
+      # @return [Array<Hash>]
+      # def fetch_relations(doc)
+      #   doc.xpath("//tr[th='Replaces']/td/a").map do |r|
+      #     fref = RelatonBib::FormattedRef.new(content: r.text, language: "en", script: "Latn")
+      #     link = fetch_link r[:href]
+      #     bibitem = BsiBibliographicItem.new(formattedref: fref, type: "standard", link: link)
+      #     { type: "complements", bibitem: bibitem }
+      #   end
+      # end
+
+      # Fetch titles.
+      # @param title [String]
+      # @return [Array<Relaton::Bib::Title>]
+      def fetch_titles(title)
+        titles = split_title(title).map do |t|
+          Relaton::Bib::Title.new(**t, language: "en", script: "Latn")
+        end
+        titles << Relaton::Bib::Title.new(type: "main", content: title, language: "en", script: "Latn")
+      end
+
+      def split_title(title)
+        ttls = title.split(/\s(?:-|\u2014)\s/) # if ttls.size == 1
+        case ttls.size
+        when 0, 1 then [{ type: "title-main",  content: ttls.first }]
+        else intro_or_part ttls
+        end
+      end
+
+      # @param ttls [Array<String>]
+      # @return [Array<Hash>]
+      def intro_or_part(ttls)
+        titles = [{ type: "title-intro", content: ttls[0] }, { type: "title-main", content: ttls[1] }]
+        parts = ttls.slice(2..-1)
+        titles << { type: "title-part", content: parts.join(" -- ") } if parts.any?
+        titles
+      end
+
+      #
+      # Fetch doctype.
+      #
+      # @param [Relaton::Bsi::Hit] hit hit
+      #
+      # @return [Relaton::Bsi::Doctype] doctype
+      #
+      def fetch_doctype(hit)
+        Doctype.new(content: doctype(hit))
+      end
+
+      def doctype(hit)
+        case hit.hit[:code]
+        when /(^|\s)Flex\s/ then "flex-standard"
+        when /(^|\s)PAS\s/ then "publicly-available-specification"
+        else hit.hit[:doctype]
+        end
+      end
+
+      # Fetch dates
+      # @param hit [Relaton::Bsi:Hit]
+      # @return [Array<Relaton::Bib::Date>]
+      def fetch_dates(hit)
+        [Relaton::Bib::Date.new(type: "published", at: hit.hit[:date])]
+      end
+
+      # Fetch contributors
+      # @param hit [Relaton::Bsi::Hit]
+      # @param data [Hash]
+      # @return [Array<Relaton::Bib::Contributor>]
+      def fetch_contributors(hit, data)
+        contribs = [create_contributor({ type: "publisher" }, owner_entity(hit))]
+
+        wg = data["committee"]&.fetch("value")
+        if wg
+          subdivision = Relaton::Bib::Subdivision.new(
+            type: "technical-committee",
+            name: [Relaton::Bib::TypedLocalizedString.new(content: wg)],
+          )
+          org = bsi_org(subdivision: [subdivision])
+          desc = Relaton::Bib::LocalizedMarkedUpString.new(content: "committee")
+          role = Relaton::Bib::Contributor::Role.new(type: "author", description: [desc])
+          contribs << Relaton::Bib::Contributor.new(role: [role], organization: org)
+        end
+        contribs
+      end
+
+      def create_contributor(role, org)
+        role = Relaton::Bib::Contributor::Role.new(**role)
+        Relaton::Bib::Contributor.new(role: [role], organization: org)
+      end
+
+      # Fetch links.
+      # @param path [String]
+      # @return [Array<Relaton::Bib::Uri>]
+      def fetch_link(path)
+        url = "#{HitCollection::DOMAIN}/products/#{path}"
+        [Relaton::Bib::Uri.new(type: "src", content: url)]
+      end
+
+      # Fetch copyright.
+      # @param hit [Relaton::Bsi::Hit]
+      # @return [Array<Relaton::Bib::Copyright>]
+      def fetch_copyright(hit)
+        owner = Relaton::Bib::ContributionInfo.new organization: owner_entity(hit)
+        from = Date.parse(hit.hit[:date]).year.to_s
+        [Relaton::Bib::Copyright.new(owner: [owner], from: from)]
+      end
+
+      # @param hit [Relaton::Bsi::Hit]
+      # @return [Relaton::Bib::Organization]
+      def owner_entity(hit)
+        case hit.hit[:publisher]
+        when "BSI" then bsi_org
+        else
+          name = Relaton::Bib::TypedLocalizedString.new(content: hit.hit[:publisher])
+          Relaton::Bib::Organization.new(name: [name])
+        end
+      end
+
+      def bsi_org(subdivision: [])
+        abbre = Relaton::Bib::LocalizedString.new(content: "BSI", language: "en", script: "Latn")
+        name = Relaton::Bib::TypedLocalizedString.new(
+          content: "British Standards Institution", language: "en", script: "Latn"
+        )
+        uri = Relaton::Bib::Uri.new(content: "https://www.bsigroup.com/")
+        Relaton::Bib::Organization.new abbreviation: abbre, name: [name], uri: [uri], subdivision: subdivision
+      end
+    end
+  end
+end

data/lib/relaton/bsi/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Bsi
+    module Util
+      extend Bib::Util
+      PROGNAME = "relaton-bsi".freeze
+    end
+  end
+end

data/lib/relaton/bsi.rb

@@ -0,0 +1,25 @@
+require "relaton/iso"
+require_relative "version"
+require_relative "bsi/util"
+require_relative "bsi/item_data"
+require_relative "bsi/model/item"
+require_relative "bsi/model/relation"
+require_relative "bsi/model/bibitem"
+require_relative "bsi/model/bibdata"
+require_relative "bsi/scraper"
+require_relative "bsi/hit_collection"
+require_relative "bsi/hit"
+require_relative "bsi/bibliography"
+
+module Relaton
+  module Bsi
+    # Returns hash of XML greammar
+    # @return [String]
+    def self.grammar_hash
+      # gem_path = File.expand_path "..", __dir__
+      # grammars_path = File.join gem_path, "grammars", "*"
+      # grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end

data/lib/relaton/calconnect/bibliography.rb

@@ -0,0 +1,95 @@
+require "mechanize"
+
+module Relaton::Calconnect
+  class Bibliography
+    class << self
+      # @param text [String]
+      # @return [RelatonCalconnect::HitCollection]
+      def search(text, year = nil, _opts = {})
+        HitCollection.new text, year
+      rescue Mechanize::ResponseCodeError, SocketError, Errno::ECONNREFUSED
+        raise Relaton::RequestError, "Could not access https://standards.calconnect.org"
+      end
+
+      # @param ref [String] the OGC standard Code to look up (e..g "8200")
+      # @param year [String] the year the standard was published (optional)
+      #
+      # @param opts [Hash] options
+      # @option opts [TrueClass, FalseClass] :all_parts restricted to all parts
+      #   if all-parts reference is required
+      # @option opts [TrueClass, FalseClass] :bibdata
+      #
+      # @return [RelatonCalconnect::CcBibliographicItem]
+      def get(ref, year = nil, opts = {}) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+        code = ref
+
+        if year.nil?
+          /^(?<code1>[^\s]+(?:\s\w+)?\s[\d-]+):?(?<year1>\d{4})?/ =~ ref
+          unless code1.nil?
+            code = code1
+            year = year1
+          end
+        end
+
+        Util.info "Fetching from Relaton repository ...", key: ref
+        result = search(code, year, opts) || (return nil)
+        ret = bib_results_filter(result, year)
+        if ret[:ret]
+          Util.info "Found: `#{ret[:ret].docidentifier.first.content}`", key: ref
+          ret[:ret]
+        else
+          Util.info "Not found.", key: ref
+          fetch_ref_err(code, year, ret[:years])
+        end
+      end
+
+      private
+
+      # Sort through the results from RelatonNist, fetching them three at a time,
+      # and return the first result that matches the code,
+      # matches the year (if provided), and which # has a title (amendments do not).
+      # Only expects the first page of results to be populated.
+      # Does not match corrigenda etc (e.g. ISO 3166-1:2006/Cor 1:2007)
+      # If no match, returns any years which caused mismatch, for error reporting
+      #
+      # @param result
+      # @param opts [Hash] options
+      #
+      # @return [Hash]
+      def bib_results_filter(result, year)
+        missed_years = Set.new
+        result.each do |r|
+          item = r.item
+          item.fetched = Date.today.to_s
+          return { ret: item } if !year
+
+          /:(?<id_year>\d{4})$/ =~ r.hit[:id]
+          return { ret: item } if year.to_i == id_year.to_i
+
+          missed_years << id_year.to_i if id_year
+
+          item.date.select { |d| d.type == "published" }.each do |d|
+            return { ret: item } if year.to_i == d.at.to_date.year
+
+            missed_years << d.at.to_date.year
+          end
+        end
+        { years: missed_years }
+      end
+
+      # @param code [Strig]
+      # @param year [String]
+      # @param missed_years [Array<Strig>]
+      def fetch_ref_err(code, year, missed_years)
+        # id = year ? "`#{code}` year `#{year}`" : code
+        # Util.info "WARNING: No match found online for #{id}. " \
+        #           "The code must be exactly like it is on the standards website."
+        unless missed_years.empty?
+          Util.info "There was no match for `#{year}`, though there " \
+                    "were matches found for `#{missed_years.join('`, `')}`."
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/relaton/calconnect/data_fetcher.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal:true
+
+require "json"
+require "mechanize"
+require "relaton/core"
+require "relaton/index"
+require_relative "scraper"
+require_relative "util"
+
+module Relaton::Calconnect
+  #
+  # Relaton-calconnect data fetcher
+  #
+  class DataFetcher < Relaton::Core::DataFetcher
+    ENDPOINT = "https://standards.calconnect.org/cc/index.json"
+
+    def etagfile
+      @etagfile ||= File.join @output, "etag.txt"
+    end
+
+    def index
+      @index = Relaton::Index.find_or_create :CC, file: "index-v1.yaml"
+    end
+
+    def log_error(msg)
+      Util.error msg
+    end
+
+    def agent
+      @agent ||= Mechanize.new
+    end
+
+    #
+    # fetch data form server and save it to file.
+    #
+    def fetch(_source = nil) # rubocop:disable Metrics/AbcSize
+      agent.request_headers["If-None-Match"] = etag if etag
+      resp = agent.get(ENDPOINT)
+      # 304 Not Modified — nothing changed since the last fetch
+      return if resp.code == "304"
+
+      data = JSON.parse resp.body
+      all_success = true
+      Array(data["documents"]).each { |doc| all_success &&= parse_page doc }
+      self.etag = resp.response["etag"] if all_success
+      index.save
+      report_errors
+    end
+
+    private
+
+    #
+    # Parse document and write it to file
+    #
+    # @param [Hash] doc
+    #
+    def parse_page(doc)
+      bib = Scraper.new(@errors).parse_page doc
+      write_doc doc["id"], bib
+      true
+    rescue StandardError => e
+      Util.warn "Document: #{doc['id']}"
+      Util.warn e.message
+      Util.warn e.backtrace[0..5].join("\n")
+      false
+    end
+
+    def write_doc(slug, bib) # rubocop:disable Metrics/MethodLength
+      file = output_file slug
+      if @files.include? file
+        Util.warn "#{file} exist"
+      else
+        @files << file
+      end
+      index.add_or_update primary_docid(bib), file
+      File.write file, serialize(bib), encoding: "UTF-8"
+    end
+
+    # Index entries are keyed by the canonical doc identifier
+    # (e.g. "CC/DIR 10005:2019"), not the upstream slug used for filenames.
+    def primary_docid(bib)
+      docid = bib.docidentifier.find(&:primary) || bib.docidentifier.first
+      docid.content
+    end
+
+    def to_yaml(bib) = bib.to_yaml
+    def to_xml(bib) = bib.to_xml(bibdata: true)
+    def to_bibxml(bib) = bib.to_rfcxml
+
+    #
+    # Read ETag from file
+    #
+    # @return [String, NilClass]
+    def etag
+      @etag ||= File.exist?(etagfile) ? File.read(etagfile, encoding: "UTF-8") : nil
+    end
+
+    #
+    # Save ETag to file
+    #
+    # @param tag [String]
+    def etag=(e_tag)
+      File.write etagfile, e_tag, encoding: "UTF-8"
+    end
+  end
+end

data/lib/relaton/calconnect/hit.rb

@@ -0,0 +1,14 @@
+require "mechanize"
+
+module Relaton::Calconnect
+  class Hit < Relaton::Core::Hit
+    # Parse page.
+    # @return [Relaton::Calconnect::ItemData]
+    def item
+      @item ||= begin
+        url = "#{HitCollection::GHURL}#{@hit[:file]}"
+        Item.from_yaml Mechanize.new.get(url).body
+      end
+    end
+  end
+end

data/lib/relaton/calconnect/hit_collection.rb

@@ -0,0 +1,84 @@
+require "yaml"
+require "fileutils"
+
+module Relaton::Calconnect
+  class HitCollection < Relaton::Core::HitCollection
+    # ENDPOINT = "https://standards.calconnect.org/relaton/index.yaml".freeze
+    # ENDPOINT = "http://127.0.0.1:4000/relaton/index.yaml".freeze
+    # DATADIR = File.expand_path ".relaton/calconnect", Dir.home
+    # DATAFILE = File.expand_path "bibliography.yml", DATADIR
+    # ETAGFILE = File.expand_path "etag.txt", DATADIR
+    GHURL = "https://raw.githubusercontent.com/relaton/relaton-data-calconnect/refs/heads/v2/".freeze
+
+    # @param ref [Strig]
+    # @param year [String]
+    # @param opts [Hash]
+    def initialize(ref, year = nil)
+      super
+      # @array = from_yaml(ref).sort_by do |hit|
+      #   hit.hit["revdate"] ? Date.parse(hit.hit["revdate"]) : Date.new
+      # end.reverse
+      index = Relaton::Index.find_or_create :CC, url: "#{GHURL}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml"
+      @array = index.search(ref).map do |row|
+        Hit.new(row, self)
+      end
+    end
+
+    # private
+
+    #
+    # Fetch data from yaml
+    #
+    # @param docid [String]
+    #
+    # @return [Array<RelatonBib::Hit>]
+    #
+    # def from_yaml(docid, **_opts)
+    #   data["root"]["items"].select do |doc|
+    #     doc["docid"] && doc["docid"]["id"].include?(docid)
+    #   end.map { |h| Hit.new(h, self) }
+    # end
+
+    #
+    # Fetches YAML data
+    #
+    # @return [Hash]
+    # def data
+    #   FileUtils.mkdir_p DATADIR
+    #   ctime = File.ctime DATAFILE if File.exist? DATAFILE
+    #   fetch_data if !ctime || ctime.to_date < Date.today
+    #   @data ||= YAML.safe_load File.read(DATAFILE, encoding: "UTF-8")
+    # end
+
+    #
+    # fetch data from server and save it to file.
+    #
+    # def fetch_data
+    #   resp = Faraday.new(ENDPOINT, headers: { "If-None-Match" => etag }).get
+    #   # return if there aren't any changes since last fetching
+    #   return unless resp.status == 200
+
+    #   self.etag = resp[:etag]
+    #   @data = YAML.safe_load resp.body
+    #   File.write DATAFILE, @data.to_yaml, encoding: "UTF-8"
+    # end
+
+    #
+    # Read ETag from file
+    #
+    # @return [String, NilClass]
+    # def etag
+    #   @etag ||= if File.exist? ETAGFILE
+    #               File.read ETAGFILE, encoding: "UTF-8"
+    #             end
+    # end
+
+    #
+    # Save ETag to file
+    #
+    # @param tag [String]
+    # def etag=(e_tag)
+    #   File.write ETAGFILE, e_tag, encoding: "UTF-8"
+    # end
+  end
+end

data/lib/relaton/calconnect/item_data.rb

@@ -0,0 +1,12 @@
+module Relaton
+  module Calconnect
+    class ItemData < Relaton::Bib::ItemData
+      def create_id(without_date: false)
+        docid = docidentifier.find(&:primary) || docidentifier.first
+        return unless docid
+
+        self.id = docid.content.gsub(/\W+/, "")
+      end
+    end
+  end
+end

data/lib/relaton/calconnect/model/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Calconnect
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/calconnect/model/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Calconnect
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/calconnect/model/doctype.rb

@@ -0,0 +1,11 @@
+module Relaton
+  module Calconnect
+    class Doctype < Bib::Doctype
+      TYPES = %W[
+        directive guide specification standard report administrative amendment technical\scorrigendum advisory
+      ].freeze
+
+      attribute :type, :string, values: TYPES
+    end
+  end
+end

data/lib/relaton/calconnect/model/ext.rb

@@ -0,0 +1,11 @@
+require_relative "doctype"
+
+module Relaton
+  module Calconnect
+    class Ext < Bib::Ext
+      attribute :doctype, Doctype
+
+      def get_schema_version = Relaton.schema_versions["relaton-model-cc"]
+    end
+  end
+end

data/lib/relaton/calconnect/model/item.rb

@@ -0,0 +1,13 @@
+require "relaton/bib"
+require_relative "../item_data"
+require_relative "ext"
+
+module Relaton
+  module Calconnect
+    class Item < Bib::Item
+      model ItemData
+
+      attribute :ext, Ext
+    end
+  end
+end