relaton-bsi 1.8.4 → 1.9.0

data/lib/relaton_bsi/scrapper.rb

@@ -1,82 +1,151 @@
 # frozen_string_literal: true
 
+require "graphql/client"
+require "graphql/client/http"
+
+Encoding.default_internal = Encoding::UTF_8
+Encoding.default_external = Encoding::UTF_8
+
 module RelatonBsi
   # Scrapper.
   module Scrapper
+    HTTP = GraphQL::Client::HTTP.new "https://shop-bsi.myshopify.com/api/2021-04/graphql.json" do
+      def headers(_context)
+        { "x-shopify-storefront-access-token": "c935c196c0b7d1d86bfb5139006cfd46" }
+      end
+    end
+
+    Schema = GraphQL::Client.load_schema File.join(__dir__, "schema.json")
+
+    Client = GraphQL::Client.new(schema: Schema, execute: HTTP)
+
+    Product = Client.parse <<~'GRAPHQL'
+      fragment ProductFragment on Product {
+        createdAt
+        publishedAt
+        updatedAt
+        productType
+        committee: metafield(namespace: "global", key: "committee") {
+          value
+        }
+        designated: metafield(namespace: "global", key: "designatedStandard") {
+          value
+        }
+        packContents: metafield(namespace: "global", key: "packContents") {
+          value
+        }
+        summary: metafield(namespace: "global", key: "summary") {
+          value
+        }
+        corrigendumHandle: metafield(namespace: "global", key: "corrigendumHandle") {
+          value
+        }
+        variants(first: 250) {
+          edges {
+            node {
+              version: metafield(namespace: "global", key: "version") {
+                value
+              }
+              isbn: metafield(namespace: "global", key: "isbn") {
+                value
+              }
+            }
+          }
+        }
+        description
+      }
+    GRAPHQL
+
+    Query = Client.parse <<~GRAPHQL
+      query GetProducts($h0: String!) {
+        productByHandle(handle: $h0) {
+          ...RelatonBsi::Scrapper::Product::ProductFragment
+        }
+      }
+    GRAPHQL
+
     class << self
       # Parse page.
       # @param hit [RelatonBsi::Hit]
       # @return [Hash]
       def parse_page(hit) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-        doc = hit.hit_collection.agent.get hit.hit[:url]
+        # doc = hit.hit_collection.agent.get hit.hit[:url]
+        result = Client.query(Query::GetProducts, variables: { h0: hit.hit[:url] })
+        data = result.data.product_by_handle.to_h
         BsiBibliographicItem.new(
           fetched: Date.today.to_s,
           type: "standard",
-          docid: fetch_docid(doc),
+          docid: fetch_docid(hit.hit[:code], data),
           language: ["en"],
           script: ["Latn"],
-          title: fetch_titles(doc),
-          doctype: "specification",
-          docstatus: fetch_status(doc),
-          ics: fetch_ics(doc),
+          title: fetch_titles(hit.hit[:title]),
+          doctype: hit.hit[:doctype],
+          docstatus: fetch_status(hit.hit[:status]),
+          ics: fetch_ics(hit.hit[:ics]),
           date: fetch_dates(hit),
-          contributor: fetch_contributors(doc),
-          editorialgroup: fetch_editorialgroup(doc),
+          contributor: fetch_contributors(hit),
+          editorialgroup: fetch_editorialgroup(data),
           structuredidentifier: fetch_structuredid(hit),
-          abstract: fetch_abstract(doc),
-          copyright: fetch_copyright(doc, hit),
-          link: fetch_link(HitCollection::DOMAIN + hit.hit[:url]),
-          relation: fetch_relations(doc),
-          place: ["London"]
+          abstract: fetch_abstract(data),
+          copyright: fetch_copyright(hit),
+          link: fetch_link(hit.hit[:url]),
+          # relation: fetch_relations(doc),
+          place: ["London"],
         )
       end
 
       private
 
-      # @param doc [Mechanize::Page]
+      # @param ics [Array<String>]
       # @return [Array<RelatonIsobib::Ics>]
-      def fetch_ics(doc)
-        doc.xpath("//tr[th='ICS']/td/node()").map(&:text).reject { |a| a.empty? }.map do |ics|
-          RelatonIsoBib::Ics.new(ics)
+      def fetch_ics(ics)
+        ics.map do |s|
+          code, = s.split
+          RelatonIsoBib::Ics.new(code)
         end
       end
 
       # Fetch abstracts.
-      # @param doc [Mechanize::Page]
+      # @param data [Hash]
       # @return [Array<Hash>]
-      def fetch_abstract(doc)
-        content = doc.at("//tr[th='Descriptors']/td")
-        [{ content: content.text, language: "en", script: "Latn", }]
+      def fetch_abstract(data)
+        return [] unless data["description"]
+
+        [{ content: data["description"], language: "en", script: "Latn" }]
       end
 
       # Fetch docid.
-      # @param doc [Mechanize::Page]
-      # @return [Array<Hash>]
-      def fetch_docid(doc)
-        docids = []
-        docid = doc.at("//tr[th[.='Standard Number']]/td").text
-        docids << RelatonBib::DocumentIdentifier.new(type: "BSI", id: docid)
-        isbn = doc.at("//tr[th[.='ISBN']]/td").text
-        docids << RelatonBib::DocumentIdentifier.new(type: "ISBN", id: isbn)
-        docids
+      # @param docid [String]
+      # @param data [Hash]
+      # @return [Array<RelatonBib::DocumentIdentifier>]
+      def fetch_docid(docid, data)
+        ids = [{ type: "BSI", id: docid }]
+        if data.any?
+          isbn = data["variants"]["edges"][0]["node"]["isbn"]["value"]
+          ids << { type: "ISBN", id: isbn }
+        end
+        ids.map do |did|
+          RelatonBib::DocumentIdentifier.new(type: did[:type], id: did[:id])
+        end
       end
 
       # Fetch status.
-      # @param doc [Mechanize::Page]
-      # @return [RelatonBib::DocumentStatus, NilClass]
-      def fetch_status(doc)
-        s = doc.at("//tr[th='Status']/td")
-        return unless s
+      # @param status [String]
+      # @return [RelatonBib::DocumentStatus, nil]
+      def fetch_status(status)
+        return unless status
 
-        RelatonBib::DocumentStatus.new(stage: s.text)
+        RelatonBib::DocumentStatus.new(stage: status)
       end
 
       # Fetch workgroup.
-      # @param doc [Mechanize::Page]
+      # @param data [Hash]
       # @return [RelatonIsoBib::EditorialGroup]
-      def fetch_editorialgroup(doc)
-        wg = doc.at("//tr[th='Committee']/td")
-        tc = RelatonIsoBib::IsoSubgroup.new name: wg.text
+      def fetch_editorialgroup(data)
+        wg = data["committee"]&.fetch("value")
+        return unless wg
+
+        tc = RelatonBib::WorkGroup.new name: wg
         RelatonIsoBib::EditorialGroup.new technical_committee: [tc]
       end
 
@@ -89,74 +158,63 @@ module RelatonBsi
       # Fetch relations.
       # @param doc [Mechanize::Page]
       # @return [Array<Hash>]
-      def fetch_relations(doc)
-        doc.xpath("//tr[th='Replaces']/td/a").map do |r|
-          fref = RelatonBib::FormattedRef.new(content: r.text, language: "en", script: "Latn")
-          link = fetch_link r[:href]
-          bibitem = BsiBibliographicItem.new(formattedref: fref, type: "standard", link: link)
-          { type: "complements", bibitem: bibitem }
-        end
-      end
+      # def fetch_relations(doc)
+      #   doc.xpath("//tr[th='Replaces']/td/a").map do |r|
+      #     fref = RelatonBib::FormattedRef.new(content: r.text, language: "en", script: "Latn")
+      #     link = fetch_link r[:href]
+      #     bibitem = BsiBibliographicItem.new(formattedref: fref, type: "standard", link: link)
+      #     { type: "complements", bibitem: bibitem }
+      #   end
+      # end
 
       # Fetch titles.
-      # @param doc [Mechanize::Page]
-      # @return [Array<Hash>]
-      def fetch_titles(doc)
-        te = doc.at("//div[@id='title']/h2").text.strip
-        ttls = RelatonBib::TypedTitleString.from_string te, "en", "Latn"
-        tf = doc.at("//tr[th[.='Title in French']]/td")
-        if tf
-          ttls += RelatonBib::TypedTitleString.from_string tf.text.strip, "fr", "Latn"
-        end
-        tf = doc.at("//tr[th[.='Title in German']]/td")
-        if tf
-          ttls += RelatonBib::TypedTitleString.from_string tf.text.strip, "de", "Latn"
-        end
-        ttls
+      # @param title [String]
+      # @return [RelatonBib::TypedTitleStringCollection]
+      def fetch_titles(title)
+        RelatonBib::TypedTitleString.from_string title, "en", "Latn"
       end
 
       # Fetch dates
       # @param hit [RelatonBsi:Hit]
       # @return [Array<Hash>]
       def fetch_dates(hit)
-        [{ type: "published", on: hit.hit[:date].to_s }]
+        [{ type: "published", on: hit.hit[:date] }]
       end
 
       # Fetch contributors
-      # @param doc [Mechanize::Page]
+      # @param hit [RelatonBsi::Hit]
       # @return [Array<Hash>]
-      def fetch_contributors(doc)
+      def fetch_contributors(hit)
         contrib = { role: [type: "publisher"] }
-        contrib[:entity] = owner_entity doc
+        contrib[:entity] = owner_entity hit
         [contrib]
       end
 
       # Fetch links.
-      # @param url [String]
+      # @param path [String]
       # @return [Array<Hash>]
-      def fetch_link(url)
+      def fetch_link(path)
+        url = "#{HitCollection::DOMAIN}/products/#{path}"
         [{ type: "src", content: url }]
       end
 
       # Fetch copyright.
-      # @param doc [Mechanize::Page]
       # @param hit [RelatonBsi::Hit]
       # @return [Array<Hash>]
-      def fetch_copyright(doc, hit)
-        owner = owner_entity doc
-        from = hit.hit[:date].year.to_s
+      def fetch_copyright(hit)
+        owner = owner_entity hit
+        from = Date.parse(hit.hit[:date]).year.to_s
         [{ owner: [owner], from: from }]
       end
 
-      # @param doc [Mechanize::Page]
+      # @param hit [RelatonBsi::Hit]
       # @return [Hash]
-      def owner_entity(doc)
-        abbrev = doc.at("//tr[th='Publisher']/td").text
-        case abbrev
+      def owner_entity(hit)
+        case hit.hit[:publisher]
         when "BSI"
-          { abbreviation: abbrev, name: "British Standards Institution", url: "https://www.bsigroup.com/" }
+          { abbreviation: hit.hit[:publisher], name: "British Standards Institution", url: "https://www.bsigroup.com/" }
         else
-          { name: abbrev }
+          { name: hit.hit[:publisher] }
         end
       end
     end

data/lib/relaton_bsi/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RelatonBsi
-  VERSION = "1.8.4"
+  VERSION = "1.9.0"
 end

data/lib/relaton_bsi/xml_parser.rb

@@ -13,15 +13,13 @@ module RelatonBsi
         ext = isoitem.at "./ext"
         return data unless ext
 
-        data[:price_code] = ext.at("./price-code")&.text
-        data[:cen_processing] = ext.at("./cen-processing")&.text
         data
       end
 
       # @param item_hash [Hash]
       # @return [RelatonBsi::BsiBibliographicItem]
       def bib_item(item_hash)
-        hash = BsiBibliographicItem.new **item_hash
+        BsiBibliographicItem.new(**item_hash)
       end
     end
   end

data/relaton_bsi.gemspec

@@ -29,21 +29,15 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  # Uncomment to register a new dependency of your gem
-  # spec.add_dependency "example-gem", "~> 1.0"
-
-  # For more information and examples about making a new gem, checkout our
-  # guide at: https://bundler.io/guides/creating_gem.html
   spec.add_development_dependency "equivalent-xml", "~> 0.6"
   spec.add_development_dependency "pry-byebug"
-  # spec.add_development_dependency "rake", "~> 13.0"
-  # spec.add_development_dependency "rspec", "~> 3.0"
-  # spec.add_development_dependency "ruby-debug-ide"
   spec.add_development_dependency "ruby-jing"
   spec.add_development_dependency "simplecov"
   spec.add_development_dependency "vcr", "~> 5.0.0"
   spec.add_development_dependency "webmock"
 
-  spec.add_dependency "mechanize", "~> 2.8.0"
-  spec.add_dependency "relaton-iso-bib", "~> 1.8.0"
+  spec.add_dependency "algolia", "~> 2.1.1"
+  spec.add_dependency "graphql-client", "~> 0.16.0"
+  # spec.add_dependency "mechanize", "~> 2.8.0"
+  spec.add_dependency "relaton-iso-bib", "~> 1.9.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-bsi
 version: !ruby/object:Gem::Version
-  version: 1.8.4
+  version: 1.9.0
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-07-01 00:00:00.000000000 Z
+date: 2021-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
@@ -95,33 +95,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: mechanize
+  name: algolia
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.8.0
+        version: 2.1.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.8.0
+        version: 2.1.1
+- !ruby/object:Gem::Dependency
+  name: graphql-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.16.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.16.0
 - !ruby/object:Gem::Dependency
   name: relaton-iso-bib
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.8.0
+        version: 1.9.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.8.0
+        version: 1.9.0
 description: 'RelatonBsi: retrieve BSI Standards for bibliographic use using the BibliographicItem
   model'
 email:
@@ -154,6 +168,7 @@ files:
 - lib/relaton_bsi/hit.rb
 - lib/relaton_bsi/hit_collection.rb
 - lib/relaton_bsi/processor.rb
+- lib/relaton_bsi/schema.json
 - lib/relaton_bsi/scrapper.rb
 - lib/relaton_bsi/version.rb
 - lib/relaton_bsi/xml_parser.rb