relaton-asme 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e288ff1fe94fd8f66ca1d903b2c2d83d97b2d3b35555547e0d8e0bfe1eb04f80
+  data.tar.gz: beb6d02b7c13f3ff37fecec8e22a4ffd9cffe4429c656e286e4c853f283d5539
+SHA512:
+  metadata.gz: 4cad93db00b8e5ccd5f75e00e07852735f19e9c16e63d5e6485e671df791f2b726c856a41178e26d6e912cc7ebe684f5faa79457bead82283b29f90f81ce9748
+  data.tar.gz: 5a7b3379cfb3d7571b5d9c1a685a83b9c822ea7dea55dcb0f6ff40255551632014ce7798a73a288674a114432a9b440863c13d7f41969fc28be2ad6bb90ea8e9

data/LICENSE

@@ -0,0 +1,25 @@
+BSD 2-Clause License
+
+Copyright (c) 2025, Ribose Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.adoc

@@ -0,0 +1,539 @@
+= Relaton for ASME
+
+image:https://img.shields.io/gem/v/relaton-asme.svg[Gem Version,link=https://rubygems.org/gems/relaton-asme]
+image:https://img.shields.io/github/license/relaton/relaton-asme.svg[License]
+image:https://github.com/relaton/relaton-asme/actions/workflows/rake.yml/badge.svg[Build Status,link=https://github.com/relaton/relaton-asme/actions/workflows/rake.yml]
+
+Fetch and parse ASME (American Society of Mechanical Engineers) standards and
+codes from the official ASME website.
+
+== Purpose
+
+Relaton for ASME provides a programmatic interface to fetch ASME standards and codes publications.
+
+It accesses the ASME JSON API for publication listings and parses JSON-LD
+structured data from publication pages to extract complete bibliographic
+metadata.
+
+== Features
+
+* <<fetching-publication-listings,Fetching publication listings>> - Access ASME JSON API with pagination
+* <<parsing-publication-metadata,Parsing publication metadata>> - Extract data from JSON-LD structured content
+* <<data-serialization,Data serialization>> - Export to JSON/YAML formats
+* <<batch-processing,Batch processing>> - Fetch multiple publications efficiently
+* <<pre-scraped-database,Pre-scraped database>> - Access 4,928 ASME editions (1967-2025)
+
+== Architecture
+
+=== General
+
+RelatonAsme uses a two-stage data pipeline with object-oriented, model-driven architecture:
+
+.Data Pipeline
+[source]
+----
+Stage 1: ASME JSON API
+    │
+    └─► IndexPage fetches from /api/products
+        │
+        └─► Returns publication URLs
+            │
+            ▼
+Stage 2: JSON-LD Parsing
+    │
+    └─► PublicationPage parses <script type="application/ld+json">
+        │
+        └─► Extracts Schema.org Product data
+            │
+            ▼
+Stage 3: Data Models
+    │
+    └─► AsmePublicationEdition (with URL, metadata, pricing)
+----
+
+=== Data model hierarchy
+
+[source]
+----
+AsmePublication (Container)
+    │
+    └── AsmePublicationEdition[] (Primary Model - Array)
+            │
+            ├── url: String (publication page URL)
+            ├── document_type: String ("standard")
+            ├── title: String (WITHOUT designator prefix)
+            ├── document_identifier: String (e.g., "BPVC.I-2025")
+            ├── designator: String (e.g., "BPVC.I")
+            ├── edition_year: String (e.g., "2025")
+            ├── publisher: String ("ASME")
+            ├── publish_date: String
+            ├── language: String ("EN")
+            ├── number_of_pages: Integer (optional)
+            ├── description: String (optional)
+            ├── topics: String[] (optional)
+            └── formats: AsmeFormat[] (optional)
+                    │
+                    ├── format_type: String (e.g., "PDF", "Print Book")
+                    ├── price: String (e.g., "$675.00")
+                    └── isbn: String (optional)
+----
+
+== Installation
+
+Add this line to your application's Gemfile:
+
+[source,ruby]
+----
+gem "relaton-asme"
+----
+
+And then execute:
+
+[source,shell]
+----
+bundle install
+----
+
+Or install it yourself as:
+
+[source,shell]
+----
+gem install relaton-asme
+----
+
+== Usage
+
+[[fetching-publication-listings]]
+=== Fetching publication listings
+
+==== General
+
+The `IndexPage` class fetches publication listings from the ASME JSON API at
+`https://www.asme.org/api/products`.
+
+==== Fetching publication URLs from API
+
+Syntax:
+
+[source,ruby]
+----
+index = RelatonAsme::Fetcher::IndexPage.new(
+  page_number: {page_number}, <1>
+  per_page: {per_page} <2>
+)
+urls = index.publication_urls <3>
+----
+<1> Page number to fetch (1-based, default: 1)
+<2> Number of results per page (default: 100, max: 100)
+<3> Returns array of publication URLs
+
+Where,
+
+`page_number`:: The page number to fetch, starting from 1. Default is 1.
+`per_page`:: Number of publications to fetch per page. Default is 100.
+`publication_urls`:: Returns an array of publication URLs for each edition/format combination.
+
+.Fetching URLs from the first page of API
+[example]
+====
+[source,ruby]
+----
+require "relaton_asme"
+
+index = RelatonAsme::Fetcher::IndexPage.new(page_number: 1, per_page: 10)
+urls = index.publication_urls
+
+puts "Found #{urls.size} publication URLs"
+urls.first(5).each { |url| puts url }
+----
+
+This queries the ASME API and returns URLs for each edition/format combination.
+
+Example output:
+----
+Found 30 publication URLs
+https://www.asme.org/codes-standards/find-codes-standards/bpvc-i-.../2025/print-book
+https://www.asme.org/codes-standards/find-codes-standards/bpvc-i-.../2023/print-book
+----
+====
+
+[[parsing-publication-metadata]]
+=== Parsing publication metadata
+
+==== General
+
+The `PublicationPage` class parses JSON-LD structured data from individual publication pages. Each page may contain multiple editions (different years) as separate offers.
+
+==== Fetching publication data from JSON-LD
+
+Syntax:
+
+[source,ruby]
+----
+page = RelatonAsme::Fetcher::PublicationPage.new({url}) <1>
+publication = page.to_data <2>
+----
+<1> Publication URL to fetch
+<2> Returns `AsmePublication` object with editions extracted from JSON-LD offers
+
+Where,
+
+`url`:: The full URL to the publication page on asme.org
+`to_data`:: Returns an `AsmePublication` object containing all editions from the page's JSON-LD data
+
+.Fetching a publication via JSON-LD
+[example]
+====
+[source,ruby]
+----
+url = "https://www.asme.org/codes-standards/find-codes-standards/" \
+      "temperature-measurement/2024/pdf"
+
+page = RelatonAsme::Fetcher::PublicationPage.new(url)
+publication = page.to_data
+
+if publication
+  publication.editions.each do |edition|
+    puts "Title: #{edition.title}"
+    puts "Year: #{edition.edition_year}"
+    puts "URL: #{edition.url}"
+    puts "Identifier: #{edition.document_identifier}"
+  end
+end
+----
+
+This parses the JSON-LD structured data and may return multiple editions (e.g.,
+2024 and 1974 versions) from a single page.
+
+Example output:
+----
+Title: Temperature Measurement (2024)
+Year: 2024
+URL: https://www.asme.org/.../2024/pdf/
+Identifier: PTC 19.3-2024
+
+Title: Temperature Measurement (1974)
+Year: 1974
+URL: https://www.asme.org/.../1974/pdf/
+Identifier: PTC 19.3-1974
+----
+====
+
+[[batch-processing]]
+=== Batch processing with Runner
+
+==== General
+
+The `Runner` class orchestrates batch fetching of publications from the API.
+
+==== Fetching all publications
+
+Syntax:
+
+[source,ruby]
+----
+runner = RelatonAsme::Fetcher::Runner.new <1>
+editions = runner.fetch_all(
+  max_pages: {max_pages}, <2>
+  per_page: {per_page} <3>
+)
+----
+<1> Create a new runner instance
+<2> Maximum number of API pages to fetch (nil for all pages)
+<3> Number of publications per page (default: 100)
+
+Where,
+
+`max_pages`:: Optional limit on the number of API pages to process. Pass `nil` to fetch all 671 publications (135 pages).
+`per_page`:: Number of publications to fetch per API page. Default is 100, maximum is 100.
+`editions`:: Returns an array of `AsmePublicationEdition` objects from all fetched publications.
+
+.Fetching publications from first 2 pages
+[example]
+====
+[source,ruby]
+----
+runner = RelatonAsme::Fetcher::Runner.new
+editions = runner.fetch_all(max_pages: 2, per_page: 10)
+
+puts "Fetched #{editions.size} editions"
+editions.first(3).each do |edition|
+  puts "#{edition.document_identifier} - #{edition.title}"
+end
+----
+
+This fetches publications from the first 2 API pages and parses their JSON-LD data.
+====
+
+==== Fetching and saving to file
+
+Syntax:
+
+[source,ruby]
+----
+runner = RelatonAsme::Fetcher::Runner.new
+runner.fetch_all_editions(
+  {output_file}, <1>
+  format: {format}, <2>
+  max_pages: {max_pages} <3>
+)
+----
+<1> Path to output file
+<2> Output format (`:json` or `:yaml`)
+<3> Optional maximum number of pages
+
+.Saving publications to YAML
+[example]
+====
+[source,ruby]
+----
+runner = RelatonAsme::Fetcher::Runner.new
+runner.fetch_all_editions(
+  "asme_publications.yml",
+  format: :yaml,
+  max_pages: 1
+)
+----
+
+This fetches publications from the first API page, parses their JSON-LD data,
+and saves them as YAML to `asme_publications.yml`.
+====
+
+[[data-serialization]]
+=== Data serialization
+
+==== General
+
+All data models support JSON and YAML serialization through `lutaml-model`.
+
+==== Serializing to JSON
+
+.Converting an edition to JSON
+[example]
+====
+[source,ruby]
+----
+edition = RelatonAsme::Fetcher::AsmePublicationEdition.new(
+  url: "https://www.asme.org/...",
+  document_type: "standard",
+  title: "BPVC Section I",
+  document_identifier: "BPVC.I-2025",
+  designator: "BPVC.I",
+  edition_year: "2025",
+  publisher: "ASME",
+  publish_date: "2025",
+  language: "EN"
+)
+
+json = edition.to_json
+puts json
+----
+
+Note: `edition_year` is a String, not an Integer.
+====
+
+[[pre-scraped-database]]
+=== Pre-scraped database
+
+==== General
+
+A complete database of ASME publications has been pre-scraped and is available in `data/asme_publication_editions.yml`.
+
+==== Database contents
+
+The database contains:
+
+* **4,928 ASME publication editions**
+* **269 unique ASME publications**
+* **Year range:** 1967 - 2025 (58 years)
+* **Formats:** Print Book, PDF, Bundle
+
+==== Loading the database
+
+.Loading pre-scraped ASME data
+[example]
+====
+[source,ruby]
+----
+require "yaml"
+
+data = YAML.load_file(
+  "data/asme_publication_editions.yml",
+  permitted_classes: [Symbol]
+)
+
+puts "Loaded #{data.size} ASME editions"
+
+# Find all editions for a specific designator
+bpvc_editions = data.select { |e| e["designator"] == "BPVC.I" }
+puts "BPVC.I has #{bpvc_editions.size} editions"
+
+# Find editions by year
+recent = data.select { |e| e["edition_year"].to_i >= 2020 }
+puts "#{recent.size} editions from 2020 onwards"
+----
+====
+
+== Data model details
+
+=== AsmePublicationEdition (Primary Model)
+
+The primary data model containing all publication metadata for a specific edition.
+
+.Attributes
+[cols="1,1,3"]
+|===
+|Attribute |Type |Description
+
+|`url`
+|String
+|Direct link to publication page on asme.org
+
+|`document_type`
+|String
+|Type of document (typically "standard")
+
+|`title`
+|String
+|Edition title WITHOUT designator prefix, with year in parentheses
+
+|`document_identifier`
+|String
+|Edition-specific identifier (e.g., "BPVC.I-2025")
+
+|`designator`
+|String
+|ASME code/SKU (e.g., "BPVC.I", "B31.3", "Y14.5")
+
+|`edition_year`
+|String
+|Publication year as string (e.g., "2025", "2023")
+
+|`publisher`
+|String
+|Publisher name (always "ASME")
+
+|`publish_date`
+|String
+|Publication date (typically same as edition_year)
+
+|`language`
+|String
+|Language code (typically "EN")
+
+|`number_of_pages`
+|Integer
+|Page count (optional, not available in JSON-LD)
+
+|`description`
+|String
+|Detailed description (optional, not available in JSON-LD)
+
+|`topics`
+|Array<String>
+|Array of topic names (optional, not available in JSON-LD)
+
+|`formats`
+|Array<AsmeFormat>
+|Available formats and pricing from JSON-LD offers
+|===
+
+=== AsmeFormat
+
+Represents a publication format with pricing information extracted from JSON-LD offers.
+
+.Attributes
+[cols="1,1,3"]
+|===
+|Attribute |Type |Description
+
+|`format_type`
+|String
+|Format type (e.g., "PDF", "Print Book", "Bundle")
+
+|`price`
+|String
+|Price formatted as string (e.g., "$675.00")
+
+|`isbn`
+|String
+|ISBN number if available (optional, not in JSON-LD)
+|===
+
+=== AsmePublication (Container)
+
+Container model for grouping related publication editions.
+
+.Attributes
+[cols="1,1,3"]
+|===
+|Attribute |Type |Description
+
+|`publication_group`
+|String
+|Publication designator/SKU used for grouping
+
+|`editions`
+|Array<AsmePublicationEdition>
+|Array of publication editions with different years/formats
+|===
+
+== Development
+
+After checking out the repo, run `bundle install` to install dependencies.
+
+Run tests:
+
+[source,shell]
+----
+bundle exec rake spec
+----
+
+Run RuboCop:
+
+[source,shell]
+----
+bundle exec rake rubocop
+----
+
+Run all checks (tests + RuboCop):
+
+[source,shell]
+----
+bundle exec rake
+----
+
+== Scraping ASME Publications
+
+To scrape ASME publications and save to a file:
+
+[source,ruby]
+----
+require "relaton_asme"
+
+runner = RelatonAsme::Fetcher::Runner.new
+
+# Fetch all publications (671 total, takes ~2 hours)
+runner.fetch_all_editions(
+  "data/asme_publications.yml",
+  format: :yaml
+)
+
+# Or fetch just a few pages for testing
+runner.fetch_all_editions(
+  "data/sample.yml",
+  format: :yaml,
+  max_pages: 2
+)
+----
+
+== Contributing
+
+Bug reports and pull requests are welcome on GitHub at
+https://github.com/relaton/relaton-asme.
+
+== Copyright & license
+
+Copyright Ribose.
+
+The gem is available as open source under the terms of the BSD-2-Clause License.

data/lib/relaton_asme/fetcher/asme_format.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+
+module RelatonAsme
+  module Fetcher
+    # Represents a publication format (e.g., PDF, Print Book, Bundle)
+    class AsmeFormat < Lutaml::Model::Serializable
+      attribute :format_type, :string
+      attribute :price, :string
+      attribute :isbn, :string
+
+      json do
+        map "format_type", to: :format_type
+        map "price", to: :price
+        map "isbn", to: :isbn
+      end
+
+      yaml do
+        map "format_type", to: :format_type
+        map "price", to: :price
+        map "isbn", to: :isbn
+      end
+    end
+  end
+end

data/lib/relaton_asme/fetcher/asme_publication.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+require_relative "asme_publication_edition"
+
+module RelatonAsme
+  module Fetcher
+    # Container model for grouping related publication editions
+    class AsmePublication < Lutaml::Model::Serializable
+      attribute :publication_group, :string
+      attribute :editions, AsmePublicationEdition, collection: true
+
+      json do
+        map "publication_group", to: :publication_group
+        map "editions", to: :editions
+      end
+
+      yaml do
+        map "publication_group", to: :publication_group
+        map "editions", to: :editions
+      end
+    end
+  end
+end

data/lib/relaton_asme/fetcher/asme_publication_edition.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+require_relative "asme_format"
+
+module RelatonAsme
+  module Fetcher
+    # Represents a specific edition of an ASME publication
+    # This is the PRIMARY data model containing all metadata
+    class AsmePublicationEdition < Lutaml::Model::Serializable
+      attribute :url, :string
+      attribute :document_type, :string
+      attribute :title, :string
+      attribute :document_identifier, :string
+      attribute :designator, :string
+      attribute :edition_year, :string
+      attribute :publisher, :string
+      attribute :publish_date, :string
+      attribute :language, :string
+      attribute :number_of_pages, :integer
+      attribute :description, :string
+      attribute :topics, :string, collection: true
+      attribute :formats, AsmeFormat, collection: true
+
+      json do
+        map "url", to: :url
+        map "document_type", to: :document_type
+        map "title", to: :title
+        map "document_identifier", to: :document_identifier
+        map "designator", to: :designator
+        map "edition_year", to: :edition_year
+        map "publisher", to: :publisher
+        map "publish_date", to: :publish_date
+        map "language", to: :language
+        map "number_of_pages", to: :number_of_pages
+        map "description", to: :description
+        map "topics", to: :topics
+        map "formats", to: :formats
+      end
+
+      yaml do
+        map "url", to: :url
+        map "document_type", to: :document_type
+        map "title", to: :title
+        map "document_identifier", to: :document_identifier
+        map "designator", to: :designator
+        map "edition_year", to: :edition_year
+        map "publisher", to: :publisher
+        map "publish_date", to: :publish_date
+        map "language", to: :language
+        map "number_of_pages", to: :number_of_pages
+        map "description", to: :description
+        map "topics", to: :topics
+        map "formats", to: :formats
+      end
+    end
+  end
+end

data/lib/relaton_asme/fetcher/basic_page.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "mechanize"
+
+module RelatonAsme
+  module Fetcher
+    # Base class for web scraping pages using Mechanize
+    class BasicPage
+      attr_reader :url, :page, :agent
+
+      # Initialize a new page fetcher
+      #
+      # @param url [String] URL to fetch
+      # @param agent [Mechanize, nil] Optional Mechanize agent to reuse
+      def initialize(url, agent: nil)
+        @url = url
+        @agent = agent || create_agent
+        @page = nil
+      end
+
+      # Fetch the page
+      #
+      # @return [Mechanize::Page] The fetched page
+      def fetch
+        @page = @agent.get(url)
+      end
+
+      # Get the page, fetching if necessary
+      #
+      # @return [Mechanize::Page] The page
+      def ensure_page
+        @ensure_page ||= fetch
+      end
+
+      private
+
+      # Create a new Mechanize agent with standard configuration
+      #
+      # @return [Mechanize] Configured agent
+      def create_agent
+        agent = Mechanize.new
+        agent.user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_2_1) " \
+                           "AppleWebKit/605.1.15 (KHTML, like Gecko) " \
+                           "Version/17.2 Safari/605.1.15"
+        agent.follow_meta_refresh = true
+        agent.redirect_ok = true
+        agent
+      end
+    end
+  end
+end

data/lib/relaton_asme/fetcher/index_page.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "basic_page"
+
+module RelatonAsme
+  module Fetcher
+    # Handles fetching ASME standards index via JSON API
+    class IndexPage < BasicPage
+      API_URL = "https://www.asme.org/api/products"
+      DEFAULT_PER_PAGE = 100
+
+      # Initialize index page for a specific page number
+      #
+      # @param page_number [Integer] Page number (1-based)
+      # @param per_page [Integer] Results per page
+      # @param agent [Mechanize, nil] Optional Mechanize agent
+      def initialize(page_number: 1, per_page: DEFAULT_PER_PAGE, agent: nil)
+        @page_number = page_number
+        @per_page = per_page
+        url = build_api_url(page_number, per_page)
+        super(url, agent: agent)
+      end
+
+      # Extract publication URLs from the API response
+      #
+      # @return [Array<String>] Array of publication URLs
+      def publication_urls
+        ensure_page
+        extract_urls_from_json
+      end
+
+      # Check if there are more pages
+      #
+      # @return [Boolean] True if more pages exist
+      def more_pages?
+        ensure_page
+        data = parse_json_response
+        return false unless data
+
+        data["page"] < data["pages"]
+      end
+
+      private
+
+      # Build the API URL with query parameters
+      #
+      # @param page_number [Integer] Page number
+      # @param per_page [Integer] Results per page
+      # @return [String] Full API URL
+      def build_api_url(page_number, per_page)
+        params = [
+          "type=Codes-Standards",
+          "page=#{page_number}",
+          "perPage=#{per_page}",
+          "sortBy=date",
+          "sortByDir=desc"
+        ].join("&")
+
+        "#{API_URL}?#{params}"
+      end
+
+      # Parse JSON response from API
+      #
+      # @return [Hash, nil] Parsed JSON data or nil
+      def parse_json_response
+        JSON.parse(page.body)
+      rescue JSON::ParserError => e
+        warn "Failed to parse JSON response: #{e.message}"
+        nil
+      end
+
+      # Extract publication URLs from JSON response
+      #
+      # @return [Array<String>] Publication URLs
+      def extract_urls_from_json
+        data = parse_json_response
+        return [] unless data && data["results"]
+
+        data["results"].map do |result|
+          build_publication_urls(result)
+        end.flatten.compact.uniq
+      end
+
+      # Build publication URLs from result data
+      #
+      # @param result [Hash] Result data from API
+      # @return [Array<String>] URLs for each edition/format combination
+      def build_publication_urls(result)
+        base_url = result.dig("WebsiteProductGrouping", "URL")
+        return [] unless base_url
+
+        editions = result.dig("WebsiteProductGrouping", "Editions") || []
+        formats = result.dig("WebsiteProductGrouping", "Formats") || []
+
+        # Generate URLs for each edition/format combination
+        urls = []
+        editions.each do |edition|
+          formats.each do |format|
+            format_path = format_to_path(format)
+            url = "https://www.asme.org#{base_url}/#{edition}/#{format_path}"
+            urls << url
+          end
+        end
+
+        urls
+      end
+
+      # Convert format name to URL path component
+      #
+      # @param format [String] Format name
+      # @return [String] URL path component
+      def format_to_path(format)
+        case format.downcase
+        when /print.*book/
+          "print-book"
+        when /pdf/
+          "pdf"
+        when /bundle/
+          "bundle"
+        else
+          format.downcase.tr(" ", "-")
+        end
+      end
+    end
+  end
+end

data/lib/relaton_asme/fetcher/publication_page.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "basic_page"
+require_relative "asme_publication"
+require_relative "asme_publication_edition"
+require_relative "asme_format"
+
+module RelatonAsme
+  module Fetcher
+    # Handles scraping of individual ASME publication pages
+    class PublicationPage < BasicPage
+      # Convert page data to AsmePublication model
+      #
+      # @return [AsmePublication, nil] Publication with editions or nil
+      def to_data
+        ensure_page
+        jsonld_data = parse_jsonld
+
+        return nil unless jsonld_data
+
+        build_publication_from_jsonld(jsonld_data)
+      end
+
+      private
+
+      # Parse JSON-LD structured data from page
+      #
+      # @return [Hash, nil] Parsed JSON-LD data or nil
+      def parse_jsonld
+        jsonld_script = page.search('script[type="application/ld+json"]').first
+        return nil unless jsonld_script
+
+        JSON.parse(jsonld_script.text)
+      rescue JSON::ParserError => e
+        warn "Failed to parse JSON-LD: #{e.message}"
+        nil
+      end
+
+      # Build AsmePublication from JSON-LD data
+      #
+      # @param jsonld_data [Hash] JSON-LD data
+      # @return [AsmePublication] Publication with editions
+      def build_publication_from_jsonld(jsonld_data)
+        offers = jsonld_data["offers"] || []
+        base_title = jsonld_data["name"]
+        designator = jsonld_data["sku"]
+
+        editions = offers.map do |offer|
+          build_edition_from_offer(offer, base_title, designator)
+        end.compact
+
+        AsmePublication.new(
+          publication_group: designator,
+          editions: editions
+        )
+      end
+
+      # Build edition from a JSON-LD offer
+      #
+      # @param offer [Hash] Offer data
+      # @param base_title [String] Base publication title
+      # @param designator [String] Publication designator/SKU
+      # @return [AsmePublicationEdition, nil] Edition or nil
+      def build_edition_from_offer(offer, base_title, designator)
+        url = offer["url"]
+        return nil unless url
+
+        # Extract year and format from URL
+        year = extract_year_from_url(url)
+        format_type = extract_format_from_url(url)
+
+        return nil unless year
+
+        # Build title without designator prefix, just base title and year
+        title = "#{base_title} (#{year})"
+        document_identifier = "#{designator}-#{year}"
+
+        AsmePublicationEdition.new(
+          url: url,
+          document_type: "standard",
+          title: title,
+          document_identifier: document_identifier,
+          designator: designator,
+          edition_year: year.to_s,
+          publisher: "ASME",
+          publish_date: year.to_s,
+          language: "EN",
+          formats: [build_format_from_offer(offer, format_type)]
+        )
+      end
+
+      # Extract year from URL
+      #
+      # @param url [String] URL
+      # @return [Integer, nil] Year or nil
+      def extract_year_from_url(url)
+        match = url.match(%r{/(\d{4})/})
+        match ? match[1].to_i : nil
+      end
+
+      # Extract format type from URL
+      #
+      # @param url [String] URL
+      # @return [String] Format type
+      def extract_format_from_url(url)
+        return "PDF" if url.include?("/pdf")
+        return "Print Book" if url.include?("/print")
+        return "Bundle" if url.include?("/bundle")
+
+        "PDF"
+      end
+
+      # Build format from offer data
+      #
+      # @param offer [Hash] Offer data
+      # @param format_type [String] Format type
+      # @return [AsmeFormat] Format object
+      def build_format_from_offer(offer, format_type)
+        price = offer["price"]
+        price_str = "$#{price}.00"
+
+        AsmeFormat.new(
+          format_type: format_type,
+          price: price_str
+        )
+      end
+    end
+  end
+end

data/lib/relaton_asme/fetcher/runner.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require_relative "index_page"
+require_relative "publication_page"
+
+module RelatonAsme
+  module Fetcher
+    # Orchestrates the fetching of ASME publications
+    class Runner
+      attr_reader :agent
+
+      # Initialize the runner
+      #
+      # @param agent [Mechanize, nil] Optional Mechanize agent to reuse
+      def initialize(agent: nil)
+        @agent = agent || create_agent
+      end
+
+      # Fetch all publications from all pages
+      #
+      # @param max_pages [Integer, nil] Maximum pages to fetch (nil = all)
+      # @param per_page [Integer] Results per page
+      # @return [Array<AsmePublicationEdition>] All editions
+      def fetch_all(max_pages: nil, per_page: 100)
+        editions = []
+        page_number = 1
+
+        loop do
+          puts "Fetching page #{page_number}..."
+          page_editions = fetch_page(page_number, per_page: per_page)
+          editions.concat(page_editions)
+
+          break if max_pages && page_number >= max_pages
+          break unless more_pages?(page_number, per_page)
+
+          page_number += 1
+        end
+
+        editions
+      end
+
+      # Fetch publications from a specific page
+      #
+      # @param page_number [Integer] Page number to fetch
+      # @param per_page [Integer] Results per page
+      # @return [Array<AsmePublicationEdition>] Editions from the page
+      def fetch_page(page_number, per_page: 100)
+        index = IndexPage.new(
+          page_number: page_number,
+          per_page: per_page,
+          agent: @agent
+        )
+
+        urls = index.publication_urls
+        puts "Found #{urls.size} publications on page #{page_number}"
+
+        fetch_publications(urls)
+      end
+
+      # Fetch multiple publications by URLs
+      #
+      # @param urls [Array<String>] Publication URLs
+      # @return [Array<AsmePublicationEdition>] All editions
+      def fetch_publications(urls)
+        editions = []
+
+        urls.each_with_index do |url, index|
+          puts "Fetching publication #{index + 1}/#{urls.size}: #{url}"
+
+          begin
+            publication = fetch_publication(url)
+            editions.concat(publication.editions) if publication&.editions
+          rescue StandardError => e
+            warn "Failed to fetch #{url}: #{e.message}"
+          end
+        end
+
+        editions
+      end
+
+      # Fetch a single publication by URL
+      #
+      # @param url [String] Publication URL
+      # @return [AsmePublication, nil] Publication with editions
+      def fetch_publication(url)
+        page = PublicationPage.new(url, agent: @agent)
+        page.to_data
+      end
+
+      # Check if more pages exist
+      #
+      # @param page_number [Integer] Current page number
+      # @param per_page [Integer] Results per page
+      # @return [Boolean] True if more pages exist
+      def more_pages?(page_number, per_page)
+        index = IndexPage.new(
+          page_number: page_number,
+          per_page: per_page,
+          agent: @agent
+        )
+        index.more_pages?
+      end
+
+      # Fetch all editions for publications and save to file
+      #
+      # @param output_file [String] Path to output file
+      # @param format [Symbol] Output format (:json or :yaml)
+      # @param max_pages [Integer, nil] Maximum pages to fetch
+      # @return [Array<AsmePublicationEdition>] All editions
+      def fetch_all_editions(output_file, format: :json, max_pages: nil)
+        editions = fetch_all(max_pages: max_pages)
+
+        save_editions(editions, output_file, format)
+
+        editions
+      end
+
+      # Save editions to file
+      #
+      # @param editions [Array<AsmePublicationEdition>] Editions to save
+      # @param output_file [String] Path to output file
+      # @param format [Symbol] Output format (:json or :yaml)
+      def save_editions(editions, output_file, format)
+        content = case format
+                  when :json
+                    serialize_to_json(editions)
+                  when :yaml
+                    serialize_to_yaml(editions)
+                  else
+                    raise ArgumentError, "Unsupported format: #{format}"
+                  end
+
+        File.write(output_file, content)
+        puts "Saved #{editions.size} editions to #{output_file}"
+      end
+
+      private
+
+      # Create a new Mechanize agent
+      #
+      # @return [Mechanize] Configured agent
+      def create_agent
+        agent = Mechanize.new
+        agent.user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_2_1) " \
+                           "AppleWebKit/605.1.15 (KHTML, like Gecko) " \
+                           "Version/17.2 Safari/605.1.15"
+        agent.follow_meta_refresh = true
+        agent.redirect_ok = true
+        agent
+      end
+
+      # Serialize editions to JSON
+      #
+      # @param editions [Array<AsmePublicationEdition>] Editions
+      # @return [String] JSON string
+      def serialize_to_json(editions)
+        JSON.pretty_generate(editions.map(&:to_hash))
+      end
+
+      # Serialize editions to YAML
+      #
+      # @param editions [Array<AsmePublicationEdition>] Editions
+      # @return [String] YAML string
+      def serialize_to_yaml(editions)
+        editions.map(&:to_hash).to_yaml
+      end
+    end
+  end
+end

data/lib/relaton_asme/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module RelatonAsme
+  VERSION = "0.1.0"
+end

data/lib/relaton_asme.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require_relative "relaton_asme/version"
+require_relative "relaton_asme/fetcher/asme_format"
+require_relative "relaton_asme/fetcher/asme_publication_edition"
+require_relative "relaton_asme/fetcher/asme_publication"
+require_relative "relaton_asme/fetcher/basic_page"
+require_relative "relaton_asme/fetcher/index_page"
+require_relative "relaton_asme/fetcher/publication_page"
+require_relative "relaton_asme/fetcher/runner"
+
+module RelatonAsme
+  class Error < StandardError; end
+end

metadata

@@ -0,0 +1,85 @@
+--- !ruby/object:Gem::Specification
+name: relaton-asme
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Ribose Inc.
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2025-12-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: lutaml-model
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Retrieve bibliographic information of ASME standards and codes.
+email:
+- open.source@ribose.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.adoc
+- lib/relaton_asme.rb
+- lib/relaton_asme/fetcher/asme_format.rb
+- lib/relaton_asme/fetcher/asme_publication.rb
+- lib/relaton_asme/fetcher/asme_publication_edition.rb
+- lib/relaton_asme/fetcher/basic_page.rb
+- lib/relaton_asme/fetcher/index_page.rb
+- lib/relaton_asme/fetcher/publication_page.rb
+- lib/relaton_asme/fetcher/runner.rb
+- lib/relaton_asme/version.rb
+homepage: https://github.com/relaton/relaton-asme
+licenses:
+- BSD-2-Clause
+metadata:
+  homepage_uri: https://github.com/relaton/relaton-asme
+  source_code_uri: https://github.com/relaton/relaton-asme
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.22
+signing_key:
+specification_version: 4
+summary: Fetch and parse ASME standards and codes
+test_files: []