relaton-etsi 2.0.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e3f3b20d51d31465037d74aff3c4a88db51e16020d894b5cd6ae994c094ab3fe
-  data.tar.gz: 38271ff60d1ccef3ca1976cb1b887e795151a36ccb3973dac253190d7765e3e3
+  metadata.gz: 637634d27039848b4e93c7aadfe70cf2b220d2e222a2a9feba7b5a9760e933f1
+  data.tar.gz: 20dd06bbbff7b1af33ff03523ac418a91b2f276588d998061bdc2536a8f42075
 SHA512:
-  metadata.gz: 81e1ce86f16cd81aebae309f16761877014db6784743b7813a413ef0af82240595ad7779f095142f0822b3e2b376254112fb419aeba22e78d77f557f0f4e3749
-  data.tar.gz: 3cc6f64e2833c524f6ab12fb582b22498093fc870b087b104635a0be04a574b895c0f767cef0f67ac8a583fc3cf70f5f92e5b653974bc8cac0652b7894ad10ce
+  metadata.gz: 2fdae3a2a629012007d6d05a0245e3569e6e3a5eb2c45547ca89e018d348e68dd5bd3fa29c703b06a8e2ca30f4c296455676862c4e55c9d4395b706ba9023844
+  data.tar.gz: 818a000c7d5d7755859f45685eb33ab59611de7b50c2edc712c947f850d9ba448cf2433e4a93fc7017c79de055da4a312a00a1aa6c3c28db2693f8a31093596b

data/CLAUDE.md

@@ -66,4 +66,5 @@ Uses Lutaml for serialization:
 
 ## Testing
 
+- **Index fixture:** `spec/fixtures/index-v1.zip` is pre-loaded into `Relaton::Index` pool in `before(:suite)` (configured in `spec/support/webmock.rb`). Run `rake spec:update_index` to refresh from relaton-data-etsi.
 Uses RSpec with VCR for HTTP interaction recording. VCR cassettes are in `spec/vcr_cassettes/`. When tests make new HTTP requests, VCR will record them.

data/README.adoc

@@ -141,6 +141,8 @@ Relaton::Etsi uses the relaton-logger gem for logging. By default, it logs to ST
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
+To update the index test fixture (used by tests), run `rake spec:update_index`. This downloads the latest `index-v1.zip` from the https://github.com/relaton/relaton-data-etsi[relaton-data-etsi] repository.
+
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
 == Contributing

data/Rakefile

@@ -10,3 +10,25 @@ require "rubocop/rake_task"
 RuboCop::RakeTask.new
 
 task default: %i[spec]
+
+namespace :spec do
+  desc "Download latest ETSI index fixture from relaton-data-etsi"
+  task :update_index do
+    require "net/http"
+    require "uri"
+
+    url = "https://raw.githubusercontent.com/relaton/relaton-data-etsi/data-v2/index-v1.zip"
+    dest = File.join(__dir__, "spec", "fixtures", "index-v1.zip")
+
+    puts "Downloading \#{url} ..."
+    uri = URI.parse(url)
+    response = Net::HTTP.get_response(uri)
+
+    if response.is_a?(Net::HTTPSuccess)
+      File.binwrite(dest, response.body)
+      puts "Updated \#{dest} (\#{response.body.bytesize} bytes)"
+    else
+      abort "Failed to download: HTTP \#{response.code}"
+    end
+  end
+end

data/grammars/biblio.rng

@@ -2015,15 +2015,11 @@ provided that it is not the entire bibliographic item that is so related</a:docu
     <a:documentation>A version of the bibliographic item (within an edition). Can be used for drafts</a:documentation>
     <element name="version">
       <optional>
-        <ref name="revision-date">
-          <a:documentation>The date at which the current version of the bibliographic item was produced</a:documentation>
-        </ref>
-      </optional>
-      <optional>
-        <ref name="draft">
-          <a:documentation>The identifier for the current draft of the bibliographic item</a:documentation>
-        </ref>
+        <attribute name="type">
+          <a:documentation>Versioning scheme, in case of multiple versioning schemes</a:documentation>
+        </attribute>
       </optional>
+      <text/>
     </element>
   </define>
   <define name="vedition">

data/lib/relaton/etsi/bibdata.rb

@@ -3,6 +3,7 @@ require_relative "item"
 module Relaton
   module Etsi
     class Bibdata < Item
+      model ItemData
       include Bib::BibdataShared
     end
   end

data/lib/relaton/etsi/bibitem.rb

@@ -3,6 +3,7 @@ require_relative "item"
 module Relaton
   module Etsi
     class Bibitem < Item
+      model ItemData
       include Bib::BibitemShared
     end
   end

data/lib/relaton/etsi/data_fetcher.rb

@@ -1,6 +1,6 @@
 require "date"
+require "json"
 require "mechanize"
-require "csv"
 require "relaton/core"
 require "relaton/index"
 require_relative "../etsi"
@@ -9,10 +9,12 @@ require_relative "data_parser"
 module Relaton
   module Etsi
     class DataFetcher < Core::DataFetcher
-      SOURCEURL = "https://www.etsi.org/?option=com_standardssearch&view=data&format=csv&includeScope=1&" \
-        "page=1&search=&title=1&etsiNumber=1&content=1&version=0&onApproval=1&published=1&withdrawn=1&" \
-        "historical=1&isCurrent=1&superseded=1&startDate=1988-01-15&endDate=%<date>s&harmonized=0&" \
-        "keyword=&TB=&stdType=&frequency=&mandate=&collection=&sort=1&x=%<timestamp>s".freeze
+      PAGE_SIZE = 50
+
+      SOURCEURL = "https://www.etsi.org/custom/standardssearch/data.php?format=json&includeScope=1&" \
+        "page=%<page>s&search=&title=1&etsiNumber=1&content=1&version=0&onApproval=1&published=1&" \
+        "withdrawn=1&historical=1&isCurrent=1&superseded=1&startDate=1988-01-15&endDate=%<date>s&" \
+        "harmonized=0&keyword=&TB=&stdType=&frequency=&mandate=&collection=&sort=1&x=%<timestamp>s".freeze
 
       def index
         @index ||= Relaton::Index.find_or_create :etsi, file: INDEX_FILE
@@ -28,18 +30,60 @@ module Relaton
       # @param [Object] _source unused, required by superclass interface
       #
       def fetch(_source = nil)
-        time = Time.now
-        date = time.to_date + 1
-        timestamp = (time.to_f * 1000).to_i
-        url = format(SOURCEURL, date: date, timestamp: timestamp)
-        csv = fetch_with_retry(url)
-        CSV.parse(csv, headers: true, col_sep: ";", skip_lines: /sep=;/).each do |row|
-          save DataParser.new(row, @errors).parse
-        end
+        first_page = fetch_page(1)
+        process_records(first_page)
+        fetch_remaining_pages(first_page)
         index.save
         report_errors
       end
 
+      def fetch_remaining_pages(first_page)
+        total = first_page.first ? first_page.first["total_count"].to_i : 0
+        total_pages = (total / PAGE_SIZE.to_f).ceil
+        (2..total_pages).each do |page|
+          records = fetch_page(page)
+          break if records.empty?
+
+          process_records(records)
+        end
+      end
+
+      def fetch_page(page)
+        date = Time.now.to_date + 1
+        timestamp = (Time.now.to_f * 1000).to_i
+        url = format(SOURCEURL, page: page, date: date, timestamp: timestamp)
+        JSON.parse(fetch_with_retry(url))
+      end
+
+      def process_records(records)
+        records.each do |record|
+          save DataParser.new(normalize(record), @errors).parse
+        end
+      end
+
+      def normalize(record)
+        {
+          "ETSI deliverable" => record["ETSI_DELIVERABLE"],
+          "title" => record["TITLE"],
+          "Details link" => "https://webapp.etsi.org/workprogram/Report_WorkItem.asp?WKI_ID=#{record['wki_id']}",
+          "PDF link" => "https://www.etsi.org/deliver/#{record['EDSpathname']}#{record['EDSPDFfilename']}",
+          "Status" => derive_status(record),
+          "Keywords" => record["Keywords"].to_s,
+          "Technical body" => record["TB"],
+          "Scope" => record["Scope"],
+        }
+      end
+
+      def derive_status(record)
+        return "Withdrawn" if record["ACTION_TYPE"] == "WD"
+
+        code = record["STATUS_CODE"].to_i
+        return "On Approval" if code < 12
+        return "Historical" if code == 13
+
+        "Published"
+      end
+
       NETWORK_ERRORS = [
         Mechanize::Error, Net::OpenTimeout, Net::ReadTimeout,
         SocketError, Errno::ECONNRESET

data/lib/relaton/etsi/data_parser.rb

@@ -13,10 +13,8 @@ module Relaton
       end
 
       def parse
-        args = ATTRS.each_with_object({}) do |attr, hash|
-          hash[attr] = send(attr)
-        end
-        ItemData.new(**args)
+        args = ATTRS.to_h { |attr| [attr, send(attr)] }
+        ItemData.new(type: "standard", **args)
       end
 
       def pubid

data/lib/relaton/etsi/ext.rb

@@ -3,7 +3,6 @@ require_relative "doctype"
 module Relaton
   module Etsi
     class Ext < Bib::Ext
-      attribute :schema_version, method: :get_schema_version
       attribute :doctype, Doctype
       attribute :marker, :string, values: %w[Current Superseded]
       attribute :frequency, :string, collection: true
@@ -19,9 +18,14 @@ module Relaton
         map_element "custom-collection", to: :custom_collection
       end
 
-      def get_schema_version
-        Relaton.schema_versions["relaton-model-etsi"]
+      key_value do
+        map_element "marker", to: :marker
+        map_element "frequency", to: :frequency
+        map_element "mandate", to: :mandate
+        map_element "custom_collection", to: :custom_collection
       end
+
+      def get_schema_version = Relaton.schema_versions["relaton-model-etsi"]
     end
   end
 end

data/lib/relaton/etsi/version.rb

@@ -2,6 +2,6 @@
 
 module Relaton
   module Etsi
-    VERSION = "2.0.0"
+    VERSION = "2.1.0"
   end
 end

metadata

@@ -1,28 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: relaton-etsi
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.1.0
 platform: ruby
 authors:
 - Ribose Inc.
+autorequire:
 bindir: exe
 cert_chain: []
-date: 1980-01-02 00:00:00.000000000 Z
+date: 2026-05-04 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: csv
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: mechanize
   requirement: !ruby/object:Gem::Requirement
@@ -43,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 2.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 2.1.0
 - !ruby/object:Gem::Dependency
   name: relaton-core
   requirement: !ruby/object:Gem::Requirement
@@ -119,6 +106,7 @@ licenses:
 - BSD-2-Clause
 metadata:
   homepage_uri: https://github.com/relaton/relaton-etsi
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -133,7 +121,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 3.5.22
+signing_key:
 specification_version: 4
 summary: 'Relaton::Etsi: retrieve ETSI Standards for bibliographic using the BibliographicItem
   model'