oddb2xml 3.0.18 → 3.0.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a4aadf4a537b9f625c491c4609116f6aef44c4f558e0a273a0d2da705ad8bfd0
-  data.tar.gz: 946a8f14539626a383dcab8e73cdc5e89410e8df2721a44d0fca50cc868cf11c
+  metadata.gz: fe41f3e5cfb0e100f111a3071f32ae8dd4b4103d3f5726bf4eab061f80389323
+  data.tar.gz: 785dee3549e33031069aceac4f9be31cc844de603d3b183d94b55f976b0c482c
 SHA512:
-  metadata.gz: d91d784a98fd256a0ce056c0cfa36959249c7d4cf6b194a2b97c360569ad6f3e503e57c4395722d8a83b5f163ec3b9f5187b8f28ca193de8cc83eff80cf59c2f
-  data.tar.gz: 75f26b3a4559da9776b77c718a916e7ee76715a49475066b8b1e575ba5aa9790b9c6797245784a799af211bb18c86711600ce7ce8cb1794feae4769bb442eaed
+  metadata.gz: b385af5fc0643374e878da0cd6e1b262e4d7521c5ac5dd3a3670971f8f130ca3492b4aa296410c5e9e6627021bcbb86b508dbd3e80bf1fdd044c61851df69664
+  data.tar.gz: 40a8baa557cdb31905f8711702bbe52f8d968b67488ae88f293233ea3bd1db689c9ea75d27b4b17019cf2369fb593eed3268c3867277daf5cb73c763b6a74791

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    oddb2xml (3.0.18)
+    oddb2xml (3.0.19)
       csv
       htmlentities
       httpi

data/History.txt

@@ -1,3 +1,9 @@
+=== 3.0.20 / 09.06.2026
+* Bugfix/robustness: handle truncated or failed Swissmedic .xlsx downloads instead of crashing later in the parser (issue #121). A download through a scanning/allow-list proxy can return an incomplete .xlsx -- a valid ZIP/Excel header (so `file` still reports "Microsoft Excel 2007+") but a missing end-of-central-directory record -- which made RubyXL die deep in rubyzip with the cryptic "Zip end of central directory signature not found". New Oddb2xml.valid_zip? verifies an archive is complete (PK header + EOCD signature in the tail). SwissmedicDownloader now validates the downloaded file and automatically re-fetches it (up to the normal retry count) when it is empty or truncated, and no longer reuses a cached broken file under --skip-download. As a last resort SwissmedicExtractor raises a clear, actionable error (pointing at connectivity / "oddb2xml --proxy-check") instead of the rubyzip backtrace.
+
+=== 3.0.19 / 09.06.2026
+* New option --proxy-check: probe connectivity/proxy reachability for every host oddb2xml could need, print a full OK/BLOCKED/UNREACHABLE report (honouring http(s)_proxy) and exit without downloading or building. Exits 0 if all hosts are reachable, 1 otherwise — handy for cron/deploy preflight on allow-list proxies (e.g. "oddb2xml --proxy-check"). Reuses the 3.0.18 proxy checker; checks run concurrently.
+
 === 3.0.18 / 09.06.2026
 * Bugfix: stop the FHIR downloader from crashing with "Errno::ENOENT @ rb_file_s_size ... foph-sl-export-latest-de.ndjson" when run with --skip-download (issue #121). FhirDownloader#skip_download? returned true on the bare --skip-download flag and then called File.size on a file that was never downloaded. Each oddb2xml run uses its own ./downloads dir, so deploy scripts that download once and then re-run with --skip-download in a fresh dir hit this every time. skip_download? now requires the target NDJSON to actually exist on disk before honouring the flag; a missing file falls through to a normal download instead of crashing.
 * New: proxy / connectivity preflight check (issue #121). At the very start of a run, oddb2xml now probes the outbound hosts it needs (honouring the http(s)_proxy environment) and prints a loud warning if any host is blocked by the proxy (HTTP 407 on an allow-list proxy such as Aspectra's Skyhigh gateway) or otherwise unreachable — surfacing the cause up front instead of a later empty-output/Errno symptom. The probed host set is option-aware (e.g. id.gs1.ch only with --firstbase, epl.bag.admin.ch only with --fhir). It only warns and never aborts the run; downloads still proceed and fail individually as before. Checks run concurrently (~6s worst case) and are skipped during tests; set ODDB2XML_SKIP_PROXY_CHECK=1 to silence it.

data/lib/oddb2xml/cli.rb

@@ -39,6 +39,11 @@ module Oddb2xml
     def run
       threads = []
       start_time = Time.now
+      if @options[:proxy_check]
+        ok = ProxyCheck.report(@options)
+        exit(ok ? 0 : 1) unless defined?(RSpec)
+        return ok
+      end
       ProxyCheck.run(@options)
       files2rm = Dir.glob(File.join(DOWNLOADS, "*"))
       FileUtils.rm_f(files2rm, verbose: true) if (files2rm.size > 0) && !Oddb2xml.skip_download?

data/lib/oddb2xml/downloader.rb

@@ -279,13 +279,21 @@ module Oddb2xml
     def download
       @file2save = File.join(DOWNLOADS, "swissmedic_#{@type}.xlsx")
       report_download(@url, @file2save)
-      if @options[:calc] && @options[:skip_download] && File.exist?(@file2save) && ((Time.now - File.ctime(@file2save)).to_i < 24 * 60 * 60)
+      if @options[:calc] && @options[:skip_download] && File.exist?(@file2save) && ((Time.now - File.ctime(@file2save)).to_i < 24 * 60 * 60) && Oddb2xml.valid_zip?(@file2save)
         Oddb2xml.log "SwissmedicDownloader #{__LINE__}: Skip downloading #{@file2save} #{File.size(@file2save)} bytes"
         return File.expand_path(@file2save)
       end
       begin
         @url = @direct_url_link
         download_as(@file2save, "w+")
+        # The Swissmedic file is an .xlsx (a ZIP). Downloads through scanning
+        # proxies are sometimes truncated (valid header, missing EOCD), which
+        # would later crash RubyXL with a cryptic rubyzip error. Verify the
+        # archive is complete and just fetch it again if not. (issue #121)
+        unless Oddb2xml.valid_zip?(@file2save)
+          raise Oddb2xml::IncompleteDownloadError,
+            "Swissmedic #{@type} xlsx is empty or truncated (#{File.size(@file2save)} bytes)"
+        end
         if @options[:artikelstamm]
           # ssconvert is in the package gnumeric (Debian)
           cmd = "ssconvert '#{@file2save}' '#{File.join(DOWNLOADS, File.basename(@file2save).sub(/\.xls.*/, ".csv"))}' 2> /dev/null"
@@ -293,8 +301,12 @@ module Oddb2xml
           system(cmd)
         end
         return File.expand_path(@file2save)
-      rescue Timeout::Error, Errno::ETIMEDOUT
-        retrievable? ? retry : raise
+      rescue Timeout::Error, Errno::ETIMEDOUT, Oddb2xml::IncompleteDownloadError => error
+        if retrievable?
+          Oddb2xml.log("Retrying Swissmedic #{@type} download: #{error.message}")
+          retry
+        end
+        raise
       ensure
         Oddb2xml.download_finished(@file2save, false)
       end

data/lib/oddb2xml/extractor.rb

@@ -260,6 +260,14 @@ module Oddb2xml
       @type = type
       Oddb2xml.log("SwissmedicExtractor #{@filename} #{File.size(@filename)} bytes")
       return unless File.exist?(@filename)
+      unless Oddb2xml.valid_zip?(@filename)
+        raise "SwissmedicExtractor: '#{@filename}' is not a usable .xlsx " \
+          "(#{File.size(@filename)} bytes). The Swissmedic '#{@type}' download failed or was " \
+          "truncated -- the file is empty, an HTML error/proxy page, or an incomplete ZIP " \
+          "(valid header but missing end-of-central-directory). Check connectivity to " \
+          "www.swissmedic.ch (run 'oddb2xml --proxy-check'); a too-small file usually means " \
+          "the proxy cut off a large download."
+      end
       @sheet = RubyXL::Parser.parse(File.expand_path(@filename)).worksheets[0]
     end
 

data/lib/oddb2xml/options.rb

@@ -46,6 +46,7 @@ module Oddb2xml
         opt :use_ra11zip, "Use the ra11.zip (a zipped transfer.dat from Galexis)",
           default: File.exist?("ra11.zip") ? "ra11.zip" : nil, type: :string
         opt :firstbase, "Build all NONPHARMA articles on firstbase (GS1 Switzerland CSV from id.gs1.ch)", short: "b", default: false
+        opt :proxy_check, "Only probe connectivity/proxy reachability for every required host, print a report and exit (no download/build). Honours http(s)_proxy. Exits 0 if all reachable, 1 otherwise.", short: :none, default: false
       end
 
       @opts[:percent] = @opts[:increment]

data/lib/oddb2xml/proxy_check.rb

@@ -43,6 +43,46 @@ module Oddb2xml
       hosts
     end
 
+    # Full union of every host any run could need, regardless of options.
+    # Used by --proxy-check so the report covers everything in one go.
+    def all_hosts
+      BASE_HOSTS.merge(
+        "epl.bag.admin.ch" => "BAG FHIR data (--fhir)",
+        "id.gs1.ch" => "GS1 NONPHARMA (--firstbase / -b)",
+        "www.spezialitaetenliste.ch" => "BAG Spezialitätenliste",
+        "www.medregbm.admin.ch" => "Medizinalberuferegister (-x address)"
+      )
+    end
+
+    # Probe every host and print a full OK/BLOCKED/UNREACHABLE table.
+    # Returns true when all hosts are reachable. Used by `oddb2xml --proxy-check`.
+    def report(_options = {})
+      proxy = proxy_uri
+      results = all_hosts.map do |host, desc|
+        Thread.new { [host, desc, check_host(host, proxy)] }
+      end.map(&:value).sort_by { |(host, _desc, _status)| host }
+
+      header = "oddb2xml connectivity check"
+      header += proxy ? " (via proxy #{proxy.host}:#{proxy.port})" : " (no proxy configured)"
+      puts header
+      results.each do |(host, desc, status)|
+        tag = case status
+        when :ok then "OK     "
+        when :blocked then "BLOCKED" # proxy returned 407
+        else "UNREACH"
+        end
+        puts format("  [%s] %-28s %s", tag, host, desc)
+      end
+      unreachable = results.reject { |(_host, _desc, status)| status == :ok }
+      if unreachable.empty?
+        puts "All #{results.size} hosts reachable."
+        true
+      else
+        puts "#{unreachable.size} of #{results.size} host(s) NOT reachable -- downloads using them will fail."
+        false
+      end
+    end
+
     # Returns :ok, :blocked (proxy 407) or :unreachable for a single host.
     def check_host(host, proxy)
       http =
@@ -85,24 +125,24 @@ module Oddb2xml
 
     def warn_about(problems, proxy)
       line = "=" * 72
-      $stderr.puts line
-      $stderr.puts " oddb2xml CONNECTIVITY WARNING"
-      $stderr.puts " The following hosts could not be reached -- the corresponding"
-      $stderr.puts " downloads will FAIL or produce incomplete data:"
+      warn line
+      warn " oddb2xml CONNECTIVITY WARNING"
+      warn " The following hosts could not be reached -- the corresponding"
+      warn " downloads will FAIL or produce incomplete data:"
       problems.each do |(host, desc, status)|
         tag = (status == :blocked) ? "BLOCKED by proxy (407)" : "UNREACHABLE          "
-        $stderr.puts format("   [%s] %-26s %s", tag, host, desc)
+        warn format("   [%s] %-26s %s", tag, host, desc)
       end
       if proxy
-        $stderr.puts ""
-        $stderr.puts " Proxy in use: #{proxy.host}:#{proxy.port}"
+        warn ""
+        warn " Proxy in use: #{proxy.host}:#{proxy.port}"
         if problems.any? { |(_h, _d, s)| s == :blocked }
-          $stderr.puts " This looks like an allow-list proxy. Ask your admin to allow the"
-          $stderr.puts " hosts above (HTTPS/443), or set credentials in http(s)_proxy."
+          warn " This looks like an allow-list proxy. Ask your admin to allow the"
+          warn " hosts above (HTTPS/443), or set credentials in http(s)_proxy."
         end
       end
-      $stderr.puts " (Set ODDB2XML_SKIP_PROXY_CHECK=1 to silence this check.)"
-      $stderr.puts line
+      warn " (Set ODDB2XML_SKIP_PROXY_CHECK=1 to silence this check.)"
+      warn line
     end
   end
 end

data/lib/oddb2xml/util.rb

@@ -3,10 +3,29 @@ require "htmlentities"
 
 module Oddb2xml
   FAKE_GTIN_START = "999999"
+
+  # Raised when a downloaded archive (zip/xlsx) is empty or truncated, so the
+  # caller can retry the download instead of crashing later in the parser.
+  class IncompleteDownloadError < StandardError; end
+
   def self.gen_prodno(iksnr, seqnr)
     sprintf("%05d", iksnr) + sprintf("%02d", seqnr)
   end
 
+  # True only for a *complete* ZIP/xlsx: a valid local-file header (PK) plus the
+  # End Of Central Directory record (PK\x05\x06) near the end. A truncated
+  # download keeps the header (so `file` still says "Microsoft Excel 2007+") but
+  # loses the EOCD, which makes rubyzip fail with the cryptic "end of central
+  # directory signature not found". See issue #121.
+  def self.valid_zip?(file)
+    return false unless file && File.exist?(file)
+    size = File.size(file)
+    return false unless size > 100
+    return false unless File.binread(file, 2) == "PK"
+    offset = [size - 66_000, 0].max
+    File.binread(file, size - offset, offset).include?("PK\x05\x06".b)
+  end
+
   def self.uri_open(url, max_retries: 3)
     retries = 0
     begin

data/lib/oddb2xml/version.rb

@@ -1,3 +1,3 @@
 module Oddb2xml
-  VERSION = "3.0.18"
+  VERSION = "3.0.20"
 end