oddb2xml 3.0.22 → 3.0.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b116a974b4b6f80e5dba15a710f53fe2c4b0b44bcd094216eb0f46678d703beb
-  data.tar.gz: e994a68fedddc58377421d5883322578ce5b6b119deaae1ba793acb4be190a9a
+  metadata.gz: 6c5dd43eea5580b800dcb7c1ede8f2b1b55ded474a7608471185bd0fab81f5c0
+  data.tar.gz: 0f2b55c4a6706b4e723efbcb77bd3c40b321eb3cf2893b58230572e03b6936a5
 SHA512:
-  metadata.gz: baf0927ae2293672d4e1db0e61ace759fc8fe11cb05e936a556eafd3740d4355d9b6cf5b5e049aa9205125760d6d363bcecaa1cb98c93d4646981502b087f6b4
-  data.tar.gz: 03217fb23800e5745aef09b836860e4f6f021b641e2ff7076d382f5eb4235f2ce618360966d0a59fdaad53647b7bd8b558cd131b15ed320ee2706ac9a98a2c97
+  metadata.gz: a55b5366832bd577609b70baa32934119ef953a441296c76848f183565c70709f7a24541f1927c23c78ded28b6b515b802ea287f34f58b5a50228ed6bba071ba
+  data.tar.gz: d4591b9b3e812a134e0598865d5385f0fcbe3546cc1b799e62c2c4b8d6c22ce8ee8347e32c7c967a3bd6051aa5e1d45ad8a262942a841b67a4ef22afd6a92579

data/.ruby-version

@@ -1 +1 @@
-3.4.5
+3.3.6

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    oddb2xml (3.0.22)
+    oddb2xml (3.0.24)
       csv
       htmlentities
       httpi

data/History.txt

@@ -1,3 +1,10 @@
+=== 3.0.24 / 16.06.2026
+* Bugfix (--skip-download): cached files fetched with a write mode were silently emptied on every --skip-download run. DownloadMethod#download_as restored the file from the ./downloads cache and then re-opened it with the caller's mode (e.g. "w+"), which truncated it to zero bytes before the read — so the returned data was empty. This blanked epha_interactions.csv (oddb_interaction.xml came out with NBR_RECORD=0) and any other source pulled with a "w+"-style mode (LPPV, Weleda/WALA SL, BAG SL group prices) whenever its file was present in the cache. It surfaced in deploy scripts that download once and rebuild several price increments from the shared cache (e.g. -b -I 45/50/55): the first build was correct, every --skip-download rebuild lost the interactions. The skip branch now opens the restored file read-only, preserving any encoding suffix ("w+:iso-8859-1:utf-8" -> "r:iso-8859-1:utf-8").
+
+=== 3.0.23 / 12.06.2026
+* Bugfix (--proxy-check): the connectivity check now follows HTTP redirects to other hosts and reports the real "forwarder" target an allow-list proxy must permit, instead of stopping at the first hop. GS1 Switzerland turned id.gs1.ch into a 301 redirect to the global resolver id.gs1.org, which 307-redirects again to apitools.gs1.ch — so allowing only id.gs1.ch is no longer enough, and the firstbase download dies on the blocked target. Previously any 3xx answer was reported as "OK", so the check was falsely green; it now shows e.g. "[BLOCKED] id.gs1.ch -> apitools.gs1.ch" plus a "must be on the proxy allow-list too" note for every cross-host redirect. id.gs1.org is also probed explicitly (added to --firstbase's host set and to the full --proxy-check report).
+* Improvement (--proxy-check): each host is now probed with the actual resource path the downloader fetches (e.g. raw.githubusercontent.com/zdavatz/…, www.spezialitaetenliste.ch/File.axd, files.refdata.ch/…/Refdata.Articles.zip) rather than "/". Probing "/" produced misleading host redirects (raw.githubusercontent.com/ -> github.com) for hosts whose real download path returns 200 directly; the genuine paths also reveal real forwarders such as www.spezialitaetenliste.ch -> sl.bag.admin.ch.
+
 === 3.0.22 / 11.06.2026
 * New (Artikelstamm / -e / -b): extend the 3.0.21 "Kapitel 70" SL recovery to WALA products (GTIN prefix 7640187…), in addition to Weleda. A third runtime CSV in github.com/zdavatz/oddb2xml_files, wala_arzneimittel.csv (bundled fallback under data/), is mapped exactly like the Weleda file: for any GTIN absent from the FHIR NDJSON, oddb2xml fills <SL_ENTRY>true</SL_ENTRY> and the BAG SL public price (<PPUB> in Artikelstamm, the standard <ARTPRI><PTYP>PPUB</PTYP> entry in oddb_article.xml for -e/-b). The WALA layout differs from Weleda: it is ";"-separated with a BOM, has no "/ SL" column (a row is SL when it carries a CSL-Code = Kapitel-70.01 group code), and the public package price is given inline in the "CSL 70.01." column -- already multiplied for the pack size (the multiplier appears only in the galenic-form text, e.g. "Solutio ad inj. 10 x 1 ml"), so it is taken verbatim instead of being re-joined against bag_sl_group_prices.csv (which would yield the 1/10 per-unit price). The FHIR/ZurRose price still always wins; this only fills a gap. Live file: 320 WALA SL products with prices.
 

data/lib/oddb2xml/downloader.rb

@@ -18,7 +18,13 @@ module Oddb2xml
       data = nil
       FileUtils.makedirs(File.dirname(file), verbose: true)
       if Oddb2xml.skip_download(file)
-        io = File.open(file, option)
+        # The file has just been restored from the download cache. Open it
+        # read-only: a write mode like "w+" would truncate the cached file to
+        # zero bytes before the read, silently emptying it (e.g. it blanked
+        # epha_interactions.csv on every --skip-download run). Preserve any
+        # encoding suffix (e.g. "w+:iso-8859-1:utf-8" -> "r:iso-8859-1:utf-8").
+        read_option = option.sub(/\A[wa]\+?/, "r")
+        io = File.open(file, read_option)
         data = io.read
       else
         begin

data/lib/oddb2xml/proxy_check.rb

@@ -25,6 +25,25 @@ module Oddb2xml
 
     TIMEOUT = 6 # seconds, per host (open + read); checks run concurrently
 
+    # Representative resource path per host -- the actual file the downloader
+    # fetches, NOT "/". Probing "/" gives misleading host redirects (e.g.
+    # raw.githubusercontent.com/ -> github.com, while the real raw file path
+    # returns 200), whereas the genuine download paths reveal the real
+    # forwarder chain the proxy must allow (id.gs1.ch -> id.gs1.org ->
+    # apitools.gs1.ch; www.spezialitaetenliste.ch/File.axd -> sl.bag.admin.ch).
+    PROBE_PATHS = {
+      "files.refdata.ch" => "/simis-public-prod/Articles/1.0/Refdata.Articles.zip",
+      "raw.githubusercontent.com" => "/zdavatz/oddb2xml_files/master/LPPV.txt",
+      "id.gs1.ch" => "/01/07612345000961",
+      "id.gs1.org" => "/01/07612345000961",
+      "www.spezialitaetenliste.ch" => "/File.axd?file=XMLPublications.zip",
+      "www.medregbm.admin.ch" => "/Publikation/"
+    }.freeze
+
+    def probe_path(host)
+      PROBE_PATHS[host] || "/"
+    end
+
     def proxy_uri
       env = ENV["https_proxy"] || ENV["HTTPS_PROXY"] || ENV["http_proxy"] || ENV["HTTP_PROXY"]
       return nil if env.nil? || env.empty?
@@ -34,10 +53,25 @@ module Oddb2xml
       nil
     end
 
+    # Redirect targets ("forwarders") that an allow-list proxy must permit in
+    # addition to the host we actually request. id.gs1.ch 301-redirects every
+    # path to the global resolver id.gs1.org, so allowing only id.gs1.ch is not
+    # enough -- the firstbase download follows the redirect and dies on the
+    # blocked target. The real firstbase chain is id.gs1.ch -> id.gs1.org ->
+    # apitools.gs1.ch, so the redirect is followed dynamically too (see
+    # check_host); this list just guarantees the known target is probed even
+    # when the redirect probe is short-circuited.
+    FORWARDERS = {
+      "id.gs1.org" => "GS1 global resolver (id.gs1.ch redirect target, --firstbase / -b)"
+    }.freeze
+
     def hosts_for(options = {})
       hosts = BASE_HOSTS.dup
       hosts["epl.bag.admin.ch"] = "BAG FHIR data (--fhir)" if options[:fhir]
-      hosts["id.gs1.ch"] = "GS1 NONPHARMA (--firstbase / -b)" if options[:firstbase]
+      if options[:firstbase]
+        hosts["id.gs1.ch"] = "GS1 NONPHARMA (--firstbase / -b)"
+        hosts["id.gs1.org"] = FORWARDERS["id.gs1.org"]
+      end
       hosts["www.spezialitaetenliste.ch"] = "BAG Spezialitätenliste" unless options[:fhir]
       hosts["www.medregbm.admin.ch"] = "Medizinalberuferegister (-x address)" if options[:address]
       hosts
@@ -51,7 +85,7 @@ module Oddb2xml
         "id.gs1.ch" => "GS1 NONPHARMA (--firstbase / -b)",
         "www.spezialitaetenliste.ch" => "BAG Spezialitätenliste",
         "www.medregbm.admin.ch" => "Medizinalberuferegister (-x address)"
-      )
+      ).merge(FORWARDERS)
     end
 
     # Probe every host and print a full OK/BLOCKED/UNREACHABLE table.
@@ -59,32 +93,40 @@ module Oddb2xml
     def report(_options = {})
       proxy = proxy_uri
       results = all_hosts.map do |host, desc|
-        Thread.new { [host, desc, check_host(host, proxy)] }
+        Thread.new { [host, desc, check_host(host, proxy, probe_path(host))] }
       end.map(&:value).sort_by { |(host, _desc, _status)| host }
 
       header = "oddb2xml connectivity check"
       header += proxy ? " (via proxy #{proxy.host}:#{proxy.port})" : " (no proxy configured)"
       puts header
       results.each do |(host, desc, status)|
-        tag = case status
+        tag = case status[:result]
         when :ok then "OK     "
         when :blocked then "BLOCKED" # proxy returned 407
         else "UNREACH"
         end
-        puts format("  [%s] %-28s %s", tag, host, desc)
+        label = status[:via] ? "#{host} -> #{status[:via]}" : host
+        puts format("  [%s] %-36s %s", tag, label, desc)
       end
-      unreachable = results.reject { |(_host, _desc, status)| status == :ok }
+      unreachable = results.reject { |(_host, _desc, status)| status[:result] == :ok }
       if unreachable.empty?
         puts "All #{results.size} hosts reachable."
         true
       else
         puts "#{unreachable.size} of #{results.size} host(s) NOT reachable -- downloads using them will fail."
+        results.select { |(_host, _desc, status)| status[:via] }.each do |(host, _desc, status)|
+          puts "  note: #{host} redirects to #{status[:via]} -- that host must be on the proxy allow-list too."
+        end
         false
       end
     end
 
-    # Returns :ok, :blocked (proxy 407) or :unreachable for a single host.
-    def check_host(host, proxy)
+    # Probe a host (following HTTP redirects to other hosts) and return a Hash:
+    #   { result: :ok | :blocked | :unreachable, via: "final.host" | nil }
+    # `:via` is set only when the host redirected to a *different* host, so the
+    # caller can surface that the redirect target (e.g. id.gs1.ch -> id.gs1.org)
+    # must be reachable too -- a 301 to a blocked host used to be reported as OK.
+    def check_host(host, proxy, path = "/", hops = 4, origin = nil)
       http =
         if proxy
           Net::HTTP.new(host, 443, proxy.host, proxy.port, proxy.user, proxy.password)
@@ -96,14 +138,27 @@ module Oddb2xml
       http.open_timeout = TIMEOUT
       http.read_timeout = TIMEOUT
       http.start do |h|
-        res = h.head("/")
-        return :blocked if res.code.to_s == "407"
-        return :ok # any HTTP answer (200/301/403/404/...) means the host is reachable
+        res = h.head(path)
+        return {result: :blocked, via: via_for(origin, host)} if res.code.to_s == "407"
+        if res.code.to_s.start_with?("3") && res["location"] && hops > 0
+          loc = URI.parse(res["location"])
+          if loc.host && loc.host != host
+            next_path = (loc.respond_to?(:request_uri) && loc.request_uri) ? loc.request_uri : "/"
+            return check_host(loc.host, proxy, next_path, hops - 1, origin || host)
+          end
+        end
+        # any other HTTP answer (200/403/404/...) means this host is reachable
+        return {result: :ok, via: via_for(origin, host)}
       end
     rescue => error
       msg = error.message.to_s.downcase
-      return :blocked if msg.include?("407") || msg.include?("authenticationrequired") || msg.include?("proxy")
-      :unreachable
+      blocked = msg.include?("407") || msg.include?("authenticationrequired") || msg.include?("proxy")
+      {result: blocked ? :blocked : :unreachable, via: via_for(origin, host)}
+    end
+
+    # The final host reached, but only when it differs from where we started.
+    def via_for(origin, host)
+      (origin && origin != host) ? host : nil
     end
 
     # Probe all relevant hosts concurrently and warn about any that fail.
@@ -114,10 +169,10 @@ module Oddb2xml
       proxy = proxy_uri
       hosts = hosts_for(options)
       results = hosts.map do |host, desc|
-        Thread.new { [host, desc, check_host(host, proxy)] }
+        Thread.new { [host, desc, check_host(host, proxy, probe_path(host))] }
       end.map(&:value)
 
-      problems = results.reject { |(_host, _desc, status)| status == :ok }
+      problems = results.reject { |(_host, _desc, status)| status[:result] == :ok }
       return if problems.empty?
 
       warn_about(problems, proxy)
@@ -130,8 +185,9 @@ module Oddb2xml
       warn " The following hosts could not be reached -- the corresponding"
       warn " downloads will FAIL or produce incomplete data:"
       problems.each do |(host, desc, status)|
-        tag = (status == :blocked) ? "BLOCKED by proxy (407)" : "UNREACHABLE          "
-        warn format("   [%s] %-26s %s", tag, host, desc)
+        tag = (status[:result] == :blocked) ? "BLOCKED by proxy (407)" : "UNREACHABLE          "
+        label = status[:via] ? "#{host} -> #{status[:via]}" : host
+        warn format("   [%s] %-34s %s", tag, label, desc)
       end
       if proxy
         warn ""

data/lib/oddb2xml/version.rb

@@ -1,3 +1,3 @@
 module Oddb2xml
-  VERSION = "3.0.22"
+  VERSION = "3.0.24"
 end

data/scripts/run_oddb2xml.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+#
+# run_oddb2xml — build the firstbase (-b) oddb2xml feed at several price
+# increments and stage the results for transfer.
+#
+# The upstream sources are downloaded ONCE: the first build fetches them, and
+# every subsequent increment re-uses the cached ./downloads via --skip-download.
+# All builds therefore run in a single shared working directory ($BUILD_DIR) —
+# the original deploy script cd'd into a separate dir per increment, where
+# --skip-download could not find downloads/ (DOWNLOADS is cwd-relative) and
+# silently re-downloaded everything each time.
+#
+# Output layout (under $OUT_DIR, default /home/zdavatz/oddb2xml):
+#   <OUT_DIR>/45/        oddb_*.xml built with +45 %
+#   <OUT_DIR>/50/        oddb_*.xml built with +50 %
+#   <OUT_DIR>/55/        oddb_*.xml built with +55 %
+#   <OUT_DIR>/default/   oddb_*.xml built with no increment
+# Each destination dir also keeps the source archive as oddb2xml.zip.
+# The working dir ($BUILD_DIR, default <OUT_DIR>-build) holds the shared
+# downloads/ cache and the transient zip; it lives OUTSIDE $OUT_DIR so the
+# transfer's `scp -r $OUT_DIR/*` never uploads the multi-hundred-MB cache.
+#
+# Configurable via environment:
+#   OUT_DIR           destination root          (default /home/zdavatz/oddb2xml)
+#   BUILD_DIR         working dir               (default <OUT_DIR>-build)
+#   INCREMENTS        space-separated percents   (default "45 50 55")
+#   ODDB2XML_BIN      oddb2xml executable        (default oddb2xml)
+#   SKIP_GEM_INSTALL  set to 1 to skip `gem install oddb2xml`
+#   RUN_TRANSFER      set to 1 to run the transfer (scripts/transfer.sh) at the end
+#   TRANSFER_CMD      transfer command (default: sudo, preserving
+#                     ODDB2XML_TRANSFER_DIR, scripts/transfer.sh next to this file)
+#
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+OUT_DIR="${OUT_DIR:-/home/zdavatz/oddb2xml}"
+BUILD_DIR="${BUILD_DIR:-${OUT_DIR%/}-build}"
+INCREMENTS="${INCREMENTS:-45 50 55}"
+ODDB2XML_BIN="${ODDB2XML_BIN:-oddb2xml}"
+TRANSFER_CMD="${TRANSFER_CMD:-$SCRIPT_DIR/transfer.sh}"
+
+log() { printf '%s %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*"; }
+
+# 1. Install / update the published gem unless told otherwise.
+if [[ "${SKIP_GEM_INSTALL:-0}" != "1" ]]; then
+  log "Installing oddb2xml gem"
+  gem install oddb2xml
+fi
+
+# 2. Fresh working dir (keeps a shared downloads/ cache across increments).
+log "Preparing build dir $BUILD_DIR"
+rm -rf "$BUILD_DIR"
+mkdir -p "$BUILD_DIR"
+cd "$BUILD_DIR"
+
+first=1
+
+# build_one <increment-percent|""> <destination-subdir>
+build_one() {
+  local inc="$1" name="$2" dest="$OUT_DIR/$2"
+  local inc_opt=() dl_opt=()
+  [[ -n "$inc" ]] && inc_opt=(-I "$inc")
+
+  if [[ $first -eq 1 ]]; then
+    first=0                      # first build downloads the sources
+  else
+    dl_opt=(--skip-download)     # the rest re-use the cached downloads/
+  fi
+
+  log "Building increment '${inc:-none}' -> $dest"
+  rm -f oddb*.zip
+  "$ODDB2XML_BIN" "${dl_opt[@]}" -b "${inc_opt[@]}" -c zip
+
+  shopt -s nullglob
+  local zips=(oddb*.zip)
+  shopt -u nullglob
+  [[ ${#zips[@]} -ge 1 ]] || { log "ERROR: no zip produced for increment '${inc:-none}'"; exit 1; }
+  local zip="${zips[0]}"
+
+  rm -rf "$dest"
+  mkdir -p "$dest"
+  unzip -o -q -d "$dest" "$zip"
+  mv "$zip" "$dest/oddb2xml.zip"
+  log "Staged $dest"
+}
+
+for inc in $INCREMENTS; do
+  build_one "$inc" "$inc"
+done
+build_one "" "default"           # final run with no increment
+
+# 3. Optional hand-off to the transfer step (scripts/transfer.sh).
+if [[ "${RUN_TRANSFER:-0}" == "1" ]]; then
+  log "Running transfer: $TRANSFER_CMD"
+  export ODDB2XML_TRANSFER_DIR="$OUT_DIR"   # keep transfer.sh in sync with OUT_DIR
+  $TRANSFER_CMD
+fi
+
+log "Done. Output under $OUT_DIR"

data/scripts/transfer.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+#
+# transfer.sh — push the generated oddb2xml feeds (plus the aips2sqlite
+# Fachinformation XML and the swissmedic-sequences CSV) to the HIN download
+# server via scp. Runs on the ywesee host; everything is user-owned, so no sudo.
+#
+# Paths default to this host's layout and can be overridden via environment:
+#   ODDB2XML_TRANSFER_DIR  dir whose contents go to .../download/oddb2xml/
+#                          (default /home/zdavatz/oddb2xml)
+#   AIPS2SQLITE_DIR        aips2sqlite output dir
+#                          (default /home/zdavatz/software/aips2sqlite/jars/output)
+#   SSH_KEY                scp identity file        (default ~/.ssh/id_ed25519)
+#   SCP_DEST               scp destination base, e.g. user@host:/path/download
+#                          (REQUIRED — no default yet; set the new download server)
+
+ODDB2XML_TRANSFER_DIR="${ODDB2XML_TRANSFER_DIR:-/home/zdavatz/oddb2xml}"
+AIPS2SQLITE_DIR="${AIPS2SQLITE_DIR:-/home/zdavatz/software/aips2sqlite/jars/output}"
+SSH_KEY="${SSH_KEY:-$HOME/.ssh/id_ed25519}"
+# TODO: set the new download-server destination.
+SCP_DEST="${SCP_DEST:?set SCP_DEST to the scp target, e.g. user@host:/var/www/.../download}"
+
+###
+### ODDB2XML
+###
+
+find "$ODDB2XML_TRANSFER_DIR/" -type d -exec chmod 755 {} \;
+find "$ODDB2XML_TRANSFER_DIR/" -type f -exec chmod 644 {} \;
+
+scp -r -i "$SSH_KEY" "$ODDB2XML_TRANSFER_DIR"/* "$SCP_DEST/oddb2xml/"
+
+###
+### aips2sqlite
+###
+
+if [ -d "$AIPS2SQLITE_DIR/fis" ]; then
+  find "$AIPS2SQLITE_DIR/fis" -name '*.xml' -type f -exec chmod 644 {} \;
+  scp -r -i "$SSH_KEY" "$AIPS2SQLITE_DIR"/fis/*.xml "$SCP_DEST/mediupdate-xml/"
+fi
+
+if [ -f "$AIPS2SQLITE_DIR/oddb2xml_swissmedic_sequences.csv" ]; then
+  chmod 644 "$AIPS2SQLITE_DIR/oddb2xml_swissmedic_sequences.csv"
+  scp -r -i "$SSH_KEY" "$AIPS2SQLITE_DIR/oddb2xml_swissmedic_sequences.csv" "$SCP_DEST/oddb2xml/"
+fi
+
+exit 0