oddb2xml 3.0.23 → 3.0.25

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b99346243a08ae356f8e123953b8aa8c0475918e79e184d9de028a74b6a572a3
-  data.tar.gz: d72e4c0d0f9e66576d6488425d91113f0842035fc7e94ccfa6a1fe392dbb3dee
+  metadata.gz: 1c80413abe4bd24c2c3555aa6eb4407c7464e381275a118c7929c686fe73f494
+  data.tar.gz: 7bf43ba8cf901d4d2de42d0e21e95bf2e25e4f1a88360b5cc4451427c52adccb
 SHA512:
-  metadata.gz: 721a6e109a22dbd57951ccf250b675ad0da0318157dd1ccd3a419a153a926690a5590552856d8e59ab9e2683ff6677a877543109a1cc71d80bb5e4a91c25647b
-  data.tar.gz: 46d8484d674fa5f7e7b81bc9f5de81490eceeb0657d5b23f50d2cb97028901b24a9030c1d790a252f0a342ec7931c8f5251bcc1e1bc015d76821a38d2911d47f
+  metadata.gz: b20195b2d9b1d6ddd0e51b7e8062f71e2bc8c2c7b2d74c817b058c1c56f18ef203b119a39cf80e55310bd9c6d06204f15bb3c0e7d2d988837967fa1381a10735
+  data.tar.gz: 26b6d78eddd658b8dbed3949b0df277639636a66b8ed86a3a54f9478a19dfbdc81874b88afdafc3221114a41f59f7184c271206f14b6c1f5135f93b895ef3fa3

data/.ruby-version

@@ -1 +1 @@
-3.4.5
+3.3.6

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    oddb2xml (3.0.23)
+    oddb2xml (3.0.25)
       csv
       htmlentities
       httpi

data/History.txt

@@ -1,3 +1,9 @@
+=== 3.0.25 / 17.06.2026
+* Bugfix (-b/--firstbase): the GS1 Switzerland firstbase CSV (id.gs1.ch) is served with a UTF-8 BOM. FirstbaseExtractor read it with encoding "UTF-8", so the BOM glued onto the first header ("Gtin") and row["Gtin"] returned nil for every row — every line was skipped as having an empty GTIN. The result: @firstbase came out empty and all NONPHARMA articles from the firstbase feed were silently missing, so oddb_article.xml shipped only the ~17k Refdata/Swissmedic base instead of the ~190k GS1 set. Now read with encoding "bom|utf-8", which strips the BOM (live file: all 192'807 rows parse).
+
+=== 3.0.24 / 16.06.2026
+* Bugfix (--skip-download): cached files fetched with a write mode were silently emptied on every --skip-download run. DownloadMethod#download_as restored the file from the ./downloads cache and then re-opened it with the caller's mode (e.g. "w+"), which truncated it to zero bytes before the read — so the returned data was empty. This blanked epha_interactions.csv (oddb_interaction.xml came out with NBR_RECORD=0) and any other source pulled with a "w+"-style mode (LPPV, Weleda/WALA SL, BAG SL group prices) whenever its file was present in the cache. It surfaced in deploy scripts that download once and rebuild several price increments from the shared cache (e.g. -b -I 45/50/55): the first build was correct, every --skip-download rebuild lost the interactions. The skip branch now opens the restored file read-only, preserving any encoding suffix ("w+:iso-8859-1:utf-8" -> "r:iso-8859-1:utf-8").
+
 === 3.0.23 / 12.06.2026
 * Bugfix (--proxy-check): the connectivity check now follows HTTP redirects to other hosts and reports the real "forwarder" target an allow-list proxy must permit, instead of stopping at the first hop. GS1 Switzerland turned id.gs1.ch into a 301 redirect to the global resolver id.gs1.org, which 307-redirects again to apitools.gs1.ch — so allowing only id.gs1.ch is no longer enough, and the firstbase download dies on the blocked target. Previously any 3xx answer was reported as "OK", so the check was falsely green; it now shows e.g. "[BLOCKED] id.gs1.ch -> apitools.gs1.ch" plus a "must be on the proxy allow-list too" note for every cross-host redirect. id.gs1.org is also probed explicitly (added to --firstbase's host set and to the full --proxy-check report).
 * Improvement (--proxy-check): each host is now probed with the actual resource path the downloader fetches (e.g. raw.githubusercontent.com/zdavatz/…, www.spezialitaetenliste.ch/File.axd, files.refdata.ch/…/Refdata.Articles.zip) rather than "/". Probing "/" produced misleading host redirects (raw.githubusercontent.com/ -> github.com) for hosts whose real download path returns 200 directly; the genuine paths also reveal real forwarders such as www.spezialitaetenliste.ch -> sl.bag.admin.ch.

data/lib/oddb2xml/downloader.rb

@@ -18,7 +18,13 @@ module Oddb2xml
       data = nil
       FileUtils.makedirs(File.dirname(file), verbose: true)
       if Oddb2xml.skip_download(file)
-        io = File.open(file, option)
+        # The file has just been restored from the download cache. Open it
+        # read-only: a write mode like "w+" would truncate the cached file to
+        # zero bytes before the read, silently emptying it (e.g. it blanked
+        # epha_interactions.csv on every --skip-download run). Preserve any
+        # encoding suffix (e.g. "w+:iso-8859-1:utf-8" -> "r:iso-8859-1:utf-8").
+        read_option = option.sub(/\A[wa]\+?/, "r")
+        io = File.open(file, read_option)
         data = io.read
       else
         begin

data/lib/oddb2xml/extractor.rb

@@ -622,7 +622,10 @@ module Oddb2xml
     def to_hash
       data = {}
       return data unless @file && File.exist?(@file)
-      CSV.foreach(@file, headers: true, encoding: "UTF-8") do |row|
+      # The GS1 firstbase CSV is served with a UTF-8 BOM. Without "bom|" the BOM
+      # glues onto the first header ("Gtin"), so row["Gtin"] is nil and every
+      # row is skipped — dropping all -b/firstbase NONPHARMA articles.
+      CSV.foreach(@file, headers: true, encoding: "bom|utf-8") do |row|
         gtin = row["Gtin"].to_s.gsub(/^0+/, "")
         next if gtin.empty?
         data[gtin] = {

data/lib/oddb2xml/version.rb

@@ -1,3 +1,3 @@
 module Oddb2xml
-  VERSION = "3.0.23"
+  VERSION = "3.0.25"
 end

data/scripts/run_oddb2xml.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+#
+# run_oddb2xml — build the firstbase (-b) oddb2xml feed at several price
+# increments and stage the results for transfer.
+#
+# The upstream sources are downloaded ONCE: the first build fetches them, and
+# every subsequent increment re-uses the cached ./downloads via --skip-download.
+# All builds therefore run in a single shared working directory ($BUILD_DIR) —
+# the original deploy script cd'd into a separate dir per increment, where
+# --skip-download could not find downloads/ (DOWNLOADS is cwd-relative) and
+# silently re-downloaded everything each time.
+#
+# Output layout (under $OUT_DIR, default /home/zdavatz/oddb2xml):
+#   <OUT_DIR>/45/        oddb_*.xml built with +45 %
+#   <OUT_DIR>/50/        oddb_*.xml built with +50 %
+#   <OUT_DIR>/55/        oddb_*.xml built with +55 %
+#   <OUT_DIR>/default/   oddb_*.xml built with no increment
+# Each destination dir also keeps the source archive as oddb2xml.zip.
+# The working dir ($BUILD_DIR, default <OUT_DIR>-build) holds the shared
+# downloads/ cache and the transient zip; it lives OUTSIDE $OUT_DIR so the
+# transfer's `scp -r $OUT_DIR/*` never uploads the multi-hundred-MB cache.
+#
+# Configurable via environment:
+#   OUT_DIR           destination root          (default /home/zdavatz/oddb2xml)
+#   BUILD_DIR         working dir               (default <OUT_DIR>-build)
+#   INCREMENTS        space-separated percents   (default "45 50 55")
+#   ODDB2XML_BIN      oddb2xml executable        (default oddb2xml)
+#   SKIP_GEM_INSTALL  set to 1 to skip `gem install oddb2xml`
+#   RUN_TRANSFER      set to 1 to run the transfer (scripts/transfer.sh) at the end
+#   TRANSFER_CMD      transfer command (default: sudo, preserving
+#                     ODDB2XML_TRANSFER_DIR, scripts/transfer.sh next to this file)
+#
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+OUT_DIR="${OUT_DIR:-/home/zdavatz/oddb2xml}"
+BUILD_DIR="${BUILD_DIR:-${OUT_DIR%/}-build}"
+INCREMENTS="${INCREMENTS:-45 50 55}"
+ODDB2XML_BIN="${ODDB2XML_BIN:-oddb2xml}"
+TRANSFER_CMD="${TRANSFER_CMD:-$SCRIPT_DIR/transfer.sh}"
+
+log() { printf '%s %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*"; }
+
+# 1. Install / update the published gem unless told otherwise.
+if [[ "${SKIP_GEM_INSTALL:-0}" != "1" ]]; then
+  log "Installing oddb2xml gem"
+  gem install oddb2xml
+fi
+
+# 2. Fresh working dir (keeps a shared downloads/ cache across increments).
+log "Preparing build dir $BUILD_DIR"
+rm -rf "$BUILD_DIR"
+mkdir -p "$BUILD_DIR"
+cd "$BUILD_DIR"
+
+first=1
+
+# build_one <increment-percent|""> <destination-subdir>
+build_one() {
+  local inc="$1" name="$2" dest="$OUT_DIR/$2"
+  local inc_opt=() dl_opt=()
+  [[ -n "$inc" ]] && inc_opt=(-I "$inc")
+
+  if [[ $first -eq 1 ]]; then
+    first=0                      # first build downloads the sources
+  else
+    dl_opt=(--skip-download)     # the rest re-use the cached downloads/
+  fi
+
+  log "Building increment '${inc:-none}' -> $dest"
+  rm -f oddb*.zip
+  "$ODDB2XML_BIN" "${dl_opt[@]}" -b "${inc_opt[@]}" -c zip
+
+  shopt -s nullglob
+  local zips=(oddb*.zip)
+  shopt -u nullglob
+  [[ ${#zips[@]} -ge 1 ]] || { log "ERROR: no zip produced for increment '${inc:-none}'"; exit 1; }
+  local zip="${zips[0]}"
+
+  rm -rf "$dest"
+  mkdir -p "$dest"
+  unzip -o -q -d "$dest" "$zip"
+  mv "$zip" "$dest/oddb2xml.zip"
+  log "Staged $dest"
+}
+
+for inc in $INCREMENTS; do
+  build_one "$inc" "$inc"
+done
+build_one "" "default"           # final run with no increment
+
+# 3. Optional hand-off to the transfer step (scripts/transfer.sh).
+if [[ "${RUN_TRANSFER:-0}" == "1" ]]; then
+  log "Running transfer: $TRANSFER_CMD"
+  export ODDB2XML_TRANSFER_DIR="$OUT_DIR"   # keep transfer.sh in sync with OUT_DIR
+  $TRANSFER_CMD
+fi
+
+log "Done. Output under $OUT_DIR"

data/scripts/transfer.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+#
+# transfer.sh — push the generated oddb2xml feeds (plus the aips2sqlite
+# Fachinformation XML and the swissmedic-sequences CSV) to the HIN download
+# server via scp. Runs on the ywesee host; everything is user-owned, so no sudo.
+#
+# Paths default to this host's layout and can be overridden via environment:
+#   ODDB2XML_TRANSFER_DIR  dir whose contents go to .../download/oddb2xml/
+#                          (default /home/zdavatz/oddb2xml)
+#   AIPS2SQLITE_DIR        aips2sqlite output dir
+#                          (default /home/zdavatz/software/aips2sqlite/jars/output)
+#   SSH_KEY                scp identity file        (default ~/.ssh/id_ed25519)
+#   SCP_DEST               scp destination base, e.g. user@host:/path/download
+#                          (REQUIRED — no default yet; set the new download server)
+
+ODDB2XML_TRANSFER_DIR="${ODDB2XML_TRANSFER_DIR:-/home/zdavatz/oddb2xml}"
+AIPS2SQLITE_DIR="${AIPS2SQLITE_DIR:-/home/zdavatz/software/aips2sqlite/jars/output}"
+SSH_KEY="${SSH_KEY:-$HOME/.ssh/id_ed25519}"
+# TODO: set the new download-server destination.
+SCP_DEST="${SCP_DEST:?set SCP_DEST to the scp target, e.g. user@host:/var/www/.../download}"
+
+###
+### ODDB2XML
+###
+
+find "$ODDB2XML_TRANSFER_DIR/" -type d -exec chmod 755 {} \;
+find "$ODDB2XML_TRANSFER_DIR/" -type f -exec chmod 644 {} \;
+
+scp -r -i "$SSH_KEY" "$ODDB2XML_TRANSFER_DIR"/* "$SCP_DEST/oddb2xml/"
+
+###
+### aips2sqlite
+###
+
+if [ -d "$AIPS2SQLITE_DIR/fis" ]; then
+  find "$AIPS2SQLITE_DIR/fis" -name '*.xml' -type f -exec chmod 644 {} \;
+  scp -r -i "$SSH_KEY" "$AIPS2SQLITE_DIR"/fis/*.xml "$SCP_DEST/mediupdate-xml/"
+fi
+
+if [ -f "$AIPS2SQLITE_DIR/oddb2xml_swissmedic_sequences.csv" ]; then
+  chmod 644 "$AIPS2SQLITE_DIR/oddb2xml_swissmedic_sequences.csv"
+  scp -r -i "$SSH_KEY" "$AIPS2SQLITE_DIR/oddb2xml_swissmedic_sequences.csv" "$SCP_DEST/oddb2xml/"
+fi
+
+exit 0