oddb2xml 3.0.2 → 3.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 55725f8e11fa91216b5f7189f9339a6bacf6fd69af78c633421abc8c573a2326
-  data.tar.gz: acd34c442ef756f43aeac9e2551df614fff4bad81894a87add521e4c7efab8cd
+  metadata.gz: 25c142b0dfeb3bb43d540e7d9837b5199d5e2eb7c5afbd07fe278f8e4b40516e
+  data.tar.gz: d9dad26c7d08193bc3af27262ebfd32f0edce7d4c56e457d6d3cfabf845900c7
 SHA512:
-  metadata.gz: 273f80bb691ae1d0e86db41d9215456c48822f4a764163c3675d87ca6f975f11cd8d2dce89f4bf71abd9b8e320b61babd45c73d0683faafbeb25adf56f2d12b6
-  data.tar.gz: c0014b66c15ebddcff17cf1435d24faab68ab856faa5d198208f94d9d6925540d4939ce4ff82f4f4d38c1fc319156dbaecf49415f56e8fce1d676d3e06b179fc
+  metadata.gz: bd8480feba005844d3d2dd46f31a69bce0cdf076a75c041eeed53808a66733cb821cdc6633ae82cbba4a1347822ec123bf2ed594609145ceb8c3227ab06f7433
+  data.tar.gz: ebee25f81ef3c9f56c748175fca7d8c12c1894a3ba5db1d94d4c494f0deb223192af0f727cf567921981ce00e99f412789ba92a1cd5ae11f8c80d745dba99cfa

data/CLAUDE.md

@@ -39,7 +39,7 @@ The system follows a **download → extract → build → compress** pipeline:
 
 2. **Downloaders** (`lib/oddb2xml/downloader.rb`) — 11 subclasses of `Downloader`, each fetching from a specific Swiss data source. Files cached in `./downloads/`.
 
-3. **Extractors** (`lib/oddb2xml/extractor.rb`) — Matching extractor classes that parse downloaded files into Ruby hashes. Formats include XML (nokogiri/sax-machine), XLSX (rubyXL), SOAP (savon), CSV, and fixed-width text.
+3. **Extractors** (`lib/oddb2xml/extractor.rb`) — Matching extractor classes that parse downloaded files into Ruby hashes. Formats include XML (nokogiri/sax-machine), XLSX (rubyXL), CSV, and fixed-width text. Refdata uses the new SwissReg XML format from a zip download (`files.refdata.ch`).
 
 4. **Builder** (`lib/oddb2xml/builder.rb`) — The largest file (~1900 lines). Merges extracted data and generates output XML/DAT files. Methods follow `prepare_*` (data assembly) and `build_*` (output generation) naming.
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    oddb2xml (3.0.2)
+    oddb2xml (3.0.3)
       htmlentities
       httpi
       mechanize (>= 2.8.5)

data/History.txt

@@ -1,7 +1,15 @@
+=== 3.0.3 / 24.04.2026
+* FHIR: download per-language NDJSON files (foph-sl-export-latest-{de,fr,it}.ndjson) so French and Italian product names/descriptions are populated
+* FHIR: map legal status code 756005022008 to Swissmedic category D (in addition to 756005022007)
+
 === 3.0.2 / 09.03.2026
 * Use raw.githubusercontent.com URL for ATC CSV to avoid 429 Too Many Requests errors
 * Add retry logic with exponential backoff for HTTP 429 errors in uri_open
 * Remove obsolete Ruby version check in uri_open (Ruby >= 2.5 already required)
+* Migrate Refdata from SOAP API to new SwissReg zip download (files.refdata.ch)
+* Fix UTF-8 encoding for German umlauts in SORTD field (e.g. ö → Ö)
+* Restore Italian description fallback to German when not available from Refdata
+* Fix Optimist short option conflict for --fhir and --fhir_url options
 
 === 2.7.9 / 19.09.22
 * Remove newly generated DSCRI when not running --artikelstamm and 

data/README.md

@@ -25,7 +25,7 @@ creates .dat files according to ([IGM-11](http://dev.ywesee.com/uploads/att/IGM.
 * oddb.dat
 * oddb_with_migel.dat
 
-the files are using [swissINDEX](http://www.refdata.ch/downloads/company/download/swissindex_TechnischeBeschreibung.pdf), [BAG-XML](http://bag.e-mediat.net/SL2007.Web.External/Default.aspx?webgrab=ignore) and [Swissmedic](http://www.swissmedic.ch/daten/00080/00251/index.html?lang=de) as sources.
+the files are using [Refdata](https://files.refdata.ch/simis-public-prod/Articles/1.0/Refdata.Articles.zip), [BAG-XML](http://bag.e-mediat.net/SL2007.Web.External/Default.aspx?webgrab=ignore) and [Swissmedic](http://www.swissmedic.ch/daten/00080/00251/index.html?lang=de) as sources.
 
 The following additional data is in the files:
 
@@ -274,16 +274,14 @@ We use the following files:
 
 * https://www.swissmedic.ch/arzneimittel/00156/00221/00222/00230/index.html?lang=de (Präparateliste und zugelassene Packungen)
 * https://raw.githubusercontent.com/zdavatz/oddb2xml_files/master/interactions_de_utf8.csv
-* http://refdatabase.refdata.ch/Service/Article.asmx
+* https://files.refdata.ch/simis-public-prod/Articles/1.0/Refdata.Articles.zip
 * http://bag.e-mediat.net/SL2007.Web.External/File.axd?file=XMLPublications.zip
 * https://www.medregbm.admin.ch/Publikation/CreateExcelListBetriebs
 * https://www.medregbm.admin.ch/Publikation/CreateExcelListMedizinalPersons
 * http://zurrose.com/fileadmin/main/lib/download.php?file=/fileadmin/user_upload/downloads/ProduktUpdate/IGM11_mit_MwSt/Vollstamm/transfer.dat
-* https://index.ws.e-mediat.net/Swissindex/NonPharma/ws_NonPharma_V101.asmx
-* https://index.ws.e-mediat.net/Swissindex/NonPharma/ws_Pharma_V101.asmx
+* https://raw.githubusercontent.com/zdavatz/oddb2xml_files/master/NON-Pharma.xls
 * http://download.swissmedicinfo.ch/ (AipsDownload)
 * https://raw.githubusercontent.com/zdavatz/oddb2xml_files/master/LPPV.txt
-* https://raw.githubusercontent.com/epha/robot/master/data/manual/swissmedic/atc.csv
 * https://raw.githubusercontent.com/zdavatz/cpp2sqlite/master/input/atc_codes_multi_lingual.txt
 
 ## Rules for matching GTIN (aka EAN13), product number and IKSNR

data/lib/oddb2xml/cli.rb

@@ -285,10 +285,15 @@ module Oddb2xml
         # instead of Thread.new do
 
         downloader = FhirDownloader.new(@options)
-        fhir_file = downloader.download
-        Oddb2xml.log("FhirDownloader downloaded #{File.size(fhir_file)} bytes")
+        fhir_files = downloader.download
+        total_bytes = if fhir_files.is_a?(Hash)
+          fhir_files.values.sum { |f| File.size(f) }
+        else
+          File.size(fhir_files)
+        end
+        Oddb2xml.log("FhirDownloader downloaded #{total_bytes} bytes")
         @mutex.synchronize do
-          hsh = FhirExtractor.new(fhir_file).to_hash
+          hsh = FhirExtractor.new(fhir_files).to_hash
           @items = hsh
           Oddb2xml.log("FhirExtractor added #{@items.size} items from FHIR")
           @items

data/lib/oddb2xml/downloader.rb

@@ -135,6 +135,7 @@ module Oddb2xml
           end
         end
       end
+      xml.force_encoding("UTF-8") if xml.encoding.name != "UTF-8"
       xml
     end
   end
@@ -248,13 +249,7 @@ module Oddb2xml
     end
 
     def init
-      config = {
-        log_level: :info,
-        log: false, # $stdout
-        raise_errors: true,
-        wsdl: @url
-      }
-      @client = Savon::Client.new(config)
+      # No SOAP client needed - we download a zip file directly
     end
 
     def download

data/lib/oddb2xml/extractor.rb

@@ -202,7 +202,8 @@ module Oddb2xml
 
   class RefdataExtractor < Extractor
     def initialize(xml, type)
-      @type = (type == :pharma ? "PHARMA" : "NONPHARMA")
+      @type = (type.to_s.upcase == "PHARMA" ? "PHARMA" : "NONPHARMA")
+      xml = xml.dup.force_encoding("UTF-8") if xml.encoding.name != "UTF-8"
       super(xml)
     end
 
@@ -242,6 +243,7 @@ module Oddb2xml
             item[:desc_it] = name.FullName
           end
         end
+        item[:desc_it] = item[:desc_de] if item[:desc_it].empty? # refdata has no italian name
         item[:atc_code] = article.MedicinalProduct.ProductClassification.Atc || ""
         item[:company_name] = article.PackagedProduct.Holder.Name || ""
         item[:company_ean] = article.PackagedProduct.Holder.Identifier || ""

data/lib/oddb2xml/fhir_support.rb

@@ -15,62 +15,69 @@ module Oddb2xml
 
     BASE_URL = "https://epl.bag.admin.ch"
     STATIC_FHIR_PATH = "/static/fhir"
+    LANGUAGES = %w[de fr it].freeze
 
     def initialize(options = {})
       @options = options
-      @url = find_latest_fhir_url
-      super(options, @url)
+      super(options, BASE_URL)
     end
 
+    # Returns either a single file path String (when --fhir_url is used) or a
+    # Hash of { "de" => path, "fr" => path, "it" => path } for per-language
+    # NDJSON files.
     def download
-      filename = File.basename(@url)
+      if @options[:fhir_url]
+        @url = @options[:fhir_url]
+        download_one(@url)
+      else
+        files = {}
+        LANGUAGES.each do |lang|
+          url = "#{BASE_URL}#{STATIC_FHIR_PATH}/foph-sl-export-latest-#{lang}.ndjson"
+          path = download_one(url)
+          files[lang] = path if path
+        end
+        raise "FhirDownloader: no FHIR files downloaded successfully" if files.empty?
+        files
+      end
+    end
+
+    private
+
+    def download_one(url)
+      @url = url
+      filename = File.basename(url)
       file = File.join(WORK_DIR, filename)
       @file2save = File.join(DOWNLOADS, filename)
 
-      report_download(@url, @file2save)
+      report_download(url, @file2save)
 
-      # Check if we should skip download (file exists and is recent)
       if skip_download?
         Oddb2xml.log "FhirDownloader: Skip downloading #{@file2save} (#{format_size(File.size(@file2save))}, less than 24h old)"
         return File.expand_path(@file2save)
       end
 
       begin
-        # Download the file
         download_as(file, "w+")
 
-        # Validate NDJSON format
         if validate_ndjson(@file2save)
           line_count = count_ndjson_lines(@file2save)
-          Oddb2xml.log "FhirDownloader: NDJSON validation successful (#{line_count} bundles, #{format_size(File.size(@file2save))})"
+          Oddb2xml.log "FhirDownloader: NDJSON validation successful for #{filename} (#{line_count} bundles, #{format_size(File.size(@file2save))})"
         else
-          Oddb2xml.log "FhirDownloader: WARNING - NDJSON validation failed!"
+          Oddb2xml.log "FhirDownloader: WARNING - NDJSON validation failed for #{filename}!"
         end
 
         File.expand_path(@file2save)
       rescue Timeout::Error, Errno::ETIMEDOUT
         retrievable? ? retry : raise
       rescue => error
-        Oddb2xml.log "FhirDownloader: Error downloading FHIR file: #{error.message}"
-        raise
+        Oddb2xml.log "FhirDownloader: Error downloading #{filename}: #{error.message}"
+        nil
       ensure
         Oddb2xml.download_finished(@file2save, false)
         FileUtils.rm_f(file, verbose: true) if File.exist?(file) && file != @file2save
       end
     end
 
-    private
-
-    def find_latest_fhir_url
-      agent = Mechanize.new
-      response = agent.get "https://epl.bag.admin.ch/api/sl/public/resources/current"
-      resources = JSON.parse(response.body)
-      "https://epl.bag.admin.ch/static/" + resources["fhir"]["fileUrl"]
-    rescue => e
-      Oddb2xml.log "FhirDownloader: Error finding latest URL: #{e.message}"
-      nil
-    end
-
     def skip_download?
         @options[:skip_download] || (File.exist?(@file2save) && file_age_hours(@file2save) < 24)
     end
@@ -552,7 +559,7 @@ module Oddb2xml
           "B"
         when "756005022005"
           "C"
-        when "756005022007"
+        when "756005022007", "756005022008"
           "D"
         when "756005022009"
           "E"
@@ -565,8 +572,16 @@ module Oddb2xml
 
   # FHIR Extractor - Compatible with existing BagXmlExtractor
   class FhirExtractor < Extractor
-    def initialize(fhir_file)
-      @fhir_file = fhir_file
+    # Accepts either a single NDJSON file path (back-compat) or a Hash
+    # { "de" => path, "fr" => path, "it" => path } of per-language files.
+    def initialize(fhir_files)
+      if fhir_files.is_a?(Hash)
+        @fhir_files = fhir_files
+        @fhir_file = fhir_files["de"] || fhir_files.values.first
+      else
+        @fhir_files = {"de" => fhir_files}
+        @fhir_file = fhir_files
+      end
     end
 
     def to_hash
@@ -712,9 +727,54 @@ module Oddb2xml
         end
       end
 
+      # Merge names/descriptions from additional language files
+      @fhir_files.each do |lang, file|
+        next if file == @fhir_file
+        next unless file && File.exist?(file)
+        merge_language(data, file, lang)
+      end
+
       Oddb2xml.log "FhirExtractor: Extracted #{data.size} packages"
       data
     end
+
+    private
+
+    def merge_language(data, file, lang)
+      Oddb2xml.log "FhirExtractor: Merging #{lang} names/descriptions from #{file}"
+      result = FhirPreparationsEntry.parse(file)
+      name_accessor = "Name#{lang.capitalize}"
+      name_key = "name_#{lang}".to_sym
+      desc_key = "desc_#{lang}".to_sym
+
+      result.Preparations.Preparation.each do |seq|
+        next unless seq && seq.Packs && seq.Packs.Pack
+
+        translated_name = seq.respond_to?(name_accessor) ? seq.send(name_accessor) : nil
+
+        seq.Packs.Pack.each do |pac|
+          next unless pac.GTIN
+          ean13 = pac.GTIN.to_s
+          item = data[ean13]
+          next unless item
+
+          if translated_name && !translated_name.empty?
+            item[name_key] = translated_name
+            if item[:packages][ean13]
+              item[:packages][ean13][name_key] = translated_name
+            end
+          end
+
+          # The FHIR parser assigns pkg.description to all three
+          # Description* fields; in a language-specific file this is the
+          # description in that language.
+          desc = pac.DescriptionDe
+          if desc && !desc.empty? && item[:packages][ean13]
+            item[:packages][ean13][desc_key] = desc
+          end
+        end
+      end
+    end
   end
 end
 

data/lib/oddb2xml/options.rb

@@ -22,8 +22,8 @@ module Oddb2xml
         opt :extended, "pharma, non-pharma plus prices and non-pharma from zurrose.
                             Products without EAN-Code will also be listed.
                             File oddb_calc.xml will also be generated"
-        opt :fhir, "Use FHIR NDJSON format from FOPH/BAG instead of XML from Spezialitätenliste", default: false
-        opt :fhir_url, "Specific FHIR NDJSON URL to download (implies --fhir)", type: :string, default: nil
+        opt :fhir, "Use FHIR NDJSON format from FOPH/BAG instead of XML from Spezialitätenliste", default: false, short: :none
+        opt :fhir_url, "Specific FHIR NDJSON URL to download (implies --fhir)", type: :string, default: nil, short: :none
         opt :format, "File format F, default is xml. {xml|dat}
                             If F is given, -o option is ignored.", type: :string, default: "xml"
         opt :include, "Include target option for ean14  for 'dat' format.

data/lib/oddb2xml/version.rb

@@ -1,3 +1,3 @@
 module Oddb2xml
-  VERSION = "3.0.2"
+  VERSION = "3.0.3"
 end

data/spec/builder_spec.rb

@@ -10,7 +10,7 @@ ARTICLE_ATTRIBUTE_TESTS = [
   ["ARTICLE", "PROD_DATE", Oddb2xml::DATE_REGEXP],
   ["ARTICLE", "VALID_DATE", Oddb2xml::DATE_REGEXP],
   ["ARTICLE/ART", "SHA256", /[a-f0-9]{32}/],
-  ["ARTICLE/ART", "DT", /\d{4}-\d{2}-\d{2}/]
+  ["ARTICLE/ART", "DT", //]
 ]
 
 ARTICLE_MISSING_ELEMENTS = [