medreg 0.1

data/README.textile

@@ -0,0 +1,22 @@
+h1. medreg
+
+* https://github.com/zdavatz/medreg
+
+h2. DESCRIPTION
+
+Create yaml file for all
+* companies in the health care fromh https://www.medregbm.admin.ch/
+* medical persons from http://www.medregom.admin.ch/
+
+h2. INSTALL:
+
+* gem install medreg
+
+h2. DEVELOPERS:
+
+* Zeno R.R. Davatz
+* Niklaus Giger
+
+h2. LICENSE:
+
+* GPLv3

data/Rakefile

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'medreg/version'
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+# dependencies are declared in medreg.gemspec
+
+desc 'Offer a gem task like hoe'
+task :gem => :build do
+  Rake::Task[:build].invoke
+end
+
+desc "Run tests"
+task :default => :test
+
+desc 'Run medreg tests'
+task :test do
+  log_file = 'suite.log'
+  res = system("bash -c 'set -o pipefail && bundle exec ruby test/suite.rb 2>&1 | tee #{log_file}'")
+  puts "Running test/suite.rb returned #{res.inspect}. Output was redirected to #{log_file}"
+  exit 1 unless res
+end
+
+require 'rake/clean'
+CLEAN.include FileList['pkg/*.gem']
+
+# vim: syntax=ruby

data/bin/medreg

@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby
+
+require 'pathname'
+root = Pathname.new(__FILE__).realpath.parent.parent
+$:.unshift root.join('lib') if $0 == __FILE__
+
+require 'optparse'
+require "date"
+require 'medreg'
+require 'medreg/version'
+
+def help
+  <<EOS
+#$0 ver.#{Medreg::VERSION}
+Usage:
+  #{File.basename(__FILE__)} [persons | companies]
+    Create yaml and csv files under data for all companies/persons known to the
+    swiss health authorities. If no parameter given, fetches companies and persons.
+EOS
+end
+
+parser = OptionParser.new
+opts = {}
+parser.on('--swiss_only') {|v| opts[:swiss_only] = true }
+parser.on_tail('-h', '--help') { puts help; exit }
+
+args = ARGV.dup
+begin
+  parser.parse!(args)
+rescue OptionParser::MissingArgument,
+       OptionParser::InvalidArgument,
+       OptionParser::InvalidOption
+  puts help
+  exit 1
+end
+
+begin
+  Medreg::run(args[0])
+rescue Interrupt
+  puts "Unterbrochen. Breche mit Fehler ab"
+  exit 1
+end
+
+puts "#{__FILE__} completed successfully" if $VERBOSE

data/lib/medreg.rb

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+
+require 'medreg/medreg'

data/lib/medreg/address.rb

@@ -0,0 +1,67 @@
+  #!/usr/bin/env ruby
+# encoding: utf-8
+
+module Medreg
+  class Address2 # copied from oddb
+    @@city_pattern = /[^0-9]+[^0-9\-](?!-)([0-9]+)?/u
+    attr_accessor :name, :additional_lines, :address, :location, :title, :fon, :fax, :canton, :type
+    alias :address_type :type
+    def initialize
+      super
+      @additional_lines = []
+      @fon = []
+      @fax = []
+    end
+    def city
+      @location
+      if(match = @@city_pattern.match(@location.to_s))
+         match.to_s.strip
+      end
+    end
+    def lines
+      lines = lines_without_title
+      if(!@title.to_s.empty?)
+        lines.unshift(@title)
+      end
+      lines
+    end
+    def lines_without_title
+      ([
+        @name,
+      ] + @additional_lines +
+      [
+        @address,
+        location_canton,
+      ]).delete_if { |line| line.to_s.empty? }
+    end
+    def location_canton
+      if(@canton && @location)
+        @location + " (#{@canton})"
+      else
+        @location
+      end
+    end
+    def number
+      if(match = /[0-9][^\s,]*/u.match(@address.to_s))
+        match.to_s.strip
+      elsif @additional_lines[-1]
+        @additional_lines[-1].split(/\s/)[-1]
+      end
+    end
+    def plz
+      if(match = /[1-9][0-9]{3}/u.match(@location.to_s))
+         match.to_s
+      end
+    end
+    def street
+      if(match = /[^0-9,]+/u.match(@address.to_s))
+        match.to_s.strip
+      elsif @additional_lines[-1]
+        @additional_lines[0].split(/\s/)[0]
+      end
+    end
+    def <=>(other)
+      self.lines <=> other.lines
+    end
+  end
+end

data/lib/medreg/ba_type.rb

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+# Business area for companies
+require 'set'
+require 'medreg'
+
+module Medreg
+  class BA_type
+    include Enumerable
+    BA_cantonal_authority =  'ba_cantonal_authority'
+    BA_doctor             =  'ba_doctor'
+    BA_health             =  'ba_health'
+    BA_hospital           =  'ba_hospital'
+    BA_hospital_pharmacy  =  'ba_hospital_pharmacy'
+    BA_info               =  'ba_info'
+    BA_insurance          =  'ba_insurance'
+    BA_pharma             =  'ba_pharma'
+    BA_public_pharmacy    =  'ba_public_pharmacy'
+    BA_research_institute =  'ba_research_institute'
+    def BA_type.collect
+      BA_types
+    end
+  end
+
+  BA_types = Set[
+    nil,
+    BA_type::BA_cantonal_authority,
+    BA_type::BA_doctor,
+    BA_type::BA_health,
+    BA_type::BA_hospital,
+    BA_type::BA_hospital_pharmacy,
+    BA_type::BA_info,
+    BA_type::BA_insurance,
+    BA_type::BA_pharma,
+    BA_type::BA_public_pharmacy,
+    BA_type::BA_research_institute,
+  ]
+end

data/lib/medreg/company.rb

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require 'medreg/address'
+require 'medreg/ba_type'
+module Medreg
+  class Company
+    attr_accessor :address_email, :business_area, :business_unit, :cl_status,
+      :competition_email, :complementary_type, :contact, :deductible_display,
+      :disable_patinfo, :ean13, :generic_type, :addresses,
+      :invoice_htmlinfos, :logo_filename, :lookandfeel_member_count, :name,
+      :powerlink, :regulatory_email, :swissmedic_email, :swissmedic_salutation,
+      :url, :ydim_id, :limit_invoice_duration, :force_new_ydim_debitor,
+      :narcotics
+    attr_reader :disabled_invoices
+    alias :fullname :name
+    alias :power_link= :powerlink=
+    alias :power_link :powerlink
+    alias :to_s :name
+    alias :email :address_email
+    def initialize
+      @addresses = [Address2.new]
+    end
+    def is_pharmacy?
+      case @business_area
+        when BA_type::BA_public_pharmacy, BA_type:: BA_hospital_pharmacy
+          return true
+        else
+          false
+        end
+    end
+  end
+end

data/lib/medreg/company_importer.rb

@@ -0,0 +1,262 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require 'medreg/address'
+require 'medreg/ba_type'
+require 'medreg/company'
+require 'medreg/resilient_loop'
+require 'rubyXL'
+require 'mechanize'
+require 'logger'
+require 'cgi'
+require 'psych' if RUBY_VERSION.match(/^1\.9/)
+require "yaml"
+
+module Medreg
+  DebugImport         = defined?(MiniTest)
+  BetriebeURL         = 'https://www.medregbm.admin.ch/Betrieb/Search'
+  BetriebeXLS_URL     = "https://www.medregbm.admin.ch/Publikation/CreateExcelListBetriebs"
+  RegExpBetriebDetail = /\/Betrieb\/Details\//
+  Companies_curr      = File.expand_path(File.join(__FILE__, "../../../data/companies_#{Time.now.strftime('%Y.%m.%d')}.xlsx"))
+  Companies_YAML      = File.expand_path(File.join(__FILE__, "../../../data/companies_#{Time.now.strftime('%Y.%m.%d')}.yaml"))
+  # MedRegURL     = 'http://www.medreg.admin.ch/'
+  CompanyInfo = Struct.new("CompanyInfo",
+                          :gln,
+                          :exam,
+                          :address,
+                          :name_1,
+                          :name_2,
+                          :addresses,
+                          :plz,
+                          :canton_giving_permit,
+                          :country,
+                          :company_type,
+                          :drug_permit,
+                          )
+#    GLN Person  Name  Vorname PLZ Ort Bewilligungskanton  Land  Diplom  BTM Berechtigung  Bewilligung Selbstdispensation  Bemerkung Selbstdispensation
+
+  COMPANY_COL = {
+    :gln                  => 0, # A
+    :name_1               => 1, # B
+    :name_2               => 2, # C
+    :street               => 3, # D
+    :street_number        => 4, # E
+    :plz                  => 5, # F
+    :locality             => 6, # G
+    :canton_giving_permit => 7, # H
+    :country              => 8, # I
+    :company_type         => 9, # J
+    :drug_permit          => 10, # K
+  }
+  class CompanyImporter
+    RECIPIENTS = []
+
+    def save_for_log(msg)
+      Medreg.log(msg)
+      withTimeStamp = "#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}: #{msg}"
+      @@logInfo << withTimeStamp
+    end
+    def initialize(glns_to_import = [])
+      @glns_to_import = glns_to_import.clone
+      @glns_to_import.delete_if {|item| item.size == 0}
+      @info_to_gln    = {}
+      @@logInfo       = []
+      FileUtils.rm_f(Companies_YAML) if File.exists?(Companies_YAML)
+      @yaml_file      = File.open(Companies_YAML, 'w+')
+      @companies_created = 0
+      @companies_updated = 0
+      @companies_skipped = 0
+      @companies_deleted = 0
+      @archive = ARCHIVE_PATH
+      @@all_companies    = []
+      setup_default_agent
+    end
+    def update
+      saved = @glns_to_import.clone
+      latest = get_latest_file
+      save_for_log "parse_xls #{latest} specified GLN ids #{saved.inspect}"
+      parse_xls(latest)
+      @info_to_gln.keys
+      get_detail_to_glns(saved.size > 0 ? saved : @glns_to_import)
+      return @companies_created, @companies_updated, @companies_deleted, @companies_skipped
+    ensure
+      File.open(Companies_YAML, 'w+') {|f| f.write(@@all_companies.to_yaml) }
+      save_for_log "Saved #{@@all_companies.size} companies in #{Companies_YAML}"
+    end
+    def setup_default_agent
+      @agent = Mechanize.new
+      @agent.user_agent = 'Mozilla/5.0 (X11; Linux x86_64; rv:31.0) Gecko/20100101 Firefox/31.0 Iceweasel/31.1.0'
+      @agent.redirect_ok         = :all
+      @agent.follow_meta_refresh_self = true
+      @agent.follow_meta_refresh = :everwhere
+      @agent.redirection_limit   = 55
+      @agent.follow_meta_refresh = true
+      @agent.ignore_bad_chunking = true
+      @agent
+    end
+    def parse_details(html, gln)
+      left = html.at('div[class="colLeft"]').text
+      right = html.at('div[class="colRight"]').text
+      btm = html.at('div[class="twoColSpan"]').text
+      infos = []
+      infos = left.split(/\r\n\s*/)
+      unless infos[2].eql?(gln.to_s)
+        Medreg.log "Mismatch between searched gln #{gln} and details #{infos[2]}"
+        return nil
+      end
+      company = Hash.new
+      company[:ean13] =  gln.to_s.clone
+      company[:name] =  infos[4]
+      idx_plz     = infos.index("PLZ \\ Ort")
+      idx_canton  = infos.index('Bewilligungskanton')
+      address = infos[6..idx_plz-1].join(' ')
+      company[:plz] = infos[idx_plz+1]
+      company[:location] = infos[idx_plz+2]
+      idx_typ = infos.index('Betriebstyp')
+      ba_type = infos[idx_typ+1]
+      company[:address] = address
+      company[:ba_type] = ba_type
+      company[:narcotics] = btm.split(/\r\n\s*/)[-1]
+      update_address(company)
+      Medreg.log company if $VERBOSE
+      company
+    end
+    Search_failure = 'search_took_to_long'
+    def get_detail_to_glns(glns)
+      r_loop = ResilientLoop.new(File.basename(__FILE__, '.rb'))
+      failure = 'Die Personensuche dauerte zu lange'
+      idx = 0
+      max_retries = 3
+      Medreg.log "get_detail_to_glns for #{glns.size} glns. first 10 are #{glns[0..9]} state_id is #{r_loop.state_id.inspect}" if DebugImport
+      glns.each { |gln|
+        idx += 1
+        if r_loop.must_skip?(gln)
+          Medreg.log "Skipping #{gln}. Waiting for #{r_loop.state_id.inspect}" if DebugImport
+          next
+        end
+        nr_tries = 0
+        success = false
+        while nr_tries < max_retries  and not success
+          begin
+            r_loop.try_run(gln, defined?(Minitest) ? 500 : 5 ) do
+              Medreg.log "Searching for company with GLN #{gln}. Skipped #{@companies_skipped}, created #{@companies_created} updated #{@companies_updated} of #{glns.size}).#{nr_tries > 0 ? ' nr_tries is ' + nr_tries.to_s : ''}"
+              page_1 = @agent.get(BetriebeURL)
+              raise Search_failure if page_1.content.match(failure)
+              hash = [
+            ['Betriebsname', ''],
+            ['Plz', ''],
+            ['Ort', ''],
+            ['GlnBetrieb', gln.to_s],
+            ['BetriebsCodeId', '0'],
+            ['KantonsCodeId', '0'],
+              ]
+              res_2 = @agent.post(BetriebeURL, hash)
+              if res_2.link(:href => RegExpBetriebDetail)
+                page_3 = res_2.link(:href => RegExpBetriebDetail).click
+                raise Search_failure if page_3.content.match(failure)
+                company = parse_details(page_3, gln)
+                store_company(company)
+                @@all_companies << company
+              else
+                Medreg.log "could not find gln #{gln}"
+                @companies_skipped += 1
+              end
+              success = true
+            end
+          rescue Timeout => e
+            nr_tries += max_retries  if defined?(MiniTest)
+            Medreg.log "rescue #{e} will retry #{max_retries - nr_tries} times"
+            nr_tries += 1
+            sleep defined?(MiniTest) ? 0.01 : 60
+          end
+          if (@companies_created + @companies_updated) % 100 == 99
+            Medreg.log "Start saving #{gln} after #{@companies_created} created #{@companies_updated} updated"
+          end
+        end
+      }
+      r_loop.finished
+    ensure
+      Medreg.log "Start saving"
+      Medreg.log "Finished"
+    end
+    def get_latest_file
+      agent = Mechanize.new
+      target = Companies_curr
+      needs_update = true
+      return target if File.exist?(target)
+      file = agent.get(BetriebeXLS_URL)
+      download = file.body
+      File.open(target, 'w+') { |f| f.write download }
+      save_for_log "saved #{file.body.size} bytes as #{target}"
+      target
+    end
+    def report
+      report = "Companies update \n\n"
+      report << "New companies: "       << @companies_created.to_s << "\n"
+      report << "Updated companies: "   << @companies_updated.to_s << "\n"
+      report << "Deleted companies: "   << @companies_deleted.to_s << "\n"
+      report
+    end
+    def update_address(data)
+      addr = Address2.new
+      addr.name    =  data[:name  ]
+      addr.address =  data[:address]
+      # addr.additional_lines = [data[:address] ]
+      addr.location = [data[:plz], data[:location]].compact.join(' ')
+      if(fon = data[:phone])
+        addr.fon = [fon]
+      end
+      if(fax = data[:fax])
+        addr.fax = [fax]
+      end
+      data[:addresses] = [addr]
+    end
+    def store_company(data)
+      @companies_created += 1
+      company = Company.new
+      action = 'create'
+      ba_type = nil
+      case  data[:ba_type]
+        when /kantonale Beh/i
+          ba_type = Medreg::BA_type::BA_cantonal_authority
+        when /ffentliche Apotheke/i
+          ba_type = Medreg::BA_type::BA_public_pharmacy
+        when /Spitalapotheke/i
+          ba_type = Medreg::BA_type::BA_hospital_pharmacy
+        when /wissenschaftliches Institut/i
+          ba_type = Medreg::BA_type::BA_research_institute
+        else
+          ba_type = 'unknown'
+      end
+      company.ean13         = data[:ean13]
+      company.name          = data[:name]
+      company.business_area = ba_type
+      company.narcotics     = data[:narcotics]
+      company.addresses     = data[:addresses]
+      Medreg.log "store_company updated #{data[:ean13]} database. ba_type #{ba_type}." if $VERBOSE
+    end
+    def parse_xls(path)
+      Medreg.log "parsing #{path}"
+      workbook = RubyXL::Parser.parse(path)
+      positions = []
+      rows = 0
+      workbook[0].each do |row|
+        next unless row and (row[COMPANY_COL[:gln]] or row[COMPANY_COL[:name_1]])
+        rows += 1
+        if rows > 1
+          info = CompanyInfo.new
+          [:gln, :name_1, :name_2, :plz, :canton_giving_permit, :country, :company_type,:drug_permit].each {
+            |field|
+            cmd = "info.#{field} = row[COMPANY_COL[#{field.inspect}]] ? row[COMPANY_COL[#{field.inspect}]].value : nil"
+            eval(cmd)
+          }
+          @info_to_gln[ row[COMPANY_COL[:gln]] ? row[COMPANY_COL[:gln]].value : row[COMPANY_COL[:name_1]].value ] = info
+        end
+      end
+      @glns_to_import = @info_to_gln.keys.sort.uniq
+    end
+    def Company.all_companies
+      @@all_companies
+    end
+  end
+end