brand2csv 0.1.1

data/spike_mechanize_swissreg.rb

@@ -0,0 +1,312 @@
+#!/usr/bin/env ruby
+require 'mechanize'
+require 'prettyprint'
+require 'optparse'
+
+Useage = "Usage: #{File.basename(__FILE__)} timespan
+    Find all brands registered in switzerland during the given timespan.
+    The following examples valid timespan periods:
+      1.10.2005
+      1.10.2005-31.10.2005
+      1.10.2005, 5.10.2005-31.10.2005
+"
+OptionParser.new do |opts|
+  opts.banner = Useage
+  opts.on("-h", "--help", "Show this help") do |v|
+    puts opts
+    exit
+  end
+end.parse!
+unless ARGV
+  puts Useage
+  exit 1
+end
+
+timespan = ARGV[0] 
+
+module Brand2csv do
+      # Weitere gesehene Fehler
+    bekannteFehler = 
+        ['Das Datum ist ung', # ültig'
+          'Es wurden keine Daten gefunden.',
+          'Die Suchkriterien sind teilweise unzul', # ässig',
+          'Geben Sie mindestens ein Suchkriterium ein',
+          'Die Suche wurde abgebrochen, da die maximale Suchzeit von 60 Sekunden',
+        ]
+end
+$base_uri = 'https://www.swissreg.ch'
+$start_uri = "#{$base_uri}/srclient/faces/jsp/start.jsp"
+  
+def writeResponse(filename, body)
+  ausgabe = File.open(filename, 'w+')
+  ausgabe.puts body
+  ausgabe.close
+end
+
+def view_state(response)
+  if match = /javax.faces.ViewState.*?value="([^"]+)"/u.match(response.force_encoding('utf-8'))
+    match[1]
+  else
+    ""
+  end
+end
+
+def parse_swissreg(timespan = "01.06.2007-10.06.2007",  # sollte 377 Treffer ergeben, für 01.06.2007-10.06.2007, 559271 wurde in diesem Zeitraum registriert
+                   marke = nil,    
+                   nummer =nil) #  nummer = "559271" ergibt genau einen treffer
+
+  a = Mechanize.new { |agent|
+  #  agent.user_agent_alias = 'Mac Safari'
+    agent.user_agent = 'Mozilla/5.0 (X11; Linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0'
+  #  agent.redirection_limit   = 5
+    agent.verify_mode = OpenSSL::SSL::VERIFY_NONE
+  }
+
+page = a.get $start_uri  # get a cookie for the session
+content = a.get_file $start_uri
+FileUtils.makedirs 'mechanize'
+writeResponse('mechanize/main.html', content)
+state = view_state(content)
+data = [
+  ["autoScroll", "0,0"],
+  ["id_swissreg:_link_hidden_", ""],
+  ["id_swissreg_SUBMIT", "1"],
+  ["id_swissreg:_idcl", "id_swissreg_sub_nav_ipiNavigation_item0"],
+  ["javax.faces.ViewState", state],
+]
+
+content = a.post($start_uri, data)  
+writeResponse('mechanize/einfache_suche.html', content.body)
+
+data = [
+  ["autoScroll", "0,0"],
+  ["id_swissreg:_link_hidden_", ""],
+  ["id_swissreg_SUBMIT", "1"],
+  ["id_swissreg:_idcl", "id_swissreg_sub_nav_ipiNavigation_item0_item3"],
+  ["javax.faces.ViewState", state],
+]
+# sr1 ist die einfache suche, sr3 die erweiterte Suche
+path = "/srclient/faces/jsp/trademark/sr3.jsp"
+response = a.post($base_uri + path, data)
+writeResponse('mechanize/erweiterte_suche.html', response.body)
+# Bis hier alles okay
+    criteria = [
+      ["autoScroll", "0,0"],
+      ["id_swissreg:_link_hidden_", ""],
+      # "id_swissreg:mainContent:id_cbxFormatChoice" 2 = Publikationsansicht 1 = Registeransicht
+      ["id_swissreg:mainContent:id_cbxFormatChoice", "1"],
+      ["id_swissreg:mainContent:id_ckbTMState", "1"], # "Hängige Gesuche 1
+#      ["id_swissreg:mainContent:id_ckbTMState", "2"], # "Gelöschte Gesuche 2
+      ["id_swissreg:mainContent:id_ckbTMState", "3"], # aktive Marken 3 
+#      ["id_swissreg:mainContent:id_ckbTMState", "4"], # gelöschte Marken 4
+      ["id_swissreg:mainContent:id_cbxCountry", "CH"], # Auswahl Länder _ALL
+#      ["id_swissreg:mainContent:id_txf_tm_no", ""],  # Marken Nr
+      ["id_swissreg:mainContent:id_txf_app_no", ""],                       # Gesuch Nr.
+      ["id_swissreg:mainContent:id_txf_applicant", ""],                    # Inhaber/in
+      ["id_swissreg:mainContent:id_txf_agent", ""],                         # Vertreter/in
+      ["id_swissreg:mainContent:id_txf_licensee", ""], # Lizenznehmer
+      ["id_swissreg:mainContent:id_txf_nizza_class", ""], # Nizza Klassifikation Nr.
+#      ["id_swissreg:mainContent:id_txf_appDate", timespan], # Hinterlegungsdatum
+      ["id_swissreg:mainContent:id_txf_expiryDate", ""], # Ablauf Schutzfrist
+      # Markenart: Individualmarke 1 Kollektivmarke 2 Garantiemarke 3
+      ["id_swissreg:mainContent:id_cbxTMTypeGrp", "_ALL"],  # Markenart
+      ["id_swissreg:mainContent:id_cbxTMForm", "_ALL"],  # Markentyp
+      ["id_swissreg:mainContent:id_cbxTMColorClaim", "_ALL"],  # Farbanspruch
+      ["id_swissreg:mainContent:id_txf_pub_date", ""], # Publikationsdatum
+#      name="id_swissreg:mainContent:id_ckbTMChoice", "tm_lbl_applicant"], # />&#160;Inhaber/in</label></td>
+      
+    # info zu Publikationsgrund id_swissreg:mainContent:id_ckbTMPubReason
+      ["id_swissreg:mainContent:id_ckbTMPubReason", "1"], #Neueintragungen
+      ["id_swissreg:mainContent:id_ckbTMPubReason", "2"], #Berichtigungen
+      ["id_swissreg:mainContent:id_ckbTMPubReason", "3"], #Verlängerungen
+      ["id_swissreg:mainContent:id_ckbTMPubReason", "4"], #Löschungen
+      ["id_swissreg:mainContent:id_ckbTMPubReason", "5"], #Inhaberänderungen
+      ["id_swissreg:mainContent:id_ckbTMPubReason", "6"], #Vertreteränderungen
+      ["id_swissreg:mainContent:id_ckbTMPubReason", "7"], #Lizenzänderungen
+      ["id_swissreg:mainContent:id_ckbTMPubReason", "8"], #Weitere Registeränderungen
+      ["id_swissreg:mainContent:id_ckbTMEmptyHits", "0"],  # Leere Trefferliste anzeigen
+      
+      # Angezeigte Spalten "id_swissreg:mainContent:id_ckbTMChoice"
+      ["id_swissreg:mainContent:id_ckbTMChoice", "tm_lbl_tm_text"], # Marke
+      # ["id_swissreg:mainContent:id_ckbTMChoice", "tm_lbl_state"], # Status
+      # ["id_swissreg:mainContent:id_ckbTMChoice", "tm_lbl_nizza_class"], # Nizza Klassifikation Nr.
+      # ["id_swissreg:mainContent:id_ckbTMChoice", "tm_lbl_no"], # disabled="disabled"], # Nummer
+      ["id_swissreg:mainContent:id_ckbTMChoice", "tm_lbl_applicant"], # Inhaber/in
+      ["id_swissreg:mainContent:id_ckbTMChoice", "tm_lbl_country"], # Land (Inhaber/in)
+      # ["id_swissreg:mainContent:id_ckbTMChoice", "tm_lbl_agent"], # Vertreter/in
+      # ["id_swissreg:mainContent:id_ckbTMChoice", "tm_lbl_licensee"], # Lizenznehmer/in
+      ["id_swissreg:mainContent:id_ckbTMChoice", "tm_lbl_app_date"], # Hinterlegungsdatum
+      # ["id_swissreg:mainContent:id_ckbTMChoice", "tm_lbl_expiry_date"], # Ablauf Schutzfrist
+      # ["id_swissreg:mainContent:id_ckbTMChoice", "tm_lbl_type_grp"], # Markenart
+      # ["id_swissreg:mainContent:id_ckbTMChoice", "tm_lbl_form"], # Markentyp
+      # ["id_swissreg:mainContent:id_ckbTMChoice", "tm_lbl_color_claim"], # Farbanspruch
+      
+      ["id_swissreg:mainContent:id_cbxHitsPerPage", "100"],   # Treffer pro Seite
+      ["id_swissreg:mainContent:id_ckbTMChoice", "tm_lbl_applicant"],
+      ["id_swissreg:mainContent:sub_fieldset:id_submit", "suchen"],
+#      ["id_swissreg:mainContent:sub_fieldset:id_reset", "0"],
+      ["id_swissreg_SUBMIT", "1"],
+      ["javax.faces.ViewState", state],
+    ]
+    if marke               # Wortlaut der Marke
+      puts "Marke ist #{marke}"
+     criteria << ["id_swissreg:mainContent:id_txf_tm_text", marke]
+    else
+      puts "Keine Marke spezifiziert. #{marke.inspect}"
+     criteria << ["id_swissreg:mainContent:id_txf_tm_text", ""]
+    end
+    if timespan               # Hinterlegungsdatum
+      puts "Hinterlegungsdatum ist #{timespan}"
+      criteria << ["id_swissreg:mainContent:id_txf_appDate", timespan] # Hinterlegungsdatum
+    else
+      puts "Keine Hinterlegungsdatum spezifiziert. #{timespan.inspect}"
+      criteria << ["id_swissreg:mainContent:id_txf_appDate", ""] # Hinterlegungsdatum
+    end
+    if nummer               
+      puts "nummer ist #{timespan}"
+      criteria << ["id_swissreg:mainContent:id_txf_tm_no", nummer] 
+    else
+      puts "Keine nummer spezifiziert. #{timespan.inspect}"
+      criteria << ["id_swissreg:mainContent:id_txf_tm_no", ""] 
+    end
+    
+path = "/srclient/faces/jsp/trademark/sr3.jsp"
+response = a.post($base_uri + path, criteria)
+writeResponse('mechanize/resultate_1.html', response.body)
+criteria<<['id_swissreg:mainContent:scroll_1idx2', 'idx2']
+if false # does not work, returns to the extended search path
+response = a.post($base_uri + path, criteria)
+writeResponse('mechanize/resultate_2.html', response.body)
+end
+
+if false # Will try later
+path = "/srclient/faces/jsp/trademark/sr3.jsp"
+data_detail = [
+  ["autoScroll", "0,0"],
+  ["id_swissreg:_link_hidden_", ""],
+  ["id_swissreg_SUBMIT", "1"],
+  ["id_swissreg:_idcl", "id_swissreg_sub_nav_ipiNavigation_item0_item3"],
+  ["javax.faces.ViewState", state],
+]
+
+response = a.post($base_uri + path, data_detail)
+https://www.swissreg.ch/srclient/faces/jsp/trademark/sr30.jsp
+end
+end # 
+# parse_swissreg("01.06.2007-10.06.2007" , 'asp*')
+require 'csv'
+$results = []
+$errors  = Hash.new
+
+class Marke < Struct.new(:name, :markennummer, :inhaber, :land, :hinterlegungsdatum, :zeile_1, :zeile_2, :zeile_3, :zeile_4, :plz, :ort)
+end
+
+AddressRegexp = /^(\d\d\d\d)\W*(.*)/
+LineSplit     = ', '
+DefaultCountry = 'Schweiz'
+
+def parseAddress(nummer, inhaber)
+  zeile_1, zeile_2, zeile_3, zeile_4, zeile_5, zeile_6 = inhaber.split(LineSplit)
+  ort = nil
+  plz = nil
+  if    m = AddressRegexp.match(zeile_2)
+    zeile_2 = nil
+    plz = m[1]; ort = m[2]
+  elsif m = AddressRegexp.match(zeile_3)
+    zeile_3 = nil
+    plz = m[1]; ort = m[2]
+  elsif m = AddressRegexp.match(zeile_4)
+    zeile_4 = nil
+    plz = m[1]; ort = m[2]
+  elsif m = AddressRegexp.match(zeile_5)
+    zeile_5 = nil
+    plz = m[1]; ort = m[2]
+  else
+    puts "Achtung! Konnte Marke #{nummer} mit Inhaber #{inhaber} nicht parsen" if $VERBOSE
+    return nil,   nil,     nil,     nil,     nil,     nil,     nil, nil
+  end
+  return zeile_1, zeile_2, zeile_3, zeile_4, zeile_5, zeile_6, plz, ort
+end
+
+def fetchDetails(nummer) # takes a long time!
+  doc = nil
+  filename = "mechanize/detail_#{nummer}.html"
+  unless File.exists?(filename)
+    url = "https://www.swissreg.ch/srclient/faces/jsp/trademark/sr300.jsp?language=de&section=tm&id=#{nummer}"
+    a = Mechanize.new { |agent|
+      #  agent.user_agent_alias = 'Mac Safari'
+        agent.user_agent = 'Mozilla/5.0 (X11; Linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0'
+      #  agent.redirection_limit   = 5
+        agent.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      }
+    page = a.get $start_uri  # get a cookie for the session
+    content = a.get_file url
+    writeResponse("mechanize/detail_#{nummer}.html", content)
+  end
+  doc = Nokogiri::Slop(File.open(filename))
+  path_name = "//html/body/form/div/div/fieldset/div/table/tbody/tr/td"
+  elem = doc.xpath(path_name).first
+  counter = 0
+  doc.xpath(path_name).each{ 
+    |td|
+      pp "#{counter}: #{td.text}" if $VERBOSE
+      counter += 1
+      next unless /^inhaber/i.match(td.text)
+      zeilen = []
+      doc.xpath(path_name)[counter].children.each{ |child| zeilen << child.text.gsub(LineSplit,'. ') unless child.text.length == 0 } # avoid adding <br>
+      if info = $errors[nummer]
+        info.inhaber = zeilen.join(LineSplit)
+        info.zeile_1, info.zeile_2, info.zeile_3, info.zeile_4, zeile_5, zeile_6, info.plz, info.ort = parseAddress(nummer, info.inhaber)
+        $results << info
+      else
+        bezeichnung =  doc.xpath(path_name)[15]
+        inhaber = zeilen.join(LineSplit)
+        zeile_1, zeile_2, zeile_3, zeile_4, zeile_5, zeile_6, plz, ort = parseAddress(nummer, inhaber)
+        hinterlegungsdatum = doc.xpath(path_name)[7]
+        marke = Marke.new(bezeichnung, nummer,  inhaber,  DefaultCountry,  hinterlegungsdatum, zeile_1, zeile_2, zeile_3, zeile_4, plz, ort )
+        $results << marke
+      end
+  }
+end
+
+def fetchresult(filename= 'mechanize/resultate_1.html')
+  nrFailures = 0
+  # doc = Nokogiri::Slop(inhalt) 
+  doc = Nokogiri::Slop(File.open(filename))
+  path_name = "//html/body/form/div/div/fieldset/table/tbody/tr/td/table/tbody/tr"
+  elem = doc.xpath(path_name).first
+  doc.xpath(path_name).each{ 
+    |elem|
+    bezeichnung = elem.elements[1].text
+    land = elem.elements[4].text
+    next unless /#{DefaultCountry}/i.match(land)
+    inhaber =  elem.elements[3].text
+    nummer  = elem.elements[2].text
+    if bezeichnung.length == 0
+      bezeichnung = elem.children[1].children[0].children[0].children[0].attribute('src').to_s
+    end
+    zeile_1, zeile_2, zeile_3, zeile_4, zeile_5, zeile_6, plz, ort = parseAddress(nummer, inhaber)
+    if zeile_1
+      $results << Marke.new(bezeichnung, elem.elements[2].text,  elem.elements[3].text,  land,  elem.elements[5].text,
+                            zeile_1, zeile_2, zeile_3, zeile_4, plz, ort )
+    else
+      nrFailures += 1
+      $errors[nummer] = Marke.new(bezeichnung, elem.elements[2].text,  elem.elements[3].text,  land,  elem.elements[5].text,
+                            zeile_1, zeile_2, zeile_3, zeile_4, plz, ort )
+    end
+  }
+  puts "Es gab #{nrFailures} Fehler beim lesen von #{filename}"  if $VERBOSE
+end
+
+pp 1
+parse_swissreg(timespan)
+pp 2
+fetchresult
+$errors.each{ 
+  |markennummer, info|
+    fetchDetails(markennummer) 
+}
+
+CSV.open('ausgabe.csv', 'w') do |csv|
+  $results.each{ |x| csv << x }
+end

data/spike_watir.rb

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+# require 'selenium'
+require 'watir'
+require 'watir-webdriver'
+require 'fileutils'
+require 'pp'
+
+# zum schnell von Hand testen
+# client = Selenium::WebDriver::Remote::Http::Default.new
+# client.timeout = 180 # seconds – default is 60
+
+$cnt = 1
+def saveStep(b, cnt = $cnt)
+  name = "watir/#{__FILE__}_#{cnt}.html".sub('.rb','')
+  ausgabe = File.open(name, "w+")
+  ausgabe.write(b.html)
+  $cnt += 1 if $cnt.to_i != 0
+
+end
+Swiss_reg_URL = 'https://www.swissreg.ch'
+client = Selenium::WebDriver::Remote::Http::Default.new
+browser  = Watir::Browser.new :firefox   #, :http_client => client
+# browser  = Watir::Browser.new :chrome   #, :http_client => client
+browser.goto Swiss_reg_URL
+#browser = Watir::Browser.start "https://www.swissreg.ch"
+saveStep(browser, 11)
+browser.link(:id, "id_swissreg_sub_nav_ipiNavigation_item0").click
+saveStep(browser, 12 )
+browser.link(:id, "id_swissreg_sub_nav_ipiNavigation_item0_item3").click
+saveStep(browser, 13 )
+browser.text_field(:id, "id_swissreg:mainContent:id_txf_appDate").set("1.10.2011-5.10.2011")
+saveStep(browser, 14 )
+browser.button(:value,"suchen").click
+saveStep(browser, 15)
+aus = browser.text
+ausgabe=File.open('watir/liste.txt','w+')
+ausgabe.puts aus
+ausgabe.close
+browser.link(:id, "id_swissreg:mainContent:data:2:tm_no_detail:id_detail").click# puts browser.text
+browser.images.each do |x|
+  idx += 1
+  # apparently the string accepted by the string method will not allow variable substitution
+  location = 'img_' + idx.to_s + '.png'
+  x.save(location)
+end
+puts browser.url
+pp browser.windows.each{ |w| 
+                         pp w.url
+#  saveStep(w)
+}
+# window2  = Watir::Browser.attach(:url, "https://www.swissreg.ch/srclient/faces/jsp/trademark/sr30.jsp") stackLevel too deep
+#window2  = Selenium::WebDriver::Firefox.attach(:url, "https://www.swissreg.ch/srclient/faces/jsp/trademark/sr30.jsp") 
+#saveStep(window2)
+#aus = window2.text
+ausgabe=File.open('watir/detail.txt','w+')
+ausgabe.puts aus
+ausgabe.close

metadata

@@ -0,0 +1,151 @@
+--- !ruby/object:Gem::Specification
+name: brand2csv
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+  prerelease: 
+platform: ruby
+authors:
+- Niklaus Giger, Zeno R.R. Davatz
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-05-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: hoe
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '3.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '3.4'
+description: ! "brand2csv creates csv files for swiss brand registered in a specific
+  time period. \n  The csv contains the brand, link to image (if present), link to
+  the detailinfo at swissreg.ch, name and address of owner (Inhaber)"
+email: yasaka@ywesee.com, zdavatz@ywesee.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- History.txt
+- LICENCE.txt
+- Manifest.txt
+- README.md
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- Gemfile.lock
+- History.txt
+- LICENCE.txt
+- LICENSE
+- Manifest.txt
+- README.md
+- Rakefile
+- lib/brand2csv.rb
+- lib/brand2csv/version.rb
+- protocol.2013.05.12.textile
+- protocol.2013.05.15.textile
+- protocol.2013.05.21.textile
+- resultat_1.html
+- spike.rb
+- spike_mechanize_swissreg.rb
+- spike_watir.rb
+homepage: https://github.com/zdavatz/brand2csv
+licenses: []
+post_install_message: 
+rdoc_options:
+- --main
+- README.md
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: brand2csv
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: brand2csv creates csv files for swiss brands.
+test_files: []