orsos 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b6fe6169334559b768985db4d2454312220ed1ca
-  data.tar.gz: 63303154856a9585e431d0f14e6cf5673c8b7dbe
+  metadata.gz: 583b7c177d70e612ff2119874e9929f089942e53
+  data.tar.gz: 6b0399d8c7e9076e019323a70cab445ba0905270
 SHA512:
-  metadata.gz: 543e7f9dc9c589fa87838165c3b9c67e6d2dc2563fe9e1e57b8a24ea936b7f627719d8f76cfd27c01ea275e7f753aa04b3e992e26d0c2744177d2b1d6002bc9e
-  data.tar.gz: 9bdd606574305f3ffbf700b2342d57cf8a3dd0d7b29353f5f3d62ceb5b9b956bf0e05350426d52301481e706b9b228c76006d8bd2609b48d74355d7154545ca9
+  metadata.gz: fa7196f3a009cb2d5b2f6ec3d55bfb5b2c2876d4d559367d526bb5c351e244ce849b924bfe293da284e11e4caeb89db4c36c37fc582397fd452255b95205a12e
+  data.tar.gz: 79182511ef087494b19b1ac2a739d977ec38f51985ae62bebb0dca419bf27d282b7e1e95d437ed2738f138eab43222e314616c5f4fcd68bf2dcf660c857ef5c0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    orsos (0.0.5)
+    orsos (0.0.6)
       mechanize
       rest_client (~> 1.7.3)
       thor

data/lib/orsos/commands/get.rb

@@ -3,13 +3,14 @@ require_relative '../webdownloader'
 
 module Orsos::Commands
   class Get < Thor
+    class_option :in2csv, type: :boolean, desc: 'use in2csv to convert downloaded xls to csv'
+    class_option :xls2csv, type: :boolean, desc: 'use xls2csv to convert downloaded xls to csv'
+    class_option :stdout, type: :boolean, desc: 'output to stdout'
+    class_option :verbose, type: :boolean, desc: 'turn on verbose logging of search'
+
     desc "transactions FROM [TO]", "Download campaign finance transactions daily between FROM till TO and saves each day to sos_transactions_{%Y%m%d}-{current time stamp}. eg., orsos get transactions 2014-10-01 2014-10-31. TO defaults to today's date"
-    option :verbose, type: :boolean, desc: 'turn on verbose logging of search'
     option :filer_id, type: :numeric, desc: 'conduct search by filer_id'
     option :single_file, type: :boolean, desc: 'search and save data in date range as a single file rather than one search per day'
-    option :in2csv, type: :boolean, desc: 'use in2csv to convert downloaded xls to csv'
-    option :xls2csv, type: :boolean, desc: 'use xls2csv to convert downloaded xls to csv'
-    option :stdout, type: :boolean, desc: 'output to stdout'
     def transactions(from, to=Date.today)
       from_date = case from
         when Date
@@ -30,48 +31,82 @@ module Orsos::Commands
       end
 
       trans_opts = options.select{|k,v| ['filer_id'].include?(k) }
-      if options['in2csv']
-        csvbin = 'in2csv'
-        fileext = 'csv'
-      elsif options['xls2csv']
-        csvbin = 'xls2csv'
-        fileext = 'csv'
-      else
-        csvbin = nil
-        fileext = 'xls'
-      end
 
       if !options['single_file'].nil?
-        filename = "sos_transactions_#{from_date.strftime("%Y%m%d")}-#{to_date.strftime("%Y%m%d")}-#{DateTime.now.strftime("%Y%m%d%H%M%S")}.#{fileext}"
-        Orsos::Webdownloader.new(options[:verbose])
+        Orsos::Webdownloader.new(get_downloader_options(filename: "sos_transactions_#{from_date.strftime("%Y%m%d")}-#{to_date.strftime("%Y%m%d")}-#{DateTime.now.strftime("%Y%m%d%H%M%S")}", options: options))
                             .save_campaign_finance_transactions from_date: from_date, 
                                                                 to_date: to_date, 
-                                                                filename: filename, 
-                                                                csvbin: csvbin,
-                                                                stdout: options['stdout'],
                                                                 options: trans_opts
 
       else
         (from_date..to_date).each do |date|
-          filename = "sos_transactions_#{date.strftime("%Y%m%d")}-#{DateTime.now.strftime("%Y%m%d%H%M%S")}.#{fileext}"
-
-          Orsos::Webdownloader.new(options[:verbose])
+          Orsos::Webdownloader.new(get_downloader_options(filename: "sos_transactions_#{date.strftime("%Y%m%d")}-#{DateTime.now.strftime("%Y%m%d%H%M%S")}", options: options))
                               .save_campaign_finance_transactions from_date: date, 
                                                                   to_date: date, 
-                                                                  filename: filename, 
-                                                                  csvbin: csvbin,
-                                                                  stdout: options['stdout'],
                                                                   options: trans_opts
         end
       end
     end
 
+    desc "committees", "Download committees information sos_committees_{search query}. eg., orsos get committees kitzhaber."
+    option :committee_name_contains, type: :string, desc: 'search by name of committee which contains.... eg., --committee_name=kitzhaber searches for records that contain kitzhaber in the name.'
+    def committees
+      Orsos::Webdownloader.new(get_downloader_options(filename: "sos_committees_#{options['committee_name_contains']}", options: options))
+                          .save_committees committee_name_contains: options['committee_name_contains']
+    end
+
+    desc "candidate_filings FROM [TO]", "Download candidate_filings between FROM till TO into sos_candidate_filings_{from %Y%m%d}-{to %Y%m%d}-{current time stamp}. eg., orsos get candidate_filings 2014-10-01 2014-10-31. TO defaults to today's date"
+    def candidate_filings(from, to=Date.today)
+      from_date = case from
+        when Date
+          from
+        when String
+          Date.parse from
+        else
+          raise 'invalid from date'
+      end
+
+      to_date = case to
+        when Date
+          to
+        when String
+          Date.parse to
+        else
+          raise 'invalid to date'
+      end
+
+      Orsos::Webdownloader.new(get_downloader_options(filename: "sos_candidate_filings_#{from_date.strftime("%Y%m%d")}-#{to_date.strftime("%Y%m%d")}-#{DateTime.now.strftime("%Y%m%d%H%M%S")}", options: options))
+                          .save_candidate_filings from_date: from_date, to_date: to_date
+    end
+
     ### FIX for help issue (see commit) ###
     package_name "get"
 
     def self.banner(command, namespace = nil, subcommand = false)
       "#{basename} #{@package_name} #{command.usage}"
     end
+
     ### END FIX ###
+    private
+
+    def get_downloader_options filename: , options: {}
+      if options['in2csv']
+        csvbin = 'in2csv'
+        fileext = 'csv'
+      elsif options['xls2csv']
+        csvbin = 'xls2csv'
+        fileext = 'csv'
+      else
+        csvbin = nil
+        fileext = 'xls'
+      end
+      
+      {
+        verbose: options[:verbose],
+        csvbin: csvbin,
+        stdout: options['stdout'],
+        filename: "#{filename}.#{fileext}"
+      }
+    end
   end
 end

data/lib/orsos/version.rb

@@ -1,3 +1,3 @@
 module Orsos
-  VERSION = "0.0.5"
+  VERSION = "0.0.6"
 end

data/lib/orsos/webdownloader.rb

@@ -7,43 +7,55 @@ module MakeMakefile::Logging
 end
 
 class Orsos::Webdownloader
-  def initialize(verbose=false)
+  def initialize(verbose: false, csvbin: nil, stdout: false, filename: nil)
     @verbose = verbose
+    @csvbin = csvbin
+    @stdout = stdout
+    @filename = filename
   end
 
-  def save_campaign_finance_transactions from_date:, to_date:, filename: , csvbin:, stdout:, options: {}
-    puts "downloading transactions for #{from_date.strftime('%Y-%m-%d')} till #{to_date.strftime('%Y-%m-%d')}" if !stdout
-
-    export_page = download_campaign_finance_transactions from_date: from_date, to_date: to_date, filer_id: options['filer_id']
-    raise "could not download campaign finance transactions" if export_page.nil?
-
-    data = if !csvbin.nil?
-      csvpath = find_executable0 csvbin
-      raise "could not find #{csvbin} in $PATH" if csvpath.nil?
-      file = Tempfile.new(['xls2csv-', '.xls'])
-      file.binmode
-      begin
-        file.write(export_page.body)
-        file.rewind
-
-        `#{csvpath} #{file.path}`
-      ensure
-        file.close
-        file.unlink
-      end
-    else
+  def save_campaign_finance_transactions from_date:, to_date:, options: {}
+    save("transactions for #{from_date.strftime('%Y-%m-%d')} till #{to_date.strftime('%Y-%m-%d')}") do
+      export_page = download_campaign_finance_transactions from_date: from_date, to_date: to_date, filer_id: options['filer_id']
+      raise "could not download campaign finance transactions" if export_page.nil?
+
+      export_page.body
+    end
+  end
+
+  def save_committees committee_name_contains:, options: {}
+    save("committees searched by #{committee_name_contains}") do
+      export_page = download_committees committee_name: committee_name_contains, committee_name_search_type: 'contains'
+      raise "could not download committees" if export_page.nil?
+      export_page.body
+    end
+  end
+
+  def save_candidate_filings from_date: , to_date: , options: {}
+    save("candidates filings for #{from_date.strftime('%Y-%m-%d')} till #{to_date.strftime('%Y-%m-%d')}") do
+      export_page = download_candidate_filings from_date: from_date, to_date: to_date
+      raise "could not download committees" if export_page.nil?
       export_page.body
     end
 
-    if stdout
+  end
+
+
+private
+  def save msg, &block
+    puts "downloading #{msg}" if !@stdout
+
+    body = block.call
+    data = !@csvbin.nil? ? convert_to_csv(body) : body
+
+    if @stdout
       $stdout.write data
     else
-      File.open(filename, 'wb') {|f| f.write(data) }
-      puts "saved transactions for #{from_date.strftime("%Y-%m-%d")} till #{to_date.strftime('%Y-%m-%d')} to #{filename}"
+      File.open(@filename, 'wb') {|f| f.write(data) }
+      puts "saved #{msg} to #{@filename}"
     end
   end
 
-private
   def download_campaign_finance_transactions from_date:, to_date:, filer_id: nil
     set_agent
     export_page = nil
@@ -64,6 +76,73 @@ private
     return export_page
   end
 
+  def download_committees committee_name: '', committee_name_search_type: 'contains'
+    set_agent
+    export_page = nil
+
+    @agent.get("#{@base_url}/orestar/GotoSearchByName.do") do |search_page|
+      @results_page = @agent.post('https://secure.sos.state.or.us/orestar/CommitteeSearchFirstPage.do', {
+        buttonName: '',
+        page: 100,
+        committeeName: committee_name,
+        committeeNameMultiboxText: committee_name_search_type,
+        committeeId: '',
+        firstName: '',
+        firstNameMultiboxText: 'contains',
+        lastName: '',
+        lastNameMultiboxText: 'contains',
+        discontinuedSOO: 'on',
+        submit: 'Submit',
+        approvedSOO: 'true',
+        pendingApprovalSOO: 'false',
+        insufficientSOO: 'false',
+        resolvedSOO: 'false',
+        rejectedSOO: 'false'
+      }) 
+
+      if link = @results_page.link_with(text: "Export To Excel Format")
+        export_page  = @agent.click(link)
+      end
+    end
+   
+    return export_page
+  end
+
+  def download_candidate_filings from_date: , to_date: nil
+    set_agent
+    export_page = nil
+
+    @agent.get("#{@base_url}/orestar/CFSearchPage.do") do |search_page|
+      search_page.form_with(name: 'cfSearchPageForm') do |form|
+        form.cfFilingFromDate = from_date.strftime("%m/%d/%Y")
+        form.cfFilingToDate = to_date.strftime("%m/%d/%Y")
+
+        @results_page = @agent.submit(form, form.button_with(value: "Submit"))
+        if link = @results_page.link_with(text: "Export")
+          export_page  = @agent.click(link)
+        end
+      end
+    end
+
+    return export_page
+  end
+
+  def convert_to_csv body
+    csvpath = find_executable0 @csvbin
+    raise "could not find #{@csvbin} in $PATH" if csvpath.nil?
+    file = Tempfile.new(['xls2csv-', '.xls'])
+    file.binmode
+    begin
+      file.write(body)
+      file.rewind
+
+      return `#{csvpath} #{file.path}`
+    ensure
+      file.close
+      file.unlink
+    end
+  end
+
   def set_source_xls_file_and_downloaded_at body, filename
     file = StringIO.new(body)
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: orsos
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Jonathan Chang