orsos 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 583b7c177d70e612ff2119874e9929f089942e53
-  data.tar.gz: 6b0399d8c7e9076e019323a70cab445ba0905270
+  metadata.gz: 4b9ba5e8b35fc60d3a36534f70e73f04f7f8962c
+  data.tar.gz: f8cb4234383f9a4d771992b794827ccb779352c9
 SHA512:
-  metadata.gz: fa7196f3a009cb2d5b2f6ec3d55bfb5b2c2876d4d559367d526bb5c351e244ce849b924bfe293da284e11e4caeb89db4c36c37fc582397fd452255b95205a12e
-  data.tar.gz: 79182511ef087494b19b1ac2a739d977ec38f51985ae62bebb0dca419bf27d282b7e1e95d437ed2738f138eab43222e314616c5f4fcd68bf2dcf660c857ef5c0
+  metadata.gz: 3a3024f06a558ea6e41021c65adcfefd0638f964bde5c611a075e12edf591f7e51d5783850fbef426f1fa4c3de27ce483be97bc10f4dafe567616cb268d75188
+  data.tar.gz: b0d0b12f00cc4e49c4c4120eb0dbcedea790c77c788b1ae22a28763a97ad6800f1fa863f31b41327932d165c6d00fd20a81311e99e0b8955e6b807e82b00eb41

data/README.md

@@ -3,8 +3,7 @@
 Oregon Secretary of State Website Client. Use this tool to easily download transactions, candidate filings and committees excel files. Also provides a client to Hack Oregon API
 
 ## TODO
-- Candidate Filings
-- Committees
+- add more query params to actions
 
 ## Requirements
 - Only tested with Ruby 2.1.2
@@ -86,8 +85,25 @@ TODO: Write usage instructions here
     
     $ orsos get help
     get commands:
-      orsos get help [COMMAND]          # Describe subcommands or one specific subcommand
-      orsos get transactions FROM [TO]  # Download campaign finance transactions daily between FROM ti...
+      orsos get candidate_filings FROM [TO]  # Download candidate_filings between FROM till TO into sos_candidate_filings_{from %Y%m%d}-{to %Y%m%d}-{current time stamp}. eg., orsos get candidate_filings 20...
+      orsos get committees                   # Download committees information sos_committees_{search query}. eg., orsos get committees kitzhaber.
+      orsos get help [COMMAND]               # Describe subcommands or one specific subcommand
+      orsos get transactions FROM [TO]       # Download campaign finance transactions daily between FROM till TO and saves each day to sos_transactions_{%Y%m%d}-{current time stamp}. eg., orsos get transac...
+
+    Options:
+      [--in2csv], [--no-in2csv]    # use in2csv to convert downloaded xls to csv
+      [--xls2csv], [--no-xls2csv]  # use xls2csv to convert downloaded xls to csv
+      [--stdout], [--no-stdout]    # output to stdout
+      [--verbose], [--no-verbose]  # turn on verbose logging of search
+
+    $ orsos get transactions 2014-10-21 2014-10-22
+    downloading transactions for 2014-10-21
+    saved transactions for 2014-10-21 to sos_transactions_20141021-20141022121911.xls
+    downloading transactions for 2014-10-22
+    saved transactions for 2014-10-22 to sos_transactions_20141022-20141022121927.xls
+
+    $ ls *.xls
+    sos_transactions_20141021-20141022121911.xls	sos_transactions_20141022-20141022121927.xls
 
     $ orsos get transactions 2014-10-21 2014-10-22
     downloading transactions for 2014-10-21
@@ -98,6 +114,24 @@ TODO: Write usage instructions here
     $ ls *.xls
     sos_transactions_20141021-20141022121911.xls	sos_transactions_20141022-20141022121927.xls
 
+    $ orsos get transactions 2014-10-01 2014-10-02 --filer_id=13920 --single-file --in2csv --stdout | head -n 5
+    Tran Id,Original Id,Tran Date,Tran Status,Filer,Contributor/Payee,Sub Type,Amount,Aggregate Amount,Contributor/Payee Committee ID,Filer Id,Attest By Name,Attest Date,Review By Name,Review Date,Due Date,Occptn Ltr Date,Pymt Sched Txt,Purp Desc,Intrst Rate,Check Nbr,Tran Stsfd Ind,Filed By Name,Filed Date,Addr book Agent Name,Book Type,Title Txt,Occptn Txt,Emp Name,Emp City,Emp State,Employ Ind,Self Employ Ind,Addr Line1,Addr Line2,City,State,Zip,Zip Plus Four,County,Purpose Codes,Exp Date
+    1850201,1850201,10/02/2014,Original,Kitzhaber for Governor,Anne Philiben,Cash Contribution,100.0,150.0,,13920,Kevin F Neely,10/02/2014,eliack,10/16/2014,10/09/2014,,,,,,N,Kevin F Neely,10/02/2014,,Individual,,,,,,Y,N,1114 SE Palmwood Ct,,Bend,OR,97702,,,,
+    1850213,1850213,10/02/2014,Original,Kitzhaber for Governor,Patrick Egan,Cash Contribution,500.0,500.0,,13920,Kevin F Neely,10/02/2014,eliack,10/16/2014,10/09/2014,,,,,,N,Kevin F Neely,10/02/2014,,Individual,,Vice president,Pacific Power,Portland,OR,N,N,55 Aquinas St,,Lake Oswego,OR,97035,,,,
+    1850216,1850216,10/02/2014,Original,Kitzhaber for Governor,Miscellaneous Cash Contributions $100 and under ,Cash Contribution,980.0,,,13920,Kevin F Neely,10/02/2014,,,10/09/2014,,,,,,N,Kevin F Neely,10/02/2014,,,,,,,,N,N,,,,,,,,,
+    1847069,1847069,10/01/2014,Original,Kitzhaber for Governor,Alan Hilles,Cash Contribution,20.0,180.0,,13920,Kevin F Neely,10/01/2014,eliack,10/16/2014,10/08/2014,,,,,,N,Kevin F Neely,10/01/2014,,Individual,,physician,Bend Memorial Clinic,Bend,OR,N,N,2874 NW Fairway Hts Dr,,Bend,OR,97701,,,,
+
+    $ orsos get committees --committee-name-contains=kitzhaber --in2csv --stdout 
+    Committee Id,Committee Name,Committee Type,Committee SubType,Candidate Office,Candidate Office Group,Filing Date,Organization Filing Date,Treasurer First Name,Treasurer Last Name,Treasurer Mailing Address,Treasurer Work Phone,Treasurer Fax,Candidate First Name,Candidate Last Name,Candidate Maling Address,Candidate Work Phone,Candidate Residence Phone,Candidate Fax,Candidate Email,Active Election,Measure
+    13920,Kitzhaber for Governor,CC,,statewide,Governor,05/28/2014,09/02/2009,Kevin,Neely,PO Box 42307 Portland OR 97242,,(503)295-0670,John,Kitzhaber,PO Box 42307 Portland OR 97242,,,,info@c-esystems.com,2014 General Election,
+    
+    $ orsos get candidate_filings 2014-08-01 2014-10-01 --in2csv --stdout | head -n 5
+    Election Txt,Election Year,Office Group,ID Nbr,Office,Candidate Office,Candidate File RSN,File Mthd Ind,Filetype Descr,Party Descr,Major Party Ind,Cand Ballot Name Txt,Occptn Txt,Education Bckgrnd Txt,Occptn Bkgrnd Txt,School - Grade - Diploma/Degree/Certificate - Course of Study,Prev Govt Bkgrnd Txt,Judge Incbnt Ind,Qlf Ind,Filed Date,File Fee Rfnd Date,Witdrw Date,Withdrw Resn Txt,Pttn File Date,Pttn Sgnr Rqd Nbr,Pttn Signr Filed Nbr,Pttn Cmplt Date,Ballot Order Nbr,Prfx Name Cd,First Name,Mdle Name,Last Name,Sufx Name,Title Txt,Mailing Addr Line 1,Mailing Addr Line 2,Mailing City Name,Mailing St Cd,Mailing Zip Code,Mailing Zip Plus Four,Residence Addr Line 1,Residence Addr Line 2,Residence City Name,Residence St Cd,Residence Zip code,Residence Zip Plus Four,Home Phone,Cell Phone,Fax Phone,Email,Work Phone,Web Address
+    2014 General Election,2014,State Representative,1251,51st District,"State Representative, 51st District",15697,P,Minor Party,Libertarian,N,Jodi Bailey,"Membership Director, Building a Better America Council",,"Small Business owner, Sales, National Builders Hardware",,Happy Valley Police & Public Safety Committee ,N,Y,08/13/2014,,,,,,,,3,,Jodi,K,Bailey,,,9803 SE Plover Dr,,Happy Valley,OR,97086,,9803 SE Plover Dr,,Happy Valley,OR,97086,,,(503)504-5611,,jodikbailey@yahoo.com,(503)724-8239,jodibailey.com
+    2014 General Election,2014,State Representative,1228,28th District,"State Representative, 28th District",15729,P,Minor Party,Working Families,N,Jeff Barker,State Representative House Dist. 28,,"State Representative 2003-Present
+
+    Portland Police Bureau 1974-2001
+
 
 ## Contributing
 

data/lib/orsos/commands/get.rb

@@ -7,6 +7,7 @@ module Orsos::Commands
     class_option :xls2csv, type: :boolean, desc: 'use xls2csv to convert downloaded xls to csv'
     class_option :stdout, type: :boolean, desc: 'output to stdout'
     class_option :verbose, type: :boolean, desc: 'turn on verbose logging of search'
+    class_option :raise_error_if_no_data, type: :boolean, desc: 'raise an error if no data returned from ORSOS website'
 
     desc "transactions FROM [TO]", "Download campaign finance transactions daily between FROM till TO and saves each day to sos_transactions_{%Y%m%d}-{current time stamp}. eg., orsos get transactions 2014-10-01 2014-10-31. TO defaults to today's date"
     option :filer_id, type: :numeric, desc: 'conduct search by filer_id'
@@ -105,7 +106,8 @@ module Orsos::Commands
         verbose: options[:verbose],
         csvbin: csvbin,
         stdout: options['stdout'],
-        filename: "#{filename}.#{fileext}"
+        filename: "#{filename}.#{fileext}",
+        raise_error_if_no_data: options['raise_error_if_no_data']
       }
     end
   end

data/lib/orsos/version.rb

@@ -1,3 +1,3 @@
 module Orsos
-  VERSION = "0.0.6"
+  VERSION = "0.0.7"
 end

data/lib/orsos/webdownloader.rb

@@ -7,53 +7,55 @@ module MakeMakefile::Logging
 end
 
 class Orsos::Webdownloader
-  def initialize(verbose: false, csvbin: nil, stdout: false, filename: nil)
+  def initialize(verbose: false, csvbin: nil, stdout: false, filename: nil, raise_error_if_no_data: false)
     @verbose = verbose
     @csvbin = csvbin
     @stdout = stdout
     @filename = filename
+    @raise_error_if_no_data = raise_error_if_no_data
   end
 
   def save_campaign_finance_transactions from_date:, to_date:, options: {}
     save("transactions for #{from_date.strftime('%Y-%m-%d')} till #{to_date.strftime('%Y-%m-%d')}") do
-      export_page = download_campaign_finance_transactions from_date: from_date, to_date: to_date, filer_id: options['filer_id']
-      raise "could not download campaign finance transactions" if export_page.nil?
-
-      export_page.body
+      download_campaign_finance_transactions from_date: from_date, to_date: to_date, filer_id: options['filer_id']
     end
   end
 
   def save_committees committee_name_contains:, options: {}
     save("committees searched by #{committee_name_contains}") do
-      export_page = download_committees committee_name: committee_name_contains, committee_name_search_type: 'contains'
-      raise "could not download committees" if export_page.nil?
-      export_page.body
+      download_committees committee_name: committee_name_contains, committee_name_search_type: 'contains'
     end
   end
 
   def save_candidate_filings from_date: , to_date: , options: {}
     save("candidates filings for #{from_date.strftime('%Y-%m-%d')} till #{to_date.strftime('%Y-%m-%d')}") do
-      export_page = download_candidate_filings from_date: from_date, to_date: to_date
-      raise "could not download committees" if export_page.nil?
-      export_page.body
+      download_candidate_filings from_date: from_date, to_date: to_date
     end
-
   end
 
-
-private
   def save msg, &block
-    puts "downloading #{msg}" if !@stdout
+    puts "[ORSOS] [#{Time.now}] downloading #{msg}" if !@stdout
 
-    body = block.call
-    data = !@csvbin.nil? ? convert_to_csv(body) : body
+    export_page = block.call
+
+    if export_page.nil?
+      if @raise_error_if_no_data
+        raise "could not download file. no data returned" 
+      elsif !@stdout
+        puts "[ORSOS] [#{Time.now}] no data returned"
+      end
 
-    if @stdout
-      $stdout.write data
     else
-      File.open(@filename, 'wb') {|f| f.write(data) }
-      puts "saved #{msg} to #{@filename}"
+      data = !@csvbin.nil? ? convert_to_csv(export.body) : export.body
+
+      if @stdout
+        $stdout.write data
+      else
+        File.open(@filename, 'wb') {|f| f.write(data) }
+        puts "[ORSOS] [#{Time.now}] saved #{msg} to #{@filename}"
+      end
     end
+
   end
 
   def download_campaign_finance_transactions from_date:, to_date:, filer_id: nil
@@ -127,6 +129,8 @@ private
     return export_page
   end
 
+  private
+
   def convert_to_csv body
     csvpath = find_executable0 @csvbin
     raise "could not find #{@csvbin} in $PATH" if csvpath.nil?

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: orsos
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - Jonathan Chang