orsos 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d63316ed0dbd8306399ebe2aaa2375a7ce4e4c2a
-  data.tar.gz: 894dfef3f82e1f8ea76f1320e265617852652564
+  metadata.gz: 366b603819a5784379c68774ac3a0287dd2dc162
+  data.tar.gz: 341c70923d8d8581671fb7ef84b3b96eb4c89f21
 SHA512:
-  metadata.gz: 1b6d948103d78c9afdd4d4a677b1f552b05c4f5cf9526684a383df20ce2e5af97e97ccbb76caecb0b28a20c0342df791dd56e26b3d02f6d22c1df5713fe1721f
-  data.tar.gz: 95610fd869f27050544ccd35ce1f91ea13626f0a6ad798a957d31507c33f1a126464cca69764b1ab968b4400f76403a0476302656a2e2d591d5840038565f79b
+  metadata.gz: 800a3742d40ca1cc86cca6be5066ac6557556aa265f3c1d6b7df1d6fd91213f248de131788232c9a9aaa3b452cf22bdc7ff866382dfefebc7f1e6a9bd8e8c2d7
+  data.tar.gz: e79d62229acf16b5316966621e19a978204d3a1d306231fc0ad742078a14c8c211b15630922689cd19cfc49592ad17612406e3aa553348b3fe83e09c8e01a8c9

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    orsos (0.0.1)
+    orsos (0.0.2)
 
 GEM
   remote: https://rubygems.org/

data/lib/orsos/commands/get.rb

@@ -4,7 +4,11 @@ require_relative '../webdownloader'
 module Orsos::Commands
   class Get < Thor
     desc "transactions FROM [TO]", "Download campaign finance transactions daily between FROM till TO and saves each day to sos_transactions_{%Y%m%d}-{current time stamp}. eg., orsos get transactions 2014-10-01 2014-10-31. TO defaults to today's date"
-    option :verbose, type: :boolean
+    option :verbose, type: :boolean, desc: 'turn on verbose logging of search'
+    option :filer_id, type: :numeric, desc: 'conduct search by filer_id'
+    option :single_file, type: :boolean, desc: 'search and save data in date range as a single file rather than one search per day'
+    option :in2csv, type: :boolean, desc: 'use in2csv to convert downloaded xls to csv'
+    option :xls2csv, type: :boolean, desc: 'use xls2csv to convert downloaded xls to csv'
     def transactions(from, to=Date.today)
       from_date = case from
         when Date
@@ -23,10 +27,39 @@ module Orsos::Commands
         else
           raise 'invalid to date'
       end
-      
-      (from_date..to_date).each do |date|
+
+      trans_opts = options.select{|k,v| ['filer_id'].include?(k) }
+      if options['in2csv']
+        csvbin = 'in2csv'
+        fileext = 'csv'
+      elsif options['xls2csv']
+        csvbin = 'xls2csv'
+        fileext = 'csv'
+      else
+        csvbin = nil
+        fileext = 'xls'
+      end
+
+      if !options['single_file'].nil?
+        filename = "sos_transactions_#{from_date.strftime("%Y%m%d")}-#{to_date.strftime("%Y%m%d")}-#{DateTime.now.strftime("%Y%m%d%H%M%S")}.#{fileext}"
         Orsos::Webdownloader.new(options[:verbose])
-                            .save_campaign_finance_transactions_to_xls date, "sos_transactions"
+                            .save_campaign_finance_transactions from_date: from_date, 
+                                                                to_date: to_date, 
+                                                                filename: filename, 
+                                                                csvbin: csvbin,
+                                                                options: trans_opts
+
+      else
+        (from_date..to_date).each do |date|
+          filename = "sos_transactions_#{date.strftime("%Y%m%d")}-#{DateTime.now.strftime("%Y%m%d%H%M%S")}.#{fileext}"
+
+          Orsos::Webdownloader.new(options[:verbose])
+                              .save_campaign_finance_transactions from_date: date, 
+                                                                  to_date: date, 
+                                                                  filename: filename, 
+                                                                  csvbin: csvbin,
+                                                                  options: trans_opts
+        end
       end
     end
 

data/lib/orsos/version.rb

@@ -1,3 +1,3 @@
 module Orsos
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/lib/orsos/webdownloader.rb

@@ -1,31 +1,52 @@
 require 'mechanize'
 require 'logger'
+require 'mkmf'
 
 class Orsos::Webdownloader
   def initialize(verbose=false)
     @verbose = verbose
   end
 
-  def save_campaign_finance_transactions_to_xls date, filename_prefix="sos_transactions"
-    puts "downloading transactions for #{date.strftime('%Y-%m-%d')}"
-    filename = "#{filename_prefix}_#{date.strftime("%Y%m%d")}-#{DateTime.now.strftime("%Y%m%d%H%M%S")}.xls"
+  def save_campaign_finance_transactions from_date:, to_date:, filename: , csvbin:, options: {}
+    puts "downloading transactions for #{from_date.strftime('%Y-%m-%d')} till #{to_date.strftime('%Y-%m-%d')}"
 
-    export_page = download_campaign_finance_transactions date
+    export_page = download_campaign_finance_transactions from_date: from_date, to_date: to_date, filer_id: options['filer_id']
     raise "could not download campaign finance transactions" if export_page.nil?
-    File.open(filename, 'wb') {|f| f.write(export_page.body) } if export_page
 
-    puts "saved transactions for #{date.strftime("%Y-%m-%d")} to #{filename}"
+    if !csvbin.nil?
+      csvpath = find_executable csvbin
+      raise "could not find #{csvbin} in $PATH" if csvpath.nil?
+      file = Tempfile.new(['xls2csv-', '.xls'])
+      file.binmode
+      begin
+        file.write(export_page.body)
+        file.rewind
+
+        data = `#{csvpath} #{file.path}`
+      
+        File.open(filename, 'wb') {|f| f.write(data) } if data
+      ensure
+        file.close
+        file.unlink
+      end
+
+      puts "saved transactions for #{from_date.strftime("%Y-%m-%d")} till #{to_date.strftime('%Y-%m-%d')} to #{filename}"
+    else
+      File.open(filename, 'wb') {|f| f.write(export_page.body) } if export_page
+      puts "saved transactions for #{from_date.strftime("%Y-%m-%d")} till #{to_date.strftime('%Y-%m-%d')} to #{filename}"
+    end
   end
 
 private
-  def download_campaign_finance_transactions date
+  def download_campaign_finance_transactions from_date:, to_date:, filer_id: nil
     set_agent
     export_page = nil
 
     @agent.get("#{@base_url}/orestar/gotoPublicTransactionSearch.do") do |search_page|
       search_page.form_with(name: 'cneSearchForm') do |form|
-        form.cneSearchTranFiledStartDate = date.strftime("%m/%d/%Y")
-        form.cneSearchTranFiledEndDate = date.strftime("%m/%d/%Y")
+        form.cneSearchTranFiledStartDate = from_date.strftime("%m/%d/%Y")
+        form.cneSearchTranFiledEndDate = to_date.strftime("%m/%d/%Y")
+        form.cneSearchFilerCommitteeId = filer_id unless filer_id.nil?
 
         @results_page = @agent.submit(form, form.button_with(value: "Search"))
         if link = @results_page.link_with(text: "Export To Excel Format")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: orsos
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Jonathan Chang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-22 00:00:00.000000000 Z
+date: 2014-10-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler