orsos 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d63316ed0dbd8306399ebe2aaa2375a7ce4e4c2a
+  data.tar.gz: 894dfef3f82e1f8ea76f1320e265617852652564
+SHA512:
+  metadata.gz: 1b6d948103d78c9afdd4d4a677b1f552b05c4f5cf9526684a383df20ce2e5af97e97ccbb76caecb0b28a20c0342df791dd56e26b3d02f6d22c1df5713fe1721f
+  data.tar.gz: 95610fd869f27050544ccd35ce1f91ea13626f0a6ad798a957d31507c33f1a126464cca69764b1ab968b4400f76403a0476302656a2e2d591d5840038565f79b

data/.gitignore

@@ -0,0 +1,50 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+
+# If you find yourself ignoring temporary files generated by your text editor
+# or operating system, you probably want to add a global ignore instead:
+#   git config --global core.excludesfile ~/.gitignore_global
+#
+# Here are some files you may want to ignore globally:
+
+# scm revert files
+**.orig
+
+# Mac finder artifacts
+.DS_Store
+
+# Netbeans project directory
+/nbproject/
+
+# RubyMine project files
+.idea
+
+# Textmate project files
+/*.tmproj
+
+# vim artifacts
+**.swp
+
+# Environment files that may contain sensitive data
+.env
+.powenv

data/.ruby-gemset

@@ -0,0 +1 @@
+hack_oregon_prototype

data/.ruby-version

@@ -0,0 +1 @@
+2.1.2

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+ruby '2.1.2'
+gem 'pry-byebug'
+gem 'repl_rake'
+# Specify your gem's dependencies in orsos.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,67 @@
+PATH
+  remote: .
+  specs:
+    orsos (0.0.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    byebug (3.5.1)
+      columnize (~> 0.8)
+      debugger-linecache (~> 1.2)
+      slop (~> 3.6)
+    coderay (1.1.0)
+    columnize (0.8.9)
+    debugger-linecache (1.2.0)
+    domain_name (0.5.21)
+      unf (>= 0.0.5, < 1.0.0)
+    http-cookie (1.0.2)
+      domain_name (~> 0.5)
+    mechanize (2.7.3)
+      domain_name (~> 0.5, >= 0.5.1)
+      http-cookie (~> 1.0)
+      mime-types (~> 2.0)
+      net-http-digest_auth (~> 1.1, >= 1.1.1)
+      net-http-persistent (~> 2.5, >= 2.5.2)
+      nokogiri (~> 1.4)
+      ntlm-http (~> 0.1, >= 0.1.1)
+      webrobots (>= 0.0.9, < 0.2)
+    method_source (0.8.2)
+    mime-types (2.3)
+    mini_portile (0.6.0)
+    net-http-digest_auth (1.4)
+    net-http-persistent (2.9.4)
+    netrc (0.7.9)
+    nokogiri (1.6.3.1)
+      mini_portile (= 0.6.0)
+    ntlm-http (0.1.1)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    pry-byebug (2.0.0)
+      byebug (~> 3.4)
+      pry (~> 0.10)
+    rake (10.3.2)
+    repl_rake (0.1.0)
+    rest_client (1.7.3)
+      netrc (~> 0.7.7)
+    slop (3.6.0)
+    thor (0.19.1)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.6)
+    webrobots (0.1.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.6)
+  mechanize
+  orsos!
+  pry-byebug
+  rake
+  repl_rake
+  rest_client (~> 1.7.3)
+  thor

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Jonathan Chang
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,105 @@
+# Orsos
+
+Oregon Secretary of State Website Client. Use this tool to easily download transactions, candidate filings and committees excel files. Also provides a client to Hack Oregon API
+
+## TODO
+- Transactions by commitee filing id
+- Candidate Filings
+- Committees
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'orsos'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install orsos
+
+## Usage
+
+TODO: Write usage instructions here
+
+    $ orsos help
+    Commands:
+      orsos api [COMMAND]   # client for api located at http://hack-oregon-prototype.herokuapp.com
+      orsos get [COMMAND]   # downloads excel files from website
+      orsos help [COMMAND]  # Describe available commands or one specific command
+
+    $ orsos api help
+    api commands:
+      orsos api candidates      # gets json or text for http://hack-oregon-prototype.herokuapp.com/api...
+      orsos api help [COMMAND]  # Describe subcommands or one specific subcommand
+
+    $ orsos api candidates | head -n 20
+    {
+      "candidates": [
+        {
+          "id": 174,
+          "ballot_name": "Adam Peterson",
+          "party_affiliation": "Nonpartisan",
+          "email": "adam@judgeadampeterson.com",
+          "cell_phone": "Exempt from public record "
+        },
+        {
+          "id": 287,
+          "ballot_name": "Adrienne Nelson",
+          "party_affiliation": "Nonpartisan",
+          "email": "adrienne.nelson@ojd.state.or.us",
+          "cell_phone": null
+        },
+        {
+          "id": 12,
+          "ballot_name": "Alex Gardner",
+          "party_affiliation": "Nonpartisan",
+
+    $ orsos api candidates --text | head -n 20
+
+    Candidates
+    ----------
+    id	ballot name	party	email
+    -------------------------
+    174	Adam Peterson	Nonpartisan	adam@judgeadampeterson.com
+    287	Adrienne Nelson	Nonpartisan	adrienne.nelson@ojd.state.or.us
+    12	Alex Gardner	Nonpartisan	alex.gardner@co.lane.or.us
+    195	Alex Hamalian	Nonpartisan	alexforjudge@gmail.com
+    180	Alicia A Fuchs	Nonpartisan	alicia.fuchs@ojd.state.or.us
+    106	Alissa Keny-Guyer	Working Families	alissa@alissakenyguyer.com
+    258	Alta Jean Brady	Nonpartisan	ghendrix@bendcable.com
+    168	A Michael Adler	Nonpartisan	michael.adler@ojd.state.or.us
+    315	Amy Holmes Hehn	Nonpartisan	amy.holmeshehn@ojd.state.or.us
+    156	Andrew (Drew) Kaza	Working Families	calmsense@nehalemtel.net
+    58	Andrew R Erwin	Nonpartisan	andyerwin@icloud.com
+    242	Andy Balyeat	Nonpartisan	andy@balyeatlaw.com
+    82	Andy Olson	Independent	andypamo@comcast.net
+    157	Angel Lopez	Nonpartisan	Angel.Lopez@ojd.state.or.us
+    22	Annette C Hillman	Nonpartisan	annette.c.hillman@ojd.state.or.us
+    46	Ann Lininger	Working Families	ann.lininger@gmail.com
+    
+    $ orsos get help
+    get commands:
+      orsos get help [COMMAND]          # Describe subcommands or one specific subcommand
+      orsos get transactions FROM [TO]  # Download campaign finance transactions daily between FROM ti...
+
+    $ orsos get transactions 2014-10-21 2014-10-22
+    downloading transactions for 2014-10-21
+    saved transactions for 2014-10-21 to sos_transactions_20141021-20141022121911.xls
+    downloading transactions for 2014-10-22
+    saved transactions for 2014-10-22 to sos_transactions_20141022-20141022121927.xls
+
+    $ ls *.xls
+    sos_transactions_20141021-20141022121911.xls	sos_transactions_20141022-20141022121927.xls
+
+
+## Contributing
+
+1. Fork it ( https://github.com/jonochang/orsos/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,3 @@
+require "bundler/gem_tasks"
+require 'repl_rake'
+ReplRake.setup

data/bin/orsos

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'orsos'
+require 'pry-byebug'
+Orsos::Command.start

data/lib/orsos/command.rb

@@ -0,0 +1,7 @@
+require 'thor'
+require_relative './commands/get'
+require_relative './commands/api'
+class Orsos::Command < Thor
+  register Orsos::Commands::Get, 'get', 'get [COMMAND]', 'downloads excel files from website'
+  register Orsos::Commands::Api, 'api', 'api [COMMAND]', "client for api located at #{Orsos::Commands::Api::ROOT_URL}"
+end

data/lib/orsos/commands/api.rb

@@ -0,0 +1,35 @@
+require 'rest_client'
+require 'json'
+module Orsos::Commands
+  class Api < Thor
+    ROOT_URL = "http://hack-oregon-prototype.herokuapp.com"
+
+    desc "candidates", "gets json or text for #{ROOT_URL}/api/candidates.json"
+    option :text, type: :boolean
+    def candidates
+      url = "#{ROOT_URL}/api/candidates.json"
+      response = RestClient.get(url)
+      if options[:text]
+        json = JSON.parse(response.body)
+        puts "Candidates"
+        puts "----------"
+        puts "id\tballot name\tparty\temail"
+        puts "-------------------------"
+        json["candidates"].each do |candidate|
+          puts "#{candidate["id"]}\t#{candidate["ballot_name"]}\t#{candidate["party_affiliation"]}\t#{candidate["email"]}"
+        end
+      else
+        puts JSON.pretty_generate(JSON.parse(response.body))
+      end
+    end
+
+    ### FIX for help issue (see commit) ###
+    package_name "api"
+
+    def self.banner(command, namespace = nil, subcommand = false)
+      "#{basename} #{@package_name} #{command.usage}"
+    end
+    ### END FIX ###
+  end
+end
+

data/lib/orsos/commands/get.rb

@@ -0,0 +1,41 @@
+require 'date'
+require_relative '../webdownloader'
+
+module Orsos::Commands
+  class Get < Thor
+    desc "transactions FROM [TO]", "Download campaign finance transactions daily between FROM till TO and saves each day to sos_transactions_{%Y%m%d}-{current time stamp}. eg., orsos get transactions 2014-10-01 2014-10-31. TO defaults to today's date"
+    option :verbose, type: :boolean
+    def transactions(from, to=Date.today)
+      from_date = case from
+        when Date
+          from
+        when String
+          Date.parse from
+        else
+          raise 'invalid from date'
+      end
+
+      to_date = case to
+        when Date
+          to
+        when String
+          Date.parse to
+        else
+          raise 'invalid to date'
+      end
+      
+      (from_date..to_date).each do |date|
+        Orsos::Webdownloader.new(options[:verbose])
+                            .save_campaign_finance_transactions_to_xls date, "sos_transactions"
+      end
+    end
+
+    ### FIX for help issue (see commit) ###
+    package_name "get"
+
+    def self.banner(command, namespace = nil, subcommand = false)
+      "#{basename} #{@package_name} #{command.usage}"
+    end
+    ### END FIX ###
+  end
+end

data/lib/orsos/version.rb

@@ -0,0 +1,3 @@
+module Orsos
+  VERSION = "0.0.1"
+end

data/lib/orsos/webdownloader.rb

@@ -0,0 +1,69 @@
+require 'mechanize'
+require 'logger'
+
+class Orsos::Webdownloader
+  def initialize(verbose=false)
+    @verbose = verbose
+  end
+
+  def save_campaign_finance_transactions_to_xls date, filename_prefix="sos_transactions"
+    puts "downloading transactions for #{date.strftime('%Y-%m-%d')}"
+    filename = "#{filename_prefix}_#{date.strftime("%Y%m%d")}-#{DateTime.now.strftime("%Y%m%d%H%M%S")}.xls"
+
+    export_page = download_campaign_finance_transactions date
+    raise "could not download campaign finance transactions" if export_page.nil?
+    File.open(filename, 'wb') {|f| f.write(export_page.body) } if export_page
+
+    puts "saved transactions for #{date.strftime("%Y-%m-%d")} to #{filename}"
+  end
+
+private
+  def download_campaign_finance_transactions date
+    set_agent
+    export_page = nil
+
+    @agent.get("#{@base_url}/orestar/gotoPublicTransactionSearch.do") do |search_page|
+      search_page.form_with(name: 'cneSearchForm') do |form|
+        form.cneSearchTranFiledStartDate = date.strftime("%m/%d/%Y")
+        form.cneSearchTranFiledEndDate = date.strftime("%m/%d/%Y")
+
+        @results_page = @agent.submit(form, form.button_with(value: "Search"))
+        if link = @results_page.link_with(text: "Export To Excel Format")
+          export_page  = @agent.click(link)
+        end
+      end
+    end
+
+    return export_page
+  end
+
+  def set_source_xls_file_and_downloaded_at body, filename
+    file = StringIO.new(body)
+  end
+
+  def parse_date source
+    return nil if source.to_s.strip.empty?
+    Date.strptime(source, '%m/%d/%Y')
+  end
+
+  def uri_path attachment
+    return nil unless attachment.exists?
+    case attachment.options[:storage]
+    when :filesystem
+      attachment.path
+    when :s3
+      attachment.url(:original, timestamp: false)
+    else
+      raise 'unsupported paperclip storage'
+    end
+  end
+
+  def set_agent
+    @agent = Mechanize.new
+    @agent.user_agent_alias = 'Mac Safari'#'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36' # Wikipedia blocks "mechanize"
+    @agent.log = Logger.new(STDOUT) if @verbose
+
+    @history = @agent.history
+    @base_url = URI 'https://secure.sos.state.or.us'
+  end
+end

data/lib/orsos.rb

@@ -0,0 +1,5 @@
+require_relative "./orsos/version"
+require_relative "./orsos/command"
+
+module Orsos
+end

data/orsos.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'orsos/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "orsos"
+  spec.version       = Orsos::VERSION
+  spec.authors       = ["Jonathan Chang"]
+  spec.email         = ["jonathan.chang@silverpond.com.au"]
+  spec.summary       = %q{Oregon Secretary of State website (http://sos.oregon.gov/Pages/default.aspx) scraper}
+  spec.description   = %q{A command line tool to download data from the Oregon Secretary of State website (http://sos.oregon.gov/Pages/default.aspx). Uses mechanize gem to post search requests to the website and download excel files.}
+  spec.homepage      = "https://github.com/jonochang/orsos"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "mechanize"
+  spec.add_development_dependency "thor"
+  spec.add_development_dependency "rest_client", "~> 1.7.3"
+end

data/orsos.rb

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+require "rubygems" # ruby1.9 doesn't "require" it though
+require "thor"
+
+class MyThorCommand < Thor
+  desc "foo", "Prints foo"
+  def foo
+    puts "foo"
+  end
+end
+
+MyThorCommand.start

metadata

@@ -0,0 +1,135 @@
+--- !ruby/object:Gem::Specification
+name: orsos
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Jonathan Chang
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-10-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rest_client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.3
+description: A command line tool to download data from the Oregon Secretary of State
+  website (http://sos.oregon.gov/Pages/default.aspx). Uses mechanize gem to post search
+  requests to the website and download excel files.
+email:
+- jonathan.chang@silverpond.com.au
+executables:
+- orsos
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".ruby-gemset"
+- ".ruby-version"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/orsos
+- lib/orsos.rb
+- lib/orsos/command.rb
+- lib/orsos/commands/api.rb
+- lib/orsos/commands/get.rb
+- lib/orsos/version.rb
+- lib/orsos/webdownloader.rb
+- orsos.gemspec
+- orsos.rb
+homepage: https://github.com/jonochang/orsos
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Oregon Secretary of State website (http://sos.oregon.gov/Pages/default.aspx)
+  scraper
+test_files: []