enigma_io 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rubocop.yml

@@ -0,0 +1,14 @@
+# This configuration was generated by `rubocop --auto-gen-config`.
+# The point is for the user to remove these configuration records
+# one by one as the offences are removed from the code base.
+
+MethodLength:
+  Max: 20
+
+TrivialAccessors:
+  Enabled: false
+
+AllCops:
+  Excludes:
+    - test/**
+    - examples/**

data/.travis.yml

@@ -0,0 +1,14 @@
+language: ruby
+rvm:
+  - "1.9.3"
+  - "2.0.0"
+  - "2.1.0"
+  - jruby-19mode # JRuby in 1.9 mode
+
+# uncomment this line if your project needs to run something other than `rake`:
+# script: bundle exec rspec spec
+before_install: gem update --remote bundler
+
+script:
+  - bundle exec rake test
+  - bundle exec rubocop

data/Gemfile

@@ -0,0 +1,11 @@
+source 'https://rubygems.org'
+gemspec
+
+group :test do
+  gem 'rake', '>= 0.8.7'
+  gem 'simplecov'
+  gem 'webmock'
+  gem 'excon', '>= 0.27.4'
+  gem 'vcr'
+  gem 'mocha'
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Stephen Pike
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,64 @@
+# Ruby client for the enigma api
+
+Ruby client for the enigma api located at https://app.enigma.io/api. Supports ruby >= 1.9.3
+
+Note that you need api key to use their api.
+
+[![Build Status](https://travis-ci.org/scpike/enigma.png?branch=master)](https://travis-ci.org/scpike/enigma)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'enigma'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install enigma
+
+## Usage
+
+Basic usage is straightforward. There are also more detailed [data api examples](examples/data.md) and [export api examples](examples/export.md) available.
+
+    # Defaults to looking for key in ENV['ENIGMA_KEY']
+
+    client = Enigma::Client.new(key: :secret_key)
+
+    client.meta('us.gov.whitehouse.visitor-list')
+
+    res = client.data('us.gov.whitehouse.visitor-list')
+
+    # get some data
+
+    res.result.map { |r| ... }
+
+    client.stats('us.gov.whitehouse.visitor-list', select: 'type_of_access')
+
+    client.export('us.gov.whitehouse.visitor-list').parse.each do |row|
+       puts row.inspect
+    end
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+### Notes
+
+You'll need to have rubocop installed.
+
+    gem install rubocop
+
+Tests are run with `rake`. Check the test coverage (printed when you
+run rake) as well as the output of rubocop (also run with rake).
+
+## License
+
+MIT license

data/Rakefile

@@ -0,0 +1,28 @@
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+task :gendoc do
+  system "yardoc"
+  system "yard stats --list-undoc"
+end
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.test_files = FileList['test/**_test.rb']
+  t.verbose = false
+  t.warning = true
+end
+
+task :rubocop do |t|
+  sh 'rubocop'
+end
+
+task :build => :gendoc do
+  system "gem build enigma_io.gemspec"
+end
+
+task :release => :build do
+  system "gem push enigma_io-#{Enigma::VERSION}.gem"
+end
+
+task default: [:test, :rubocop]

data/enigma_io.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'enigma/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'enigma_io'
+  spec.version       = Enigma::VERSION
+  spec.authors       = ['Stephen Pike']
+  spec.email         = ['steve@scpike.net']
+  spec.description   = 'Ruby client for the Enigma API'
+  spec.summary       = 'Ruby client for the Enigma API'
+  spec.homepage      = 'https://github.com/scpike/enigma'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
+  spec.test_files    = Dir.glob("{test/**/*}")
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.3'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'yard'
+
+  spec.add_runtime_dependency 'typhoeus'
+  spec.add_runtime_dependency 'hashie'
+  spec.add_runtime_dependency 'rubyzip', '>= 1.0.0'
+
+  spec.required_ruby_version = '>= 1.9.3'
+end

data/examples/data.md

@@ -0,0 +1,94 @@
+# Accessing the Enigma metadata and data apis
+
+#### Create a client, key should be set in the environment variable 'ENIGMA_KEY'
+
+    client = Enigma::Client.new
+
+#### Let's look at internet use in the United States
+
+    res = client.meta('us.gov.census.internet.usage11')
+
+#### The result is a Hashie::Mash object, so you can access it like a hash or like an object
+
+    puts res.keys
+
+    => ["datapath", "success", "info", "result", "raw"]
+
+`raw` is the raw repsonse from the Enigma api. Everything else is
+the contents of the api response
+
+#### Verify this is a table (something we can get data about):
+
+    res.info.result_type
+
+    => "table"
+
+#### Let's see the columns
+
+    res.result.columns.each { |c| puts "#{c.id}: #{c.description}" }
+
+> region: State Region
+>
+> total_over3: Total (thousands)
+>
+> number_somewhere: Number of individuals accessing the Internet from some location (thousands). "Some location" means Internet access that occurs either inside or outside the householder's home.
+>
+> percent_somewhere: Percent of individuals accessing the Internet from some location. "Some location" means Internet access that occurs either inside or outside the householder's home.
+>
+> number_inhome: Individual lives in household with Internet (thousands). At least one member of the individual's household reported using the Internet from home.
+>
+> percent_inhome: Percent of individual living in household with Internet. At least one member of the individual's household reported using the Internet from home.
+>
+> serialid: Serialid
+
+#### Now let's get the data
+
+    data = client.data('us.gov.census.internet.usage11')
+
+    data.result.each { |r| puts "#{r.region} #{r.percent_inhome}" }
+
+> United States 76.50
+>
+> Alaska 80.00
+>
+> Alabama 69.50
+>
+> Arizona 76.00
+>
+> Arkansas 68.50
+>
+> ...
+
+#### Let's find the state withthe highest percentage. We also only care about the region and percent_inhome columns
+
+    res = client.data('us.gov.census.internet.usage11',
+                      sort: 'percent_inhome',
+                      select: [ 'region', 'percent_inhome' ])
+
+    puts res.result.first.to_hash
+
+    => {"region" =>"New Hampshire", "percent_inhome" =>"87.10"}
+
+##### What about the lowest?
+
+    puts res.result.last.to_hash
+
+    => {"region" =>"Mississippi", "percent_inhome" =>"61.40"}
+
+##### Let's search for Pennsylvania
+
+    res = client.data('us.gov.census.internet.usage11',
+                      select: [ 'region', 'percent_inhome' ],
+                      search: { region: 'Pennsylvania })
+
+#### Verify only one result found
+
+    > res.info.total_results
+
+    => 1
+
+#### Check it out
+
+    > puts res.first.to_hash
+
+    => {"region" =>"Pennsylvania", "percent_inhome" =>"75.40"}

data/examples/export.md

@@ -0,0 +1,35 @@
+# Export API
+
+#### Start the connection
+
+    client = Enigma::Client.new
+
+    dl = client.export('us.gov.census.internet.usage11')
+
+#### This did *not* download anything. The api replied with a link to where the file will eventually be
+
+    puts dl.download_url
+
+> Secret one time url
+
+#### If you want the actual contents, you can ask the client to poll until the file is available
+
+    dl.get # puts the zipped contents in dl.raw_download
+
+#### Write the zipped contents to a file
+
+    zipfile = File.open('out.zip', 'wb')
+
+    dl.write zipfile
+
+#### Write the unzipped contents to a file
+
+    csvfile = File.open('out.csv', 'wb')
+
+    dl.write_csv csvfile
+
+#### Read the contents of the csv file
+
+    dl.parse.each do |row|
+       puts row.inspect
+    end

data/lib/enigma/client.rb

@@ -0,0 +1,26 @@
+# coding: utf-8
+
+module Enigma
+  # Connects to the Enigma api at https://app.enigma.io/api.
+  #
+  class Client
+    # Creates a new client connection
+    #
+    # @option opts [String] key - Enigma API key.
+    #
+    # The api key Defaults to the ENIGMA_KEY environment variable
+    #
+    def initialize(opts = {})
+      Enigma.key = ENV['ENIGMA_KEY'] || opts[:key]
+      fail ArgumentError, 'API key is required' unless Enigma.key
+    end
+
+    # Each endpoint becomes a method like `client.meta`
+    #
+    Endpoint.descendants.each do |klass|
+      define_method klass.url_chunk do |*args|
+        klass.new(*args).request
+      end
+    end
+  end
+end

data/lib/enigma/download.rb

@@ -0,0 +1,81 @@
+# coding: utf-8
+
+module Enigma
+  # Handle polling the url returned by the Export api until we're able
+  # to download it. Will load the zipped contents of the file into
+  # memory, and can then either write it to disk (`write`), write the
+  # unzipped version to disk (`write_csv`) or parse the unzipped
+  # contents using the CSV library as an array of hashes
+  #
+  class Download
+    DELAY = 1 # How long to wait between polling attempts, in seconds
+    attr_accessor :response, :raw_download, :download_contents
+
+    def initialize(res)
+      self.response = res
+    end
+
+    def download_url
+      response.export_url
+    end
+
+    def datapath
+      response.datapath
+    end
+
+    def get
+      @raw_download ||= do_download
+    end
+
+    def write(io)
+      get
+      io.write raw_download
+    end
+
+    def write_tmp
+      tmp = Tempfile.new(datapath)
+      write(tmp)
+      tmp.rewind
+      tmp
+    end
+
+    def unzip
+      @download_contents ||=
+        begin
+          tmp = write_tmp
+          contents = nil
+          Zip::File.open(tmp.path) do |zipfile|
+            contents = zipfile.first.get_input_stream.read
+          end
+          contents
+        end
+    end
+
+    def write_csv(io)
+      unzip
+      io.write download_contents
+    end
+
+    def parse(opts = {})
+      opts = { headers: true, header_converters: :symbol }
+      CSV.parse(unzip, opts.merge(opts || {}))
+    end
+
+    def do_download
+      Enigma.logger.info "Trying to download #{download_url}"
+      success = false
+      until success
+        req = Typhoeus::Request.new(download_url)
+        req.on_complete do |response|
+          if response.response_code == 404
+            sleep DELAY
+          else
+            success = true
+            return response.body
+          end
+        end
+        req.run
+      end
+    end
+  end
+end

data/lib/enigma/endpoint.rb

@@ -0,0 +1,108 @@
+# coding: utf-8
+module Enigma
+  # Generic Enigma endpoint. Knows how to construct a URL, request it,
+  # and wrap the response in an `Enigma::Response`
+  #
+  # Assumes the api endpoints for its descendants are a lowercase
+  # version of their class names
+  #
+  # Handles some nice conversion of select, search, and where clauses
+  # from ruby hashes to string parameters
+  #
+  class Endpoint
+    attr_accessor :params, :datapath, :url
+
+    def self.descendants
+      ObjectSpace.each_object(Class).select { |klass| klass < self }
+    end
+
+    def initialize(datapath, opts = {})
+      self.datapath = datapath
+      self.params = opts
+    end
+
+    def self.url_chunk
+      to_s.gsub(/.*::/, '').downcase
+    end
+
+    def params
+      @params[:where] = serialize_where(@params[:where]) if @params[:where]
+      @params[:search] = serialize_search(@params[:search]) if @params[:search]
+      @params[:select] = serialize_select(@params[:select]) if @params[:select]
+      @params
+    end
+
+    # Serialize a where clause. Allows you to pass in a hash and have
+    # it converted to an equality where
+    #
+    # >  Filter results with a SQL-style "where" clause. Only applies to
+    # >  numerical columns - use the "search" parameter for strings. Valid
+    # >  operators are >, < and =. Only one "where" clause per request is
+    # >  currently supported.
+    #
+    # @param [String|Hash] where clause to convert
+    # @return [String] parameter ready for the request
+    #
+    def serialize_where(where)
+      if where.is_a? Hash
+        column, value = where.first
+        "#{column}=#{value}"
+      else
+        where
+      end
+    end
+
+    # Serialize a search clause. Allows you to pass in a hash of
+    # one or more fieldName: value pairs
+    #
+    # @param [String|Hash] search clause to convert
+    # @return [String] parameter ready for the request
+    #
+    def serialize_search(search)
+      if search.is_a? Hash
+        search.map do |field, value|
+          value = [value].flatten.join('|')
+          "@#{field} (#{value})"
+        end.join ' '
+      else
+        search
+      end
+    end
+
+    # Serialize a search clause. Allows you to pass in an array of
+    # column names
+    #
+    # @param [String|Array] select clause to convert
+    # @return [String] parameter ready for the request
+    #
+    def serialize_select(select)
+      if select.is_a? Enumerable
+        select.join(',')
+      else
+        select
+      end
+    end
+
+    # Endpoints show up in urls as a lowercase version of their class
+    # names
+    def url_chunk
+      self.class.url_chunk
+    end
+
+    def path
+      [Enigma.api_version, url_chunk, Enigma.key, datapath].join('/')
+    end
+
+    def url
+      URI.join(Enigma.root_url, path).to_s
+    end
+
+    def request
+      Enigma.logger.info "Making request to #{url}"
+      req = Typhoeus::Request.new(url, method: :get, params: params).run
+      Response.parse(req)
+    end
+  end
+end
+
+Dir[File.dirname(__FILE__) + '/endpoints/*.rb'].each { |file|  require(file) }

data/lib/enigma/endpoints/data.rb

@@ -0,0 +1,5 @@
+# coding: utf-8
+module Enigma
+  class Data < Endpoint
+  end
+end

data/lib/enigma/endpoints/export.rb

@@ -0,0 +1,22 @@
+# coding: utf-8
+module Enigma
+  # The export endpoint has no filtering, but it returns a URL where
+  # the download will eventually be available. This client will
+  # default to waiting for the file to be available before continuing
+  #
+  class Export < Endpoint
+    attr_accessor :download, :unzip
+
+    # The API request responds with a URL to poll until it's
+    # ready. Create a new download object with that URL and return it
+    #
+    # @return [Download]
+    #
+    def request
+      req = Typhoeus::Request.new(url, method: :get, params: params).run
+      Enigma.logger.info req.body
+      res = Response.parse(req)
+      Download.new(res)
+    end
+  end
+end

data/lib/enigma/endpoints/meta.rb

@@ -0,0 +1,5 @@
+# coding: utf-8
+module Enigma
+  class Meta < Endpoint
+  end
+end

data/lib/enigma/endpoints/stats.rb

@@ -0,0 +1,5 @@
+# coding: utf-8
+module Enigma
+  class Stats < Endpoint
+  end
+end

data/lib/enigma/response.rb

@@ -0,0 +1,17 @@
+# coding: utf-8
+
+module Enigma
+  # We'll wrap the response in a Hashie::Mash subclass so that you can
+  # access the attributes as function calls
+  #
+  # Raw response is kept in the `raw` attribute
+  #
+  class Response
+    def self.parse(res)
+      mash = Hashie::Mash.new(JSON.parse(res.body))
+      mash.raw = res
+      fail mash.message.to_s if mash.info && mash.info.error
+      mash
+    end
+  end
+end

data/lib/enigma/version.rb

@@ -0,0 +1,6 @@
+# coding: utf-8
+
+# Enigma version
+module Enigma
+  VERSION = '0.0.1'
+end

data/lib/enigma.rb

@@ -0,0 +1,49 @@
+# coding: utf-8
+require 'rubygems'
+
+# Required gems
+require 'hashie'
+require 'typhoeus'
+
+# Core dependencies
+require 'net/https'
+require 'uri'
+require 'json'
+require 'zip'
+require 'csv'
+
+# Library
+require 'enigma/version'
+require 'enigma/download'
+require 'enigma/endpoint'
+require 'enigma/response'
+require 'enigma/client'
+
+# Access to the engima API
+module Enigma
+  attr_accessor :key, :root_url, :api_version
+
+  def self.root_url
+    'https://api.enigma.io/'
+  end
+
+  def self.api_version
+    'v2'
+  end
+
+  def self.key
+    @key
+  end
+  def self.key=(k)
+    @key = k
+  end
+
+  def self.logger
+    @logger ||=
+      begin
+        logger = Logger.new(STDOUT)
+        logger.level = Logger::INFO
+        logger
+      end
+  end
+end