chimps 0.1.0

data/lib/chimps/commands/query.rb

@@ -0,0 +1,59 @@
+module Chimps
+  module Commands
+
+    # A command to issue a GET request against the Infochimps paid
+    # query API.
+    class Query < Chimps::Command
+
+      BANNER = "usage: chimps query [OPTIONS] DATASET [PROP=VALUE] ..."
+      HELP   = <<EOF
+
+Make a query of the given DATASET on the Infochimps paid query API
+(not the main Infochimps site).
+
+Properties and values can be supplied directly on the command line,
+from an input YAML file, or multiple YAML documents streamed in via
+STDIN, in order of decreasing precedence.
+
+You can learn more about the Infochimps query API, discover datasets
+to query, and look up the available parameters at
+
+  http://api.infochimps.com
+
+You can learn about the main Infochimps site API at
+
+  http://infochimps.org/api
+EOF
+
+      include Chimps::Utils::UsesYamlData
+      IGNORE_YAML_FILES_ON_COMMAND_LINE = true # must come after include
+
+      # The dataset to query.
+      #
+      # @return [String]
+      def dataset
+        raise CLIError.new("Must provide a dataset to query.") if argv.first.blank?
+        argv.first
+      end
+
+      # The path on the Infochimps query API to query.
+      #
+      # @return [String]
+      def path
+        dataset + ".json"
+      end
+      
+      # Issue the GET request.
+      def execute!
+        response = QueryRequest.new(path, :query_params => data, :authenticate => true).get
+        if response.error?
+          response.print
+        else
+          puts response.inspect
+        end
+      end
+      
+    end
+  end
+end
+

data/lib/chimps/commands/search.rb

@@ -0,0 +1,59 @@
+module Chimps
+  module Commands
+
+    # A command to issue a GET request to create a search at
+    # Infochimps.
+    class Search < Chimps::Command
+
+      # Default number of search results returned.
+      # DEFAULT_LIMIT  = 20
+      
+      BANNER = "usage: chimps search [OPTIONS] QUERY"
+      HELP   = <<EOF
+
+Perform a search on Infochimps.  By default the search will be of
+datasets and will return all matches for the given QUERY.
+EOF
+      
+      # Path to search resource
+      PATH   = 'search.json'
+      
+      # Models this command applies to (default first)
+      MODELS = %w[dataset collection source license]
+      include Chimps::Utils::UsesModel
+
+      # FIXME have to implement this on the server side.
+      # def limit
+      #   @limit ||= DEFAULT_LIMIT
+      # end
+
+      def define_options
+        # on_tail("-n", "--num-results NUM", "Return the given number of results instead of the default #{DEFAULT_LIMIT}") do |n|
+        #   @limit = n.to_i
+        # end
+
+        on_tail("-s", "--[no-]skip-column-names", "don't print column names in output.") do |s|
+          @skip_column_names = s
+        end
+        
+      end
+
+      def query
+        raise CLIError.new("Must provide a query to search for") if argv.blank?
+        argv.join(' ')
+      end
+
+      def params
+        {
+          :query => query,
+          :model => model
+        }
+      end
+
+      def execute!
+        Chimps::Request.new(PATH, :params => params).get.print(:skip_column_names => @skip_column_names)
+      end
+    end
+  end
+end
+

data/lib/chimps/commands/show.rb

@@ -0,0 +1,32 @@
+module Chimps
+  module Commands
+
+    class Show < Chimps::Command
+
+      BANNER = "usage: chimps show [OPTIONS] ID_OR_HANDLE"
+      HELP   = <<EOF
+
+Return a description of the resource (defaults to dataset) with the
+given ID or HANDLE
+EOF
+
+      # Models this command applies to (default first)
+      MODELS = %w[dataset collection source license tag category]
+      include Chimps::Utils::UsesModel
+
+      # The path of the URL to send a Request to.
+      #
+      # This is different from Chimps::Commands::UsesModel in that it
+      # submits to the YAML path.
+      def model_path
+        "#{plural_model}/#{model_identifier}.yaml"
+      end
+
+      def execute!
+        puts Chimps::Request.new(model_path).get.body
+      end
+
+    end
+  end
+end
+

data/lib/chimps/commands/test.rb

@@ -0,0 +1,40 @@
+module Chimps
+  module Commands
+
+    # A command to test whether API authentication with Infochimps is
+    # working.
+    class Test < Chimps::Command
+
+      BANNER = "usage: chimps test"
+      HELP = <<EOF
+
+Print diagnostic information on the API credentials being used by chimps
+and send a test request to Infochimps to make sure the API credentials
+work.
+
+EOF
+
+      # Path to submit test requests to.
+      def path
+        "api_accounts/#{Chimps::CONFIG[:site][:key]}"
+      end
+
+      # Issue the request.
+      def execute!
+        puts "Reading identity file at #{CONFIG[:identity_file]}" if Chimps.verbose?
+        response = Chimps::Request.new(path, :sign => true).get
+        if response.error?
+          case 
+          when response.code == 404 then puts "ERROR Unrecognized API key" # record not found
+          when response.code == 401 then puts "ERROR Signature does not match API key and query.  Is your secret key correct?" # unauthorized
+          else
+            nil                 # response gets printed anyway
+          end
+        end
+        response.print
+      end
+
+    end
+  end
+end
+

data/lib/chimps/commands/update.rb

@@ -0,0 +1,33 @@
+module Chimps
+  module Commands
+
+    # A command to issue a PUT request to update a resource at
+    # Infochimps.
+    class Update < Chimps::Command
+
+      BANNER = "usage: chimps update [OPTIONS] ID_OR_HANDLE [PROP=VALUE] ..."
+      HELP   = <<EOF
+
+Updates a single resource of a given type (defaults to dataset)
+identified by ID_OR_HANDLE using the properties and values supplied.
+
+Properties and values can be supplied directly on the command line,
+from an input YAML file, or multiple YAML documents streamed in via
+STDIN, in order of decreasing precedence.
+EOF
+
+      # Models this command applies to (default first)
+      MODELS = %w[dataset source license]
+      include Chimps::Utils::UsesModel
+      include Chimps::Utils::UsesYamlData
+
+      # Issue the PUT request.
+      def execute!
+        ensure_data_is_present!
+        Request.new(model_path, :data => {model.to_sym => data } , :authenticate => true).put.print
+      end
+      
+    end
+  end
+end
+

data/lib/chimps/commands/upload.rb

@@ -0,0 +1,63 @@
+module Chimps
+  module Commands
+
+    # A command for uploading data to Infochimps.
+    class Upload < Chimps::Command
+
+      BANNER = "usage: chimps upload [OPTIONS] ID_OR_HANDLE PATH [PATH] ..."
+      HELP   = <<EOF
+
+Upload data from your local machine for an existing dataset identified
+by ID_OR_HANDLE on Infochimps.
+
+chimps will package all paths supplied into a local archive and then
+upload this archive to Infochimps.  The local archive defaults to a
+sensible name in the current directory but can also be customized.
+
+If the only file to be packaged is already a package (.zip, .tar,
+.tar.gz, &.c) then it will not be packaged again.
+EOF
+
+      # The path to the archive
+      attr_reader :archive
+
+      # The data format to annotate the upload with.
+      #
+      # Chimps will try to guess if this isn't given.
+      attr_reader :fmt
+
+      # The ID or handle of the dataset to upload data for.
+      #
+      # @return [String]
+      def dataset
+        raise CLIError.new("Must provide an ID or URL-escaped handle as the first argument") if argv.first.blank?
+        argv.first
+      end
+
+      # A list of local paths to upload.
+      #
+      # @return [Array<String>]
+      def local_paths
+        raise CLIError.new("Must provide some paths to upload") if argv.length < 2
+        argv[1..-1]
+      end
+      
+      def define_upload_options
+        on_tail("-a", "--archive-path", "Path to the archive to be created.  Defaults to a timestamped ZIP file named after the dataset in the current directory.") do |path|
+          @archive = path
+        end
+
+        on_tail("-f", "--format FORMAT", "Data format to annotate upload with.  Tries to guess if not given.") do |f|
+          @fmt = f
+        end
+          
+      end
+
+      # Upload the data.
+      def execute!
+        Chimps::Workflows::Uploader.new(:dataset => dataset, :archive => archive, :local_paths => local_paths, :fmt => fmt).execute!
+      end
+    end
+  end
+end
+

data/lib/chimps/commands.rb

@@ -0,0 +1,46 @@
+module Chimps
+
+  # A namespace to hold the various commands Chimps defines.
+  module Commands
+
+    # Construct a new command from the given +command_name+ and +argv.
+    # The resulting command will be initialized but will not have been
+    # executed.
+    #
+    # @param [String] command_name
+    # @param [Array<String>] argv
+    # @return [Chimps::Command]
+    def self.construct command_name, argv
+      self.constants.each do |constant_name|
+        return "Chimps::Commands::#{constant_name}".constantize.new(argv) if constant_name.downcase == command_name
+      end
+      raise CLIError.new("Invalid command: #{command_name}.  Try running `chimps help'")
+    end
+
+    # Construct a new command from the given +command_name+ and
+    # +argv+.
+    #
+    # Delegates to Chimps::Commands.construct, so see its
+    # documentation for more information.
+    def construct command_name, argv
+      Chimps::Commands.construct command_name, argv
+    end
+
+    # A list of all the commmand names defined by Chimps.  Each name
+    # maps to a corresponding subclass of Chimps::Command living in
+    # the Chimps::Commands module.
+    NAMES = %w[search help test create show update destroy upload list download batch query]
+    
+    NAMES.each do |name|
+      autoload name.capitalize.to_sym, "chimps/commands/#{name}"
+    end
+
+    # Is +name+ a Chimps command name?
+    #
+    # @param [String] name
+    # @return [true, false]
+    def command_name? name
+      NAMES.include?(name)
+    end
+  end
+end

data/lib/chimps/config.rb

@@ -0,0 +1,57 @@
+module Chimps
+
+  # Default configuration for Chimps.  User-specific configuration
+  # usually lives in a YAML file <tt>~/.chimps</tt>.
+  CONFIG = {
+    :query => {
+      :host => 'http://api.infochimps.com'
+    },
+    :site => {
+      :host => 'http://infochimps.org'
+    },
+    :identity_file    => File.expand_path(ENV["CHIMPS_RC"] || "~/.chimps"),
+    :verbose          => nil,
+    :timestamp_format => "%Y-%m-%d_%H-%M-%S"
+  }
+
+  # Is Chimps in verbose mode?
+  #
+  # @return [true, false]
+  def self.verbose?
+    CONFIG[:verbose]
+  end
+
+  # The username Chimps will pass to Infochimps.
+  #
+  # @return [String]
+  def self.username
+    CONFIG[:site][:username] or raise AuthenticationError.new("No site username set in #{Chimps::CONFIG[:identity_file]}")
+  end
+
+  # Defines methods to load the Chimps configuration.
+  module Config
+
+    # The root of the Chimps source base.
+    #
+    # @return [String]
+    def self.chimps_root
+      File.expand_path File.join(File.dirname(__FILE__), '../..')
+    end
+
+    # Load the configuration settings from the configuration/identity
+    # file.
+    def self.load
+      # FIXME this is a terrible hack...and it only goes to 2 deep!
+      if File.exist?(CONFIG[:identity_file])
+        require 'yaml'
+        YAML.load_file(CONFIG[:identity_file]).each_pair do |key, value|
+          if value.is_a?(Hash) && CONFIG.include?(key)
+            CONFIG[key].merge!(value)
+          else
+            CONFIG[key] = value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chimps/request.rb

@@ -0,0 +1,302 @@
+require 'restclient'
+
+module Chimps
+
+  # A class to encapsulate requests made of Infochimps.
+  #
+  # Essentialy a wrapper for RestClient::Resource with added
+  # funcionality for automatically signing requests and parsing
+  # Infochimps API responses.
+  class Request < RestClient::Resource
+
+    # Default headers to pass with every request.
+    DEFAULT_HEADERS = { :content_type => 'application/json', :accept => 'application/json' }
+
+    # Path of the URL to submit to.  Must be a String.
+    attr_accessor :path
+
+    # Parameters to include in the query string of the URL to submit
+    # to.  Must be a Hash.
+    attr_accessor :query_params
+
+    # Data to include in the body of the request.  Must be a Hash.
+    attr_accessor :data
+
+    # Initialize a Request to the given +path+.
+    #
+    # Query parameters and data can be passed in as hashes named
+    # <tt>:params</tt> and <tt>:data</tt>, respectively.
+    #
+    # If <tt>:sign</tt> is passed in the +options+ then the URL of
+    # this request will be signed with the Chimps user's Infochimps
+    # API key and secret.  Failure to properly sign will raise an
+    # error.
+    #
+    # If <tt>:sign_if_possible</tt> is passed in the +options+ then an
+    # attemp to sign the URL will be made though an error will _not_
+    # raise an error.
+    #
+    # @param [String] path
+    # @param [Hash] options
+    # @option options [Hash] params Query parameters to include in the URL
+    # @option options [Hash] data Data to include in the request body
+    # @option options [true, false] sign Sign this request, raising an error on failure
+    # @option options [true, false] sign_if_possible Sign this request, no error on failure
+    # @return [Chimps::Request]
+    def initialize path, options={}
+      @path         = path
+      @query_params = options[:query_params] || options[:params] || {}
+      @data         = options[:data]         || {}
+      @authentication_required      = [:authenticate, :authenticated, :authenticate_if_possible, :sign, :signed, :sign_if_possible].any? { |key| options.include?(key) }
+      @forgive_authentication_error = options[:sign_if_possible] || options[:authenticate_if_possible]
+      authenticate_if_necessary!
+      super url_with_query_string
+    end
+
+    # Should the request be authenticated?
+    #
+    # @return [true, false]
+    def authenticate?
+      @authentication_required
+    end
+    alias_method :sign?, :authenticate?
+
+    # Is this request authentiable (has the Chimps user specified an
+    # API key and secret in their configuration file)?
+    #
+    # @return [true, false]
+    def authenticable?
+      !Chimps::CONFIG[:site][:key].blank? && !Chimps::CONFIG[:site][:secret].blank?
+    end
+    alias_method :signable?, :authenticable?
+
+    # The host to send requests to.
+    #
+    # @return [String]
+    def host
+      @host ||= ENV["CHIMPS_HOST"] || Chimps::CONFIG[:site][:host]
+    end
+
+    # Return the URL for this request with the (signed, if necessary)
+    # query string appended.
+    #
+    # @return [String]
+    def url_with_query_string
+      base_url = File.join(host, path)
+      base_url += "?#{query_string}" unless query_string.blank?
+      base_url
+    end
+
+    # Return the query string for this request, signed if necessary.
+    #
+    # @return [String]
+    def query_string
+      (authenticate? && authenticable?) ? signed_query_string : unsigned_query_string
+    end
+
+    # Perform a GET request to this URL, returning a parsed response.
+    #
+    # Any headers in +options+ will passed to
+    # RestClient::Resource.get.
+    #
+    # @param [Hash] options
+    # @return [Chimps::Response]
+    def get options={}
+      handle_exceptions do
+        puts "GET #{url}" if Chimps.verbose?
+        Response.new(super(DEFAULT_HEADERS.merge(options)))
+      end
+    end
+
+    # Perform a POST request to this URL, returning a parsed response.
+    #
+    # Any headers in +options+ will passed to
+    # RestClient::Resource.post.
+    #
+    # @param [Hash] options
+    # @return [Chimps::Response]
+    def post options={}
+      handle_exceptions do
+        puts "POST #{url}" if Chimps.verbose?
+        Response.new(super(data_text, DEFAULT_HEADERS.merge(options)))
+      end
+    end
+
+    # Perform a PUT request to this URL, returning a parsed response.
+    #
+    # Any headers in +options+ will passed to
+    # RestClient::Resource.put.
+    #
+    # @param [Hash] options
+    # @return [Chimps::Response]
+    def put options={}
+      handle_exceptions do
+        puts "PUT #{url}" if Chimps.verbose?
+        Response.new(super(data_text, DEFAULT_HEADERS.merge(options)))
+      end
+    end
+
+    # Perform a DELETE request to this URL, returning a parsed
+    # response.
+    #
+    # Any headers in +options+ will passed to
+    # RestClient::Resource.delete.
+    #
+    # @param [Hash] options
+    # @return [Chimps::Response]
+    def delete options={}
+      handle_exceptions do
+        puts "DELETE #{url}" if Chimps.verbose?
+        Response.new(super(DEFAULT_HEADERS.merge(options)))
+      end
+    end
+    
+    protected
+    # Yield to +block+ but rescue any RestClient errors by wrapping
+    # them in a Chimps::Response.
+    def handle_exceptions &block
+      begin
+        yield
+      rescue RestClient::Exception => e
+        return Response.new(e.response, :error => e.message)
+      end
+    end
+
+    # Authenticate this request by stuffing the <tt>:requested_at</tt>
+    # and <tt>:api_key</tt> properties into its <tt>:query_params</tt>
+    # hash.
+    #
+    # Will do nothing at all if Chimps::Request#authenticate? returns
+    # false.
+    def authenticate_if_necessary!
+      return unless authenticate?
+      raise Chimps::AuthenticationError.new("API key or secret missing from #{CONFIG[:identity_file]}") unless (authenticable? || @forgive_authentication_error)
+      query_params[:requested_at] = Time.now.to_i.to_s
+      query_params[:api_key]      = Chimps::CONFIG[:site][:key]
+    end
+
+    # Return the sorted keys of the query params.
+    #
+    # @return [Array]
+    def alphabetical_params
+      query_params.keys.map(&:to_s).sort
+    end
+
+    # Return an unsigned query string for this request.
+    #
+    # Query parameters will be used in alphabetical order.
+    #
+    # @return [String]
+    def unsigned_query_string
+      # alphabetical_params.map { |key| "#{CGI::escape(key.to_s)}=#{CGI::escape(query_params[key.to_sym].to_s)}" }.join("&") # doesn't flatten nested hashes properly
+      RestClient::Payload.generate(query_params)
+    end
+
+    # Return an unsigned query string for this request without the
+    # <tt>&</tt> and <tt>=</tt> characters.
+    #
+    # This is the text that will be signed for GET and DELETE
+    # requests.
+    #
+    # @return [String]
+    def unsigned_query_string_stripped
+      require 'cgi'      
+      @query_params_text ||= alphabetical_params.map { |key| CGI::escape(key.to_s) + CGI::escape(query_params[key.to_sym].to_s) }.join('')
+    end
+
+    # Return the data of this request as a string.
+    #
+    # This is the text that will be signed for POST and PUT requests.
+    #
+    # @return [String]
+    def data_text
+      require 'json'
+      @data_text ||= data.to_json
+    end
+
+    # Sign +string+ by concatenting it with the secret and computing
+    # the MD5 digest of the whole thing.
+    #
+    # @param [String]
+    # @return [String]
+    def sign string
+      raise Chimps::AuthenticationError.new("No API secret stored in #{CONFIG[:identity_file]}.") unless (authenticable? || @forgive_authentication_error)
+      require 'digest/md5'
+      Digest::MD5.hexdigest(string + CONFIG[:site][:secret])
+    end
+
+    # Append the signature to the unsigned query string.
+    #
+    # The signature made from the Chimps user's API secret and either
+    # the query string text (stripped of <tt>&</tt> and <tt>=</tt>)
+    # for GET and DELETE requests or the request body for POST and PUT
+    # requests.
+    #
+    # @return [String]
+    def signed_query_string
+      signature = sign(data.blank? ? unsigned_query_string_stripped : data_text)
+      "#{unsigned_query_string}&signature=#{signature}"
+    end
+
+  end
+
+  # A class to encapsulate requests made against the Infochimps paid
+  # query API.
+  class QueryRequest < Request
+
+    # Is this request authentiable (has the Chimps user specified an
+    # API key and secret in their configuration file)?
+    #
+    # @return [true, false]
+    def authenticable?
+      !Chimps::CONFIG[:query][:key].blank? && !Chimps::CONFIG[:query][:secret].blank?
+    end
+
+    # The host to send requests to.
+    #
+    # @return [String]
+    def host
+      @host ||= ENV["CHIMPS_QUERY_HOST"] || Chimps::CONFIG[:query][:host]
+    end
+
+    # Authenticate this request by stuffing the <tt>:requested_at</tt>
+    # and <tt>:api_key</tt> properties into its <tt>:query_params</tt>
+    # hash.
+    #
+    # Will do nothing at all if Chimps::Request#authenticate? returns
+    # false.
+    def authenticate_if_necessary!
+      return unless authenticate?
+      raise Chimps::AuthenticationError.new("API key or secret missing from #{CONFIG[:identity_file]}") unless (authenticable? || @forgive_authentication_error)
+      query_params[:requested_at] = Time.now.to_i.to_s
+      query_params[:apikey]       = Chimps::CONFIG[:query][:key]
+    end
+
+    # Sign +string+ by concatenting it with the secret and computing
+    # the MD5 digest of the whole thing.
+    #
+    # @param [String]
+    # @return [String]
+    def sign string
+      raise Chimps::AuthenticationError.new("No API secret stored in #{CONFIG[:identity_file]}.") unless (authenticable? || @forgive_authentication_error)
+      require 'digest/md5'
+      Digest::MD5.hexdigest(string + CONFIG[:key][:secret])
+    end
+
+    # Append the signature to the unsigned query string.
+    #
+    # The signature made from the Chimps user's API secret and either
+    # the query string text (stripped of <tt>&</tt> and <tt>=</tt>)
+    # for GET and DELETE requests or the request body for POST and PUT
+    # requests.
+    #
+    # @return [String]
+    def signed_query_string
+      unsigned_query_string
+    end
+    
+    
+    
+  end
+
+end