chimps 0.1.0

data/README.rdoc

@@ -0,0 +1,48 @@
+= Searching
+
+Search datasets
+
+  chimps search statisical abstract
+
+Search sources
+
+  chimps search -m source department of justice
+
+Search datasets with particular tags
+
+  chimps search -t government,finance statistical abstract
+
+or categories
+
+  chimps search -c education statistical abstract
+
+= Browsing
+
+  chimps describe dataset 3923
+  chimps describe source us-doj
+  chimps describe field length
+
+= Downloading
+
+  chimps download 39283
+
+= Creating
+
+  chimps create data.yaml
+
+also
+
+  chimps schema source
+  chimps schema dataset
+
+and of course
+
+  chimps upload 39283 path/to/my/data
+
+= General Options
+
+Work as someone other than the usual user
+
+  chimps -i path/to/my/identify_file.yml create data.yaml
+
+

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/bin/chimps

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require "chimps"
+require 'chimps/cli'
+Chimps::CLI.execute!(ARGV.dup) if $0 == __FILE__

data/examples/batch.yaml

@@ -0,0 +1,69 @@
+---
+
+# This is an example of the expected structure of a file used in a
+# bulk update.
+#
+# It consists of one large array itself consisting of mappings from a
+# resource name (dataset, license, source) to properties about that
+# resource.
+#
+# Many such documents can be combined together in a YAML stream and
+# sent as a single request.
+
+- source:
+    id: ~
+    request_id: foobar
+    title: Foobar the Fake Source
+    description: |
+      
+      Foobar generates great data!  Unfortunately, it's all of it
+      fake!
+
+- dataset:
+    # this dataset lacks an ID property so it will be created
+    title: expatriatoro
+    description: |
+      Just some description of my dataset
+
+    tag_list:
+      - foo
+      - bar
+      - baz
+
+    sources:
+      # must all exist
+      - Census Bureau
+      - foobar
+
+    fields:
+      - title: length
+        units: m
+
+      - title: mass
+        units: kg
+    local_paths:
+      - /tmp/waste/waste.c
+
+- dataset:
+    id: 1
+    title: f
+    description: |
+      Just some description of my dataset
+
+    tag_list:
+      - foo
+      - bar
+      - baz
+
+    sources:
+      # must all exist
+      - Census Bureau
+      - foobar
+
+    snippet:
+      columns: foo, bar,baz
+      data:
+        - [1,2,3]
+        - [4,5,6]
+        - [7,8,9]
+

data/lib/chimps/cli.rb

@@ -0,0 +1,102 @@
+module Chimps
+
+  # Defines methods for choosing which Chimps::Command class should be
+  # instantiated from the ARGV passed in on the command line.
+  module CLI
+
+    # Execute the Chimps command specified by +argv+.
+    #
+    # @param [Array<String>] argv
+    def self.execute! argv
+      begin
+        Runner.new(argv).execute!    
+      rescue Chimps::Error => e
+        puts e.message
+        exit 1
+      rescue => e
+        $stderr.puts("#{e.message} (#{e.class})")
+        $stderr.puts(e.backtrace.join("\n"))
+        exit 1
+      end
+    end
+
+    # Defines methods to parse the original ARGV and from it choose
+    # and instantiate the appropriate Chimps::Command subclass with
+    # the appropriate arguments.
+    class Runner
+      include Chimps::Commands
+
+      # The original ARGV passed in by the user.
+      attr_reader :argv
+
+      # Create a new Chimps::CLI::Runner from +argv+.
+      #
+      # @param [Array<String>] argv
+      # @return [Chimps::CLI::Runner]
+      def initialize argv
+        @argv = argv
+      end
+
+      # Execute this Runner's chosen and initialized command.
+      def execute!
+        command.execute!
+      end
+
+      # The chosen and initialized command for this Runner.
+      #
+      # @return [Chimps::Command]
+      def command
+        @command ||= construct(command_name, argv_for_command)
+      end
+
+      protected
+
+      # Return the index in ARGV of the command name to run.
+      #
+      # It may not always be the first element of ARGV because
+      #
+      #   chimps show my-dataset
+      #   chimps -v show my-dataset
+      #   chimps show -v my-dataset
+      #
+      # should all have the same behavior.
+      #
+      # @return [Integer] the index in ARGV of the command name.
+      def command_index
+        return @command_index if @command_index
+        argv.each_with_index do |arg, index|
+          if command_name?(arg)
+            @command_index = index
+            break
+          end
+        end
+        @command_index or raise CLIError.new("Must specify a command.  Try running `chimps help'")
+      end
+
+      # The name of the command to run.
+      #
+      # @return [String]
+      def command_name
+        @command_name ||= argv[command_index]
+      end
+
+      # The ARGV to pass to the command chosen to run.
+      #
+      # It differs from the original ARGV only in that the command's
+      # name has been stripped:
+      #
+      #   Chimps::CLI::Runner.new('show', '-v', 'my-dataset').argv_for_command
+      #   => ['-v','my-dataset']
+      #
+      # This does not always return "all but the first element" of
+      # ARGV; see Chimps::CLI::Runner#command_index for details.
+      def argv_for_command
+        returning(argv.dup) do |new_argv|
+          new_argv.delete_at(command_index)
+        end
+      end
+    end
+
+  end
+end
+

data/lib/chimps/commands/base.rb

@@ -0,0 +1,107 @@
+require 'optparse'
+
+module Chimps
+
+  # A base class from which to subclass specific commands.  A subclass
+  # should
+  #
+  # - define class constants <tt>BANNER</tt> and <tt>HELP</tt> which
+  # - will display the appropriate help to the user.
+  # 
+  # - add specific options by defining a method that begins with
+  #   +define+ and ends with +options+ (i.e. - +define_output_options+
+  #   to add options related to output).
+  #
+  # - define a method <tt>execute!</tt> which will actually run the
+  #   command.
+  class Command < OptionParser
+
+    # Appears when printing help for this command, as the very first
+    # line.  Should be one-line summary of how to use this command.
+    BANNER = "Define #{self}::BANNER when you subclass Chimps::Command"
+
+    # Appears when printing help for this command.  Should consist of
+    # general help or examples of the command iteslf.  Help on
+    # specific options is automatcally generated.
+    HELP   = "Define #{self}::HELP when you subclass Chimps::Command"
+
+    # The (processed) ARGV for this command.
+    attr_reader :argv
+
+    # Create a new command.  Will define options specific to
+    # subclases, parse the given +argv+, and load the global Chimps
+    # configuration.  Will _not_ execute the command.
+    #
+    # @param [Array<String>] argv
+    # @return [Chimps::Command]
+    def initialize argv
+      super self.class::BANNER
+      @argv = argv
+      run_options_definers
+      parse_command_line!
+      Chimps::Config.load
+    end
+
+    # The name of this command, including the
+    # <tt>Chimps::Commands</tt> prefix.
+    #
+    # @return [String]
+    def self.name
+      self.to_s.downcase
+    end
+
+    # The name of this command, excluding the
+    # <tt>Chimps::Commands</tt> prefix.
+    #
+    # @return [String]
+    def name
+      self.class.name.split('::').last
+    end
+    
+    protected
+    
+    # Parse the command line.
+    def parse_command_line!
+      begin
+        parse!(argv)
+      rescue OptionParser::InvalidOption => e
+        raise CLIError.new("#{e.message}.  Try `chimps help #{name}'")
+      end
+    end
+      
+    # Run all methods beginning with +define+ and ending with +option+
+    # or +options+.
+    #
+    # This is (hackish) mechanism for subclasses of Chimps::Command to
+    # define their own specific options.
+    def run_options_definers
+      # FIXME there's a better way to do this...      
+      methods.grep(/^define.+options?$/).each { |method| send method }
+    end
+
+    # Define options common to all Chimps' commands.  The two only two
+    # such options at the moment are <tt>-v</tt> (or
+    # <tt>--[no-]verbose</tt>) for verbosity, and <tt>-i</tt> (or
+    # <tt>--identity-file</tt>) for setting the identify file to use.
+    def define_common_options
+      separator self.class::HELP
+      separator "\nOptions include:"
+      
+      on("-v", "--[no-]verbose", "Be verbose, or not.") do |v|
+        Chimps::CONFIG[:verbose] = v
+      end
+      
+      on("-i", "--identity-file PATH", "Use the given YAML identify file to authenticate with Infochimps instead of the default (~/.chimps) ") do |i|
+        Chimps::CONFIG[:identity_file] = File.expand_path(i)
+      end
+    end
+
+    # Run this command.
+    #
+    # Will raise a NotImplementedError for Chimps::Command itself --
+    # subclasses are expected to redefine this method.
+    def execute!
+      raise NotImplementedError.new("Redefine the `execute!' method in a subclass of #{self.class}.")
+    end
+  end
+end

data/lib/chimps/commands/batch.rb

@@ -0,0 +1,68 @@
+module Chimps
+  module Commands
+
+    # A command for performing batch updates.
+    class Batch < Chimps::Command
+
+      BANNER = "usage: chimps batch [OPTIONS] [INPUT_PATH] ..."
+      HELP   = <<EOF
+
+Perform a batch operation on Infochimps by reading YAML input files.
+
+The input files should collectively define an array of resources to
+make create or update requests on.  Each request in the array is
+treated separately (even though the entire array is processed as one
+POST request) and so it is possible that some will succeed and others
+fail.
+
+It is also possible to upload data in this batch process.  Each
+(successful) request which defined a 'local_paths' property in the
+original input files will have the data at these paths uploaded to
+Infochimps.  These uploads will proceed one at a time following the
+initial batch POST request.
+
+The format of the YAML input files is given at
+
+  http://infochimps.org/api
+EOF
+
+      # A path to store the intermediate batch response.  Useful for
+      # debugging.
+      attr_accessor :output_path
+
+      # The data format to annotate the upload with.
+      #
+      # Chimps will try to guess if this isn't given.
+      attr_reader :fmt
+
+      # Whether to continue to upload even if some of the resources
+      # had errors on update/create.
+      attr_accessor :upload_even_if_errors
+
+      include Chimps::Utils::UsesYamlData
+
+      def define_options
+        on_tail("-o", "--output PATH", "Store the response from the server at PATH") do |o|
+          @output_path = File.expand_path(o)
+        end
+
+        on_tail("-e", "--force", "Attempt to upload data even when there were errors in the batch update request") do |e|
+          @upload_even_if_errors = e
+        end
+
+        on_tail("-f", "--format FORMAT", "Data format to annotate EACH upload with.  Tries to guess if not given.") do |f|
+          @fmt = f
+        end
+        
+      end
+
+      # Perform the batch update and upload.
+      def execute!
+        ensure_data_is_present!
+        Chimps::Workflows::BatchUpdater.new(data, :output_path => output_path, :upload_even_if_errors => upload_even_if_errors, :fmt => fmt).execute!
+      end
+      
+    end
+  end
+end
+

data/lib/chimps/commands/create.rb

@@ -0,0 +1,33 @@
+module Chimps
+  module Commands
+
+    # A command to issue a POST requst to create a resource at
+    # Infochimps.
+    class Create < Chimps::Command
+
+      BANNER = "usage: chimps create [OPTIONS] [PROP=VALUE] ..."
+      HELP   = <<EOF
+
+Create a single resource (defaults to a dataset) using the properties
+and values supplied.
+
+Properties and values can be supplied directly on the command line,
+from an input YAML file, or multiple YAML documents streamed in via
+STDIN, in order of decreasing precedence.
+EOF
+
+      # Models this command applies to (default first)
+      MODELS = %w[dataset source license]
+      include Chimps::Utils::UsesModel
+      include Chimps::Utils::UsesYamlData
+
+      # Issue the POST request.
+      def execute!
+        ensure_data_is_present!
+        Request.new(models_path, :data => {model.to_sym => data } , :authenticate => true).post.print
+      end
+      
+    end
+  end
+end
+

data/lib/chimps/commands/destroy.rb

@@ -0,0 +1,28 @@
+module Chimps
+  module Commands
+
+    # A command to issue a DELETE request against a resource at
+    # Infochimps.
+    class Destroy < Chimps::Command
+
+      BANNER = "usage: chimps destroy [OPTIONS] ID_OR_HANDLE"
+      HELP   = <<EOF
+
+Destroys a resource of a given type (defaults to dataset) identified
+by ID_OR_HANDLE.
+
+EOF
+
+      # Models this command applies to (default first)
+      MODELS = %w[dataset package source license]
+      include Chimps::Utils::UsesModel
+
+      # Issue the DELETE request.
+      def execute!
+        Request.new(model_path, :authenticate => true).delete.print
+      end
+      
+    end
+  end
+end
+

data/lib/chimps/commands/download.rb

@@ -0,0 +1,76 @@
+module Chimps
+  module Commands
+
+    # A command to download data from Infochimps.
+    class Download < Chimps::Command
+
+      BANNER = "usage: chimps download [OPTIONS] ID_OR_HANDLE"
+      HELP   = <<EOF
+
+Download a dataset identified by the given ID_OR_HANDLE to the current
+directory (you can also specify a particular path).
+
+If the dataset isn't freely downloadable, you'll have to have
+purchased it first via the Web.
+EOF
+
+      # The chosen data format.
+      attr_reader :fmt
+
+      # The chosen package format.
+      attr_reader :pkg_fmt
+
+      # The local path to download the data to.
+      attr_reader :local_path
+      
+      # Set the format for the download token.
+      #
+      # Will try to normalize the input somewhat (downcasing,
+      # stripping leading periods)
+      #
+      # @param [String] new_fmt
+      def fmt= new_fmt
+        @fmt = new_fmt.downcase.strip.gsub(/^\./, '')
+      end
+
+      # Set the package format for the download token.
+      # 
+      # Will try to normalize the input somewhat (downcasing,
+      # stripping leading periods)
+      #
+      # @param [String] new_pkg_fmt
+      def pkg_fmt= new_pkg_fmt
+        @pkg_fmt = new_pkg_fmt.downcase.strip.gsub(/^\./, '')
+      end
+      
+      # The ID of the package to download.
+      def dataset
+        raise CLIError.new("Must provide an ID or handle of a dataset to download.") if argv.first.blank?
+        argv.first
+      end
+
+      # Issue the request for the token and the request for the
+      # download.
+      def execute!
+        Chimps::Workflows::Downloader.new(:dataset => dataset, :fmt => fmt, :pkg_fmt => pkg_fmt, :local_path => local_path).execute!
+      end
+
+      def define_options
+        on_tail("-o", "--output PATH", "Path to download file to") do |o|
+          @local_path = File.expand_path(o)
+        end
+
+        on_tail("-f", "--format FORMAT", "Choose a particular data format (csv, tsv, excel, &c.)") do |f|
+          self.fmt = f
+        end
+
+        on_tail("-p", "--package PACKAGE", "Choose a particular package type (zip or tar.bz2)") do |p|
+          self.pkg_fmt = p
+        end
+        
+      end
+      
+    end
+  end
+end
+

data/lib/chimps/commands/help.rb

@@ -0,0 +1,89 @@
+module Chimps
+  module Commands
+    class Help < Chimps::Command
+
+      BANNER = "usage: chimps help [COMMAND]"
+      HELP = <<EOF
+
+This is the Infochimps command-line client.  You can use it to search,
+browse, create, edit, or delete data and metadata in the Infochimps
+repository at http://infochimps.org.
+
+Before you can create, edit, or delete anything you'll need to get an
+Infochimps account and sign up for an API key:
+
+  http://infochimps.org/signup
+
+But you can still browse, search, and download (free) data
+immediately.
+
+Learn more about the Infochimps API which powers this tool at
+
+  http://infochimps.org/api
+
+= Commands
+
+chimps is a wrapper over the RESTful Infochimps API.  It exposes the
+following actions
+
+  chimps list
+  chimps show
+  chimps create
+  chimps update
+  chimps destroy
+
+for datasets (as well as other selected resources).  It also helps
+automate the workflow of uploading and downloading data and making
+batch changes with
+
+  chimps upload
+  chimps download
+  chimps batch
+
+You can also make queries against the Infochimps paid query API with
+
+  chimps query
+
+Finally, you can test that your system is configured properly and that
+you can authenticate with Infochimps with
+
+  chimps test
+
+If you're confused try running
+
+  chimps help COMMAND
+
+for any of the commands above.
+
+= Setup
+
+Once you have obtained an API key and secret from Infochimps, place them
+in a file Chimps::CONFIG[:identity_file] in your home directory with the
+following format
+
+  ---
+  # API credentials for use on the main Infochimps site
+  :site:
+    :username: your_site_name
+    :key: oreeph6giedaeL3
+    :secret: Queechei6cu8chiuyiig8cheg5Ahx0boolaizi1ohtarooFu1doo5ohj5ohp9eehae5hakoongahghohgoi7yeihohx1eidaeng0eaveefohchoh6WeeV1EM
+
+  # API credentials for use on the Infochimps paid query API
+  :query:
+    :username: your_query_name
+    :key: zei7eeloShoah3Ce
+    :secret: eixairaichaxaaRe8eeya5moh8Uthahf0pi4eig7SoirohPhei6sai8aereu0yuepiefeipoozoegahchaeheedee8uphohoo9moongae8Fa0aih4BooSeiM
+EOF
+      
+      def execute!
+        if argv.first.blank?
+          puts self
+        else
+          puts Chimps::Commands.construct(argv.first, [])
+        end
+      end
+
+    end
+  end
+end
+

data/lib/chimps/commands/list.rb

@@ -0,0 +1,54 @@
+module Chimps
+  module Commands
+
+    # A command to issue a GET request against an index of resources
+    # at Infochimps.
+    class List < Chimps::Command
+
+      BANNER = "chimps list [OPTIONS]"
+      HELP   = <<EOF
+
+List resources of a given type (defaults to dataset).
+
+Lists your resources by default but see options below.
+
+EOF
+
+      # Models that can be indexed (default first)
+      MODELS = %w[dataset license source]
+      include Chimps::Utils::UsesModel
+
+      def define_options
+        on_tail("-a", "--all", "List all resources, not just those owned by you.") do |a|
+          @all = a
+        end
+
+        on_tail("-s", "--[no-]skip-column-names", "Don't print column names in output.") do |s|
+          @skip_column_names = s
+        end
+        
+      end
+
+      # List all resources or just those owned by the Chimps user?
+      def all?
+        @all
+      end
+
+      # Parameters to include in the query.
+      #
+      # If listing all resources, then return +nil+.
+      #
+      # @return [Hash, nil]
+      def params
+        return { :id => Chimps.username } unless all?
+      end
+
+      # Issue the GET request.
+      def execute!
+        Request.new(models_path, :params => params).get.print(:skip_column_names => @skip_column_names)
+      end
+
+    end
+  end
+end
+