chimps 0.2.2 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +3 -9
- data/Gemfile.lock +14 -10
- data/README.rdoc +146 -240
- data/Rakefile +4 -33
- data/VERSION +1 -1
- data/lib/chimps/config.rb +35 -21
- data/lib/chimps/{utils/error.rb → error.rb} +1 -12
- data/lib/chimps/query_request.rb +67 -0
- data/lib/chimps/request.rb +82 -108
- data/lib/chimps/response.rb +62 -22
- data/lib/chimps/utils/typewriter.rb +90 -0
- data/lib/chimps/utils/uses_curl.rb +22 -12
- data/lib/chimps/utils.rb +50 -6
- data/lib/chimps/workflows/download.rb +72 -0
- data/lib/chimps/workflows/upload.rb +113 -0
- data/lib/chimps.rb +12 -12
- data/spec/chimps/query_request_spec.rb +44 -0
- data/spec/chimps/request_spec.rb +92 -0
- data/spec/chimps/response_spec.rb +0 -1
- data/spec/chimps/workflows/download_spec.rb +48 -0
- data/spec/spec_helper.rb +2 -19
- metadata +46 -91
- data/.document +0 -5
- data/.gitignore +0 -32
- data/CHANGELOG.textile +0 -4
- data/bin/chimps +0 -5
- data/lib/chimps/cli.rb +0 -28
- data/lib/chimps/commands/base.rb +0 -65
- data/lib/chimps/commands/batch.rb +0 -40
- data/lib/chimps/commands/create.rb +0 -31
- data/lib/chimps/commands/destroy.rb +0 -26
- data/lib/chimps/commands/download.rb +0 -46
- data/lib/chimps/commands/help.rb +0 -100
- data/lib/chimps/commands/list.rb +0 -41
- data/lib/chimps/commands/query.rb +0 -82
- data/lib/chimps/commands/search.rb +0 -48
- data/lib/chimps/commands/show.rb +0 -30
- data/lib/chimps/commands/test.rb +0 -39
- data/lib/chimps/commands/update.rb +0 -34
- data/lib/chimps/commands/upload.rb +0 -50
- data/lib/chimps/commands.rb +0 -125
- data/lib/chimps/typewriter.rb +0 -349
- data/lib/chimps/utils/log.rb +0 -48
- data/lib/chimps/utils/uses_model.rb +0 -34
- data/lib/chimps/utils/uses_yaml_data.rb +0 -93
- data/lib/chimps/workflows/batch.rb +0 -127
- data/lib/chimps/workflows/downloader.rb +0 -102
- data/lib/chimps/workflows/up.rb +0 -149
- data/lib/chimps/workflows/upload/bundler.rb +0 -249
- data/lib/chimps/workflows/upload/notifier.rb +0 -59
- data/lib/chimps/workflows/upload/token.rb +0 -77
- data/lib/chimps/workflows/upload/uploader.rb +0 -51
- data/lib/chimps/workflows.rb +0 -12
- data/spec/chimps/typewriter_spec.rb +0 -114
- data/spec/chimps/workflows/upload/bundler_spec.rb +0 -75
- data/spec/chimps/workflows/upload/token_spec.rb +0 -6
@@ -1,82 +0,0 @@
|
|
1
|
-
module Chimps
|
2
|
-
module Commands
|
3
|
-
|
4
|
-
# A command to issue a GET request against the Infochimps paid
|
5
|
-
# query API.
|
6
|
-
class Query < Chimps::Command
|
7
|
-
|
8
|
-
USAGE = "usage: chimps query [OPTIONS] DATASET [PROP=VALUE] ..."
|
9
|
-
HELP = <<EOF
|
10
|
-
|
11
|
-
Make a query of the given DATASET on the Infochimps paid query API
|
12
|
-
(not the main Infochimps site).
|
13
|
-
|
14
|
-
Properties and values can be supplied directly on the command line,
|
15
|
-
from an input YAML file, or multiple YAML documents streamed in via
|
16
|
-
STDIN, in order of decreasing precedence.
|
17
|
-
|
18
|
-
You can learn more about the Infochimps query API, discover datasets
|
19
|
-
to query, and look up the available parameters at
|
20
|
-
|
21
|
-
http://api.infochimps.com
|
22
|
-
|
23
|
-
You can learn about the main Infochimps site API at
|
24
|
-
|
25
|
-
http://infochimps.org/api
|
26
|
-
EOF
|
27
|
-
|
28
|
-
include Chimps::Utils::UsesYamlData
|
29
|
-
def ignore_first_arg_on_command_line
|
30
|
-
true
|
31
|
-
end
|
32
|
-
|
33
|
-
# The dataset to query.
|
34
|
-
#
|
35
|
-
# @return [String]
|
36
|
-
def dataset
|
37
|
-
raise CLIError.new("Must provide a dataset to query.") if config.argv.first.blank?
|
38
|
-
config.argv.first
|
39
|
-
end
|
40
|
-
|
41
|
-
# The path on the Infochimps query API to query.
|
42
|
-
#
|
43
|
-
# @return [String]
|
44
|
-
def path
|
45
|
-
dataset + ".json"
|
46
|
-
end
|
47
|
-
|
48
|
-
# Should the query output be pretty-printed?
|
49
|
-
#
|
50
|
-
# @return [true, nil]
|
51
|
-
def pretty_print?
|
52
|
-
config[:pretty_print]
|
53
|
-
end
|
54
|
-
|
55
|
-
# The requests that will be sent to the server.
|
56
|
-
#
|
57
|
-
# @return [Array<Chimps::QueryRequest>]
|
58
|
-
def requests
|
59
|
-
ensure_data_is_present!
|
60
|
-
if data.is_a?(Hash)
|
61
|
-
[QueryRequest.new(path, :query_params => data, :authenticate => true)]
|
62
|
-
else # it's an Array, see Chimps::Utils::UsesYamlData
|
63
|
-
data.map { |params| QueryRequest.new(path, :query_params => params, :authenticate => true) }
|
64
|
-
end
|
65
|
-
end
|
66
|
-
|
67
|
-
# Issue the GET request.
|
68
|
-
def execute!
|
69
|
-
requests.each do |request|
|
70
|
-
response = request.get
|
71
|
-
if response.error?
|
72
|
-
response.print :to => $stderr
|
73
|
-
else
|
74
|
-
puts pretty_print? ? JSON.pretty_generate(response.data) : response.body
|
75
|
-
end
|
76
|
-
end
|
77
|
-
end
|
78
|
-
|
79
|
-
end
|
80
|
-
end
|
81
|
-
end
|
82
|
-
|
@@ -1,48 +0,0 @@
|
|
1
|
-
module Chimps
|
2
|
-
module Commands
|
3
|
-
|
4
|
-
# A command to issue a GET request to create a search at
|
5
|
-
# Infochimps.
|
6
|
-
class Search < Chimps::Command
|
7
|
-
|
8
|
-
# Describes usage of search command.
|
9
|
-
USAGE = "usage: chimps search [OPTIONS] QUERY"
|
10
|
-
|
11
|
-
# Default number of search results returned.
|
12
|
-
# DEFAULT_LIMIT = 20
|
13
|
-
|
14
|
-
HELP = <<EOF
|
15
|
-
|
16
|
-
Perform a search on Infochimps. By default the search will be of
|
17
|
-
datasets and will return all matches for the given QUERY.
|
18
|
-
EOF
|
19
|
-
|
20
|
-
# Path to search resource
|
21
|
-
PATH = 'search.json'
|
22
|
-
|
23
|
-
include Chimps::Utils::UsesModel
|
24
|
-
|
25
|
-
# FIXME have to implement this on the server side.
|
26
|
-
# def limit
|
27
|
-
# @limit ||= DEFAULT_LIMIT
|
28
|
-
# end
|
29
|
-
|
30
|
-
def query
|
31
|
-
raise CLIError.new("Must provide a query to search for") if config.argv.blank?
|
32
|
-
config.argv.join(' ')
|
33
|
-
end
|
34
|
-
|
35
|
-
def params
|
36
|
-
{
|
37
|
-
:query => query,
|
38
|
-
:model => model
|
39
|
-
}
|
40
|
-
end
|
41
|
-
|
42
|
-
def execute!
|
43
|
-
Chimps::Request.new(PATH, :params => params).get.print(:skip_column_names => config[:skip_column_names])
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
|
-
|
data/lib/chimps/commands/show.rb
DELETED
@@ -1,30 +0,0 @@
|
|
1
|
-
module Chimps
|
2
|
-
module Commands
|
3
|
-
|
4
|
-
class Show < Chimps::Command
|
5
|
-
|
6
|
-
USAGE = "usage: chimps show [OPTIONS] ID_OR_HANDLE"
|
7
|
-
HELP = <<EOF
|
8
|
-
|
9
|
-
Return a description of the resource (defaults to dataset) with the
|
10
|
-
given ID or HANDLE
|
11
|
-
EOF
|
12
|
-
|
13
|
-
include Chimps::Utils::UsesModel
|
14
|
-
|
15
|
-
# The path of the URL to send a Request to.
|
16
|
-
#
|
17
|
-
# This is different from Chimps::Commands::UsesModel in that it
|
18
|
-
# submits to the YAML path.
|
19
|
-
def model_path
|
20
|
-
"#{plural_model}/#{model_identifier}.yaml"
|
21
|
-
end
|
22
|
-
|
23
|
-
def execute!
|
24
|
-
puts Chimps::Request.new(model_path).get.body
|
25
|
-
end
|
26
|
-
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
data/lib/chimps/commands/test.rb
DELETED
@@ -1,39 +0,0 @@
|
|
1
|
-
module Chimps
|
2
|
-
module Commands
|
3
|
-
|
4
|
-
# A command to test whether API authentication with Infochimps is
|
5
|
-
# working.
|
6
|
-
class Test < Chimps::Command
|
7
|
-
|
8
|
-
USAGE = "usage: chimps test"
|
9
|
-
HELP = <<EOF
|
10
|
-
|
11
|
-
Print diagnostic information on the API credentials being used by chimps
|
12
|
-
and send a test request to Infochimps to make sure the API credentials
|
13
|
-
work.
|
14
|
-
|
15
|
-
EOF
|
16
|
-
|
17
|
-
# Path to submit test requests to.
|
18
|
-
def path
|
19
|
-
"api_accounts/#{Chimps::Config[:site][:key]}"
|
20
|
-
end
|
21
|
-
|
22
|
-
# Issue the request.
|
23
|
-
def execute!
|
24
|
-
response = Chimps::Request.new(path, :sign => true).get
|
25
|
-
if response.error?
|
26
|
-
case
|
27
|
-
when response.code == 404 then puts "ERROR Unrecognized API key" # record not found
|
28
|
-
when response.code == 401 then puts "ERROR Signature does not match API key and query. Is your secret key correct?" # unauthorized
|
29
|
-
else
|
30
|
-
nil # response gets printed anyway
|
31
|
-
end
|
32
|
-
end
|
33
|
-
response.print
|
34
|
-
end
|
35
|
-
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
@@ -1,34 +0,0 @@
|
|
1
|
-
module Chimps
|
2
|
-
module Commands
|
3
|
-
|
4
|
-
# A command to issue a PUT request to update a resource at
|
5
|
-
# Infochimps.
|
6
|
-
class Update < Chimps::Command
|
7
|
-
|
8
|
-
USAGE = "usage: chimps update [OPTIONS] ID_OR_HANDLE [PROP=VALUE] ..."
|
9
|
-
HELP = <<EOF
|
10
|
-
|
11
|
-
Updates a single resource of a given type (defaults to dataset)
|
12
|
-
identified by ID_OR_HANDLE using the properties and values supplied.
|
13
|
-
|
14
|
-
Properties and values can be supplied directly on the command line,
|
15
|
-
from an input YAML file, or multiple YAML documents streamed in via
|
16
|
-
STDIN, in order of decreasing precedence.
|
17
|
-
EOF
|
18
|
-
|
19
|
-
include Chimps::Utils::UsesModel
|
20
|
-
include Chimps::Utils::UsesYamlData
|
21
|
-
def ignore_first_arg_on_command_line
|
22
|
-
true
|
23
|
-
end
|
24
|
-
|
25
|
-
# Issue the PUT request.
|
26
|
-
def execute!
|
27
|
-
ensure_data_is_present!
|
28
|
-
Request.new(model_path, :data => {model.to_sym => data } , :authenticate => true).put.print
|
29
|
-
end
|
30
|
-
|
31
|
-
end
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
@@ -1,50 +0,0 @@
|
|
1
|
-
module Chimps
|
2
|
-
module Commands
|
3
|
-
|
4
|
-
# A command for uploading data to Infochimps.
|
5
|
-
class Upload < Chimps::Command
|
6
|
-
|
7
|
-
USAGE = "usage: chimps upload [OPTIONS] ID_OR_HANDLE PATH [PATH] ..."
|
8
|
-
HELP = <<EOF
|
9
|
-
|
10
|
-
Upload data from your local machine for an existing dataset identified
|
11
|
-
by ID_OR_HANDLE on Infochimps.
|
12
|
-
|
13
|
-
chimps will package all paths supplied into a local archive and then
|
14
|
-
upload this archive to Infochimps. The local archive defaults to a
|
15
|
-
sensible name in the current directory but can also be customized.
|
16
|
-
|
17
|
-
If the only file to be packaged is already a package (.zip, .tar,
|
18
|
-
.tar.gz, &c.) then it will not be packaged again.
|
19
|
-
|
20
|
-
Supplied paths are allowed to be remote files so someting like
|
21
|
-
|
22
|
-
chimps upload my-dataset path/to/local/file.txt http://my-site.com/path/to/remote/file.txt
|
23
|
-
|
24
|
-
will work.
|
25
|
-
EOF
|
26
|
-
|
27
|
-
# The ID or handle of the dataset to upload data for.
|
28
|
-
#
|
29
|
-
# @return [String]
|
30
|
-
def dataset
|
31
|
-
raise CLIError.new("Must provide an ID or URL-escaped handle as the first argument") if config.argv.first.blank?
|
32
|
-
config.argv.first
|
33
|
-
end
|
34
|
-
|
35
|
-
# A list of paths to upload.
|
36
|
-
#
|
37
|
-
# @return [Array<String>]
|
38
|
-
def paths
|
39
|
-
raise CLIError.new("Must provide some paths to upload") if config.argv.length < 2
|
40
|
-
config.argv[1..-1]
|
41
|
-
end
|
42
|
-
|
43
|
-
# Upload the data.
|
44
|
-
def execute!
|
45
|
-
Chimps::Workflows::Up.new(:dataset => dataset, :archive => config[:archive], :paths => paths, :fmt => config[:format]).execute!.print
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
data/lib/chimps/commands.rb
DELETED
@@ -1,125 +0,0 @@
|
|
1
|
-
require 'configliere'
|
2
|
-
|
3
|
-
module Chimps
|
4
|
-
|
5
|
-
Config.use :commands
|
6
|
-
|
7
|
-
# A namespace to hold the various commands Chimps defines.
|
8
|
-
module Commands
|
9
|
-
|
10
|
-
def self.class_for name
|
11
|
-
"Chimps::Commands::#{name.to_s.capitalize}".constantize
|
12
|
-
end
|
13
|
-
|
14
|
-
def self.included obj
|
15
|
-
obj.extend(ClassMethods)
|
16
|
-
end
|
17
|
-
|
18
|
-
module ClassMethods
|
19
|
-
# Create a new command from the given +command_name+. The
|
20
|
-
# resulting command will be initialized but will not have been
|
21
|
-
# executed.
|
22
|
-
#
|
23
|
-
# @return [Chimps::Command]
|
24
|
-
def command
|
25
|
-
raise Chimps::CLIError.new("Must specify a command. Try `chimps help'.") unless Chimps::Config.command
|
26
|
-
Chimps::Config.command_settings.resolve!
|
27
|
-
Chimps::Commands.class_for(Chimps::Config.command_name).new(Chimps::Config.command_settings)
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
protected
|
32
|
-
|
33
|
-
def self.define_skip_column_names command
|
34
|
-
command.define :skip_column_names, :description => "Don't print column names in output", :flag => :s, :type => :boolean
|
35
|
-
end
|
36
|
-
|
37
|
-
def self.define_model command, models=%w[dataset collection source license]
|
38
|
-
models_string = models[0..-2].map { |m| "'#{m}'" }.join(', ') + ", or '#{models.last}'"
|
39
|
-
command.define :model, :description => "Model to search (one of #{models_string})", :flag => :m, :default => models.first, :type => String
|
40
|
-
end
|
41
|
-
|
42
|
-
def self.define_data_file command
|
43
|
-
command.define :data_file, :description => "Path to a file containing YAML data.", :flag => :d
|
44
|
-
end
|
45
|
-
|
46
|
-
#
|
47
|
-
# Core REST actions
|
48
|
-
#
|
49
|
-
|
50
|
-
Chimps::Config.define_command :list, :description => "List resources" do |command|
|
51
|
-
define_skip_column_names(command)
|
52
|
-
define_model(command)
|
53
|
-
command.define :all, :description => "List all resources, not just those owned by you", :flag => :a
|
54
|
-
end
|
55
|
-
|
56
|
-
Chimps::Config.define_command :show, :description => "Show a resource in detail" do |command|
|
57
|
-
define_model(command, %w[dataset collection source license category tag])
|
58
|
-
end
|
59
|
-
|
60
|
-
Chimps::Config.define_command :create, :description => "Create a new resource" do |command|
|
61
|
-
define_model(command, %w[dataset source license])
|
62
|
-
define_data_file(command)
|
63
|
-
end
|
64
|
-
|
65
|
-
Chimps::Config.define_command :update, :description => "Update an existing resource" do |command|
|
66
|
-
define_model(command, %w[dataset source license])
|
67
|
-
define_data_file(command)
|
68
|
-
end
|
69
|
-
|
70
|
-
Chimps::Config.define_command :destroy, :description => "Destroy an existing resource" do |command|
|
71
|
-
define_model(command, %w[dataset package source license])
|
72
|
-
end
|
73
|
-
|
74
|
-
#
|
75
|
-
# Workflows
|
76
|
-
#
|
77
|
-
|
78
|
-
Chimps::Config.define_command :download, :description => "Download a dataset" do |command|
|
79
|
-
command.define :output, :description => "Path to output file (defaults to sensible path in current directory)", :flag => :o, :type => String
|
80
|
-
command.define :format, :description => "Preferred data-format (csv, tsv, xls, &c.)", :flag => :f, :type => String
|
81
|
-
command.define :pkg_fmt, :description => "Preferred package-format (zip, tar.bz2, &c.)", :flag => :p, :type => String
|
82
|
-
end
|
83
|
-
|
84
|
-
Chimps::Config.define_command :upload, :description => "Upload a dataset" do |command|
|
85
|
-
command.define :archive, :description => "Path to the local archive that will be created (defaults to a sensibly named ZIP file in the current directory)", :type => String, :flag => :a
|
86
|
-
command.define :format, :description => "Data format (tsv, csv, xls, &c.) of the uploaded data (will guess if not given)", :type => String, :flag => :f
|
87
|
-
end
|
88
|
-
|
89
|
-
Chimps::Config.define_command :batch, :description => "Perform a batch processing request" do |command|
|
90
|
-
command.define :output, :description => "Path to store the server's response", :type => String, :flag => :o
|
91
|
-
command.define :force, :description => "Force upload of data even if there were errors in the batch request.", :flag => :F
|
92
|
-
command.define :format, :description => "Data format to annotate each upload with (will guess if not given)", :type => String, :flag => :f
|
93
|
-
define_data_file(command)
|
94
|
-
end
|
95
|
-
|
96
|
-
#
|
97
|
-
# Miscellaneous
|
98
|
-
#
|
99
|
-
|
100
|
-
Chimps::Config.define_help_command!
|
101
|
-
|
102
|
-
Chimps::Config.define_command :search, :description => 'Search resources' do |command|
|
103
|
-
define_skip_column_names(command)
|
104
|
-
define_model(command)
|
105
|
-
end
|
106
|
-
|
107
|
-
Chimps::Config.define_command :query, :description => "Get a response from the Query API" do |command|
|
108
|
-
command.define :pretty_print, :description => "Pretty-print output", :flag => :p
|
109
|
-
define_data_file(command)
|
110
|
-
end
|
111
|
-
|
112
|
-
Chimps::Config.define_command :test, :description => "Test your authentication credentials with Infochimps"
|
113
|
-
|
114
|
-
|
115
|
-
# A list of all the commmand names defined by Chimps. Each name
|
116
|
-
# maps to a corresponding subclass of Chimps::Command living in
|
117
|
-
# the Chimps::Commands module.
|
118
|
-
#Configliere::COMMANDS += %w[search help test create show update destroy upload list download batch query]
|
119
|
-
|
120
|
-
Chimps::Config.commands.keys.each do |command|
|
121
|
-
autoload command.to_s.capitalize.to_sym, "chimps/commands/#{command}"
|
122
|
-
end
|
123
|
-
|
124
|
-
end
|
125
|
-
end
|