chimps 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +3 -9
- data/Gemfile.lock +14 -10
- data/README.rdoc +146 -240
- data/Rakefile +4 -33
- data/VERSION +1 -1
- data/lib/chimps/config.rb +35 -21
- data/lib/chimps/{utils/error.rb → error.rb} +1 -12
- data/lib/chimps/query_request.rb +67 -0
- data/lib/chimps/request.rb +82 -108
- data/lib/chimps/response.rb +62 -22
- data/lib/chimps/utils/typewriter.rb +90 -0
- data/lib/chimps/utils/uses_curl.rb +22 -12
- data/lib/chimps/utils.rb +50 -6
- data/lib/chimps/workflows/download.rb +72 -0
- data/lib/chimps/workflows/upload.rb +113 -0
- data/lib/chimps.rb +12 -12
- data/spec/chimps/query_request_spec.rb +44 -0
- data/spec/chimps/request_spec.rb +92 -0
- data/spec/chimps/response_spec.rb +0 -1
- data/spec/chimps/workflows/download_spec.rb +48 -0
- data/spec/spec_helper.rb +2 -19
- metadata +46 -91
- data/.document +0 -5
- data/.gitignore +0 -32
- data/CHANGELOG.textile +0 -4
- data/bin/chimps +0 -5
- data/lib/chimps/cli.rb +0 -28
- data/lib/chimps/commands/base.rb +0 -65
- data/lib/chimps/commands/batch.rb +0 -40
- data/lib/chimps/commands/create.rb +0 -31
- data/lib/chimps/commands/destroy.rb +0 -26
- data/lib/chimps/commands/download.rb +0 -46
- data/lib/chimps/commands/help.rb +0 -100
- data/lib/chimps/commands/list.rb +0 -41
- data/lib/chimps/commands/query.rb +0 -82
- data/lib/chimps/commands/search.rb +0 -48
- data/lib/chimps/commands/show.rb +0 -30
- data/lib/chimps/commands/test.rb +0 -39
- data/lib/chimps/commands/update.rb +0 -34
- data/lib/chimps/commands/upload.rb +0 -50
- data/lib/chimps/commands.rb +0 -125
- data/lib/chimps/typewriter.rb +0 -349
- data/lib/chimps/utils/log.rb +0 -48
- data/lib/chimps/utils/uses_model.rb +0 -34
- data/lib/chimps/utils/uses_yaml_data.rb +0 -93
- data/lib/chimps/workflows/batch.rb +0 -127
- data/lib/chimps/workflows/downloader.rb +0 -102
- data/lib/chimps/workflows/up.rb +0 -149
- data/lib/chimps/workflows/upload/bundler.rb +0 -249
- data/lib/chimps/workflows/upload/notifier.rb +0 -59
- data/lib/chimps/workflows/upload/token.rb +0 -77
- data/lib/chimps/workflows/upload/uploader.rb +0 -51
- data/lib/chimps/workflows.rb +0 -12
- data/spec/chimps/typewriter_spec.rb +0 -114
- data/spec/chimps/workflows/upload/bundler_spec.rb +0 -75
- data/spec/chimps/workflows/upload/token_spec.rb +0 -6
@@ -1,82 +0,0 @@
|
|
1
|
-
module Chimps
|
2
|
-
module Commands
|
3
|
-
|
4
|
-
# A command to issue a GET request against the Infochimps paid
|
5
|
-
# query API.
|
6
|
-
class Query < Chimps::Command
|
7
|
-
|
8
|
-
USAGE = "usage: chimps query [OPTIONS] DATASET [PROP=VALUE] ..."
|
9
|
-
HELP = <<EOF
|
10
|
-
|
11
|
-
Make a query of the given DATASET on the Infochimps paid query API
|
12
|
-
(not the main Infochimps site).
|
13
|
-
|
14
|
-
Properties and values can be supplied directly on the command line,
|
15
|
-
from an input YAML file, or multiple YAML documents streamed in via
|
16
|
-
STDIN, in order of decreasing precedence.
|
17
|
-
|
18
|
-
You can learn more about the Infochimps query API, discover datasets
|
19
|
-
to query, and look up the available parameters at
|
20
|
-
|
21
|
-
http://api.infochimps.com
|
22
|
-
|
23
|
-
You can learn about the main Infochimps site API at
|
24
|
-
|
25
|
-
http://infochimps.org/api
|
26
|
-
EOF
|
27
|
-
|
28
|
-
include Chimps::Utils::UsesYamlData
|
29
|
-
def ignore_first_arg_on_command_line
|
30
|
-
true
|
31
|
-
end
|
32
|
-
|
33
|
-
# The dataset to query.
|
34
|
-
#
|
35
|
-
# @return [String]
|
36
|
-
def dataset
|
37
|
-
raise CLIError.new("Must provide a dataset to query.") if config.argv.first.blank?
|
38
|
-
config.argv.first
|
39
|
-
end
|
40
|
-
|
41
|
-
# The path on the Infochimps query API to query.
|
42
|
-
#
|
43
|
-
# @return [String]
|
44
|
-
def path
|
45
|
-
dataset + ".json"
|
46
|
-
end
|
47
|
-
|
48
|
-
# Should the query output be pretty-printed?
|
49
|
-
#
|
50
|
-
# @return [true, nil]
|
51
|
-
def pretty_print?
|
52
|
-
config[:pretty_print]
|
53
|
-
end
|
54
|
-
|
55
|
-
# The requests that will be sent to the server.
|
56
|
-
#
|
57
|
-
# @return [Array<Chimps::QueryRequest>]
|
58
|
-
def requests
|
59
|
-
ensure_data_is_present!
|
60
|
-
if data.is_a?(Hash)
|
61
|
-
[QueryRequest.new(path, :query_params => data, :authenticate => true)]
|
62
|
-
else # it's an Array, see Chimps::Utils::UsesYamlData
|
63
|
-
data.map { |params| QueryRequest.new(path, :query_params => params, :authenticate => true) }
|
64
|
-
end
|
65
|
-
end
|
66
|
-
|
67
|
-
# Issue the GET request.
|
68
|
-
def execute!
|
69
|
-
requests.each do |request|
|
70
|
-
response = request.get
|
71
|
-
if response.error?
|
72
|
-
response.print :to => $stderr
|
73
|
-
else
|
74
|
-
puts pretty_print? ? JSON.pretty_generate(response.data) : response.body
|
75
|
-
end
|
76
|
-
end
|
77
|
-
end
|
78
|
-
|
79
|
-
end
|
80
|
-
end
|
81
|
-
end
|
82
|
-
|
@@ -1,48 +0,0 @@
|
|
1
|
-
module Chimps
|
2
|
-
module Commands
|
3
|
-
|
4
|
-
# A command to issue a GET request to create a search at
|
5
|
-
# Infochimps.
|
6
|
-
class Search < Chimps::Command
|
7
|
-
|
8
|
-
# Describes usage of search command.
|
9
|
-
USAGE = "usage: chimps search [OPTIONS] QUERY"
|
10
|
-
|
11
|
-
# Default number of search results returned.
|
12
|
-
# DEFAULT_LIMIT = 20
|
13
|
-
|
14
|
-
HELP = <<EOF
|
15
|
-
|
16
|
-
Perform a search on Infochimps. By default the search will be of
|
17
|
-
datasets and will return all matches for the given QUERY.
|
18
|
-
EOF
|
19
|
-
|
20
|
-
# Path to search resource
|
21
|
-
PATH = 'search.json'
|
22
|
-
|
23
|
-
include Chimps::Utils::UsesModel
|
24
|
-
|
25
|
-
# FIXME have to implement this on the server side.
|
26
|
-
# def limit
|
27
|
-
# @limit ||= DEFAULT_LIMIT
|
28
|
-
# end
|
29
|
-
|
30
|
-
def query
|
31
|
-
raise CLIError.new("Must provide a query to search for") if config.argv.blank?
|
32
|
-
config.argv.join(' ')
|
33
|
-
end
|
34
|
-
|
35
|
-
def params
|
36
|
-
{
|
37
|
-
:query => query,
|
38
|
-
:model => model
|
39
|
-
}
|
40
|
-
end
|
41
|
-
|
42
|
-
def execute!
|
43
|
-
Chimps::Request.new(PATH, :params => params).get.print(:skip_column_names => config[:skip_column_names])
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
|
-
|
data/lib/chimps/commands/show.rb
DELETED
@@ -1,30 +0,0 @@
|
|
1
|
-
module Chimps
|
2
|
-
module Commands
|
3
|
-
|
4
|
-
class Show < Chimps::Command
|
5
|
-
|
6
|
-
USAGE = "usage: chimps show [OPTIONS] ID_OR_HANDLE"
|
7
|
-
HELP = <<EOF
|
8
|
-
|
9
|
-
Return a description of the resource (defaults to dataset) with the
|
10
|
-
given ID or HANDLE
|
11
|
-
EOF
|
12
|
-
|
13
|
-
include Chimps::Utils::UsesModel
|
14
|
-
|
15
|
-
# The path of the URL to send a Request to.
|
16
|
-
#
|
17
|
-
# This is different from Chimps::Commands::UsesModel in that it
|
18
|
-
# submits to the YAML path.
|
19
|
-
def model_path
|
20
|
-
"#{plural_model}/#{model_identifier}.yaml"
|
21
|
-
end
|
22
|
-
|
23
|
-
def execute!
|
24
|
-
puts Chimps::Request.new(model_path).get.body
|
25
|
-
end
|
26
|
-
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
data/lib/chimps/commands/test.rb
DELETED
@@ -1,39 +0,0 @@
|
|
1
|
-
module Chimps
|
2
|
-
module Commands
|
3
|
-
|
4
|
-
# A command to test whether API authentication with Infochimps is
|
5
|
-
# working.
|
6
|
-
class Test < Chimps::Command
|
7
|
-
|
8
|
-
USAGE = "usage: chimps test"
|
9
|
-
HELP = <<EOF
|
10
|
-
|
11
|
-
Print diagnostic information on the API credentials being used by chimps
|
12
|
-
and send a test request to Infochimps to make sure the API credentials
|
13
|
-
work.
|
14
|
-
|
15
|
-
EOF
|
16
|
-
|
17
|
-
# Path to submit test requests to.
|
18
|
-
def path
|
19
|
-
"api_accounts/#{Chimps::Config[:site][:key]}"
|
20
|
-
end
|
21
|
-
|
22
|
-
# Issue the request.
|
23
|
-
def execute!
|
24
|
-
response = Chimps::Request.new(path, :sign => true).get
|
25
|
-
if response.error?
|
26
|
-
case
|
27
|
-
when response.code == 404 then puts "ERROR Unrecognized API key" # record not found
|
28
|
-
when response.code == 401 then puts "ERROR Signature does not match API key and query. Is your secret key correct?" # unauthorized
|
29
|
-
else
|
30
|
-
nil # response gets printed anyway
|
31
|
-
end
|
32
|
-
end
|
33
|
-
response.print
|
34
|
-
end
|
35
|
-
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
@@ -1,34 +0,0 @@
|
|
1
|
-
module Chimps
|
2
|
-
module Commands
|
3
|
-
|
4
|
-
# A command to issue a PUT request to update a resource at
|
5
|
-
# Infochimps.
|
6
|
-
class Update < Chimps::Command
|
7
|
-
|
8
|
-
USAGE = "usage: chimps update [OPTIONS] ID_OR_HANDLE [PROP=VALUE] ..."
|
9
|
-
HELP = <<EOF
|
10
|
-
|
11
|
-
Updates a single resource of a given type (defaults to dataset)
|
12
|
-
identified by ID_OR_HANDLE using the properties and values supplied.
|
13
|
-
|
14
|
-
Properties and values can be supplied directly on the command line,
|
15
|
-
from an input YAML file, or multiple YAML documents streamed in via
|
16
|
-
STDIN, in order of decreasing precedence.
|
17
|
-
EOF
|
18
|
-
|
19
|
-
include Chimps::Utils::UsesModel
|
20
|
-
include Chimps::Utils::UsesYamlData
|
21
|
-
def ignore_first_arg_on_command_line
|
22
|
-
true
|
23
|
-
end
|
24
|
-
|
25
|
-
# Issue the PUT request.
|
26
|
-
def execute!
|
27
|
-
ensure_data_is_present!
|
28
|
-
Request.new(model_path, :data => {model.to_sym => data } , :authenticate => true).put.print
|
29
|
-
end
|
30
|
-
|
31
|
-
end
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
@@ -1,50 +0,0 @@
|
|
1
|
-
module Chimps
|
2
|
-
module Commands
|
3
|
-
|
4
|
-
# A command for uploading data to Infochimps.
|
5
|
-
class Upload < Chimps::Command
|
6
|
-
|
7
|
-
USAGE = "usage: chimps upload [OPTIONS] ID_OR_HANDLE PATH [PATH] ..."
|
8
|
-
HELP = <<EOF
|
9
|
-
|
10
|
-
Upload data from your local machine for an existing dataset identified
|
11
|
-
by ID_OR_HANDLE on Infochimps.
|
12
|
-
|
13
|
-
chimps will package all paths supplied into a local archive and then
|
14
|
-
upload this archive to Infochimps. The local archive defaults to a
|
15
|
-
sensible name in the current directory but can also be customized.
|
16
|
-
|
17
|
-
If the only file to be packaged is already a package (.zip, .tar,
|
18
|
-
.tar.gz, &c.) then it will not be packaged again.
|
19
|
-
|
20
|
-
Supplied paths are allowed to be remote files so someting like
|
21
|
-
|
22
|
-
chimps upload my-dataset path/to/local/file.txt http://my-site.com/path/to/remote/file.txt
|
23
|
-
|
24
|
-
will work.
|
25
|
-
EOF
|
26
|
-
|
27
|
-
# The ID or handle of the dataset to upload data for.
|
28
|
-
#
|
29
|
-
# @return [String]
|
30
|
-
def dataset
|
31
|
-
raise CLIError.new("Must provide an ID or URL-escaped handle as the first argument") if config.argv.first.blank?
|
32
|
-
config.argv.first
|
33
|
-
end
|
34
|
-
|
35
|
-
# A list of paths to upload.
|
36
|
-
#
|
37
|
-
# @return [Array<String>]
|
38
|
-
def paths
|
39
|
-
raise CLIError.new("Must provide some paths to upload") if config.argv.length < 2
|
40
|
-
config.argv[1..-1]
|
41
|
-
end
|
42
|
-
|
43
|
-
# Upload the data.
|
44
|
-
def execute!
|
45
|
-
Chimps::Workflows::Up.new(:dataset => dataset, :archive => config[:archive], :paths => paths, :fmt => config[:format]).execute!.print
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
data/lib/chimps/commands.rb
DELETED
@@ -1,125 +0,0 @@
|
|
1
|
-
require 'configliere'
|
2
|
-
|
3
|
-
module Chimps
|
4
|
-
|
5
|
-
Config.use :commands
|
6
|
-
|
7
|
-
# A namespace to hold the various commands Chimps defines.
|
8
|
-
module Commands
|
9
|
-
|
10
|
-
def self.class_for name
|
11
|
-
"Chimps::Commands::#{name.to_s.capitalize}".constantize
|
12
|
-
end
|
13
|
-
|
14
|
-
def self.included obj
|
15
|
-
obj.extend(ClassMethods)
|
16
|
-
end
|
17
|
-
|
18
|
-
module ClassMethods
|
19
|
-
# Create a new command from the given +command_name+. The
|
20
|
-
# resulting command will be initialized but will not have been
|
21
|
-
# executed.
|
22
|
-
#
|
23
|
-
# @return [Chimps::Command]
|
24
|
-
def command
|
25
|
-
raise Chimps::CLIError.new("Must specify a command. Try `chimps help'.") unless Chimps::Config.command
|
26
|
-
Chimps::Config.command_settings.resolve!
|
27
|
-
Chimps::Commands.class_for(Chimps::Config.command_name).new(Chimps::Config.command_settings)
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
protected
|
32
|
-
|
33
|
-
def self.define_skip_column_names command
|
34
|
-
command.define :skip_column_names, :description => "Don't print column names in output", :flag => :s, :type => :boolean
|
35
|
-
end
|
36
|
-
|
37
|
-
def self.define_model command, models=%w[dataset collection source license]
|
38
|
-
models_string = models[0..-2].map { |m| "'#{m}'" }.join(', ') + ", or '#{models.last}'"
|
39
|
-
command.define :model, :description => "Model to search (one of #{models_string})", :flag => :m, :default => models.first, :type => String
|
40
|
-
end
|
41
|
-
|
42
|
-
def self.define_data_file command
|
43
|
-
command.define :data_file, :description => "Path to a file containing YAML data.", :flag => :d
|
44
|
-
end
|
45
|
-
|
46
|
-
#
|
47
|
-
# Core REST actions
|
48
|
-
#
|
49
|
-
|
50
|
-
Chimps::Config.define_command :list, :description => "List resources" do |command|
|
51
|
-
define_skip_column_names(command)
|
52
|
-
define_model(command)
|
53
|
-
command.define :all, :description => "List all resources, not just those owned by you", :flag => :a
|
54
|
-
end
|
55
|
-
|
56
|
-
Chimps::Config.define_command :show, :description => "Show a resource in detail" do |command|
|
57
|
-
define_model(command, %w[dataset collection source license category tag])
|
58
|
-
end
|
59
|
-
|
60
|
-
Chimps::Config.define_command :create, :description => "Create a new resource" do |command|
|
61
|
-
define_model(command, %w[dataset source license])
|
62
|
-
define_data_file(command)
|
63
|
-
end
|
64
|
-
|
65
|
-
Chimps::Config.define_command :update, :description => "Update an existing resource" do |command|
|
66
|
-
define_model(command, %w[dataset source license])
|
67
|
-
define_data_file(command)
|
68
|
-
end
|
69
|
-
|
70
|
-
Chimps::Config.define_command :destroy, :description => "Destroy an existing resource" do |command|
|
71
|
-
define_model(command, %w[dataset package source license])
|
72
|
-
end
|
73
|
-
|
74
|
-
#
|
75
|
-
# Workflows
|
76
|
-
#
|
77
|
-
|
78
|
-
Chimps::Config.define_command :download, :description => "Download a dataset" do |command|
|
79
|
-
command.define :output, :description => "Path to output file (defaults to sensible path in current directory)", :flag => :o, :type => String
|
80
|
-
command.define :format, :description => "Preferred data-format (csv, tsv, xls, &c.)", :flag => :f, :type => String
|
81
|
-
command.define :pkg_fmt, :description => "Preferred package-format (zip, tar.bz2, &c.)", :flag => :p, :type => String
|
82
|
-
end
|
83
|
-
|
84
|
-
Chimps::Config.define_command :upload, :description => "Upload a dataset" do |command|
|
85
|
-
command.define :archive, :description => "Path to the local archive that will be created (defaults to a sensibly named ZIP file in the current directory)", :type => String, :flag => :a
|
86
|
-
command.define :format, :description => "Data format (tsv, csv, xls, &c.) of the uploaded data (will guess if not given)", :type => String, :flag => :f
|
87
|
-
end
|
88
|
-
|
89
|
-
Chimps::Config.define_command :batch, :description => "Perform a batch processing request" do |command|
|
90
|
-
command.define :output, :description => "Path to store the server's response", :type => String, :flag => :o
|
91
|
-
command.define :force, :description => "Force upload of data even if there were errors in the batch request.", :flag => :F
|
92
|
-
command.define :format, :description => "Data format to annotate each upload with (will guess if not given)", :type => String, :flag => :f
|
93
|
-
define_data_file(command)
|
94
|
-
end
|
95
|
-
|
96
|
-
#
|
97
|
-
# Miscellaneous
|
98
|
-
#
|
99
|
-
|
100
|
-
Chimps::Config.define_help_command!
|
101
|
-
|
102
|
-
Chimps::Config.define_command :search, :description => 'Search resources' do |command|
|
103
|
-
define_skip_column_names(command)
|
104
|
-
define_model(command)
|
105
|
-
end
|
106
|
-
|
107
|
-
Chimps::Config.define_command :query, :description => "Get a response from the Query API" do |command|
|
108
|
-
command.define :pretty_print, :description => "Pretty-print output", :flag => :p
|
109
|
-
define_data_file(command)
|
110
|
-
end
|
111
|
-
|
112
|
-
Chimps::Config.define_command :test, :description => "Test your authentication credentials with Infochimps"
|
113
|
-
|
114
|
-
|
115
|
-
# A list of all the commmand names defined by Chimps. Each name
|
116
|
-
# maps to a corresponding subclass of Chimps::Command living in
|
117
|
-
# the Chimps::Commands module.
|
118
|
-
#Configliere::COMMANDS += %w[search help test create show update destroy upload list download batch query]
|
119
|
-
|
120
|
-
Chimps::Config.commands.keys.each do |command|
|
121
|
-
autoload command.to_s.capitalize.to_sym, "chimps/commands/#{command}"
|
122
|
-
end
|
123
|
-
|
124
|
-
end
|
125
|
-
end
|