chimps 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +3 -9
- data/Gemfile.lock +14 -10
- data/README.rdoc +146 -240
- data/Rakefile +4 -33
- data/VERSION +1 -1
- data/lib/chimps/config.rb +35 -21
- data/lib/chimps/{utils/error.rb → error.rb} +1 -12
- data/lib/chimps/query_request.rb +67 -0
- data/lib/chimps/request.rb +82 -108
- data/lib/chimps/response.rb +62 -22
- data/lib/chimps/utils/typewriter.rb +90 -0
- data/lib/chimps/utils/uses_curl.rb +22 -12
- data/lib/chimps/utils.rb +50 -6
- data/lib/chimps/workflows/download.rb +72 -0
- data/lib/chimps/workflows/upload.rb +113 -0
- data/lib/chimps.rb +12 -12
- data/spec/chimps/query_request_spec.rb +44 -0
- data/spec/chimps/request_spec.rb +92 -0
- data/spec/chimps/response_spec.rb +0 -1
- data/spec/chimps/workflows/download_spec.rb +48 -0
- data/spec/spec_helper.rb +2 -19
- metadata +46 -91
- data/.document +0 -5
- data/.gitignore +0 -32
- data/CHANGELOG.textile +0 -4
- data/bin/chimps +0 -5
- data/lib/chimps/cli.rb +0 -28
- data/lib/chimps/commands/base.rb +0 -65
- data/lib/chimps/commands/batch.rb +0 -40
- data/lib/chimps/commands/create.rb +0 -31
- data/lib/chimps/commands/destroy.rb +0 -26
- data/lib/chimps/commands/download.rb +0 -46
- data/lib/chimps/commands/help.rb +0 -100
- data/lib/chimps/commands/list.rb +0 -41
- data/lib/chimps/commands/query.rb +0 -82
- data/lib/chimps/commands/search.rb +0 -48
- data/lib/chimps/commands/show.rb +0 -30
- data/lib/chimps/commands/test.rb +0 -39
- data/lib/chimps/commands/update.rb +0 -34
- data/lib/chimps/commands/upload.rb +0 -50
- data/lib/chimps/commands.rb +0 -125
- data/lib/chimps/typewriter.rb +0 -349
- data/lib/chimps/utils/log.rb +0 -48
- data/lib/chimps/utils/uses_model.rb +0 -34
- data/lib/chimps/utils/uses_yaml_data.rb +0 -93
- data/lib/chimps/workflows/batch.rb +0 -127
- data/lib/chimps/workflows/downloader.rb +0 -102
- data/lib/chimps/workflows/up.rb +0 -149
- data/lib/chimps/workflows/upload/bundler.rb +0 -249
- data/lib/chimps/workflows/upload/notifier.rb +0 -59
- data/lib/chimps/workflows/upload/token.rb +0 -77
- data/lib/chimps/workflows/upload/uploader.rb +0 -51
- data/lib/chimps/workflows.rb +0 -12
- data/spec/chimps/typewriter_spec.rb +0 -114
- data/spec/chimps/workflows/upload/bundler_spec.rb +0 -75
- data/spec/chimps/workflows/upload/token_spec.rb +0 -6
@@ -0,0 +1,90 @@
|
|
1
|
+
module Chimps
|
2
|
+
|
3
|
+
module Utils
|
4
|
+
|
5
|
+
# There are two Chimpanzees using a typewriter. One of them presses
|
6
|
+
# most of the keys and writes to $stdout. The other only hits the
|
7
|
+
# spacebar and writes to $stderr. He's crazy.
|
8
|
+
#
|
9
|
+
# These two Chimps together manage to line everything up just right.
|
10
|
+
class Typewriter < Array
|
11
|
+
|
12
|
+
# The response that this Typewriter will print.
|
13
|
+
attr_accessor :response
|
14
|
+
|
15
|
+
# Widths of columns as determined by the maximum number of
|
16
|
+
# characters in any row.
|
17
|
+
attr_accessor :column_widths
|
18
|
+
|
19
|
+
# Separates rows.
|
20
|
+
attr_accessor :row_separator
|
21
|
+
|
22
|
+
# Separates columns.
|
23
|
+
attr_accessor :column_separator
|
24
|
+
|
25
|
+
# Default row separator
|
26
|
+
ROW_SEPARATOR = "\n"
|
27
|
+
|
28
|
+
# Default columnn separator
|
29
|
+
COLUMN_SEPARATOR = "\t"
|
30
|
+
|
31
|
+
# FIXME
|
32
|
+
def spacer
|
33
|
+
2
|
34
|
+
end
|
35
|
+
|
36
|
+
# Return a Typewriter to print +response+.
|
37
|
+
#
|
38
|
+
# @param [Chimps::Response] response
|
39
|
+
# @return [Chimps::Typewriter]
|
40
|
+
def initialize response, options={}
|
41
|
+
super()
|
42
|
+
@response = response
|
43
|
+
@column_widths = {}
|
44
|
+
self.row_separator = (options[:row_separator] || ROW_SEPARATOR)
|
45
|
+
self.column_separator = (options[:column_separator] || COLUMN_SEPARATOR)
|
46
|
+
accumulate(response)
|
47
|
+
end
|
48
|
+
|
49
|
+
# Print the accumulated lines in this Typewriter.
|
50
|
+
#
|
51
|
+
# Will first calculate appropriate column widths for each line and
|
52
|
+
# then pad with spaces each entry so that the columns line up.
|
53
|
+
#
|
54
|
+
# The spaces are written to $stderr and the rest of the characters
|
55
|
+
# to $stdout. This lets you pipe output from a Typewriter into
|
56
|
+
# other processes and preserve the TSV structure.
|
57
|
+
def print
|
58
|
+
$stdout.sync = true ; $stderr.sync = true
|
59
|
+
each do |row|
|
60
|
+
row.each_with_index do |entry, field|
|
61
|
+
$stdout.write entry
|
62
|
+
max_width = column_widths[field] + spacer
|
63
|
+
unless entry.size >= max_width
|
64
|
+
num_spaces = max_width - entry.size
|
65
|
+
pad = " " * num_spaces
|
66
|
+
$stderr.write(pad)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
$stdout.write "\n"
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
# Accumulate lines to print from +string+.
|
74
|
+
#
|
75
|
+
# Updates internal width counters as it accumulates
|
76
|
+
#
|
77
|
+
# @param [Array, Hash, String] obj
|
78
|
+
def accumulate response
|
79
|
+
response.body.strip.split(row_separator).each do |line|
|
80
|
+
self << [].tap do |row|
|
81
|
+
line.split(column_separator).each_with_index do |entry, field|
|
82
|
+
column_widths[field] = entry.size if entry.size > (column_widths[field] || 0)
|
83
|
+
row << entry
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
@@ -5,22 +5,32 @@ module Chimps
|
|
5
5
|
# system call.
|
6
6
|
module UsesCurl
|
7
7
|
|
8
|
-
def
|
8
|
+
def curl_program
|
9
9
|
`which curl`.chomp
|
10
10
|
end
|
11
11
|
|
12
|
-
#
|
13
|
-
#
|
14
|
-
#
|
15
|
-
#
|
16
|
-
#
|
17
|
-
#
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
12
|
+
# Curl invocations (specifically those that do S3 HTTP POST) are
|
13
|
+
# sometimes sensitive about the order of parameters. Instead of
|
14
|
+
# a Hash we therefore take an Array of pairs here.
|
15
|
+
#
|
16
|
+
# @param [Array<Array<String>>] array
|
17
|
+
# @return [String]
|
18
|
+
def curl_params params
|
19
|
+
params.map do |param, value|
|
20
|
+
"-F #{param}='#{value}'"
|
21
|
+
end.join(' ')
|
22
|
+
end
|
23
23
|
|
24
|
+
def curl url, options={}
|
25
|
+
options = {:method => "GET", :output => '/dev/null', :params => []}.merge(options)
|
26
|
+
progress_meter = Chimps.verbose? ? '' : '-s -S'
|
27
|
+
command = "#{curl_program} #{progress_meter} -X #{options[:method]} -o #{options[:output]}"
|
28
|
+
command += " #{curl_params(options[:params])}" unless options[:params].empty?
|
29
|
+
command += " '#{url}'"
|
30
|
+
Chimps.log.info(command)
|
31
|
+
system(command)
|
32
|
+
end
|
33
|
+
|
24
34
|
end
|
25
35
|
end
|
26
36
|
end
|
data/lib/chimps/utils.rb
CHANGED
@@ -1,13 +1,57 @@
|
|
1
1
|
require 'chimps/config'
|
2
|
-
require 'chimps/utils/extensions'
|
3
|
-
require 'chimps/
|
4
|
-
require 'chimps/utils/log'
|
2
|
+
# require 'chimps/utils/extensions'
|
3
|
+
require 'chimps/error'
|
5
4
|
|
6
5
|
module Chimps
|
6
|
+
|
7
7
|
module Utils
|
8
|
-
autoload :UsesCurl,
|
9
|
-
autoload :
|
10
|
-
|
8
|
+
autoload :UsesCurl, 'chimps/utils/uses_curl'
|
9
|
+
autoload :Typewriter, 'chimps/utils/typewriter'
|
10
|
+
end
|
11
|
+
|
12
|
+
# The Chimps logger. Set via Chimps.config[:log] and defaults
|
13
|
+
# to $stdout.
|
14
|
+
#
|
15
|
+
# @return [Logger]
|
16
|
+
def self.log
|
17
|
+
@log ||= Log.new_logger
|
18
|
+
end
|
19
|
+
|
20
|
+
# Set the Chimps logger.
|
21
|
+
#
|
22
|
+
# @param [Logger] new_log
|
23
|
+
def self.log= new_log
|
24
|
+
@log = new_log
|
11
25
|
end
|
12
26
|
|
27
|
+
# Module for initializing the Chimps logger from configuration
|
28
|
+
# settings.
|
29
|
+
module Log
|
30
|
+
|
31
|
+
# Initialize a new Logger instance with the log level set by
|
32
|
+
# Chimps.verbose?
|
33
|
+
#
|
34
|
+
# @return [Logger]
|
35
|
+
def self.new_logger
|
36
|
+
require 'logger'
|
37
|
+
Logger.new(log_file).tap do |log|
|
38
|
+
log.progname = "Chimps"
|
39
|
+
log.level = Chimps.verbose? ? Logger::INFO : Logger::WARN
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
# Return either the path to the log file in Chimps.config[:log]
|
44
|
+
# or $stdout if the path is blank or equal to `-'.
|
45
|
+
#
|
46
|
+
# @return [String, $stdout] the path to the log or $stdout
|
47
|
+
def self.log_file
|
48
|
+
if Chimps.config[:log]
|
49
|
+
Chimps.config[:log].strip == '-' ? $stdout : Chimps.config[:log]
|
50
|
+
else
|
51
|
+
$stdout
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
13
55
|
end
|
56
|
+
|
57
|
+
|
@@ -0,0 +1,72 @@
|
|
1
|
+
module Chimps
|
2
|
+
|
3
|
+
# A download is composted of an initial POST request which obtains a
|
4
|
+
# signed and expiring token from Infochimps followed by a GET to the
|
5
|
+
# URL provided in the token.
|
6
|
+
class Download
|
7
|
+
|
8
|
+
# The slug or (ID) of the dataset to download
|
9
|
+
attr_accessor :dataset
|
10
|
+
|
11
|
+
# Provides the use of curl to download the file.
|
12
|
+
include Chimps::Utils::UsesCurl
|
13
|
+
|
14
|
+
# Create a new download for the dataset named by the given slug or
|
15
|
+
# ID.
|
16
|
+
#
|
17
|
+
# @param [String] dataset
|
18
|
+
def initialize dataset
|
19
|
+
self.dataset = dataset
|
20
|
+
end
|
21
|
+
|
22
|
+
# Download data to +path+.
|
23
|
+
#
|
24
|
+
# If +path+ is a directory then the resulting file will be put
|
25
|
+
# there with a basename determined sensibly from +signed_url+.
|
26
|
+
# Otherwise it will be placed at +path+ itself.
|
27
|
+
#
|
28
|
+
# @param [String] path
|
29
|
+
# @return [Integer] the exit code of the curl command used to download the data
|
30
|
+
def download path
|
31
|
+
if File.directory?(path)
|
32
|
+
basename = File.basename(signed_url).split('?').first
|
33
|
+
path = File.join(path, basename)
|
34
|
+
end
|
35
|
+
curl signed_url, :output => path
|
36
|
+
end
|
37
|
+
|
38
|
+
# The request for obtaining a download token from Infochimps.
|
39
|
+
#
|
40
|
+
# @return [Chimps::Request]
|
41
|
+
def token_request
|
42
|
+
@token_request ||= Request.new("/datasets/#{dataset}/downloads", :sign_if_possible => true)
|
43
|
+
end
|
44
|
+
|
45
|
+
# A download token from Infochimps containing a signed URL from
|
46
|
+
# which data can be downloaded.
|
47
|
+
#
|
48
|
+
# @return [Chimps::Response]
|
49
|
+
def token
|
50
|
+
@token ||= token_request.post do |response, request, result, &block|
|
51
|
+
case response.code
|
52
|
+
when 301, 302, 307
|
53
|
+
response.follow_redirection(request, result, &block)
|
54
|
+
when 200
|
55
|
+
response.return!(request, result, &block)
|
56
|
+
else
|
57
|
+
raise Error.new("Could not obtain download token from Infochimps")
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
# Return the signed URL as parsed from the download token.
|
63
|
+
#
|
64
|
+
# @return [String] the token's signed URL
|
65
|
+
def signed_url
|
66
|
+
token.parse
|
67
|
+
raise Error.new("Malformed download token received from Infochimps") unless token['download_token'].is_a?(Hash) && token['download_token']['signed_url']
|
68
|
+
token['download_token']['signed_url']
|
69
|
+
end
|
70
|
+
|
71
|
+
end
|
72
|
+
end
|
@@ -0,0 +1,113 @@
|
|
1
|
+
module Chimps
|
2
|
+
|
3
|
+
# An upload at Infochimps is a process attached to a dataset which
|
4
|
+
# carries a state.
|
5
|
+
#
|
6
|
+
# A dataset typically does not have an "upload" associated with it
|
7
|
+
# but anyone authorized to update the dataset can *create* an upload
|
8
|
+
# for it. This upload object is empty by default. You can submit
|
9
|
+
# files or links to upload. When you're done you can submit the
|
10
|
+
# entire upload for processing. You can view the status of the
|
11
|
+
# upload at any time.
|
12
|
+
class Upload
|
13
|
+
|
14
|
+
# The slug or (ID) of the dataset to upload for
|
15
|
+
attr_accessor :slug
|
16
|
+
|
17
|
+
# Gives the ability to use curl to upload local files.
|
18
|
+
include Chimps::Utils::UsesCurl
|
19
|
+
|
20
|
+
# Create a new Upload for the dataset with the given +slug+ or ID.
|
21
|
+
#
|
22
|
+
# @return [Chimps::Upload]
|
23
|
+
def initialize slug
|
24
|
+
self.slug = slug
|
25
|
+
end
|
26
|
+
|
27
|
+
# Show this upload.
|
28
|
+
#
|
29
|
+
# @return [Chimps::Response]
|
30
|
+
def show
|
31
|
+
follow_redirects_on :get, "/datasets/#{slug}/upload.yaml"
|
32
|
+
end
|
33
|
+
|
34
|
+
# Create this upload on Infochimps.
|
35
|
+
#
|
36
|
+
# @return [Chimps::Response]
|
37
|
+
def create
|
38
|
+
follow_redirects_on :post, "/datasets/#{slug}/upload.json", :body => true do |response, request, result, &block|
|
39
|
+
if response.code == 409
|
40
|
+
response # upload already exists
|
41
|
+
else
|
42
|
+
response.return!(request, result, &block)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def update params={}
|
48
|
+
follow_redirects_on :put, "/datasets/#{slug}/upload.json", params
|
49
|
+
end
|
50
|
+
|
51
|
+
def upload_files *paths
|
52
|
+
paths.map { |p| File.expand_path(p) }.each do |path|
|
53
|
+
upload_file(upload_token)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def upload_token
|
58
|
+
follow_redirects_on :get, "/datasets/#{slug}/upload.json", :query => { :token => true }
|
59
|
+
end
|
60
|
+
|
61
|
+
def upload_file path, token
|
62
|
+
token.parse
|
63
|
+
p token
|
64
|
+
raise UploadError.new("#{path} does not exist") unless File.exist?(path)
|
65
|
+
raise UploadError.new("#{path} is a directory -- can only upload files") if File.directory?(path)
|
66
|
+
params = %w[AWSAccessKeyId acl key policy success_action_status signature].map do |param|
|
67
|
+
[param, token[param]]
|
68
|
+
end
|
69
|
+
params << ['file', '@' + path] # this is how you tell curl to upload a file
|
70
|
+
Chimps.log.info("Uploading #{path} for dataset #{slug}")
|
71
|
+
curl token['url'], :method => "POST", :params => params
|
72
|
+
end
|
73
|
+
|
74
|
+
def remove_files *uuids
|
75
|
+
follow_redirects_on :put, "/datasets/#{slug}/upload.json", :body => { :upload => { :remove_files => uuids }}
|
76
|
+
end
|
77
|
+
|
78
|
+
def create_links *links
|
79
|
+
follow_redirects_on :put, "/datasets/#{slug}/upload.json", :body => { :upload => { :add_links => links }}
|
80
|
+
end
|
81
|
+
|
82
|
+
def remove_links *uuids
|
83
|
+
follow_redirects_on :put, "/datasets/#{slug}/upload.json", :body => { :upload => { :remove_links => uuids }}
|
84
|
+
end
|
85
|
+
|
86
|
+
def start
|
87
|
+
follow_redirects_on :put, "/datasets/#{slug}/upload.json", :query => { :submit => true }
|
88
|
+
end
|
89
|
+
|
90
|
+
def destroy
|
91
|
+
follow_redirects_on :delete, "/datasets/#{slug}/upload.json"
|
92
|
+
end
|
93
|
+
|
94
|
+
def restart
|
95
|
+
follow_redirects_on :delete, "/datasets/#{slug}/upload.json", :query => { :restart => true }
|
96
|
+
end
|
97
|
+
|
98
|
+
def follow_redirects_on method, url, options={}, &block
|
99
|
+
Request.new(url, {:sign => true}.merge(options)).send(method) do |response, request, result, &block|
|
100
|
+
if [301, 302, 307].include?(response.code)
|
101
|
+
response.follow_redirection(request, result, &block)
|
102
|
+
else
|
103
|
+
if response.code != 200 && block_given?
|
104
|
+
response.return!(request, result, &block)
|
105
|
+
else
|
106
|
+
response.return!(request, result)
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
end
|
data/lib/chimps.rb
CHANGED
@@ -1,29 +1,29 @@
|
|
1
1
|
require 'rubygems'
|
2
|
+
ENV["BUNDLE_GEMFILE"] ||= File.expand_path('../Gemfile', File.dirname(__FILE__))
|
2
3
|
require 'bundler/setup'
|
4
|
+
require 'chimps/config'
|
3
5
|
require 'chimps/utils'
|
4
6
|
|
5
|
-
# The Chimps module
|
6
|
-
#
|
7
|
+
# The Chimps module provides classes which make making requests at
|
8
|
+
# Infochimps easy.
|
7
9
|
#
|
8
10
|
# Using this tool you can search, download, edit, and upload data and
|
9
11
|
# metadata to and from Infochimps.
|
10
12
|
module Chimps
|
11
13
|
|
12
|
-
autoload :Config, 'chimps/config'
|
13
|
-
autoload :CLI, 'chimps/cli'
|
14
|
-
autoload :Command, 'chimps/commands/base'
|
15
|
-
autoload :Commands, 'chimps/commands'
|
16
14
|
autoload :Request, 'chimps/request'
|
17
|
-
autoload :QueryRequest, 'chimps/
|
15
|
+
autoload :QueryRequest, 'chimps/query_request'
|
18
16
|
autoload :Response, 'chimps/response'
|
19
|
-
autoload :
|
20
|
-
autoload :
|
17
|
+
autoload :Download, 'chimps/workflows/download'
|
18
|
+
autoload :Upload, 'chimps/workflows/upload'
|
21
19
|
|
22
20
|
# Load and resolve configuration.
|
23
21
|
def self.boot!
|
24
|
-
|
25
|
-
|
26
|
-
|
22
|
+
config.read config[:site_config] if config[:site_config] && File.exist?(config[:site_config])
|
23
|
+
config.read config[:config] if config[:config] && File.exist?(config[:config])
|
24
|
+
config.resolve!
|
25
|
+
config[:dataset] = config[:site] if (! config[:dataset]) && config[:site] # backwards compatibility
|
26
|
+
true
|
27
27
|
end
|
28
28
|
|
29
29
|
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
require File.expand_path('../spec_helper', File.dirname(__FILE__))
|
2
|
+
|
3
|
+
describe Chimps::QueryRequest do
|
4
|
+
|
5
|
+
before do
|
6
|
+
Chimps.config[:query][:host] = 'http://qubar.com'
|
7
|
+
Chimps.config[:query][:key] = 'spec_key'
|
8
|
+
end
|
9
|
+
|
10
|
+
describe "generating the base URL with query string" do
|
11
|
+
it "should join the path to the Infochimps query host" do
|
12
|
+
Chimps::QueryRequest.new('/path/to/something').base_url.should == 'http://qubar.com/path/to/something'
|
13
|
+
end
|
14
|
+
it "should generate the same base URL regardless of whether the path has a leading '/' or not" do
|
15
|
+
Chimps::QueryRequest.new('/path/to/something').base_url.should == Chimps::QueryRequest.new('path/to/something').base_url
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
describe "generating the query string" do
|
20
|
+
it "should add apikey and requested_at params by default" do
|
21
|
+
qs = Chimps::QueryRequest.new('/path/to/something').query_string
|
22
|
+
qs.should include('apikey')
|
23
|
+
qs.should include('requested_at')
|
24
|
+
qs.should_not include('signature')
|
25
|
+
end
|
26
|
+
|
27
|
+
it "should properly URL encode the query string it generates" do
|
28
|
+
Chimps::QueryRequest.new('/path/to/something', :query_params => {:foo => 'bar baz'}).query_string.should include('foo=bar%20baz')
|
29
|
+
end
|
30
|
+
|
31
|
+
it "should raise an error if no credentials are available" do
|
32
|
+
Chimps.config[:query][:key] = nil
|
33
|
+
lambda { Chimps::QueryRequest.new('/path/to/something').query_string }.should raise_error(Chimps::AuthenticationError)
|
34
|
+
end
|
35
|
+
|
36
|
+
it "should allow setting a raw query string" do
|
37
|
+
Chimps::QueryRequest.new('/path/to/something', :query => 'foo=bar', :raw => true).query_string.should == 'foo=bar'
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
|
43
|
+
|
44
|
+
|
@@ -0,0 +1,92 @@
|
|
1
|
+
require File.expand_path('../spec_helper', File.dirname(__FILE__))
|
2
|
+
|
3
|
+
describe Chimps::Request do
|
4
|
+
|
5
|
+
before do
|
6
|
+
Chimps.config[:dataset][:host] = 'http://foobar.com'
|
7
|
+
Chimps.config[:dataset][:key] = 'spec_key'
|
8
|
+
Chimps.config[:dataset][:secret] = 'secret'
|
9
|
+
end
|
10
|
+
|
11
|
+
describe "generating the base URL with query string" do
|
12
|
+
it "should join the path to the Infochimps site host" do
|
13
|
+
Chimps::Request.new('/path/to/something').base_url.should == 'http://foobar.com/path/to/something'
|
14
|
+
end
|
15
|
+
it "should generate the same base URL regardless of whether the path has a leading '/' or not" do
|
16
|
+
Chimps::Request.new('/path/to/something').base_url.should == Chimps::Request.new('path/to/something').base_url
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
describe "generating the query string" do
|
21
|
+
it "should generate no query string by default" do
|
22
|
+
Chimps::Request.new('/path/to/something').query_string.should_not include('?')
|
23
|
+
end
|
24
|
+
|
25
|
+
it "should encode a Hash of query string parameters when given" do
|
26
|
+
Chimps::Request.new('/path/to/something', :query_params => {:foo => 'bar', :fuzz => 'booz'}).query_string.should == 'foo=bar&fuzz=booz'
|
27
|
+
end
|
28
|
+
|
29
|
+
it "should properly URL encode the query string it generates" do
|
30
|
+
Chimps::Request.new('/path/to/something', :query_params => {:foo => 'bar baz'}).query_string.should == 'foo=bar%20baz'
|
31
|
+
end
|
32
|
+
|
33
|
+
it "should sign the URL it generates if asked to" do
|
34
|
+
qs = Chimps::Request.new('/path/to/something', :sign => true).query_string
|
35
|
+
qs.should include('apikey')
|
36
|
+
qs.should include('requested_at')
|
37
|
+
qs.should include('signature')
|
38
|
+
end
|
39
|
+
|
40
|
+
it "should raise an error if asked to sign and no credentials are available" do
|
41
|
+
Chimps.config[:dataset][:key] = nil
|
42
|
+
lambda { Chimps::Request.new('/path/to/something', :sign => true).query_string }.should raise_error(Chimps::AuthenticationError)
|
43
|
+
end
|
44
|
+
|
45
|
+
it "should not raise an error if asked to sign_if_possible and no credentials are avialable" do
|
46
|
+
Chimps.config[:dataset][:key] = nil
|
47
|
+
lambda { Chimps::Request.new('/path/to/something', :sign_if_possible => true).query_string }.should_not raise_error(Chimps::AuthenticationError)
|
48
|
+
end
|
49
|
+
|
50
|
+
it "should allow setting a raw query string" do
|
51
|
+
Chimps::Request.new('/path/to/something', :query => 'foo=bar', :raw => true).query_string.should == 'foo=bar'
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
describe "generating the request body" do
|
56
|
+
it "should have no body by default" do
|
57
|
+
Chimps::Request.new('/path/to/something').body.should be_blank
|
58
|
+
end
|
59
|
+
|
60
|
+
it "should encode a Hash of parameters when given" do
|
61
|
+
Chimps::Request.new('/path/to/something', :body => { :foo => 'bar' }).encoded_body.should == '{"foo":"bar"}'
|
62
|
+
end
|
63
|
+
|
64
|
+
it "should sign the body when it exists" do
|
65
|
+
request = Chimps::Request.new('/path/to/something', :body => { :foo => 'bar'}, :sign => true)
|
66
|
+
request.should_receive(:sign).with('{"foo":"bar"}')
|
67
|
+
request.query_string
|
68
|
+
end
|
69
|
+
|
70
|
+
it "should allow setting a raw body" do
|
71
|
+
Chimps::Request.new('/path/to/something', :body => '{"foo": "bar"}', :raw => true).encoded_body.should == '{"foo": "bar"}'
|
72
|
+
end
|
73
|
+
|
74
|
+
end
|
75
|
+
|
76
|
+
describe "making a request" do
|
77
|
+
it "should swallow low-level networking errors" do
|
78
|
+
Chimps::Request.new('/some/made/up/path').get.code.should == 404
|
79
|
+
end
|
80
|
+
|
81
|
+
it "should swallow application-level errors" do
|
82
|
+
Chimps.config[:dataset][:host] = 'http://www.infochimps.com'
|
83
|
+
Chimps::Request.new('/some/made/up/path').get.code.should == 404
|
84
|
+
end
|
85
|
+
|
86
|
+
end
|
87
|
+
|
88
|
+
|
89
|
+
end
|
90
|
+
|
91
|
+
|
92
|
+
|
@@ -0,0 +1,48 @@
|
|
1
|
+
require File.expand_path('../../spec_helper', File.dirname(__FILE__))
|
2
|
+
|
3
|
+
describe Chimps::Download do
|
4
|
+
before do
|
5
|
+
@download = Chimps::Download.new('foobar')
|
6
|
+
end
|
7
|
+
|
8
|
+
describe "downloading" do
|
9
|
+
before do
|
10
|
+
@basename = "data.tar.gz"
|
11
|
+
@signed_url = "http://bucket.aws.amazon.com/path/to/#{@basename}?this=is&aFake=SignedURL"
|
12
|
+
@download.stub!(:signed_url).and_return(@signed_url)
|
13
|
+
end
|
14
|
+
|
15
|
+
it "should write to a sensibly named file when given a directory" do
|
16
|
+
@download.should_receive(:curl).with(@signed_url, { :output => File.join('/tmp', @basename) })
|
17
|
+
@download.download('/tmp')
|
18
|
+
end
|
19
|
+
|
20
|
+
it "should write to a path when given a path" do
|
21
|
+
@download.should_receive(:curl).with(@signed_url, { :output => '/wukka/wukka.tar.gz' })
|
22
|
+
@download.download('/wukka/wukka.tar.gz')
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
describe "extracting a signed URL from a download token" do
|
27
|
+
before do
|
28
|
+
@token = {}
|
29
|
+
@token.stub!(:parse)
|
30
|
+
@download.stub!(:token).and_return(@token)
|
31
|
+
end
|
32
|
+
|
33
|
+
it "should raise an Error if the token doesn't have a signed URL " do
|
34
|
+
lambda { @download.signed_url }.should raise_error(Chimps::Error)
|
35
|
+
@token['download_token'] = {'foo' => 'bar'}
|
36
|
+
lambda { @download.signed_url }.should raise_error(Chimps::Error)
|
37
|
+
end
|
38
|
+
|
39
|
+
it "should return the signed URL from the token when present" do
|
40
|
+
@token['download_token'] = {'signed_url' => 'foobar'}
|
41
|
+
@download.signed_url.should == 'foobar'
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
46
|
+
|
47
|
+
|
48
|
+
|
data/spec/spec_helper.rb
CHANGED
@@ -4,27 +4,10 @@ CHIMPS_LIB_DIR = File.join(CHIMPS_ROOT_DIR, 'lib') unle
|
|
4
4
|
$: << CHIMPS_LIB_DIR
|
5
5
|
|
6
6
|
require 'rubygems'
|
7
|
-
require '
|
7
|
+
require 'rspec'
|
8
8
|
require 'chimps'
|
9
9
|
|
10
10
|
Dir[File.dirname(__FILE__) + "/support/**/*.rb"].each { |path| require path }
|
11
|
-
|
12
|
-
module Chimps
|
13
|
-
module Test
|
14
|
-
TMP_DIR = "/tmp/chimps_test" unless defined?(TMP_DIR)
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
Spec::Runner.configure do |config|
|
11
|
+
RSpec.configure do |config|
|
19
12
|
config.include Chimps::Test::CustomMatchers
|
20
|
-
|
21
|
-
config.before do
|
22
|
-
FileUtils.mkdir_p Chimps::Test::TMP_DIR
|
23
|
-
FileUtils.cd Chimps::Test::TMP_DIR
|
24
|
-
end
|
25
|
-
|
26
|
-
config.after do
|
27
|
-
FileUtils.rm_rf Chimps::Test::TMP_DIR
|
28
|
-
end
|
29
|
-
|
30
13
|
end
|