nightcrawler_swift 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Changelog.md +11 -0
- data/Gemfile.lock +1 -1
- data/README.md +40 -25
- data/bin/nswift +8 -0
- data/lib/nightcrawler_swift/cli.rb +208 -0
- data/lib/nightcrawler_swift/command.rb +12 -2
- data/lib/nightcrawler_swift/{delete.rb → commands/delete.rb} +1 -1
- data/lib/nightcrawler_swift/{download.rb → commands/download.rb} +1 -1
- data/lib/nightcrawler_swift/{list.rb → commands/list.rb} +0 -0
- data/lib/nightcrawler_swift/{sync.rb → commands/sync.rb} +2 -2
- data/lib/nightcrawler_swift/commands/upload.rb +25 -0
- data/lib/nightcrawler_swift/connection.rb +44 -23
- data/lib/nightcrawler_swift/exceptions.rb +1 -0
- data/lib/nightcrawler_swift/tasks/asset_sync.rake +7 -1
- data/lib/nightcrawler_swift/version.rb +1 -1
- data/lib/nightcrawler_swift.rb +26 -9
- data/spec/fixtures/auth_success.json +6 -6
- data/spec/lib/nightcrawler_swift/cli_spec.rb +293 -0
- data/spec/lib/nightcrawler_swift/command_spec.rb +55 -14
- data/spec/lib/nightcrawler_swift/{delete_spec.rb → commands/delete_spec.rb} +15 -8
- data/spec/lib/nightcrawler_swift/{download_spec.rb → commands/download_spec.rb} +16 -7
- data/spec/lib/nightcrawler_swift/{list_spec.rb → commands/list_spec.rb} +12 -8
- data/spec/lib/nightcrawler_swift/{sync_spec.rb → commands/sync_spec.rb} +10 -1
- data/spec/lib/nightcrawler_swift/{upload_spec.rb → commands/upload_spec.rb} +23 -8
- data/spec/lib/nightcrawler_swift/connection_spec.rb +32 -20
- data/spec/lib/nightcrawler_swift/tasks/asset_sync_spec.rb +19 -3
- data/spec/lib/nightcrawler_swift_spec.rb +46 -15
- metadata +23 -18
- data/lib/nightcrawler_swift/upload.rb +0 -17
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 616e1624837906ed1b5381823a1de196ba8b41e7
|
4
|
+
data.tar.gz: 2e4d0b8eb39afb34e6cda6d5e7725c6938a3a817
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 193eccd937c649b468e4b24cff3728d22a6acd7de8ac824876732bff14fbb145e33dafb222ff6069e350912c849b7c55d5e75a56e25a11357e6a8e9d368ece6c
|
7
|
+
data.tar.gz: 466c00ebec3b67d64efca29ec820afea48ad74a0e702617dc91e570cf0c8c08bfca7a0808b2d8c8d01d3c09d398c64d7df7e7766312e1c35826fa500dd1af264
|
data/Changelog.md
CHANGED
@@ -1,5 +1,16 @@
|
|
1
1
|
# Changelog
|
2
2
|
|
3
|
+
## 0.4.0
|
4
|
+
|
5
|
+
- Better catalog selection
|
6
|
+
- Treatment for no catalogs returned
|
7
|
+
- Splited asset_sync task in two other tasks: sync and asset_sync
|
8
|
+
- Configurable verify_ssl and timeout
|
9
|
+
- Automatic connect/reconnect of commands
|
10
|
+
- Bugfix: download command was not using the bucket/container name
|
11
|
+
- Bugfix: the etag header must not be quoted
|
12
|
+
- CLI with basic commands (list, download, upload and delete)
|
13
|
+
|
3
14
|
## 0.3.0
|
4
15
|
|
5
16
|
- Included ```Cache-Control``` header to upload command through max_age configuration parameter
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -1,7 +1,4 @@
|
|
1
|
-
[](https://codeclimate.com/github/tulios/nightcrawler_swift)
|
2
|
-
[](https://travis-ci.org/tulios/nightcrawler_swift)
|
3
|
-
[](http://badge.fury.io/rb/nightcrawler_swift)
|
4
|
-
# Nightcrawler Swift
|
1
|
+
# Nightcrawler Swift [](https://codeclimate.com/github/tulios/nightcrawler_swift) [](https://travis-ci.org/tulios/nightcrawler_swift) [](http://badge.fury.io/rb/nightcrawler_swift)
|
5
2
|
|
6
3
|
Like the X-Men nightcrawler this gem teleports your assets to a OpenStack Swift bucket/container. It was designed to sync your assets with OpenStack Swift and allow some operations with your buckets/containers.
|
7
4
|
|
@@ -32,9 +29,17 @@ config.nightcrawler_swift.tenant_name = "nightcrawler"
|
|
32
29
|
config.nightcrawler_swift.username = "my_username1"
|
33
30
|
config.nightcrawler_swift.password = "my_password1"
|
34
31
|
config.nightcrawler_swift.auth_url = "https://auth.url.com:123/v2.0/tokens"
|
35
|
-
config.nightcrawler_swift.max_age = 3600 #optional
|
36
32
|
```
|
37
|
-
|
33
|
+
|
34
|
+
__Optional configurations:__
|
35
|
+
|
36
|
+
```ruby
|
37
|
+
config.nightcrawler_swift.max_age = 3600 # default: nil
|
38
|
+
config.nightcrawler_swift.verify_ssl = true # default: false
|
39
|
+
config.nightcrawler_swift.timeout = 10 # in seconds, default: nil
|
40
|
+
```
|
41
|
+
|
42
|
+
_max_age_ will be used to define *Cache-Control:max-age=<value>* header.
|
38
43
|
|
39
44
|
By default it will use ```Rails.logger``` as logger, to change that use a different logger in configurations, like:
|
40
45
|
|
@@ -48,7 +53,7 @@ config.nightcrawler_swift.logger = Logger.new(STDOUT)
|
|
48
53
|
rake nightcrawler_swift:rails:asset_sync
|
49
54
|
```
|
50
55
|
|
51
|
-
It will invoke ```rake assets:precompile``` and will copy your public directory to swift bucket/container.
|
56
|
+
It will invoke ```rake assets:precompile``` and will copy your public directory to swift bucket/container. To sync the public directory without the asset precompilation use the task: ```nightcrawler_swift:rails:sync```
|
52
57
|
|
53
58
|
### Programatically
|
54
59
|
|
@@ -57,14 +62,21 @@ It will invoke ```rake assets:precompile``` and will copy your public directory
|
|
57
62
|
```ruby
|
58
63
|
NightcrawlerSwift.configure({
|
59
64
|
bucket: "rogue",
|
60
|
-
tenant_name: "nightcrawler"
|
65
|
+
tenant_name: "nightcrawler",
|
61
66
|
username: "my_username1",
|
62
67
|
password: "my_password1",
|
63
|
-
auth_url: "https://auth.url.com:123/v2.0/tokens"
|
64
|
-
max_age: 3600 #optional
|
68
|
+
auth_url: "https://auth.url.com:123/v2.0/tokens"
|
65
69
|
})
|
66
70
|
```
|
67
71
|
|
72
|
+
__Optional configurations:__
|
73
|
+
|
74
|
+
```ruby
|
75
|
+
max_age: 3600,
|
76
|
+
verify_ssl: true,
|
77
|
+
timeout: 10
|
78
|
+
```
|
79
|
+
|
68
80
|
By default it will use ```Logger.new(STDOUT)``` as logger, to change that use:
|
69
81
|
|
70
82
|
```ruby
|
@@ -79,21 +91,7 @@ NightcrawlerSwift.sync File.expand_path("./my-dir")
|
|
79
91
|
|
80
92
|
## Commands
|
81
93
|
|
82
|
-
NightcrawlerSwift has some useful built-in commands. All commands require the configuration and
|
83
|
-
|
84
|
-
To Establish the connection, use:
|
85
|
-
|
86
|
-
```ruby
|
87
|
-
NightcrawlerSwift.connection.connect!
|
88
|
-
```
|
89
|
-
|
90
|
-
To check if the connection is still valid, use:
|
91
|
-
|
92
|
-
```ruby
|
93
|
-
NightcrawlerSwift.connection.connected?
|
94
|
-
```
|
95
|
-
|
96
|
-
To reconnect just use ```NightcrawlerSwift.connection.connect!``` again.
|
94
|
+
NightcrawlerSwift has some useful built-in commands. All commands require the configuration and will __automatically__ connect/reconnect to keystone when necessary.
|
97
95
|
|
98
96
|
### Upload
|
99
97
|
|
@@ -131,6 +129,23 @@ delete.execute "my_file_path.txt"
|
|
131
129
|
# true / false
|
132
130
|
```
|
133
131
|
|
132
|
+
## Connection
|
133
|
+
|
134
|
+
To manually establish the connection with keystone, use:
|
135
|
+
|
136
|
+
```ruby
|
137
|
+
NightcrawlerSwift.connection.connect!
|
138
|
+
```
|
139
|
+
|
140
|
+
To check if the connection is still valid, use:
|
141
|
+
|
142
|
+
```ruby
|
143
|
+
NightcrawlerSwift.connection.connected?
|
144
|
+
```
|
145
|
+
|
146
|
+
To reconnect just use ```NightcrawlerSwift.connection.connect!``` again.
|
147
|
+
|
148
|
+
|
134
149
|
## Contributing
|
135
150
|
|
136
151
|
1. Fork it ( https://github.com/tulios/nightcrawler_swift/fork )
|
data/bin/nswift
ADDED
@@ -0,0 +1,208 @@
|
|
1
|
+
require "optparse"
|
2
|
+
require "json"
|
3
|
+
|
4
|
+
module NightcrawlerSwift
|
5
|
+
class CLI
|
6
|
+
CONFIG_FILE = ".nswiftrc"
|
7
|
+
CACHE_FILE = ".nswift_cache"
|
8
|
+
COMMANDS = {
|
9
|
+
"list" => {
|
10
|
+
description: "Lists all files of the bucket/container. Ex: nswift list",
|
11
|
+
command: NightcrawlerSwift::List
|
12
|
+
},
|
13
|
+
|
14
|
+
"download" => {
|
15
|
+
description: "Downloads a file by path. Format: nswift download <swift path> Ex: nswift download assets/robots.txt > my-robots.txt",
|
16
|
+
command: NightcrawlerSwift::Download
|
17
|
+
},
|
18
|
+
|
19
|
+
"upload" => {
|
20
|
+
description: "Uploads some file. Format: nswift upload <real path> <swift path> Ex: nswift upload assets/robots.txt robots.txt",
|
21
|
+
command: NightcrawlerSwift::Upload
|
22
|
+
},
|
23
|
+
|
24
|
+
"delete" => {
|
25
|
+
description: "Deletes a file by path. Format: nswift delete <swift path> Ex: nswift delete robots.txt",
|
26
|
+
command: NightcrawlerSwift::Delete
|
27
|
+
}
|
28
|
+
}
|
29
|
+
|
30
|
+
attr_reader :opt_parser, :options
|
31
|
+
attr_accessor :argv
|
32
|
+
|
33
|
+
def initialize argv
|
34
|
+
@argv = argv
|
35
|
+
STDOUT.sync = true
|
36
|
+
NightcrawlerSwift.logger.formatter = lambda {|severity, datetime, progname, msg| "#{msg}\n"}
|
37
|
+
end
|
38
|
+
|
39
|
+
def run
|
40
|
+
configure_default_options
|
41
|
+
parse_parameters
|
42
|
+
@command_name = argv.shift
|
43
|
+
validate_command_and_options
|
44
|
+
execute_command if @command_name
|
45
|
+
end
|
46
|
+
|
47
|
+
protected
|
48
|
+
def command_list command
|
49
|
+
array = command.new.execute
|
50
|
+
array.each {|hash| log hash["name"]}
|
51
|
+
end
|
52
|
+
|
53
|
+
def command_download command
|
54
|
+
filepath = argv.first
|
55
|
+
log command.new.execute(filepath)
|
56
|
+
end
|
57
|
+
|
58
|
+
def command_upload command
|
59
|
+
realpath = argv.shift
|
60
|
+
swiftpath = argv.shift
|
61
|
+
|
62
|
+
uploaded = command.new.execute swiftpath, File.open(File.expand_path(realpath), "r")
|
63
|
+
log(uploaded ? "success" : "failure")
|
64
|
+
end
|
65
|
+
|
66
|
+
def command_delete command
|
67
|
+
filepath = argv.first
|
68
|
+
deleted = command.new.execute(filepath).to_json
|
69
|
+
log(deleted ? "success" : "failure")
|
70
|
+
end
|
71
|
+
|
72
|
+
def user_home_dir
|
73
|
+
Dir.home
|
74
|
+
end
|
75
|
+
|
76
|
+
def log string
|
77
|
+
NightcrawlerSwift.logger.info string
|
78
|
+
end
|
79
|
+
|
80
|
+
private
|
81
|
+
def configure_default_options
|
82
|
+
@options = OpenStruct.new
|
83
|
+
@options.configured = true
|
84
|
+
@options.default_config_file = true
|
85
|
+
@options.config_file = File.expand_path(File.join(user_home_dir, CONFIG_FILE))
|
86
|
+
@options.cache_file = File.expand_path(File.join(user_home_dir, CACHE_FILE))
|
87
|
+
@options.command = nil
|
88
|
+
end
|
89
|
+
|
90
|
+
def validate_command_and_options
|
91
|
+
if @command_name.nil? or argv.nil?
|
92
|
+
log opt_parser.help
|
93
|
+
exit
|
94
|
+
end
|
95
|
+
|
96
|
+
unless options.configured
|
97
|
+
log "You must configure your swift credentials, take a look at:\n #{options.config_file}"
|
98
|
+
exit
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
def config_hash
|
103
|
+
@config_hash ||= JSON.parse(File.read(options.config_file))
|
104
|
+
|
105
|
+
rescue Errno::ENOENT => e
|
106
|
+
log "No such file or directory - #{options.config_file}"
|
107
|
+
exit 1
|
108
|
+
end
|
109
|
+
|
110
|
+
def execute_command
|
111
|
+
NightcrawlerSwift.configure config_hash
|
112
|
+
connect_and_execute do
|
113
|
+
if command = COMMANDS[@command_name]
|
114
|
+
command_module = command[:command]
|
115
|
+
command_method = "command_#{@command_name}"
|
116
|
+
self.send(command_method, command_module)
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
rescue Exceptions::BaseError => e
|
121
|
+
log e.message
|
122
|
+
exit 1
|
123
|
+
end
|
124
|
+
|
125
|
+
def connect_and_execute &block
|
126
|
+
path = options.cache_file
|
127
|
+
if File.exist?(path)
|
128
|
+
hash = JSON.parse File.read(path)
|
129
|
+
NightcrawlerSwift.connection.auth_response = OpenStruct.new(hash)
|
130
|
+
NightcrawlerSwift.connection.configure
|
131
|
+
end
|
132
|
+
|
133
|
+
begin
|
134
|
+
block.call
|
135
|
+
ensure
|
136
|
+
File.open(path, "w") do |f|
|
137
|
+
f.write(NightcrawlerSwift.connection.auth_response.to_h.to_json)
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
def parse_parameters
|
143
|
+
configure_opt_parser
|
144
|
+
opt_parser.parse!(argv)
|
145
|
+
check_rcfile if options.default_config_file
|
146
|
+
|
147
|
+
rescue OptionParser::InvalidOption => e
|
148
|
+
log e.message
|
149
|
+
exit 1
|
150
|
+
end
|
151
|
+
|
152
|
+
def configure_opt_parser
|
153
|
+
@opt_parser = OptionParser.new do |opts|
|
154
|
+
opts.banner = "nswift #{NightcrawlerSwift::VERSION}"
|
155
|
+
opts.separator "Usage: nswift command [options]"
|
156
|
+
|
157
|
+
opts.separator ""
|
158
|
+
opts.separator "commands:"
|
159
|
+
COMMANDS.keys.each do |key|
|
160
|
+
opts.separator " #{key}\t\t\t #{COMMANDS[key][:description]}"
|
161
|
+
end
|
162
|
+
|
163
|
+
opts.separator ""
|
164
|
+
opts.separator "options:"
|
165
|
+
|
166
|
+
opts.on("-c", "--config=PATH", String, "Alternative '#{CONFIG_FILE}' file") do |path|
|
167
|
+
path = File.expand_path(path.strip)
|
168
|
+
log "Using custom config file at: #{path}"
|
169
|
+
options.config_file = path
|
170
|
+
options.default_config_file = false
|
171
|
+
end
|
172
|
+
|
173
|
+
opts.on_tail("-h", "--help", "Show this message") do
|
174
|
+
log opts.help
|
175
|
+
exit
|
176
|
+
end
|
177
|
+
|
178
|
+
opts.on_tail("-v", "--version", "Show version") do
|
179
|
+
log NightcrawlerSwift::VERSION
|
180
|
+
exit
|
181
|
+
end
|
182
|
+
end
|
183
|
+
end
|
184
|
+
|
185
|
+
def check_rcfile
|
186
|
+
unless File.exist?(options.config_file)
|
187
|
+
File.open(options.config_file, "w") { |f|
|
188
|
+
f.write(sample_rcfile)
|
189
|
+
}
|
190
|
+
end
|
191
|
+
|
192
|
+
if sample_rcfile == File.read(options.config_file)
|
193
|
+
options.configured = false
|
194
|
+
end
|
195
|
+
end
|
196
|
+
|
197
|
+
def sample_rcfile
|
198
|
+
JSON.pretty_generate({
|
199
|
+
bucket: "<bucket/container name>",
|
200
|
+
tenant_name: "<tenant name>",
|
201
|
+
username: "<username>",
|
202
|
+
password: "<password>",
|
203
|
+
auth_url: "<auth url, ex: https://auth.url.com:123/v2.0/tokens>"
|
204
|
+
})
|
205
|
+
end
|
206
|
+
|
207
|
+
end
|
208
|
+
end
|
@@ -2,7 +2,13 @@ module NightcrawlerSwift
|
|
2
2
|
class Command
|
3
3
|
|
4
4
|
def connection
|
5
|
-
NightcrawlerSwift.connection
|
5
|
+
NightcrawlerSwift.connection.tap do |conn|
|
6
|
+
conn.connect! unless conn.connected?
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
def options
|
11
|
+
NightcrawlerSwift.options
|
6
12
|
end
|
7
13
|
|
8
14
|
def execute
|
@@ -32,7 +38,11 @@ module NightcrawlerSwift
|
|
32
38
|
private
|
33
39
|
|
34
40
|
def resource_for url
|
35
|
-
RestClient::Resource.new
|
41
|
+
RestClient::Resource.new(
|
42
|
+
url,
|
43
|
+
verify_ssl: options.verify_ssl,
|
44
|
+
timeout: options.timeout
|
45
|
+
)
|
36
46
|
end
|
37
47
|
|
38
48
|
def prepare_params params
|
@@ -3,7 +3,7 @@ module NightcrawlerSwift
|
|
3
3
|
|
4
4
|
def execute path
|
5
5
|
response = delete "#{connection.upload_url}/#{path}", headers: {accept: :json }
|
6
|
-
|
6
|
+
[200, 201].include?(response.code)
|
7
7
|
|
8
8
|
rescue RestClient::ResourceNotFound => e
|
9
9
|
raise Exceptions::NotFoundError.new(e)
|
File without changes
|
@@ -7,12 +7,12 @@ module NightcrawlerSwift
|
|
7
7
|
end
|
8
8
|
|
9
9
|
def execute dir_path
|
10
|
-
@logger.info "dir_path: #{dir_path}"
|
10
|
+
@logger.info "[NightcrawlerSwift] dir_path: #{dir_path}"
|
11
11
|
Dir["#{dir_path}/**/**"].each do |fullpath|
|
12
12
|
path = fullpath.gsub("#{dir_path}/", "")
|
13
13
|
|
14
14
|
unless File.directory?(fullpath)
|
15
|
-
@logger.info path
|
15
|
+
@logger.info "[NightcrawlerSwift] #{path}"
|
16
16
|
@upload.execute path, File.open(fullpath, "r")
|
17
17
|
end
|
18
18
|
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module NightcrawlerSwift
|
2
|
+
class Upload < Command
|
3
|
+
|
4
|
+
def execute path, file
|
5
|
+
content = file.read
|
6
|
+
headers = {etag: etag(content), content_type: content_type(file)}
|
7
|
+
headers.merge!(cache_control: "max-age=#{options.max_age}") if options.max_age
|
8
|
+
response = put "#{connection.upload_url}/#{path}", body: content, headers: headers
|
9
|
+
[200, 201].include?(response.code)
|
10
|
+
|
11
|
+
rescue RestClient::UnprocessableEntity => e
|
12
|
+
raise Exceptions::ValidationError.new(e)
|
13
|
+
end
|
14
|
+
|
15
|
+
private
|
16
|
+
def content_type file
|
17
|
+
MultiMime.by_file(file).content_type
|
18
|
+
end
|
19
|
+
|
20
|
+
def etag content
|
21
|
+
Digest::MD5.hexdigest(content)
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
25
|
+
end
|