nightcrawler_swift 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. checksums.yaml +4 -4
  2. data/Changelog.md +11 -0
  3. data/Gemfile.lock +1 -1
  4. data/README.md +40 -25
  5. data/bin/nswift +8 -0
  6. data/lib/nightcrawler_swift/cli.rb +208 -0
  7. data/lib/nightcrawler_swift/command.rb +12 -2
  8. data/lib/nightcrawler_swift/{delete.rb → commands/delete.rb} +1 -1
  9. data/lib/nightcrawler_swift/{download.rb → commands/download.rb} +1 -1
  10. data/lib/nightcrawler_swift/{list.rb → commands/list.rb} +0 -0
  11. data/lib/nightcrawler_swift/{sync.rb → commands/sync.rb} +2 -2
  12. data/lib/nightcrawler_swift/commands/upload.rb +25 -0
  13. data/lib/nightcrawler_swift/connection.rb +44 -23
  14. data/lib/nightcrawler_swift/exceptions.rb +1 -0
  15. data/lib/nightcrawler_swift/tasks/asset_sync.rake +7 -1
  16. data/lib/nightcrawler_swift/version.rb +1 -1
  17. data/lib/nightcrawler_swift.rb +26 -9
  18. data/spec/fixtures/auth_success.json +6 -6
  19. data/spec/lib/nightcrawler_swift/cli_spec.rb +293 -0
  20. data/spec/lib/nightcrawler_swift/command_spec.rb +55 -14
  21. data/spec/lib/nightcrawler_swift/{delete_spec.rb → commands/delete_spec.rb} +15 -8
  22. data/spec/lib/nightcrawler_swift/{download_spec.rb → commands/download_spec.rb} +16 -7
  23. data/spec/lib/nightcrawler_swift/{list_spec.rb → commands/list_spec.rb} +12 -8
  24. data/spec/lib/nightcrawler_swift/{sync_spec.rb → commands/sync_spec.rb} +10 -1
  25. data/spec/lib/nightcrawler_swift/{upload_spec.rb → commands/upload_spec.rb} +23 -8
  26. data/spec/lib/nightcrawler_swift/connection_spec.rb +32 -20
  27. data/spec/lib/nightcrawler_swift/tasks/asset_sync_spec.rb +19 -3
  28. data/spec/lib/nightcrawler_swift_spec.rb +46 -15
  29. metadata +23 -18
  30. data/lib/nightcrawler_swift/upload.rb +0 -17
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 792e1b2a4c65b85291d7a5c733fbd238f6b4f531
4
- data.tar.gz: b34fa251f341f191b7627d8ac200bb0daa43fdbd
3
+ metadata.gz: 616e1624837906ed1b5381823a1de196ba8b41e7
4
+ data.tar.gz: 2e4d0b8eb39afb34e6cda6d5e7725c6938a3a817
5
5
  SHA512:
6
- metadata.gz: be4f048df14fe2453b4fc22ed8e495434c5f4fb51009f8e5765fae6f61cbc160dcf12ae4c2d274e4aa69096884b629d8e83538f03bd14dd4992ad39dc1e66f91
7
- data.tar.gz: d9bd21e55c32c1ee73f16e817c7edf2c529552d94402c615a63a37026f01eb5ba477b21c461878f35fcd54953e574553dda79e5ce6f50cf5b745e61ad55030a7
6
+ metadata.gz: 193eccd937c649b468e4b24cff3728d22a6acd7de8ac824876732bff14fbb145e33dafb222ff6069e350912c849b7c55d5e75a56e25a11357e6a8e9d368ece6c
7
+ data.tar.gz: 466c00ebec3b67d64efca29ec820afea48ad74a0e702617dc91e570cf0c8c08bfca7a0808b2d8c8d01d3c09d398c64d7df7e7766312e1c35826fa500dd1af264
data/Changelog.md CHANGED
@@ -1,5 +1,16 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.4.0
4
+
5
+ - Better catalog selection
6
+ - Treatment for no catalogs returned
7
+ - Splited asset_sync task in two other tasks: sync and asset_sync
8
+ - Configurable verify_ssl and timeout
9
+ - Automatic connect/reconnect of commands
10
+ - Bugfix: download command was not using the bucket/container name
11
+ - Bugfix: the etag header must not be quoted
12
+ - CLI with basic commands (list, download, upload and delete)
13
+
3
14
  ## 0.3.0
4
15
 
5
16
  - Included ```Cache-Control``` header to upload command through max_age configuration parameter
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- nightcrawler_swift (0.3.0)
4
+ nightcrawler_swift (0.4.0)
5
5
  multi_mime (>= 1.0.1)
6
6
  rest-client
7
7
 
data/README.md CHANGED
@@ -1,7 +1,4 @@
1
- [![Code Climate](https://codeclimate.com/github/tulios/nightcrawler_swift/badges/gpa.svg)](https://codeclimate.com/github/tulios/nightcrawler_swift)
2
- [![Travis](https://api.travis-ci.org/tulios/nightcrawler_swift.svg?branch=master)](https://travis-ci.org/tulios/nightcrawler_swift)
3
- [![Gem Version](https://badge.fury.io/rb/nightcrawler_swift.svg)](http://badge.fury.io/rb/nightcrawler_swift)
4
- # Nightcrawler Swift
1
+ # Nightcrawler Swift [![Code Climate](https://codeclimate.com/github/tulios/nightcrawler_swift/badges/gpa.svg)](https://codeclimate.com/github/tulios/nightcrawler_swift) [![Travis](https://api.travis-ci.org/tulios/nightcrawler_swift.svg?branch=master)](https://travis-ci.org/tulios/nightcrawler_swift) [![Gem Version](https://badge.fury.io/rb/nightcrawler_swift.svg)](http://badge.fury.io/rb/nightcrawler_swift)
5
2
 
6
3
  Like the X-Men nightcrawler this gem teleports your assets to a OpenStack Swift bucket/container. It was designed to sync your assets with OpenStack Swift and allow some operations with your buckets/containers.
7
4
 
@@ -32,9 +29,17 @@ config.nightcrawler_swift.tenant_name = "nightcrawler"
32
29
  config.nightcrawler_swift.username = "my_username1"
33
30
  config.nightcrawler_swift.password = "my_password1"
34
31
  config.nightcrawler_swift.auth_url = "https://auth.url.com:123/v2.0/tokens"
35
- config.nightcrawler_swift.max_age = 3600 #optional
36
32
  ```
37
- **max_age** will be used to define *Cache-Control:max-age=<value>* header. It's not required.
33
+
34
+ __Optional configurations:__
35
+
36
+ ```ruby
37
+ config.nightcrawler_swift.max_age = 3600 # default: nil
38
+ config.nightcrawler_swift.verify_ssl = true # default: false
39
+ config.nightcrawler_swift.timeout = 10 # in seconds, default: nil
40
+ ```
41
+
42
+ _max_age_ will be used to define *Cache-Control:max-age=<value>* header.
38
43
 
39
44
  By default it will use ```Rails.logger``` as logger, to change that use a different logger in configurations, like:
40
45
 
@@ -48,7 +53,7 @@ config.nightcrawler_swift.logger = Logger.new(STDOUT)
48
53
  rake nightcrawler_swift:rails:asset_sync
49
54
  ```
50
55
 
51
- It will invoke ```rake assets:precompile``` and will copy your public directory to swift bucket/container.
56
+ It will invoke ```rake assets:precompile``` and will copy your public directory to swift bucket/container. To sync the public directory without the asset precompilation use the task: ```nightcrawler_swift:rails:sync```
52
57
 
53
58
  ### Programatically
54
59
 
@@ -57,14 +62,21 @@ It will invoke ```rake assets:precompile``` and will copy your public directory
57
62
  ```ruby
58
63
  NightcrawlerSwift.configure({
59
64
  bucket: "rogue",
60
- tenant_name: "nightcrawler"
65
+ tenant_name: "nightcrawler",
61
66
  username: "my_username1",
62
67
  password: "my_password1",
63
- auth_url: "https://auth.url.com:123/v2.0/tokens",
64
- max_age: 3600 #optional
68
+ auth_url: "https://auth.url.com:123/v2.0/tokens"
65
69
  })
66
70
  ```
67
71
 
72
+ __Optional configurations:__
73
+
74
+ ```ruby
75
+ max_age: 3600,
76
+ verify_ssl: true,
77
+ timeout: 10
78
+ ```
79
+
68
80
  By default it will use ```Logger.new(STDOUT)``` as logger, to change that use:
69
81
 
70
82
  ```ruby
@@ -79,21 +91,7 @@ NightcrawlerSwift.sync File.expand_path("./my-dir")
79
91
 
80
92
  ## Commands
81
93
 
82
- NightcrawlerSwift has some useful built-in commands. All commands require the configuration and a established connection.
83
-
84
- To Establish the connection, use:
85
-
86
- ```ruby
87
- NightcrawlerSwift.connection.connect!
88
- ```
89
-
90
- To check if the connection is still valid, use:
91
-
92
- ```ruby
93
- NightcrawlerSwift.connection.connected?
94
- ```
95
-
96
- To reconnect just use ```NightcrawlerSwift.connection.connect!``` again.
94
+ NightcrawlerSwift has some useful built-in commands. All commands require the configuration and will __automatically__ connect/reconnect to keystone when necessary.
97
95
 
98
96
  ### Upload
99
97
 
@@ -131,6 +129,23 @@ delete.execute "my_file_path.txt"
131
129
  # true / false
132
130
  ```
133
131
 
132
+ ## Connection
133
+
134
+ To manually establish the connection with keystone, use:
135
+
136
+ ```ruby
137
+ NightcrawlerSwift.connection.connect!
138
+ ```
139
+
140
+ To check if the connection is still valid, use:
141
+
142
+ ```ruby
143
+ NightcrawlerSwift.connection.connected?
144
+ ```
145
+
146
+ To reconnect just use ```NightcrawlerSwift.connection.connect!``` again.
147
+
148
+
134
149
  ## Contributing
135
150
 
136
151
  1. Fork it ( https://github.com/tulios/nightcrawler_swift/fork )
data/bin/nswift ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ $LOAD_PATH.unshift File.dirname(__FILE__) + "/../lib"
4
+
5
+ require "nightcrawler_swift"
6
+ require "nightcrawler_swift/cli"
7
+
8
+ NightcrawlerSwift::CLI.new(ARGV).run
@@ -0,0 +1,208 @@
1
+ require "optparse"
2
+ require "json"
3
+
4
+ module NightcrawlerSwift
5
+ class CLI
6
+ CONFIG_FILE = ".nswiftrc"
7
+ CACHE_FILE = ".nswift_cache"
8
+ COMMANDS = {
9
+ "list" => {
10
+ description: "Lists all files of the bucket/container. Ex: nswift list",
11
+ command: NightcrawlerSwift::List
12
+ },
13
+
14
+ "download" => {
15
+ description: "Downloads a file by path. Format: nswift download <swift path> Ex: nswift download assets/robots.txt > my-robots.txt",
16
+ command: NightcrawlerSwift::Download
17
+ },
18
+
19
+ "upload" => {
20
+ description: "Uploads some file. Format: nswift upload <real path> <swift path> Ex: nswift upload assets/robots.txt robots.txt",
21
+ command: NightcrawlerSwift::Upload
22
+ },
23
+
24
+ "delete" => {
25
+ description: "Deletes a file by path. Format: nswift delete <swift path> Ex: nswift delete robots.txt",
26
+ command: NightcrawlerSwift::Delete
27
+ }
28
+ }
29
+
30
+ attr_reader :opt_parser, :options
31
+ attr_accessor :argv
32
+
33
+ def initialize argv
34
+ @argv = argv
35
+ STDOUT.sync = true
36
+ NightcrawlerSwift.logger.formatter = lambda {|severity, datetime, progname, msg| "#{msg}\n"}
37
+ end
38
+
39
+ def run
40
+ configure_default_options
41
+ parse_parameters
42
+ @command_name = argv.shift
43
+ validate_command_and_options
44
+ execute_command if @command_name
45
+ end
46
+
47
+ protected
48
+ def command_list command
49
+ array = command.new.execute
50
+ array.each {|hash| log hash["name"]}
51
+ end
52
+
53
+ def command_download command
54
+ filepath = argv.first
55
+ log command.new.execute(filepath)
56
+ end
57
+
58
+ def command_upload command
59
+ realpath = argv.shift
60
+ swiftpath = argv.shift
61
+
62
+ uploaded = command.new.execute swiftpath, File.open(File.expand_path(realpath), "r")
63
+ log(uploaded ? "success" : "failure")
64
+ end
65
+
66
+ def command_delete command
67
+ filepath = argv.first
68
+ deleted = command.new.execute(filepath).to_json
69
+ log(deleted ? "success" : "failure")
70
+ end
71
+
72
+ def user_home_dir
73
+ Dir.home
74
+ end
75
+
76
+ def log string
77
+ NightcrawlerSwift.logger.info string
78
+ end
79
+
80
+ private
81
+ def configure_default_options
82
+ @options = OpenStruct.new
83
+ @options.configured = true
84
+ @options.default_config_file = true
85
+ @options.config_file = File.expand_path(File.join(user_home_dir, CONFIG_FILE))
86
+ @options.cache_file = File.expand_path(File.join(user_home_dir, CACHE_FILE))
87
+ @options.command = nil
88
+ end
89
+
90
+ def validate_command_and_options
91
+ if @command_name.nil? or argv.nil?
92
+ log opt_parser.help
93
+ exit
94
+ end
95
+
96
+ unless options.configured
97
+ log "You must configure your swift credentials, take a look at:\n #{options.config_file}"
98
+ exit
99
+ end
100
+ end
101
+
102
+ def config_hash
103
+ @config_hash ||= JSON.parse(File.read(options.config_file))
104
+
105
+ rescue Errno::ENOENT => e
106
+ log "No such file or directory - #{options.config_file}"
107
+ exit 1
108
+ end
109
+
110
+ def execute_command
111
+ NightcrawlerSwift.configure config_hash
112
+ connect_and_execute do
113
+ if command = COMMANDS[@command_name]
114
+ command_module = command[:command]
115
+ command_method = "command_#{@command_name}"
116
+ self.send(command_method, command_module)
117
+ end
118
+ end
119
+
120
+ rescue Exceptions::BaseError => e
121
+ log e.message
122
+ exit 1
123
+ end
124
+
125
+ def connect_and_execute &block
126
+ path = options.cache_file
127
+ if File.exist?(path)
128
+ hash = JSON.parse File.read(path)
129
+ NightcrawlerSwift.connection.auth_response = OpenStruct.new(hash)
130
+ NightcrawlerSwift.connection.configure
131
+ end
132
+
133
+ begin
134
+ block.call
135
+ ensure
136
+ File.open(path, "w") do |f|
137
+ f.write(NightcrawlerSwift.connection.auth_response.to_h.to_json)
138
+ end
139
+ end
140
+ end
141
+
142
+ def parse_parameters
143
+ configure_opt_parser
144
+ opt_parser.parse!(argv)
145
+ check_rcfile if options.default_config_file
146
+
147
+ rescue OptionParser::InvalidOption => e
148
+ log e.message
149
+ exit 1
150
+ end
151
+
152
+ def configure_opt_parser
153
+ @opt_parser = OptionParser.new do |opts|
154
+ opts.banner = "nswift #{NightcrawlerSwift::VERSION}"
155
+ opts.separator "Usage: nswift command [options]"
156
+
157
+ opts.separator ""
158
+ opts.separator "commands:"
159
+ COMMANDS.keys.each do |key|
160
+ opts.separator " #{key}\t\t\t #{COMMANDS[key][:description]}"
161
+ end
162
+
163
+ opts.separator ""
164
+ opts.separator "options:"
165
+
166
+ opts.on("-c", "--config=PATH", String, "Alternative '#{CONFIG_FILE}' file") do |path|
167
+ path = File.expand_path(path.strip)
168
+ log "Using custom config file at: #{path}"
169
+ options.config_file = path
170
+ options.default_config_file = false
171
+ end
172
+
173
+ opts.on_tail("-h", "--help", "Show this message") do
174
+ log opts.help
175
+ exit
176
+ end
177
+
178
+ opts.on_tail("-v", "--version", "Show version") do
179
+ log NightcrawlerSwift::VERSION
180
+ exit
181
+ end
182
+ end
183
+ end
184
+
185
+ def check_rcfile
186
+ unless File.exist?(options.config_file)
187
+ File.open(options.config_file, "w") { |f|
188
+ f.write(sample_rcfile)
189
+ }
190
+ end
191
+
192
+ if sample_rcfile == File.read(options.config_file)
193
+ options.configured = false
194
+ end
195
+ end
196
+
197
+ def sample_rcfile
198
+ JSON.pretty_generate({
199
+ bucket: "<bucket/container name>",
200
+ tenant_name: "<tenant name>",
201
+ username: "<username>",
202
+ password: "<password>",
203
+ auth_url: "<auth url, ex: https://auth.url.com:123/v2.0/tokens>"
204
+ })
205
+ end
206
+
207
+ end
208
+ end
@@ -2,7 +2,13 @@ module NightcrawlerSwift
2
2
  class Command
3
3
 
4
4
  def connection
5
- NightcrawlerSwift.connection
5
+ NightcrawlerSwift.connection.tap do |conn|
6
+ conn.connect! unless conn.connected?
7
+ end
8
+ end
9
+
10
+ def options
11
+ NightcrawlerSwift.options
6
12
  end
7
13
 
8
14
  def execute
@@ -32,7 +38,11 @@ module NightcrawlerSwift
32
38
  private
33
39
 
34
40
  def resource_for url
35
- RestClient::Resource.new url, verify_ssl: false
41
+ RestClient::Resource.new(
42
+ url,
43
+ verify_ssl: options.verify_ssl,
44
+ timeout: options.timeout
45
+ )
36
46
  end
37
47
 
38
48
  def prepare_params params
@@ -3,7 +3,7 @@ module NightcrawlerSwift
3
3
 
4
4
  def execute path
5
5
  response = delete "#{connection.upload_url}/#{path}", headers: {accept: :json }
6
- JSON.parse(response.body)
6
+ [200, 201].include?(response.code)
7
7
 
8
8
  rescue RestClient::ResourceNotFound => e
9
9
  raise Exceptions::NotFoundError.new(e)
@@ -2,7 +2,7 @@ module NightcrawlerSwift
2
2
  class Download < Command
3
3
 
4
4
  def execute path
5
- response = get "#{connection.public_url}/#{path}"
5
+ response = get "#{connection.public_url}/#{options.bucket}/#{path}"
6
6
  response.body
7
7
 
8
8
  rescue RestClient::ResourceNotFound => e
@@ -7,12 +7,12 @@ module NightcrawlerSwift
7
7
  end
8
8
 
9
9
  def execute dir_path
10
- @logger.info "dir_path: #{dir_path}"
10
+ @logger.info "[NightcrawlerSwift] dir_path: #{dir_path}"
11
11
  Dir["#{dir_path}/**/**"].each do |fullpath|
12
12
  path = fullpath.gsub("#{dir_path}/", "")
13
13
 
14
14
  unless File.directory?(fullpath)
15
- @logger.info path
15
+ @logger.info "[NightcrawlerSwift] #{path}"
16
16
  @upload.execute path, File.open(fullpath, "r")
17
17
  end
18
18
  end
@@ -0,0 +1,25 @@
1
+ module NightcrawlerSwift
2
+ class Upload < Command
3
+
4
+ def execute path, file
5
+ content = file.read
6
+ headers = {etag: etag(content), content_type: content_type(file)}
7
+ headers.merge!(cache_control: "max-age=#{options.max_age}") if options.max_age
8
+ response = put "#{connection.upload_url}/#{path}", body: content, headers: headers
9
+ [200, 201].include?(response.code)
10
+
11
+ rescue RestClient::UnprocessableEntity => e
12
+ raise Exceptions::ValidationError.new(e)
13
+ end
14
+
15
+ private
16
+ def content_type file
17
+ MultiMime.by_file(file).content_type
18
+ end
19
+
20
+ def etag content
21
+ Digest::MD5.hexdigest(content)
22
+ end
23
+
24
+ end
25
+ end