right_develop 1.2.2 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.ruby-version +1 -0
- data/CHANGELOG.rdoc +6 -1
- data/Rakefile +5 -3
- data/VERSION +1 -1
- data/lib/right_develop/ci/java_spec_formatter.rb +1 -1
- data/lib/right_develop/commands/git.rb +28 -2
- data/lib/right_develop/git/rake_task.rb +102 -0
- data/lib/right_develop/git.rb +27 -19
- data/lib/right_develop/parsers/sax_parser.rb +22 -5
- data/lib/right_develop/parsers/xml_post_parser.rb +11 -2
- data/lib/right_develop/parsers.rb +1 -1
- data/lib/right_develop/s3/interface.rb +298 -0
- data/lib/right_develop/s3/rake_task.rb +168 -0
- data/lib/right_develop/s3.rb +31 -0
- data/lib/right_develop/utility/git.rb +364 -0
- data/lib/right_develop/utility/shell.rb +131 -0
- data/lib/right_develop/utility/versioning.rb +183 -0
- data/lib/right_develop/utility.rb +32 -0
- data/lib/right_develop.rb +3 -1
- data/right_develop.gemspec +25 -298
- data/right_develop.rconf +3 -3
- metadata +184 -1729
- data/lib/right_develop/git/branch.rb +0 -74
- data/lib/right_develop/git/branch_collection.rb +0 -72
- data/lib/right_develop/git/commit.rb +0 -30
- data/lib/right_develop/git/repository.rb +0 -57
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
1.8.7-p371
|
data/CHANGELOG.rdoc
CHANGED
data/Rakefile
CHANGED
@@ -11,6 +11,8 @@ require 'rspec/core/rake_task'
|
|
11
11
|
require 'cucumber/rake/task'
|
12
12
|
|
13
13
|
# We use RightDevelop's CI harness in its own Rakefile. Hooray dogfood!
|
14
|
+
lib_dir = File.expand_path('../lib', __FILE__)
|
15
|
+
$: << lib_dir unless $:.include?(lib_dir)
|
14
16
|
require 'right_develop'
|
15
17
|
|
16
18
|
# But, we have a very special need, because OUR Cucumbers need to run with a pristine
|
@@ -62,10 +64,10 @@ Jeweler::Tasks.new do |gem|
|
|
62
64
|
gem.files.exclude "spec/**/*"
|
63
65
|
end
|
64
66
|
|
65
|
-
|
66
|
-
# push this gem to the public!
|
67
|
-
#Jeweler::RubygemsDotOrgTasks.new
|
67
|
+
Jeweler::RubygemsDotOrgTasks.new
|
68
68
|
|
69
69
|
CLEAN.include('pkg')
|
70
70
|
|
71
71
|
RightDevelop::CI::RakeTask.new
|
72
|
+
RightDevelop::Git::RakeTask.new
|
73
|
+
RightDevelop::S3::RakeTask.new
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
2.0.1
|
@@ -25,7 +25,7 @@ require 'time'
|
|
25
25
|
require 'builder'
|
26
26
|
|
27
27
|
# Try to load RSpec 2.x - 1.x formatters
|
28
|
-
['rspec/core', 'spec', 'rspec/core/formatters/
|
28
|
+
['rspec/core', 'spec', 'rspec/core/formatters/base_formatter', 'spec/runner/formatter/base_text_formatter'].each do |f|
|
29
29
|
begin
|
30
30
|
require f
|
31
31
|
rescue LoadError
|
@@ -1,3 +1,27 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2013 RightScale Inc
|
3
|
+
#
|
4
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
5
|
+
# a copy of this software and associated documentation files (the
|
6
|
+
# "Software"), to deal in the Software without restriction, including
|
7
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
8
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
10
|
+
# the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be
|
13
|
+
# included in all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
18
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
19
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
20
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
21
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'right_git'
|
24
|
+
require 'right_develop'
|
1
25
|
require "action_view"
|
2
26
|
|
3
27
|
module RightDevelop::Commands
|
@@ -46,8 +70,10 @@ EOS
|
|
46
70
|
|
47
71
|
case task
|
48
72
|
when "prune"
|
49
|
-
|
50
|
-
|
73
|
+
repo = ::RightGit::Repository.new(
|
74
|
+
::Dir.pwd,
|
75
|
+
::RightDevelop::Utility::Git::DEFALT_REPO_OPTIONS)
|
76
|
+
self.new(repo, :prune, options)
|
51
77
|
else
|
52
78
|
Trollop.die "unknown task #{task}"
|
53
79
|
end
|
@@ -0,0 +1,102 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2013 RightScale Inc
|
3
|
+
#
|
4
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
5
|
+
# a copy of this software and associated documentation files (the
|
6
|
+
# "Software"), to deal in the Software without restriction, including
|
7
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
8
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
10
|
+
# the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be
|
13
|
+
# included in all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
18
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
19
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
20
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
21
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
22
|
+
|
23
|
+
# Make sure the rest of RightDevelop & Gitis required, since this file can be
|
24
|
+
# required directly.
|
25
|
+
require 'right_develop/git'
|
26
|
+
|
27
|
+
# Once this file is required, the Rake DSL is loaded - don't do this except inside Rake!!
|
28
|
+
require 'rake/tasklib'
|
29
|
+
|
30
|
+
module RightDevelop::Git
|
31
|
+
|
32
|
+
class RakeTask < ::Rake::TaskLib
|
33
|
+
DEFAULT_OPTIONS = {
|
34
|
+
:git_namespace => :git,
|
35
|
+
:pre_checkout_step => nil,
|
36
|
+
:post_checkout_step => nil,
|
37
|
+
:pre_verify_step => nil,
|
38
|
+
:post_verify_step => nil,
|
39
|
+
}
|
40
|
+
|
41
|
+
include ::Rake::DSL if defined?(::Rake::DSL)
|
42
|
+
|
43
|
+
attr_accessor :git_namespace
|
44
|
+
attr_accessor :pre_checkout_step, :post_checkout_step
|
45
|
+
attr_accessor :pre_verify_step, :post_verify_step
|
46
|
+
|
47
|
+
def initialize(options = {})
|
48
|
+
# Let client provide options object-style, in our initializer
|
49
|
+
options = DEFAULT_OPTIONS.merge(options)
|
50
|
+
self.git_namespace = options[:git_namespace]
|
51
|
+
self.pre_checkout_step = options[:pre_checkout_step]
|
52
|
+
self.post_checkout_step = options[:post_checkout_step]
|
53
|
+
self.pre_verify_step = options[:pre_verify_step]
|
54
|
+
self.post_verify_step = options[:post_verify_step]
|
55
|
+
|
56
|
+
# Let client provide options DSL-style by calling our writers
|
57
|
+
yield(self) if block_given?
|
58
|
+
|
59
|
+
namespace self.git_namespace do
|
60
|
+
|
61
|
+
desc "Perform 'git submodule update --init --recursive'"
|
62
|
+
task :setup do
|
63
|
+
git.setup
|
64
|
+
end
|
65
|
+
|
66
|
+
desc "If HEAD is a branch or tag ref, ensure that all submodules are checked out to the same tag or branch or ensure consistency for SHA"
|
67
|
+
task :verify, [:revision, :base_dir] do |_, args|
|
68
|
+
revision = args[:revision].to_s.strip
|
69
|
+
base_dir = args[:base_dir].to_s.strip
|
70
|
+
revision = nil if revision.empty?
|
71
|
+
base_dir = '.' if base_dir.empty?
|
72
|
+
::Dir.chdir(base_dir) do
|
73
|
+
pre_verify_step.call(revision) if pre_verify_step
|
74
|
+
git.verify_revision(revision)
|
75
|
+
post_verify_step.call(revision) if post_verify_step
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
desc "Checkout supermodule and all submodules to given tag, branch or SHA"
|
80
|
+
task :checkout, [:revision, :base_dir] do |_, args|
|
81
|
+
revision = args[:revision].to_s.strip
|
82
|
+
base_dir = args[:base_dir].to_s.strip
|
83
|
+
raise ::ArgumentError, 'revision is required' if revision.empty?
|
84
|
+
base_dir = '.' if base_dir.empty?
|
85
|
+
::Dir.chdir(base_dir) do
|
86
|
+
pre_checkout_step.call(revision) if pre_checkout_step
|
87
|
+
git.checkout_revision(revision, :force => true, :recursive => true)
|
88
|
+
post_checkout_step.call(revision) if post_checkout_step
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
end # namespace
|
93
|
+
end # initialize
|
94
|
+
|
95
|
+
private
|
96
|
+
|
97
|
+
def git
|
98
|
+
::RightDevelop::Utility::Git
|
99
|
+
end
|
100
|
+
|
101
|
+
end # RakeTask
|
102
|
+
end # RightDevelop::Git
|
data/lib/right_develop/git.rb
CHANGED
@@ -1,22 +1,30 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
1
|
+
#
|
2
|
+
# Copyright (c) 2013 RightScale Inc
|
3
|
+
#
|
4
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
5
|
+
# a copy of this software and associated documentation files (the
|
6
|
+
# "Software"), to deal in the Software without restriction, including
|
7
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
8
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
10
|
+
# the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be
|
13
|
+
# included in all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
18
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
19
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
20
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
21
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
6
22
|
|
7
|
-
|
8
|
-
|
9
|
-
lines = message.split("\n").map { |l| l.strip }.reject { |l| l.empty? }
|
10
|
-
super(lines.last || @output)
|
11
|
-
end
|
12
|
-
end
|
23
|
+
# ancestor
|
24
|
+
require 'right_develop'
|
13
25
|
|
14
|
-
|
15
|
-
|
26
|
+
module RightDevelop
|
27
|
+
module Git
|
28
|
+
autoload :RakeTask, 'right_develop/git/rake_task'
|
16
29
|
end
|
17
|
-
|
18
|
-
require "right_develop/git/branch"
|
19
|
-
require "right_develop/git/branch_collection"
|
20
|
-
require "right_develop/git/commit"
|
21
|
-
require "right_develop/git/repository"
|
22
|
-
end
|
30
|
+
end
|
@@ -1,9 +1,18 @@
|
|
1
|
-
require 'xml/libxml'
|
2
|
-
require 'active_support/inflector'
|
3
|
-
require 'right_develop/parsers/xml_post_parser.rb'
|
4
|
-
|
5
1
|
module RightDevelop::Parsers
|
6
2
|
class SaxParser
|
3
|
+
begin
|
4
|
+
# libxml-ruby requires a C library and headers to be available before it will install; thus,
|
5
|
+
# we do not call it out as a gemspec dependency
|
6
|
+
require 'xml/libxml'
|
7
|
+
|
8
|
+
# ActiveSupport is not a runtime
|
9
|
+
require 'active_support/inflector'
|
10
|
+
|
11
|
+
AVAILABLE = true
|
12
|
+
rescue LoadError => e
|
13
|
+
AVAILABLE = false
|
14
|
+
end
|
15
|
+
|
7
16
|
extend XmlPostParser
|
8
17
|
|
9
18
|
# Parses XML into a ruby hash
|
@@ -14,6 +23,10 @@ module RightDevelop::Parsers
|
|
14
23
|
# the return content of the initial xml parser.
|
15
24
|
# @return [Array or Hash] returns rubified XML in Hash and Array format
|
16
25
|
def self.parse(text, opts = {})
|
26
|
+
unless AVAILABLE
|
27
|
+
raise NotImplementedError, "#{self.name} is unavailable on this system because libxml-ruby and/or active_support are not installed"
|
28
|
+
end
|
29
|
+
|
17
30
|
# Parse the xml text
|
18
31
|
# http://libxml.rubyforge.org/rdoc/
|
19
32
|
xml = ::XML::SaxParser::string(text)
|
@@ -32,6 +45,10 @@ module RightDevelop::Parsers
|
|
32
45
|
end
|
33
46
|
|
34
47
|
def initialize
|
48
|
+
unless AVAILABLE
|
49
|
+
raise NotImplementedError, "#{self.name} is unavailable on this system because libxml-ruby and/or active_support are not installed"
|
50
|
+
end
|
51
|
+
|
35
52
|
@tag = {}
|
36
53
|
@path = []
|
37
54
|
end
|
@@ -136,4 +153,4 @@ module RightDevelop::Parsers
|
|
136
153
|
def on_end_document
|
137
154
|
end
|
138
155
|
end
|
139
|
-
end
|
156
|
+
end
|
@@ -1,7 +1,12 @@
|
|
1
|
-
require 'active_support/inflector'
|
2
|
-
|
3
1
|
module RightDevelop::Parsers
|
4
2
|
module XmlPostParser
|
3
|
+
begin
|
4
|
+
require 'active_support/inflector'
|
5
|
+
|
6
|
+
AVAILABLE = true
|
7
|
+
rescue LoadError => e
|
8
|
+
AVAILABLE = false
|
9
|
+
end
|
5
10
|
|
6
11
|
# Parses a rubified XML hash/array, removing the top level xml tag, along with
|
7
12
|
# any arrays encoded with singular/plural for parent/child nodes.
|
@@ -40,6 +45,10 @@ module RightDevelop::Parsers
|
|
40
45
|
# @return [Array or Hash] returns a ruby Array or Hash with top level xml tags removed,
|
41
46
|
# as well as any extra XML encoded array tags.
|
42
47
|
def self.remove_nesting(xml_object)
|
48
|
+
unless AVAILABLE
|
49
|
+
raise NotImplementedError, "#{self.name} is unavailable on this system because libxml-ruby and/or active_support are not installed"
|
50
|
+
end
|
51
|
+
|
43
52
|
if xml_object.length != 1 || (!xml_object.is_a?(Hash) && !xml_object.is_a?(Array))
|
44
53
|
raise ArgumentError, "xml_object format doesn't have a single top level entry"
|
45
54
|
end
|
@@ -0,0 +1,298 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2013 RightScale Inc
|
3
|
+
#
|
4
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
5
|
+
# a copy of this software and associated documentation files (the
|
6
|
+
# "Software"), to deal in the Software without restriction, including
|
7
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
8
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
10
|
+
# the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be
|
13
|
+
# included in all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
18
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
19
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
20
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
21
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'right_aws'
|
24
|
+
|
25
|
+
module RightDevelop
|
26
|
+
module S3
|
27
|
+
|
28
|
+
# Provides a Ruby OOP interface to Amazon S3.
|
29
|
+
#
|
30
|
+
# Note: filters are used as options for multiple storage actions below and
|
31
|
+
# refers to an array of Regexp or wildcard-style filter strings
|
32
|
+
# (e.g. '*.txt'). they are used to match file paths relative to a given
|
33
|
+
# subdirectory or else from the root of the bucket or directory on disk).
|
34
|
+
class Interface
|
35
|
+
NO_SLASHES_REGEXP = /^[^\/]+$/
|
36
|
+
|
37
|
+
DEFAULT_OPTIONS = {
|
38
|
+
:filters => nil,
|
39
|
+
:subdirectory => nil,
|
40
|
+
:recursive => true,
|
41
|
+
:aws_access_key_id => nil,
|
42
|
+
:aws_secret_access_key => nil,
|
43
|
+
:logger => nil
|
44
|
+
}.freeze
|
45
|
+
|
46
|
+
# @option options [String] :aws_access_key_id defaults to using env var value
|
47
|
+
# @option options [String] :aws_secret_access_key defaults to using env var value
|
48
|
+
# @option options [Logger] :logger or nil to log to STDOUT
|
49
|
+
def initialize(options={})
|
50
|
+
options = DEFAULT_OPTIONS.merge(options)
|
51
|
+
|
52
|
+
aws_access_key_id = options[:aws_access_key_id]
|
53
|
+
aws_secret_access_key = options[:aws_secret_access_key]
|
54
|
+
unless aws_access_key_id && aws_secret_access_key
|
55
|
+
raise ::ArgumentError,
|
56
|
+
'Missing one or both mandatory options - :aws_access_key_id and :aws_secret_access_key'
|
57
|
+
end
|
58
|
+
|
59
|
+
@logger = options[:logger] || Logger.new(STDOUT)
|
60
|
+
@s3 = ::RightAws::S3Interface.new(aws_access_key_id, aws_secret_access_key, :logger => @logger)
|
61
|
+
end
|
62
|
+
|
63
|
+
attr_accessor :logger
|
64
|
+
|
65
|
+
# Lists the files in the given bucket.
|
66
|
+
#
|
67
|
+
# @param [String] bucket to query
|
68
|
+
# @option options [String] :subdirectory to start from or nil
|
69
|
+
# @option options [TrueClass|FalseClass] :recursive true if recursive (default)
|
70
|
+
# @option options [Array] :filters for returned paths or nil or empty
|
71
|
+
# @return [Array] list of relative file paths or empty
|
72
|
+
def list_files(bucket, options={})
|
73
|
+
options = DEFAULT_OPTIONS.dup.merge(options)
|
74
|
+
prefix = normalize_subdirectory_path(options[:subdirectory])
|
75
|
+
filters = normalize_filters(options)
|
76
|
+
files = []
|
77
|
+
trivial_filters = filters.select { |filter| filter.is_a?(String) }
|
78
|
+
if trivial_filters.empty?
|
79
|
+
@s3.incrementally_list_bucket(bucket, 'prefix' => prefix) do |response|
|
80
|
+
incremental_files = response[:contents].map do |details|
|
81
|
+
details[:key][(prefix.length)..-1]
|
82
|
+
end
|
83
|
+
files += filter_files(incremental_files, filters)
|
84
|
+
end
|
85
|
+
else
|
86
|
+
trivial_filters.each do |filename|
|
87
|
+
begin
|
88
|
+
# use head to query file existence.
|
89
|
+
@s3.head(bucket, "#{prefix}#{filename}")
|
90
|
+
files << filename
|
91
|
+
rescue RightAws::AwsError => e
|
92
|
+
# do nothing if file not found
|
93
|
+
raise unless '404' == e.http_code
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
return files
|
98
|
+
end
|
99
|
+
|
100
|
+
# Downloads all files from the given bucket to the given directory.
|
101
|
+
#
|
102
|
+
# @param [String] bucket for download
|
103
|
+
# @param [String] to_dir_path source directory to upload
|
104
|
+
# @option options [String] :subdirectory to start from or nil
|
105
|
+
# @option options [TrueClass|FalseClass] :recursive true if recursive (default)
|
106
|
+
# @option options [Array] :filters for returned paths or nil or empty
|
107
|
+
# @return [Fixnum] count of uploaded files
|
108
|
+
def download_files(bucket, to_dir_path, options={})
|
109
|
+
options = DEFAULT_OPTIONS.dup.merge(options)
|
110
|
+
prefix = normalize_subdirectory_path(options[:subdirectory])
|
111
|
+
files = list_files(bucket, options)
|
112
|
+
if files.empty?
|
113
|
+
logger.info("No files found in \"#{bucket}/#{prefix}\"")
|
114
|
+
else
|
115
|
+
logger.info("Downloading #{files.count} files...")
|
116
|
+
prefix = normalize_subdirectory_path(options[:subdirectory])
|
117
|
+
downloaded = 0
|
118
|
+
files.each do |path|
|
119
|
+
key = "#{prefix}#{path}"
|
120
|
+
to_file_path = File.join(to_dir_path, path)
|
121
|
+
parent_path = File.dirname(to_file_path)
|
122
|
+
FileUtils.mkdir_p(parent_path) unless File.directory?(parent_path)
|
123
|
+
|
124
|
+
disk_file = to_file_path
|
125
|
+
file_md5 = File.exist?(disk_file) && Digest::MD5.hexdigest(File.read(disk_file))
|
126
|
+
|
127
|
+
if file_md5
|
128
|
+
head = @s3.head(bucket, key) rescue nil
|
129
|
+
key_md5 = head && head['etag'].gsub(/[^0-9a-fA-F]/, '')
|
130
|
+
skip = (key_md5 == file_md5)
|
131
|
+
end
|
132
|
+
|
133
|
+
if skip
|
134
|
+
logger.info("Skipping #{bucket}/#{key} (identical contents)")
|
135
|
+
else
|
136
|
+
logger.info("Downloading #{bucket}/#{key}")
|
137
|
+
::File.open(to_file_path, 'wb') do |f|
|
138
|
+
@s3.get(bucket, key) { |chunk| f.write(chunk) }
|
139
|
+
end
|
140
|
+
downloaded += 1
|
141
|
+
end
|
142
|
+
|
143
|
+
logger.info("Downloaded to \"#{to_file_path}\"")
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
downloaded
|
148
|
+
end
|
149
|
+
|
150
|
+
# Uploads all files from the given directory (ignoring any empty
|
151
|
+
# directories) to the given bucket.
|
152
|
+
#
|
153
|
+
# @param [String] bucket for upload
|
154
|
+
# @param [String] from_dir_path source directory to upload
|
155
|
+
# @option options [String] :subdirectory to start from or nil
|
156
|
+
# @option options [TrueClass|FalseClass] :recursive true if recursive (default)
|
157
|
+
# @option options [Array] :filters for returned paths or nil or empty
|
158
|
+
# @option options [String] :visibility for uploaded files, defaults to 'public-read'
|
159
|
+
# @return [Fixnum] count of downloaded files
|
160
|
+
def upload_files(bucket, from_dir_path, options={})
|
161
|
+
Dir.chdir(from_dir_path) do
|
162
|
+
logger.info("Working in #{Dir.pwd.inspect}")
|
163
|
+
options = DEFAULT_OPTIONS.dup.merge(options)
|
164
|
+
prefix = normalize_subdirectory_path(options[:subdirectory])
|
165
|
+
filters = normalize_filters(options)
|
166
|
+
pattern = options[:recursive] ? '**/*' : '*'
|
167
|
+
files = Dir.glob(pattern).select { |path| File.file?(path) }
|
168
|
+
filter_files(files, filters)
|
169
|
+
access = normalize_access(options)
|
170
|
+
uploaded = 0
|
171
|
+
files.each do |path|
|
172
|
+
key = "#{prefix}#{path}"
|
173
|
+
file_md5 = Digest::MD5.hexdigest(File.read(path))
|
174
|
+
File.open(path, 'rb') do |f|
|
175
|
+
head = @s3.head(bucket, key) rescue nil
|
176
|
+
key_md5 = head && head['etag'].gsub(/[^0-9a-fA-F]/, '')
|
177
|
+
|
178
|
+
if file_md5 == key_md5
|
179
|
+
logger.info("Skipping #{bucket}/#{key} (identical contents)")
|
180
|
+
else
|
181
|
+
logger.info("Uploading to #{bucket}/#{key}")
|
182
|
+
@s3.put(bucket, key, f, 'x-amz-acl' => access)
|
183
|
+
uploaded += 1
|
184
|
+
end
|
185
|
+
end
|
186
|
+
end
|
187
|
+
|
188
|
+
uploaded
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
# Deletes all files from the given bucket.
|
193
|
+
#
|
194
|
+
# @param [String] bucket for delete
|
195
|
+
# @option options [String] :subdirectory to start from or nil
|
196
|
+
# @option options [TrueClass|FalseClass] :recursive true if recursive (default)
|
197
|
+
# @option options [Regexp] :filter for files to delete or nil
|
198
|
+
# @return [Fixnum] count of deleted files
|
199
|
+
def delete_files(bucket, options={})
|
200
|
+
options = DEFAULT_OPTIONS.dup.merge(options)
|
201
|
+
prefix = normalize_subdirectory_path(options[:subdirectory])
|
202
|
+
files = list_files(bucket, options)
|
203
|
+
if files.empty?
|
204
|
+
logger.info("No files found in \"#{bucket}/#{prefix}\"")
|
205
|
+
else
|
206
|
+
logger.info("Deleting #{files.count} files...")
|
207
|
+
files.each do |path|
|
208
|
+
@s3.delete(bucket, "#{prefix}#{path}")
|
209
|
+
logger.info("Deleted \"#{bucket}/#{prefix}#{path}\"")
|
210
|
+
end
|
211
|
+
end
|
212
|
+
return files.size
|
213
|
+
end
|
214
|
+
|
215
|
+
protected
|
216
|
+
|
217
|
+
# Normalizes a relative file path for use with S3.
|
218
|
+
#
|
219
|
+
# @param [String] subdirectory
|
220
|
+
def normalize_file_path(path)
|
221
|
+
# remove leading and trailing slashes and change any multiple slashes to single.
|
222
|
+
return (path || '').gsub("\\", '/').gsub(/^\/+/, '').gsub(/\/+$/, '').gsub(/\/+/, '/')
|
223
|
+
end
|
224
|
+
|
225
|
+
# Normalizes subdirectory path for use with S3.
|
226
|
+
#
|
227
|
+
# @param [String] path
|
228
|
+
# @return [String] normalized path
|
229
|
+
def normalize_subdirectory_path(path)
|
230
|
+
path = normalize_file_path(path)
|
231
|
+
path += '/' unless path.empty?
|
232
|
+
return path
|
233
|
+
end
|
234
|
+
|
235
|
+
# Normalizes storage filters from options.
|
236
|
+
#
|
237
|
+
# @option options [Array] :filters for returned paths or nil or empty
|
238
|
+
def normalize_filters(options)
|
239
|
+
initial_filters = Array(options[:filters])
|
240
|
+
normalized_filters = nil
|
241
|
+
|
242
|
+
# support trivial filters as simple string array for direct lookup of
|
243
|
+
# one or more S3 object (since listing entire buckets can be slow).
|
244
|
+
# recursion always requires a listing so that cannot be trivial.
|
245
|
+
if !options[:recursive] && initial_filters.size == 1
|
246
|
+
# filter is trivial unless it contains wildcards. more than one
|
247
|
+
# non-wildcard filenames delimited by semicolon can be trivial.
|
248
|
+
filter = initial_filters.first
|
249
|
+
if filter.kind_of?(String) && filter == filter.gsub('*', '').gsub('?', '')
|
250
|
+
normalized_filters = filter.split(';').uniq
|
251
|
+
end
|
252
|
+
end
|
253
|
+
unless normalized_filters
|
254
|
+
normalized_filters = []
|
255
|
+
normalized_filters << NO_SLASHES_REGEXP unless options[:recursive]
|
256
|
+
initial_filters.each do |filter|
|
257
|
+
if filter.kind_of?(String)
|
258
|
+
# split on semicolon (;) and OR the result into one regular expression.
|
259
|
+
# example: "*.tar;*.tgz;*.zip" -> /^.*\.tar|.*\.tgz|.*\.zip$/
|
260
|
+
#
|
261
|
+
# convert wildcard-style filter string (e.g. '*.txt') to Regexp.
|
262
|
+
escaped = Regexp.escape(filter).gsub("\\*", '.*').gsub("\\?", '.').gsub(';', '|')
|
263
|
+
regexp = Regexp.compile("^#{escaped}$")
|
264
|
+
filter = regexp
|
265
|
+
end
|
266
|
+
normalized_filters << filter unless normalized_filters.index(filter)
|
267
|
+
end
|
268
|
+
end
|
269
|
+
return normalized_filters
|
270
|
+
end
|
271
|
+
|
272
|
+
# Normalizes access from options (for uploading files).
|
273
|
+
#
|
274
|
+
# Note: access strings are AWS S3-style but can easily be mapped to any
|
275
|
+
# bucket storage implementation which supports ACLs.
|
276
|
+
#
|
277
|
+
# @option options [String] :access requested ACL or nil for public-read
|
278
|
+
# @return @return [String] normalized access
|
279
|
+
def normalize_access(options)
|
280
|
+
access = options[:access].to_s.empty? ? nil : options[:access]
|
281
|
+
return access || 'public-read'
|
282
|
+
end
|
283
|
+
|
284
|
+
# Filters the given list of file paths using the given filters, if any.
|
285
|
+
#
|
286
|
+
# @param [Array] files to filter
|
287
|
+
# @param [Array] filters for matching or empty
|
288
|
+
# @return [Array] filtered files
|
289
|
+
def filter_files(files, filters)
|
290
|
+
return files if filters.empty?
|
291
|
+
|
292
|
+
# select each path only if it matches all filters.
|
293
|
+
return files.select { |path| filters.all? { |filter| filter.match(path) } }
|
294
|
+
end
|
295
|
+
|
296
|
+
end # Interface
|
297
|
+
end # Buckets
|
298
|
+
end # RightDevelop
|