right_scraper 3.2.6 → 5.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/lib/right_scraper.rb +16 -34
- data/lib/right_scraper/builders.rb +32 -0
- data/lib/right_scraper/builders/base.rb +19 -20
- data/lib/right_scraper/builders/filesystem.rb +8 -6
- data/lib/right_scraper/builders/union.rb +4 -1
- data/lib/right_scraper/loggers.rb +31 -0
- data/lib/right_scraper/loggers/base.rb +113 -0
- data/lib/right_scraper/loggers/default.rb +98 -0
- data/lib/right_scraper/{scraper.rb → main.rb} +53 -9
- data/lib/right_scraper/processes.rb +33 -0
- data/lib/right_scraper/processes/shell.rb +227 -0
- data/lib/right_scraper/processes/{ssh.rb → ssh_agent.rb} +4 -0
- data/lib/right_scraper/processes/svn_client.rb +117 -0
- data/lib/right_scraper/processes/warden.rb +358 -0
- data/lib/right_scraper/registered_base.rb +154 -0
- data/lib/right_scraper/repositories.rb +33 -0
- data/lib/right_scraper/repositories/base.rb +271 -232
- data/lib/right_scraper/repositories/download.rb +8 -6
- data/lib/right_scraper/repositories/git.rb +8 -9
- data/lib/right_scraper/repositories/svn.rb +8 -8
- data/lib/right_scraper/resources.rb +32 -0
- data/lib/right_scraper/resources/base.rb +5 -1
- data/lib/right_scraper/resources/cookbook.rb +34 -27
- data/lib/right_scraper/resources/workflow.rb +27 -28
- data/lib/right_scraper/retrievers.rb +34 -0
- data/lib/right_scraper/retrievers/base.rb +80 -84
- data/lib/right_scraper/retrievers/checkout_base.rb +178 -0
- data/lib/right_scraper/retrievers/download.rb +125 -117
- data/lib/right_scraper/retrievers/git.rb +377 -223
- data/lib/right_scraper/retrievers/svn.rb +102 -62
- data/lib/right_scraper/scanners.rb +37 -0
- data/lib/right_scraper/scanners/base.rb +77 -80
- data/lib/right_scraper/scanners/cookbook_manifest.rb +31 -30
- data/lib/right_scraper/scanners/cookbook_metadata.rb +380 -35
- data/lib/right_scraper/scanners/cookbook_s3_upload.rb +56 -53
- data/lib/right_scraper/scanners/union.rb +61 -58
- data/lib/right_scraper/scanners/workflow_manifest.rb +55 -54
- data/lib/right_scraper/scanners/workflow_metadata.rb +41 -39
- data/lib/right_scraper/scanners/workflow_s3_upload.rb +59 -55
- data/lib/right_scraper/scrapers.rb +32 -0
- data/lib/right_scraper/scrapers/base.rb +217 -205
- data/lib/right_scraper/scrapers/cookbook.rb +42 -40
- data/lib/right_scraper/scrapers/workflow.rb +57 -58
- data/lib/right_scraper/version.rb +3 -0
- data/right_scraper.gemspec +12 -16
- metadata +57 -163
- data/Gemfile +0 -15
- data/Rakefile +0 -89
- data/lib/right_scraper/logger.rb +0 -107
- data/lib/right_scraper/loggers/noisy.rb +0 -85
- data/lib/right_scraper/repositories/mock.rb +0 -70
- data/lib/right_scraper/retrievers/checkout.rb +0 -79
- data/lib/right_scraper/scraper_logger.rb +0 -66
- data/lib/right_scraper/svn_client.rb +0 -164
- data/right_scraper.rconf +0 -13
- data/spec/builder_spec.rb +0 -50
- data/spec/cookbook_helper.rb +0 -73
- data/spec/cookbook_manifest_spec.rb +0 -93
- data/spec/cookbook_s3_upload_spec.rb +0 -159
- data/spec/download/download_retriever_spec.rb +0 -118
- data/spec/download/download_retriever_spec_helper.rb +0 -72
- data/spec/download/download_spec.rb +0 -128
- data/spec/download/multi_dir_spec.rb +0 -106
- data/spec/download/multi_dir_spec_helper.rb +0 -40
- data/spec/git/cookbook_spec.rb +0 -165
- data/spec/git/demokey +0 -27
- data/spec/git/demokey.pub +0 -1
- data/spec/git/password_key +0 -30
- data/spec/git/password_key.pub +0 -1
- data/spec/git/repository_spec.rb +0 -110
- data/spec/git/retriever_spec.rb +0 -553
- data/spec/git/retriever_spec_helper.rb +0 -112
- data/spec/git/scraper_spec.rb +0 -151
- data/spec/git/ssh_spec.rb +0 -174
- data/spec/git/url_spec.rb +0 -103
- data/spec/logger_spec.rb +0 -185
- data/spec/repository_spec.rb +0 -111
- data/spec/retriever_spec_helper.rb +0 -146
- data/spec/scanner_spec.rb +0 -61
- data/spec/scraper_helper.rb +0 -88
- data/spec/scraper_spec.rb +0 -147
- data/spec/spec_helper.rb +0 -185
- data/spec/svn/cookbook_spec.rb +0 -96
- data/spec/svn/multi_svn_spec.rb +0 -64
- data/spec/svn/multi_svn_spec_helper.rb +0 -40
- data/spec/svn/repository_spec.rb +0 -72
- data/spec/svn/retriever_spec.rb +0 -266
- data/spec/svn/scraper_spec.rb +0 -90
- data/spec/svn/svn_retriever_spec_helper.rb +0 -90
- data/spec/svn/url_spec.rb +0 -47
- data/spec/url_spec.rb +0 -164
@@ -1,90 +0,0 @@
|
|
1
|
-
#--
|
2
|
-
# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
|
3
|
-
#
|
4
|
-
# Permission is hereby granted, free of charge, to any person obtaining
|
5
|
-
# a copy of this software and associated documentation files (the
|
6
|
-
# 'Software'), to deal in the Software without restriction, including
|
7
|
-
# without limitation the rights to use, copy, modify, merge, publish,
|
8
|
-
# distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
-
# permit persons to whom the Software is furnished to do so, subject to
|
10
|
-
# the following conditions:
|
11
|
-
#
|
12
|
-
# The above copyright notice and this permission notice shall be
|
13
|
-
# included in all copies or substantial portions of the Software.
|
14
|
-
#
|
15
|
-
# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
16
|
-
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
-
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
18
|
-
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
19
|
-
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
20
|
-
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
21
|
-
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
22
|
-
#++
|
23
|
-
|
24
|
-
require File.expand_path(File.join(File.dirname(__FILE__), '..', 'retriever_spec_helper'))
|
25
|
-
|
26
|
-
module RightScraper
|
27
|
-
|
28
|
-
# SVN implementation of retriever spec helper
|
29
|
-
# See parent class for methods headers comments
|
30
|
-
class SvnRetrieverSpecHelper < RetrieverSpecHelper
|
31
|
-
include RightScraper::SvnClient
|
32
|
-
|
33
|
-
def svn_repo_path
|
34
|
-
File.join(@tmpdir, "svn")
|
35
|
-
end
|
36
|
-
|
37
|
-
attr_reader :repo
|
38
|
-
alias_method :repository, :repo
|
39
|
-
|
40
|
-
def scraper_path
|
41
|
-
File.join(@tmpdir, "scraper")
|
42
|
-
end
|
43
|
-
|
44
|
-
def repo_url
|
45
|
-
file_prefix = 'file://'
|
46
|
-
file_prefix += '/' if RUBY_PLATFORM =~ /mswin|mingw/
|
47
|
-
url = "#{file_prefix}#{svn_repo_path}"
|
48
|
-
end
|
49
|
-
|
50
|
-
alias_method :repo_dir, :repo_path
|
51
|
-
|
52
|
-
def initialize
|
53
|
-
super()
|
54
|
-
FileUtils.mkdir(svn_repo_path)
|
55
|
-
@repo = RightScraper::Repositories::Base.from_hash(
|
56
|
-
:display_name => 'test repo',
|
57
|
-
:repo_type => :svn,
|
58
|
-
:url => repo_url)
|
59
|
-
output = `svnadmin create #{svn_repo_path}`
|
60
|
-
raise "Can't create repo: #{output}" unless $?.success?
|
61
|
-
run_svn_no_chdir "checkout", repo_url, repo_path
|
62
|
-
make_cookbooks
|
63
|
-
commit_content
|
64
|
-
end
|
65
|
-
|
66
|
-
def make_cookbooks
|
67
|
-
create_cookbook(repo_path, repo_content)
|
68
|
-
end
|
69
|
-
|
70
|
-
def delete(location, log="")
|
71
|
-
run_svn "delete", location
|
72
|
-
end
|
73
|
-
|
74
|
-
def commit_content(commit_message='commit message')
|
75
|
-
run_svn "add", Dir.glob(File.join(repo_path, '**/*'))
|
76
|
-
run_svn "commit", "--message", commit_message
|
77
|
-
end
|
78
|
-
|
79
|
-
def commit_id(index_from_last=0)
|
80
|
-
lines = run_svn_with_buffered_output "log", "-l", (index_from_last + 1).to_s, "-r", "HEAD:0"
|
81
|
-
id = nil
|
82
|
-
lines.each do |line|
|
83
|
-
if line =~ /^r(\d+)/
|
84
|
-
id = $1
|
85
|
-
end
|
86
|
-
end
|
87
|
-
id
|
88
|
-
end
|
89
|
-
end
|
90
|
-
end
|
data/spec/svn/url_spec.rb
DELETED
@@ -1,47 +0,0 @@
|
|
1
|
-
#--
|
2
|
-
# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
|
3
|
-
#
|
4
|
-
# Permission is hereby granted, free of charge, to any person obtaining
|
5
|
-
# a copy of this software and associated documentation files (the
|
6
|
-
# 'Software'), to deal in the Software without restriction, including
|
7
|
-
# without limitation the rights to use, copy, modify, merge, publish,
|
8
|
-
# distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
-
# permit persons to whom the Software is furnished to do so, subject to
|
10
|
-
# the following conditions:
|
11
|
-
#
|
12
|
-
# The above copyright notice and this permission notice shall be
|
13
|
-
# included in all copies or substantial portions of the Software.
|
14
|
-
#
|
15
|
-
# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
16
|
-
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
-
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
18
|
-
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
19
|
-
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
20
|
-
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
21
|
-
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
22
|
-
#++
|
23
|
-
|
24
|
-
require File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec_helper'))
|
25
|
-
|
26
|
-
describe RightScraper::Repositories::Svn do
|
27
|
-
def make_repo(url)
|
28
|
-
@repo = RightScraper::Repositories::Svn.from_hash(:display_name => 'test repo',
|
29
|
-
:repo_type => :svn,
|
30
|
-
:url => url,
|
31
|
-
:first_credential => "foo:b/ar",
|
32
|
-
:second_credential => "blah")
|
33
|
-
end
|
34
|
-
include RightScraper::SpecHelpers::ProductionModeEnvironment
|
35
|
-
|
36
|
-
it 'should not throw an error when creating a repository for SVN URIs' do
|
37
|
-
lambda do
|
38
|
-
make_repo "http://rightscale.com/%20CBLAH"
|
39
|
-
end.should_not raise_exception
|
40
|
-
lambda do
|
41
|
-
make_repo "svn://rightscale.com/%20CBLAH"
|
42
|
-
end.should_not raise_exception
|
43
|
-
lambda do
|
44
|
-
make_repo "svn+ssh://rightscale.com/%20CBLAH"
|
45
|
-
end.should_not raise_exception
|
46
|
-
end
|
47
|
-
end
|
data/spec/url_spec.rb
DELETED
@@ -1,164 +0,0 @@
|
|
1
|
-
#--
|
2
|
-
# Copyright: Copyright (c) 2010-2011 RightScale, Inc.
|
3
|
-
#
|
4
|
-
# Permission is hereby granted, free of charge, to any person obtaining
|
5
|
-
# a copy of this software and associated documentation files (the
|
6
|
-
# 'Software'), to deal in the Software without restriction, including
|
7
|
-
# without limitation the rights to use, copy, modify, merge, publish,
|
8
|
-
# distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
-
# permit persons to whom the Software is furnished to do so, subject to
|
10
|
-
# the following conditions:
|
11
|
-
#
|
12
|
-
# The above copyright notice and this permission notice shall be
|
13
|
-
# included in all copies or substantial portions of the Software.
|
14
|
-
#
|
15
|
-
# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
16
|
-
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
-
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
18
|
-
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
19
|
-
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
20
|
-
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
21
|
-
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
22
|
-
#++
|
23
|
-
|
24
|
-
require File.expand_path(File.join(File.dirname(__FILE__), 'spec_helper'))
|
25
|
-
require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'right_scraper'))
|
26
|
-
|
27
|
-
describe RightScraper::Repositories::Svn do
|
28
|
-
def make_repo(url)
|
29
|
-
@repo = RightScraper::Repositories::Svn.from_hash(
|
30
|
-
:display_name => 'test repo',
|
31
|
-
:repo_type => :download,
|
32
|
-
:url => url,
|
33
|
-
:first_credential => "foo:b/ar",
|
34
|
-
:second_credential => "foo@bar")
|
35
|
-
end
|
36
|
-
before(:each) do
|
37
|
-
@oldtest = ENV['DEVELOPMENT']
|
38
|
-
end
|
39
|
-
after(:each) do
|
40
|
-
if @oldtest.nil?
|
41
|
-
ENV.delete('DEVELOPMENT')
|
42
|
-
else
|
43
|
-
ENV['DEVELOPMENT'] = @oldtest
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
context 'in production mode' do
|
48
|
-
before(:each) do
|
49
|
-
ENV.delete('DEVELOPMENT')
|
50
|
-
end
|
51
|
-
|
52
|
-
it 'should not throw an error when creating a repository for a normal URI' do
|
53
|
-
lambda do
|
54
|
-
make_repo "http://rightscale.com/%20CBLAH"
|
55
|
-
end.should_not raise_exception
|
56
|
-
lambda do
|
57
|
-
make_repo "http://172.12.3.42/%20CBLAH"
|
58
|
-
end.should_not raise_exception
|
59
|
-
lambda do
|
60
|
-
make_repo "http://110.12.3.42/%20CBLAH"
|
61
|
-
end.should_not raise_exception
|
62
|
-
end
|
63
|
-
|
64
|
-
it 'should refuse to create repositories for hosts that don\'t exist' do
|
65
|
-
lambda do
|
66
|
-
make_repo "http://nonexistent.invalid/%20CBLAH"
|
67
|
-
end.should raise_exception(RuntimeError, /Invalid URI/)
|
68
|
-
end
|
69
|
-
|
70
|
-
it 'should refuse to create repositories for private networks' do
|
71
|
-
["10.0.2.43", "172.18.3.42", "192.168.4.2"].each do |ip|
|
72
|
-
lambda do
|
73
|
-
make_repo "http://#{ip}/%20CBLAH"
|
74
|
-
end.should raise_exception(RuntimeError, /Invalid URI/)
|
75
|
-
end
|
76
|
-
end
|
77
|
-
|
78
|
-
it 'should refuse to create repositories for loopback addresses' do
|
79
|
-
lambda do
|
80
|
-
make_repo "http://localhost/%20CBLAH"
|
81
|
-
end.should raise_exception(RuntimeError, /Invalid URI/)
|
82
|
-
lambda do
|
83
|
-
make_repo "http://127.0.0.1/%20CBLAH"
|
84
|
-
end.should raise_exception(RuntimeError, /Invalid URI/)
|
85
|
-
lambda do
|
86
|
-
make_repo "http://127.3.2.1/%20CBLAH"
|
87
|
-
end.should raise_exception(RuntimeError, /Invalid URI/)
|
88
|
-
end
|
89
|
-
|
90
|
-
it 'should refuse to create repositories for file:/// URIs' do
|
91
|
-
lambda do
|
92
|
-
make_repo "file://var/run/something/%20CBLAH"
|
93
|
-
end.should raise_exception(RuntimeError, /Invalid URI/)
|
94
|
-
end
|
95
|
-
|
96
|
-
it 'should refuse to create repositories for our EC2 metadata server' do
|
97
|
-
lambda do
|
98
|
-
make_repo "http://169.254.169.254/%20CBLAH"
|
99
|
-
end.should raise_exception(RuntimeError, /Invalid URI/)
|
100
|
-
end
|
101
|
-
|
102
|
-
it 'should refuse to create repositories for even valid IPv6 addresses' do
|
103
|
-
lambda do
|
104
|
-
make_repo "http://[::ffff:128.111.1.1]/%20CBLAH"
|
105
|
-
end.should raise_exception(RuntimeError, /Invalid URI/)
|
106
|
-
end
|
107
|
-
end
|
108
|
-
|
109
|
-
context 'in development mode' do
|
110
|
-
before(:each) do
|
111
|
-
ENV['DEVELOPMENT'] = "yes"
|
112
|
-
end
|
113
|
-
|
114
|
-
it 'should not throw an error when creating a repository for a normal URI' do
|
115
|
-
lambda do
|
116
|
-
make_repo "http://rightscale.com/%20CBLAH"
|
117
|
-
end.should_not raise_exception
|
118
|
-
end
|
119
|
-
|
120
|
-
it 'should create repositories for hosts that don\'t exist' do
|
121
|
-
lambda do
|
122
|
-
make_repo "http://nonexistent.invalid/%20CBLAH"
|
123
|
-
end.should_not raise_exception
|
124
|
-
end
|
125
|
-
|
126
|
-
it 'should create repositories for private networks' do
|
127
|
-
["10.0.2.43", "172.18.3.42", "192.168.4.2"].each do |ip|
|
128
|
-
lambda do
|
129
|
-
make_repo "http://#{ip}/%20CBLAH"
|
130
|
-
end.should_not raise_exception
|
131
|
-
end
|
132
|
-
end
|
133
|
-
|
134
|
-
it 'should create repositories for loopback addresses' do
|
135
|
-
lambda do
|
136
|
-
make_repo "http://localhost/%20CBLAH"
|
137
|
-
end.should_not raise_exception
|
138
|
-
lambda do
|
139
|
-
make_repo "http://127.0.0.1/%20CBLAH"
|
140
|
-
end.should_not raise_exception
|
141
|
-
lambda do
|
142
|
-
make_repo "http://127.3.2.1/%20CBLAH"
|
143
|
-
end.should_not raise_exception
|
144
|
-
end
|
145
|
-
|
146
|
-
it 'should create repositories for file:/// URIs' do
|
147
|
-
lambda do
|
148
|
-
make_repo "file://var/run/something/%20CBLAH"
|
149
|
-
end.should_not raise_exception
|
150
|
-
end
|
151
|
-
|
152
|
-
it 'should create repositories for our EC2 metadata server' do
|
153
|
-
lambda do
|
154
|
-
make_repo "http://169.254.169.254/%20CBLAH"
|
155
|
-
end.should_not raise_exception
|
156
|
-
end
|
157
|
-
|
158
|
-
it 'should create repositories for even valid IPv6 addresses' do
|
159
|
-
lambda do
|
160
|
-
make_repo "http://[::ffff:128.111.1.1]/%20CBLAH"
|
161
|
-
end.should_not raise_exception
|
162
|
-
end
|
163
|
-
end
|
164
|
-
end
|