right_scraper 3.2.6 → 5.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (92) hide show
  1. checksums.yaml +7 -0
  2. data/lib/right_scraper.rb +16 -34
  3. data/lib/right_scraper/builders.rb +32 -0
  4. data/lib/right_scraper/builders/base.rb +19 -20
  5. data/lib/right_scraper/builders/filesystem.rb +8 -6
  6. data/lib/right_scraper/builders/union.rb +4 -1
  7. data/lib/right_scraper/loggers.rb +31 -0
  8. data/lib/right_scraper/loggers/base.rb +113 -0
  9. data/lib/right_scraper/loggers/default.rb +98 -0
  10. data/lib/right_scraper/{scraper.rb → main.rb} +53 -9
  11. data/lib/right_scraper/processes.rb +33 -0
  12. data/lib/right_scraper/processes/shell.rb +227 -0
  13. data/lib/right_scraper/processes/{ssh.rb → ssh_agent.rb} +4 -0
  14. data/lib/right_scraper/processes/svn_client.rb +117 -0
  15. data/lib/right_scraper/processes/warden.rb +358 -0
  16. data/lib/right_scraper/registered_base.rb +154 -0
  17. data/lib/right_scraper/repositories.rb +33 -0
  18. data/lib/right_scraper/repositories/base.rb +271 -232
  19. data/lib/right_scraper/repositories/download.rb +8 -6
  20. data/lib/right_scraper/repositories/git.rb +8 -9
  21. data/lib/right_scraper/repositories/svn.rb +8 -8
  22. data/lib/right_scraper/resources.rb +32 -0
  23. data/lib/right_scraper/resources/base.rb +5 -1
  24. data/lib/right_scraper/resources/cookbook.rb +34 -27
  25. data/lib/right_scraper/resources/workflow.rb +27 -28
  26. data/lib/right_scraper/retrievers.rb +34 -0
  27. data/lib/right_scraper/retrievers/base.rb +80 -84
  28. data/lib/right_scraper/retrievers/checkout_base.rb +178 -0
  29. data/lib/right_scraper/retrievers/download.rb +125 -117
  30. data/lib/right_scraper/retrievers/git.rb +377 -223
  31. data/lib/right_scraper/retrievers/svn.rb +102 -62
  32. data/lib/right_scraper/scanners.rb +37 -0
  33. data/lib/right_scraper/scanners/base.rb +77 -80
  34. data/lib/right_scraper/scanners/cookbook_manifest.rb +31 -30
  35. data/lib/right_scraper/scanners/cookbook_metadata.rb +380 -35
  36. data/lib/right_scraper/scanners/cookbook_s3_upload.rb +56 -53
  37. data/lib/right_scraper/scanners/union.rb +61 -58
  38. data/lib/right_scraper/scanners/workflow_manifest.rb +55 -54
  39. data/lib/right_scraper/scanners/workflow_metadata.rb +41 -39
  40. data/lib/right_scraper/scanners/workflow_s3_upload.rb +59 -55
  41. data/lib/right_scraper/scrapers.rb +32 -0
  42. data/lib/right_scraper/scrapers/base.rb +217 -205
  43. data/lib/right_scraper/scrapers/cookbook.rb +42 -40
  44. data/lib/right_scraper/scrapers/workflow.rb +57 -58
  45. data/lib/right_scraper/version.rb +3 -0
  46. data/right_scraper.gemspec +12 -16
  47. metadata +57 -163
  48. data/Gemfile +0 -15
  49. data/Rakefile +0 -89
  50. data/lib/right_scraper/logger.rb +0 -107
  51. data/lib/right_scraper/loggers/noisy.rb +0 -85
  52. data/lib/right_scraper/repositories/mock.rb +0 -70
  53. data/lib/right_scraper/retrievers/checkout.rb +0 -79
  54. data/lib/right_scraper/scraper_logger.rb +0 -66
  55. data/lib/right_scraper/svn_client.rb +0 -164
  56. data/right_scraper.rconf +0 -13
  57. data/spec/builder_spec.rb +0 -50
  58. data/spec/cookbook_helper.rb +0 -73
  59. data/spec/cookbook_manifest_spec.rb +0 -93
  60. data/spec/cookbook_s3_upload_spec.rb +0 -159
  61. data/spec/download/download_retriever_spec.rb +0 -118
  62. data/spec/download/download_retriever_spec_helper.rb +0 -72
  63. data/spec/download/download_spec.rb +0 -128
  64. data/spec/download/multi_dir_spec.rb +0 -106
  65. data/spec/download/multi_dir_spec_helper.rb +0 -40
  66. data/spec/git/cookbook_spec.rb +0 -165
  67. data/spec/git/demokey +0 -27
  68. data/spec/git/demokey.pub +0 -1
  69. data/spec/git/password_key +0 -30
  70. data/spec/git/password_key.pub +0 -1
  71. data/spec/git/repository_spec.rb +0 -110
  72. data/spec/git/retriever_spec.rb +0 -553
  73. data/spec/git/retriever_spec_helper.rb +0 -112
  74. data/spec/git/scraper_spec.rb +0 -151
  75. data/spec/git/ssh_spec.rb +0 -174
  76. data/spec/git/url_spec.rb +0 -103
  77. data/spec/logger_spec.rb +0 -185
  78. data/spec/repository_spec.rb +0 -111
  79. data/spec/retriever_spec_helper.rb +0 -146
  80. data/spec/scanner_spec.rb +0 -61
  81. data/spec/scraper_helper.rb +0 -88
  82. data/spec/scraper_spec.rb +0 -147
  83. data/spec/spec_helper.rb +0 -185
  84. data/spec/svn/cookbook_spec.rb +0 -96
  85. data/spec/svn/multi_svn_spec.rb +0 -64
  86. data/spec/svn/multi_svn_spec_helper.rb +0 -40
  87. data/spec/svn/repository_spec.rb +0 -72
  88. data/spec/svn/retriever_spec.rb +0 -266
  89. data/spec/svn/scraper_spec.rb +0 -90
  90. data/spec/svn/svn_retriever_spec_helper.rb +0 -90
  91. data/spec/svn/url_spec.rb +0 -47
  92. data/spec/url_spec.rb +0 -164
@@ -1,90 +0,0 @@
1
- #--
2
- # Copyright: Copyright (c) 2010-2011 RightScale, Inc.
3
- #
4
- # Permission is hereby granted, free of charge, to any person obtaining
5
- # a copy of this software and associated documentation files (the
6
- # 'Software'), to deal in the Software without restriction, including
7
- # without limitation the rights to use, copy, modify, merge, publish,
8
- # distribute, sublicense, and/or sell copies of the Software, and to
9
- # permit persons to whom the Software is furnished to do so, subject to
10
- # the following conditions:
11
- #
12
- # The above copyright notice and this permission notice shall be
13
- # included in all copies or substantial portions of the Software.
14
- #
15
- # THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
16
- # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
- # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
18
- # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
19
- # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
20
- # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
21
- # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
- #++
23
-
24
- require File.expand_path(File.join(File.dirname(__FILE__), '..', 'retriever_spec_helper'))
25
-
26
- module RightScraper
27
-
28
- # SVN implementation of retriever spec helper
29
- # See parent class for methods headers comments
30
- class SvnRetrieverSpecHelper < RetrieverSpecHelper
31
- include RightScraper::SvnClient
32
-
33
- def svn_repo_path
34
- File.join(@tmpdir, "svn")
35
- end
36
-
37
- attr_reader :repo
38
- alias_method :repository, :repo
39
-
40
- def scraper_path
41
- File.join(@tmpdir, "scraper")
42
- end
43
-
44
- def repo_url
45
- file_prefix = 'file://'
46
- file_prefix += '/' if RUBY_PLATFORM =~ /mswin|mingw/
47
- url = "#{file_prefix}#{svn_repo_path}"
48
- end
49
-
50
- alias_method :repo_dir, :repo_path
51
-
52
- def initialize
53
- super()
54
- FileUtils.mkdir(svn_repo_path)
55
- @repo = RightScraper::Repositories::Base.from_hash(
56
- :display_name => 'test repo',
57
- :repo_type => :svn,
58
- :url => repo_url)
59
- output = `svnadmin create #{svn_repo_path}`
60
- raise "Can't create repo: #{output}" unless $?.success?
61
- run_svn_no_chdir "checkout", repo_url, repo_path
62
- make_cookbooks
63
- commit_content
64
- end
65
-
66
- def make_cookbooks
67
- create_cookbook(repo_path, repo_content)
68
- end
69
-
70
- def delete(location, log="")
71
- run_svn "delete", location
72
- end
73
-
74
- def commit_content(commit_message='commit message')
75
- run_svn "add", Dir.glob(File.join(repo_path, '**/*'))
76
- run_svn "commit", "--message", commit_message
77
- end
78
-
79
- def commit_id(index_from_last=0)
80
- lines = run_svn_with_buffered_output "log", "-l", (index_from_last + 1).to_s, "-r", "HEAD:0"
81
- id = nil
82
- lines.each do |line|
83
- if line =~ /^r(\d+)/
84
- id = $1
85
- end
86
- end
87
- id
88
- end
89
- end
90
- end
data/spec/svn/url_spec.rb DELETED
@@ -1,47 +0,0 @@
1
- #--
2
- # Copyright: Copyright (c) 2010-2011 RightScale, Inc.
3
- #
4
- # Permission is hereby granted, free of charge, to any person obtaining
5
- # a copy of this software and associated documentation files (the
6
- # 'Software'), to deal in the Software without restriction, including
7
- # without limitation the rights to use, copy, modify, merge, publish,
8
- # distribute, sublicense, and/or sell copies of the Software, and to
9
- # permit persons to whom the Software is furnished to do so, subject to
10
- # the following conditions:
11
- #
12
- # The above copyright notice and this permission notice shall be
13
- # included in all copies or substantial portions of the Software.
14
- #
15
- # THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
16
- # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
- # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
18
- # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
19
- # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
20
- # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
21
- # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
- #++
23
-
24
- require File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec_helper'))
25
-
26
- describe RightScraper::Repositories::Svn do
27
- def make_repo(url)
28
- @repo = RightScraper::Repositories::Svn.from_hash(:display_name => 'test repo',
29
- :repo_type => :svn,
30
- :url => url,
31
- :first_credential => "foo:b/ar",
32
- :second_credential => "blah")
33
- end
34
- include RightScraper::SpecHelpers::ProductionModeEnvironment
35
-
36
- it 'should not throw an error when creating a repository for SVN URIs' do
37
- lambda do
38
- make_repo "http://rightscale.com/%20CBLAH"
39
- end.should_not raise_exception
40
- lambda do
41
- make_repo "svn://rightscale.com/%20CBLAH"
42
- end.should_not raise_exception
43
- lambda do
44
- make_repo "svn+ssh://rightscale.com/%20CBLAH"
45
- end.should_not raise_exception
46
- end
47
- end
data/spec/url_spec.rb DELETED
@@ -1,164 +0,0 @@
1
- #--
2
- # Copyright: Copyright (c) 2010-2011 RightScale, Inc.
3
- #
4
- # Permission is hereby granted, free of charge, to any person obtaining
5
- # a copy of this software and associated documentation files (the
6
- # 'Software'), to deal in the Software without restriction, including
7
- # without limitation the rights to use, copy, modify, merge, publish,
8
- # distribute, sublicense, and/or sell copies of the Software, and to
9
- # permit persons to whom the Software is furnished to do so, subject to
10
- # the following conditions:
11
- #
12
- # The above copyright notice and this permission notice shall be
13
- # included in all copies or substantial portions of the Software.
14
- #
15
- # THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
16
- # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
- # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
18
- # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
19
- # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
20
- # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
21
- # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
- #++
23
-
24
- require File.expand_path(File.join(File.dirname(__FILE__), 'spec_helper'))
25
- require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'right_scraper'))
26
-
27
- describe RightScraper::Repositories::Svn do
28
- def make_repo(url)
29
- @repo = RightScraper::Repositories::Svn.from_hash(
30
- :display_name => 'test repo',
31
- :repo_type => :download,
32
- :url => url,
33
- :first_credential => "foo:b/ar",
34
- :second_credential => "foo@bar")
35
- end
36
- before(:each) do
37
- @oldtest = ENV['DEVELOPMENT']
38
- end
39
- after(:each) do
40
- if @oldtest.nil?
41
- ENV.delete('DEVELOPMENT')
42
- else
43
- ENV['DEVELOPMENT'] = @oldtest
44
- end
45
- end
46
-
47
- context 'in production mode' do
48
- before(:each) do
49
- ENV.delete('DEVELOPMENT')
50
- end
51
-
52
- it 'should not throw an error when creating a repository for a normal URI' do
53
- lambda do
54
- make_repo "http://rightscale.com/%20CBLAH"
55
- end.should_not raise_exception
56
- lambda do
57
- make_repo "http://172.12.3.42/%20CBLAH"
58
- end.should_not raise_exception
59
- lambda do
60
- make_repo "http://110.12.3.42/%20CBLAH"
61
- end.should_not raise_exception
62
- end
63
-
64
- it 'should refuse to create repositories for hosts that don\'t exist' do
65
- lambda do
66
- make_repo "http://nonexistent.invalid/%20CBLAH"
67
- end.should raise_exception(RuntimeError, /Invalid URI/)
68
- end
69
-
70
- it 'should refuse to create repositories for private networks' do
71
- ["10.0.2.43", "172.18.3.42", "192.168.4.2"].each do |ip|
72
- lambda do
73
- make_repo "http://#{ip}/%20CBLAH"
74
- end.should raise_exception(RuntimeError, /Invalid URI/)
75
- end
76
- end
77
-
78
- it 'should refuse to create repositories for loopback addresses' do
79
- lambda do
80
- make_repo "http://localhost/%20CBLAH"
81
- end.should raise_exception(RuntimeError, /Invalid URI/)
82
- lambda do
83
- make_repo "http://127.0.0.1/%20CBLAH"
84
- end.should raise_exception(RuntimeError, /Invalid URI/)
85
- lambda do
86
- make_repo "http://127.3.2.1/%20CBLAH"
87
- end.should raise_exception(RuntimeError, /Invalid URI/)
88
- end
89
-
90
- it 'should refuse to create repositories for file:/// URIs' do
91
- lambda do
92
- make_repo "file://var/run/something/%20CBLAH"
93
- end.should raise_exception(RuntimeError, /Invalid URI/)
94
- end
95
-
96
- it 'should refuse to create repositories for our EC2 metadata server' do
97
- lambda do
98
- make_repo "http://169.254.169.254/%20CBLAH"
99
- end.should raise_exception(RuntimeError, /Invalid URI/)
100
- end
101
-
102
- it 'should refuse to create repositories for even valid IPv6 addresses' do
103
- lambda do
104
- make_repo "http://[::ffff:128.111.1.1]/%20CBLAH"
105
- end.should raise_exception(RuntimeError, /Invalid URI/)
106
- end
107
- end
108
-
109
- context 'in development mode' do
110
- before(:each) do
111
- ENV['DEVELOPMENT'] = "yes"
112
- end
113
-
114
- it 'should not throw an error when creating a repository for a normal URI' do
115
- lambda do
116
- make_repo "http://rightscale.com/%20CBLAH"
117
- end.should_not raise_exception
118
- end
119
-
120
- it 'should create repositories for hosts that don\'t exist' do
121
- lambda do
122
- make_repo "http://nonexistent.invalid/%20CBLAH"
123
- end.should_not raise_exception
124
- end
125
-
126
- it 'should create repositories for private networks' do
127
- ["10.0.2.43", "172.18.3.42", "192.168.4.2"].each do |ip|
128
- lambda do
129
- make_repo "http://#{ip}/%20CBLAH"
130
- end.should_not raise_exception
131
- end
132
- end
133
-
134
- it 'should create repositories for loopback addresses' do
135
- lambda do
136
- make_repo "http://localhost/%20CBLAH"
137
- end.should_not raise_exception
138
- lambda do
139
- make_repo "http://127.0.0.1/%20CBLAH"
140
- end.should_not raise_exception
141
- lambda do
142
- make_repo "http://127.3.2.1/%20CBLAH"
143
- end.should_not raise_exception
144
- end
145
-
146
- it 'should create repositories for file:/// URIs' do
147
- lambda do
148
- make_repo "file://var/run/something/%20CBLAH"
149
- end.should_not raise_exception
150
- end
151
-
152
- it 'should create repositories for our EC2 metadata server' do
153
- lambda do
154
- make_repo "http://169.254.169.254/%20CBLAH"
155
- end.should_not raise_exception
156
- end
157
-
158
- it 'should create repositories for even valid IPv6 addresses' do
159
- lambda do
160
- make_repo "http://[::ffff:128.111.1.1]/%20CBLAH"
161
- end.should_not raise_exception
162
- end
163
- end
164
- end