right_scraper 3.2.6 → 5.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. checksums.yaml +7 -0
  2. data/lib/right_scraper.rb +16 -34
  3. data/lib/right_scraper/builders.rb +32 -0
  4. data/lib/right_scraper/builders/base.rb +19 -20
  5. data/lib/right_scraper/builders/filesystem.rb +8 -6
  6. data/lib/right_scraper/builders/union.rb +4 -1
  7. data/lib/right_scraper/loggers.rb +31 -0
  8. data/lib/right_scraper/loggers/base.rb +113 -0
  9. data/lib/right_scraper/loggers/default.rb +98 -0
  10. data/lib/right_scraper/{scraper.rb → main.rb} +53 -9
  11. data/lib/right_scraper/processes.rb +33 -0
  12. data/lib/right_scraper/processes/shell.rb +227 -0
  13. data/lib/right_scraper/processes/{ssh.rb → ssh_agent.rb} +4 -0
  14. data/lib/right_scraper/processes/svn_client.rb +117 -0
  15. data/lib/right_scraper/processes/warden.rb +358 -0
  16. data/lib/right_scraper/registered_base.rb +154 -0
  17. data/lib/right_scraper/repositories.rb +33 -0
  18. data/lib/right_scraper/repositories/base.rb +271 -232
  19. data/lib/right_scraper/repositories/download.rb +8 -6
  20. data/lib/right_scraper/repositories/git.rb +8 -9
  21. data/lib/right_scraper/repositories/svn.rb +8 -8
  22. data/lib/right_scraper/resources.rb +32 -0
  23. data/lib/right_scraper/resources/base.rb +5 -1
  24. data/lib/right_scraper/resources/cookbook.rb +34 -27
  25. data/lib/right_scraper/resources/workflow.rb +27 -28
  26. data/lib/right_scraper/retrievers.rb +34 -0
  27. data/lib/right_scraper/retrievers/base.rb +80 -84
  28. data/lib/right_scraper/retrievers/checkout_base.rb +178 -0
  29. data/lib/right_scraper/retrievers/download.rb +125 -117
  30. data/lib/right_scraper/retrievers/git.rb +377 -223
  31. data/lib/right_scraper/retrievers/svn.rb +102 -62
  32. data/lib/right_scraper/scanners.rb +37 -0
  33. data/lib/right_scraper/scanners/base.rb +77 -80
  34. data/lib/right_scraper/scanners/cookbook_manifest.rb +31 -30
  35. data/lib/right_scraper/scanners/cookbook_metadata.rb +380 -35
  36. data/lib/right_scraper/scanners/cookbook_s3_upload.rb +56 -53
  37. data/lib/right_scraper/scanners/union.rb +61 -58
  38. data/lib/right_scraper/scanners/workflow_manifest.rb +55 -54
  39. data/lib/right_scraper/scanners/workflow_metadata.rb +41 -39
  40. data/lib/right_scraper/scanners/workflow_s3_upload.rb +59 -55
  41. data/lib/right_scraper/scrapers.rb +32 -0
  42. data/lib/right_scraper/scrapers/base.rb +217 -205
  43. data/lib/right_scraper/scrapers/cookbook.rb +42 -40
  44. data/lib/right_scraper/scrapers/workflow.rb +57 -58
  45. data/lib/right_scraper/version.rb +3 -0
  46. data/right_scraper.gemspec +12 -16
  47. metadata +57 -163
  48. data/Gemfile +0 -15
  49. data/Rakefile +0 -89
  50. data/lib/right_scraper/logger.rb +0 -107
  51. data/lib/right_scraper/loggers/noisy.rb +0 -85
  52. data/lib/right_scraper/repositories/mock.rb +0 -70
  53. data/lib/right_scraper/retrievers/checkout.rb +0 -79
  54. data/lib/right_scraper/scraper_logger.rb +0 -66
  55. data/lib/right_scraper/svn_client.rb +0 -164
  56. data/right_scraper.rconf +0 -13
  57. data/spec/builder_spec.rb +0 -50
  58. data/spec/cookbook_helper.rb +0 -73
  59. data/spec/cookbook_manifest_spec.rb +0 -93
  60. data/spec/cookbook_s3_upload_spec.rb +0 -159
  61. data/spec/download/download_retriever_spec.rb +0 -118
  62. data/spec/download/download_retriever_spec_helper.rb +0 -72
  63. data/spec/download/download_spec.rb +0 -128
  64. data/spec/download/multi_dir_spec.rb +0 -106
  65. data/spec/download/multi_dir_spec_helper.rb +0 -40
  66. data/spec/git/cookbook_spec.rb +0 -165
  67. data/spec/git/demokey +0 -27
  68. data/spec/git/demokey.pub +0 -1
  69. data/spec/git/password_key +0 -30
  70. data/spec/git/password_key.pub +0 -1
  71. data/spec/git/repository_spec.rb +0 -110
  72. data/spec/git/retriever_spec.rb +0 -553
  73. data/spec/git/retriever_spec_helper.rb +0 -112
  74. data/spec/git/scraper_spec.rb +0 -151
  75. data/spec/git/ssh_spec.rb +0 -174
  76. data/spec/git/url_spec.rb +0 -103
  77. data/spec/logger_spec.rb +0 -185
  78. data/spec/repository_spec.rb +0 -111
  79. data/spec/retriever_spec_helper.rb +0 -146
  80. data/spec/scanner_spec.rb +0 -61
  81. data/spec/scraper_helper.rb +0 -88
  82. data/spec/scraper_spec.rb +0 -147
  83. data/spec/spec_helper.rb +0 -185
  84. data/spec/svn/cookbook_spec.rb +0 -96
  85. data/spec/svn/multi_svn_spec.rb +0 -64
  86. data/spec/svn/multi_svn_spec_helper.rb +0 -40
  87. data/spec/svn/repository_spec.rb +0 -72
  88. data/spec/svn/retriever_spec.rb +0 -266
  89. data/spec/svn/scraper_spec.rb +0 -90
  90. data/spec/svn/svn_retriever_spec_helper.rb +0 -90
  91. data/spec/svn/url_spec.rb +0 -47
  92. data/spec/url_spec.rb +0 -164
@@ -1,90 +0,0 @@
1
- #--
2
- # Copyright: Copyright (c) 2010-2011 RightScale, Inc.
3
- #
4
- # Permission is hereby granted, free of charge, to any person obtaining
5
- # a copy of this software and associated documentation files (the
6
- # 'Software'), to deal in the Software without restriction, including
7
- # without limitation the rights to use, copy, modify, merge, publish,
8
- # distribute, sublicense, and/or sell copies of the Software, and to
9
- # permit persons to whom the Software is furnished to do so, subject to
10
- # the following conditions:
11
- #
12
- # The above copyright notice and this permission notice shall be
13
- # included in all copies or substantial portions of the Software.
14
- #
15
- # THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
16
- # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
- # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
18
- # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
19
- # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
20
- # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
21
- # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
- #++
23
-
24
- require File.expand_path(File.join(File.dirname(__FILE__), '..', 'retriever_spec_helper'))
25
-
26
- module RightScraper
27
-
28
- # SVN implementation of retriever spec helper
29
- # See parent class for methods headers comments
30
- class SvnRetrieverSpecHelper < RetrieverSpecHelper
31
- include RightScraper::SvnClient
32
-
33
- def svn_repo_path
34
- File.join(@tmpdir, "svn")
35
- end
36
-
37
- attr_reader :repo
38
- alias_method :repository, :repo
39
-
40
- def scraper_path
41
- File.join(@tmpdir, "scraper")
42
- end
43
-
44
- def repo_url
45
- file_prefix = 'file://'
46
- file_prefix += '/' if RUBY_PLATFORM =~ /mswin|mingw/
47
- url = "#{file_prefix}#{svn_repo_path}"
48
- end
49
-
50
- alias_method :repo_dir, :repo_path
51
-
52
- def initialize
53
- super()
54
- FileUtils.mkdir(svn_repo_path)
55
- @repo = RightScraper::Repositories::Base.from_hash(
56
- :display_name => 'test repo',
57
- :repo_type => :svn,
58
- :url => repo_url)
59
- output = `svnadmin create #{svn_repo_path}`
60
- raise "Can't create repo: #{output}" unless $?.success?
61
- run_svn_no_chdir "checkout", repo_url, repo_path
62
- make_cookbooks
63
- commit_content
64
- end
65
-
66
- def make_cookbooks
67
- create_cookbook(repo_path, repo_content)
68
- end
69
-
70
- def delete(location, log="")
71
- run_svn "delete", location
72
- end
73
-
74
- def commit_content(commit_message='commit message')
75
- run_svn "add", Dir.glob(File.join(repo_path, '**/*'))
76
- run_svn "commit", "--message", commit_message
77
- end
78
-
79
- def commit_id(index_from_last=0)
80
- lines = run_svn_with_buffered_output "log", "-l", (index_from_last + 1).to_s, "-r", "HEAD:0"
81
- id = nil
82
- lines.each do |line|
83
- if line =~ /^r(\d+)/
84
- id = $1
85
- end
86
- end
87
- id
88
- end
89
- end
90
- end
data/spec/svn/url_spec.rb DELETED
@@ -1,47 +0,0 @@
1
- #--
2
- # Copyright: Copyright (c) 2010-2011 RightScale, Inc.
3
- #
4
- # Permission is hereby granted, free of charge, to any person obtaining
5
- # a copy of this software and associated documentation files (the
6
- # 'Software'), to deal in the Software without restriction, including
7
- # without limitation the rights to use, copy, modify, merge, publish,
8
- # distribute, sublicense, and/or sell copies of the Software, and to
9
- # permit persons to whom the Software is furnished to do so, subject to
10
- # the following conditions:
11
- #
12
- # The above copyright notice and this permission notice shall be
13
- # included in all copies or substantial portions of the Software.
14
- #
15
- # THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
16
- # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
- # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
18
- # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
19
- # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
20
- # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
21
- # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
- #++
23
-
24
- require File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec_helper'))
25
-
26
- describe RightScraper::Repositories::Svn do
27
- def make_repo(url)
28
- @repo = RightScraper::Repositories::Svn.from_hash(:display_name => 'test repo',
29
- :repo_type => :svn,
30
- :url => url,
31
- :first_credential => "foo:b/ar",
32
- :second_credential => "blah")
33
- end
34
- include RightScraper::SpecHelpers::ProductionModeEnvironment
35
-
36
- it 'should not throw an error when creating a repository for SVN URIs' do
37
- lambda do
38
- make_repo "http://rightscale.com/%20CBLAH"
39
- end.should_not raise_exception
40
- lambda do
41
- make_repo "svn://rightscale.com/%20CBLAH"
42
- end.should_not raise_exception
43
- lambda do
44
- make_repo "svn+ssh://rightscale.com/%20CBLAH"
45
- end.should_not raise_exception
46
- end
47
- end
data/spec/url_spec.rb DELETED
@@ -1,164 +0,0 @@
1
- #--
2
- # Copyright: Copyright (c) 2010-2011 RightScale, Inc.
3
- #
4
- # Permission is hereby granted, free of charge, to any person obtaining
5
- # a copy of this software and associated documentation files (the
6
- # 'Software'), to deal in the Software without restriction, including
7
- # without limitation the rights to use, copy, modify, merge, publish,
8
- # distribute, sublicense, and/or sell copies of the Software, and to
9
- # permit persons to whom the Software is furnished to do so, subject to
10
- # the following conditions:
11
- #
12
- # The above copyright notice and this permission notice shall be
13
- # included in all copies or substantial portions of the Software.
14
- #
15
- # THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
16
- # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
- # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
18
- # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
19
- # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
20
- # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
21
- # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
- #++
23
-
24
- require File.expand_path(File.join(File.dirname(__FILE__), 'spec_helper'))
25
- require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'right_scraper'))
26
-
27
- describe RightScraper::Repositories::Svn do
28
- def make_repo(url)
29
- @repo = RightScraper::Repositories::Svn.from_hash(
30
- :display_name => 'test repo',
31
- :repo_type => :download,
32
- :url => url,
33
- :first_credential => "foo:b/ar",
34
- :second_credential => "foo@bar")
35
- end
36
- before(:each) do
37
- @oldtest = ENV['DEVELOPMENT']
38
- end
39
- after(:each) do
40
- if @oldtest.nil?
41
- ENV.delete('DEVELOPMENT')
42
- else
43
- ENV['DEVELOPMENT'] = @oldtest
44
- end
45
- end
46
-
47
- context 'in production mode' do
48
- before(:each) do
49
- ENV.delete('DEVELOPMENT')
50
- end
51
-
52
- it 'should not throw an error when creating a repository for a normal URI' do
53
- lambda do
54
- make_repo "http://rightscale.com/%20CBLAH"
55
- end.should_not raise_exception
56
- lambda do
57
- make_repo "http://172.12.3.42/%20CBLAH"
58
- end.should_not raise_exception
59
- lambda do
60
- make_repo "http://110.12.3.42/%20CBLAH"
61
- end.should_not raise_exception
62
- end
63
-
64
- it 'should refuse to create repositories for hosts that don\'t exist' do
65
- lambda do
66
- make_repo "http://nonexistent.invalid/%20CBLAH"
67
- end.should raise_exception(RuntimeError, /Invalid URI/)
68
- end
69
-
70
- it 'should refuse to create repositories for private networks' do
71
- ["10.0.2.43", "172.18.3.42", "192.168.4.2"].each do |ip|
72
- lambda do
73
- make_repo "http://#{ip}/%20CBLAH"
74
- end.should raise_exception(RuntimeError, /Invalid URI/)
75
- end
76
- end
77
-
78
- it 'should refuse to create repositories for loopback addresses' do
79
- lambda do
80
- make_repo "http://localhost/%20CBLAH"
81
- end.should raise_exception(RuntimeError, /Invalid URI/)
82
- lambda do
83
- make_repo "http://127.0.0.1/%20CBLAH"
84
- end.should raise_exception(RuntimeError, /Invalid URI/)
85
- lambda do
86
- make_repo "http://127.3.2.1/%20CBLAH"
87
- end.should raise_exception(RuntimeError, /Invalid URI/)
88
- end
89
-
90
- it 'should refuse to create repositories for file:/// URIs' do
91
- lambda do
92
- make_repo "file://var/run/something/%20CBLAH"
93
- end.should raise_exception(RuntimeError, /Invalid URI/)
94
- end
95
-
96
- it 'should refuse to create repositories for our EC2 metadata server' do
97
- lambda do
98
- make_repo "http://169.254.169.254/%20CBLAH"
99
- end.should raise_exception(RuntimeError, /Invalid URI/)
100
- end
101
-
102
- it 'should refuse to create repositories for even valid IPv6 addresses' do
103
- lambda do
104
- make_repo "http://[::ffff:128.111.1.1]/%20CBLAH"
105
- end.should raise_exception(RuntimeError, /Invalid URI/)
106
- end
107
- end
108
-
109
- context 'in development mode' do
110
- before(:each) do
111
- ENV['DEVELOPMENT'] = "yes"
112
- end
113
-
114
- it 'should not throw an error when creating a repository for a normal URI' do
115
- lambda do
116
- make_repo "http://rightscale.com/%20CBLAH"
117
- end.should_not raise_exception
118
- end
119
-
120
- it 'should create repositories for hosts that don\'t exist' do
121
- lambda do
122
- make_repo "http://nonexistent.invalid/%20CBLAH"
123
- end.should_not raise_exception
124
- end
125
-
126
- it 'should create repositories for private networks' do
127
- ["10.0.2.43", "172.18.3.42", "192.168.4.2"].each do |ip|
128
- lambda do
129
- make_repo "http://#{ip}/%20CBLAH"
130
- end.should_not raise_exception
131
- end
132
- end
133
-
134
- it 'should create repositories for loopback addresses' do
135
- lambda do
136
- make_repo "http://localhost/%20CBLAH"
137
- end.should_not raise_exception
138
- lambda do
139
- make_repo "http://127.0.0.1/%20CBLAH"
140
- end.should_not raise_exception
141
- lambda do
142
- make_repo "http://127.3.2.1/%20CBLAH"
143
- end.should_not raise_exception
144
- end
145
-
146
- it 'should create repositories for file:/// URIs' do
147
- lambda do
148
- make_repo "file://var/run/something/%20CBLAH"
149
- end.should_not raise_exception
150
- end
151
-
152
- it 'should create repositories for our EC2 metadata server' do
153
- lambda do
154
- make_repo "http://169.254.169.254/%20CBLAH"
155
- end.should_not raise_exception
156
- end
157
-
158
- it 'should create repositories for even valid IPv6 addresses' do
159
- lambda do
160
- make_repo "http://[::ffff:128.111.1.1]/%20CBLAH"
161
- end.should_not raise_exception
162
- end
163
- end
164
- end