turbot 0.1.36 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +8 -0
  3. data/.rspec +3 -0
  4. data/.travis.yml +15 -0
  5. data/.yardopts +3 -0
  6. data/Gemfile +4 -0
  7. data/LICENSE +22 -0
  8. data/README.md +44 -25
  9. data/Rakefile +16 -0
  10. data/appveyor.yml +35 -0
  11. data/bin/turbot +2 -16
  12. data/data/schema.json +134 -0
  13. data/{templates → data/templates}/LICENSE.txt +0 -0
  14. data/{templates → data/templates}/manifest.json +0 -0
  15. data/{templates → data/templates}/python/scraper.py +0 -0
  16. data/{templates → data/templates}/ruby/scraper.rb +0 -0
  17. data/dist/deb.rake +32 -0
  18. data/dist/gem.rake +16 -0
  19. data/dist/manifest.rake +9 -0
  20. data/dist/pkg.rake +60 -0
  21. data/dist/resources/deb/control +10 -0
  22. data/dist/resources/deb/postinst +45 -0
  23. data/dist/resources/deb/turbot +25 -0
  24. data/dist/resources/deb/turbot-release-key.txt +30 -0
  25. data/dist/resources/pkg/Distribution.erb +15 -0
  26. data/dist/resources/pkg/PackageInfo.erb +6 -0
  27. data/dist/resources/pkg/postinstall +45 -0
  28. data/dist/resources/pkg/turbot +24 -0
  29. data/dist/resources/tgz/turbot +24 -0
  30. data/dist/rpm.rake +35 -0
  31. data/dist/tgz.rake +26 -0
  32. data/dist/zip.rake +40 -0
  33. data/lib/turbot.rb +18 -15
  34. data/lib/turbot/cli.rb +10 -27
  35. data/lib/turbot/command.rb +59 -212
  36. data/lib/turbot/command/auth.rb +72 -34
  37. data/lib/turbot/command/base.rb +22 -61
  38. data/lib/turbot/command/bots.rb +251 -300
  39. data/lib/turbot/command/help.rb +57 -110
  40. data/lib/turbot/command/version.rb +6 -10
  41. data/lib/turbot/handlers/base_handler.rb +21 -0
  42. data/lib/turbot/handlers/dump_handler.rb +10 -0
  43. data/lib/turbot/handlers/preview_handler.rb +30 -0
  44. data/lib/turbot/handlers/validation_handler.rb +17 -0
  45. data/lib/turbot/helpers.rb +14 -482
  46. data/lib/turbot/helpers/api_helper.rb +41 -0
  47. data/lib/turbot/helpers/netrc_helper.rb +66 -0
  48. data/lib/turbot/helpers/shell_helper.rb +36 -0
  49. data/lib/turbot/version.rb +1 -1
  50. data/spec/fixtures/bad_permissions +0 -0
  51. data/spec/fixtures/empty +0 -0
  52. data/spec/fixtures/netrc +6 -0
  53. data/spec/spec_helper.rb +17 -219
  54. data/spec/support/bot_helper.rb +102 -0
  55. data/spec/support/command_helper.rb +20 -0
  56. data/spec/support/custom_matchers.rb +5 -0
  57. data/spec/support/fixture_helper.rb +9 -0
  58. data/spec/support/netrc_helper.rb +21 -0
  59. data/spec/turbot/command/auth_spec.rb +202 -20
  60. data/spec/turbot/command/base_spec.rb +22 -58
  61. data/spec/turbot/command/bots_spec.rb +580 -89
  62. data/spec/turbot/command/help_spec.rb +32 -75
  63. data/spec/turbot/command/version_spec.rb +11 -10
  64. data/spec/turbot/command_spec.rb +55 -87
  65. data/spec/turbot/helpers_spec.rb +28 -44
  66. data/turbot.gemspec +31 -0
  67. metadata +88 -178
  68. data/data/cacert.pem +0 -3988
  69. data/lib/turbot/auth.rb +0 -315
  70. data/lib/turbot/client.rb +0 -757
  71. data/lib/turbot/client/cisaurus.rb +0 -25
  72. data/lib/turbot/client/pgbackups.rb +0 -113
  73. data/lib/turbot/client/rendezvous.rb +0 -111
  74. data/lib/turbot/client/ssl_endpoint.rb +0 -25
  75. data/lib/turbot/client/turbot_postgresql.rb +0 -148
  76. data/lib/turbot/command/ssl.rb +0 -43
  77. data/lib/turbot/deprecated.rb +0 -5
  78. data/lib/turbot/deprecated/help.rb +0 -38
  79. data/lib/turbot/distribution.rb +0 -9
  80. data/lib/turbot/errors.rb +0 -28
  81. data/lib/turbot/excon.rb +0 -11
  82. data/lib/turbot/helpers/log_displayer.rb +0 -70
  83. data/lib/turbot/helpers/pg_dump_restore.rb +0 -115
  84. data/lib/turbot/helpers/turbot_postgresql.rb +0 -213
  85. data/lib/turbot/plugin.rb +0 -165
  86. data/lib/turbot/updater.rb +0 -171
  87. data/lib/vendor/turbot/okjson.rb +0 -598
  88. data/spec/helper/legacy_help.rb +0 -16
  89. data/spec/helper/pg_dump_restore_spec.rb +0 -67
  90. data/spec/spec.opts +0 -1
  91. data/spec/support/display_message_matcher.rb +0 -49
  92. data/spec/support/dummy_api.rb +0 -120
  93. data/spec/support/openssl_mock_helper.rb +0 -8
  94. data/spec/support/organizations_mock_helper.rb +0 -11
  95. data/spec/turbot/auth_spec.rb +0 -214
  96. data/spec/turbot/client/pgbackups_spec.rb +0 -43
  97. data/spec/turbot/client/rendezvous_spec.rb +0 -62
  98. data/spec/turbot/client/ssl_endpoint_spec.rb +0 -48
  99. data/spec/turbot/client/turbot_postgresql_spec.rb +0 -71
  100. data/spec/turbot/client_spec.rb +0 -548
  101. data/spec/turbot/helpers/turbot_postgresql_spec.rb +0 -181
  102. data/spec/turbot/plugin_spec.rb +0 -172
  103. data/spec/turbot/updater_spec.rb +0 -44
@@ -0,0 +1,20 @@
1
+ module CommandHelper
2
+ def execute(command_line)
3
+ original_stderr, original_stdout = $stderr, $stdout
4
+
5
+ $stderr = captured_stderr = StringIO.new
6
+ $stdout = captured_stdout = StringIO.new
7
+
8
+ begin
9
+ Turbot::CLI.start(*command_line.split(' '))
10
+ rescue Exception => e
11
+ unless SystemExit === e
12
+ p e
13
+ end
14
+ ensure
15
+ $stderr, $stdout = original_stderr, original_stdout
16
+ end
17
+
18
+ [captured_stderr.string, captured_stdout.string]
19
+ end
20
+ end
@@ -0,0 +1,5 @@
1
+ RSpec::Matchers.define :equal_lines do |expected|
2
+ match do |actual|
3
+ actual.split(/\r?\n/) == expected.split(/\r?\n/)
4
+ end
5
+ end
@@ -0,0 +1,9 @@
1
+ module FixtureHelper
2
+ def fixture(path, chmod = 0600)
3
+ filename = File.expand_path(File.join('..', '..', 'fixtures', path), __FILE__)
4
+ if File.exist?(filename)
5
+ FileUtils.chmod(chmod, filename)
6
+ end
7
+ filename
8
+ end
9
+ end
@@ -0,0 +1,21 @@
1
+ module NetrcHelper
2
+ def spec_open_netrc
3
+ Netrc.read(Netrc.default_path)
4
+ end
5
+
6
+ def spec_save_netrc_entry(data)
7
+ netrc = spec_open_netrc
8
+ netrc['api.http://turbot.opencorporates.com'] = data
9
+ netrc.save
10
+ end
11
+
12
+ def spec_delete_netrc_entry
13
+ netrc = spec_open_netrc
14
+ netrc.delete('api.http://turbot.opencorporates.com')
15
+ netrc.save
16
+ end
17
+
18
+ def spec_read_netrc
19
+ spec_open_netrc['api.http://turbot.opencorporates.com']
20
+ end
21
+ end
@@ -1,38 +1,220 @@
1
- require "spec_helper"
2
- require "turbot/command/auth"
1
+ require 'spec_helper'
2
+ require 'turbot/command/auth'
3
3
 
4
4
  describe Turbot::Command::Auth do
5
- describe "auth" do
6
- it "displays turbot help auth" do
7
- stderr, stdout = execute("auth")
5
+ describe 'auth' do
6
+ it 'displays help for auth commands' do
7
+ stderr, stdout = execute('auth')
8
8
 
9
- stderr.should == ""
10
- stdout.should include "Additional commands"
11
- stdout.should include "auth:login"
12
- stdout.should include "auth:logout"
9
+ expect(stderr).to eq('')
10
+ expect(stdout).to include('turbot auth')
11
+ expect(stdout).to include('Additional commands')
12
+ expect(stdout).to include('auth:whoami')
13
13
  end
14
14
  end
15
15
 
16
- describe "auth:token" do
16
+ context 'when editing the .netrc file' do
17
+ before do
18
+ allow(Netrc).to receive(:default_path).and_return(fixture('writable'))
19
+ end
17
20
 
18
- it "displays the user's api key" do
19
- stderr, stdout = execute("auth:token")
20
- stderr.should == ""
21
- stdout.should == <<-STDOUT
22
- apikey01
21
+ describe 'auth:login' do
22
+ it 'logs the user in' do
23
+ stub_request(:get, 'http://turbot.opencorporates.com/api/user/api_key?api_key=&email=email@example.com&password=password').to_return({
24
+ :status => 200,
25
+ :body => JSON.dump('api_key' => 'apikey01'),
26
+ })
27
+ stub_request(:get, 'http://turbot.opencorporates.com/api/user?api_key=apikey01').to_return({
28
+ :status => 200,
29
+ :body => JSON.dump('api_key' => 'apikey01'),
30
+ })
31
+ allow(STDIN).to receive(:gets).and_return('email@example.com', 'password')
32
+ allow_any_instance_of(Turbot::Command::Auth).to receive(:ask_for_password_on_windows).and_return('password')
33
+
34
+ spec_delete_netrc_entry
35
+
36
+ stderr, stdout = execute('auth:login')
37
+
38
+ expect(spec_read_netrc.to_a).to eq(['email@example.com', 'apikey01'])
39
+
40
+ expect(stderr).to eq('')
41
+ expect(stdout).to eq <<-STDOUT
42
+ Enter your Turbot email and password.
43
+ Email: Password (typing will be hidden):
44
+ Authentication successful.
45
+ STDOUT
46
+ end
47
+
48
+ it 'displays an error message' do
49
+ stub_request(:get, 'http://turbot.opencorporates.com/api/user/api_key?api_key=&email=&password=').to_return(:status => 200, :body => '{"api_key":""}')
50
+ allow(STDIN).to receive(:gets).and_return('', '')
51
+ allow_any_instance_of(Turbot::Command::Auth).to receive(:ask_for_password_on_windows).and_return('')
52
+
53
+ spec_delete_netrc_entry
54
+
55
+ stderr, stdout = execute('auth:login')
56
+
57
+ expect(spec_read_netrc).to eq(nil)
58
+
59
+ expect(stdout).to eq <<-STDOUT
60
+ Enter your Turbot email and password.
61
+ Email: Password (typing will be hidden):
23
62
  STDOUT
63
+ expect(stderr).to eq <<-STDERR
64
+ ! Authentication failed.
65
+ STDERR
66
+ end
67
+ end
68
+
69
+ describe 'auth:logout' do
70
+ it 'logs the user out' do
71
+ spec_save_netrc_entry(['user', 'pass'])
72
+
73
+ stderr, stdout = execute('auth:logout')
74
+
75
+ expect(spec_read_netrc).to eq(nil)
76
+
77
+ expect(stderr).to eq('')
78
+ expect(stdout).to eq <<-STDOUT
79
+ Deleted Turbot credentials.
80
+ STDOUT
81
+ end
24
82
  end
25
83
  end
26
84
 
27
- describe "auth:whoami" do
28
- it "displays the user's email address" do
29
- stderr, stdout = execute("auth:whoami")
30
- stderr.should == ""
31
- stdout.should == <<-STDOUT
85
+ context 'when logged in' do
86
+ before do
87
+ allow(Netrc).to receive(:default_path).and_return(fixture('netrc'))
88
+ end
89
+
90
+ describe 'auth:token' do
91
+ it "displays the user's api key" do
92
+ stderr, stdout = execute('auth:token')
93
+
94
+ expect(stderr).to eq('')
95
+ expect(stdout).to eq <<-STDOUT
96
+ apikey01
97
+ STDOUT
98
+ end
99
+ end
100
+
101
+ describe 'auth:whoami' do
102
+ it "displays the user's email address" do
103
+ stderr, stdout = execute('auth:whoami')
104
+
105
+ expect(stderr).to eq('')
106
+ expect(stdout).to eq <<-STDOUT
32
107
  email@example.com
33
108
  STDOUT
109
+ end
34
110
  end
35
111
 
112
+ context 'with TURBOT_HOST set' do
113
+ around(:each) do |example|
114
+ ENV['TURBOT_HOST'] = 'http://turbot.example.com'
115
+ example.run
116
+ ENV['TURBOT_HOST'] = nil
117
+ end
118
+
119
+ describe 'auth:token' do
120
+ it "displays the user's api key" do
121
+ stderr, stdout = execute('auth:token')
122
+
123
+ expect(stderr).to eq('')
124
+ expect(stdout).to eq <<-STDOUT
125
+ apikey02
126
+ STDOUT
127
+ end
128
+ end
129
+
130
+ describe 'auth:whoami' do
131
+ it "displays the user's email address" do
132
+ stderr, stdout = execute('auth:whoami')
133
+
134
+ expect(stderr).to eq('')
135
+ expect(stdout).to eq <<-STDOUT
136
+ example@email.com
137
+ STDOUT
138
+ end
139
+ end
140
+ end
141
+
142
+ context 'with TURBOT_API_KEY set' do
143
+ around(:each) do |example|
144
+ ENV['TURBOT_API_KEY'] = 'apikey99'
145
+ example.run
146
+ ENV['TURBOT_API_KEY'] = nil
147
+ end
148
+
149
+ describe 'auth:token' do
150
+ it "displays the user's api key" do
151
+ stderr, stdout = execute('auth:token')
152
+
153
+ expect(stderr).to eq('')
154
+ expect(stdout).to eq <<-STDOUT
155
+ apikey99
156
+ STDOUT
157
+ end
158
+ end
159
+
160
+ describe 'auth:whoami' do
161
+ it "displays nothing" do
162
+ stderr, stdout = execute('auth:whoami')
163
+
164
+ expect(stderr).to eq('')
165
+ expect(stdout).to eq <<-STDOUT
166
+
167
+ STDOUT
168
+ end
169
+ end
170
+ end
36
171
  end
37
172
 
173
+ context 'when logged out' do
174
+ ['empty', 'nonexistent'].each do |path|
175
+ before do
176
+ allow(Netrc).to receive(:default_path).and_return(fixture(path))
177
+ end
178
+
179
+ describe 'auth:token' do
180
+ it 'displays an error message' do
181
+ stderr, stdout = execute('auth:token')
182
+
183
+ expect(stdout).to eq('')
184
+ expect(stderr).to eq <<-STDERR
185
+ ! not logged in
186
+ STDERR
187
+ end
188
+ end
189
+
190
+ describe 'auth:whoami' do
191
+ it 'displays an error message' do
192
+ stderr, stdout = execute('auth:whoami')
193
+
194
+ expect(stdout).to eq('')
195
+ expect(stderr).to eq <<-STDERR
196
+ ! not logged in
197
+ STDERR
198
+ end
199
+ end
200
+ end
201
+ end
202
+
203
+ # Windows NTFS and FAT don't support `File.chmod`.
204
+ unless RUBY_PLATFORM =~ /mswin32|mingw32/
205
+ context 'with a bad .netrc file' do
206
+ before do
207
+ allow(Netrc).to receive(:default_path).and_return(fixture('bad_permissions', 0644))
208
+ end
209
+
210
+ describe 'auth:whoami' do
211
+ it 'displays an error message' do
212
+ stderr, stdout = execute('auth:whoami')
213
+
214
+ expect(stdout).to eq('')
215
+ expect(stderr).to match(%r{\A ! Permission bits for '.+/spec/fixtures/bad_permissions' should be 0600, but are 644\n\z})
216
+ end
217
+ end
218
+ end
219
+ end
38
220
  end
@@ -1,66 +1,30 @@
1
- require "spec_helper"
2
- require "turbot/command/base"
1
+ require 'spec_helper'
3
2
 
4
- module Turbot::Command
5
- describe Base do
6
- before do
7
- @base = Base.new
8
- @base.stub!(:display)
9
- @client = mock('turbot client', :host => 'turbot.com')
10
- end
11
-
12
- describe "confirming" do
13
- it "confirms the bot via --confirm" do
14
- Turbot::Command.stub(:current_options).and_return(:confirm => "example")
15
- @base.stub(:bot).and_return("example")
16
- @base.confirm_command.should be_true
17
- end
18
-
19
- it "does not confirms the bot via --confirm on a mismatch" do
20
- Turbot::Command.stub(:current_options).and_return(:confirm => "badapp")
21
- @base.stub(:bot).and_return("example")
22
- lambda { @base.confirm_command}.should raise_error CommandFailed
23
- end
24
-
25
- it "confirms the bot interactively via ask" do
26
- @base.stub(:bot).and_return("example")
27
- @base.stub(:ask).and_return("example")
28
- Turbot::Command.stub(:current_options).and_return({})
29
- @base.confirm_command.should be_true
30
- end
3
+ describe Turbot::Command::Base do
4
+ before do
5
+ allow(subject).to receive(:display)
6
+ @client = double('turbot client', :host => 'turbot.com')
7
+ end
31
8
 
32
- it "fails if the interactive confirm doesn't match" do
33
- @base.stub(:bot).and_return("example")
34
- @base.stub(:ask).and_return("badresponse")
35
- Turbot::Command.stub(:current_options).and_return({})
36
- capture_stderr do
37
- lambda { @base.confirm_command }.should raise_error(SystemExit)
38
- end.should == <<-STDERR
39
- ! Confirmation did not match example. Aborted.
40
- STDERR
41
- end
9
+ context 'detecting the bot' do
10
+ it 'attempts to find the bot via the --bot option' do
11
+ allow(subject).to receive(:options).and_return(:bot => 'example')
12
+ expect(subject.bot).to eq('example')
42
13
  end
43
14
 
44
- context "detecting the bot" do
45
- it "attempts to find the bot via the --bot option" do
46
- @base.stub!(:options).and_return(:bot => "example")
47
- @base.bot.should == "example"
48
- end
49
-
50
- it "attempts to find the bot via TURBOT_BOT when not explicitly specified" do
51
- ENV['TURBOT_BOT'] = "myenvapp"
52
- @base.bot.should == "myenvapp"
53
- @base.stub!(:options).and_return([])
54
- @base.bot.should == "myenvapp"
55
- ENV.delete('TURBOT_BOT')
56
- end
15
+ it 'attempts to find the bot via TURBOT_BOT when not explicitly specified' do
16
+ ENV['TURBOT_BOT'] = 'myenvapp'
17
+ expect(subject.bot).to eq('myenvapp')
18
+ allow(subject).to receive(:options).and_return([])
19
+ expect(subject.bot).to eq('myenvapp')
20
+ ENV.delete('TURBOT_BOT')
21
+ end
57
22
 
58
- it "overrides TURBOT_BOT when explicitly specified" do
59
- ENV['TURBOT_BOT'] = "myenvapp"
60
- @base.stub!(:options).and_return(:bot => "example")
61
- @base.bot.should == "example"
62
- ENV.delete('TURBOT_BOT')
63
- end
23
+ it 'overrides TURBOT_BOT when explicitly specified' do
24
+ ENV['TURBOT_BOT'] = 'myenvapp'
25
+ allow(subject).to receive(:options).and_return(:bot => 'example')
26
+ expect(subject.bot).to eq('example')
27
+ ENV.delete('TURBOT_BOT')
64
28
  end
65
29
  end
66
30
  end
@@ -1,133 +1,624 @@
1
- require "spec_helper"
2
- require "turbot/command/bots"
1
+ require 'spec_helper'
2
+ require 'turbot/command/bots'
3
+ require 'turbot_runner'
3
4
 
4
5
  describe Turbot::Command::Bots do
5
- describe "validate" do
6
- let :working_directory do
7
- Dir.mktmpdir
6
+ context 'when unauthenticated' do
7
+ before do
8
+ allow(Netrc).to receive(:default_path).and_return(fixture('empty'))
8
9
  end
9
10
 
10
- let :schemas_directory do
11
- Dir.mktmpdir
11
+ describe 'bots' do
12
+ it 'lists no bots' do
13
+ stub_request(:get, 'http://turbot.opencorporates.com/api/bots?api_key=').to_return({
14
+ :status => 401,
15
+ :body => JSON.dump('message' => 'No API key provided'),
16
+ })
17
+
18
+ stderr, stdout = execute('bots')
19
+
20
+ expect(stdout).to eq('')
21
+ expect(stderr).to eq <<-STDERR
22
+ ! No API key provided (HTTP 401)
23
+ STDERR
24
+ end
12
25
  end
26
+ end
13
27
 
28
+ context 'when authenticated' do
14
29
  before do
15
- config = {
16
- 'bot_id' => 'dummy bot',
17
- 'data_type' => 'dummy',
18
- 'identifying_fields' => ['name'],
19
- 'files' => 'scraper.rb',
20
- 'language' => 'ruby',
21
- 'publisher' => {
22
- 'name' => 'Dummy',
23
- 'url' => 'http://example.com/',
24
- 'terms' => 'MIT',
25
- 'terms_url' => 'http://opensource.org/licenses/MIT',
26
- },
27
- }
28
- Turbot::Command::Bots.any_instance.stub(:parsed_manifest).and_return(config)
29
-
30
- # Create a manifest.json file for TurbotRunner to find.
31
- File.open(File.join(working_directory, 'manifest.json'), 'w') { |f| f << JSON.dump(config) }
32
- Turbot::Command::Bots.any_instance.stub(:working_directory).and_return(working_directory)
33
-
34
- # Change the path to TurbotRunner's schemas.
35
- begin
36
- old_verbose, $VERBOSE = $VERBOSE, nil
37
- TurbotRunner::SCHEMAS_PATH = File.expand_path('../../../schemas', __FILE__)
38
- ensure
39
- $VERBOSE = old_verbose
30
+ allow(Netrc).to receive(:default_path).and_return(fixture('netrc'))
31
+ end
32
+
33
+ describe 'bots' do
34
+ it 'lists bots' do
35
+ stub_request(:get, 'http://turbot.opencorporates.com/api/bots?api_key=apikey01').to_return({
36
+ :status => 200,
37
+ :body => JSON.dump({
38
+ 'data' => [{
39
+ 'bot_id' => 'dummy_bot',
40
+ 'created_at' => '2010-01-01T00:00:00.000Z',
41
+ 'updated_at' => '2010-01-02T00:00:00.000Z',
42
+ 'state' => 'scheduled',
43
+ }],
44
+ }),
45
+ })
46
+
47
+ stderr, stdout = execute('bots')
48
+
49
+ expect(stderr).to eq('')
50
+ expect(stdout).to eq <<-STDOUT
51
+ dummy_bot
52
+ STDOUT
53
+ end
54
+
55
+ it 'lists no bots' do
56
+ stub_request(:get, 'http://turbot.opencorporates.com/api/bots?api_key=apikey01').to_return({
57
+ :status => 200,
58
+ :body => JSON.dump('data' => []),
59
+ })
60
+
61
+ stderr, stdout = execute('bots')
62
+
63
+ expect(stderr).to eq('')
64
+ expect(stdout).to eq <<-STDOUT
65
+ You have no bots.
66
+ STDOUT
40
67
  end
41
68
  end
42
69
 
43
- after do
44
- FileUtils.remove_entry_secure(working_directory)
45
- FileUtils.remove_entry_secure(schemas_directory)
70
+ describe 'bots:info' do
71
+ it "shows the given bot's details" do
72
+ stub_bot_info
73
+
74
+ stderr, stdout = execute('bots:info --bot example')
75
+
76
+ expect(stderr).to eq('')
77
+ expect(stdout).to eq <<-STDOUT
78
+ bot_id: dummy_bot
79
+ created_at: 2010-01-01T00:00:00.000Z
80
+ updated_at: 2010-01-02T00:00:00.000Z
81
+ state: scheduled
82
+ STDOUT
83
+ end
84
+
85
+ it "shows the inferred bot's details" do
86
+ stub_bot_info
87
+
88
+ bot_directory = create_bot_directory
89
+ create_manifest_file(bot_directory)
90
+
91
+ stderr, stdout = execute_in_directory('bots:info', bot_directory)
92
+
93
+ expect(stderr).to eq('')
94
+ expect(stdout).to eq <<-STDOUT
95
+ bot_id: dummy_bot
96
+ created_at: 2010-01-01T00:00:00.000Z
97
+ updated_at: 2010-01-02T00:00:00.000Z
98
+ state: scheduled
99
+ STDOUT
100
+ end
101
+
102
+ it 'errors if no local bot is found' do
103
+ stderr, stdout = execute('bots:info')
104
+
105
+ expect(stdout).to eq('')
106
+ expect(stderr).to eq <<-STDERR
107
+ ! No bot specified.
108
+ ! Run this command from a bot directory containing a `manifest.json` file, or specify the bot with --bot BOT.
109
+ STDERR
110
+ end
111
+
112
+ it 'errors if no bot exists in Turbot' do
113
+ stub_bot_info_error
114
+
115
+ stderr, stdout = execute('bots:info --bot example')
116
+
117
+ expect(stdout).to eq('')
118
+ expect(stderr).to eq <<-STDERR
119
+ ! No bot registered for bot_id example
120
+ ! If you have renamed your bot, ... (HTTP 402: bot-not-found)
121
+ STDERR
122
+ end
46
123
  end
47
124
 
48
- def define_scraper(hash)
49
- File.open(File.join(working_directory, 'scraper.rb'), 'w') do |f|
50
- f << <<-EOL
51
- require 'json'
52
- puts JSON.dump(#{hash})
53
- EOL
125
+ describe 'bots:generate' do
126
+ it 'generates a Ruby bot template' do
127
+ stub_bot_info_error('rb')
128
+
129
+ working_directory = Dir.mktmpdir
130
+
131
+ stderr, stdout = execute_in_directory('bots:generate --bot rb', working_directory)
132
+
133
+ expect(stderr).to eq('')
134
+ expect(stdout).to eq <<-STDOUT
135
+ Created new bot template for rb!
136
+ STDOUT
137
+
138
+ expect(File.exist?(File.join(working_directory, 'rb', 'LICENSE.txt'))).to eq(true)
139
+ expect(File.exist?(File.join(working_directory, 'rb', 'manifest.json'))).to eq(true)
140
+ expect(File.exist?(File.join(working_directory, 'rb', 'scraper.rb'))).to eq(true)
141
+ end
142
+
143
+ it 'generates a Python bot template' do
144
+ stub_bot_info_error('py')
145
+
146
+ working_directory = Dir.mktmpdir
147
+
148
+ stderr, stdout = execute_in_directory('bots:generate --bot py --language python', working_directory)
149
+
150
+ expect(stderr).to eq('')
151
+ expect(stdout).to eq <<-STDOUT
152
+ Created new bot template for py!
153
+ STDOUT
154
+
155
+ expect(File.exist?(File.join(working_directory, 'py', 'LICENSE.txt'))).to eq(true)
156
+ expect(File.exist?(File.join(working_directory, 'py', 'manifest.json'))).to eq(true)
157
+ expect(File.exist?(File.join(working_directory, 'py', 'scraper.py'))).to eq(true)
158
+ end
159
+
160
+ it 'errors if the bot exists in Turbot' do
161
+ stub_bot_info
162
+
163
+ stderr, stdout = execute('bots:generate --bot example')
164
+
165
+ expect(stdout).to eq('')
166
+ expect(stderr).to eq <<-STDERR
167
+ ! There's already a bot named example in Turbot. Try another name.
168
+ STDERR
169
+ end
170
+
171
+ it 'errors if the bot name is invalid' do
172
+ stub_bot_info_error('example!')
173
+
174
+ stderr, stdout = execute('bots:generate --bot example!')
175
+
176
+ expect(stdout).to eq('')
177
+ expect(stderr).to include('The bot name example! is invalid.')
178
+ end
179
+
180
+ it 'errors if the directory exists' do
181
+ stub_bot_info_error
182
+
183
+ working_directory = Dir.mktmpdir
184
+ bot_directory = create_bot_directory(working_directory)
185
+
186
+ stderr, stdout = execute_in_directory('bots:generate --bot example', working_directory)
187
+
188
+ expect(stdout).to eq('')
189
+ expect(stderr).to eq <<-STDERR
190
+ ! There's already a directory named example. Move it, delete it, change directory, or try another name.
191
+ STDERR
192
+ end
193
+
194
+ it 'errors if the language is unsupported' do
195
+ stub_bot_info_error
196
+
197
+ stderr, stdout = execute('bots:generate --bot example --language go')
198
+
199
+ expect(stdout).to eq('')
200
+ expect(stderr).to eq <<-STDOUT
201
+ ! The language go is unsupported.
202
+ STDOUT
54
203
  end
55
204
  end
56
205
 
57
- context "for data_type with schema" do
58
- it "says bot is valid if its output matches the schema" do
59
- define_scraper(name: 'One')
206
+ describe 'bots:register' do
207
+ it 'registers the bot' do
208
+ stub_bot_info_error
209
+ stub_request(:post, 'http://turbot.opencorporates.com/api/bots').to_return(:status => 200, :body => '{}')
60
210
 
61
- stderr, stdout = execute("bots:validate")
211
+ bot_directory = create_bot_directory
212
+ create_manifest_file(bot_directory)
62
213
 
63
- stdout.should include 'Validated 1 records!'
64
- stderr.should == ""
214
+ stderr, stdout = execute_in_directory('bots:register', bot_directory)
215
+
216
+ expect(stderr).to eq('')
217
+ expect(stdout).to eq <<-STDOUT
218
+ Registered example!
219
+ STDOUT
65
220
  end
66
221
 
67
- it "says bot is invalid if its output doesn't match the schema" do
68
- define_scraper(name: 123)
222
+ it 'errors if no local bot is found' do
223
+ stub_bot_info_error
69
224
 
70
- stderr, stdout = execute("bots:validate")
225
+ stderr, stdout = execute('bots:register')
71
226
 
72
- stdout.should include 'Property of wrong type'
73
- stderr.should == ""
227
+ expect(stdout).to eq('')
228
+ expect(stderr).to eq <<-STDERR
229
+ ! No bot specified.
230
+ ! Run this command from a bot directory containing a `manifest.json` file, or specify the bot with --bot BOT.
231
+ STDERR
74
232
  end
75
233
 
76
- context "for bot that doesn't output identifying fields" do
77
- it "says bot is invalid" do
78
- define_scraper(title: 'One')
234
+ it 'errors if the bot exists in Turbot' do
235
+ stub_bot_info
79
236
 
80
- stderr, stdout = execute("bots:validate")
237
+ bot_directory = create_bot_directory
238
+ create_manifest_file(bot_directory)
81
239
 
82
- stdout.should include 'There were no values provided for any of the identifying fields'
83
- stderr.should == ""
84
- end
240
+ stderr, stdout = execute_in_directory('bots:register', bot_directory)
241
+
242
+ expect(stdout).to eq('')
243
+ expect(stderr).to eq <<-STDERR
244
+ ! There's already a bot named example in Turbot. Try another name.
245
+ STDERR
246
+ end
247
+ end
248
+
249
+ describe 'bots:push' do
250
+ it 'pushes to Turbot' do
251
+ allow(STDIN).to receive(:gets).and_return('y')
252
+ stub_request(:put, 'http://turbot.opencorporates.com/api/bots/example/code').to_return(:status => 200, :body => '{}')
253
+
254
+ bot_directory = create_bot_directory
255
+ create_manifest_file(bot_directory)
256
+ create_scraper_file(bot_directory)
257
+
258
+ stderr, stdout = execute_in_directory('bots:push', bot_directory)
259
+
260
+ expect(stderr).to eq('')
261
+ expect(stdout).to eq <<-STDOUT
262
+ This will submit your bot and its data for review.
263
+ Are you happy your bot produces valid data (e.g. with `turbot bots:validate`)? [Y/n]
264
+ Your bot has been pushed to Turbot and will be reviewed for inclusion as soon as we can. THANK YOU!
265
+ STDOUT
266
+ end
267
+
268
+ it 'skips confirmation' do
269
+ allow(STDIN).to receive(:gets).and_return('y')
270
+ stub_request(:put, 'http://turbot.opencorporates.com/api/bots/example/code').to_return(:status => 200, :body => '{}')
271
+
272
+ bot_directory = create_bot_directory
273
+ create_manifest_file(bot_directory)
274
+ create_scraper_file(bot_directory)
275
+
276
+ stderr, stdout = execute_in_directory('bots:push --yes', bot_directory)
277
+
278
+ expect(stderr).to eq('')
279
+ expect(stdout).to eq <<-STDOUT
280
+ Your bot has been pushed to Turbot and will be reviewed for inclusion as soon as we can. THANK YOU!
281
+ STDOUT
282
+ end
283
+
284
+ it 'errors if no local bot is found' do
285
+ stderr, stdout = execute('bots:push')
286
+
287
+ expect(stdout).to eq('')
288
+ expect(stderr).to eq <<-STDERR
289
+ ! No bot specified.
290
+ ! Run this command from a bot directory containing a `manifest.json` file, or specify the bot with --bot BOT.
291
+ STDERR
292
+ end
293
+
294
+ it 'aborts if push not confirmed' do
295
+ allow(STDIN).to receive(:gets).and_return('n')
296
+
297
+ bot_directory = create_bot_directory
298
+ create_manifest_file(bot_directory)
299
+
300
+ stderr, stdout = execute_in_directory('bots:push', bot_directory)
301
+
302
+ expect(stdout).to eq <<-STDOUT
303
+ This will submit your bot and its data for review.
304
+ Are you happy your bot produces valid data (e.g. with `turbot bots:validate`)? [Y/n]
305
+ STDOUT
306
+ expect(stderr).to eq <<-STDERR
307
+ ! Aborted.
308
+ STDERR
85
309
  end
86
310
  end
87
311
 
88
- context "for data_type without schema" do
89
- it "says bot is invalid" do
90
- define_scraper({})
312
+ describe 'bots:validate' do
313
+ before do
314
+ set_turbot_runner_schemas
315
+ end
316
+
317
+ it 'validates valid records' do
318
+ bot_directory = create_bot_directory
319
+ create_manifest_file(bot_directory)
320
+ create_scraper_file(bot_directory)
321
+
322
+ stderr, stdout = execute_in_directory('bots:validate', bot_directory)
323
+
324
+ expect(stderr).to eq('')
325
+ expect(stdout).to eq <<-STDOUT
326
+ Validated 1 records!
327
+ STDOUT
328
+ end
329
+
330
+ it 'reports invalid records' do
331
+ bot_directory = create_bot_directory
332
+ create_manifest_file(bot_directory)
333
+ create_scraper_file(bot_directory, [{'name' => 1}])
334
+
335
+ stderr, stdout = execute_in_directory('bots:validate', bot_directory)
336
+
337
+ expect(stderr).to eq('')
338
+ expect(stdout).to eq <<-STDOUT
339
+
340
+ The following record is invalid:
341
+ {"name":1}
342
+ * Property of wrong type: name (must be of type string)
343
+
344
+ Validated 0 records before bot failed!
345
+ STDOUT
346
+ end
347
+
348
+ it 'reports invalid JSON' do
349
+ bot_directory = create_bot_directory
350
+ create_manifest_file(bot_directory)
351
+ create_scraper_file(bot_directory, ['{'])
352
+
353
+ stderr, stdout = execute_in_directory('bots:validate', bot_directory)
354
+
355
+ expect(stderr).to eq('')
356
+ expect(stdout).to equal_lines <<-STDOUT
357
+
358
+ The following line was not valid JSON:
359
+ "{"
360
+ Validated 0 records before bot failed!
361
+ STDOUT
362
+ end
363
+
364
+ it 'reports records without identifying fields' do
365
+ bot_directory = create_bot_directory
366
+ create_manifest_file(bot_directory)
367
+ create_scraper_file(bot_directory, [{}])
368
+
369
+ stderr, stdout = execute_in_directory('bots:validate', bot_directory)
370
+
371
+ expect(stderr).to eq('')
372
+ expect(stdout).to eq <<-STDOUT
91
373
 
92
- stderr, stdout = execute("bots:validate")
374
+ The following record is invalid:
375
+ {}
376
+ * There were no values provided for any of the identifying fields: name
93
377
 
94
- stdout.should include "Validated 0 records before bot failed!"
95
- stderr.should == ""
378
+ Validated 0 records before bot failed!
379
+ STDOUT
380
+ end
381
+
382
+ it 'errors if no local bot is found' do
383
+ stderr, stdout = execute('bots:validate')
384
+
385
+ expect(stdout).to eq('')
386
+ expect(stderr).to eq <<-STDERR
387
+ ! No bot specified.
388
+ ! Run this command from a bot directory containing a `manifest.json` file, or specify the bot with --bot BOT.
389
+ STDERR
390
+ end
391
+
392
+ it 'errors if manifest is invalid JSON' do
393
+ bot_directory = create_bot_directory
394
+ create_manifest_file(bot_directory, '{')
395
+
396
+ stderr, stdout = execute_in_directory('bots:validate', bot_directory)
397
+
398
+ expect(stdout).to eq('')
399
+ expect(stderr).to eq <<-STDERR
400
+ ! `manifest.json` is invalid JSON. Consider validating it at http://pro.jsonlint.com/
401
+ STDERR
402
+ end
403
+
404
+ it 'warns if deprecated fields are used' do
405
+ bot_directory = create_bot_directory
406
+ create_scraper_file(bot_directory)
407
+ create_manifest_file(bot_directory, JSON.dump(valid_manifest.merge({
408
+ 'allow_duplicates' => true,
409
+ 'author' => 'John Q. Public',
410
+ 'incremental' => true,
411
+ 'public_repository' => 'http://example.com/',
412
+ })))
413
+
414
+ stderr, stdout = execute_in_directory('bots:validate', bot_directory)
415
+
416
+ expect(stderr).to eq('')
417
+ expect(stdout).to eq <<-STDOUT
418
+ WARNING: "allow_duplicates" is deprecated. Use "duplicates_allowed" instead.
419
+ WARNING: "author" is deprecated. Use "publisher" instead.
420
+ WARNING: "incremental" is deprecated. Use "manually_end_run" instead.
421
+ WARNING: "public_repository" is deprecated. Use "public_repo_url" instead.
422
+ Validated 1 records!
423
+ STDOUT
424
+ end
425
+
426
+ it 'errors if manifest is invalid' do
427
+ bot_directory = create_bot_directory
428
+ create_scraper_file(bot_directory)
429
+ create_manifest_file(bot_directory, JSON.dump(valid_manifest.merge({
430
+ 'bot_id' => 'example!',
431
+ 'title' => 1,
432
+ })))
433
+
434
+ stderr, stdout = execute_in_directory('bots:validate', bot_directory)
435
+
436
+ expect(stdout).to eq('')
437
+ expect(stderr).to eq <<-STDERR
438
+ ! `manifest.json` is invalid. Please correct the errors:
439
+ ! * The property '#/bot_id' value "example!" did not match the regex '^[A-Za-z0-9._-]+$'
440
+ ! * The property '#/title' of type Fixnum did not match the following type: string
441
+ STDERR
442
+ end
443
+
444
+ it 'errors if transformer files not in files list' do
445
+ bot_directory = create_bot_directory
446
+ create_scraper_file(bot_directory)
447
+ create_manifest_file(bot_directory, JSON.dump(valid_manifest.merge({
448
+ 'transformers' => [{
449
+ 'data_type' => 'dummy',
450
+ 'file' => 'transformer.rb',
451
+ 'identifying_fields' => ['name'],
452
+ }]
453
+ })))
454
+
455
+ stderr, stdout = execute_in_directory('bots:validate', bot_directory)
456
+
457
+ expect(stdout).to eq('')
458
+ expect(stderr).to eq <<-STDERR
459
+ ! `manifest.json` is invalid. Please correct the errors:
460
+ ! * Some transformer files are not listed in the top-level files: transformer.rb
461
+ STDERR
462
+ end
463
+
464
+ it 'errors if data_type is invalid' do
465
+ bot_directory = create_bot_directory
466
+ create_scraper_file(bot_directory)
467
+ create_manifest_file(bot_directory, JSON.dump(valid_manifest.merge({
468
+ 'data_type' => 'invalid',
469
+ })))
470
+
471
+ stderr, stdout = execute_in_directory('bots:validate', bot_directory)
472
+
473
+ expect(stdout).to eq('')
474
+ expect(stderr).to eq <<-STDERR
475
+ ! `manifest.json` is invalid. Please correct the errors:
476
+ ! * The property '#/data_type' value "invalid" is not a supported data type.
477
+ STDERR
96
478
  end
97
479
  end
98
480
 
99
- context "for bot with manifest missing some required fields" do
100
- it "says bot is invalid" do
101
- config = {
102
- 'bot_id' => 'dummy bot',
103
- 'identifying_fields' => ['name'],
104
- 'files' => 'scraper.rb',
105
- }
106
- Turbot::Command::Bots.any_instance.stub(:parsed_manifest).and_return(config)
481
+ describe 'bots:preview' do
482
+ it 'submits records' do
483
+ stub_preview
484
+
485
+ set_turbot_runner_schemas
486
+
487
+ bot_directory = create_bot_directory
488
+ create_manifest_file(bot_directory)
489
+ create_scraper_file(bot_directory)
107
490
 
108
- stderr, stdout = execute("bots:validate")
491
+ allow_any_instance_of(Turbot::Command::Base).to receive(:working_directory).and_return(bot_directory)
492
+ stderr, stdout = execute('bots:preview')
493
+ restore_working_directory_method
109
494
 
110
- stdout.should == ""
111
- stderr.should include 'Manifest is missing data_type'
495
+ expect(stderr).to eq('')
496
+ expect(stdout).to eq <<-STDOUT
497
+ Sending to Turbot...
498
+ Submitted 1 records to Turbot.
499
+ View your records at http://example.com/
500
+ STDOUT
501
+ end
502
+
503
+ it 'errors if no local bot is found' do
504
+ stderr, stdout = execute('bots:preview')
505
+
506
+ expect(stdout).to eq('')
507
+ expect(stderr).to eq <<-STDERR
508
+ ! No bot specified.
509
+ ! Run this command from a bot directory containing a `manifest.json` file, or specify the bot with --bot BOT.
510
+ STDERR
511
+ end
512
+
513
+ it 'errors if the scraper is broken' do
514
+ stub_preview
515
+
516
+ set_turbot_runner_schemas
517
+
518
+ bot_directory = create_bot_directory
519
+ create_manifest_file(bot_directory)
520
+ create_broken_scraper_file(bot_directory)
521
+
522
+ allow_any_instance_of(Turbot::Command::Base).to receive(:working_directory).and_return(bot_directory)
523
+ stderr, stdout = execute('bots:preview')
524
+ restore_working_directory_method
525
+
526
+ expect(stderr).to eq('')
527
+ expect(stdout).to eq <<-STDOUT
528
+ Sending to Turbot...
529
+ Bot failed!
530
+ STDOUT
112
531
  end
113
532
  end
114
- end
115
533
 
116
- describe '#create_zip_archive' do
117
- it 'adds all given files to archive' do
118
- dirs = ['a', 'b', 'a/p', 'b/p']
119
- paths = ['a/p/x', 'a/y', 'b/p/x', 'b/y', 'z']
534
+ describe 'bots:dump' do
535
+ it 'dumps records' do
536
+ set_turbot_runner_schemas
537
+
538
+ bot_directory = create_bot_directory
539
+ create_manifest_file(bot_directory)
540
+ create_scraper_file(bot_directory)
541
+
542
+ allow_any_instance_of(Turbot::Command::Base).to receive(:working_directory).and_return(bot_directory)
543
+ stderr, stdout = execute('bots:dump')
544
+ restore_working_directory_method
545
+
546
+ expect(stderr).to eq('')
547
+ expect(stdout).to eq <<-STDOUT
548
+ {"name":"foo"}
549
+ Bot ran successfully!
550
+ STDOUT
551
+ end
120
552
 
121
- base_dir = Dir.mktmpdir
122
- dirs.each {|dir| Dir.mkdir(File.join(base_dir, dir))}
123
- paths.each {|path| FileUtils.touch(File.join(base_dir, path))}
553
+ it 'reports only validation errors' do
554
+ set_turbot_runner_schemas
124
555
 
125
- command = Turbot::Command::Bots.new
126
- archive_path = Tempfile.new('test').path
127
- command.send(:create_zip_archive, archive_path, base_dir, ['a', 'b/p/x', 'b/y', 'z'])
556
+ bot_directory = create_bot_directory
557
+ create_manifest_file(bot_directory)
558
+ create_scraper_file(bot_directory, [{'name' => 'foo'}, {'name' => 1}])
128
559
 
129
- Zip::File.open(archive_path) do |zipfile|
130
- expect(zipfile.map {|entry| entry.to_s}).to match_array(paths + ['a/p/'])
560
+ allow_any_instance_of(Turbot::Command::Base).to receive(:working_directory).and_return(bot_directory)
561
+ stderr, stdout = execute('bots:dump --quiet')
562
+ restore_working_directory_method
563
+
564
+ expect(stderr).to eq('')
565
+ expect(stdout).to eq <<-STDOUT
566
+
567
+ The following record is invalid:
568
+ {"name":1}
569
+ * Property of wrong type: name (must be of type string)
570
+
571
+ Bot failed!
572
+ STDOUT
573
+ end
574
+
575
+ it 'errors if no local bot is found' do
576
+ stderr, stdout = execute('bots:dump')
577
+
578
+ expect(stdout).to eq('')
579
+ expect(stderr).to eq <<-STDERR
580
+ ! No bot specified.
581
+ ! Run this command from a bot directory containing a `manifest.json` file, or specify the bot with --bot BOT.
582
+ STDERR
583
+ end
584
+
585
+ it 'errors if the scraper is broken' do
586
+ set_turbot_runner_schemas
587
+
588
+ bot_directory = create_bot_directory
589
+ create_manifest_file(bot_directory)
590
+ create_broken_scraper_file(bot_directory)
591
+
592
+ allow_any_instance_of(Turbot::Command::Base).to receive(:working_directory).and_return(bot_directory)
593
+ stderr, stdout = execute('bots:dump')
594
+ restore_working_directory_method
595
+
596
+ expect(stderr).to eq('')
597
+ expect(stdout).to eq <<-STDOUT
598
+ Bot failed!
599
+ STDOUT
600
+ end
601
+ end
602
+
603
+ describe '#create_zip_archive' do
604
+ it 'adds all given files to archive' do
605
+ dirs = ['a', 'b', 'a/p', 'b/p']
606
+ paths = ['a/p/x', 'a/y', 'b/p/x', 'b/y', 'z']
607
+ working_directory = Dir.mktmpdir
608
+ dirs.each { |dir| Dir.mkdir(File.join(working_directory, dir)) }
609
+ paths.each { |path| FileUtils.touch(File.join(working_directory, path)) }
610
+
611
+ tempfile = Tempfile.new('test')
612
+ tempfile.close
613
+ archive_path = "#{tempfile.path}.zip"
614
+
615
+ allow_any_instance_of(Turbot::Command::Base).to receive(:working_directory).and_return(working_directory)
616
+ Turbot::Command::Bots.new.send(:create_zip_archive, archive_path, ['a', 'b/p/x', 'b/y', 'z'])
617
+ restore_working_directory_method
618
+
619
+ Zip::File.open(archive_path) do |zipfile|
620
+ expect(zipfile.map { |entry| entry.to_s }).to match_array(paths + ['a/p/'])
621
+ end
131
622
  end
132
623
  end
133
624
  end