turbot 0.1.36 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +8 -0
  3. data/.rspec +3 -0
  4. data/.travis.yml +15 -0
  5. data/.yardopts +3 -0
  6. data/Gemfile +4 -0
  7. data/LICENSE +22 -0
  8. data/README.md +44 -25
  9. data/Rakefile +16 -0
  10. data/appveyor.yml +35 -0
  11. data/bin/turbot +2 -16
  12. data/data/schema.json +134 -0
  13. data/{templates → data/templates}/LICENSE.txt +0 -0
  14. data/{templates → data/templates}/manifest.json +0 -0
  15. data/{templates → data/templates}/python/scraper.py +0 -0
  16. data/{templates → data/templates}/ruby/scraper.rb +0 -0
  17. data/dist/deb.rake +32 -0
  18. data/dist/gem.rake +16 -0
  19. data/dist/manifest.rake +9 -0
  20. data/dist/pkg.rake +60 -0
  21. data/dist/resources/deb/control +10 -0
  22. data/dist/resources/deb/postinst +45 -0
  23. data/dist/resources/deb/turbot +25 -0
  24. data/dist/resources/deb/turbot-release-key.txt +30 -0
  25. data/dist/resources/pkg/Distribution.erb +15 -0
  26. data/dist/resources/pkg/PackageInfo.erb +6 -0
  27. data/dist/resources/pkg/postinstall +45 -0
  28. data/dist/resources/pkg/turbot +24 -0
  29. data/dist/resources/tgz/turbot +24 -0
  30. data/dist/rpm.rake +35 -0
  31. data/dist/tgz.rake +26 -0
  32. data/dist/zip.rake +40 -0
  33. data/lib/turbot.rb +18 -15
  34. data/lib/turbot/cli.rb +10 -27
  35. data/lib/turbot/command.rb +59 -212
  36. data/lib/turbot/command/auth.rb +72 -34
  37. data/lib/turbot/command/base.rb +22 -61
  38. data/lib/turbot/command/bots.rb +251 -300
  39. data/lib/turbot/command/help.rb +57 -110
  40. data/lib/turbot/command/version.rb +6 -10
  41. data/lib/turbot/handlers/base_handler.rb +21 -0
  42. data/lib/turbot/handlers/dump_handler.rb +10 -0
  43. data/lib/turbot/handlers/preview_handler.rb +30 -0
  44. data/lib/turbot/handlers/validation_handler.rb +17 -0
  45. data/lib/turbot/helpers.rb +14 -482
  46. data/lib/turbot/helpers/api_helper.rb +41 -0
  47. data/lib/turbot/helpers/netrc_helper.rb +66 -0
  48. data/lib/turbot/helpers/shell_helper.rb +36 -0
  49. data/lib/turbot/version.rb +1 -1
  50. data/spec/fixtures/bad_permissions +0 -0
  51. data/spec/fixtures/empty +0 -0
  52. data/spec/fixtures/netrc +6 -0
  53. data/spec/spec_helper.rb +17 -219
  54. data/spec/support/bot_helper.rb +102 -0
  55. data/spec/support/command_helper.rb +20 -0
  56. data/spec/support/custom_matchers.rb +5 -0
  57. data/spec/support/fixture_helper.rb +9 -0
  58. data/spec/support/netrc_helper.rb +21 -0
  59. data/spec/turbot/command/auth_spec.rb +202 -20
  60. data/spec/turbot/command/base_spec.rb +22 -58
  61. data/spec/turbot/command/bots_spec.rb +580 -89
  62. data/spec/turbot/command/help_spec.rb +32 -75
  63. data/spec/turbot/command/version_spec.rb +11 -10
  64. data/spec/turbot/command_spec.rb +55 -87
  65. data/spec/turbot/helpers_spec.rb +28 -44
  66. data/turbot.gemspec +31 -0
  67. metadata +88 -178
  68. data/data/cacert.pem +0 -3988
  69. data/lib/turbot/auth.rb +0 -315
  70. data/lib/turbot/client.rb +0 -757
  71. data/lib/turbot/client/cisaurus.rb +0 -25
  72. data/lib/turbot/client/pgbackups.rb +0 -113
  73. data/lib/turbot/client/rendezvous.rb +0 -111
  74. data/lib/turbot/client/ssl_endpoint.rb +0 -25
  75. data/lib/turbot/client/turbot_postgresql.rb +0 -148
  76. data/lib/turbot/command/ssl.rb +0 -43
  77. data/lib/turbot/deprecated.rb +0 -5
  78. data/lib/turbot/deprecated/help.rb +0 -38
  79. data/lib/turbot/distribution.rb +0 -9
  80. data/lib/turbot/errors.rb +0 -28
  81. data/lib/turbot/excon.rb +0 -11
  82. data/lib/turbot/helpers/log_displayer.rb +0 -70
  83. data/lib/turbot/helpers/pg_dump_restore.rb +0 -115
  84. data/lib/turbot/helpers/turbot_postgresql.rb +0 -213
  85. data/lib/turbot/plugin.rb +0 -165
  86. data/lib/turbot/updater.rb +0 -171
  87. data/lib/vendor/turbot/okjson.rb +0 -598
  88. data/spec/helper/legacy_help.rb +0 -16
  89. data/spec/helper/pg_dump_restore_spec.rb +0 -67
  90. data/spec/spec.opts +0 -1
  91. data/spec/support/display_message_matcher.rb +0 -49
  92. data/spec/support/dummy_api.rb +0 -120
  93. data/spec/support/openssl_mock_helper.rb +0 -8
  94. data/spec/support/organizations_mock_helper.rb +0 -11
  95. data/spec/turbot/auth_spec.rb +0 -214
  96. data/spec/turbot/client/pgbackups_spec.rb +0 -43
  97. data/spec/turbot/client/rendezvous_spec.rb +0 -62
  98. data/spec/turbot/client/ssl_endpoint_spec.rb +0 -48
  99. data/spec/turbot/client/turbot_postgresql_spec.rb +0 -71
  100. data/spec/turbot/client_spec.rb +0 -548
  101. data/spec/turbot/helpers/turbot_postgresql_spec.rb +0 -181
  102. data/spec/turbot/plugin_spec.rb +0 -172
  103. data/spec/turbot/updater_spec.rb +0 -44
@@ -1,19 +1,14 @@
1
- require "fileutils"
2
- require "turbot/auth"
3
- require "turbot/client/rendezvous"
4
- require "turbot/command"
5
-
6
1
  class Turbot::Command::Base
7
2
  include Turbot::Helpers
8
3
 
9
4
  def self.namespace
10
- self.to_s.split("::").last.downcase
5
+ self.to_s.split('::').last.downcase
11
6
  end
12
7
 
13
8
  attr_reader :args
14
9
  attr_reader :options
15
10
 
16
- def initialize(args=[], options={})
11
+ def initialize(args = [], options = {})
17
12
  @args = args
18
13
  @options = options
19
14
  end
@@ -21,24 +16,13 @@ class Turbot::Command::Base
21
16
  def bot
22
17
  @bot ||= if options[:bot].is_a?(String)
23
18
  options[:bot]
24
- elsif ENV.has_key?('TURBOT_BOT')
19
+ elsif ENV['TURBOT_BOT']
25
20
  ENV['TURBOT_BOT']
26
- elsif bot_from_manifest = extract_bot_from_manifest(Dir.pwd)
27
- bot_from_manifest
28
- else
29
- # raise instead of using error command to enable rescuing when bot is optional
30
- raise Turbot::Command::CommandFailed.new("No bot specified.\nRun this command from a bot folder containing a `manifest.json`, or specify which bot to use with --bot BOT_ID.") unless options[:ignore_no_bot]
21
+ elsif manifest = parse_manifest
22
+ manifest['bot_id']
31
23
  end
32
24
  end
33
25
 
34
- def api
35
- Turbot::Auth.api
36
- end
37
-
38
- def turbot
39
- Turbot::Auth.client
40
- end
41
-
42
26
  protected
43
27
 
44
28
  def self.inherited(klass)
@@ -53,13 +37,13 @@ protected
53
37
  end
54
38
 
55
39
  def self.method_added(method)
56
- return if self == Turbot::Command::Base
57
- return if private_method_defined?(method)
58
- return if protected_method_defined?(method)
40
+ if self == Turbot::Command::Base || private_method_defined?(method) || protected_method_defined?(method)
41
+ return
42
+ end
59
43
 
60
44
  help = extract_help_from_caller(caller.first)
61
45
  resolved_method = (method.to_s == "index") ? nil : method.to_s
62
- command = [ self.namespace, resolved_method ].compact.join(":")
46
+ command = [self.namespace, resolved_method].compact.join(":")
63
47
  banner = extract_banner(help) || command
64
48
 
65
49
  Turbot::Command.register_command(
@@ -73,18 +57,10 @@ protected
73
57
  :description => extract_description(help),
74
58
  :options => extract_options(help)
75
59
  )
76
-
77
- alias_command command.gsub(/_/, '-'), command if command =~ /_/
78
- end
79
-
80
- def self.alias_command(new, old)
81
- raise "no such command: #{old}" unless Turbot::Command.commands[old]
82
- Turbot::Command.command_aliases[new] = old
83
60
  end
84
61
 
85
- def extract_bot
86
- output_with_bang "Command::Base#extract_bot has been deprecated. Please use Command::Base#bot instead. #{caller.first}"
87
- bot
62
+ def self.alias_command(command_alias, command)
63
+ Turbot::Command.command_aliases[command_alias] = command
88
64
  end
89
65
 
90
66
  #
@@ -113,7 +89,7 @@ protected
113
89
  buffer = []
114
90
  lines = Turbot::Command.files[file]
115
91
 
116
- (line_number.to_i-2).downto(0) do |i|
92
+ (line_number.to_i - 2).downto(0) do |i|
117
93
  line = lines[i]
118
94
  case line[0..0]
119
95
  when ""
@@ -156,37 +132,22 @@ protected
156
132
  Turbot::Command.current_command
157
133
  end
158
134
 
159
- def extract_option(key)
160
- options[key.dup.gsub('-','_').to_sym]
161
- end
162
-
163
- def invalid_arguments
164
- Turbot::Command.invalid_arguments
165
- end
166
-
167
- def shift_argument
168
- Turbot::Command.shift_argument
169
- end
170
-
171
135
  def validate_arguments!
172
136
  Turbot::Command.validate_arguments!
173
137
  end
174
138
 
175
- def extract_bot_from_manifest(dir)
176
- begin
177
- config = JSON.load(open("#{dir}/manifest.json").read)
178
- config && config["bot_id"]
179
- rescue Errno::ENOENT
139
+ def parse_manifest
140
+ path = File.join(working_directory, 'manifest.json')
141
+ if File.exists?(path)
142
+ begin
143
+ JSON.load(File.read(path))
144
+ rescue JSON::ParserError => e
145
+ error "`manifest.json` is invalid JSON. Consider validating it at http://pro.jsonlint.com/"
146
+ end
180
147
  end
181
148
  end
182
149
 
183
- def escape(value)
184
- turbot.escape(value)
185
- end
186
- end
187
-
188
- module Turbot::Command
189
- unless const_defined?(:BaseWithApp)
190
- BaseWithApp = Base
150
+ def working_directory
151
+ Dir.pwd
191
152
  end
192
153
  end
@@ -1,217 +1,267 @@
1
- require "turbot/command/base"
2
- require 'active_support/core_ext/object/blank'
3
- require 'active_support/core_ext/hash/slice'
4
- require 'zip'
5
- require 'open3'
6
- require 'base64'
7
- require 'shellwords'
8
- require 'turbot_runner'
9
-
10
- # manage bots (generate skeleton, validate data, submit code)
1
+ #Manage bots (generate template, validate data, submit code)
11
2
  #
12
3
  class Turbot::Command::Bots < Turbot::Command::Base
4
+ BOT_ID_RE = /\A[A-Za-z0-9_-]+\z/.freeze
5
+
6
+ def initialize(*args)
7
+ super
8
+
9
+ require 'turbot_runner'
10
+ require 'turbot/handlers/base_handler'
11
+ require 'turbot/handlers/dump_handler'
12
+ require 'turbot/handlers/preview_handler'
13
+ require 'turbot/handlers/validation_handler'
14
+ end
13
15
 
14
16
  # bots
15
17
  #
16
- # list your bots
18
+ #List your bots.
17
19
  #
18
20
  #Example:
19
21
  #
20
- # $ turbot bots
21
- # === My Bots
22
- # example
23
- # example2
22
+ # $ turbot bots
23
+ # example1
24
+ # example2
24
25
  #
25
26
  def index
26
27
  validate_arguments!
27
- bots = api.list_bots.data
28
- unless bots.empty?
29
- styled_header("Bots")
30
- styled_array(bots.map{|bot| bot[:bot_id]})
28
+
29
+ response = api.list_bots
30
+ if response.is_a?(Turbot::API::SuccessResponse)
31
+ if response.data.empty?
32
+ puts 'You have no bots.'
33
+ else
34
+ response.data.each do |bot|
35
+ puts bot[:bot_id]
36
+ end
37
+ end
31
38
  else
32
- display("You have no bots.")
39
+ error_message(response)
33
40
  end
34
41
  end
42
+ alias_command 'list', 'bots'
35
43
 
36
- alias_command "list", "bots"
37
-
38
- # bots:info
39
- #
40
- # show detailed bot information
44
+ # bots:info [--bot BOT]
41
45
  #
42
- # -s, --shell # output more shell friendly key/value pairs
46
+ #Show a bot's details.
43
47
  #
44
- #Examples:
48
+ # -b, --bot BOT # a bot ID
45
49
  #
46
- # $ turbot bots:info
47
- # === example
48
- # Last run status: OK
49
- # Last run ended: 2001/01/01
50
- # ...
50
+ #Example:
51
51
  #
52
- # $ turbot bots:info --shell
53
- # last_run_status: OK
54
- # last_run_ended: 2001/01/01
55
- # ...
52
+ # $ turbot bots:info --bot example
53
+ # bot_id: example
54
+ # created_at: 2010-01-01T00:00:00.000Z
55
+ # updated_at: 2010-01-02T00:00:00.000Z
56
+ # state: scheduled
56
57
  #
57
58
  def info
58
59
  validate_arguments!
59
- bot_data = api.get_bot(bot)
60
- unless options[:shell]
61
- styled_header(bot_data["name"])
62
- end
60
+ error_if_no_local_bot_found
63
61
 
64
- if options[:shell]
65
- bot_data.keys.sort_by { |a| a.to_s }.each do |key|
66
- hputs("#{key}=#{bot_data[key]}")
62
+ response = api.show_bot(bot)
63
+ if response.is_a?(Turbot::API::SuccessResponse)
64
+ response.data.each do |key,value|
65
+ puts "#{key}: #{value}"
67
66
  end
68
67
  else
69
- data = {}
70
- if bot_data["last_run_status"]
71
- data["Last run status"] = bot_data["last_run_status"]
72
- end
73
- if bot_data["last_run_ended"]
74
- data["Last run ended"] = format_date(bot_data["last_run_ended"]) if bot_data["last_run_ended"]
75
- end
76
- data["Git URL"] = bot_data["git_url"]
77
- data["Repo Size"] = format_bytes(bot_data["repo_size"]) if bot_data["repo_size"]
78
- styled_hash(data)
68
+ error_message(response)
79
69
  end
80
70
  end
71
+ alias_command 'info', 'bots:info'
81
72
 
82
- alias_command "info", "bots:info"
83
-
84
-
85
- # bots:generate --bot name_of_bot
73
+ # bots:generate --bot BOT
86
74
  #
87
- # Generate stub code for a bot in specified language
75
+ #Generate a bot template in the specified language.
76
+ #
77
+ # -b, --bot BOT # a bot ID
78
+ # -l, --language LANGUAGE # ruby (default) or python
79
+ #
80
+ #Example:
81
+ #
82
+ # $ turbot bots:generate --bot my_amazing_bot --language ruby
83
+ # Created new bot template for my_amazing_bot!
88
84
  #
89
- # -l, --language LANGUAGE # language to generate (currently `ruby` (default) or `python`)
90
-
91
- # $ turbot bots:generate --language=ruby --bot my_amazing_bot
92
- # Created new bot template at my_amazing_bot!
93
-
94
85
  def generate
95
86
  validate_arguments!
96
- response = api.show_bot(bot)
97
- if response.is_a? Turbot::API::SuccessResponse
98
- error("There's already a bot called #{bot} registered with Turbot. Bot names must be unique.")
99
- end
87
+ error_if_bot_exists_in_turbot
100
88
 
101
- language = options[:language] || "ruby"
102
- scraper_template = File.expand_path("../../../../templates/#{language}", __FILE__)
103
- error("unsupported language #{language}") if !File.exists?(scraper_template)
89
+ # Check the bot name.
90
+ unless bot[BOT_ID_RE]
91
+ error "The bot name #{bot} is invalid. Bot names must contain only lowercase letters (a-z), numbers (0-9), underscore (_) or hyphen (-)."
92
+ end
104
93
 
105
- manifest_template = File.expand_path("../../../../templates/manifest.json", __FILE__)
106
- license_template = File.expand_path("../../../../templates/LICENSE.txt", __FILE__)
107
- manifest = open(manifest_template).read.sub("{{bot_id}}", bot)
108
- scraper_name = case language
109
- when "ruby"
110
- "scraper.rb"
111
- when "python"
112
- "scraper.py"
113
- end
94
+ # Check collision with existing directory.
95
+ bot_directory = File.join(working_directory, bot)
96
+ if File.exists?(bot_directory)
97
+ error "There's already a directory named #{bot}. Move it, delete it, change directory, or try another name."
98
+ end
114
99
 
115
- manifest = manifest.sub("{{scraper_name}}", scraper_name)
116
- manifest = manifest.sub("{{language}}", language)
100
+ language = (options[:language] || 'ruby').downcase
101
+ scraper_template = File.expand_path("../../../../data/templates/#{language}", __FILE__)
117
102
 
118
- # Language-specific stuff
119
- # Language-specific stuff:
120
- if File.exists? bot
121
- error("There's already a folder called #{bot}; move it out the way or try a different name")
103
+ # Check language.
104
+ unless File.exists?(scraper_template)
105
+ error "The language #{language} is unsupported."
122
106
  end
123
- FileUtils.mkdir(bot)
124
- FileUtils.cp_r(Dir["#{scraper_template}/*"], bot)
125
107
 
126
- # Same for all languages
127
- FileUtils.cp(license_template, "#{bot}/LICENSE.txt")
128
- open("#{bot}/manifest.json", "w") do |f|
129
- f.write(JSON.pretty_generate(JSON.parse(manifest)))
108
+ scraper_name = case language
109
+ when 'ruby'
110
+ 'scraper.rb'
111
+ when 'python'
112
+ 'scraper.py'
130
113
  end
131
114
 
132
- response = api.create_bot(bot, JSON.parse(manifest))
133
- if response.is_a? Turbot::API::SuccessResponse
134
- puts "Created new bot template at #{bot}!"
135
- else
136
- error(response.message)
115
+ # Create the scraper.
116
+ FileUtils.mkdir(bot_directory)
117
+ FileUtils.cp(File.join(scraper_template, scraper_name), File.join(bot_directory, scraper_name))
118
+
119
+ # Create the license.
120
+ license_template = File.expand_path('../../../../data/templates/LICENSE.txt', __FILE__)
121
+ FileUtils.cp(license_template, File.join(bot_directory, 'LICENSE.txt'))
122
+
123
+ # Create the manifest.
124
+ manifest_template = File.expand_path('../../../../data/templates/manifest.json', __FILE__)
125
+ manifest = File.read(manifest_template).
126
+ sub('{{bot_id}}', bot).
127
+ sub('{{scraper_name}}', scraper_name).
128
+ sub('{{language}}', language)
129
+ File.open(File.join(bot_directory, 'manifest.json'), 'w') do |f|
130
+ f.write(JSON.pretty_generate(JSON.load(manifest)))
137
131
  end
138
- end
139
132
 
133
+ puts "Created new bot template for #{bot}!"
134
+ end
140
135
 
141
136
  # bots:register
142
137
  #
143
- # Register a bot with turbot. Must be run from a folder containing scraper and manifest.json
144
-
145
- # $ turbot bots:register
146
- # Registered my_amazing_bot!
147
-
138
+ #Register a bot with Turbot. Must be run from a bot directory containing a `manifest.json` file.
139
+ #
140
+ #Example:
141
+ #
142
+ # $ turbot bots:register
143
+ # Registered my_amazing_bot!
144
+ #
148
145
  def register
149
- response = api.show_bot(bot)
150
- if response.is_a? Turbot::API::SuccessResponse
151
- error("There's already a bot called #{bot} registered with Turbot. Bot names must be unique.")
152
- end
146
+ validate_arguments!
147
+ error_if_no_local_bot_found
148
+ error_if_bot_exists_in_turbot
153
149
 
154
- manifest = parsed_manifest(working_directory)
155
- response = api.create_bot(bot, manifest)
156
- if response.is_a? Turbot::API::FailureResponse
157
- error(response.message)
158
- else
150
+ response = api.create_bot(bot, parse_manifest)
151
+ if response.is_a?(Turbot::API::SuccessResponse)
159
152
  puts "Registered #{bot}!"
153
+ else
154
+ error_message(response)
160
155
  end
161
156
  end
162
157
 
163
158
  # bots:push
164
159
  #
165
- # Push bot code to the turbot server. Must be run from a local bot checkout.
160
+ #Push the bot's code to Turbot. Must be run from a bot directory containing a `manifest.json` file.
161
+ #
162
+ # -y, --yes # skip confirmation
163
+ #
164
+ #Example:
165
+ #
166
+ # $ turbot bots:push
167
+ # This will submit your bot and its data for review.
168
+ # Are you happy your bot produces valid data (e.g. with `turbot bots:validate`)? [Y/n]
169
+ # Your bot has been pushed to Turbot and will be reviewed for inclusion as soon as we can. THANK YOU!
166
170
  #
167
- # $ turbot bots:push
168
- # Your bot has been pushed to Turbot and will be reviewed for inclusion as soon as we can. THANKYOU!
169
-
170
171
  def push
171
172
  validate_arguments!
172
- puts "This will submit your bot and its data for review."
173
- puts "Are you happy your bot produces valid data (e.g. with `turbot bots:validate`)? [Y/n]"
174
- confirmed = ask
175
- error("Aborting push") if !confirmed.downcase.empty? && confirmed.downcase != "y"
176
- manifest = parsed_manifest(working_directory)
177
- archive = Tempfile.new(bot)
178
- archive_path = "#{archive.path}.zip"
179
- create_zip_archive(archive_path, working_directory, manifest['files'] + ['manifest.json'])
180
-
181
- response = File.open(archive_path) {|file| api.update_code(bot, file)}
182
- case response
183
- when Turbot::API::SuccessResponse
184
- puts "Your bot has been pushed to Turbot and will be reviewed for inclusion as soon as we can. THANK YOU!"
185
- when Turbot::API::FailureResponse
186
- error(response.message)
173
+ error_if_no_local_bot_found
174
+
175
+ unless options[:yes]
176
+ puts 'This will submit your bot and its data for review.'
177
+ puts 'Are you happy your bot produces valid data (e.g. with `turbot bots:validate`)? [Y/n]'
178
+ answer = ask
179
+ unless ['', 'y'].include?(answer.downcase.strip)
180
+ error 'Aborted.'
181
+ end
182
+ end
183
+
184
+ # TODO Validate the manifest.json file.
185
+
186
+ manifest = parse_manifest
187
+ tempfile = Tempfile.new(bot)
188
+ tempfile.close # Windows will raise Errno::EACCES on Zip::File.open below
189
+ archive_path = "#{tempfile.path}.zip"
190
+ create_zip_archive(archive_path, manifest['files'] + ['manifest.json'])
191
+
192
+ response = File.open(archive_path) do |f|
193
+ api.update_code(bot, f)
194
+ end
195
+ if response.is_a?(Turbot::API::SuccessResponse)
196
+ puts 'Your bot has been pushed to Turbot and will be reviewed for inclusion as soon as we can. THANK YOU!'
197
+ else
198
+ error_message(response)
187
199
  end
188
200
  end
201
+ alias_command 'push', 'bots:push'
189
202
 
190
- alias_command "push", "bots:push"
191
203
 
192
204
  # bots:validate
193
205
  #
194
- # Validate bot output against its schema
206
+ #Validate the `manifest.json` file and validate the bot's output against its schema.
207
+ #
208
+ #Example:
209
+ #
210
+ # $ turbot bots:validate
211
+ # Validated 2 records!
195
212
  #
196
- # $ turbot bots:validate
197
- # Validating example... done
198
-
199
213
  def validate
200
- scraper_path = shift_argument || scraper_file(working_directory)
201
214
  validate_arguments!
202
- config = parsed_manifest(working_directory)
215
+ error_if_no_local_bot_found
216
+
217
+ manifest = parse_manifest
203
218
 
204
- %w(bot_id data_type identifying_fields files language publisher).each do |key|
205
- error("Manifest is missing #{key}") unless config.has_key?(key)
219
+ { 'allow_duplicates' => 'duplicates_allowed',
220
+ 'author' => 'publisher',
221
+ 'incremental' => 'manually_end_run',
222
+ 'public_repository' => 'public_repo_url',
223
+ }.each do |deprecated,field|
224
+ if manifest[deprecated]
225
+ puts %(WARNING: "#{deprecated}" is deprecated. Use "#{field}" instead.)
226
+ end
227
+ end
228
+
229
+ schema = JSON.load(File.read(File.expand_path('../../../../data/schema.json', __FILE__)))
230
+ validator = JSON::Validator.new(schema, manifest, {
231
+ clear_cache: false,
232
+ parse_data: false,
233
+ record_errors: true,
234
+ errors_as_objects: true,
235
+ })
236
+
237
+ errors = validator.validate
238
+ if errors.any?
239
+ messages = ['`manifest.json` is invalid. Please correct the errors:']
240
+ errors.each do |error|
241
+ messages << "* #{error.fetch(:message).sub(/ in schema \S+\z/, '')}"
242
+ end
243
+ error messages.join("\n")
206
244
  end
207
245
 
208
- type = config["data_type"]
246
+ if manifest['transformers']
247
+ difference = manifest['transformers'].map { |transformer| transformer['file'] } - manifest['files']
248
+ if difference.any?
249
+ messages = ['`manifest.json` is invalid. Please correct the errors:']
250
+ messages << "* Some transformer files are not listed in the top-level files: #{difference.join(', ')}"
251
+ error messages.join("\n")
252
+ end
253
+ end
209
254
 
210
- handler = ValidationHandler.new
255
+ handler = Turbot::Handlers::ValidationHandler.new
211
256
  runner = TurbotRunner::Runner.new(working_directory, :record_handler => handler)
212
- rc = runner.run
257
+ begin
258
+ rc = runner.run
259
+ rescue TurbotRunner::InvalidDataType
260
+ messages = ['`manifest.json` is invalid. Please correct the errors:']
261
+ messages << %(* The property '#/data_type' value "#{manifest['data_type']}" is not a supported data type.)
262
+ error messages.join("\n")
263
+ end
213
264
 
214
- puts
215
265
  if rc == TurbotRunner::Runner::RC_OK
216
266
  puts "Validated #{handler.count} records!"
217
267
  else
@@ -221,216 +271,117 @@ class Turbot::Command::Bots < Turbot::Command::Base
221
271
 
222
272
  # bots:dump
223
273
  #
224
- # Execute bot locally (writes to STDOUT)
274
+ #Execute the bot locally and write the bot's output to STDOUT.
275
+ #
276
+ # -q, --quiet # only output validation errors
277
+ #
278
+ #Example:
279
+ #
280
+ # $ turbot bots:dump
281
+ # {'foo': 'bar'}
282
+ # {'foo2': 'bar2'}
225
283
  #
226
- # $ turbot bots:dump
227
- # {'foo': 'bar'}
228
- # {'foo2': 'bar2'}
229
-
230
284
  def dump
231
285
  validate_arguments!
286
+ error_if_no_local_bot_found
232
287
 
233
- handler = DumpHandler.new
288
+ if options[:quiet]
289
+ handler = Turbot::Handlers::BaseHandler.new
290
+ else
291
+ handler = Turbot::Handlers::DumpHandler.new
292
+ end
234
293
  runner = TurbotRunner::Runner.new(working_directory, :record_handler => handler)
235
294
  rc = runner.run
236
295
 
237
- puts
238
296
  if rc == TurbotRunner::Runner::RC_OK
239
- puts "Bot ran successfully!"
297
+ puts 'Bot ran successfully!'
240
298
  else
241
- puts "Bot failed!"
299
+ puts 'Bot failed!'
242
300
  end
243
301
  end
244
302
 
245
- # # bots:single
246
- # #
247
- # # Execute bot in same way as OpenCorporates single-record update
248
- # #
249
- # # $ turbot bots:single
250
- # # Enter argument (as JSON object):
251
- # # {"id": "frob123"}
252
- # # {"id": "frob123", "stuff": "updated-data-for-this-record"}
253
- #
254
- # def single
255
- # # This will need to be language-aware, eventually
256
- # scraper_path = shift_argument || scraper_file(working_directory)
257
- # validate_arguments!
258
- # print 'Arguments (as JSON object, e.g. {"id":"ABC123"}: '
259
- # arg = ask
260
- # count = 0
261
- # run_scraper_each_line("#{scraper_path} #{bot} #{Shellwords.shellescape(arg)}") do |line|
262
- # raise "Your scraper returned more than one value!" if count > 1
263
- # puts line
264
- # count += 1
265
- # end
266
- # end
267
-
268
-
269
303
  # bots:preview
270
304
  #
271
- # Send bot data to Turbot for remote previewing / sharing
305
+ #Send bot data to Turbot for remote previewing or sharing.
306
+ #
307
+ #Example:
308
+ #
309
+ # $ turbot bots:preview
310
+ # Sending to Turbot...
311
+ # Submitted 2 records to Turbot.
312
+ # View your records at http://turbot.opencorporates.com/..
272
313
  #
273
- # Sending example to turbot... done
274
314
  def preview
275
315
  validate_arguments!
316
+ error_if_no_local_bot_found
276
317
 
277
- config = parsed_manifest(working_directory)
278
-
279
- response = api.update_bot(bot, parsed_manifest(working_directory))
280
- if !response.is_a? Turbot::API::SuccessResponse
281
- error(response.message)
318
+ response = api.update_bot(bot, parse_manifest)
319
+ if response.is_a?(Turbot::API::FailureResponse)
320
+ error_message(response)
282
321
  end
283
322
 
284
323
  response = api.destroy_draft_data(bot)
285
- if !response.is_a? Turbot::API::SuccessResponse
286
- error(response.message)
324
+ if response.is_a?(Turbot::API::FailureResponse)
325
+ error_message(response)
287
326
  end
288
327
 
289
- puts "Sending to turbot... "
328
+ puts 'Sending to Turbot...'
290
329
 
291
- handler = PreviewHandler.new(bot, api)
330
+ handler = Turbot::Handlers::PreviewHandler.new(bot, api)
292
331
  runner = TurbotRunner::Runner.new(working_directory, :record_handler => handler)
293
332
  rc = runner.run
294
333
 
295
- puts
296
-
297
334
  if rc == TurbotRunner::Runner::RC_OK
298
335
  response = handler.submit_batch
299
- if response.is_a? Turbot::API::SuccessResponse
336
+ if response.is_a?(Turbot::API::SuccessResponse)
300
337
  if handler.count > 0
301
- puts "Submitted #{handler.count} records to turbot"
302
- puts "View your records at #{response.data[:url]}"
338
+ puts "Submitted #{handler.count} records to Turbot.\nView your records at #{response.data[:url]}"
303
339
  else
304
- puts "No records sent"
340
+ puts 'No records sent.'
305
341
  end
306
342
  else
307
- error(response.message)
343
+ error_message(response)
308
344
  end
309
345
  else
310
- puts
311
- puts "Bot failed!"
346
+ puts 'Bot failed!'
312
347
  end
313
348
  end
314
349
 
315
- private
316
- def spinner(p)
317
- parts = "\|/-" * 2
318
- print parts[p % parts.length] + "\r"
319
- end
350
+ private
320
351
 
321
- def parsed_manifest(dir)
322
- begin
323
- JSON.parse(open(manifest_path).read)
324
- rescue Errno::ENOENT
325
- raise "This command must be run from a directory including `manifest.json`"
352
+ def error_if_no_local_bot_found
353
+ unless bot
354
+ error "No bot specified.\nRun this command from a bot directory containing a `manifest.json` file, or specify the bot with --bot BOT."
326
355
  end
327
356
  end
328
357
 
329
- def scraper_file(dir)
330
- Dir.glob("scraper*").reject{|n| !n.match(/(rb|py)$/)}.first
331
- end
332
-
333
- def working_directory
334
- Dir.pwd
358
+ def error_if_bot_exists_in_turbot
359
+ if api.show_bot(bot).is_a?(Turbot::API::SuccessResponse)
360
+ error "There's already a bot named #{bot} in Turbot. Try another name."
361
+ end
335
362
  end
336
363
 
337
- def manifest_path
338
- File.join(working_directory, 'manifest.json')
364
+ def error_message(response)
365
+ suffix = response.error_code && ": #{response.error_code}"
366
+ error "#{response.message} (HTTP #{response.code}#{suffix})"
339
367
  end
340
368
 
341
- def create_zip_archive(archive_path, basepath, subpaths)
369
+ def create_zip_archive(archive_path, basenames)
342
370
  Zip.continue_on_exists_proc = true
371
+
343
372
  Zip::File.open(archive_path, Zip::File::CREATE) do |zipfile|
344
- subpaths.each do |subpath|
345
- path = File.join(basepath, subpath)
373
+ basenames.each do |basename|
374
+ filename = File.join(working_directory, basename)
346
375
 
347
- if File.directory?(path)
348
- Dir["#{path}/**/*"].each do |path1|
349
- subpath1 = Pathname.new(path1).relative_path_from(Pathname.new(basepath))
350
- zipfile.add(subpath1, path1)
376
+ if File.directory?(filename)
377
+ Dir["#{filename}/**/*"].each do |filename1|
378
+ basename1 = Pathname.new(filename1).relative_path_from(Pathname.new(working_directory))
379
+ zipfile.add(basename1, filename1)
351
380
  end
352
381
  else
353
- zipfile.add(subpath, path)
382
+ zipfile.add(basename, filename)
354
383
  end
355
384
  end
356
385
  end
357
386
  end
358
387
  end
359
-
360
- class TurbotClientHandler < TurbotRunner::BaseHandler
361
- def handle_invalid_record(record, data_type, error_message)
362
- puts
363
- puts "The following record is invalid:"
364
- puts record.to_json
365
- puts " * #{error_message}"
366
- puts
367
- end
368
-
369
- def handle_non_json_output(line)
370
- puts
371
- puts "The following line was not valid JSON:"
372
- puts line
373
- end
374
- end
375
-
376
-
377
- class DumpHandler < TurbotClientHandler
378
- def handle_valid_record(record, data_type)
379
- puts record.to_json
380
- end
381
- end
382
-
383
-
384
- class ValidationHandler < TurbotClientHandler
385
- attr_reader :count
386
-
387
- def initialize(*)
388
- @count = 0
389
- super
390
- end
391
-
392
- def handle_valid_record(record, data_type)
393
- @count += 1
394
- STDOUT.write('.')
395
- end
396
-
397
- def handle_invalid_record(record, data_type, error_message)
398
- puts
399
- puts "The following record is invalid:"
400
- puts record.to_json
401
- puts " * #{error_message}"
402
- puts
403
- end
404
-
405
- def handle_invalid_json(line)
406
- puts
407
- puts "The following line was not valid JSON:"
408
- puts line
409
- end
410
- end
411
-
412
-
413
- class PreviewHandler < TurbotClientHandler
414
- attr_reader :count
415
-
416
- def initialize(bot_name, api)
417
- @bot_name = bot_name
418
- @api = api
419
- @batch = []
420
- @count = 0
421
- super()
422
- end
423
-
424
- def handle_valid_record(record, data_type)
425
- @count += 1
426
- STDOUT.write('.')
427
- @batch << record.merge(:data_type => data_type)
428
- submit_batch if @count % 20 == 0
429
- end
430
-
431
- def submit_batch
432
- result = @api.create_draft_data(@bot_name, @batch.to_json)
433
- @batch = []
434
- result
435
- end
436
- end