turbot 0.1.36 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +8 -0
- data/.rspec +3 -0
- data/.travis.yml +15 -0
- data/.yardopts +3 -0
- data/Gemfile +4 -0
- data/LICENSE +22 -0
- data/README.md +44 -25
- data/Rakefile +16 -0
- data/appveyor.yml +35 -0
- data/bin/turbot +2 -16
- data/data/schema.json +134 -0
- data/{templates → data/templates}/LICENSE.txt +0 -0
- data/{templates → data/templates}/manifest.json +0 -0
- data/{templates → data/templates}/python/scraper.py +0 -0
- data/{templates → data/templates}/ruby/scraper.rb +0 -0
- data/dist/deb.rake +32 -0
- data/dist/gem.rake +16 -0
- data/dist/manifest.rake +9 -0
- data/dist/pkg.rake +60 -0
- data/dist/resources/deb/control +10 -0
- data/dist/resources/deb/postinst +45 -0
- data/dist/resources/deb/turbot +25 -0
- data/dist/resources/deb/turbot-release-key.txt +30 -0
- data/dist/resources/pkg/Distribution.erb +15 -0
- data/dist/resources/pkg/PackageInfo.erb +6 -0
- data/dist/resources/pkg/postinstall +45 -0
- data/dist/resources/pkg/turbot +24 -0
- data/dist/resources/tgz/turbot +24 -0
- data/dist/rpm.rake +35 -0
- data/dist/tgz.rake +26 -0
- data/dist/zip.rake +40 -0
- data/lib/turbot.rb +18 -15
- data/lib/turbot/cli.rb +10 -27
- data/lib/turbot/command.rb +59 -212
- data/lib/turbot/command/auth.rb +72 -34
- data/lib/turbot/command/base.rb +22 -61
- data/lib/turbot/command/bots.rb +251 -300
- data/lib/turbot/command/help.rb +57 -110
- data/lib/turbot/command/version.rb +6 -10
- data/lib/turbot/handlers/base_handler.rb +21 -0
- data/lib/turbot/handlers/dump_handler.rb +10 -0
- data/lib/turbot/handlers/preview_handler.rb +30 -0
- data/lib/turbot/handlers/validation_handler.rb +17 -0
- data/lib/turbot/helpers.rb +14 -482
- data/lib/turbot/helpers/api_helper.rb +41 -0
- data/lib/turbot/helpers/netrc_helper.rb +66 -0
- data/lib/turbot/helpers/shell_helper.rb +36 -0
- data/lib/turbot/version.rb +1 -1
- data/spec/fixtures/bad_permissions +0 -0
- data/spec/fixtures/empty +0 -0
- data/spec/fixtures/netrc +6 -0
- data/spec/spec_helper.rb +17 -219
- data/spec/support/bot_helper.rb +102 -0
- data/spec/support/command_helper.rb +20 -0
- data/spec/support/custom_matchers.rb +5 -0
- data/spec/support/fixture_helper.rb +9 -0
- data/spec/support/netrc_helper.rb +21 -0
- data/spec/turbot/command/auth_spec.rb +202 -20
- data/spec/turbot/command/base_spec.rb +22 -58
- data/spec/turbot/command/bots_spec.rb +580 -89
- data/spec/turbot/command/help_spec.rb +32 -75
- data/spec/turbot/command/version_spec.rb +11 -10
- data/spec/turbot/command_spec.rb +55 -87
- data/spec/turbot/helpers_spec.rb +28 -44
- data/turbot.gemspec +31 -0
- metadata +88 -178
- data/data/cacert.pem +0 -3988
- data/lib/turbot/auth.rb +0 -315
- data/lib/turbot/client.rb +0 -757
- data/lib/turbot/client/cisaurus.rb +0 -25
- data/lib/turbot/client/pgbackups.rb +0 -113
- data/lib/turbot/client/rendezvous.rb +0 -111
- data/lib/turbot/client/ssl_endpoint.rb +0 -25
- data/lib/turbot/client/turbot_postgresql.rb +0 -148
- data/lib/turbot/command/ssl.rb +0 -43
- data/lib/turbot/deprecated.rb +0 -5
- data/lib/turbot/deprecated/help.rb +0 -38
- data/lib/turbot/distribution.rb +0 -9
- data/lib/turbot/errors.rb +0 -28
- data/lib/turbot/excon.rb +0 -11
- data/lib/turbot/helpers/log_displayer.rb +0 -70
- data/lib/turbot/helpers/pg_dump_restore.rb +0 -115
- data/lib/turbot/helpers/turbot_postgresql.rb +0 -213
- data/lib/turbot/plugin.rb +0 -165
- data/lib/turbot/updater.rb +0 -171
- data/lib/vendor/turbot/okjson.rb +0 -598
- data/spec/helper/legacy_help.rb +0 -16
- data/spec/helper/pg_dump_restore_spec.rb +0 -67
- data/spec/spec.opts +0 -1
- data/spec/support/display_message_matcher.rb +0 -49
- data/spec/support/dummy_api.rb +0 -120
- data/spec/support/openssl_mock_helper.rb +0 -8
- data/spec/support/organizations_mock_helper.rb +0 -11
- data/spec/turbot/auth_spec.rb +0 -214
- data/spec/turbot/client/pgbackups_spec.rb +0 -43
- data/spec/turbot/client/rendezvous_spec.rb +0 -62
- data/spec/turbot/client/ssl_endpoint_spec.rb +0 -48
- data/spec/turbot/client/turbot_postgresql_spec.rb +0 -71
- data/spec/turbot/client_spec.rb +0 -548
- data/spec/turbot/helpers/turbot_postgresql_spec.rb +0 -181
- data/spec/turbot/plugin_spec.rb +0 -172
- data/spec/turbot/updater_spec.rb +0 -44
data/lib/turbot/command/base.rb
CHANGED
@@ -1,19 +1,14 @@
|
|
1
|
-
require "fileutils"
|
2
|
-
require "turbot/auth"
|
3
|
-
require "turbot/client/rendezvous"
|
4
|
-
require "turbot/command"
|
5
|
-
|
6
1
|
class Turbot::Command::Base
|
7
2
|
include Turbot::Helpers
|
8
3
|
|
9
4
|
def self.namespace
|
10
|
-
self.to_s.split(
|
5
|
+
self.to_s.split('::').last.downcase
|
11
6
|
end
|
12
7
|
|
13
8
|
attr_reader :args
|
14
9
|
attr_reader :options
|
15
10
|
|
16
|
-
def initialize(args=[], options={})
|
11
|
+
def initialize(args = [], options = {})
|
17
12
|
@args = args
|
18
13
|
@options = options
|
19
14
|
end
|
@@ -21,24 +16,13 @@ class Turbot::Command::Base
|
|
21
16
|
def bot
|
22
17
|
@bot ||= if options[:bot].is_a?(String)
|
23
18
|
options[:bot]
|
24
|
-
elsif ENV
|
19
|
+
elsif ENV['TURBOT_BOT']
|
25
20
|
ENV['TURBOT_BOT']
|
26
|
-
elsif
|
27
|
-
|
28
|
-
else
|
29
|
-
# raise instead of using error command to enable rescuing when bot is optional
|
30
|
-
raise Turbot::Command::CommandFailed.new("No bot specified.\nRun this command from a bot folder containing a `manifest.json`, or specify which bot to use with --bot BOT_ID.") unless options[:ignore_no_bot]
|
21
|
+
elsif manifest = parse_manifest
|
22
|
+
manifest['bot_id']
|
31
23
|
end
|
32
24
|
end
|
33
25
|
|
34
|
-
def api
|
35
|
-
Turbot::Auth.api
|
36
|
-
end
|
37
|
-
|
38
|
-
def turbot
|
39
|
-
Turbot::Auth.client
|
40
|
-
end
|
41
|
-
|
42
26
|
protected
|
43
27
|
|
44
28
|
def self.inherited(klass)
|
@@ -53,13 +37,13 @@ protected
|
|
53
37
|
end
|
54
38
|
|
55
39
|
def self.method_added(method)
|
56
|
-
|
57
|
-
|
58
|
-
|
40
|
+
if self == Turbot::Command::Base || private_method_defined?(method) || protected_method_defined?(method)
|
41
|
+
return
|
42
|
+
end
|
59
43
|
|
60
44
|
help = extract_help_from_caller(caller.first)
|
61
45
|
resolved_method = (method.to_s == "index") ? nil : method.to_s
|
62
|
-
command = [
|
46
|
+
command = [self.namespace, resolved_method].compact.join(":")
|
63
47
|
banner = extract_banner(help) || command
|
64
48
|
|
65
49
|
Turbot::Command.register_command(
|
@@ -73,18 +57,10 @@ protected
|
|
73
57
|
:description => extract_description(help),
|
74
58
|
:options => extract_options(help)
|
75
59
|
)
|
76
|
-
|
77
|
-
alias_command command.gsub(/_/, '-'), command if command =~ /_/
|
78
|
-
end
|
79
|
-
|
80
|
-
def self.alias_command(new, old)
|
81
|
-
raise "no such command: #{old}" unless Turbot::Command.commands[old]
|
82
|
-
Turbot::Command.command_aliases[new] = old
|
83
60
|
end
|
84
61
|
|
85
|
-
def
|
86
|
-
|
87
|
-
bot
|
62
|
+
def self.alias_command(command_alias, command)
|
63
|
+
Turbot::Command.command_aliases[command_alias] = command
|
88
64
|
end
|
89
65
|
|
90
66
|
#
|
@@ -113,7 +89,7 @@ protected
|
|
113
89
|
buffer = []
|
114
90
|
lines = Turbot::Command.files[file]
|
115
91
|
|
116
|
-
(line_number.to_i-2).downto(0) do |i|
|
92
|
+
(line_number.to_i - 2).downto(0) do |i|
|
117
93
|
line = lines[i]
|
118
94
|
case line[0..0]
|
119
95
|
when ""
|
@@ -156,37 +132,22 @@ protected
|
|
156
132
|
Turbot::Command.current_command
|
157
133
|
end
|
158
134
|
|
159
|
-
def extract_option(key)
|
160
|
-
options[key.dup.gsub('-','_').to_sym]
|
161
|
-
end
|
162
|
-
|
163
|
-
def invalid_arguments
|
164
|
-
Turbot::Command.invalid_arguments
|
165
|
-
end
|
166
|
-
|
167
|
-
def shift_argument
|
168
|
-
Turbot::Command.shift_argument
|
169
|
-
end
|
170
|
-
|
171
135
|
def validate_arguments!
|
172
136
|
Turbot::Command.validate_arguments!
|
173
137
|
end
|
174
138
|
|
175
|
-
def
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
139
|
+
def parse_manifest
|
140
|
+
path = File.join(working_directory, 'manifest.json')
|
141
|
+
if File.exists?(path)
|
142
|
+
begin
|
143
|
+
JSON.load(File.read(path))
|
144
|
+
rescue JSON::ParserError => e
|
145
|
+
error "`manifest.json` is invalid JSON. Consider validating it at http://pro.jsonlint.com/"
|
146
|
+
end
|
180
147
|
end
|
181
148
|
end
|
182
149
|
|
183
|
-
def
|
184
|
-
|
185
|
-
end
|
186
|
-
end
|
187
|
-
|
188
|
-
module Turbot::Command
|
189
|
-
unless const_defined?(:BaseWithApp)
|
190
|
-
BaseWithApp = Base
|
150
|
+
def working_directory
|
151
|
+
Dir.pwd
|
191
152
|
end
|
192
153
|
end
|
data/lib/turbot/command/bots.rb
CHANGED
@@ -1,217 +1,267 @@
|
|
1
|
-
|
2
|
-
require 'active_support/core_ext/object/blank'
|
3
|
-
require 'active_support/core_ext/hash/slice'
|
4
|
-
require 'zip'
|
5
|
-
require 'open3'
|
6
|
-
require 'base64'
|
7
|
-
require 'shellwords'
|
8
|
-
require 'turbot_runner'
|
9
|
-
|
10
|
-
# manage bots (generate skeleton, validate data, submit code)
|
1
|
+
#Manage bots (generate template, validate data, submit code)
|
11
2
|
#
|
12
3
|
class Turbot::Command::Bots < Turbot::Command::Base
|
4
|
+
BOT_ID_RE = /\A[A-Za-z0-9_-]+\z/.freeze
|
5
|
+
|
6
|
+
def initialize(*args)
|
7
|
+
super
|
8
|
+
|
9
|
+
require 'turbot_runner'
|
10
|
+
require 'turbot/handlers/base_handler'
|
11
|
+
require 'turbot/handlers/dump_handler'
|
12
|
+
require 'turbot/handlers/preview_handler'
|
13
|
+
require 'turbot/handlers/validation_handler'
|
14
|
+
end
|
13
15
|
|
14
16
|
# bots
|
15
17
|
#
|
16
|
-
#
|
18
|
+
#List your bots.
|
17
19
|
#
|
18
20
|
#Example:
|
19
21
|
#
|
20
|
-
#
|
21
|
-
#
|
22
|
-
#
|
23
|
-
# example2
|
22
|
+
# $ turbot bots
|
23
|
+
# example1
|
24
|
+
# example2
|
24
25
|
#
|
25
26
|
def index
|
26
27
|
validate_arguments!
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
28
|
+
|
29
|
+
response = api.list_bots
|
30
|
+
if response.is_a?(Turbot::API::SuccessResponse)
|
31
|
+
if response.data.empty?
|
32
|
+
puts 'You have no bots.'
|
33
|
+
else
|
34
|
+
response.data.each do |bot|
|
35
|
+
puts bot[:bot_id]
|
36
|
+
end
|
37
|
+
end
|
31
38
|
else
|
32
|
-
|
39
|
+
error_message(response)
|
33
40
|
end
|
34
41
|
end
|
42
|
+
alias_command 'list', 'bots'
|
35
43
|
|
36
|
-
|
37
|
-
|
38
|
-
# bots:info
|
39
|
-
#
|
40
|
-
# show detailed bot information
|
44
|
+
# bots:info [--bot BOT]
|
41
45
|
#
|
42
|
-
#
|
46
|
+
#Show a bot's details.
|
43
47
|
#
|
44
|
-
#
|
48
|
+
# -b, --bot BOT # a bot ID
|
45
49
|
#
|
46
|
-
#
|
47
|
-
# === example
|
48
|
-
# Last run status: OK
|
49
|
-
# Last run ended: 2001/01/01
|
50
|
-
# ...
|
50
|
+
#Example:
|
51
51
|
#
|
52
|
-
#
|
53
|
-
#
|
54
|
-
#
|
55
|
-
#
|
52
|
+
# $ turbot bots:info --bot example
|
53
|
+
# bot_id: example
|
54
|
+
# created_at: 2010-01-01T00:00:00.000Z
|
55
|
+
# updated_at: 2010-01-02T00:00:00.000Z
|
56
|
+
# state: scheduled
|
56
57
|
#
|
57
58
|
def info
|
58
59
|
validate_arguments!
|
59
|
-
|
60
|
-
unless options[:shell]
|
61
|
-
styled_header(bot_data["name"])
|
62
|
-
end
|
60
|
+
error_if_no_local_bot_found
|
63
61
|
|
64
|
-
|
65
|
-
|
66
|
-
|
62
|
+
response = api.show_bot(bot)
|
63
|
+
if response.is_a?(Turbot::API::SuccessResponse)
|
64
|
+
response.data.each do |key,value|
|
65
|
+
puts "#{key}: #{value}"
|
67
66
|
end
|
68
67
|
else
|
69
|
-
|
70
|
-
if bot_data["last_run_status"]
|
71
|
-
data["Last run status"] = bot_data["last_run_status"]
|
72
|
-
end
|
73
|
-
if bot_data["last_run_ended"]
|
74
|
-
data["Last run ended"] = format_date(bot_data["last_run_ended"]) if bot_data["last_run_ended"]
|
75
|
-
end
|
76
|
-
data["Git URL"] = bot_data["git_url"]
|
77
|
-
data["Repo Size"] = format_bytes(bot_data["repo_size"]) if bot_data["repo_size"]
|
78
|
-
styled_hash(data)
|
68
|
+
error_message(response)
|
79
69
|
end
|
80
70
|
end
|
71
|
+
alias_command 'info', 'bots:info'
|
81
72
|
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
# bots:generate --bot name_of_bot
|
73
|
+
# bots:generate --bot BOT
|
86
74
|
#
|
87
|
-
#
|
75
|
+
#Generate a bot template in the specified language.
|
76
|
+
#
|
77
|
+
# -b, --bot BOT # a bot ID
|
78
|
+
# -l, --language LANGUAGE # ruby (default) or python
|
79
|
+
#
|
80
|
+
#Example:
|
81
|
+
#
|
82
|
+
# $ turbot bots:generate --bot my_amazing_bot --language ruby
|
83
|
+
# Created new bot template for my_amazing_bot!
|
88
84
|
#
|
89
|
-
# -l, --language LANGUAGE # language to generate (currently `ruby` (default) or `python`)
|
90
|
-
|
91
|
-
# $ turbot bots:generate --language=ruby --bot my_amazing_bot
|
92
|
-
# Created new bot template at my_amazing_bot!
|
93
|
-
|
94
85
|
def generate
|
95
86
|
validate_arguments!
|
96
|
-
|
97
|
-
if response.is_a? Turbot::API::SuccessResponse
|
98
|
-
error("There's already a bot called #{bot} registered with Turbot. Bot names must be unique.")
|
99
|
-
end
|
87
|
+
error_if_bot_exists_in_turbot
|
100
88
|
|
101
|
-
|
102
|
-
|
103
|
-
|
89
|
+
# Check the bot name.
|
90
|
+
unless bot[BOT_ID_RE]
|
91
|
+
error "The bot name #{bot} is invalid. Bot names must contain only lowercase letters (a-z), numbers (0-9), underscore (_) or hyphen (-)."
|
92
|
+
end
|
104
93
|
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
"scraper.rb"
|
111
|
-
when "python"
|
112
|
-
"scraper.py"
|
113
|
-
end
|
94
|
+
# Check collision with existing directory.
|
95
|
+
bot_directory = File.join(working_directory, bot)
|
96
|
+
if File.exists?(bot_directory)
|
97
|
+
error "There's already a directory named #{bot}. Move it, delete it, change directory, or try another name."
|
98
|
+
end
|
114
99
|
|
115
|
-
|
116
|
-
|
100
|
+
language = (options[:language] || 'ruby').downcase
|
101
|
+
scraper_template = File.expand_path("../../../../data/templates/#{language}", __FILE__)
|
117
102
|
|
118
|
-
#
|
119
|
-
|
120
|
-
|
121
|
-
error("There's already a folder called #{bot}; move it out the way or try a different name")
|
103
|
+
# Check language.
|
104
|
+
unless File.exists?(scraper_template)
|
105
|
+
error "The language #{language} is unsupported."
|
122
106
|
end
|
123
|
-
FileUtils.mkdir(bot)
|
124
|
-
FileUtils.cp_r(Dir["#{scraper_template}/*"], bot)
|
125
107
|
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
108
|
+
scraper_name = case language
|
109
|
+
when 'ruby'
|
110
|
+
'scraper.rb'
|
111
|
+
when 'python'
|
112
|
+
'scraper.py'
|
130
113
|
end
|
131
114
|
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
115
|
+
# Create the scraper.
|
116
|
+
FileUtils.mkdir(bot_directory)
|
117
|
+
FileUtils.cp(File.join(scraper_template, scraper_name), File.join(bot_directory, scraper_name))
|
118
|
+
|
119
|
+
# Create the license.
|
120
|
+
license_template = File.expand_path('../../../../data/templates/LICENSE.txt', __FILE__)
|
121
|
+
FileUtils.cp(license_template, File.join(bot_directory, 'LICENSE.txt'))
|
122
|
+
|
123
|
+
# Create the manifest.
|
124
|
+
manifest_template = File.expand_path('../../../../data/templates/manifest.json', __FILE__)
|
125
|
+
manifest = File.read(manifest_template).
|
126
|
+
sub('{{bot_id}}', bot).
|
127
|
+
sub('{{scraper_name}}', scraper_name).
|
128
|
+
sub('{{language}}', language)
|
129
|
+
File.open(File.join(bot_directory, 'manifest.json'), 'w') do |f|
|
130
|
+
f.write(JSON.pretty_generate(JSON.load(manifest)))
|
137
131
|
end
|
138
|
-
end
|
139
132
|
|
133
|
+
puts "Created new bot template for #{bot}!"
|
134
|
+
end
|
140
135
|
|
141
136
|
# bots:register
|
142
137
|
#
|
143
|
-
#
|
144
|
-
|
145
|
-
#
|
146
|
-
#
|
147
|
-
|
138
|
+
#Register a bot with Turbot. Must be run from a bot directory containing a `manifest.json` file.
|
139
|
+
#
|
140
|
+
#Example:
|
141
|
+
#
|
142
|
+
# $ turbot bots:register
|
143
|
+
# Registered my_amazing_bot!
|
144
|
+
#
|
148
145
|
def register
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
end
|
146
|
+
validate_arguments!
|
147
|
+
error_if_no_local_bot_found
|
148
|
+
error_if_bot_exists_in_turbot
|
153
149
|
|
154
|
-
|
155
|
-
response
|
156
|
-
if response.is_a? Turbot::API::FailureResponse
|
157
|
-
error(response.message)
|
158
|
-
else
|
150
|
+
response = api.create_bot(bot, parse_manifest)
|
151
|
+
if response.is_a?(Turbot::API::SuccessResponse)
|
159
152
|
puts "Registered #{bot}!"
|
153
|
+
else
|
154
|
+
error_message(response)
|
160
155
|
end
|
161
156
|
end
|
162
157
|
|
163
158
|
# bots:push
|
164
159
|
#
|
165
|
-
#
|
160
|
+
#Push the bot's code to Turbot. Must be run from a bot directory containing a `manifest.json` file.
|
161
|
+
#
|
162
|
+
# -y, --yes # skip confirmation
|
163
|
+
#
|
164
|
+
#Example:
|
165
|
+
#
|
166
|
+
# $ turbot bots:push
|
167
|
+
# This will submit your bot and its data for review.
|
168
|
+
# Are you happy your bot produces valid data (e.g. with `turbot bots:validate`)? [Y/n]
|
169
|
+
# Your bot has been pushed to Turbot and will be reviewed for inclusion as soon as we can. THANK YOU!
|
166
170
|
#
|
167
|
-
# $ turbot bots:push
|
168
|
-
# Your bot has been pushed to Turbot and will be reviewed for inclusion as soon as we can. THANKYOU!
|
169
|
-
|
170
171
|
def push
|
171
172
|
validate_arguments!
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
173
|
+
error_if_no_local_bot_found
|
174
|
+
|
175
|
+
unless options[:yes]
|
176
|
+
puts 'This will submit your bot and its data for review.'
|
177
|
+
puts 'Are you happy your bot produces valid data (e.g. with `turbot bots:validate`)? [Y/n]'
|
178
|
+
answer = ask
|
179
|
+
unless ['', 'y'].include?(answer.downcase.strip)
|
180
|
+
error 'Aborted.'
|
181
|
+
end
|
182
|
+
end
|
183
|
+
|
184
|
+
# TODO Validate the manifest.json file.
|
185
|
+
|
186
|
+
manifest = parse_manifest
|
187
|
+
tempfile = Tempfile.new(bot)
|
188
|
+
tempfile.close # Windows will raise Errno::EACCES on Zip::File.open below
|
189
|
+
archive_path = "#{tempfile.path}.zip"
|
190
|
+
create_zip_archive(archive_path, manifest['files'] + ['manifest.json'])
|
191
|
+
|
192
|
+
response = File.open(archive_path) do |f|
|
193
|
+
api.update_code(bot, f)
|
194
|
+
end
|
195
|
+
if response.is_a?(Turbot::API::SuccessResponse)
|
196
|
+
puts 'Your bot has been pushed to Turbot and will be reviewed for inclusion as soon as we can. THANK YOU!'
|
197
|
+
else
|
198
|
+
error_message(response)
|
187
199
|
end
|
188
200
|
end
|
201
|
+
alias_command 'push', 'bots:push'
|
189
202
|
|
190
|
-
alias_command "push", "bots:push"
|
191
203
|
|
192
204
|
# bots:validate
|
193
205
|
#
|
194
|
-
#
|
206
|
+
#Validate the `manifest.json` file and validate the bot's output against its schema.
|
207
|
+
#
|
208
|
+
#Example:
|
209
|
+
#
|
210
|
+
# $ turbot bots:validate
|
211
|
+
# Validated 2 records!
|
195
212
|
#
|
196
|
-
# $ turbot bots:validate
|
197
|
-
# Validating example... done
|
198
|
-
|
199
213
|
def validate
|
200
|
-
scraper_path = shift_argument || scraper_file(working_directory)
|
201
214
|
validate_arguments!
|
202
|
-
|
215
|
+
error_if_no_local_bot_found
|
216
|
+
|
217
|
+
manifest = parse_manifest
|
203
218
|
|
204
|
-
|
205
|
-
|
219
|
+
{ 'allow_duplicates' => 'duplicates_allowed',
|
220
|
+
'author' => 'publisher',
|
221
|
+
'incremental' => 'manually_end_run',
|
222
|
+
'public_repository' => 'public_repo_url',
|
223
|
+
}.each do |deprecated,field|
|
224
|
+
if manifest[deprecated]
|
225
|
+
puts %(WARNING: "#{deprecated}" is deprecated. Use "#{field}" instead.)
|
226
|
+
end
|
227
|
+
end
|
228
|
+
|
229
|
+
schema = JSON.load(File.read(File.expand_path('../../../../data/schema.json', __FILE__)))
|
230
|
+
validator = JSON::Validator.new(schema, manifest, {
|
231
|
+
clear_cache: false,
|
232
|
+
parse_data: false,
|
233
|
+
record_errors: true,
|
234
|
+
errors_as_objects: true,
|
235
|
+
})
|
236
|
+
|
237
|
+
errors = validator.validate
|
238
|
+
if errors.any?
|
239
|
+
messages = ['`manifest.json` is invalid. Please correct the errors:']
|
240
|
+
errors.each do |error|
|
241
|
+
messages << "* #{error.fetch(:message).sub(/ in schema \S+\z/, '')}"
|
242
|
+
end
|
243
|
+
error messages.join("\n")
|
206
244
|
end
|
207
245
|
|
208
|
-
|
246
|
+
if manifest['transformers']
|
247
|
+
difference = manifest['transformers'].map { |transformer| transformer['file'] } - manifest['files']
|
248
|
+
if difference.any?
|
249
|
+
messages = ['`manifest.json` is invalid. Please correct the errors:']
|
250
|
+
messages << "* Some transformer files are not listed in the top-level files: #{difference.join(', ')}"
|
251
|
+
error messages.join("\n")
|
252
|
+
end
|
253
|
+
end
|
209
254
|
|
210
|
-
handler = ValidationHandler.new
|
255
|
+
handler = Turbot::Handlers::ValidationHandler.new
|
211
256
|
runner = TurbotRunner::Runner.new(working_directory, :record_handler => handler)
|
212
|
-
|
257
|
+
begin
|
258
|
+
rc = runner.run
|
259
|
+
rescue TurbotRunner::InvalidDataType
|
260
|
+
messages = ['`manifest.json` is invalid. Please correct the errors:']
|
261
|
+
messages << %(* The property '#/data_type' value "#{manifest['data_type']}" is not a supported data type.)
|
262
|
+
error messages.join("\n")
|
263
|
+
end
|
213
264
|
|
214
|
-
puts
|
215
265
|
if rc == TurbotRunner::Runner::RC_OK
|
216
266
|
puts "Validated #{handler.count} records!"
|
217
267
|
else
|
@@ -221,216 +271,117 @@ class Turbot::Command::Bots < Turbot::Command::Base
|
|
221
271
|
|
222
272
|
# bots:dump
|
223
273
|
#
|
224
|
-
#
|
274
|
+
#Execute the bot locally and write the bot's output to STDOUT.
|
275
|
+
#
|
276
|
+
# -q, --quiet # only output validation errors
|
277
|
+
#
|
278
|
+
#Example:
|
279
|
+
#
|
280
|
+
# $ turbot bots:dump
|
281
|
+
# {'foo': 'bar'}
|
282
|
+
# {'foo2': 'bar2'}
|
225
283
|
#
|
226
|
-
# $ turbot bots:dump
|
227
|
-
# {'foo': 'bar'}
|
228
|
-
# {'foo2': 'bar2'}
|
229
|
-
|
230
284
|
def dump
|
231
285
|
validate_arguments!
|
286
|
+
error_if_no_local_bot_found
|
232
287
|
|
233
|
-
|
288
|
+
if options[:quiet]
|
289
|
+
handler = Turbot::Handlers::BaseHandler.new
|
290
|
+
else
|
291
|
+
handler = Turbot::Handlers::DumpHandler.new
|
292
|
+
end
|
234
293
|
runner = TurbotRunner::Runner.new(working_directory, :record_handler => handler)
|
235
294
|
rc = runner.run
|
236
295
|
|
237
|
-
puts
|
238
296
|
if rc == TurbotRunner::Runner::RC_OK
|
239
|
-
puts
|
297
|
+
puts 'Bot ran successfully!'
|
240
298
|
else
|
241
|
-
puts
|
299
|
+
puts 'Bot failed!'
|
242
300
|
end
|
243
301
|
end
|
244
302
|
|
245
|
-
# # bots:single
|
246
|
-
# #
|
247
|
-
# # Execute bot in same way as OpenCorporates single-record update
|
248
|
-
# #
|
249
|
-
# # $ turbot bots:single
|
250
|
-
# # Enter argument (as JSON object):
|
251
|
-
# # {"id": "frob123"}
|
252
|
-
# # {"id": "frob123", "stuff": "updated-data-for-this-record"}
|
253
|
-
#
|
254
|
-
# def single
|
255
|
-
# # This will need to be language-aware, eventually
|
256
|
-
# scraper_path = shift_argument || scraper_file(working_directory)
|
257
|
-
# validate_arguments!
|
258
|
-
# print 'Arguments (as JSON object, e.g. {"id":"ABC123"}: '
|
259
|
-
# arg = ask
|
260
|
-
# count = 0
|
261
|
-
# run_scraper_each_line("#{scraper_path} #{bot} #{Shellwords.shellescape(arg)}") do |line|
|
262
|
-
# raise "Your scraper returned more than one value!" if count > 1
|
263
|
-
# puts line
|
264
|
-
# count += 1
|
265
|
-
# end
|
266
|
-
# end
|
267
|
-
|
268
|
-
|
269
303
|
# bots:preview
|
270
304
|
#
|
271
|
-
#
|
305
|
+
#Send bot data to Turbot for remote previewing or sharing.
|
306
|
+
#
|
307
|
+
#Example:
|
308
|
+
#
|
309
|
+
# $ turbot bots:preview
|
310
|
+
# Sending to Turbot...
|
311
|
+
# Submitted 2 records to Turbot.
|
312
|
+
# View your records at http://turbot.opencorporates.com/..
|
272
313
|
#
|
273
|
-
# Sending example to turbot... done
|
274
314
|
def preview
|
275
315
|
validate_arguments!
|
316
|
+
error_if_no_local_bot_found
|
276
317
|
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
if !response.is_a? Turbot::API::SuccessResponse
|
281
|
-
error(response.message)
|
318
|
+
response = api.update_bot(bot, parse_manifest)
|
319
|
+
if response.is_a?(Turbot::API::FailureResponse)
|
320
|
+
error_message(response)
|
282
321
|
end
|
283
322
|
|
284
323
|
response = api.destroy_draft_data(bot)
|
285
|
-
if
|
286
|
-
|
324
|
+
if response.is_a?(Turbot::API::FailureResponse)
|
325
|
+
error_message(response)
|
287
326
|
end
|
288
327
|
|
289
|
-
puts
|
328
|
+
puts 'Sending to Turbot...'
|
290
329
|
|
291
|
-
handler = PreviewHandler.new(bot, api)
|
330
|
+
handler = Turbot::Handlers::PreviewHandler.new(bot, api)
|
292
331
|
runner = TurbotRunner::Runner.new(working_directory, :record_handler => handler)
|
293
332
|
rc = runner.run
|
294
333
|
|
295
|
-
puts
|
296
|
-
|
297
334
|
if rc == TurbotRunner::Runner::RC_OK
|
298
335
|
response = handler.submit_batch
|
299
|
-
if response.is_a?
|
336
|
+
if response.is_a?(Turbot::API::SuccessResponse)
|
300
337
|
if handler.count > 0
|
301
|
-
puts "Submitted #{handler.count} records to
|
302
|
-
puts "View your records at #{response.data[:url]}"
|
338
|
+
puts "Submitted #{handler.count} records to Turbot.\nView your records at #{response.data[:url]}"
|
303
339
|
else
|
304
|
-
puts
|
340
|
+
puts 'No records sent.'
|
305
341
|
end
|
306
342
|
else
|
307
|
-
|
343
|
+
error_message(response)
|
308
344
|
end
|
309
345
|
else
|
310
|
-
puts
|
311
|
-
puts "Bot failed!"
|
346
|
+
puts 'Bot failed!'
|
312
347
|
end
|
313
348
|
end
|
314
349
|
|
315
|
-
|
316
|
-
def spinner(p)
|
317
|
-
parts = "\|/-" * 2
|
318
|
-
print parts[p % parts.length] + "\r"
|
319
|
-
end
|
350
|
+
private
|
320
351
|
|
321
|
-
def
|
322
|
-
|
323
|
-
|
324
|
-
rescue Errno::ENOENT
|
325
|
-
raise "This command must be run from a directory including `manifest.json`"
|
352
|
+
def error_if_no_local_bot_found
|
353
|
+
unless bot
|
354
|
+
error "No bot specified.\nRun this command from a bot directory containing a `manifest.json` file, or specify the bot with --bot BOT."
|
326
355
|
end
|
327
356
|
end
|
328
357
|
|
329
|
-
def
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
def working_directory
|
334
|
-
Dir.pwd
|
358
|
+
def error_if_bot_exists_in_turbot
|
359
|
+
if api.show_bot(bot).is_a?(Turbot::API::SuccessResponse)
|
360
|
+
error "There's already a bot named #{bot} in Turbot. Try another name."
|
361
|
+
end
|
335
362
|
end
|
336
363
|
|
337
|
-
def
|
338
|
-
|
364
|
+
def error_message(response)
|
365
|
+
suffix = response.error_code && ": #{response.error_code}"
|
366
|
+
error "#{response.message} (HTTP #{response.code}#{suffix})"
|
339
367
|
end
|
340
368
|
|
341
|
-
def create_zip_archive(archive_path,
|
369
|
+
def create_zip_archive(archive_path, basenames)
|
342
370
|
Zip.continue_on_exists_proc = true
|
371
|
+
|
343
372
|
Zip::File.open(archive_path, Zip::File::CREATE) do |zipfile|
|
344
|
-
|
345
|
-
|
373
|
+
basenames.each do |basename|
|
374
|
+
filename = File.join(working_directory, basename)
|
346
375
|
|
347
|
-
if File.directory?(
|
348
|
-
Dir["#{
|
349
|
-
|
350
|
-
zipfile.add(
|
376
|
+
if File.directory?(filename)
|
377
|
+
Dir["#{filename}/**/*"].each do |filename1|
|
378
|
+
basename1 = Pathname.new(filename1).relative_path_from(Pathname.new(working_directory))
|
379
|
+
zipfile.add(basename1, filename1)
|
351
380
|
end
|
352
381
|
else
|
353
|
-
zipfile.add(
|
382
|
+
zipfile.add(basename, filename)
|
354
383
|
end
|
355
384
|
end
|
356
385
|
end
|
357
386
|
end
|
358
387
|
end
|
359
|
-
|
360
|
-
class TurbotClientHandler < TurbotRunner::BaseHandler
|
361
|
-
def handle_invalid_record(record, data_type, error_message)
|
362
|
-
puts
|
363
|
-
puts "The following record is invalid:"
|
364
|
-
puts record.to_json
|
365
|
-
puts " * #{error_message}"
|
366
|
-
puts
|
367
|
-
end
|
368
|
-
|
369
|
-
def handle_non_json_output(line)
|
370
|
-
puts
|
371
|
-
puts "The following line was not valid JSON:"
|
372
|
-
puts line
|
373
|
-
end
|
374
|
-
end
|
375
|
-
|
376
|
-
|
377
|
-
class DumpHandler < TurbotClientHandler
|
378
|
-
def handle_valid_record(record, data_type)
|
379
|
-
puts record.to_json
|
380
|
-
end
|
381
|
-
end
|
382
|
-
|
383
|
-
|
384
|
-
class ValidationHandler < TurbotClientHandler
|
385
|
-
attr_reader :count
|
386
|
-
|
387
|
-
def initialize(*)
|
388
|
-
@count = 0
|
389
|
-
super
|
390
|
-
end
|
391
|
-
|
392
|
-
def handle_valid_record(record, data_type)
|
393
|
-
@count += 1
|
394
|
-
STDOUT.write('.')
|
395
|
-
end
|
396
|
-
|
397
|
-
def handle_invalid_record(record, data_type, error_message)
|
398
|
-
puts
|
399
|
-
puts "The following record is invalid:"
|
400
|
-
puts record.to_json
|
401
|
-
puts " * #{error_message}"
|
402
|
-
puts
|
403
|
-
end
|
404
|
-
|
405
|
-
def handle_invalid_json(line)
|
406
|
-
puts
|
407
|
-
puts "The following line was not valid JSON:"
|
408
|
-
puts line
|
409
|
-
end
|
410
|
-
end
|
411
|
-
|
412
|
-
|
413
|
-
class PreviewHandler < TurbotClientHandler
|
414
|
-
attr_reader :count
|
415
|
-
|
416
|
-
def initialize(bot_name, api)
|
417
|
-
@bot_name = bot_name
|
418
|
-
@api = api
|
419
|
-
@batch = []
|
420
|
-
@count = 0
|
421
|
-
super()
|
422
|
-
end
|
423
|
-
|
424
|
-
def handle_valid_record(record, data_type)
|
425
|
-
@count += 1
|
426
|
-
STDOUT.write('.')
|
427
|
-
@batch << record.merge(:data_type => data_type)
|
428
|
-
submit_batch if @count % 20 == 0
|
429
|
-
end
|
430
|
-
|
431
|
-
def submit_batch
|
432
|
-
result = @api.create_draft_data(@bot_name, @batch.to_json)
|
433
|
-
@batch = []
|
434
|
-
result
|
435
|
-
end
|
436
|
-
end
|