turbot 0.0.2

data/lib/turbot/command/auth.rb

@@ -0,0 +1,85 @@
+require "turbot/command/base"
+
+# authentication (login, logout)
+#
+class Turbot::Command::Auth < Turbot::Command::Base
+
+  # auth
+  #
+  # Authenticate, display token and current user
+  def index
+    validate_arguments!
+
+    Turbot::Command::Help.new.send(:help_for_command, current_command)
+  end
+
+  # auth:login
+  #
+  # log in with your turbot credentials
+  #
+  #Example:
+  #
+  # $ turbot auth:login
+  # Enter your Turbot credentials:
+  # Email: email@example.com
+  # Password (typing will be hidden):
+  # Authentication successful.
+  #
+  def login
+    validate_arguments!
+
+    Turbot::Auth.login
+    display "Authentication successful."
+  end
+
+  alias_command "login", "auth:login"
+
+  # auth:logout
+  #
+  # clear local authentication credentials
+  #
+  #Example:
+  #
+  # $ turbot auth:logout
+  # Local credentials cleared.
+  #
+  def logout
+    validate_arguments!
+
+    Turbot::Auth.logout
+    display "Local credentials cleared."
+  end
+
+  alias_command "logout", "auth:logout"
+
+  # auth:token
+  #
+  # display your api token
+  #
+  #Example:
+  #
+  # $ turbot auth:token
+  # ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789ABCD
+  #
+  def token
+    validate_arguments!
+
+    display Turbot::Auth.api_key
+  end
+
+  # auth:whoami
+  #
+  # display your turbot email address
+  #
+  #Example:
+  #
+  # $ turbot auth:whoami
+  # email@example.com
+  #
+  def whoami
+    validate_arguments!
+
+    display Turbot::Auth.user
+  end
+
+end

data/lib/turbot/command/base.rb

@@ -0,0 +1,192 @@
+require "fileutils"
+require "turbot/auth"
+require "turbot/client/rendezvous"
+require "turbot/command"
+
+class Turbot::Command::Base
+  include Turbot::Helpers
+
+  def self.namespace
+    self.to_s.split("::").last.downcase
+  end
+
+  attr_reader :args
+  attr_reader :options
+
+  def initialize(args=[], options={})
+    @args = args
+    @options = options
+  end
+
+  def bot
+    @bot ||= if options[:bot].is_a?(String)
+      options[:bot]
+    elsif ENV.has_key?('TURBOT_BOT')
+      ENV['TURBOT_BOT']
+    elsif bot_from_manifest = extract_bot_from_manifest(Dir.pwd)
+      bot_from_manifest
+    else
+      # raise instead of using error command to enable rescuing when bot is optional
+      raise Turbot::Command::CommandFailed.new("No bot specified.\nRun this command from a bot folder containing a `manifest.json`,  or specify which bot to use with --bot BOT_ID.") unless options[:ignore_no_bot]
+    end
+  end
+
+  def api
+    Turbot::Auth.api
+  end
+
+  def turbot
+    Turbot::Auth.client
+  end
+
+protected
+
+  def self.inherited(klass)
+    unless klass == Turbot::Command::Base
+      help = extract_help_from_caller(caller.first)
+
+      Turbot::Command.register_namespace(
+        :name => klass.namespace,
+        :description => help.first
+      )
+    end
+  end
+
+  def self.method_added(method)
+    return if self == Turbot::Command::Base
+    return if private_method_defined?(method)
+    return if protected_method_defined?(method)
+
+    help = extract_help_from_caller(caller.first)
+    resolved_method = (method.to_s == "index") ? nil : method.to_s
+    command = [ self.namespace, resolved_method ].compact.join(":")
+    banner = extract_banner(help) || command
+
+    Turbot::Command.register_command(
+      :klass       => self,
+      :method      => method,
+      :namespace   => self.namespace,
+      :command     => command,
+      :banner      => banner.strip,
+      :help        => help.join("\n"),
+      :summary     => extract_summary(help),
+      :description => extract_description(help),
+      :options     => extract_options(help)
+    )
+
+    alias_command command.gsub(/_/, '-'), command if command =~ /_/
+  end
+
+  def self.alias_command(new, old)
+    raise "no such command: #{old}" unless Turbot::Command.commands[old]
+    Turbot::Command.command_aliases[new] = old
+  end
+
+  def extract_bot
+    output_with_bang "Command::Base#extract_bot has been deprecated. Please use Command::Base#bot instead.  #{caller.first}"
+    bot
+  end
+
+  #
+  # Parse the caller format and identify the file and line number as identified
+  # in : http://www.ruby-doc.org/core/classes/Kernel.html#M001397.  This will
+  # look for a colon followed by a digit as the delimiter.  The biggest
+  # complication is windows paths, which have a colon after the drive letter.
+  # This regex will match paths as anything from the beginning to a colon
+  # directly followed by a number (the line number).
+  #
+  # Examples of the caller format :
+  # * c:/Ruby192/lib/.../lib/turbot/command/addons.rb:8:in `<module:Command>'
+  # * c:/Ruby192/lib/.../turbot-2.0.1/lib/turbot/command/pg.rb:96:in `<class:Pg>'
+  # * /Users/ph7/...../xray-1.1/lib/xray/thread_dump_signal_handler.rb:9
+  #
+  def self.extract_help_from_caller(line)
+    # pull out of the caller the information for the file path and line number
+    if line =~ /^(.+?):(\d+)/
+      extract_help($1, $2)
+    else
+      raise("unable to extract help from caller: #{line}")
+    end
+  end
+
+  def self.extract_help(file, line_number)
+    buffer = []
+    lines = Turbot::Command.files[file]
+
+    (line_number.to_i-2).downto(0) do |i|
+      line = lines[i]
+      case line[0..0]
+        when ""
+        when "#"
+          buffer.unshift(line[1..-1])
+        else
+          break
+      end
+    end
+
+    buffer
+  end
+
+  def self.extract_banner(help)
+    help.first
+  end
+
+  def self.extract_summary(help)
+    extract_description(help).split("\n")[2].to_s.split("\n").first
+  end
+
+  def self.extract_description(help)
+    help.reject do |line|
+      line =~ /^\s+-(.+)#(.+)/
+    end.join("\n")
+  end
+
+  def self.extract_options(help)
+    help.select do |line|
+      line =~ /^\s+-(.+)#(.+)/
+    end.inject([]) do |options, line|
+      args = line.split('#', 2).first
+      args = args.split(/,\s*/).map {|arg| arg.strip}.sort.reverse
+      name = args.last.split(' ', 2).first[2..-1]
+      options << { :name => name, :args => args }
+    end
+  end
+
+  def current_command
+    Turbot::Command.current_command
+  end
+
+  def extract_option(key)
+    options[key.dup.gsub('-','_').to_sym]
+  end
+
+  def invalid_arguments
+    Turbot::Command.invalid_arguments
+  end
+
+  def shift_argument
+    Turbot::Command.shift_argument
+  end
+
+  def validate_arguments!
+    Turbot::Command.validate_arguments!
+  end
+
+  def extract_bot_from_manifest(dir)
+    begin
+      config = JSON.load(open("#{dir}/manifest.json").read)
+      config && config["bot_id"]
+    rescue Errno::ENOENT
+    end
+  end
+
+  def escape(value)
+    turbot.escape(value)
+  end
+end
+
+module Turbot::Command
+  unless const_defined?(:BaseWithApp)
+    BaseWithApp = Base
+  end
+end

data/lib/turbot/command/bots.rb

@@ -0,0 +1,326 @@
+require "turbot/command/base"
+require 'zip'
+require 'json-schema'
+require 'open3'
+require 'base64'
+require 'shellwords'
+
+# manage bots (create, destroy)
+#
+class Turbot::Command::Bots < Turbot::Command::Base
+
+  # bots
+  #
+  # list your bots
+  #
+  #Example:
+  #
+  # $ turbot bots
+  # === My Bots
+  # example
+  # example2
+  #
+  def index
+    validate_arguments!
+    bots = api.get_bots
+    unless bots.empty?
+      styled_header("Bots")
+      styled_array(bots.map{|k,data| data['name']})
+    else
+      display("You have no bots.")
+    end
+  end
+
+  alias_command "list", "bots"
+
+  # bots:info
+  #
+  # show detailed bot information
+  #
+  # -s, --shell  # output more shell friendly key/value pairs
+  #
+  #Examples:
+  #
+  # $ turbot bots:info
+  # === example
+  # Last run status: OK
+  # Last run ended: 2001/01/01
+  # ...
+  #
+  # $ turbot bots:info --shell
+  # last_run_status: OK
+  # last_run_ended: 2001/01/01
+  # ...
+  #
+  def info
+    validate_arguments!
+    bot_data = api.get_bot(bot)
+    unless options[:shell]
+      styled_header(bot_data["name"])
+    end
+
+    if options[:shell]
+      bot_data.keys.sort_by { |a| a.to_s }.each do |key|
+        hputs("#{key}=#{bot_data[key]}")
+      end
+    else
+      data = {}
+      if bot_data["last_run_status"]
+        data["Last run status"] = bot_data["last_run_status"]
+      end
+      if bot_data["last_run_ended"]
+        data["Last run ended"] = format_date(bot_data["last_run_ended"]) if bot_data["last_run_ended"]
+      end
+      data["Git URL"] = bot_data["git_url"]
+      data["Repo Size"] = format_bytes(bot_data["repo_size"]) if bot_data["repo_size"]
+      styled_hash(data)
+    end
+  end
+
+  alias_command "info", "bots:info"
+
+
+  # bots:generate --bot name_of_bot
+  #
+  # Generate stub code for a bot in specified language
+  #
+  #   -l, --language LANGUAGE # language to generate (currently `ruby` (default) or `python`)
+
+  # $ turbot bots:generate --language=ruby --bot my_amazing_bot
+  # Created new bot template at my_amazing_bot!
+
+  def generate
+    validate_arguments!
+    language = options[:language] || "ruby"
+    puts "Generating #{language} codes..."
+    FileUtils.mkdir(bot)
+    case language
+    when "ruby"
+      scraper = "scraper.rb"
+    when "python"
+      scraper = "scraper.py"
+    end
+    manifest_template = File.expand_path("../../../../templates/manifest.json", __FILE__)
+    scraper_template = File.expand_path("../../../../templates/#{scraper}", __FILE__)
+    manifest = open(manifest_template).read.sub(/{{bot_id}}/, bot)
+    FileUtils.cp(scraper_template, "#{bot}/#{scraper}")
+    open("#{bot}/manifest.json", "w") do |f|
+      f.write(manifest)
+    end
+    puts "Created new bot template at #{bot}!"
+  end
+
+
+  # bots:push
+  #
+  # Push bot code to the turbot server. Must be run from a local bot checkout.
+  #
+  # $ turbot bots:push
+  # Creating example... done
+
+  def push
+    validate_arguments!
+
+    working_dir = Dir.pwd
+    manifest = parsed_manifest(working_dir)
+    #archive_file = File.join(working_dir, 'tmp', "#{manifest['bot_id']}.zip")
+    archive = Tempfile.new(bot)
+    archive_path = "#{archive.path}.zip"
+
+    Zip.continue_on_exists_proc = true
+    Zip::File.open(archive_path, Zip::File::CREATE) do |zipfile|
+      zipfile.add("manifest.json", manifest_path)
+      manifest['files'].each { |f| zipfile.add(f, File.join(working_dir,f)) }
+    end
+
+    File.open(archive_path) do |file|
+      params = {
+        "bot[archive]" => file,
+        "bot[manifest]" => manifest
+      }
+      api.post_bot(params)
+    end
+  end
+
+  alias_command "push", "bots:push"
+
+  # bots:validate
+  #
+  # Validate bot output against its schema
+  #
+  # $ heroku bots:validate
+  # Validating example... done
+
+  def validate
+    scraper_path    = shift_argument || scraper_file(Dir.pwd)
+    validate_arguments!
+    config = parsed_manifest(Dir.pwd)
+    type = config["data_type"]
+
+    schema = get_schema(type)
+
+    if !schema || !File.exists?(schema)
+      error("No schema found for data_type: #{type}")
+    end
+
+    count = 0
+
+    run_scraper_each_line("#{scraper_path} #{bot}") do |line|
+      errors = ""
+      errors = JSON::Validator.fully_validate(
+        schema,
+        line,
+        {:errors_as_objects => true})
+
+      if !errors.empty?
+        error("LINE WITH ERROR: #{line}\n\nERRORS: #{errors}")
+      end
+      count += 1
+    end
+    puts "Validated #{count} records successfully!"
+  end
+
+  # bots:dump
+  #
+  # Execute bot locally (writes to STDOUT)
+  #
+  # $ heroku bots:dump
+  # {'foo': 'bar'}
+  # {'foo2': 'bar2'}
+
+  def dump
+    # This will need to be language-aware, eventually
+    scraper_path    = shift_argument || scraper_file(Dir.pwd)
+    validate_arguments!
+    count = 0
+    run_scraper_each_line("#{scraper_path} #{bot}") do |line|
+      puts line
+      count += 1
+    end
+  end
+
+  # bots:single
+  #
+  # Execute bot in same way as OpenCorporates single-record update
+  #
+  # $ heroku bots:single
+  # Enter argument (as JSON object):
+  # {"id": "frob123"}
+  # {"id": "frob123", "stuff": "updated-data-for-this-record"}
+
+  def single
+    # This will need to be language-aware, eventually
+    scraper_path    = shift_argument || scraper_file(Dir.pwd)
+    validate_arguments!
+    print 'Arguments (as JSON object, e.g. {"id":"ABC123"}: '
+    arg = ask
+    count = 0
+    run_scraper_each_line("#{scraper_path} #{bot} #{Shellwords.shellescape(arg)}") do |line|
+      raise "Your scraper returned more than one value!" if count > 1
+      puts line
+      count += 1
+    end
+  end
+
+
+  # bots:preview
+  #
+  # Send bot data to Angler for remote previewing / sharing
+  #
+  # Sending example to Angler... done
+  def preview
+    scraper_path    = shift_argument || scraper_file(Dir.pwd)
+    validate_arguments!
+
+    bots = api.get_bots
+    raise "You have not pushed your bot" unless bots.include?(bot)
+
+    batch = []
+    count = 0
+    puts "Sending to angler... "
+    result = ""
+    run_scraper_each_line("#{scraper_path} #{bot}") do |line|
+      batch << JSON.parse(line)
+      spinner(count)
+      if count % 20 == 0
+        result = api.send_drafts_to_angler(bot, batch.to_json)
+        batch = []
+      end
+      count += 1
+    end
+    if !batch.empty?
+      result = api.send_drafts_to_angler(bot, batch.to_json)
+    end
+    puts "Sent #{count} records."
+    puts "View your records at #{JSON.parse(result)['url']}"
+  end
+
+  private
+
+  def spinner(p)
+    parts = "\|/-" * 2
+    print parts[p % parts.length] + "\r"
+  end
+
+  def run_scraper_each_line(scraper_path, options={})
+    case scraper_path
+    when /scraper.rb /
+      interpreter = "ruby"
+    when /scraper.py /
+      interpreter = "python"
+    else
+      raise "Unsupported file extension at #{scraper_path}"
+    end
+
+    command = "#{interpreter} #{scraper_path}"
+    Open3::popen3(command, options) do |_, stdout, stderr, wait_thread|
+      loop do
+        check_output_with_timeout(stdout)
+
+        begin
+          result = stdout.readline.strip
+          yield result unless result.empty?
+          # add run id and bot name
+        rescue EOFError
+          break
+        end
+      end
+      status = wait_thread.value.exitstatus
+      if status > 0
+        message = "Bot <#{command}> exited with status #{status}: #{stderr.read}"
+        raise RuntimeError.new(message)
+      end
+    end
+  end
+
+  def check_output_with_timeout(stdout, initial_interval = 10, timeout = 21600)
+    interval = initial_interval
+    loop do
+      reads, _, _ = IO.select([stdout], [], [], interval)
+      break if !reads.nil?
+      raise "Timeout! - could not read from external bot after #{timeout} seconds" if reads.nil? && interval > timeout
+      interval *= 2
+    end
+  end
+
+  def parsed_manifest(dir)
+    begin
+      JSON.parse(open(manifest_path).read)
+    rescue Errno::ENOENT
+      raise "This command must be run from a directory including `manifest.json`"
+    end
+  end
+
+  def scraper_file(dir)
+    Dir.glob("scraper*").first
+  end
+
+  def manifest_path
+    File.join(Dir.pwd, 'manifest.json')
+  end
+
+  def get_schema(type)
+    hyphenated_name = type.to_s.gsub("_", "-").gsub(" ", "-")
+    schema = File.expand_path("../../../../schema/schemas/#{hyphenated_name}-schema.json", __FILE__)
+  end
+end

data/lib/turbot/command/config.rb

@@ -0,0 +1,123 @@
+require "turbot/command/base"
+
+# manage bot config vars
+#
+class Turbot::Command::Config < Turbot::Command::Base
+
+  # config
+  #
+  # display the config vars for an bot
+  #
+  # -s, --shell  # output config vars in shell format
+  #
+  #Examples:
+  #
+  # $ turbot config
+  # A: one
+  # B: two
+  #
+  # $ turbot config --shell
+  # A=one
+  # B=two
+  #
+  def index
+    validate_arguments!
+
+    vars = api.get_config_vars(bot)
+    if vars.empty?
+      display("#{bot} has no config vars.")
+    else
+      vars.each {|key, value| vars[key] = value.to_s.strip}
+      if options[:shell]
+        vars.keys.sort.each do |key|
+          display(%{#{key}=#{vars[key]}})
+        end
+      else
+        styled_header("#{bot} Config Vars")
+        styled_hash(vars)
+      end
+    end
+  end
+
+  # config:set KEY1=VALUE1 [KEY2=VALUE2 ...]
+  #
+  # set one or more config vars
+  #
+  #Example:
+  #
+  # $ turbot config:set A=one
+  # Setting config vars and restarting example... done, v123
+  # A: one
+  #
+  # $ turbot config:set A=one B=two
+  # Setting config vars and restarting example... done, v123
+  # A: one
+  # B: two
+  #
+  def set
+    unless args.size > 0 and args.all? { |a| a.include?('=') }
+      error("Usage: turbot config:set KEY1=VALUE1 [KEY2=VALUE2 ...]\nMust specify KEY and VALUE to set.")
+    end
+
+    vars = args.inject({}) do |vars, arg|
+      key, value = arg.split('=', 2)
+      vars[key] = value
+      vars
+    end
+
+    action("Setting config vars and restarting #{bot}") do
+      api.put_config_vars(bot, vars)
+    end
+
+    vars.each {|key, value| vars[key] = value.to_s}
+    styled_hash(vars)
+  end
+
+  alias_command "config:add", "config:set"
+
+  # config:get KEY
+  #
+  # display a config value for an bot
+  #
+  #Examples:
+  #
+  # $ turbot config:get A
+  # one
+  #
+  def get
+    unless key = shift_argument
+      error("Usage: turbot config:get KEY\nMust specify KEY.")
+    end
+    validate_arguments!
+
+    vars = api.get_config_vars(bot)
+    key, value = vars.detect {|k,v| k == key}
+    display(value.to_s)
+  end
+
+  # config:unset KEY1 [KEY2 ...]
+  #
+  # unset one or more config vars
+  #
+  # $ turbot config:unset A
+  # Unsetting A and restarting example... done, v123
+  #
+  # $ turbot config:unset A B
+  # Unsetting A and restarting example... done, v123
+  # Unsetting B and restarting example... done, v124
+  #
+  def unset
+    if args.empty?
+      error("Usage: turbot config:unset KEY1 [KEY2 ...]\nMust specify KEY to unset.")
+    end
+
+    args.each do |key|
+      action("Unsetting #{key} and restarting #{bot}") do
+        api.delete_config_var(bot, key)
+      end
+    end
+  end
+
+  alias_command "config:remove", "config:unset"
+
+end