turbot 0.0.25 → 0.0.27

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ZWYyNTZiNTliZWE4MjgyMWFjNDFjZDMxN2M5ZmQwNDFjYjRjYmE4NA==
+  data.tar.gz: !binary |-
+    OTFkZTQzZWRiZmM5NTI0OWVhYjlmMTQwMThlODk3OTY5MGQyNTRkZA==
+SHA512:
+  metadata.gz: !binary |-
+    YjY4NWE3ZWU1ZjZjYzIwZmEzNDI2MzIzZjQwOGFlZjVmN2E3NzgwZWNmZTE1
+    ZTI0ODEyZmMwZTY5ZTI0N2E5N2Q4M2I0MWUyODJkNGExNjllYzMyNDlhNDAz
+    Yjg4ZmJkN2M3NjM1NjlhMjA3ZDZhNDI4MzQ3NTU5ZjQ5Yjk3MjM=
+  data.tar.gz: !binary |-
+    Njk0ZjEzZGExOTY3ZGYxY2Y4ZTJkMDkyZDQ0Y2IzMWEwNTMxMDViYjA0NjFj
+    OWU1NDdlZTc5MzQ0YTFjMTUwY2VlYWU4MGM0MjMyMWNkYjRjYWQ2ZDg5OTVm
+    ZDg0MDA1Y2ZmZDNiZDcyZmFhYTI4NTMyODkxMzU5MTAzNTg5ZTM=

data/README.md

@@ -28,6 +28,10 @@ If you're working on the CLI and you can smoke-test your changes:
 
     $ bundle exec turbot
 
+If you need to do this from a different folder, try:
+
+    alias tb="RBENV_GEMSETS=.gems TURBOT_HOST=http://localhost:3000 RBENV_VERSION=[version] BUNDLE_GEMFILE=/path/to/turbot-client/Gemfile bundle exec turbot"
+
 Meta
 ----
 

data/lib/turbot/auth.rb

@@ -206,7 +206,7 @@ class Turbot::Auth
         delete_credentials
         raise e
       end
-      check_for_associated_ssh_key unless Turbot::Command.current_command == "keys:add"
+      # check_for_associated_ssh_key unless Turbot::Command.current_command == "keys:add"
       @credentials
     end
 

data/lib/turbot/command/bots.rb

@@ -6,6 +6,7 @@ require 'json-schema'
 require 'open3'
 require 'base64'
 require 'shellwords'
+require 'turbot_runner'
 
 # manage bots (create, submit data and code)
 #
@@ -92,14 +93,17 @@ class Turbot::Command::Bots < Turbot::Command::Base
   # Created new bot template at my_amazing_bot!
 
   def generate
+    validate_arguments!
     response = api.show_bot(bot)
     if response.is_a? Turbot::API::SuccessResponse
-      error("There's already a bot called #{bot}")
+      error("There's already a bot called #{bot} registered with Turbot. Bot names must be unique.")
     end
-    validate_arguments!
+
     language = options[:language] || "ruby"
-    manifest_template = File.expand_path("../../../../templates/manifest.json", __FILE__)
     scraper_template = File.expand_path("../../../../templates/#{language}", __FILE__)
+    error("unsupported language #{language}") if !File.exists?(scraper_template)
+
+    manifest_template = File.expand_path("../../../../templates/manifest.json", __FILE__)
     license_template = File.expand_path("../../../../templates/LICENSE.txt", __FILE__)
     manifest = open(manifest_template).read.sub("{{bot_id}}", bot)
     scraper_name = case language
@@ -112,6 +116,10 @@ class Turbot::Command::Bots < Turbot::Command::Base
     manifest = manifest.sub("{{scraper_name}}", scraper_name)
 
     # Language-specific stuff
+    # Language-specific stuff:
+    if File.exists? bot
+      error("There's already a folder called #{bot}; move it out the way or try a different name")
+    end
     FileUtils.mkdir(bot)
     FileUtils.cp_r(Dir["#{scraper_template}/*"], bot)
 
@@ -177,10 +185,10 @@ class Turbot::Command::Bots < Turbot::Command::Base
   #
   # Validate bot output against its schema
   #
-  # $ heroku bots:validate
+  # $ turbot bots:validate
   # Validating example... done
 
-  def validate(opts={})
+  def validate
     scraper_path    = shift_argument || scraper_file(Dir.pwd)
     validate_arguments!
     config = parsed_manifest(Dir.pwd)
@@ -197,45 +205,29 @@ class Turbot::Command::Bots < Turbot::Command::Base
       error("No schema found for data_type: #{type}")
     end
 
-    count = 0
-
-    run_scraper_each_line("#{scraper_path} #{bot}") do |line|
-      errors = JSON::Validator.fully_validate(
-        schema,
-        line,
-        {:errors_as_objects => true})
-
-      if !errors.empty?
-        error("LINE WITH ERROR: #{line}\n\nERRORS: #{errors}")
-      end
-      puts line if opts[:dump]
-      if JSON.parse(line).slice(*config['identifying_fields']).blank?
-        error = "Couldn't find any of #{config['identifying_fields']}"
-        error("LINE WITH ERROR: #{line}\n\nERRORS: #{error}")
-      end
-
-      count += 1
-    end
-    puts "Validated #{count} records successfully!" if !opts[:dump]
+    runner = ValidationRunner.new(Dir.pwd)
+    runner.run
   end
 
   # bots:dump
   #
   # Execute bot locally (writes to STDOUT)
   #
-  # $ heroku bots:dump
+  # $ turbot bots:dump
   # {'foo': 'bar'}
   # {'foo2': 'bar2'}
 
   def dump
-    validate(:dump => true)
+    validate_arguments!
+    runner = DumpRunner.new(Dir.pwd)
+    runner.run
   end
 
 #  # bots:single
 #  #
 #  # Execute bot in same way as OpenCorporates single-record update
 #  #
-#  # $ heroku bots:single
+#  # $ turbot bots:single
 #  # Enter argument (as JSON object):
 #  # {"id": "frob123"}
 #  # {"id": "frob123", "stuff": "updated-data-for-this-record"}
@@ -261,98 +253,23 @@ class Turbot::Command::Bots < Turbot::Command::Base
   #
   # Sending example to turbot... done
   def preview
-    scraper_path    = shift_argument || scraper_file(Dir.pwd)
     validate_arguments!
 
-    batch = []
-    count = 0
     config = parsed_manifest(Dir.pwd)
-    puts "Sending to turbot... "
-
+    api.update_bot(bot, parsed_manifest(Dir.pwd))
     api.destroy_draft_data(bot)
+    puts "Sending to turbot... "
 
-    type = config["data_type"]
-    schema = get_schema(type)
-
-    result = ""
-    run_scraper_each_line("#{scraper_path} #{bot}") do |line|
-      spinner(count)
-
-      errors = JSON::Validator.fully_validate(
-        schema,
-        line,
-        {:errors_as_objects => true})
-
-      if errors.empty?
-        batch << JSON.parse(line)
-        if count % 20 == 0
-          result = api.create_draft_data(bot, config, batch.to_json)
-          batch = []
-        end
-        count += 1
-      else
-        puts "The following record was not sent to turbot because it didn't validate against the schema:"
-        puts line
-        puts "The validation error was:"
-        puts errors
-      end
-
-    end
-    if !batch.empty?
-      result = api.create_draft_data(bot, config, batch.to_json)
-    end
-    puts "Sent #{count} records."
-    puts "View your records at #{result.data[:url]}"
+    runner = PreviewRunner.new(bot, api)
+    runner.run
   end
 
   private
-
   def spinner(p)
     parts = "\|/-" * 2
     print parts[p % parts.length] + "\r"
   end
 
-  def run_scraper_each_line(scraper_path, options={})
-    case scraper_path
-    when /scraper.rb /
-      interpreter = "ruby"
-    when /scraper.py /
-      interpreter = "python"
-    else
-      raise "Unsupported file extension at #{scraper_path}"
-    end
-
-    command = "#{interpreter} #{scraper_path}"
-    Open3::popen3(command, options) do |_, stdout, stderr, wait_thread|
-      loop do
-        check_output_with_timeout(stdout)
-
-        begin
-          result = stdout.readline.strip
-          yield result unless result.empty?
-          # add run id and bot name
-        rescue EOFError
-          break
-        end
-      end
-      status = wait_thread.value.exitstatus
-      if status > 0
-        message = "Bot <#{command}> exited with status #{status}: #{stderr.read}"
-        raise RuntimeError.new(message)
-      end
-    end
-  end
-
-  def check_output_with_timeout(stdout, initial_interval = 10, timeout = 21600)
-    interval = initial_interval
-    loop do
-      reads, _, _ = IO.select([stdout], [], [], interval)
-      break if !reads.nil?
-      raise "Timeout! - could not read from external bot after #{timeout} seconds" if reads.nil? && interval > timeout
-      interval *= 2
-    end
-  end
-
   def parsed_manifest(dir)
     begin
       JSON.parse(open(manifest_path).read)
@@ -374,3 +291,96 @@ class Turbot::Command::Bots < Turbot::Command::Base
     File.expand_path("../../../../schema/schemas/#{hyphenated_name}-schema.json", __FILE__)
   end
 end
+
+class PreviewRunner < TurbotRunner::BaseRunner
+  def initialize(bot_name, api)
+    @bot_name = bot_name
+    @api = api
+    @batch = []
+    @count = 0
+    super(Dir.pwd)
+  end
+
+  def handle_valid_record(record, data_type)
+    #spinner(@count)
+    @count += 1
+    @batch << record
+
+    if @count % 20 == 0
+      result = submit_batch
+    end
+  end
+
+  def handle_invalid_record(record, data_type, errors)
+    puts
+    puts "The following record was not sent to turbot because it didn't validate against the schema:"
+    puts record.to_json
+    errors.each {|error| puts " * #{error}"}
+    puts
+  end
+
+  def handle_successful_run
+    result = submit_batch
+    puts "Sent #{@count} records."
+    puts "View your records at #{result.data[:url]}"
+  end
+
+  private
+  def submit_batch
+    result = @api.create_draft_data(@bot_name, @batch.to_json)
+    @batch = []
+    result
+  end
+end
+
+class DumpRunner < TurbotRunner::BaseRunner
+  def handle_valid_record(record, data_type)
+    puts record.to_json
+  end
+
+  def handle_invalid_record(record, data_type, errors)
+    puts
+    puts "The following record is invalid:"
+    puts record.to_json
+    errors.each {|error| puts " * #{error}"}
+    puts
+  end
+
+  def handle_failed_run(output)
+    puts "Bot did not run to completion:"
+    puts output
+  end
+end
+
+class ValidationRunner < TurbotRunner::BaseRunner
+  def initialize(*)
+    @count = 0
+    super
+  end
+
+  def handle_valid_record(record, data_type)
+    @count += 1
+  end
+
+  def handle_invalid_record(record, data_type, errors)
+    puts
+    puts "The following record is invalid:"
+    puts record.to_json
+    errors.each {|error| puts " * #{error}"}
+    puts
+    interrupt
+  end
+
+  def handle_failed_run(output)
+    puts "Bot did not run to completion:"
+    puts output
+  end
+
+  def handle_interrupted_run
+    puts "Validated #{@count} records before finding invalid record"
+  end
+
+  def handle_successful_run
+    puts "Validated #{@count} records"
+  end
+end

data/lib/turbot/version.rb

@@ -1,3 +1,3 @@
 module Turbot
-  VERSION = "0.0.25"
+  VERSION = "0.0.27"
 end

data/schema/schemas/simple-licence-schema.json

@@ -0,0 +1,50 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "Simple Licence Schema",
+  "type": "object",
+  "properties": {
+    "source_url": {
+      "description": "Place where this fact can be verified",
+      "type": "string"
+    },
+    "sample_date": {
+      "description": "Date on which this fact was known to be true",
+      "type": "string"
+    },
+    "confidence": {
+      "description": "Confidence in accuracy of data",
+      "type": "string",
+      "enum": ["HIGH", "MEDIUM", "LOW"]
+    },
+    "company_name": {
+      "description": "Name of the company holding the licence",
+      "type": "string"
+    },
+    "company_jurisdiction": {
+      "description": "Jurisdiction where company is based",
+      "type": "string"
+    },
+    "licence_number": {
+      "description": "Licence number or code",
+      "type": "string"
+    },
+    "jurisdiction_classification": {
+      "description": "Description of how regulator classifies licence",
+      "type": "string"
+    },
+    "regulator": {
+      "description": "The regulating body that issued the licence",
+      "type": "string"
+    },
+    "licence_jurisdiction": {
+      "description": "Jurisdiction where licence is issued",
+      "type": "string"
+    },
+    "category": {
+      "description": "Category of licence",
+      "type": "string",
+      "enum": ["Financial"]
+    }
+  },
+  "required": ["source_url", "sample_date", "company_name", "company_jurisdiction"]
+}

data/templates/manifest.json

@@ -1,5 +1,6 @@
 {
  "bot_id": "{{bot_id}}",
+ "title": "My simple bot",
  "description": "This is a simple bot",
  "data_type": "primary data",
  "identifying_fields": ["number"],

data/templates/python/scraper.py

@@ -6,4 +6,5 @@ for n in range(0,20):
             "message": "Hello %s" % n,
             "sample_date": datetime.datetime.now().isoformat(),
             "source_url": "http://somewhere.com/%s" % n}
+    # The Turbot specification simply requires us to output lines of JSON
     print json.dumps(data)

data/templates/ruby/scraper.rb

@@ -7,5 +7,6 @@ require 'json'
     sample_date: Time.now,
     source_url: "http://somewhere.com/#{n}"
   }
+  # The Turbot specification simply requires us to output lines of JSON
   puts JSON.dump(data)
 end

metadata

@@ -1,20 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: turbot
 version: !ruby/object:Gem::Version
-  version: 0.0.25
-  prerelease: 
+  version: 0.0.27
 platform: ruby
 authors:
 - Turbot
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-16 00:00:00.000000000 Z
+date: 2014-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: netrc
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -30,7 +27,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -38,7 +34,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -46,7 +41,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: launchy
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -54,7 +48,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -62,7 +55,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -70,7 +62,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -78,7 +69,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: json-schema
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -86,7 +76,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -94,7 +83,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - '='
       - !ruby/object:Gem::Version
@@ -102,7 +90,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - '='
       - !ruby/object:Gem::Version
@@ -110,23 +97,34 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: turbot-api
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.0.8
+        version: 0.0.11
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.0.8
+        version: 0.0.11
+- !ruby/object:Gem::Dependency
+  name: turbot-runner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.2
 - !ruby/object:Gem::Dependency
   name: excon
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -134,7 +132,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -178,6 +175,23 @@ files:
 - lib/turbot/updater.rb
 - lib/turbot/version.rb
 - lib/vendor/turbot/okjson.rb
+- schema/schemas/company-schema.json
+- schema/schemas/includes/address.json
+- schema/schemas/includes/base-statement.json
+- schema/schemas/includes/company.json
+- schema/schemas/includes/filing.json
+- schema/schemas/includes/license-data.json
+- schema/schemas/includes/officer.json
+- schema/schemas/includes/previous_name.json
+- schema/schemas/includes/share-parcel-data.json
+- schema/schemas/includes/share-parcel.json
+- schema/schemas/includes/subsidiary-relationship-data.json
+- schema/schemas/includes/total-shares.json
+- schema/schemas/licence-schema.json
+- schema/schemas/primary-data-schema.json
+- schema/schemas/share-parcel-schema.json
+- schema/schemas/simple-licence-schema.json
+- schema/schemas/subsidiary-relationship-schema.json
 - spec/helper/legacy_help.rb
 - spec/helper/pg_dump_restore_spec.rb
 - spec/schemas/dummy_schema.json
@@ -211,45 +225,29 @@ files:
 - templates/manifest.json
 - templates/python/scraper.py
 - templates/ruby/scraper.rb
-- schema/schemas/company-schema.json
-- schema/schemas/includes/address.json
-- schema/schemas/includes/base-statement.json
-- schema/schemas/includes/company.json
-- schema/schemas/includes/filing.json
-- schema/schemas/includes/license-data.json
-- schema/schemas/includes/officer.json
-- schema/schemas/includes/previous_name.json
-- schema/schemas/includes/share-parcel-data.json
-- schema/schemas/includes/share-parcel.json
-- schema/schemas/includes/subsidiary-relationship-data.json
-- schema/schemas/includes/total-shares.json
-- schema/schemas/licence-schema.json
-- schema/schemas/primary-data-schema.json
-- schema/schemas/share-parcel-schema.json
-- schema/schemas/subsidiary-relationship-schema.json
 homepage: http://turbot.com/
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: 1.9.2
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23.2
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Client library and CLI to deploy apps on Turbot.
 test_files: []
+has_rdoc: