turbot 0.0.51 → 0.1.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MTE2NmZhMWExMDIyYjg4MWYzYmFlZGE2YzgzODcxOGIyZTg4ODhlMA==
+    NmExMjA4OTczZWFmMzMxYTQxMDg1YjU2OGVmMGI3NWYwNDQ4NDhkOQ==
   data.tar.gz: !binary |-
-    N2FmNDJiODE2ODkxMWFhMmZiMWYwYWVmMTY3MDFhOWUxZGIyNzhjYw==
+    MGJhYzJlMDkwNjI0M2Q5M2ZmZWIwYjE0NmJiYmU0YWYxNjRmZGQ3NA==
 SHA512:
   metadata.gz: !binary |-
-    NDc3MTNmZmRkNTc5MWI3YzM3M2FmZDM2NmU3OWQ3Yzg4ZmY5ZDRhMTNhMDMx
-    ZDI4ZWFhNmU2ODhiMmY3NTJiYzYzZmYzZGEzNTFmNmQxNDZiYTYxNGQ2ZWY3
-    OGY0MTMwMGU3MDFjZTc3YTMzYmYxYmZlOWFiOTM0Y2ExYWJhN2Y=
+    NTY1YWM4ZDhjM2QxN2YyMjNmMjQxYTc0YWFkOTcyNTA5MWVkM2M4MWYwM2I2
+    NDA2ZDhkNTRhZmVkYzZjZjlmMTJmNjRlZDMxNDdhOTBjOTQ5NWM4YzdlNDJl
+    YjU3MTA4YWRmMTBiZWVhMzg0YzQ1NmRiMGRkYzAxMzgzOWJlNWM=
   data.tar.gz: !binary |-
-    OTczY2UwZGYwYTZhYTAzMGVhOTllMTUzNjRkNDc5NmQxYWZjYThlNDU0NjE4
-    Mzg2YmIzYTMyM2ViM2JjOGZlMzM5MGIwNDE3NWU2ZGI3NGQwNTdkNjQxZGFm
-    OTI2M2YyZTk5YWRkMzNkZDU1YmM1YWY5OTFmMDA0ODY2M2FjYzU=
+    NGJmYzIzYzgwMDA3Y2M4Y2NmMTM3ZWQ4NGYyNDg0ZTJmOTIwY2FiYmM4YjNk
+    NTZiYmVlNTIyZWQxMzFmMzNjMmFiYjFkZTRmNzEzZGU4ZmNlNGJjNTdiNjYz
+    M2ZlMzVkZmJkYTdmYTllN2Q4MTQxNDI2M2RkMzA4OTBkMTlmZTg=

data/lib/turbot/command/bots.rb

@@ -199,7 +199,7 @@ class Turbot::Command::Bots < Turbot::Command::Base
     validate_arguments!
     config = parsed_manifest(Dir.pwd)
 
-    %w(bot_id data_type identifying_fields files).each do |key|
+    %w(bot_id data_type identifying_fields files publisher).each do |key|
       error("Manifest is missing #{key}") unless config.has_key?(key)
     end
 
@@ -211,8 +211,16 @@ class Turbot::Command::Bots < Turbot::Command::Base
       error("No schema found for data_type: #{type}")
     end
 
-    runner = ValidationRunner.new(Dir.pwd)
-    runner.run
+    handler = ValidationHandler.new
+    runner = TurbotRunner::Runner.new(Dir.pwd, :record_handler => handler)
+    rc = runner.run
+
+    puts
+    if rc
+      puts "Validated #{handler.count} records!"
+    else
+      puts "Validated #{handler.count} records before bot failed!"
+    end
   end
 
   # bots:dump
@@ -225,8 +233,17 @@ class Turbot::Command::Bots < Turbot::Command::Base
 
   def dump
     validate_arguments!
-    runner = DumpRunner.new(Dir.pwd)
-    runner.run
+
+    handler = DumpHandler.new
+    runner = TurbotRunner::Runner.new(Dir.pwd, :record_handler => handler)
+    rc = runner.run
+
+    puts
+    if rc
+      puts "Bot ran successfully!"
+    else
+      puts "Bot failed!"
+    end
   end
 
 #  # bots:single
@@ -266,12 +283,24 @@ class Turbot::Command::Bots < Turbot::Command::Base
     api.destroy_draft_data(bot)
     puts "Sending to turbot... "
 
-    runner = PreviewRunner.new(bot, api)
-    Signal.trap("INT") do
-      Process.kill("ABRT", runner.wait_thread[:pid])
-      runner.interrupt
+    handler = PreviewHandler.new(bot, api)
+    runner = TurbotRunner::Runner.new(Dir.pwd, :record_handler => handler)
+    rc = runner.run
+
+    puts
+
+    if rc
+      response = handler.submit_batch
+      if handler.count > 0
+        puts "Submitted #{handler.count} records to turbot"
+        puts "View your records at #{response.data[:url]}"
+      else
+        puts "No records sent"
+      end
+    else
+      puts
+      puts "Bot failed!"
     end
-    runner.run
   end
 
   private
@@ -320,28 +349,10 @@ class Turbot::Command::Bots < Turbot::Command::Base
   end
 end
 
-class PreviewRunner < TurbotRunner::BaseRunner
-  def initialize(bot_name, api)
-    @bot_name = bot_name
-    @api = api
-    @batch = []
-    @count = 0
-    super(Dir.pwd)
-  end
-
-  def handle_valid_record(record, data_type)
-    #spinner(@count)
-    @count += 1
-    @batch << record.merge(:data_type => data_type)
-
-    if @count % 20 == 0
-      result = submit_batch
-    end
-  end
-
+class TurbotClientHandler < TurbotRunner::BaseHandler
   def handle_invalid_record(record, data_type, errors)
     puts
-    puts "The following record was not sent to turbot because it didn't validate against the schema:"
+    puts "The following record is invalid:"
     puts record.to_json
     errors.each {|error| puts " * #{error}"}
     puts
@@ -351,72 +362,20 @@ class PreviewRunner < TurbotRunner::BaseRunner
     puts
     puts "The following line was not valid JSON:"
     puts line
-    interrupt
-  end
-
-  def handle_interrupted_run
-    puts
-    result = submit_batch
-    puts "Run interrupted!"
-    puts "Sent #{@count} records."
-    puts "View your records at #{result.data[:url]}"
-  end
-
-  def handle_successful_run
-    result = submit_batch
-    if result.is_a? Turbot::API::FailureResponse
-      @status = :failed
-      handle_failed_run(result.message)
-    else
-      puts "Run completed!"
-      if @count > 0
-        puts "Sent #{@count} records"
-        puts "View your records at #{result.data[:url]}"
-      else
-        puts "No records sent"
-      end
-    end
-  end
-
-  def handle_failed_run(message=nil)
-    STDERR.puts("Error:")
-    STDERR.puts(message) if message
-  end
-
-  private
-  def submit_batch
-    STDOUT.write('.')
-    result = @api.create_draft_data(@bot_name, @batch.to_json)
-    @batch = []
-    result
   end
 end
 
-class DumpRunner < TurbotRunner::BaseRunner
+
+class DumpHandler < TurbotClientHandler
   def handle_valid_record(record, data_type)
     puts record.to_json
   end
+end
 
-  def handle_invalid_record(record, data_type, errors)
-    puts
-    puts "The following record is invalid:"
-    puts record.to_json
-    errors.each {|error| puts " * #{error}"}
-    puts
-  end
 
-  def handle_non_json_output(line)
-    puts "The following line was not valid JSON:"
-    puts line
-    interrupt
-  end
+class ValidationHandler < TurbotClientHandler
+  attr_reader :count
 
-  def handle_failed_run
-    puts "Bot did not run to completion:"
-  end
-end
-
-class ValidationRunner < TurbotRunner::BaseRunner
   def initialize(*)
     @count = 0
     super
@@ -424,6 +383,7 @@ class ValidationRunner < TurbotRunner::BaseRunner
 
   def handle_valid_record(record, data_type)
     @count += 1
+    STDOUT.write('.')
   end
 
   def handle_invalid_record(record, data_type, errors)
@@ -432,24 +392,37 @@ class ValidationRunner < TurbotRunner::BaseRunner
     puts record.to_json
     errors.each {|error| puts " * #{error}"}
     puts
-    interrupt
   end
 
-  def handle_non_json_output(line)
+  def handle_invalid_json(line)
+    puts
     puts "The following line was not valid JSON:"
     puts line
-    interrupt
   end
+end
 
-  def handle_failed_run
-    puts "Bot did not run to completion:"
+
+class PreviewHandler < TurbotClientHandler
+  attr_reader :count
+
+  def initialize(bot_name, api)
+    @bot_name = bot_name
+    @api = api
+    @batch = []
+    @count = 0
+    super()
   end
 
-  def handle_interrupted_run
-    puts "Validated #{@count} records before finding invalid record"
+  def handle_valid_record(record, data_type)
+    @count += 1
+    STDOUT.write('.')
+    @batch << record.merge(:data_type => data_type)
+    submit_batch if @count % 20 == 0
   end
 
-  def handle_successful_run
-    puts "Validated #{@count} records"
+  def submit_batch
+    result = @api.create_draft_data(@bot_name, @batch.to_json)
+    @batch = []
+    result
   end
 end

data/lib/turbot/version.rb

@@ -1,3 +1,3 @@
 module Turbot
-  VERSION = "0.0.51"
+  VERSION = "0.1.0"
 end

data/templates/manifest.json

@@ -6,5 +6,11 @@
  "data_type": "primary data",
  "identifying_fields": ["number"],
  "files": ["{{scraper_name}}"],
- "frequency": "monthly"
+ "frequency": "monthly",
+ "publisher": {
+    "name": "Publisher of the data",
+    "url": "Publisher's website",
+    "terms": "Copyright terms (e.g. Open Government License, n/a, etc)",
+    "terms_url": "A place where these terms can be checked or verified"
+ }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: turbot
 version: !ruby/object:Gem::Version
-  version: 0.0.51
+  version: 0.1.0
 platform: ruby
 authors:
 - Turbot
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-16 00:00:00.000000000 Z
+date: 2014-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: netrc
@@ -114,14 +114,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.0.19
+        version: 0.1.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.0.19
+        version: 0.1.4
 - !ruby/object:Gem::Dependency
   name: excon
   requirement: !ruby/object:Gem::Requirement