turbot-runner-morph 0.0.1

data/schema/schemas/subsidiary-relationship-schema.json

@@ -0,0 +1,46 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "A relationship of control between two companies",
+  "type": "object",
+  "properties": {
+    "sample_date": {
+      "type": "string",
+      "format": "date"
+    },
+    "start_date": {
+      "type": "string",
+      "format": "date"
+    },
+    "start_date_type": {
+      "type": "string"
+    },
+    "end_date": {
+      "type": "string",
+      "format": "date"
+    },
+    "end_date_type": {
+      "type": "string"
+    },
+    "source_jurisdiction": {
+      "description": "Jurisdiction of the source of the data",
+      "type": "string"
+    },
+    "company": {
+      "$ref": "includes/company.json"
+    },
+    "data": {
+      "type": "array",
+      "description": "This is an array of data objects, that is the objects which actually contain the data about the datum. Generally there will only be one element in this, although in for some types of data_types there may naturally be several releated to the same company, e.g. shareholders",
+      "items": {
+        "$ref": "includes/subsidiary-relationship-data.json"
+      },
+      "additionalItems": false
+    }
+  },
+  "additionalProperties": false,
+  "required": [
+    "company",
+    "data",
+    "sample_date"
+  ]
+}

data/spec/bots/bot-that-crashes-immediately/manifest.json

@@ -0,0 +1,15 @@
+{
+    "bot_id": "bot-that-crashes-immediately",
+    "description": "This is a bot that crashes immediately",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb", "transformer1.rb"],
+    "transformers": [
+        {
+            "file": "transformer1.rb",
+            "data_type": "simple-licence",
+            "identifying_fields": ["licence_number"]
+        }
+    ]
+}

data/spec/bots/bot-that-crashes-immediately/scraper.rb

@@ -0,0 +1 @@
+raise 'Oh no'

data/spec/bots/bot-that-crashes-immediately/transformer1.rb

@@ -0,0 +1,15 @@
+require 'json'
+
+STDIN.each_line do |line|
+  raw_record = JSON.parse(line)
+
+  transformed_record = {
+    :company_name => 'Foo Widgets',
+    :company_jurisdiction => 'gb',
+    :licence_number => raw_record['licence_number'],
+    :source_url => raw_record['source_url'],
+    :sample_date => raw_record['sample_date'],
+  }
+
+  puts transformed_record.to_json
+end

data/spec/bots/bot-that-crashes-in-scraper/manifest.json

@@ -0,0 +1,15 @@
+{
+    "bot_id": "bot-that-crashes-in-scraper",
+    "description": "This is a bot that crashes in the scraper",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb", "transformer1.rb"],
+    "transformers": [
+        {
+            "file": "transformer1.rb",
+            "data_type": "simple-licence",
+            "identifying_fields": ["licence_number"]
+        }
+    ]
+}

data/spec/bots/bot-that-crashes-in-scraper/scraper.rb

@@ -0,0 +1,11 @@
+require 'json'
+
+0.upto(9) do |n|
+  record = {
+    :licence_number => "XYZ#{n}",
+    :source_url => 'http://example.com',
+    :sample_date => '2014-06-01'
+  }
+  puts(record.to_json)
+  raise 'Oh no' if n == 4
+end

data/spec/bots/bot-that-crashes-in-scraper/transformer1.rb

@@ -0,0 +1,15 @@
+require 'json'
+
+STDIN.each_line do |line|
+  raw_record = JSON.parse(line)
+
+  transformed_record = {
+    :company_name => 'Foo Widgets',
+    :company_jurisdiction => 'gb',
+    :licence_number => raw_record['licence_number'],
+    :source_url => raw_record['source_url'],
+    :sample_date => raw_record['sample_date'],
+  }
+
+  puts transformed_record.to_json
+end

data/spec/bots/bot-that-crashes-in-transformer/manifest.json

@@ -0,0 +1,20 @@
+{
+    "bot_id": "bot-that-crashes-in-transformer",
+    "description": "This is a bot that crashes in the transformer",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb", "transformer1.rb", "transformer2.rb"],
+    "transformers": [
+        {
+            "file": "transformer1.rb",
+            "data_type": "simple-licence",
+            "identifying_fields": ["licence_number"]
+        },
+        {
+            "file": "transformer2.rb",
+            "data_type": "simple-licence",
+            "identifying_fields": ["licence_number"]
+        }
+    ]
+}

data/spec/bots/bot-that-crashes-in-transformer/scraper.rb

@@ -0,0 +1,10 @@
+require 'json'
+
+0.upto(9) do |n|
+  record = {
+    :licence_number => "XYZ#{n}",
+    :source_url => 'http://example.com',
+    :sample_date => '2014-06-01'
+  }
+  puts(record.to_json)
+end

data/spec/bots/bot-that-crashes-in-transformer/transformer1.rb

@@ -0,0 +1,15 @@
+require 'json'
+
+STDIN.each_line do |line|
+  raw_record = JSON.parse(line)
+
+  transformed_record = {
+    :company_name => 'Foo Widgets',
+    :company_jurisdiction => 'gb',
+    :licence_number => raw_record['licence_number'],
+    :source_url => raw_record['source_url'],
+    :sample_date => raw_record['sample_date'],
+  }
+
+  puts transformed_record.to_json
+end

data/spec/bots/bot-that-crashes-in-transformer/transformer2.rb

@@ -0,0 +1,17 @@
+require 'json'
+
+STDIN.each_line do |line|
+  raw_record = JSON.parse(line)
+
+  transformed_record = {
+    :company_name => 'Foo Widgets',
+    :company_jurisdiction => 'gb',
+    :licence_number => raw_record['licence_number'],
+    :source_url => raw_record['source_url'],
+    :sample_date => raw_record['sample_date'],
+  }
+
+  puts transformed_record.to_json
+
+  raise 'Oh no' if raw_record['licence_number'] == 'XYZ4'
+end

data/spec/bots/bot-that-emits-run-ended/manifest.json

@@ -0,0 +1,8 @@
+{
+    "bot_id": "bot-that-emits-run-ended",
+    "description": "This is a bot that emits RUN ENDED",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb"]
+}

data/spec/bots/bot-that-emits-run-ended/scraper.rb

@@ -0,0 +1,11 @@
+require 'json'
+
+0.upto(2) do |n|
+  record = {
+    :licence_number => "XYZ#{n}",
+    :source_url => 'http://example.com',
+    :sample_date => '2014-06-01'
+  }
+  puts(record.to_json)
+end
+puts "RUN ENDED"

data/spec/bots/bot-that-expects-file/manifest.json

@@ -0,0 +1,8 @@
+{
+    "bot_id": "bot-that-expects-file",
+    "description": "This bot depends on being able to open a file in the current directory",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb", "something.txt"]
+}

data/spec/bots/bot-that-expects-file/scraper.rb

@@ -0,0 +1,11 @@
+require 'json'
+
+asd = open("something.txt", "r")
+0.upto(2) do |n|
+  record = {
+    :licence_number => "XYZ#{n}",
+    :source_url => 'http://example.com',
+    :sample_date => '2014-06-01'
+  }
+  puts(record.to_json)
+end

data/spec/bots/bot-that-expects-file/something.txt

@@ -0,0 +1 @@
+ehlo

data/spec/bots/bot-with-invalid-data-type/manifest.json

@@ -0,0 +1,8 @@
+{
+    "bot_id": "bot-with-invalid-data-type",
+    "description": "This bot has an invalid data type",
+    "language": "ruby",
+    "data_type": "fishmonger licence",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb"]
+}

data/spec/bots/bot-with-invalid-data-type/scraper.rb

@@ -0,0 +1,10 @@
+require 'json'
+
+0.upto(9) do |n|
+  record = {
+    :licence_number => "XYZ#{n}",
+    :source_url => 'http://example.com',
+    :sample_date => '2014-06-01'
+  }
+  puts(record.to_json)
+end

data/spec/bots/bot-with-invalid-sample-date/manifest.json

@@ -0,0 +1,8 @@
+{
+    "bot_id": "bot-with-invalid-sample-date",
+    "description": "This bot produces data with an invalid sample date",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb"]
+}

data/spec/bots/bot-with-invalid-sample-date/scraper.rb

@@ -0,0 +1,10 @@
+require 'json'
+
+0.upto(9) do |n|
+  record = {
+    :licence_number => "XYZ#{n}",
+    :source_url => 'http://example.com',
+    :sample_date => '01/06/2014'
+  }
+  puts(record.to_json)
+end

data/spec/bots/bot-with-pause/manifest.json

@@ -0,0 +1,8 @@
+{
+    "bot_id": "bot-with-pause",
+    "description": "This bot pauses for ten second halfway after outputting five lines",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb"]
+}

data/spec/bots/bot-with-pause/scraper.rb

@@ -0,0 +1,16 @@
+require 'json'
+
+0.upto(9) do |n|
+  record = {
+    :licence_number => "XYZ#{n}",
+    :source_url => 'http://example.com',
+    :sample_date => '2014-06-01'
+  }
+  puts(record.to_json)
+
+  if n == 4
+    $stderr.puts 'The scraper will sleep for ten seconds...'
+    sleep 10
+    $stderr.puts 'The scraper is resuming...'
+  end
+end

data/spec/bots/bot-with-transformer/manifest.json

@@ -0,0 +1,15 @@
+{
+    "bot_id": "bot-with-transformer",
+    "description": "This is a bot with a transformer",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb", "transformer1.rb", "transformer2.rb"],
+    "transformers": [
+        {
+            "file": "transformer.rb",
+            "data_type": "simple-licence",
+            "identifying_fields": ["licence_number"]
+        }
+    ]
+}

data/spec/bots/bot-with-transformer/scraper.rb

@@ -0,0 +1,10 @@
+require 'json'
+
+0.upto(9) do |n|
+  record = {
+    :licence_number => "XYZ#{n}",
+    :source_url => 'http://example.com',
+    :sample_date => '2014-06-01'
+  }
+  puts(record.to_json)
+end

data/spec/bots/bot-with-transformer/transformer.rb

@@ -0,0 +1,15 @@
+require 'json'
+
+STDIN.each_line do |line|
+  raw_record = JSON.parse(line)
+
+  transformed_record = {
+    :company_name => 'Foo Widgets',
+    :company_jurisdiction => 'gb',
+    :licence_number => raw_record['licence_number'],
+    :source_url => raw_record['source_url'],
+    :sample_date => raw_record['sample_date'],
+  }
+
+  puts transformed_record.to_json
+end

data/spec/bots/bot-with-transformers/manifest.json

@@ -0,0 +1,20 @@
+{
+    "bot_id": "bot-with-transformers",
+    "description": "This is a bot with multiple transformers",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb"],
+    "transformers": [
+        {
+            "file": "transformer1.rb",
+            "data_type": "simple-licence",
+            "identifying_fields": ["licence_number"]
+        },
+        {
+            "file": "transformer2.rb",
+            "data_type": "simple-licence",
+            "identifying_fields": ["licence_number"]
+        }
+    ]
+}

data/spec/bots/bot-with-transformers/scraper.rb

@@ -0,0 +1,10 @@
+require 'json'
+
+0.upto(9) do |n|
+  record = {
+    :licence_number => "XYZ#{n}",
+    :source_url => 'http://example.com',
+    :sample_date => '2014-06-01'
+  }
+  puts(record.to_json)
+end

data/spec/bots/bot-with-transformers/transformer1.rb

@@ -0,0 +1,15 @@
+require 'json'
+
+STDIN.each_line do |line|
+  raw_record = JSON.parse(line)
+
+  transformed_record = {
+    :company_name => 'Foo Widgets',
+    :company_jurisdiction => 'gb',
+    :licence_number => raw_record['licence_number'],
+    :source_url => raw_record['source_url'],
+    :sample_date => raw_record['sample_date'],
+  }
+
+  puts transformed_record.to_json
+end

data/spec/bots/bot-with-transformers/transformer2.rb

@@ -0,0 +1,15 @@
+require 'json'
+
+STDIN.each_line do |line|
+  raw_record = JSON.parse(line)
+
+  transformed_record = {
+    :company_name => 'Foo Widgets',
+    :company_jurisdiction => 'gb',
+    :licence_number => raw_record['licence_number'],
+    :source_url => raw_record['source_url'],
+    :sample_date => raw_record['sample_date'],
+  }
+
+  puts transformed_record.to_json
+end

data/spec/bots/invalid-json-bot/manifest.json

@@ -0,0 +1,8 @@
+{
+    "bot_id": "invalid-json-bot",
+    "description": "This is a bot that produces invalid JSON",
+    "language": "ruby",
+    "data_type": "primary data",
+    "identifying_fields": ["licence_number"],
+    "files": ["scraper.rb"]
+}

data/spec/bots/invalid-json-bot/scraper.rb

@@ -0,0 +1,11 @@
+require 'json'
+
+0.upto(9) do |n|
+  record = {
+    :licence_number => "XYZ#{n}",
+    :source_url => 'http://example.com',
+    :sample_date => '2014-06-01'
+  }
+  record = 'This is not JSON' if n == 5
+  puts(record.to_json)
+end