openc_bot 0.0.27 → 0.0.46

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    MGI3OTY2YWNlZWFlNDk4M2EwOTVhOTUwYTAwNjViODZmZDVlYjY2MQ==
-  data.tar.gz: !binary |-
-    MTQ0OGM2ZWZjOWYwNzQ5MGQ3Y2YxZDRiOGYyM2FiY2Y4MzBjNDIzZQ==
+SHA1:
+  metadata.gz: 1db3c143f46fc934729ee27c6cc5b4047fb2a5c5
+  data.tar.gz: 4c2de3f8f0ecc62f77689386dc6e50ed26290714
 SHA512:
-  metadata.gz: !binary |-
-    ZTFkYmVlMWFmZmIwMGNiYmIxZTA1ZDQzMTY5YWE0MjEyZWFlYmZiNTUxMjZj
-    ZGUxNDc2ZTZkNzlkOTYzOWFlYTRkZDM5ZDgwMTRmYmE0ZTM3ZTAxMWIwNGFm
-    YTA4MTgxNDc4OGI2OWRhZTk0NmQ3ODc4MmY5NWE4YmE4YzRlMjc=
-  data.tar.gz: !binary |-
-    MjQ2NmFhOGY4NDFjZDE2NGJjNTlkNGUwNWJjMzUyMDJjNGM4YjBiMGYxMzMw
-    YTE4ZmIxZmE0YTU0N2Y1NWE0NDU4ZGUzZjc1ODExZmZmZDAxNmZmZWMzY2Qx
-    MWQyNWU4NmEzYmQ5MjdiYzIxYTFlYTkyZjMzMWZjYWY0NjkwYWM=
+  metadata.gz: 6046b31e46416716606c0540ab60cc3b4c20d53043b7ec5701103dde77e47ccf6d0abb77e284bae0928dcaf66471d025c83bb51fbc6591c882ea401263cd4ae7
+  data.tar.gz: 22bd41bdd6639ea13f9d4787204fc7f13321d0c08ddf48748354b89a88bef8c0a73dda93f47e73cc20589332a7f6940400efa4998cc17c5ca7ebe8445222e65f

data/.travis.yml

@@ -4,6 +4,7 @@ rvm:
   - "1.9.3"
   - "2.0.0"
   - "2.1.0"
+  - "2.2.0"
   # - jruby-18mode # JRuby in 1.8 mode
   # - jruby-19mode # JRuby in 1.9 mode
   # - rbx

data/Gemfile

@@ -1,8 +1,10 @@
 source 'https://rubygems.org'
 gem "sqlite_magic", :git => 'https://github.com/openc/sqlite_magic.git'
+
 gem "pry", :group => [:development,:test]
 # Specify your gem's dependencies in openc_bot.gemspec
 gemspec
 
+
 # we need to do pull request and bump version
 # gem 'scraperwiki', '>=3.0.2', :git => 'https://github.com/openc/scraperwiki-ruby.git'

data/lib/openc_bot.rb

@@ -80,10 +80,25 @@ module OpencBot
     end
   end
 
+  def db_location
+    File.expand_path(File.join(@@app_directory, 'db', db_name))
+  end
+
   # Override default in ScraperWiki gem
   def sqlite_magic_connection
     db = @config ? @config[:db] : File.expand_path(File.join(@@app_directory, 'db', db_name))
-    @sqlite_magic_connection ||= SqliteMagic::Connection.new(db)
+    options = sqlite_busy_timeout ? {:busy_timeout => sqlite_busy_timeout} : {:busy_timeout => 10000}
+    @sqlite_magic_connection ||= SqliteMagic::Connection.new(db, options)
+  end
+
+  def sqlite_busy_timeout
+    self.const_defined?('SQLITE_BUSY_TIMEOUT') && self.const_get('SQLITE_BUSY_TIMEOUT')
+  end
+
+  def table_summary
+    field_names = sqlite_magic_connection.execute('PRAGMA table_info(ocdata)').collect{|c| c['name']}
+    select_sql = "COUNT(1) Total, " + field_names.collect{ |fn| "COUNT(#{fn}) #{fn}_not_null" }.join(', ') + " FROM ocdata"
+    select(select_sql).first
   end
 
 end

data/lib/openc_bot/company_fetcher_bot.rb

@@ -1,6 +1,8 @@
 require 'openc_bot'
 require 'openc_bot/helpers/incremental_search'
 require 'openc_bot/helpers/alpha_search'
+# require 'openc_bot/asana_notifier'
+require 'mail'
 
 
 module OpencBot
@@ -9,6 +11,8 @@ module OpencBot
     include OpencBot::Helpers::IncrementalSearch
     include OpencBot::Helpers::AlphaSearch
 
+    STDOUT.sync = true
+    STDERR.sync = true
     # This is called by #update_datum
     def fetch_datum(company_number)
       company_page = fetch_registry_page(company_number)
@@ -42,5 +46,51 @@ module OpencBot
       super || 'company-schema'
     end
 
+    def update_data(options={})
+      fetch_data
+      update_stale
+      send_run_report
+    rescue Exception => e
+      send_error_report(e)
+      raise e
+    end
+
+    private
+    def mark_bot_as_failing_on_asana(exception)
+      # error_description = "Code for this bot: https://github.com/openc/external_bots/tree/master/#{inferred_jurisdiction_code}_companies_fetcher\nError details: #{exception.inspect}.\nBacktrace:\n#{exception.backtrace}"
+      # params = {
+      #   :tag => inferred_jurisdiction_code,
+      #   :asana_api_key => ENV['ASANA_API_KEY'],
+      #   :workspace => ENV['ASANA_WORKSPACE'],
+      #   :title => exception.message,
+      #   :description => error_description
+      # }
+      # AsanaNotifier.create_failed_bot_task(params)
+    end
+
+    def send_error_report(e)
+      subject = "Error running #{self.name}: #{e}"
+      body = "Error details: #{e.inspect}.\nBacktrace:\n#{e.backtrace}"
+      mark_bot_as_failing_on_asana(e) if ENV['CREATE_ASANA_TASKS_FOR_BOT_FAILURES']
+      send_report(:subject => subject, :body => body)
+    end
+
+    def send_run_report
+      subject = "#{self.name} successfully ran"
+      db_filesize = File.size?(db_location)
+      body = "No problems to report. db is #{db_location}, #{db_filesize} bytes. Last modified: #{File.stat(db_location).mtime}"
+      send_report(:subject => subject, :body => body)
+    end
+
+    def send_report(params)
+      Mail.deliver do
+        from     'admin@opencorporates.com'
+        to       'bots@opencorporates.com'
+        subject  params[:subject]
+        body     params[:body]
+      end
+    end
+
+
   end
 end

data/lib/openc_bot/helpers/register_methods.rb

@@ -16,6 +16,10 @@ module OpencBot
         !!select("ocdata.#{primary_key_name} FROM ocdata WHERE #{primary_key_name} = ? LIMIT 1", uid).first
       end
 
+      def default_stale_count
+        self.const_defined?('STALE_COUNT') ? self.const_get('STALE_COUNT') : 1000
+      end
+
       # fetches and saves data. By default assumes an incremental search, or an alpha search
       # if USE_ALPHA_SEARCH is set. This method should be overridden if you are going to do a
       # different type of data import, e.g from a CSV file.
@@ -35,24 +39,25 @@ module OpencBot
       end
 
       def fetch_registry_page(company_number)
+        sleep_before_http_req
         _client.get_content(registry_url(company_number))
       end
 
       def prepare_and_save_data(all_data,options={})
         data_to_be_saved = prepare_for_saving(all_data)
-        fail_count, retry_interval = 0, 5
+        # fail_count, retry_interval = 0, 5
         begin
           insert_or_update([primary_key_name], data_to_be_saved)
         rescue SQLite3::BusyException => e
-          fail_count += 1
-          if fail_count <= MAX_BUSY_RETRIES
-            puts "#{e.inspect} raised #{fail_count} times saving:\n#{all_data}\n\nNow retrying in #{retry_interval} seconds" if verbose?
-            sleep retry_interval
-            retry_interval = retry_interval * 2
-            retry
-          else
-            raise e
-          end
+          # fail_count += 1
+          # if fail_count <= MAX_BUSY_RETRIES
+          puts "#{e.inspect} raised saving:\n#{all_data}\n\n" if verbose?
+          #   sleep retry_interval
+          #   retry_interval = retry_interval * 2
+          #   retry
+          # else
+          raise e
+          # end
         end
 
       end
@@ -61,6 +66,10 @@ module OpencBot
         self.const_defined?('PRIMARY_KEY_NAME') ? self.const_get('PRIMARY_KEY_NAME') : :uid
       end
 
+      def raise_when_saving_invalid_record
+        !!self.const_defined?('RAISE_WHEN_SAVING_INVALID_RECORD')
+      end
+
       # sensible default. Either uses computed version or registry_url in db
       def registry_url(uid)
         computed_registry_url(uid) || registry_url_from_db(uid)
@@ -94,7 +103,7 @@ module OpencBot
       end
 
       def stale_entry_uids(stale_count=nil)
-        stale_count ||= 1000
+        stale_count ||= default_stale_count
         sql_query = "ocdata.* from ocdata WHERE retrieved_at IS NULL OR strftime('%s', retrieved_at) < strftime('%s',  '#{Date.today - 30}') LIMIT #{stale_count.to_i}"
         raw_data = select(sql_query).each do |res|
           yield res[primary_key_name.to_s]
@@ -108,6 +117,24 @@ module OpencBot
         end
       end
 
+      def get_raw_data(uid, format=nil)
+        file_location = raw_data_file_location(uid, format)
+        File.read(file_location) if File.exist?(file_location)
+      end
+
+      def save_raw_data(raw_data, uid, format=nil)
+        file_location = raw_data_file_location(uid, format)
+        File.open(file_location, 'w') { |f| f.print raw_data }
+      end
+
+      def raw_data_file_location(uid, format=nil)
+        normalised_uid = uid.gsub(/[^[[:alnum:]]]/,'')
+        directory = File.join(*([root_directory,'data',normalised_uid.gsub(/^0+/,'').split(//).first(5)].flatten))
+        FileUtils.mkdir_p(directory) unless Dir.exist?(directory)
+        filename = format ? "#{normalised_uid}.#{format}" : normalised_uid
+        File.join(directory, filename)
+      end
+
       def update_data(options={})
         fetch_data
         update_stale
@@ -130,13 +157,14 @@ module OpencBot
       # or, if output_as_json is requested then the validation error is included
       # in the JSON error message
       def update_datum(uid, output_as_json=false,replace_existing_data=false)
+        # XXX here we refuse to run depending on run algorithm
         return unless raw_data = fetch_datum(uid)
         default_options = {primary_key_name => uid, :retrieved_at => Time.now}
         return unless base_processed_data = process_datum(raw_data)
         processed_data = default_options.merge(base_processed_data)
         # prepare the data for saving (converting Arrays, Hashes to json) and
         # save the original data too, as we may not extracting everything from it yet
-        save_entity(processed_data.merge(:data => raw_data))
+        raise_when_saving_invalid_record ? save_entity!(processed_data.merge(:data => raw_data)) : save_entity(processed_data.merge(:data => raw_data))
         if output_as_json
           puts processed_data.to_json
         else
@@ -152,11 +180,18 @@ module OpencBot
         end
       end
 
+      # at a rate of 1.16 companies per second, and allowing 12 hours
+      # running per day. a 3m register would be updated in 2 months:
+      MAX_STALE_COUNT = 100_000
       def update_stale(stale_count=nil)
-        stale_entry_uids(stale_count) do |stale_entry_uid|
-          update_datum(stale_entry_uid)
+        # XXX here set an arbitrarily large number and then rely on the system to stop
+        # XXX wrap this with timings to work out per-record rate
+        rate_limiter do |limiter|
+          stale_entry_uids(MAX_STALE_COUNT) do |stale_entry_uid|
+            update_datum(stale_entry_uid)
+            limiter.checkpoint
+          end
         end
-
       end
 
       def validate_datum(record)
@@ -196,6 +231,15 @@ module OpencBot
         prepared_data
       end
 
+      def sleep_before_http_req
+        if self.const_defined?('SLEEP_BEFORE_HTTP_REQ')
+          sleep_time = self.const_get('SLEEP_BEFORE_HTTP_REQ')
+          puts "#{self.name} about to sleep for #{sleep_time} before fetching data. Time now: #{Time.now}" if verbose?
+          sleep(sleep_time)
+          puts "#{self.name} slept for #{sleep_time}: Time now #{Time.now}" if verbose?
+        end
+      end
+
       def _client(options={})
         return @client if @client
         @client = HTTPClient.new(options.delete(:proxy))

data/lib/openc_bot/tasks.rb

@@ -3,6 +3,8 @@ require 'optparse'
 require 'json'
 require 'fileutils'
 
+PID_DIR = "/oc/pids"
+
 namespace :bot do
   desc "create a skeleton bot that can be used in OpenCorporates"
   task :create do
@@ -134,6 +136,17 @@ namespace :bot do
     end
   end
 
+  desc 'Lists count of non-null values in each field in ocdata table'
+  task :table_summary do
+    only_process_running('table_summary') do
+      bot_name = get_bot_name
+      require_relative File.join(Dir.pwd,'lib', bot_name)
+      runner = callable_from_file_name(bot_name)
+      res = runner.table_summary
+      res.each {|k,v| puts "#{k}:\t#{v}"}
+    end
+  end
+
   desc 'Summarise data for quality checking (only works for licences at the moment)'
   task :summarise_data do
     def as_sorted_hash(name, data)
@@ -327,7 +340,7 @@ EOF
         puts "Created #{new_file}"
       end
     end
-    
+
     #Add rspec debugger to gemfile
     File.open(File.join(working_dir,'Gemfile'),'a') do |file|
       file.puts "group :test do\n  gem 'rspec'\n  gem 'debugger'\nend"
@@ -341,7 +354,7 @@ EOF
   end
 
   def only_process_running(task_name)
-    pid_path = File.join(Dir.pwd, 'pids', task_name)
+    pid_path = File.join(PID_DIR, 'pids', task_name)
 
     raise_if_already_running(pid_path)
     write_pid_file(pid_path)

data/lib/openc_bot/templates/spec/spec_helper.rb

@@ -11,3 +11,7 @@ end
 def dummy_response(response_name, options={})
   IO.read(File.join(File.dirname(__FILE__),"dummy_responses",response_name.to_s), options)
 end
+
+Mail.defaults do
+  delivery_method :test # no, don't send emails when testing,
+end

data/lib/openc_bot/version.rb

@@ -1,3 +1,3 @@
 module OpencBot
-  VERSION = "0.0.27"
+  VERSION = "0.0.46"
 end

data/openc_bot.gemspec

@@ -35,14 +35,17 @@ Gem::Specification.new do |gem|
   gem.add_dependency "rake"
   gem.add_dependency "activesupport", "4.1.4"
   gem.add_dependency "nokogiri"
-  # gem.add_dependency "sqlite3"
+  gem.add_dependency "sqlite_magic", "0.0.6"
   gem.add_dependency "json"
   gem.add_dependency "json-schema"
   gem.add_dependency "httpclient"
   gem.add_dependency "backports"
   gem.add_dependency "scraperwiki", "3.0.2"
+  gem.add_dependency "mail"
+  # gem.add_dependency "openc-asana" unless RUBY_VERSION < '2.0'
 
-  gem.add_development_dependency "perftools.rb"
-  gem.add_development_dependency "debugger"
+  # gem.add_development_dependency "perftools.rb"
+  gem.add_development_dependency "byebug" unless RUBY_VERSION < '2.0'
+  gem.add_development_dependency "debugger" if RUBY_VERSION < '2.0'
   gem.add_development_dependency "rspec"
 end

data/spec/lib/company_fetcher_bot_spec.rb

@@ -3,6 +3,10 @@ require_relative '../spec_helper'
 require 'openc_bot'
 require 'openc_bot/company_fetcher_bot'
 
+Mail.defaults do
+  delivery_method :test # no, don't send emails when testing
+end
+
 module TestCompaniesFetcher
   extend OpencBot::CompanyFetcherBot
 end
@@ -121,4 +125,23 @@ describe "A module that extends CompanyFetcherBot" do
       end
     end
   end
+
+  describe '#update_data' do
+
+    before do
+      TestCompaniesFetcher.stub(:fetch_data_via_incremental_search)
+      TestCompaniesFetcher.stub(:update_stale)
+     #this can be any file that we can stat
+      TestCompaniesFetcher.stub(:db_location).
+        and_return(File.join(File.dirname(__FILE__),"company_fetcher_bot_spec.rb"))
+
+      Mail::TestMailer.deliveries.clear
+      TestCompaniesFetcher.update_data
+    end
+
+    it 'should send success email' do
+      Mail::TestMailer.deliveries.first.subject.should match /successfully ran/
+    end
+
+  end
 end

data/spec/lib/helpers/register_methods_spec.rb

@@ -8,6 +8,8 @@ module ModuleThatIncludesRegisterMethods
   extend OpencBot::Helpers::RegisterMethods
   PRIMARY_KEY_NAME = :custom_uid
   SCHEMA_NAME = 'company-schema'
+  SLEEP_BEFORE_HTTP_REQ = 2
+  RAISE_WHEN_SAVING_INVALID_RECORD = true
 end
 
 module ModuleWithNoCustomPrimaryKey
@@ -29,7 +31,6 @@ describe 'a module that includes RegisterMethods' do
   describe "#datum_exists?" do
     before do
       ModuleThatIncludesRegisterMethods.stub(:select).and_return([])
-
     end
 
     it "should select_data from database" do
@@ -233,10 +234,12 @@ describe 'a module that includes RegisterMethods' do
 
     context 'and SQLite3::BusyException raised' do
       it 'should retry up to 3 times' do
+        pending "deciding whether to allow this in some circumstances"
         ModuleThatIncludesRegisterMethods.should_receive(:insert_or_update).exactly(4).times.and_raise(SQLite3::BusyException)
         lambda { ModuleThatIncludesRegisterMethods.prepare_and_save_data(@params) }.should raise_error(SQLite3::BusyException)
       end
       it 'should not raise error if successful before limit' do
+        pending "deciding whether to allow this in some circumstances"
         ModuleThatIncludesRegisterMethods.should_receive(:insert_or_update).exactly(3).times.ordered.and_raise(SQLite3::BusyException)
         ModuleThatIncludesRegisterMethods.should_receive(:insert_or_update).ordered
         lambda { ModuleThatIncludesRegisterMethods.prepare_and_save_data(@params) }.should_not raise_error
@@ -372,6 +375,15 @@ describe 'a module that includes RegisterMethods' do
         end
       end
 
+      context 'and errors returned validating data' do
+        it "should validate processed data" do
+          ModuleThatIncludesRegisterMethods.stub(:validate_datum).and_return([{:failed_attribute => 'foo', :message => 'Something not right'}])
+          lambda { ModuleThatIncludesRegisterMethods.update_datum(@uid)}.should raise_error
+        end
+
+
+      end
+
       context 'and process_datum returns nil' do
         before do
           ModuleThatIncludesRegisterMethods.stub(:process_datum).and_return(nil)
@@ -405,11 +417,12 @@ describe 'a module that includes RegisterMethods' do
     end
   end
 
-  describe "#fetch_registry_page for company_number" do
+  describe "#fetch_registry_page for uid" do
     before do
       @dummy_client = double('http_client', :get_content => nil)
       ModuleThatIncludesRegisterMethods.stub(:_client).and_return(@dummy_client)
       ModuleThatIncludesRegisterMethods.stub(:registry_url).and_return('http://some.registry.url')
+      @dummy_client.stub(:get_content).and_return(:registry_page_html)
     end
 
     it "should GET registry_page for registry_url for company_number" do
@@ -423,6 +436,24 @@ describe 'a module that includes RegisterMethods' do
       @dummy_client.stub(:get_content).and_return(:registry_page_html)
       ModuleThatIncludesRegisterMethods.fetch_registry_page('76543').should == :registry_page_html
     end
+
+    context 'and SLEEP_BEFORE_HTTP_REQ is set' do
+      it 'should sleep for given period' do
+        ModuleThatIncludesRegisterMethods.should_receive(:sleep).with(2)
+        ModuleThatIncludesRegisterMethods.fetch_registry_page('76543')
+      end
+    end
+
+    context 'and SLEEP_BEFORE_HTTP_REQ is not set' do
+      before do
+        ModuleWithNoCustomPrimaryKey.stub(:_client).and_return(@dummy_client)
+      end
+
+      it 'should sleep for given period' do
+        ModuleWithNoCustomPrimaryKey.should_not_receive(:sleep)
+        ModuleWithNoCustomPrimaryKey.fetch_registry_page('76543')
+      end
+    end
   end
 
   describe "#validate_datum" do
@@ -594,4 +625,91 @@ describe 'a module that includes RegisterMethods' do
     end
   end
 
+  describe 'raise_when_saving_invalid_record' do
+    describe '#primary_key_name' do
+      it 'should return false if RAISE_WHEN_SAVING_INVALID_RECORD not set' do
+        ModuleWithNoCustomPrimaryKey.send(:raise_when_saving_invalid_record).should == false
+      end
+
+      it 'should return true if RAISE_WHEN_SAVING_INVALID_RECORD set' do
+        ModuleThatIncludesRegisterMethods.send(:raise_when_saving_invalid_record).should == true
+      end
+    end
+  end
+
+  describe '#raw_data_file_location for a uid' do
+    before do
+      @dummy_root_directory = File.join(File.dirname(__FILE__),'..','..','tmp')
+      Dir.mkdir(@dummy_root_directory) unless Dir.exist?(@dummy_root_directory)
+
+      ModuleThatIncludesRegisterMethods.stub(:root_directory).and_return(@dummy_root_directory)
+    end
+
+    after do
+      FileUtils.rmdir(File.join(@dummy_root_directory, 'data'))
+    end
+
+    it 'should return directory built from uid inside root data directory' do
+      ModuleThatIncludesRegisterMethods.raw_data_file_location('123456', 'html').should == File.join(@dummy_root_directory, 'data', '1','2','3','4','5', '123456.html')
+    end
+
+    it 'should create directory structure if it doesnt exist' do
+      ModuleThatIncludesRegisterMethods.raw_data_file_location('123456', 'html')
+      Dir.exist?(File.join(@dummy_root_directory, 'data', '1','2','3','4','5')).should == true
+    end
+
+    it 'should ignore leading zeroes when building directory' do
+      ModuleThatIncludesRegisterMethods.raw_data_file_location('001234', 'html').should == File.join(@dummy_root_directory, 'data', '1','2','3','4', '001234.html')
+    end
+
+    it 'should ignore non alphanum chars when building directory' do
+      ModuleThatIncludesRegisterMethods.raw_data_file_location('12a-b/3456', 'html').should == File.join(@dummy_root_directory, 'data', '1','2','a','b','3', '12ab3456.html')
+    end
+
+    it 'should allow format to be missing' do
+      ModuleThatIncludesRegisterMethods.raw_data_file_location('12a-b/3456').should == File.join(@dummy_root_directory, 'data', '1','2','a','b','3', '12ab3456')
+    end
+
+    it 'should allow format to be nil' do
+      ModuleThatIncludesRegisterMethods.raw_data_file_location('12a-b/3456', nil).should == File.join(@dummy_root_directory, 'data', '1','2','a','b','3', '12ab3456')
+    end
+  end
+
+  describe '#save_raw_data' do
+    before do
+      @dummy_root_directory = File.join(File.dirname(__FILE__),'..','..','tmp')
+      Dir.mkdir(@dummy_root_directory) unless Dir.exist?(@dummy_root_directory)
+
+      ModuleThatIncludesRegisterMethods.stub(:root_directory).and_return(@dummy_root_directory)
+    end
+
+    it 'should save raw data as in computed raw_data_file_location' do
+      ModuleThatIncludesRegisterMethods.save_raw_data('foo bar', '12a-b/3456', 'html')
+      File.read(File.join(@dummy_root_directory, 'data', '1','2','a','b','3', '12ab3456.html')).should == 'foo bar'
+    end
+
+    it 'should allow format to be missing' do
+      ModuleThatIncludesRegisterMethods.save_raw_data('foo bar', '12a-b/3456')
+      File.read(File.join(@dummy_root_directory, 'data', '1','2','a','b','3', '12ab3456')).should == 'foo bar'
+    end
+  end
+
+  describe '#get_raw_data' do
+    before do
+      @dummy_root_directory = File.join(File.dirname(__FILE__),'..','..','tmp')
+      Dir.mkdir(@dummy_root_directory) unless Dir.exist?(@dummy_root_directory)
+
+      ModuleThatIncludesRegisterMethods.stub(:root_directory).and_return(@dummy_root_directory)
+    end
+
+    it 'should read raw data in computed raw_data_file_location' do
+      File.open(File.join(@dummy_root_directory, 'data', '1','2','a','b','3', '12ab3456.html'),'w') { |f| f.print 'foo bar' }
+      ModuleThatIncludesRegisterMethods.get_raw_data('12a-b/3456', 'html').should == 'foo bar'
+    end
+
+    it 'should allow format to be missing' do
+      File.open(File.join(@dummy_root_directory, 'data', '1','2','a','b','3', '12ab3456'),'w') { |f| f.print 'foo bar' }
+      ModuleThatIncludesRegisterMethods.get_raw_data('12a-b/3456').should == 'foo bar'
+    end
+  end
 end