openc_bot 0.0.11

data/lib/openc_bot/helpers/register_methods.rb

@@ -0,0 +1,205 @@
+# encoding: UTF-8
+require 'json-schema'
+require 'active_support/core_ext'
+
+module OpencBot
+  module Helpers
+    module RegisterMethods
+
+      def use_alpha_search
+        self.const_defined?('USE_ALPHA_SEARCH') && self.const_get('USE_ALPHA_SEARCH')
+      end
+
+      def datum_exists?(uid)
+        !!select("ocdata.#{primary_key_name} FROM ocdata WHERE #{primary_key_name} = ? LIMIT 1", uid).first
+      end
+
+      # fetches and saves data. By default assumes an incremental search, or an alpha search
+      # if USE_ALPHA_SEARCH is set. This method should be overridden if you are going to do a
+      # different type of data import, e.g from a CSV file.
+      def fetch_data
+        if use_alpha_search
+          fetch_data_via_alpha_search
+        else
+          fetch_data_via_incremental_search
+        end
+      end
+
+      def export_data
+        sql_query = "ocdata.* from ocdata"
+        select(sql_query).each do |res|
+          yield post_process(res, true)
+        end
+      end
+
+      def fetch_registry_page(company_number)
+        _client.get_content(registry_url(company_number))
+      end
+
+      def prepare_and_save_data(all_data,options={})
+        data_to_be_saved = prepare_for_saving(all_data)
+        insert_or_update([primary_key_name], data_to_be_saved)
+      end
+
+      def primary_key_name
+        self.const_defined?('PRIMARY_KEY_NAME') ? self.const_get('PRIMARY_KEY_NAME') : :uid
+      end
+
+      # sensible default. Either uses computed version or registry_url in db
+      def registry_url(uid)
+        computed_registry_url(uid) || registry_url_from_db(uid)
+      end
+
+      # stub method. Override in including module if this can be computed from uid
+      def computed_registry_url(uid)
+      end
+
+      # stub method. Override in including module if this can be pulled from db (i.e. it is stored there)
+      def registry_url_from_db(uid)
+      end
+
+      def save_entity(entity_datum)
+        validation_errors = validate_datum(entity_datum.except(:data))
+        return unless validation_errors.blank?
+        prepare_and_save_data(entity_datum)
+      end
+
+      # Behaves like +save_entity+ but raises RecordInvalid exception if
+      # record is not valid (validation errors are available in the
+      # excpetion's +validation_errors+ method)
+      def save_entity!(entity_datum)
+        validation_errors = validate_datum(entity_datum.except(:data))
+        raise OpencBot::RecordInvalid.new(validation_errors) unless validation_errors.blank?
+        prepare_and_save_data(entity_datum)
+      end
+
+      def schema_name
+        self.const_defined?('SCHEMA_NAME') ? self.const_get('SCHEMA_NAME') : nil
+      end
+
+      def stale_entry_uids(stale_count=nil)
+        stale_count ||= 1000
+        sql_query = "ocdata.* from ocdata WHERE retrieved_at IS NULL OR strftime('%s', retrieved_at) < strftime('%s',  '#{Date.today - 30}') LIMIT #{stale_count.to_i}"
+        raw_data = select(sql_query).each do |res|
+          yield res[primary_key_name.to_s]
+        end
+      rescue SQLite3::SQLException => e
+        if e.message[/no such column: retrieved_at/]
+          sqlite_magic_connection.add_columns('ocdata', ['retrieved_at'])
+          retry
+        else
+          raise e
+        end
+      end
+
+      def update_data(options={})
+        fetch_data
+        update_stale
+        save_run_report(:status => 'success')
+      end
+
+      # This method updates a datum given by a uid (e.g. a company_number), by fetching new data, processing it
+      # and then saving it. It assumes the methods for doing this (#fetch_datum and #process_datum) are implemented
+      # in the module that includes this method.
+      #
+      # If no second argument is passed to this method, or false is passed, the
+      # method will return the processed data hash
+      # If true is passed as the second argument, the method will output the
+      # updated result as json to STDOUT, which can then be consumed by, say,
+      # something which triggered this method, for example if it was called by
+      # a rake task, which in turn might have been called by the main
+      # OpenCorporates application
+      def update_datum(uid, output_as_json=false,replace_existing_data=false)
+        return unless raw_data = fetch_datum(uid)
+        default_options = {primary_key_name => uid, :retrieved_at => Time.now}
+        processed_data = default_options.merge(process_datum(raw_data))
+        # prepare the data for saving (converting Arrays, Hashes to json) and
+        # save the original data too, as we may not extracting everything from it yet
+        save_entity(processed_data.merge(:data => raw_data))
+        if output_as_json
+          puts processed_data.to_json
+        else
+          processed_data
+        end
+      rescue Exception => e
+        output_json_error_message(e) if output_as_json
+      end
+
+      def update_stale(stale_count=nil)
+        stale_entry_uids(stale_count) do |stale_entry_uid|
+          update_datum(stale_entry_uid)
+        end
+
+      end
+
+      def validate_datum(record)
+        schema = File.expand_path("../../../../schemas/#{schema_name}.json", __FILE__)
+        errors = JSON::Validator.fully_validate(
+          schema,
+          record.to_json,
+          {:errors_as_objects => true})
+      end
+
+      def post_process(row_hash, skip_nulls=false)
+        # many of the fields will be serialized json and so we convert to ruby objects
+        convert_json_to_ruby(row_hash.except(:data), skip_nulls)
+      end
+
+      private
+      # This is a utility method for outputting an error message as json to STDOUT
+      # (which can then be handled by the importer)
+      def output_json_error_message(err_obj)
+        err_msg = {'error' => {'klass' => err_obj.class.to_s, 'message' => err_obj.message, 'backtrace' => err_obj.backtrace}}
+        puts err_msg.to_json
+      end
+
+      def prepare_for_saving(raw_data_hash)
+        prepared_data = deep_clone_hash(raw_data_hash)
+        #This jsonifies each value that is an an array or hash so that it can be saved as a string in sqlite
+        prepared_data.each do |k,v|
+          case v
+          when Array, Hash
+            prepared_data[k] = v.to_json
+          when Date, Time, DateTime
+            prepared_data[k] = v.iso8601
+          end
+        end
+        prepared_data
+      end
+
+      def _client(options={})
+        return @client if @client
+        @client = HTTPClient.new(options.delete(:proxy))
+        @client.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE if options.delete(:skip_ssl_verification)
+        @client.agent_name = options.delete(:user_agent)
+        @client.ssl_config.ssl_version = options.delete(:ssl_version) if options[:ssl_version]
+        if ssl_certificate = options.delete(:ssl_certificate)
+          @client.ssl_config.add_trust_ca(ssl_certificate) # Above cert
+        end
+        @client
+      end
+
+      def deep_clone_hash(given_hash)
+        Marshal.load( Marshal.dump(given_hash) )
+      end
+
+      def convert_json_to_ruby(data_hash, skip_nulls=false)
+        data_hash.each do |k,v|
+          parsed_data = JSON.parse(v) if v.is_a?(String) && v[/^[\{\[]+\"|^\[\]$|^{}$/] rescue v
+          case parsed_data
+          when Hash
+            parsed_data = parsed_data.with_indifferent_access
+          when Array
+            parsed_data.collect!{ |e| e.is_a?(Hash) ? e.with_indifferent_access : e  }
+          end
+          if skip_nulls && v.nil?
+            data_hash.delete(k)
+          else
+            data_hash[k] = parsed_data if parsed_data
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/openc_bot/helpers/text.rb

@@ -0,0 +1,18 @@
+# encoding: UTF-8
+module OpencBot
+  module Helpers
+    module Text
+      extend self
+
+      def normalise_utf8_spaces(raw_text)
+        raw_text&&raw_text.gsub(/\xC2\xA0/, ' ')
+      end
+
+      def strip_all_spaces(text)
+        text&&normalise_utf8_spaces(text).strip.gsub(/\s+/,' ')
+      end
+
+
+    end
+  end
+end

data/lib/openc_bot/incrementers.rb

@@ -0,0 +1,2 @@
+require 'openc_bot/incrementers/base'
+require 'openc_bot/incrementers/common'

data/lib/openc_bot/incrementers/base.rb

@@ -0,0 +1,214 @@
+require 'backports/2.0.0/enumerable/lazy'
+require 'json'
+module OpencBot
+  class BaseIncrementer
+
+    def initialize(name, opts={})
+      @name = name
+      @expected_count = opts[:expected_count]
+      @count = 0
+      @app_path = opts[:app_path]
+      @show_progress = opts[:show_progress] || (opts[:show_progress].nil? && true)
+      @reset_iterator = opts[:reset_iterator]
+      @max_iterations = opts[:max_iterations]
+      @opts = opts
+    end
+
+    def self.new(*args)
+      path, = caller[0].partition(":")
+      path = File.expand_path(File.join(File.dirname(path), ".."))
+      args << {} if args.count == 1
+      args[1][:app_path] = path if !args[1][:app_path]
+      super(*args)
+    end
+
+    def log_progress(percent)
+      puts "Iterator #{@name} progress: " + (percent.to_s + "%") if @show_progress
+    end
+
+    def progress_percent
+      (@count.to_f / @expected_count * 100).round(2) if @expected_count
+    end
+
+    def each
+      Enumerator.new do |yielder|
+        increment_yielder do |result|
+          if result.is_a? Hash
+            formatted_result = result.to_json
+          else
+            formatted_result = result
+          end
+          write_current(formatted_result)
+          yielder.yield(result)
+          @count += 1
+          log_progress(progress_percent)
+        end
+        reset_current
+      end.lazy
+    end
+
+    def resumable
+      enum = each
+      enum = resuming_enum(enum) unless @reset_iterator
+      enum = enum.take(@max_iterations) if @max_iterations
+      enum
+    end
+
+    def resuming_enum(enum)
+      start_from = read_current
+      preset_show_progress = @show_progress
+      @show_progress = false
+      if start_from && start_from != ""
+        enum = enum.drop_while do |x|
+          found_start_point = (x.to_s == start_from)
+          @show_progress = preset_show_progress && found_start_point
+          !found_start_point
+        end
+      end
+      enum
+    end
+
+    def position_file_name
+      "#{@app_path}/db/#{db_name}-iterator-position.txt"
+    end
+
+    def db_name
+      @name
+    end
+
+    # this is done with a file, rather than SQL, for speed reasons
+    def reset_current
+      File.open(position_file_name, "w") do |f|
+        f.write("")
+      end
+    end
+
+    def write_current(val)
+      File.open(position_file_name, "w") do |f|
+        f.write(val.to_s)
+      end
+    end
+
+    def read_current
+      begin
+        File.open(position_file_name, "r") do |f|
+          f.read
+        end
+      rescue Errno::ENOENT
+        nil
+      end
+    end
+  end
+
+  class ManualIncrementer < OpencBot::BaseIncrementer
+
+    include ScraperWiki
+
+    ITEMS_TABLE = "items"
+
+    def single_transaction
+      sqlite_magic_connection.execute("BEGIN TRANSACTION")
+      yield(self)
+      sqlite_magic_connection.execute("COMMIT")
+    end
+
+    def initialize(name, opts={})
+      super(name, opts)
+      raise "Fields must be defined for this Record" if opts[:fields].nil?
+      query = "CREATE TABLE IF NOT EXISTS #{ITEMS_TABLE} (#{opts[:fields].join(',')}, _id INTEGER PRIMARY KEY)"
+      sqlite_magic_connection.execute query
+      query = "CREATE UNIQUE INDEX IF NOT EXISTS #{opts[:fields].join('_')} " +
+        "ON #{ITEMS_TABLE} (#{opts[:fields].join(',')})"
+      sqlite_magic_connection.execute query
+    end
+
+    # Override default in ScraperWiki gem
+    def sqlite_magic_connection
+      db = File.expand_path(File.join(@app_path, 'db', "#{db_name}.db"))
+      @sqlite_magic_connection ||= SqliteMagic::Connection.new(db)
+    end
+
+    def increment_yielder(start_row=nil)
+      start_id = start_row && start_row["_id"].to_i
+      @expected_count = count_all_items
+      @count = count_processed_items(start_id)
+      loop do
+        result = read_batch(start_id).each do |row|
+          yield row
+          start_id = row["_id"].to_i + 1
+        end
+        raise StopIteration if result.empty?
+      end
+    end
+
+    def populated
+      begin
+        result = select("populated FROM misc").first['populated']
+        result && result == "true"
+      rescue SqliteMagic::NoSuchTable
+      end
+    end
+
+    def populated=(val)
+      if val && val == "true" || val == true
+        save_sqlite([:populated], {:populated => "true"}, "misc")
+      end
+    end
+
+    def enum(*args)
+      self.populated = true
+      each
+    end
+
+    def add_row(val)
+      sqlite_magic_connection.insert_or_update(
+        val.keys, val, ITEMS_TABLE, :update_unique_keys => true)
+    end
+
+    def count_processed_items(start_id)
+      if start_id
+        begin
+          result = select("count(*) as count FROM #{ITEMS_TABLE} WHERE _id < #{start_id}").first
+          result && result['count']
+        rescue SqliteMagic::NoSuchTable
+          0
+        end
+      else
+        0
+      end
+    end
+
+    def count_all_items
+      begin
+        select("count(*) as count FROM #{ITEMS_TABLE}").first['count']
+      rescue SqliteMagic::NoSuchTable
+      end
+    end
+
+    def read_batch(start_id=nil)
+      sql = "* FROM #{ITEMS_TABLE}"
+      if start_id
+        sql += " WHERE _id >= #{start_id}"
+      end
+      sql += " LIMIT 100"
+      select(sql)
+    end
+
+    # override superclass definition for more efficient version
+    def resuming_enum(enum)
+      current_row = read_current && read_current != "" && JSON.parse(read_current)
+      if current_row
+        enum = Enumerator.new do |yielder|
+          increment_yielder(current_row) do |result|
+            write_current(result.to_json)
+            yielder.yield(result)
+            @count += 1
+            log_progress(progress_percent)
+          end
+          reset_current
+        end.lazy
+      end
+      enum
+    end
+  end
+end

data/lib/openc_bot/incrementers/common.rb

@@ -0,0 +1,47 @@
+module OpencBot
+  class NumericIncrementer < OpencBot::BaseIncrementer
+    def initialize(name, opts={})
+      raise "You must specify an end_val for a NumericIncrementer" if ! opts[:end_val]
+      @start_val = opts[:start_val] || 0
+      @end_val = opts[:end_val]
+      super(name, opts)
+    end
+
+    def increment_yielder
+      @expected_count = @end_val
+      i = @start_val
+      loop do
+        if i > @end_val
+          raise StopIteration
+        end
+        yield i
+        i += 1
+      end
+    end
+  end
+
+  class AsciiIncrementer < OpencBot::BaseIncrementer
+    def initialize(name, opts={})
+      @size = opts[:size] || 3
+      super(name, opts)
+    end
+
+    def increment_yielder
+      alnum = (0...36).map{|i|i.to_s 36} # 0...z
+      all_perms = alnum.repeated_permutation(@size)
+      case @size
+      when 1
+        @expected_count = 36
+      when 2
+        @expected_count = 1296
+      when 3
+        @expected_count = 46656
+      when 4
+        @expected_count = 1679616
+      end
+      all_perms.each do |perm|
+        yield perm.join
+      end
+    end
+  end
+end