apisonator 2.100.0

data/lib/3scale/backend/analytics/kinesis.rb

@@ -0,0 +1,3 @@
+require_relative 'kinesis/adapter'
+require_relative 'kinesis/exporter'
+require_relative 'kinesis/job'

data/lib/3scale/backend/analytics/kinesis/adapter.rb

@@ -0,0 +1,180 @@
+require '3scale/backend/logging'
+
+module ThreeScale
+  module Backend
+    module Analytics
+      module Kinesis
+        class Adapter
+          # Each Kinesis record is rounded to the nearest 5KB to calculate the
+          # cost. Each of our events is a hash with a few keys: service,
+          # metric, period, time, value, etc. This means that the size of one
+          # of our events is nowhere near 5KB. For that reason, we need to make
+          # sure that we send many events in each record.
+          # The max size for each record is 1000KB. In each record batch, Kinesis
+          # accepts a maximum of 4MB.
+          #
+          # We will try to optimize the batching process later. For now, I will
+          # just put 1000 events in each record. And batches of 5 records max.
+          #
+          # When we receive a number of events not big enough to fill a record,
+          # those events are marked as pending events.
+          # Kinesis can return errors, when that happens, the events of the
+          # records that failed are re-enqueued as pending events.
+          # The list of pending events is stored in Redis, so we do not fail to
+          # process any events in case of downtime or errors.
+
+          include Logging
+
+          EVENTS_PER_RECORD = 1000
+          private_constant :EVENTS_PER_RECORD
+
+          MAX_RECORDS_PER_BATCH = 5
+          private_constant :MAX_RECORDS_PER_BATCH
+
+          EVENTS_PER_BATCH = EVENTS_PER_RECORD*MAX_RECORDS_PER_BATCH
+          private_constant :EVENTS_PER_BATCH
+
+          KINESIS_PENDING_EVENTS_KEY = 'send_to_kinesis:pending_events'
+          private_constant :KINESIS_PENDING_EVENTS_KEY
+
+          # We need to limit the number of pending events stored in Redis.
+          # The Redis database can grow very quickly if a few consecutive jobs
+          # fail. I am going to limit the number of pending events to 600k
+          # (10 jobs approx.). If that limit is reached, we will disable the
+          # creation of buckets in the system, but we will continue trying to
+          # send the failed events. We will lose data, but that is better than
+          # collapsing the whole Redis.
+          # We will try to find a better alternative once we cannot afford to
+          # miss events. Right now, we are just deleting the stats keys with
+          # period = minute, so we can restore everything else.
+          MAX_PENDING_EVENTS = 600_000
+          private_constant :MAX_PENDING_EVENTS
+
+          MAX_PENDING_EVENTS_REACHED_MSG =
+            'Bucket creation has been disabled. Max pending events reached'.freeze
+          private_constant :MAX_PENDING_EVENTS_REACHED_MSG
+
+          def initialize(stream_name, kinesis_client, storage)
+            @stream_name = stream_name
+            @kinesis_client = kinesis_client
+            @storage = storage
+          end
+
+          def send_events(events)
+            pending_events = stored_pending_events + events
+
+            # Only disable indicating emergency if bucket storage is enabled.
+            # We do not want to indicate emergency if it was disabled manually.
+            if limit_pending_events_reached?(pending_events.size) && Stats::Storage.enabled?
+              Stats::Storage.disable!(true)
+              log_bucket_creation_disabled
+            end
+
+            # Batch events until we can fill at least one record
+            if pending_events.size >= EVENTS_PER_RECORD
+              failed_events = send_events_in_batches(pending_events)
+              store_pending_events(failed_events)
+            else
+              store_pending_events(pending_events)
+            end
+          end
+
+          # Sends the pending events to Kinesis, even if there are not enough of
+          # them to fill 1 record.
+          # Returns the number of events correctly sent to Kinesis
+          def flush(limit = nil)
+            pending_events = stored_pending_events
+            events_to_flush = limit ? pending_events.take(limit) : pending_events
+            failed_events = send_events_in_batches(events_to_flush)
+            store_pending_events(pending_events - events_to_flush + failed_events)
+            events_to_flush.size - failed_events.size
+          end
+
+          def num_pending_events
+            storage.scard(KINESIS_PENDING_EVENTS_KEY)
+          end
+
+          private
+
+          attr_reader :stream_name, :kinesis_client, :storage
+
+          def stored_pending_events
+            storage.smembers(KINESIS_PENDING_EVENTS_KEY).map do |pending_event|
+              JSON.parse(pending_event, symbolize_names: true)
+            end
+          end
+
+          def limit_pending_events_reached?(count)
+            count > MAX_PENDING_EVENTS
+          end
+
+          def log_bucket_creation_disabled
+            logger.info(MAX_PENDING_EVENTS_REACHED_MSG)
+          end
+
+          # Returns the failed events
+          def send_events_in_batches(events)
+            failed_events = []
+
+            events.each_slice(EVENTS_PER_BATCH) do |events_slice|
+              begin
+                kinesis_resp = kinesis_client.put_record_batch(
+                  { delivery_stream_name: stream_name,
+                    records: events_to_kinesis_records(events_slice) })
+                failed_events << failed_events_kinesis_resp(
+                  kinesis_resp[:request_responses], events_slice)
+              rescue Aws::Firehose::Errors::ServiceError
+                failed_events << events_slice
+              end
+            end
+
+            failed_events.flatten
+          end
+
+          def events_to_kinesis_records(events)
+            # Record format expected by Kinesis:
+            # [{ data: "data_event_group_1" }, { data: "data_event_group_2" }]
+            events.each_slice(EVENTS_PER_RECORD).map do |events_slice|
+              { data: events_to_pseudo_json(events_slice) }
+            end
+          end
+
+          # We want to send to Kinesis events that can be read by Redshift.
+          # Redshift expects events in JSON format without the '[]' and
+          # without separating them with commas.
+          # We put each event in a separated line, that will make their parsing
+          # easier, but it is not needed by Redshift.
+          def events_to_pseudo_json(events)
+            events.map { |event| event.to_json }.join("\n") + "\n"
+          end
+
+          def failed_events_kinesis_resp(request_responses, events)
+            failed_records_indexes = failed_records_indexes(request_responses)
+            failed_records_indexes.flat_map do |failed_record_index|
+              events_index_start = failed_record_index*EVENTS_PER_RECORD
+              events_index_end = events_index_start + EVENTS_PER_RECORD - 1
+              events[events_index_start..events_index_end]
+            end
+          end
+
+          def failed_records_indexes(request_responses)
+            result = []
+            request_responses.each_with_index do |response, index|
+              result << index unless response[:error_code].nil?
+            end
+            result
+          end
+
+          def store_pending_events(events)
+            storage.pipelined do
+              storage.del(KINESIS_PENDING_EVENTS_KEY)
+              events.each do |event|
+                storage.sadd(KINESIS_PENDING_EVENTS_KEY, event.to_json)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/analytics/kinesis/exporter.rb

@@ -0,0 +1,86 @@
+module ThreeScale
+  module Backend
+    module Analytics
+      module Kinesis
+
+        # The main responsibility of this class is to schedule Kinesis jobs.
+        # We know that the distributed locking algorithm that we are using
+        # guarantees that two jobs will not be running at the same time except
+        # in some corner cases, like in the case of a failure of one of the Redis
+        # masters. However, this is not a problem in our case. If two Kinesis
+        # jobs run at the same time, they will probably export the same events to
+        # Kinesis. However, they will not be imported twice into Redshift because
+        # the import method that we use detects that two events are the same and
+        # only imports one. This detection is done using the 'time_gen' field
+        # that we attach to each event before they are send to Kinesis.
+        class Exporter
+          SEND_TO_KINESIS_ENABLED_KEY = 'send_to_kinesis:enabled'.freeze
+          private_constant :SEND_TO_KINESIS_ENABLED_KEY
+
+          TTL_JOB_RUNNING_KEY_SEC = 360
+          private_constant :TTL_JOB_RUNNING_KEY_SEC
+
+          class << self
+            def enable
+              storage.set(SEND_TO_KINESIS_ENABLED_KEY, '1')
+            end
+
+            def disable
+              storage.del(SEND_TO_KINESIS_ENABLED_KEY)
+            end
+
+            def enabled?
+              storage.get(SEND_TO_KINESIS_ENABLED_KEY).to_i == 1
+            end
+
+            def schedule_job
+              if enabled?
+                lock_key = dist_lock.lock
+                if lock_key
+                  Resque.enqueue(Job, Time.now.utc, lock_key, Time.now.utc.to_f)
+                end
+              end
+            end
+
+            def flush_pending_events(limit = nil)
+              flushed_events = 0
+              if enabled?
+                lock_key = dist_lock.lock
+                if lock_key
+                  flushed_events = kinesis_adapter.flush(limit)
+                  job_finished(lock_key) # flush is not asynchronous
+                end
+              end
+              flushed_events
+            end
+
+            def num_pending_events
+              kinesis_adapter.num_pending_events
+            end
+
+            # To be called by a kinesis job once it exits so other jobs can run
+            def job_finished(lock_key)
+              dist_lock.unlock if lock_key == dist_lock.current_lock_key
+            end
+
+            private
+
+            def storage
+              Backend::Storage.instance
+            end
+
+            def kinesis_adapter
+              Stats::Storage.kinesis_adapter
+            end
+
+            def dist_lock
+              @dist_lock ||= DistributedLock.new(self.name,
+                                                 TTL_JOB_RUNNING_KEY_SEC,
+                                                 storage)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/analytics/kinesis/job.rb

@@ -0,0 +1,135 @@
+require '3scale/backend/logging'
+
+module ThreeScale
+  module Backend
+    module Analytics
+      module Kinesis
+        # This job works as follows:
+        #   1) Reads the pending events from the buckets that have not been read.
+        #   2) Parses and filters those events.
+        #   3) Sends the events to the Kinesis adapter.
+        #   4) Updates the latest bucket read, to avoid processing buckets more
+        #      than once.
+        # The events are sent in batches to Kinesis, but the component that does
+        # that batching is the Kinesis adapter.
+        #
+        # Before sending the events to Kinesis, we attach a 'time_gen' attribute
+        # to each of them. This is a timestamp that indicates approximately when
+        # the event was generated based on the bucket where it was stored.
+        # We need this attribute because we will have repeated event keys in
+        # Redis and we will need to know which one contains the most updated
+        # value.
+        # Notice that we do not send all the events that are in the buckets to
+        # Kinesis. This job reads several buckets each time it runs. Some events
+        # can be repeated across those buckets. However, the job will only send
+        # to Kinesis the latest value (the one in the most recent bucket). This
+        # reduces the information that we need to parse, filter, and send.
+        # We need the extra field 'time_gen', because we cannot safely assume any
+        # order in S3 when sending events to Kinesis.
+        class Job < BackgroundJob
+          @queue = :stats
+
+          FILTERED_EVENT_PERIODS = %w(week eternity)
+          private_constant :FILTERED_EVENT_PERIODS
+
+          # We need to limit the amount of buckets that a job can process.
+          # Otherwise, there is the possibility that the job would not finish
+          # before its expiration time, and the next one would start processing
+          # the same buckets.
+          MAX_BUCKETS = 60
+          private_constant :MAX_BUCKETS
+
+          FILTERED_EVENT_PERIODS_STR = FILTERED_EVENT_PERIODS.map do |period|
+            "/#{period}".freeze
+          end.freeze
+          private_constant :FILTERED_EVENT_PERIODS_STR
+
+          class << self
+            include Logging
+
+            def perform_logged(end_time_utc, lock_key, _enqueue_time)
+              # end_time_utc will be a string when the worker processes this job.
+              # The parameter is passed through Redis as a string. We need to
+              # convert it back.
+              events_sent = 0
+
+              end_time = DateTime.parse(end_time_utc).to_time.utc
+              pending_events = bucket_reader.pending_events_in_buckets(
+                end_time_utc: end_time, max_buckets: MAX_BUCKETS)
+
+              unless pending_events[:events].empty?
+                events = prepare_events(pending_events[:latest_bucket],
+                                        pending_events[:events])
+                kinesis_adapter.send_events(events)
+                bucket_reader.latest_bucket_read = pending_events[:latest_bucket]
+                events_sent = events.size
+
+                # We might use a different strategy to delete buckets in the
+                # future, but for now, we are going to delete the buckets as they
+                # are read
+                bucket_storage.delete_range(pending_events[:latest_bucket])
+              end
+
+              Exporter.job_finished(lock_key)
+              [true, msg_events_sent(events_sent)]
+            end
+
+            private
+
+            def prepare_events(bucket, events)
+              filter_events(events)
+              parsed_events = parse_events(events.lazy)
+              add_time_gen_to_events(parsed_events, bucket_to_timestamp(bucket)).force
+            end
+
+            # Parses the events and discards the invalid ones
+            def parse_events(events)
+              events.map do |k, v|
+                begin
+                  Stats::StatsParser.parse(k, v)
+                rescue Stats::StatsParser::StatsKeyValueInvalid
+                  logger.notify("Invalid stats key-value. k: #{k}. v: #{v}")
+                  nil
+                end
+              end.reject(&:nil?)
+            end
+
+            # We do not want to send all the events to Kinesis.
+            # This method filters them.
+            def filter_events(events)
+              events.reject! do |event|
+                FILTERED_EVENT_PERIODS_STR.any? do |filtered_period|
+                  event.include?(filtered_period)
+                end
+              end
+            end
+
+            def add_time_gen_to_events(events, time_gen)
+              events.map { |event| event[:time_gen] = time_gen; event }
+            end
+
+            def bucket_to_timestamp(bucket)
+              DateTime.parse(bucket).to_time.utc.strftime('%Y%m%d %H:%M:%S')
+            end
+
+            def msg_events_sent(n_events)
+              "#{n_events} events have been sent to the Kinesis adapter"
+            end
+
+            def bucket_storage
+              Stats::Storage.bucket_storage
+            end
+
+            def bucket_reader
+              Stats::Storage.bucket_reader
+            end
+
+            def kinesis_adapter
+              Stats::Storage.kinesis_adapter
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/analytics/redshift.rb

@@ -0,0 +1,3 @@
+require_relative 'redshift/adapter'
+require_relative 'redshift/importer'
+require_relative 'redshift/job'

data/lib/3scale/backend/analytics/redshift/adapter.rb

@@ -0,0 +1,367 @@
+require 'pg'
+
+module ThreeScale
+  module Backend
+    module Analytics
+      module Redshift
+        # This class imports the events stored by Kinesis in S3 into Redshift.
+        # It keeps track of the events that have been imported so it does not
+        # read twice the same S3 path.
+        #
+        # We store 'repeated' events in S3. This means that we can find several
+        # times the same {service, instance, uinstance, metric, period, timestamp}
+        # combination.
+        #
+        # In order to avoid storing repeated information in Redshift we need to
+        # perform UPSERTs. The algorithm followed is the one explained in the
+        # official Redshift documentation:
+        # http://docs.aws.amazon.com/redshift/latest/dg/t_updating-inserting-using-staging-tables-.html
+        # The process is as follows:
+        #  1) Create a temporary table with the data imported from S3, including
+        #     duplicates.
+        #     Two attributes can have nulls: cinstance and uinstance. We replace
+        #     those nulls with ''. I have observed substantial performance gains
+        #     because of this.
+        #  2) Perform the necessary operations in the temp table to remove
+        #     duplicates. (In our case this basically consists of an inner-join).
+        #  3) Inside a transaction, delete all the events that are in the temp
+        #     table from the final table. Next, insert the ones in the temp
+        #     table into the final table. Finally, remove the temp table.
+        #  4) Last, we perform a vacuum, because Redshift does not automatically
+        #     reclaim and reuse space that has been freed after deletes or
+        #     updates. The vacuum operation also leaves the table sorted.
+        #     More info:
+        #     http://docs.aws.amazon.com/redshift/latest/dg/t_Reclaiming_storage_space202.html
+        #     Right now, we are going to vacuum every time we insert new data,
+        #     we will see if for performance reasons we need to do it less often.
+        class Adapter
+
+          module SQL
+            SCHEMA = 'backend'.freeze
+
+            # This importer relies on some tables or views that are created in
+            # Redshift to function correctly.
+            TABLES = { events: "#{SCHEMA}.events".freeze,
+                       latest_s3_path_read: "#{SCHEMA}.latest_s3_path_read".freeze,
+                       temp: "#{SCHEMA}.temp_events".freeze,
+                       unique_imported_events: "#{SCHEMA}.unique_imported_events".freeze }.freeze
+
+            EVENT_ATTRS = %w(service cinstance uinstance metric period timestamp time_gen).freeze
+            JOIN_EVENT_ATTRS = (EVENT_ATTRS - ['time_gen']).freeze
+
+            EXISTING_TABLES =
+              'SELECT table_name '\
+              'FROM information_schema.tables '\
+              "WHERE table_schema = '#{SCHEMA}';".freeze
+
+            CREATE_TEMP_TABLES =
+              "DROP TABLE IF EXISTS #{TABLES[:temp]} CASCADE; "\
+              "CREATE TABLE #{TABLES[:temp]} (LIKE #{TABLES[:events]}); "\
+              "DROP TABLE IF EXISTS #{TABLES[:unique_imported_events]} CASCADE; "\
+              "CREATE TABLE #{TABLES[:unique_imported_events]} (LIKE #{TABLES[:events]}); "\
+              'COMMIT;'.freeze
+
+            CLEAN_TEMP_TABLES =
+              "DROP TABLE #{TABLES[:unique_imported_events]}; "\
+              "DROP TABLE #{TABLES[:temp]};".freeze
+
+            LATEST_TIMESTAMP_READ = "SELECT s3_path FROM #{TABLES[:latest_s3_path_read]}".freeze
+
+            VACUUM = "VACUUM FULL #{TABLES[:events]}".freeze
+
+            class << self
+
+              def insert_imported_events
+                'BEGIN TRANSACTION; '\
+                  "DELETE FROM #{TABLES[:events]} "\
+                  "USING #{TABLES[:unique_imported_events]} u "\
+                  "WHERE #{TABLES[:events]}.timestamp >= "\
+                  "(SELECT MIN(timestamp) FROM #{TABLES[:unique_imported_events]}) "\
+                  "AND #{join_comparisons(TABLES[:events], 'u', JOIN_EVENT_ATTRS)} "\
+                  "AND (#{TABLES[:events]}.time_gen < u.time_gen); "\
+                  "INSERT INTO #{TABLES[:events]} "\
+                  "SELECT * FROM #{TABLES[:unique_imported_events]};" \
+                  'END TRANSACTION;'.freeze
+              end
+
+              # In order to get unique events, I use an inner-join with the same
+              # table. There might be several rows with the same {service, instance,
+              # uinstance, metric, period, timestamp} and different time_gen and
+              # value. From those rows, we want to get just the one with the highest
+              # time_gen. We cannot get the one with the highest value because we
+              # support SET operations. That means that a value of '0' can be more
+              # recent than '50'.
+              #
+              # The way to solve this is as follows: find out the max time_gen
+              # grouping the 'repeated' events, and then perform an inner-join to
+              # select the row with the most recent data.
+              #
+              # Note that we are only getting events with period != 'minute' and
+              # service = master. This is what is required for the dashboard project.
+              # We will need to change this when we start importing data to a
+              # Redshift cluster used as a source for the stats API.
+              def fill_table_unique_imported
+                "INSERT INTO #{TABLES[:unique_imported_events]} "\
+                  'SELECT e.service, e.cinstance, e.uinstance, e.metric, e.period, '\
+                  'e.timestamp, e.time_gen, e.value '\
+                  'FROM '\
+                  '(SELECT service, cinstance, uinstance, metric, period, '\
+                  'MAX(time_gen) AS max_time_gen, timestamp '\
+                  "FROM #{TABLES[:temp]} "\
+                  "WHERE period != 'minute' AND service = '#{master_service}' "\
+                  'GROUP BY service, cinstance, uinstance, metric, period, timestamp) AS e1 '\
+                  "INNER JOIN #{TABLES[:temp]} e "\
+                  "ON #{join_comparisons('e', 'e1', JOIN_EVENT_ATTRS)} "\
+                  'AND e.time_gen = e1.max_time_gen ' \
+                  'GROUP BY e.service, e.cinstance, e.uinstance, e.metric, e.period, '\
+                  'e.timestamp, e.time_gen, e.value'.freeze
+              end
+
+              # Once we have imported some events and have made sure that we have
+              # selected only the ones that are more recent, we need to delete the
+              # ones that do not need to be imported. Those are the ones that have
+              # a time_gen older than that of the same event in the events table.
+              def delete_outdated_from_unique_imported
+                "DELETE FROM #{TABLES[:unique_imported_events]} "\
+                  'USING (SELECT * '\
+                  "FROM #{TABLES[:events]} e "\
+                  'WHERE e.time_gen >= (SELECT MIN(time_gen) '\
+                  "FROM #{TABLES[:unique_imported_events]})) AS e "\
+                  "WHERE #{join_comparisons(
+                TABLES[:unique_imported_events], 'e', JOIN_EVENT_ATTRS)} "\
+                "AND (#{TABLES[:unique_imported_events]}.time_gen <= e.time_gen);".freeze
+              end
+
+              def import_s3_path(path, access_key_id, secret_access_key)
+                "COPY #{TABLES[:temp]} "\
+                  "FROM '#{path}' "\
+                  "CREDENTIALS '#{amazon_credentials(access_key_id,
+                                                 secret_access_key)}' "\
+                                                 "FORMAT AS JSON 'auto' "\
+                                                 "TIMEFORMAT 'auto';"
+              end
+
+              def delete_nulls_from_imported
+                attrs_with_nulls = %w(cinstance uinstance)
+                attrs_with_nulls.map do |attr|
+                  replace_nulls(TABLES[:temp], attr, '')
+                end.join(' ')
+              end
+
+              def store_timestamp_read(timestamp)
+                "DELETE FROM #{TABLES[:latest_s3_path_read]}; "\
+                  "INSERT INTO #{TABLES[:latest_s3_path_read]} VALUES ('#{timestamp}');"
+              end
+
+              def duplicated_events
+                'SELECT COUNT(*) '\
+                  'FROM (SELECT COUNT(*) AS count '\
+                  "FROM #{TABLES[:events]} "\
+                  "GROUP BY #{JOIN_EVENT_ATTRS.join(',')}) AS group_counts "\
+                  'WHERE group_counts.count > 1;'
+              end
+
+              private
+
+              def amazon_credentials(access_key_id, secret_access_key)
+                "aws_access_key_id=#{access_key_id};"\
+                  "aws_secret_access_key=#{secret_access_key}"
+              end
+
+              def replace_nulls(table, attr, value)
+                "UPDATE #{table} "\
+                  "SET #{attr} = '#{value}' "\
+                  "WHERE #{attr} IS NULL;"
+              end
+
+              # Given 2 tables and an array of attributes, generates a string
+              # like this:
+              # table1.attr1 = table2.attr1 AND table1.attr2 = table2.attr2 AND ...
+              # This is helpful to build the WHERE clauses of certain JOINs.
+              def join_comparisons(table1, table2, attrs)
+                attrs.map do |attr|
+                  "#{table1}.#{attr} = #{table2}.#{attr}"
+                end.join(' AND ') + ' '
+              end
+
+              def master_service
+                Backend.configuration.master_service_id
+              end
+
+            end
+          end
+
+          # This private class is the responsible for calculating the S3 paths
+          # that we have not imported to Redshift yet.
+          class S3EventPaths
+
+            # The events in our S3 bucket are classified in paths.
+            # Paths are created every hour.
+            DIR_CREATION_INTERVAL = 60*60
+            private_constant :DIR_CREATION_INTERVAL
+
+            # When we read a path we want to be sure that no more events will be stored
+            # For that reason, we will wait a few minutes after the hour ends just to
+            # be safe. For example, we will not read the path '2016/02/25/00' until
+            # 2016-02-25 01:00 + DIR_BACKUP_TIME_S
+            DIR_BACKUP_TIME_S = 60*10
+            private_constant :DIR_BACKUP_TIME_S
+
+            class << self
+
+              def pending_paths(latest_read)
+                time_now = Time.now.utc
+                start_time = DateTime.parse(latest_read).to_time.utc + DIR_CREATION_INTERVAL
+
+                (start_time.to_i..time_now.to_i).step(DIR_CREATION_INTERVAL).inject([]) do |res, time|
+                  t = Time.at(time)
+                  break res unless can_get_events?(time_now, t)
+                  res << t.utc
+                end
+              end
+
+              private
+
+              def can_get_events?(now, time)
+                now - time > DIR_CREATION_INTERVAL + DIR_BACKUP_TIME_S
+              end
+
+            end
+
+          end
+          private_constant :S3EventPaths
+
+          S3_BUCKET = 'backend-events'.freeze
+          private_constant :S3_BUCKET
+
+          S3_EVENTS_BASE_PATH = "s3://#{S3_BUCKET}/".freeze
+          private_constant :S3_EVENTS_BASE_PATH
+
+          REQUIRED_TABLES = [SQL::TABLES[:events],
+                             SQL::TABLES[:latest_s3_path_read]].freeze
+          private_constant :REQUIRED_TABLES
+
+          MissingRequiredTables = Class.new(ThreeScale::Backend::Error)
+          MissingLatestS3PathRead = Class.new(ThreeScale::Backend::Error)
+
+          class << self
+
+            def insert_pending_events(silent = false)
+              check_redshift_tables
+
+              pending_times_utc = S3EventPaths.pending_paths(latest_timestamp_read)
+              pending_times_utc.each do |pending_time_utc|
+                puts "Loading events generated in hour: #{pending_time_utc}" unless silent
+                save_in_redshift(s3_path(pending_time_utc))
+                save_latest_read(pending_time_utc)
+              end
+              pending_times_utc.last
+            end
+
+            # This method import a specific S3 path into Redshift.
+            # Right now, its main use case consists of uploading past events to
+            # a path and importing only that path.
+            def insert_path(path)
+              # Need to check that the 'events' table exists. Do not care about
+              # 'latest_s3_path_read' in this case.
+              unless existing_tables_with_schema.include?(SQL::TABLES[:events])
+                raise MissingRequiredTables, 'Events table is missing'
+              end
+
+              save_in_redshift("#{S3_EVENTS_BASE_PATH}#{path}")
+            end
+
+            # Returns a timestamp with format 'YYYYMMDDHH' or nil if the latest
+            # timestamp read does not exist in the DB.
+            def latest_timestamp_read
+              query_result = execute_command(SQL::LATEST_TIMESTAMP_READ)
+              return nil if query_result.ntuples == 0
+              query_result.first['s3_path']
+            end
+
+            # Returns whether the data in the DB is consistent. Right now, this
+            # method only checks if there are duplicated events, but it could be
+            # extended in the future.
+            def consistent_data?
+              execute_command(SQL::duplicated_events).first['count'].to_i.zero?
+            end
+
+            private
+
+            def config
+              Backend.configuration
+            end
+
+            def redshift_config
+              config.redshift.to_h
+            end
+
+            def redshift_connection
+              @connection ||= PGconn.new(redshift_config)
+            end
+
+            def execute_command(command)
+              redshift_connection.exec(command)
+            end
+
+            def check_redshift_tables
+              unless required_tables_exist?
+                raise MissingRequiredTables, 'Some of the required tables are not in Redshift.'
+              end
+
+              unless latest_timestamp_read_exists?
+                raise MissingLatestS3PathRead,
+                  "The 'latest read' table does not contain any values"
+              end
+            end
+
+            def existing_tables
+              execute_command(SQL::EXISTING_TABLES).map { |row| row['table_name'] }
+            end
+
+            def existing_tables_with_schema
+              existing_tables.map { |table| "#{SQL::SCHEMA}.#{table}" }
+            end
+
+            def required_tables_exist?
+              db_tables_with_schema = existing_tables_with_schema
+              REQUIRED_TABLES.all? do |required_table|
+                db_tables_with_schema.include?(required_table)
+              end
+            end
+
+            def save_in_redshift(path)
+              import_s3_path(path)
+              [SQL.delete_nulls_from_imported,
+               SQL.fill_table_unique_imported,
+               SQL.delete_outdated_from_unique_imported,
+               SQL.insert_imported_events,
+               SQL::CLEAN_TEMP_TABLES,
+               SQL::VACUUM].each { |command| execute_command(command) }
+            end
+
+            def save_latest_read(time_utc)
+              execute_command(SQL.store_timestamp_read(time_utc.strftime('%Y%m%d%H')))
+            end
+
+            def import_s3_path(path)
+              execute_command(SQL::CREATE_TEMP_TABLES)
+              execute_command(SQL.import_s3_path(
+                path, config.aws_access_key_id, config.aws_secret_access_key))
+            end
+
+            def latest_timestamp_read_exists?
+              execute_command(SQL::LATEST_TIMESTAMP_READ).ntuples > 0
+            end
+
+            def s3_path(time_utc)
+              "#{S3_EVENTS_BASE_PATH}#{time_utc.strftime('%Y/%m/%d/%H')}"
+            end
+
+          end
+
+        end
+      end
+    end
+  end
+end