bricolage-streamingload 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ed2342a1e0f9db4cbe53fbb97be7b72dcd362def
+  data.tar.gz: 1d1705ad9811bb4becee2de4168bb7f40f9a0b63
+SHA512:
+  metadata.gz: 9a09e1ec19569e03a13e01e8d2aa767de54bd21ac22cb455c9ccb92798a410f655f3435b4b0e7ff4e8bde429dcf2fd904a75f5f3d303f7bf815cfa0ca08b5631
+  data.tar.gz: 3cfb076a0144c04592db35309236e2bfce5021525e34fe26b32e5f61ef000572d8d6641bdbee078f741a0f79a85d9ef9fd4cc47ce167338cec1ad4577c051bd2

data/README.md

@@ -0,0 +1,19 @@
+# Bricolage Streaming Load
+
+Streaming load daemon based on Bricolage.
+
+This software is written in working time in Cookpad, Inc.
+
+## License
+
+MIT license.
+See LICENSES file for details.
+
+## Running Test
+
+% rake test
+
+## Author
+
+- Minero Aoki
+- Shimpei Kodama

data/bin/bricolage-streaming-dispatcher

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+Bundler.require(:default) if defined?(Bundler)
+require 'bricolage/streamingload/dispatcher'
+
+Bricolage::StreamingLoad::Dispatcher.main

data/bin/bricolage-streaming-loader

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+Bundler.require(:default) if defined?(Bundler)
+require 'bricolage/streamingload/loaderservice'
+
+Bricolage::StreamingLoad::LoaderService.main

data/lib/bricolage/sqsdatasource.rb

@@ -0,0 +1,299 @@
+require 'bricolage/datasource'
+require 'bricolage/sqswrapper'
+require 'securerandom'
+require 'aws-sdk'
+require 'json'
+require 'time'
+
+module Bricolage
+
+  class SQSDataSource < DataSource
+
+    declare_type 'sqs'
+
+    def initialize(region: 'ap-northeast-1', url:, access_key_id:, secret_access_key:,
+        visibility_timeout:, max_number_of_messages: 10, max_delete_batch_size: 10, wait_time_seconds: 20, noop: false)
+      @region = region
+      @url = url
+      @access_key_id = access_key_id
+      @secret_access_key = secret_access_key
+      @visibility_timeout = visibility_timeout
+      @max_number_of_messages = max_number_of_messages
+      @max_delete_batch_size = max_delete_batch_size
+      @wait_time_seconds = wait_time_seconds
+      @noop = noop
+    end
+
+    attr_reader :region
+    attr_reader :url
+    attr_reader :access_key_id
+    attr_reader :secret_access_key
+
+    def client
+      @client ||= begin
+        c = @noop ? DummySQSClient.new : Aws::SQS::Client.new(region: @region, access_key_id: @access_key_id, secret_access_key: @secret_access_key)
+        SQSClientWrapper.new(c, logger: logger)
+      end
+    end
+
+    #
+    # High-Level Polling Interface
+    #
+
+    def main_handler_loop(handlers)
+      trap_signals
+
+      n_zero = 0
+      until terminating?
+        insert_handler_wait(n_zero)
+        n_msg = handle_messages(handlers)
+        if n_msg == 0
+          n_zero += 1
+        else
+          n_zero = 0
+        end
+      end
+      @delete_message_buffer.flush if @delete_message_buffer
+      logger.info "shutdown gracefully"
+    end
+
+    def trap_signals
+      # Allows graceful stop
+      Signal.trap(:TERM) {
+        initiate_terminate
+      }
+      Signal.trap(:INT) {
+        initiate_terminate
+      }
+    end
+
+    def initiate_terminate
+      # No I/O allowed in this method
+      @terminating = true
+    end
+
+    def terminating?
+      @terminating
+    end
+
+    def insert_handler_wait(n_zero)
+      sec = 2 ** [n_zero, 6].min   # max 64s
+      logger.info "queue wait: sleep #{sec}" if n_zero > 0
+      sleep sec
+    end
+
+    def handle_messages(handlers:, message_class:)
+      n_msg = foreach_message(message_class) do |msg|
+        logger.debug "handling message: #{msg.inspect}" if logger.debug?
+        mid = "handle_#{msg.message_type}"
+        # just ignore unknown event to make app migration easy
+        if handlers.respond_to?(mid, true)
+          handlers.__send__(mid, msg)
+        else
+          logger.error "unknown SQS message type: #{msg.message_type.inspect} (message-id: #{msg.message_id})"
+        end
+      end
+      n_msg
+    end
+
+    def foreach_message(message_class, &block)
+      result = receive_messages()
+      unless result and result.successful?
+        logger.error "ReceiveMessage failed: #{result ? result.error.message : '(result=nil)'}"
+        return nil
+      end
+      logger.info "receive #{result.messages.size} messages" unless result.messages.empty?
+      msgs = message_class.for_sqs_result(result)
+      msgs.each(&block)
+      msgs.size
+    end
+
+    #
+    # API-Level Interface
+    #
+
+    def receive_messages
+      result = client.receive_message(
+        queue_url: @url,
+        attribute_names: ["All"],
+        message_attribute_names: ["All"],
+        max_number_of_messages: @max_number_of_messages,
+        visibility_timeout: @visibility_timeout,
+        wait_time_seconds: @wait_time_seconds
+      )
+      result
+    end
+
+    def delete_message(msg)
+      # TODO: use batch request?
+      client.delete_message(
+        queue_url: @url,
+        receipt_handle: msg.receipt_handle
+      )
+    end
+
+    def buffered_delete_message(msg)
+      delete_message_buffer.put(msg)
+    end
+
+    def delete_message_buffer
+      @delete_message_buffer ||= DeleteMessageBuffer.new(client, @url, @max_delete_batch_size, logger)
+    end
+
+    def put(msg)
+      send_message(msg)
+    end
+
+    def send_message(msg)
+      client.send_message(
+        queue_url: @url,
+        message_body: { 'Records' => [msg.body] }.to_json,
+        delay_seconds: msg.delay_seconds
+      )
+    end
+
+    class DeleteMessageBuffer
+
+      def initialize(sqs_client, url, max_buffer_size, logger)
+        @sqs_client = sqs_client
+        @url = url
+        @max_buffer_size = max_buffer_size
+        @logger = logger
+        @buf = {}
+        @retry_counts = Hash.new(0)
+      end
+
+      MAX_RETRY_COUNT = 3
+
+      def put(msg)
+        @buf[SecureRandom.uuid] = msg
+        flush if size >= @max_buffer_size
+      end
+
+      def size
+        @buf.size
+      end
+
+      def flush
+        return unless size > 0
+        response = @sqs_client.delete_message_batch({
+          queue_url: @url,
+          entries: @buf.to_a.map {|item| {id: item[0], receipt_handle: item[1].receipt_handle} }
+        })
+        clear_successes(response.successful)
+        retry_failures(response.failed)
+        @logger.debug "DeleteMessageBatch executed: #{response.successful.size} succeeded, #{response.failed.size} failed."
+      end
+
+      private
+
+      def clear_successes(successes)
+        successes.each do |s|
+          @buf.delete s.id
+        end
+      end
+
+      def retry_failures(failures)
+        return unless failures.size > 0
+        failures.each do |f|
+          @logger.info "DeleteMessageBatch failed to retry for: id=#{f.id}, sender_fault=#{f.sender_fault}, code=#{f.code}, message=#{f.message}"
+        end
+        flush
+        @buf.keys.map {|k| @retry_counts[k] += 1 }
+        @retry_counts.select {|k, v| v >= MAX_RETRY_COUNT }.each do |k, v|
+          @logger.warn "DeleteMessageBatch failed #{MAX_RETRY_COUNT} times for: message_id=#{@buf[k].message_id}, receipt_handle=#{@buf[k].receipt_handle}"
+          @buf.delete k
+          @retry_counts.delete k
+        end
+      end
+
+    end # DeleteMessageBuffer
+
+  end # class SQSDataSource
+
+
+  class SQSMessage
+
+    SQS_EVENT_SOURCE = 'bricolage:system'
+
+    # Writer interface
+    def SQSMessage.create(
+        name:,
+        time: Time.now.getutc,
+        source: SQS_EVENT_SOURCE,
+        delay_seconds: 0,
+        **message_params)
+      new(name: name, time: time, source: source, delay_seconds: delay_seconds, **message_params)
+    end
+
+    def SQSMessage.for_sqs_result(result)
+      result.messages.flat_map {|msg|
+        body = JSON.parse(msg.body)
+        records = body['Records'] or next []
+        records.map {|rec| get_concrete_class(msg, rec).for_sqs_record(msg, rec) }
+      }
+    end
+
+    # abstract SQSMessage.get_concrete_class(msg, rec)
+
+    def SQSMessage.for_sqs_record(msg, rec)
+      new(** SQSMessage.parse_sqs_record(msg, rec).merge(parse_sqs_record(msg, rec)))
+    end
+
+    def SQSMessage.parse_sqs_record(msg, rec)
+      time_str = rec['eventTime']
+      tm = time_str ? (Time.parse(time_str) rescue nil) : nil
+      {
+        message_id: msg.message_id,
+        receipt_handle: msg.receipt_handle,
+        name: rec['eventName'],
+        time: tm,
+        source: rec['eventSource']
+      }
+    end
+
+    def initialize(name:, time:, source:,
+        message_id: nil, receipt_handle: nil, delay_seconds: nil,
+        **message_params)
+      @name = name
+      @time = time
+      @source = source
+
+      @message_id = message_id
+      @receipt_handle = receipt_handle
+
+      @delay_seconds = delay_seconds
+
+      init_message(**message_params)
+    end
+
+    # abstract init_message(**message_params)
+
+    attr_reader :name
+    attr_reader :time
+    attr_reader :source
+
+    # Valid only for received messages
+
+    attr_reader :message_id
+    attr_reader :receipt_handle
+
+    # Valid only for sending messages
+
+    attr_reader :delay_seconds
+
+    def body
+      obj = {}
+      [
+        ['eventName', @name],
+        ['eventTime', (@time ? @time.iso8601 : nil)],
+        ['eventSource', @source]
+      ].each do |name, value|
+        obj[name] = value if value
+      end
+      obj
+    end
+
+  end   # class SQSMessage
+
+end   # module Bricolage

data/lib/bricolage/sqswrapper.rb

@@ -0,0 +1,77 @@
+require 'json'
+
+module Bricolage
+
+  class SQSClientWrapper
+    def initialize(sqs, logger:)
+      @sqs = sqs
+      @logger = logger
+    end
+
+    def receive_message(**args)
+      @logger.debug "receive_message(#{args.inspect})"
+      @sqs.receive_message(**args)
+    end
+
+    def send_message(**args)
+      @logger.debug "send_message(#{args.inspect})"
+      @sqs.send_message(**args)
+    end
+
+    def delete_message(**args)
+      @logger.debug "delete_message(#{args.inspect})"
+      @sqs.delete_message(**args)
+    end
+
+    def delete_message_batch(**args)
+      @logger.debug "delete_message_batch(#{args.inspect})"
+      @sqs.delete_message_batch(**args)
+    end
+  end
+
+
+  class DummySQSClient
+    def initialize(queue = [])
+      @queue = queue
+    end
+
+    def receive_message(**args)
+      msg_recs = @queue.shift or return EMPTY_RESULT
+      msgs = msg_recs.map {|recs| Message.new({'Records' => recs}.to_json) }
+      Result.new(true, msgs)
+    end
+
+    def send_message(**args)
+      SUCCESS_RESULT
+    end
+
+    def delete_message(**args)
+      SUCCESS_RESULT
+    end
+
+    class Result
+      def initialize(successful, messages = nil)
+        @successful = successful
+        @messages = messages
+      end
+
+      def successful?
+        @successful
+      end
+
+      attr_reader :messages
+    end
+
+    SUCCESS_RESULT = Result.new(true)
+    EMPTY_RESULT = Result.new(true, [])
+
+    class Message
+      def initialize(body)
+        @body = body
+      end
+
+      attr_reader :body
+    end
+  end
+
+end   # module Bricolage

data/lib/bricolage/streamingload/dispatcher.rb

@@ -0,0 +1,181 @@
+require 'bricolage/exception'
+require 'bricolage/version'
+require 'bricolage/sqsdatasource'
+require 'bricolage/logger'
+require 'bricolage/streamingload/event'
+require 'bricolage/streamingload/objectbuffer'
+require 'bricolage/streamingload/urlpatterns'
+require 'aws-sdk'
+require 'yaml'
+require 'optparse'
+require 'fileutils'
+
+module Bricolage
+
+  module StreamingLoad
+
+    class Dispatcher
+
+      def Dispatcher.main
+        opts = DispatcherOptions.new(ARGV)
+        opts.parse
+        unless opts.rest_arguments.size == 1
+          $stderr.puts opts.usage
+          exit 1
+        end
+        config_path, * = opts.rest_arguments
+        config = YAML.load(File.read(config_path))
+        logger = opts.log_file_path ? new_logger(opts.log_file_path, config) : nil
+        ctx = Context.for_application('.', environment: opts.environment, logger: logger)
+        event_queue = ctx.get_data_source('sqs', config.fetch('event-queue-ds'))
+        task_queue = ctx.get_data_source('sqs', config.fetch('task-queue-ds'))
+
+        object_buffer = ObjectBuffer.new(
+          control_data_source: ctx.get_data_source('sql', config.fetch('ctl-postgres-ds')),
+          logger: ctx.logger
+        )
+
+        url_patterns = URLPatterns.for_config(config.fetch('url_patterns'))
+
+        dispatcher = Dispatcher.new(
+          event_queue: event_queue,
+          task_queue: task_queue,
+          object_buffer: object_buffer,
+          url_patterns: url_patterns,
+          dispatch_interval: 60,
+          logger: ctx.logger
+        )
+
+        Process.daemon(true) if opts.daemon?
+        create_pid_file opts.pid_file_path if opts.pid_file_path
+        dispatcher.set_dispatch_timer
+        dispatcher.event_loop
+      end
+
+      def Dispatcher.new_logger(path, config)
+        Logger.new(
+          device: path,
+          rotation_period: config.fetch('log-rotation-period', 'daily'),
+          rotation_size: config.fetch('log-rotation-size', nil)
+        )
+      end
+
+      def Dispatcher.create_pid_file(path)
+        File.open(path, 'w') {|f|
+          f.puts $$
+        }
+      rescue
+        # ignore
+      end
+
+      def initialize(event_queue:, task_queue:, object_buffer:, url_patterns:, dispatch_interval:, logger:)
+        @event_queue = event_queue
+        @task_queue = task_queue
+        @object_buffer = object_buffer
+        @url_patterns = url_patterns
+        @dispatch_interval = dispatch_interval
+        @dispatch_message_id = nil
+        @logger = logger
+      end
+
+      def event_loop
+        @event_queue.main_handler_loop(handlers: self, message_class: Event)
+      end
+
+      def handle_shutdown(e)
+        @event_queue.initiate_terminate
+        @event_queue.delete_message(e)
+      end
+
+      def handle_data(e)
+        unless e.created?
+          @event_queue.delete_message(e)
+          return
+        end
+        obj = e.loadable_object(@url_patterns)
+        @object_buffer.put(obj)
+        @event_queue.buffered_delete_message(e)
+      end
+
+      def handle_dispatch(e)
+        if @dispatch_message_id == e.message_id
+          tasks = @object_buffer.flush_tasks
+          tasks.each {|task| @task_queue.put task }
+          set_dispatch_timer
+        end
+        @event_queue.delete_message(e)
+      end
+
+      def set_dispatch_timer
+        resp = @event_queue.send_message DispatchEvent.create(delay_seconds: @dispatch_interval)
+        @dispatch_message_id = resp.message_id
+      end
+
+      def delete_events(events)
+        events.each do |e|
+          @event_queue.delete_message(e)
+        end
+      end
+
+    end
+
+
+    class DispatcherOptions
+
+      def initialize(argv)
+        @argv = argv
+        @daemon = false
+        @log_file_path = nil
+        @pid_file_path = nil
+        @rest_arguments = nil
+
+        @opts = opts = OptionParser.new("Usage: #{$0} CONFIG_PATH")
+        opts.on('--task-id=id', 'Execute oneshot load task (implicitly disables daemon mode).') {|task_id|
+          @task_id = task_id
+        }
+        opts.on('-e', '--environment=NAME', "Sets execution environment [default: #{Context::DEFAULT_ENV}]") {|env|
+          @environment = env
+        }
+        opts.on('--daemon', 'Becomes daemon in server mode.') {
+          @daemon = true
+        }
+        opts.on('--log-file=PATH', 'Log file path') {|path|
+          @log_file_path = path
+        }
+        opts.on('--pid-file=PATH', 'Creates PID file.') {|path|
+          @pid_file_path = path
+        }
+        opts.on('--help', 'Prints this message and quit.') {
+          puts opts.help
+          exit 0
+        }
+        opts.on('--version', 'Prints version and quit.') {
+          puts "#{File.basename($0)} version #{VERSION}"
+          exit 0
+        }
+      end
+
+      def usage
+        @opts.help
+      end
+
+      def parse
+        @opts.parse!(@argv)
+        @rest_arguments = @argv.dup
+      rescue OptionParser::ParseError => err
+        raise OptionError, err.message
+      end
+
+      attr_reader :rest_arguments, :environment, :log_file_path
+
+      def daemon?
+        @daemon
+      end
+
+      attr_reader :pid_file_path
+
+    end
+
+  end   # module StreamingLoad
+
+end   # module Bricolage