simple_map_reduce 0.1.0

data/lib/simple_map_reduce/server/job_worker.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'sinatra'
+require 'sinatra/json'
+require 'sinatra/reloader' if development?
+
+module SimpleMapReduce
+  module Server
+    class JobWorker < Sinatra::Base
+      configure do
+        use Rack::Lock
+      end
+      configure :development do
+        register Sinatra::Reloader
+      end
+
+      post '/map_tasks' do
+        raw_body = request.body.read
+        job = SimpleMapReduce::Server::Job.deserialize(raw_body)
+        self.class.job_manager.enqueue_job!(SimpleMapReduce::Worker::RunMapTaskWorker, args: [job, self.class.worker_id])
+
+        json(succeeded: true, job_id: job.id)
+      end
+
+      post '/reduce_tasks' do
+        raw_body = request.body.read
+        task = SimpleMapReduce::Server::Task.deserialize(raw_body)
+
+        self.class.job_manager.enqueue_job!(SimpleMapReduce::Worker::RunReduceTaskWorker, args: [task, self.class.worker_id])
+
+        json(succeeded: true, job_id: task.job_id, task_id: task.id)
+      end
+
+      class << self
+        attr_accessor :worker_id
+
+        def setup_worker
+          check_s3_access
+          register_myself_to_job_tracker
+          job_manager
+          logger.info('All setup process is done successfully. This worker is operation ready.')
+          logger.info("This job worker url: #{SimpleMapReduce.job_worker_url}, id: #{worker_id}")
+          logger.info("The job tracker url: #{SimpleMapReduce.job_tracker_url}")
+        end
+
+        def check_s3_access
+          s3_client.list_buckets
+          logger.info('[OK] s3 connection test')
+        end
+
+        def register_myself_to_job_tracker
+          response = http_client.post do |request|
+            request.url('/workers')
+            request.body = { url: SimpleMapReduce.job_worker_url }.to_json
+          end
+
+          body = JSON.parse(response.body, symbolize_names: true)
+          self.worker_id = body[:id]
+          logger.info("[OK] registering this worker to the job_tracker #{SimpleMapReduce.job_worker_url}")
+        end
+
+        def job_manager
+          @job_manager ||= ::Rasteira::EmbedWorker::Manager.run
+        end
+
+        def http_client
+          @http_client ||= ::Faraday.new(
+            url: SimpleMapReduce.job_tracker_url,
+            headers: {
+              'Accept' => 'application/json',
+                'Content-Type' => 'application/json'
+            }
+          ) do |faraday|
+            faraday.response :raise_error
+            faraday.adapter  Faraday.default_adapter
+          end
+        end
+
+        def s3_client
+          SimpleMapReduce::S3Client.instance.client
+        end
+
+        def logger
+          SimpleMapReduce.logger
+        end
+
+        # @override
+        def quit!
+          job_manager.shutdown_workers!
+          super
+        end
+      end
+    end
+  end
+end

data/lib/simple_map_reduce/server/task.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require 'msgpack'
+require 'securerandom'
+
+module SimpleMapReduce
+  module Server
+    class Task
+      extend Forwardable
+      include AASM
+
+      attr_reader :job_id,
+                  :task_class_name, :task_script,
+                  :task_input_bucket_name, :task_input_file_path,
+                  :task_output_bucket_name, :task_output_directory_path,
+                  :worker
+
+      delegate current_state: :aasm
+      alias state current_state
+
+      aasm do
+        state :ready, initial: true
+        state :in_process
+        state :succeeded
+        state :failed
+
+        event :start do
+          transitions ready: :in_process
+        end
+
+        event :succeed do
+          transitions from: :in_process, to: :succeeded
+        end
+
+        event :fail do
+          transitions from: :in_process, to: :failed
+        end
+      end
+
+      def initialize(id: nil,
+                     job_id:,
+                     task_class_name:,
+                     task_script:,
+                     task_input_bucket_name:,
+                     task_input_file_path:,
+                     task_output_bucket_name:,
+                     task_output_directory_path:,
+                     worker: nil)
+        @id = id
+        @job_id = job_id
+        @task_class_name = task_class_name
+        @task_script = task_script
+        @task_input_bucket_name = task_input_bucket_name
+        @task_input_file_path = task_input_file_path
+        @task_output_bucket_name = task_output_bucket_name
+        @task_output_directory_path = task_output_directory_path
+        @worker = worker
+      end
+
+      def id
+        @id ||= SecureRandom.uuid
+      end
+
+      def to_h
+        {
+          id: id,
+          job_id: @job_id,
+          task_class_name: @task_class_name,
+          task_script: @task_script,
+          task_input_bucket_name: @task_input_bucket_name,
+          task_input_file_path: @task_input_file_path,
+          task_output_bucket_name: @task_output_bucket_name,
+          task_output_directory_path: @task_output_directory_path
+        }
+      end
+
+      def serialize
+        to_h.to_msgpack
+      end
+
+      def dump
+        to_h.merge(state: state)
+      end
+
+      class << self
+        def deserialize(data)
+          new(Hash[MessagePack.unpack(data).map { |k, v| [k.to_sym, v] }])
+        end
+      end
+    end
+  end
+end

data/lib/simple_map_reduce/server/worker.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+require 'forwardable'
+require 'aasm'
+
+module SimpleMapReduce
+  module Server
+    class Worker
+      extend Forwardable
+      include AASM
+
+      attr_accessor :url
+
+      delegate current_state: :aasm
+      alias state current_state
+
+      aasm do
+        state :ready, initial: true
+        state :reserved
+        state :working
+
+        event :ready do
+          transitions to: :ready
+        end
+
+        event :reserve do
+          transitions from: %i(ready working), to: :reserved
+        end
+
+        event :work do
+          transitions from: :reserved, to: :working
+        end
+      end
+
+      def initialize(url:)
+        @url = url
+        unless valid?
+          raise ArgumentError, 'invalid url'
+        end
+      end
+
+      def id
+        @id ||= SecureRandom.uuid
+      end
+
+      def dump
+        {
+          id: id,
+          url: @url,
+          state: state
+        }
+      end
+
+      # update Job
+      # @params [Hash] attributes
+      # @options attributes [String] url
+      # @options attributes [String] event
+      def update!(url: nil, event: nil)
+        if url
+          self.url = url
+        end
+
+        if event
+          public_send(event.to_sym)
+        end
+      end
+
+      private
+
+      def valid?
+        !@url.to_s.empty? && @url =~ URI::DEFAULT_PARSER.make_regexp
+      end
+    end
+  end
+end

data/lib/simple_map_reduce/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module SimpleMapReduce
+  VERSION = '0.1.0'
+end

data/lib/simple_map_reduce/worker/register_map_task_worker.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module SimpleMapReduce
+  module Worker
+    class RegisterMapTaskWorker
+      def perform(job)
+        logger.info('register map task worker start!')
+        client = http_client(job.map_worker.url)
+        response = client.post do |request|
+          request.url('/map_tasks')
+          request.body = job.serialize
+        end
+        logger.debug(response.body)
+
+        job.map_worker.work!
+        job.start!
+      rescue => e
+        logger.error(e.inspect)
+        logger.error(e.backtrace.take(50))
+        SimpleMapReduce::Server::JobTracker.store_worker(job.map_worker)
+        job.failed!
+      end
+
+      private
+
+      HTTP_MSGPACK_HEADER = {
+        'Accept' => 'application/x-msgpack',
+        'Content-Type' => 'application/x-msgpack'
+      }.freeze
+
+      def http_client(url)
+        ::Faraday.new(
+          url: url,
+          headers: HTTP_MSGPACK_HEADER,
+          request: {
+            open_timeout: 10,
+            timeout: 15
+          }
+        ) do |faraday|
+          faraday.response :logger
+          faraday.response :raise_error
+          faraday.adapter  Faraday.default_adapter
+        end
+      end
+
+      def logger
+        SimpleMapReduce.logger
+      end
+    end
+  end
+end

data/lib/simple_map_reduce/worker/run_map_task_worker.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+module SimpleMapReduce
+  module Worker
+    class RunMapTaskWorker
+      def perform(job, map_worker_id)
+        task_wrapper_class_name = "TaskWrapper#{job.id.delete('-')}"
+        self.class.class_eval("class #{task_wrapper_class_name}; end", 'Task Wrapper Class')
+        task_wrapper_class = self.class.const_get(task_wrapper_class_name)
+        task_wrapper_class.class_eval(job.map_script, 'Map task script')
+        map_task = task_wrapper_class.const_get(job.map_class_name, false).new
+        unless map_task.respond_to?(:map)
+          # TODO: notify job_tracker
+          logger.error('no map method')
+          return
+        end
+        logger.info('map task start')
+
+        local_input_cache = Tempfile.new
+        s3_client.get_object(
+          response_target: local_input_cache.path,
+          bucket: job.job_input_bucket_name,
+          key: job.job_input_directory_path
+        )
+        local_input_cache.rewind
+
+        local_output_cache = Tempfile.new
+        local_input_cache.each_line(chomp: true, rs: "\n") do |line|
+          map_task.map(line, local_output_cache)
+        end
+
+        local_output_cache.rewind
+        logger.debug("output data size: #{local_output_cache.size}")
+        logger.debug('---map output digest---')
+        local_output_cache.take(5).each do |line|
+          logger.debug(line)
+        end
+        logger.debug('---map output digest---')
+
+        response = http_client(SimpleMapReduce.job_tracker_url).post do |request|
+          request.url('/workers/reserve')
+          # TODO: providing a way to specify worker_size
+          request.body = { worker_size: 2 }.to_json
+        end
+        logger.debug(response.body)
+
+        # {"succeeded":true,"workers":[{"id":70157882164440,"url":"http://localhost:4569","state":'reserved'}]}
+        reserved_workers = JSON.parse(response.body, symbolize_names: true)[:reserved_workers]
+        if reserved_workers.count == 0
+          # keep working with same worker
+          reserved_workers << { id: map_worker_id, url: job.map_worker_url, state: 'working' }
+        end
+
+        shuffle(job, reserved_workers, local_output_cache)
+
+        unless reserved_workers.map { |w| w[:id] }.include?(map_worker_id)
+          response = http_client(SimpleMapReduce.job_tracker_url).put do |request|
+            request.url("/workers/#{map_worker_id}")
+            request.body = { event: 'ready' }.to_json
+          end
+          logger.debug(response.body)
+        end
+      rescue => e
+        logger.error(e.inspect)
+        logger.error(e.backtrace.take(50))
+        # TODO: notifying to job_tracker that this task have failed
+      ensure
+        local_input_cache&.delete
+        local_output_cache&.delete
+        reserved_workers&.each do |worker|
+          worker[:shuffled_local_output]&.delete
+        end
+        if self.class.const_defined?(task_wrapper_class_name.to_sym)
+          self.class.send(:remove_const, task_wrapper_class_name.to_sym)
+        end
+        logger.info('map task end')
+      end
+
+      private
+
+      def s3_client
+        SimpleMapReduce::S3Client.instance.client
+      end
+
+      def logger
+        SimpleMapReduce.logger
+      end
+
+      HTTP_JSON_HEADER = {
+        'Accept' => 'application/x-msgpack',
+        'Content-Type' => 'application/x-msgpack'
+      }.freeze
+
+      def http_client(url)
+        ::Faraday.new(
+          url: url,
+          headers: HTTP_JSON_HEADER
+        ) do |faraday|
+          faraday.response :logger
+          faraday.response :raise_error
+          faraday.adapter  Faraday.default_adapter
+        end
+      end
+
+      def shuffle(job, workers, local_output_cache)
+        workers_count = workers.count
+        raise 'No workers' unless workers_count > 0
+
+        workers.each do |worker|
+          worker[:shuffled_local_output] = Tempfile.new
+        end
+
+        local_output_cache.each_line(rs: "\n") do |raw_line|
+          output = JSON.parse(raw_line, symbolize_names: true)
+          partition_id = output[:key].hash % workers_count
+          workers[partition_id][:shuffled_local_output].puts(output.to_json)
+        end
+
+        task_script = job.reduce_script
+        task_class_name = job.reduce_class_name
+        task_input_bucket_name = SimpleMapReduce.s3_intermediate_bucket_name
+        task_output_bucket_name = job.job_output_bucket_name
+        task_output_directory_path = job.job_output_directory_path
+        task_input_file_path_prefix = "#{job.id}/map_output_#{Time.now.to_i}/"
+
+        workers.each_with_index do |worker, partition_id|
+          reduce_task = ::SimpleMapReduce::Server::Task.new(
+            job_id: job.id,
+            task_class_name: task_class_name,
+            task_script: task_script,
+            task_input_bucket_name: task_input_bucket_name,
+            task_input_file_path: "#{task_input_file_path_prefix}#{partition_id}_map_output.txt",
+            task_output_bucket_name: task_output_bucket_name,
+            task_output_directory_path: task_output_directory_path
+          )
+
+          local_output_cache = worker[:shuffled_local_output]
+          local_output_cache.rewind
+          s3_client.put_object(
+            body: local_output_cache.read,
+            bucket: reduce_task.task_input_bucket_name,
+            key: reduce_task.task_input_file_path
+          )
+
+          response = http_client(worker[:url]).post do |request|
+            request.url('/reduce_tasks')
+            request.body = reduce_task.serialize
+          end
+          logger.debug(response.body)
+
+          next if worker[:state] == 'working'
+          response = http_client(SimpleMapReduce.job_tracker_url).put do |request|
+            request.url("/workers/#{worker[:id]}")
+            request.body = { event: 'work' }.to_json
+          end
+          logger.debug(response.body)
+        end
+      end
+    end
+  end
+end