simple_map_reduce 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d3935042506d22ac79df4b198afd843a43e390da4271c9759aac3085d4089cb7
-  data.tar.gz: 63e0b638ccf73c436b06503b581d0d36e42b84ffb98fa00b39db5386a37fb0b3
+  metadata.gz: f96ea39ae49bed4226271a6ab428e2eb7258cc921704fb747839b8fe2c096ae3
+  data.tar.gz: b3e60d6060546c708811b9b893007cd3ff1816e4b951015cf83f4294ca822351
 SHA512:
-  metadata.gz: 4553f2db3b813c84ae47b38e1e101af879c85c34c58198a36ffc5c5b5ad44d359e3dd6b509a539f3f673cbb108c17ea5f3b846aa8e2e90a93b5c2f62635c6e53
-  data.tar.gz: 2bc3bde335613c60d77b90ca69052391c6842478cd317b60658c6e7aaf54f40637d7537cece15dc307a7407d49ee670b2c175cf0fd925bb1b99d91a586327515
+  metadata.gz: 16f03783a4eb7f7ed62987de371828d33debde02c8c0b23ec24c9c80e531f1a8053a1d6682542efd682e2f180237fb74e1ed29056d11eb6cba15b3af090cf4cc
+  data.tar.gz: 6f5e703dedf1c432bcc1e16a652ffa6713584d7d082b23b9253840e68ccdeef7167737ab8b51aef1c7c303743eaa501cd161b117d3d0672e8e9a76abf780471f

data/.github/ISSUE_TEMPLATE.md

@@ -0,0 +1 @@
+# What is this issue about ?

data/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1 @@
+# What will this PR change ?

data/README.md

@@ -1,3 +1,6 @@
+[![Build Status](https://travis-ci.org/serihiro/simple_map_reduce.svg?branch=master)](https://travis-ci.org/serihiro/simple_map_reduce)
+[![Gem Version](https://badge.fury.io/rb/simple_map_reduce.svg)](https://badge.fury.io/rb/simple_map_reduce)
+
 # SimpleMapReduce
 
 - This is a [MapReduce](https://research.google.com/archive/mapreduce.html) implementation distributed framework written in ruby.

data/lib/simple_map_reduce.rb

@@ -19,6 +19,9 @@ end
 
 require 'simple_map_reduce/version'
 require 'simple_map_reduce/s3_client'
+require 'simple_map_reduce/data_stores/default_data_store'
+require 'simple_map_reduce/data_stores/remote_data_store'
+require 'simple_map_reduce/data_store_factory'
 require 'simple_map_reduce/driver/config'
 require 'simple_map_reduce/driver/job'
 require 'simple_map_reduce/server/confg'
@@ -30,3 +33,4 @@ require 'simple_map_reduce/server/job_worker'
 require 'simple_map_reduce/worker/register_map_task_worker'
 require 'simple_map_reduce/worker/run_map_task_worker'
 require 'simple_map_reduce/worker/run_reduce_task_worker'
+require 'simple_map_reduce/worker/polling_workers_status_worker'

data/lib/simple_map_reduce/data_store_factory.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module SimpleMapReduce
+  class DataStoreFactory
+    TYPES = %w(default remote).freeze
+
+    class << self
+      def create(data_store_type, options = {})
+        unless TYPES.include?(data_store_type)
+          raise ArgumentError, "Unsupported data_store_type: `#{data_store_type}`"
+        end
+
+        case data_store_type
+        when 'default'
+          SimpleMapReduce::DataStores::DefaultDataStore.new(options)
+        when 'remote'
+          SimpleMapReduce::DataStores::RemoteDataStore.new(options)
+        end
+      end
+    end
+  end
+end

data/lib/simple_map_reduce/data_stores/default_data_store.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module SimpleMapReduce
+  module DataStores
+    class DefaultDataStore
+      def initialize(_options)
+        # do nothing
+      end
+
+      def save_state(_evnet)
+        # do nothing
+      end
+    end
+  end
+end

data/lib/simple_map_reduce/data_stores/remote_data_store.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module SimpleMapReduce
+  module DataStores
+    class RemoteDataStore
+      def initialize(options)
+        @resource_name = options[:resource_name]
+        @resource_id = options[:resource_id]
+        @server_url = options[:server_url]
+      end
+
+      def save_state(event)
+        http_client.put do |request|
+          request.url("/#{@resource_name}/#{@resource_id}")
+          request.body = { event: event }.to_json
+        end
+      end
+
+      private
+
+      HTTP_JSON_HEADER = {
+        'Accept' => 'application/json',
+        'Content-Type' => 'application/json'
+      }.freeze
+
+      def http_client
+        @http_client ||= ::Faraday.new(
+          url: @server_url,
+          headers: HTTP_JSON_HEADER,
+          request: {
+            open_timeout: 10,
+            timeout: 15
+          }
+        ) do |faraday|
+          faraday.response :logger
+          faraday.response :raise_error
+          faraday.adapter  Faraday.default_adapter
+        end
+      end
+    end
+  end
+end

data/lib/simple_map_reduce/server/job.rb

@@ -19,6 +19,8 @@ module SimpleMapReduce
       alias state current_state
 
       aasm do
+        before_all_events :save_state
+
         state :ready, initial: true
         state :in_process
         state :succeeded
@@ -47,7 +49,8 @@ module SimpleMapReduce
                      job_output_bucket_name:,
                      job_output_directory_path:,
                      map_worker_url: nil,
-                     map_worker: nil)
+                     map_worker: nil,
+                     data_store_type: 'default')
 
         @id = id
         @map_script = map_script&.strip
@@ -62,6 +65,10 @@ module SimpleMapReduce
         if @map_worker.nil? && map_worker_url
           @map_worker = SimpleMapReduce::Server::Worker.new(url: map_worker_url)
         end
+        @data_store = SimpleMapReduce::DataStoreFactory.create(data_store_type,
+                                                               server_url: SimpleMapReduce.job_tracker_url,
+                                                               resource_name: 'jobs',
+                                                               resource_id: @id)
 
         unless valid?
           raise ArgumentError, 'invalid Job parameters are detected'
@@ -121,9 +128,17 @@ module SimpleMapReduce
 
       class << self
         def deserialize(data)
-          new(Hash[MessagePack.unpack(data).map { |k, v| [k.to_sym, v] }])
+          params = Hash[MessagePack.unpack(data).map { |k, v| [k.to_sym, v] }]
+          params[:data_store_type] = 'remote'
+          new(params)
         end
       end
+
+      private
+
+      def save_state
+        @data_store.save_state(aasm.current_event)
+      end
     end
   end
 end

data/lib/simple_map_reduce/server/job_tracker.rb

@@ -173,6 +173,7 @@ module SimpleMapReduce
           check_s3_access
           create_s3_buckets_if_not_existing
           job_manager
+          start_polling_workers
           logger.info('All setup process is done successfully. The job tracker is operation ready.')
           logger.info("This job tracker url: #{SimpleMapReduce.job_tracker_url}")
         end
@@ -234,7 +235,7 @@ module SimpleMapReduce
         end
 
         def register_worker(url:)
-          worker = ::SimpleMapReduce::Server::Worker.new(url: url)
+          worker = ::SimpleMapReduce::Server::Worker.new(url: url, data_store_type: 'remote')
           if @workers.nil?
             @workers = {}
           end
@@ -255,9 +256,16 @@ module SimpleMapReduce
             ready_workers = ready_workers.keys.take(worker_size)
 
             ready_workers.map do |retry_worker_id|
-              @workers[retry_worker_id].reserve
-              @workers[retry_worker_id]
-            end
+              begin
+                @workers[retry_worker_id].reserve!
+              rescue => e
+                logger.error("Failed to transit the worker state: `#{@workers[retry_worker_id]}`")
+                logger.error(e.inspect)
+                nil
+              else
+                @workers[retry_worker_id]
+              end
+            end.compact
           else
             return []
           end
@@ -277,6 +285,22 @@ module SimpleMapReduce
           mutex.unlock
         end
 
+        POLLING_INTERVAL = 10
+
+        def start_polling_workers
+          @keep_polling_workers = true
+
+          @polling_workers_thread = Thread.new do
+            loop do
+              break unless @keep_polling_workers
+
+              job_manager.enqueue_job!(SimpleMapReduce::Worker::PollingWorkersStatusWorker, args: @workers || {})
+              sleep(POLLING_INTERVAL)
+            end
+          end
+          @polling_workers_thread.run
+        end
+
         def job_manager
           @job_manager ||= ::Rasteira::EmbedWorker::Manager.run
         end
@@ -295,6 +319,8 @@ module SimpleMapReduce
 
         # @override
         def quit!
+          @keep_polling_workers = false
+          @polling_workers_thread.kill
           job_manager.shutdown_workers!
           super
         end

data/lib/simple_map_reduce/server/job_worker.rb

@@ -18,7 +18,9 @@ module SimpleMapReduce
       post '/map_tasks' do
         raw_body = request.body.read
         job = SimpleMapReduce::Server::Job.deserialize(raw_body)
-        self.class.job_manager.enqueue_job!(SimpleMapReduce::Worker::RunMapTaskWorker, args: [job, self.class.worker_id])
+        self.class.worker.work!
+        job.start!
+        self.class.job_manager.enqueue_job!(SimpleMapReduce::Worker::RunMapTaskWorker, args: [job, self.class.worker])
 
         json(succeeded: true, job_id: job.id)
       end
@@ -26,14 +28,42 @@ module SimpleMapReduce
       post '/reduce_tasks' do
         raw_body = request.body.read
         task = SimpleMapReduce::Server::Task.deserialize(raw_body)
-
-        self.class.job_manager.enqueue_job!(SimpleMapReduce::Worker::RunReduceTaskWorker, args: [task, self.class.worker_id])
+        self.class.worker.work!
+        self.class.job_manager.enqueue_job!(SimpleMapReduce::Worker::RunReduceTaskWorker, args: [task, self.class.worker])
 
         json(succeeded: true, job_id: task.job_id, task_id: task.id)
       end
 
+      put '/workers/:id' do
+        body = JSON.parse(request.body.read, symbolize_names: true)
+        if params[:id] != self.class.worker_id
+          status 404
+          json(succeeded: false, error_message: 'The specified worker id was not found.')
+          return
+        end
+
+        begin
+          self.class.worker.update!(body)
+          json(succeeded: true, worker: self.class.worker.dump)
+        rescue => e
+          puts e.inspect
+          status 400
+          json(succeeded: false, error_class: e.class.to_s, error_message: e.message)
+        end
+      end
+
+      get '/workers/:id' do
+        if params[:id] != self.class.worker_id
+          status 404
+          json(succeeded: false, error_message: 'The specified worker id was not found.')
+        else
+          json(succeeded: true, worker: self.class.worker.dump)
+        end
+      end
+
       class << self
-        attr_accessor :worker_id
+        attr_reader :worker_id
+        attr_reader :worker
 
         def setup_worker
           check_s3_access
@@ -41,6 +71,7 @@ module SimpleMapReduce
           job_manager
           logger.info('All setup process is done successfully. This worker is operation ready.')
           logger.info("This job worker url: #{SimpleMapReduce.job_worker_url}, id: #{worker_id}")
+          logger.info("This job worker status url: #{SimpleMapReduce.job_worker_url}/workers/#{worker_id}")
           logger.info("The job tracker url: #{SimpleMapReduce.job_tracker_url}")
         end
 
@@ -56,7 +87,11 @@ module SimpleMapReduce
           end
 
           body = JSON.parse(response.body, symbolize_names: true)
-          self.worker_id = body[:id]
+          @worker_id = body[:id]
+          @worker = SimpleMapReduce::Server::Worker.new(
+            id: @worker_id,
+            url: SimpleMapReduce.job_worker_url
+          )
           logger.info("[OK] registering this worker to the job_tracker #{SimpleMapReduce.job_worker_url}")
         end
 

data/lib/simple_map_reduce/server/worker.rb

@@ -15,7 +15,11 @@ module SimpleMapReduce
       delegate current_state: :aasm
       alias state current_state
 
+      STATES = %i(ready reserved working).freeze
+
       aasm do
+        before_all_events :save_state
+
         state :ready, initial: true
         state :reserved
         state :working
@@ -29,12 +33,20 @@ module SimpleMapReduce
         end
 
         event :work do
-          transitions from: :reserved, to: :working
+          transitions from: %i(reserved working), to: :working
         end
       end
 
-      def initialize(url:)
+      def initialize(url:, id: nil, state: nil, data_store_type: 'default')
         @url = url
+        @id = id
+        if STATES.include?(state)
+          aasm_write_state_without_persistence(state)
+        end
+        @data_store = SimpleMapReduce::DataStoreFactory.create(data_store_type,
+                                                             server_url: url,
+                                                             resource_name: 'workers',
+                                                             resource_id: self.id)
         unless valid?
           raise ArgumentError, 'invalid url'
         end
@@ -71,6 +83,10 @@ module SimpleMapReduce
       def valid?
         !@url.to_s.empty? && @url =~ URI::DEFAULT_PARSER.make_regexp
       end
+
+      def save_state
+        @data_store.save_state(aasm.current_event)
+      end
     end
   end
 end

data/lib/simple_map_reduce/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SimpleMapReduce
-  VERSION = '0.1.0'
+  VERSION = '0.2.0'
 end

data/lib/simple_map_reduce/worker/polling_workers_status_worker.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module SimpleMapReduce
+  module Worker
+    class PollingWorkersStatusWorker
+      def perform(workers)
+        logger.debug("begin polling workers: #{workers.keys}")
+        workers.each do |id, worker|
+          begin
+            response = http_client(worker.url).get("/workers/#{worker.id}")
+            body = JSON.parse(response.body, symbolize_names: true)[:worker]
+            worker.aasm.current_state = body[:state].to_sym
+          rescue => e
+            logger.error(e.inspect)
+            logger.error(e&.response&.inspect)
+            logger.info("Worker #{worker.id} is removed from workers")
+            workers.delete(id)
+          end
+        end
+        logger.debug("finish polling workers: #{workers.keys}")
+      end
+
+      private
+
+      HTTP_JSON_HEADER = {
+        'Accept' => 'application/json',
+        'Content-Type' => 'application/json'
+      }.freeze
+
+      def http_client(url)
+        ::Faraday.new(
+          url: url,
+          headers: HTTP_JSON_HEADER,
+          request: {
+            open_timeout: 5,
+            timeout: 10
+          }
+        ) do |faraday|
+          faraday.response :logger
+          faraday.response :raise_error
+          faraday.adapter  Faraday.default_adapter
+        end
+      end
+
+      def logger
+        SimpleMapReduce.logger
+      end
+    end
+  end
+end

data/lib/simple_map_reduce/worker/register_map_task_worker.rb

@@ -11,9 +11,6 @@ module SimpleMapReduce
           request.body = job.serialize
         end
         logger.debug(response.body)
-
-        job.map_worker.work!
-        job.start!
       rescue => e
         logger.error(e.inspect)
         logger.error(e.backtrace.take(50))

data/lib/simple_map_reduce/worker/run_map_task_worker.rb

@@ -3,16 +3,16 @@
 module SimpleMapReduce
   module Worker
     class RunMapTaskWorker
-      def perform(job, map_worker_id)
+      class InvalidMapTaskError < StandardError; end
+
+      def perform(job, map_worker)
         task_wrapper_class_name = "TaskWrapper#{job.id.delete('-')}"
         self.class.class_eval("class #{task_wrapper_class_name}; end", 'Task Wrapper Class')
         task_wrapper_class = self.class.const_get(task_wrapper_class_name)
         task_wrapper_class.class_eval(job.map_script, 'Map task script')
         map_task = task_wrapper_class.const_get(job.map_class_name, false).new
         unless map_task.respond_to?(:map)
-          # TODO: notify job_tracker
-          logger.error('no map method')
-          return
+          raise InvalidMapTaskError, 'no map method'
         end
         logger.info('map task start')
 
@@ -45,31 +45,37 @@ module SimpleMapReduce
         logger.debug(response.body)
 
         # {"succeeded":true,"workers":[{"id":70157882164440,"url":"http://localhost:4569","state":'reserved'}]}
-        reserved_workers = JSON.parse(response.body, symbolize_names: true)[:reserved_workers]
+        reserved_workers = JSON.parse(response.body, symbolize_names: true)[:reserved_workers].map do |worker|
+          SimpleMapReduce::Server::Worker.new(
+            id: worker[:id],
+            url: worker[:url],
+            state: worker[:state].to_sym,
+            data_store_type: 'remote'
+          )
+        end
         if reserved_workers.count == 0
           # keep working with same worker
-          reserved_workers << { id: map_worker_id, url: job.map_worker_url, state: 'working' }
+          reserved_workers << map_worker
         end
 
         shuffle(job, reserved_workers, local_output_cache)
 
-        unless reserved_workers.map { |w| w[:id] }.include?(map_worker_id)
-          response = http_client(SimpleMapReduce.job_tracker_url).put do |request|
-            request.url("/workers/#{map_worker_id}")
-            request.body = { event: 'ready' }.to_json
+        if reserved_workers.all? { |w| w.id != map_worker.id }
+          begin
+            map_worker.ready!
+          rescue => notify_error
+            logger.fatal(notify_error.inspect)
+            logger.fatal(notify_error.backtrace.take(50))
           end
-          logger.debug(response.body)
         end
       rescue => e
         logger.error(e.inspect)
         logger.error(e.backtrace.take(50))
+        job.failed!
         # TODO: notifying to job_tracker that this task have failed
       ensure
         local_input_cache&.delete
         local_output_cache&.delete
-        reserved_workers&.each do |worker|
-          worker[:shuffled_local_output]&.delete
-        end
         if self.class.const_defined?(task_wrapper_class_name.to_sym)
           self.class.send(:remove_const, task_wrapper_class_name.to_sym)
         end
@@ -106,14 +112,11 @@ module SimpleMapReduce
         workers_count = workers.count
         raise 'No workers' unless workers_count > 0
 
-        workers.each do |worker|
-          worker[:shuffled_local_output] = Tempfile.new
-        end
-
+        shuffled_local_outputs = Array.new(workers_count, Tempfile.new)
         local_output_cache.each_line(rs: "\n") do |raw_line|
           output = JSON.parse(raw_line, symbolize_names: true)
           partition_id = output[:key].hash % workers_count
-          workers[partition_id][:shuffled_local_output].puts(output.to_json)
+          shuffled_local_outputs[partition_id].puts(output.to_json)
         end
 
         task_script = job.reduce_script
@@ -134,7 +137,7 @@ module SimpleMapReduce
             task_output_directory_path: task_output_directory_path
           )
 
-          local_output_cache = worker[:shuffled_local_output]
+          local_output_cache = shuffled_local_outputs[partition_id]
           local_output_cache.rewind
           s3_client.put_object(
             body: local_output_cache.read,
@@ -142,18 +145,15 @@ module SimpleMapReduce
             key: reduce_task.task_input_file_path
           )
 
-          response = http_client(worker[:url]).post do |request|
+          response = http_client(worker.url).post do |request|
             request.url('/reduce_tasks')
             request.body = reduce_task.serialize
           end
           logger.debug(response.body)
-
-          next if worker[:state] == 'working'
-          response = http_client(SimpleMapReduce.job_tracker_url).put do |request|
-            request.url("/workers/#{worker[:id]}")
-            request.body = { event: 'work' }.to_json
-          end
-          logger.debug(response.body)
+        end
+      ensure
+        shuffled_local_outputs&.each do |output|
+          output.delete
         end
       end
     end

data/lib/simple_map_reduce/worker/run_reduce_task_worker.rb

@@ -3,7 +3,7 @@
 module SimpleMapReduce
   module Worker
     class RunReduceTaskWorker
-      def perform(task, reduce_worker_id)
+      def perform(task, reduce_worker)
         task_wrapper_class_name = "TaskWrapper#{task.id.delete('-')}"
         self.class.class_eval("class #{task_wrapper_class_name}; end", 'Task Wrapper Class')
         task_wrapper_class = self.class.const_get(task_wrapper_class_name)
@@ -54,11 +54,7 @@ module SimpleMapReduce
         end
 
         begin
-          response = http_client(SimpleMapReduce.job_tracker_url).put do |request|
-            request.url("/workers/#{reduce_worker_id}")
-            request.body = { event: 'ready' }.to_json
-          end
-          logger.debug(response.body)
+          reduce_worker.ready!
         rescue => notify_error
           logger.fatal(notify_error.inspect)
           logger.fatal(notify_error.backtrace.take(50))

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: simple_map_reduce
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Kazuhiro Serizawa
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-01-08 00:00:00.000000000 Z
+date: 2018-02-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -228,6 +228,8 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/ISSUE_TEMPLATE.md"
+- ".github/PULL_REQUEST_TEMPLATE.md"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
@@ -249,6 +251,9 @@ files:
 - docker-compose.yml
 - exe/simple_map_reduce
 - lib/simple_map_reduce.rb
+- lib/simple_map_reduce/data_store_factory.rb
+- lib/simple_map_reduce/data_stores/default_data_store.rb
+- lib/simple_map_reduce/data_stores/remote_data_store.rb
 - lib/simple_map_reduce/driver/config.rb
 - lib/simple_map_reduce/driver/job.rb
 - lib/simple_map_reduce/s3_client.rb
@@ -259,6 +264,7 @@ files:
 - lib/simple_map_reduce/server/task.rb
 - lib/simple_map_reduce/server/worker.rb
 - lib/simple_map_reduce/version.rb
+- lib/simple_map_reduce/worker/polling_workers_status_worker.rb
 - lib/simple_map_reduce/worker/register_map_task_worker.rb
 - lib/simple_map_reduce/worker/run_map_task_worker.rb
 - lib/simple_map_reduce/worker/run_reduce_task_worker.rb