reduce_map 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 7315d208b60e9ee8eebaa9919ff385f4446144d7b6215d0d5fa164be30be8105
+  data.tar.gz: 845b4cc210894649511a36bbdf30ab4d8da9ae49c0c482d24e9b1240ea5d0f64
+SHA512:
+  metadata.gz: 6e083efed9b116274fd1323dbad7997fa2c52d422e59a58646b56bc5f1e9298ca18c18b78a1429a6aeee75a809bb4ff58bf4fc8e854cd341ec3dc4d725752d7a
+  data.tar.gz: 1762e721e164282bd37602b1db69fb4abcdfa3236630960cd5bd36e2f7bda35bc3818ee17f2a532038f6c2ecff172842e8689160f924ba57349ab67861318196

data/lib/grpc/server_pb.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: server.proto
+
+require 'google/protobuf'
+
+
+descriptor_data = "\n\x0cserver.proto\"\x07\n\x05\x45mpty\"=\n\nWorkerInfo\x12\x0c\n\x04uuid\x18\x01 \x01(\t\x12\x0f\n\x07success\x18\x02 \x01(\t\x12\x10\n\x08\x66ilename\x18\x03 \x01(\t\"&\n\x14RegisterWorkerResult\x12\x0e\n\x06result\x18\x01 \x01(\x08\"?\n\x15RegisterWorkerRequest\x12\x0c\n\x04uuid\x18\x01 \x01(\t\x12\n\n\x02ip\x18\x02 \x01(\t\x12\x0c\n\x04type\x18\x03 \x01(\t2q\n\x0fMapReduceMaster\x12\x41\n\x0eRegisterWorker\x12\x16.RegisterWorkerRequest\x1a\x15.RegisterWorkerResult\"\x00\x12\x1b\n\x04Ping\x12\x0b.WorkerInfo\x1a\x06.Emptyb\x06proto3"
+
+pool = Google::Protobuf::DescriptorPool.generated_pool
+pool.add_serialized_file(descriptor_data)
+
+Empty = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("Empty").msgclass
+WorkerInfo = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("WorkerInfo").msgclass
+RegisterWorkerResult = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("RegisterWorkerResult").msgclass
+RegisterWorkerRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("RegisterWorkerRequest").msgclass

data/lib/grpc/server_services_pb.rb

@@ -0,0 +1,21 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# Source: server.proto for package ''
+
+require 'grpc'
+require_relative 'server_pb'
+
+module MapReduceMaster
+  class Service
+
+    include ::GRPC::GenericService
+
+    self.marshal_class_method = :encode
+    self.unmarshal_class_method = :decode
+    self.service_name = 'MapReduceMaster'
+
+    rpc :RegisterWorker, ::RegisterWorkerRequest, ::RegisterWorkerResult
+    rpc :Ping, ::WorkerInfo, ::Empty
+  end
+
+  Stub = Service.rpc_stub_class
+end

data/lib/grpc/worker_pb.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: worker.proto
+
+require 'google/protobuf'
+
+require_relative 'server_pb'
+
+
+descriptor_data = "\n\x0cworker.proto\x1a\x0cserver.proto\"-\n\rMapInfoResult\x12\x0c\n\x04uuid\x18\x01 \x01(\t\x12\x0e\n\x06result\x18\x02 \x01(\x08\"7\n\x07MapInfo\x12\x10\n\x08\x66ilename\x18\x01 \x01(\t\x12\r\n\x05\x62lock\x18\x02 \x01(\t\x12\x0b\n\x03key\x18\x03 \x01(\t\":\n\nReduceInfo\x12\x10\n\x08\x66ilename\x18\x01 \x01(\t\x12\r\n\x05\x62lock\x18\x02 \x01(\t\x12\x0b\n\x03key\x18\x03 \x01(\t2R\n\x06Worker\x12 \n\x0cMapOperation\x12\x08.MapInfo\x1a\x06.Empty\x12&\n\x0fReduceOperation\x12\x0b.ReduceInfo\x1a\x06.Emptyb\x06proto3"
+
+pool = Google::Protobuf::DescriptorPool.generated_pool
+pool.add_serialized_file(descriptor_data)
+
+MapInfoResult = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("MapInfoResult").msgclass
+MapInfo = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("MapInfo").msgclass
+ReduceInfo = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("ReduceInfo").msgclass

data/lib/grpc/worker_services_pb.rb

@@ -0,0 +1,21 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# Source: worker.proto for package ''
+
+require 'grpc'
+require_relative 'worker_pb'
+
+module WorkerServer
+  class Service
+
+    include ::GRPC::GenericService
+
+    self.marshal_class_method = :encode
+    self.unmarshal_class_method = :decode
+    self.service_name = 'Worker'
+
+    rpc :MapOperation, ::MapInfo, ::Empty
+    rpc :ReduceOperation, ::ReduceInfo, ::Empty
+  end
+
+  Stub = Service.rpc_stub_class
+end

data/lib/map_reduce.rb

@@ -0,0 +1,139 @@
+require_relative './grpc/server_services_pb'
+require_relative './grpc/worker_services_pb'
+require 'grpc'
+require 'async'
+require 'async/semaphore'
+require 'pathname'
+require_relative 'worker'
+require 'digest'
+require 'method_source'
+
+class MapReduce < MapReduceMaster::Service
+  attr_accessor :logger, :worker_count, :map_count, :data, :files, :map_finished
+
+  def initialize(logger:, map_count: 5, file:)
+    @file = file
+    @map_count = map_count
+    @worker_count = 0
+    @logger = logger
+    @data = []
+    @files = nil
+    @map_finished = false
+  end
+
+  def ping(worker_req, _)
+    uuid = worker_req.uuid
+    success = worker_req.success
+    worker = data.find { |w| w[:uuid] == uuid }
+    worker[:status] = 0
+
+    if success == 'true'
+      logger.info("[Master] Worker #{uuid} completed the map operation succesful")
+    else
+      logger.info("[Master] Worker #{uuid} failed to complete the map opeartion successful")
+      @files << worker_req.filename
+    end
+
+    Empty.new
+  end
+
+  def register_worker(worker_req, _)
+    uuid = worker_req.uuid
+    type = worker_req.type
+    ip = worker_req.ip
+    mutex = Mutex.new
+    mutex.lock
+    data << ({ uuid:, ip:, status: 0 })
+    # That count is being back by the ruby GIL
+    @worker_count += 1
+    @logger.info('[Master] Worker register success')
+    RegisterWorkerResult.new(result: true)
+  ensure
+    mutex.unlock
+  end
+
+  def wait_for_enough_workers
+    logger.info('[Master] Wait for the creation of workers')
+    Worker.start_worker(logger, map_count)
+    logger.info('[Master] Finished!')
+  end
+
+  def reduce(&block)
+    Thread.new do
+      loop do
+        next unless @map_finished == true
+
+        block = block.source.sub(/^\s*master\.reduce do\s*\n/, '').sub(/^\s*end\s*\n/, '')
+        message = Base64.encode64(block)
+        worker = data.select { |w| w[:status] == 0 }.first
+
+        stub = WorkerServer::Stub.new(worker[:ip], :this_channel_is_insecure)
+        request = ReduceInfo.new(filename: "files/#{@encrypt_key}/map.txt", block: message, key: @encrypt_key)
+        worker[:status] = 'processing'
+        stub.reduce_operation(request)
+
+        break
+      end
+    end
+  end
+
+  def map(&block)
+    block = block.source.sub(/^\s*master\.map do\s*\n/, '').sub(/^\s*end\s*\n/, '')
+    message = Base64.encode64(block)
+    Thread.new do
+      loop do
+        @map_finished = true if files.empty?
+        break if files.empty?
+
+        Async do
+          workers = data.select { |w| w[:status] == 0 }.first(files.count)
+
+          semaphore = Async::Semaphore.new(workers.count)
+          tasks = []
+
+          workers.each do |worker|
+            tasks << semaphore.async do
+              stub = WorkerServer::Stub.new(worker[:ip], :this_channel_is_insecure)
+              request = MapInfo.new(filename: files.pop, block: message, key: @encrypt_key)
+              worker[:status] = 'processing'
+              stub.map_operation(request)
+            end
+          end
+          tasks.each(&:wait)
+        end.wait
+      end
+    end
+  end
+
+  def distribute_input
+    path_name = @file
+    key = path_name.to_path
+    logger.info('[Master] Start to distribute input')
+    @files = split_files(key, path_name)
+  end
+
+  private
+
+  def split_files(key, file)
+    @encrypt_key = generate_digest_key(key)
+    p "Going to pritn encrypt_key = #{@encrypt_key}"
+    FileUtils.mkdir_p("./files/#{@encrypt_key}")
+    p ""
+    line_maximum = (File.open(file).count / @map_count).to_i
+    file_data = file.readlines.map(&:chomp)
+    file_number = file_data.length / line_maximum
+    files = []
+    file_number.times do |index|
+      path = "./files/#{@encrypt_key}/file_#{index}"
+      File.write(path, file_data.slice!(0..line_maximum))
+      files << path
+    end
+    files
+  end
+
+  def generate_digest_key(key)
+    digest = Digest::SHA256.new
+    digest.update(key)
+    digest.hexdigest
+  end
+end

data/lib/worker.rb

@@ -0,0 +1,118 @@
+require_relative './grpc/worker_services_pb'
+require_relative './grpc/server_services_pb'
+require_relative './grpc/server_pb'
+require 'base64'
+require 'google/protobuf'
+require 'async'
+require 'logger'
+
+class Worker < WorkerServer::Service
+  attr_accessor :worker_number, :master_ip, :port, :logger, :uuid, :result
+
+  def initialize(worker_number:, master_ip:, port:, logger:)
+    @worker_number = worker_number
+    @uuid = generate_uuid
+    @master_ip = master_ip
+    @port = port
+    @logger = logger
+    @result = []
+  end
+
+  def map_operation(worker_req, _)
+    block = eval(Base64.decode64(worker_req.block))
+    block.call(File.read(worker_req.filename))
+    File.open("files/#{worker_req.key}/map.txt", 'a') do |file|
+      result.each do |array|
+        file.puts array.inspect
+      end
+    end
+    logger.info("[Worker] Worker #{uuid} gRPC finished the map operation")
+    stub = MapReduceMaster::Stub.new(@master_ip, :this_channel_is_insecure)
+    request = WorkerInfo.new(uuid: @uuid, success: 'true', filename: worker_req.filename)
+    stub.ping(request)
+    Empty.new
+  rescue StandardError => e
+    logger.error("[Worker] #{uuid} with error #{e}")
+    stub = MapReduceMaster::Stub.new(@master_ip, :this_channel_is_insecure)
+    request = WorkerInfo.new(uuid: @uuid, success: nil, filename: worker_req.filename)
+    stub.ping(request)
+  end
+
+  def reduce_operation(worker_req, _)
+    data = sort_map_file(worker_req.filename)
+    unique_keys = data.map { |item| item[0] }.uniq
+    file_path = "files/#{worker_req.key}/reduce.txt"
+    logger.info('[Worker] Starting Reduce Operation')
+    Async do
+      1.upto(unique_keys.count) do |i|
+        Async do
+          results = data.select { |item| item[0] == unique_keys[i] }
+          block = eval(Base64.decode64(worker_req.block))
+          response = block.call(results)
+          File.open(file_path, 'a') do |file|
+            file.puts response
+          end
+        end
+      end
+    end
+    logger.info('[Worker] Finished Reduce Operation')
+    logger.info("[Worker] File stored at #{file_path}")
+    Empty.new
+  end
+
+  def start
+    grpc_server = GRPC::RpcServer.new
+    grpc_server.add_http2_port("0.0.0.0:#{port}", :this_port_is_insecure)
+    grpc_server.handle(self)
+    Thread.new do
+      grpc_server.run_till_terminated
+    ensure
+      logger.info('[Worker] Worker gRPC thread failed')
+    end
+    logger.info('[Worker] Worker gRPC thread start')
+
+    logger.info('[Worker] load functions finish')
+    register_worker
+  end
+
+  def self.start_worker(logger, worker_number)
+    master_ip = '0.0.0.0:50051'
+    Async do
+      1.upto(worker_number) do |i|
+        Async do
+          worker = new(worker_number:, master_ip:, port: "3000#{i}", logger:)
+          worker.start
+        end
+      end
+    end
+  end
+
+  private
+
+  def sort_map_file(file_path)
+    file_data = []
+    File.open(file_path, 'r') do |file|
+      file.each_line do |line|
+        file_data << eval(line.strip)
+      end
+    end
+    file_data.sort_by { |item| item[0] }
+  end
+
+  def emit_intermediate(k, count:)
+    result << [k, count]
+  end
+
+  alias emit emit_intermediate
+
+  def generate_uuid
+    SecureRandom.uuid
+  end
+
+  def register_worker
+    stub = MapReduceMaster::Stub.new(@master_ip, :this_channel_is_insecure)
+    request = RegisterWorkerRequest.new(uuid: @uuid, ip: "localhost:#{@port}", type: 'map')
+    stub.register_worker(request)
+    @logger.info('[Worker] Worker register itself finish')
+  end
+end

metadata

@@ -0,0 +1,77 @@
+--- !ruby/object:Gem::Specification
+name: reduce_map
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Francisco Paradela
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2024-07-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: async
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.12.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.12.0
+- !ruby/object:Gem::Dependency
+  name: grpc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.62.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.62.0
+description: Ruby map/reduce framework
+email: franciscoleite.dev@protonmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/grpc/server_pb.rb
+- lib/grpc/server_services_pb.rb
+- lib/grpc/worker_pb.rb
+- lib/grpc/worker_services_pb.rb
+- lib/map_reduce.rb
+- lib/worker.rb
+homepage: https://rubygems.org/gems/map_reduce
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.9
+signing_key: 
+specification_version: 4
+summary: This framework is designed to provide a fully multi-threaded, distributed
+  and asynchronous approach to MapReduce processing.
+test_files: []