simple_map_reduce 0.1.0

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'simple_map_reduce'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/job_tracker

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+cd `dirname $0`
+if [ "$USE_DOCKER" == "" ];then
+  job_tracker_url="http://localhost:4567"
+  s3_endpoint="http://localhost:9000"
+else
+  job_tracker_url="http://job_tracker:4567"
+  s3_endpoint="http://minio:9000"
+fi
+
+../exe/simple_map_reduce run_job_tracker! \
+  --job-tracker-url=`echo $job_tracker_url` \
+  --s3_config=access_key_id:'MINIO_ACCESS_KEY' \
+              secret_access_key:'MINIO_SECRET_KEY' \
+              endpoint:`echo $s3_endpoint` \
+              region:'us-east-1' \
+              force_path_style:true

data/bin/job_worker1

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+cd `dirname $0`
+if [ "$USE_DOCKER" == "" ];then
+  job_tracker_url="http://localhost:4567"
+  job_worker_url="http://localhost:4568"
+  s3_endpoint="http://localhost:9000"
+else
+  job_tracker_url="http://job_tracker:4567"
+  job_worker_url="http://job_worker1:4568"
+  s3_endpoint="http://minio:9000"
+fi
+
+../exe/simple_map_reduce run_job_worker! \
+  --job-tracker-url=`echo $job_tracker_url` \
+  --job-worker-url=`echo $job_worker_url` \
+  --server-port=4568 \
+  --s3_config=access_key_id:'MINIO_ACCESS_KEY' \
+              secret_access_key:'MINIO_SECRET_KEY' \
+              endpoint:`echo $s3_endpoint` \
+              region:'us-east-1' \
+              force_path_style:true

data/bin/job_worker2

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+cd `dirname $0`
+if [ "$USE_DOCKER" == "" ];then
+  job_tracker_url="http://localhost:4567"
+  job_worker_url="http://localhost:4569"
+  s3_endpoint="http://localhost:9000"
+else
+  job_tracker_url="http://job_tracker:4567"
+  job_worker_url="http://job_worker2:4569"
+  s3_endpoint="http://minio:9000"
+fi
+
+../exe/simple_map_reduce run_job_worker! \
+  --job-tracker-url=`echo $job_tracker_url` \
+  --job-worker-url=`echo $job_worker_url` \
+  --server-port=4569 \
+  --s3_config=access_key_id:'MINIO_ACCESS_KEY' \
+              secret_access_key:'MINIO_SECRET_KEY' \
+              endpoint:`echo $s3_endpoint` \
+              region:'us-east-1' \
+              force_path_style:true

data/bin/job_worker3

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+cd `dirname $0`
+if [ "$USE_DOCKER" == "" ];then
+  job_tracker_url="http://localhost:4567"
+  job_worker_url="http://localhost:4570"
+  s3_endpoint="http://localhost:9000"
+else
+  job_tracker_url="http://job_tracker:4567"
+  job_worker_url="http://job_worker3:4570"
+  s3_endpoint="http://minio:9000"
+fi
+
+../exe/simple_map_reduce run_job_worker! \
+  --job-tracker-url=`echo $job_tracker_url` \
+  --job-worker-url=`echo $job_worker_url` \
+  --server-port=4570 \
+  --s3_config=access_key_id:'MINIO_ACCESS_KEY' \
+              secret_access_key:'MINIO_SECRET_KEY' \
+              endpoint:`echo $s3_endpoint` \
+              region:'us-east-1' \
+              force_path_style:true

data/bin/register_word_count_job

@@ -0,0 +1,80 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'simple_map_reduce'
+
+map_class_name = 'WordCount'
+map_script = <<-'EOS'
+  class WordCount
+    def map(input_data, output_io)
+      input_data.split(' ').each do |raw_word|
+        word = raw_word.strip
+        next if word.empty?
+        word.delete!('_=,.[]()#\'"-=~|&%')
+        word.downcase!
+
+        output_io.puts({ key: word, value: 1 }.to_json)
+      end
+    end
+  end
+EOS
+
+reduce_class_name = 'WordCount'
+reduce_script = <<-'EOS'
+  require 'json'
+  class WordCount
+    def reduce(input_io, output_io)
+      output = Hash.new(0)
+      count = 0
+      input_io.each_line(chomp: true, rs: "\n") do |line|
+        input = JSON.parse(line, symbolize_names: true)
+        output[input[:key]] += input[:value]
+        count += 1
+        if count % 100 == 0
+          puts "current count: #{count}"
+        end
+      end
+
+      output.each do |key, value|
+        output_io.puts(JSON.generate(Hash[key, value]))
+      end
+    end
+  end
+EOS
+
+job_input_directory_path = 'input.txt'
+job_input_bucket_name = 'input'
+
+job_output_directory_path = 'word_count'
+job_output_bucket_name = 'output'
+
+job = ::SimpleMapReduce::Server::Job.new(
+  map_script: map_script,
+  map_class_name: map_class_name,
+  reduce_script: reduce_script,
+  reduce_class_name: reduce_class_name,
+  job_input_directory_path: job_input_directory_path,
+  job_input_bucket_name: job_input_bucket_name,
+  job_output_directory_path: job_output_directory_path,
+  job_output_bucket_name: job_output_bucket_name
+)
+
+url = 'http://localhost:4567'
+http_client = ::Faraday.new(
+  url: url,
+  headers: {
+    'Accept' => 'application/json ',
+    'Content-Type' => 'application/json'
+  }
+) do |faraday|
+  faraday.response :logger
+  faraday.adapter  ::Faraday.default_adapter
+end
+
+response = http_client.post do |request|
+  request.url('/jobs')
+  request.body = job.to_h.to_json
+end
+
+puts response.inspect

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/docker-compose.yml

@@ -0,0 +1,98 @@
+version: '2'
+services:
+  job_tracker:
+    build: .
+    volumes:
+      - ./bin:/app/bin
+      - ./examples:/app/examples
+      - ./exe:/app/exe
+      - ./lib:/app/lib
+      - ./spec:/app/spec
+      - ./Gemfile:/app/Gemfile
+      - ./Rakefile:/app/Rakefile
+      - ./simple_map_reduce.gemspec:/app/simple_map_reduce.gemspec
+      - ./.gitignore:/app/.gitignore
+      - ./.rspec:/app/.rspec
+      - ./.rubocop.yml:/app/.rubocop.yml
+    ports:
+      - '4567:4567'
+    links:
+      - minio
+    environment:
+      USE_DOCKER: 'true'
+    command: bash -c "bin/job_tracker"
+  job_worker1:
+    build: .
+    volumes:
+      - ./bin:/app/bin
+      - ./examples:/app/examples
+      - ./exe:/app/exe
+      - ./lib:/app/lib
+      - ./spec:/app/spec
+      - ./Gemfile:/app/Gemfile
+      - ./Rakefile:/app/Rakefile
+      - ./simple_map_reduce.gemspec:/app/simple_map_reduce.gemspec
+      - ./.gitignore:/app/.gitignore
+      - ./.rspec:/app/.rspec
+      - ./.rubocop.yml:/app/.rubocop.yml
+    ports:
+      - '4568:4568'
+    links:
+      - job_tracker
+      - minio
+    environment:
+      USE_DOCKER: 'true'
+    command: bash -c "bin/job_worker1"
+  job_worker2:
+    build: .
+    volumes:
+      - ./bin:/app/bin
+      - ./examples:/app/examples
+      - ./exe:/app/exe
+      - ./lib:/app/lib
+      - ./spec:/app/spec
+      - ./Gemfile:/app/Gemfile
+      - ./Rakefile:/app/Rakefile
+      - ./simple_map_reduce.gemspec:/app/simple_map_reduce.gemspec
+      - ./.gitignore:/app/.gitignore
+      - ./.rspec:/app/.rspec
+      - ./.rubocop.yml:/app/.rubocop.yml
+    ports:
+      - '4569:4569'
+    environment:
+      USE_DOCKER: 'true'
+    links:
+      - job_tracker
+      - minio
+    command: bash -c "bin/job_worker2"
+  job_worker3:
+    build: .
+    volumes:
+      - ./bin:/app/bin
+      - ./examples:/app/examples
+      - ./exe:/app/exe
+      - ./lib:/app/lib
+      - ./spec:/app/spec
+      - ./Gemfile:/app/Gemfile
+      - ./Rakefile:/app/Rakefile
+      - ./simple_map_reduce.gemspec:/app/simple_map_reduce.gemspec
+      - ./.gitignore:/app/.gitignore
+      - ./.rspec:/app/.rspec
+      - ./.rubocop.yml:/app/.rubocop.yml
+    ports:
+      - '4570:4570'
+    environment:
+      USE_DOCKER: 'true'
+    links:
+      - job_tracker
+      - minio
+    command: bash -c "bin/job_worker3"
+  minio:
+    image: minio/minio
+    ports:
+      - '9000:9000'
+    environment:
+      - 'MINIO_ACCESS_KEY=MINIO_ACCESS_KEY'
+      - 'MINIO_SECRET_KEY=MINIO_SECRET_KEY'
+      - 'MINIO_REGION=us-east-1'
+    command: server /export

data/exe/simple_map_reduce

@@ -0,0 +1,183 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require_relative '../lib/simple_map_reduce'
+require 'thor'
+require 'faker'
+require 'aws-sdk'
+
+module SimpleMapReduce
+  class Cli < ::Thor
+    desc 'run_job_tracker!', 'start job_tracker server'
+    method_option :job_tracker_url, type: :string, required: true
+    method_option :server_port, type: :numeric
+    method_option :s3_config, type: :hash
+    method_option :s3_input_bucket_name, type: :string
+    method_option :s3_intermediate_bucket_name, type: :string
+    method_option :s3_output_bucket_name, type: :string
+    def run_job_tracker!
+      opts = Hash[options.to_h.to_a.map { |v| [v[0].to_sym, v[1]] }]
+      config = SimpleMapReduce::Server::Config.new(
+        job_tracker_url: opts[:job_tracker_url],
+        server_port: opts[:server_port],
+        s3_config: opts[:s3_config],
+        s3_input_bucket_name: opts[:s3_input_bucket_name],
+        s3_intermediate_bucket_name: opts[:s3_intermediate_bucket_name],
+        s3_output_bucket_name: opts[:s3_output_bucket_name]
+      )
+      assign_config_parameters(config)
+      SimpleMapReduce::Server::JobTracker.run!(port: config.server_port, bind: '0.0.0.0') do
+        SimpleMapReduce::Server::JobTracker.setup_job_tracker
+      end
+    end
+
+    desc 'run_job_worker!!', 'start job_worker server'
+    method_option :job_tracker_url, type: :string, required: true
+    method_option :job_worker_url, type: :string, required: true
+    method_option :server_port, type: :numeric
+    method_option :s3_config, type: :hash
+    method_option :s3_input_bucket_name, type: :string
+    method_option :s3_intermediate_bucket_name, type: :string
+    method_option :s3_output_bucket_name, type: :string
+    def run_job_worker!
+      opts = Hash[options.to_h.to_a.map { |v| [v[0].to_sym, v[1]] }]
+      config = SimpleMapReduce::Server::Config.new(
+        job_tracker_url: opts[:job_tracker_url],
+        job_worker_url: opts[:job_worker_url],
+        server_port: opts[:server_port],
+        s3_config: opts[:s3_config],
+        s3_input_bucket_name: opts[:s3_input_bucket_name],
+        s3_intermediate_bucket_name: opts[:s3_intermediate_bucket_name],
+        s3_output_bucket_name: opts[:s3_output_bucket_name]
+      )
+      assign_config_parameters(config)
+      SimpleMapReduce::Server::JobWorker.run!(port: config.server_port, bind: '0.0.0.0') do
+        SimpleMapReduce::Server::JobWorker.setup_worker
+      end
+    end
+
+    desc 'generate_lorem_text_data', 'generate test data'
+    method_option :count, type: :numeric, default: 1000, desc: 'The number of line count you want'
+    method_option :upload, type: :boolean, default: false, desc: 'The flag to upload to local minio bucket'
+    def generate_lorem_text_data
+      base_path = Dir.pwd
+      file = File.open(File.join(base_path, 'input.txt'), 'w+')
+      file.write(Faker::Lorem.paragraphs(options[:count]).join("\n"))
+      puts 'input.txt generated'
+
+      if options[:upload]
+        config = {
+          access_key_id: 'MINIO_ACCESS_KEY',
+          secret_access_key: 'MINIO_SECRET_KEY',
+          endpoint: ENV['USE_DOCKER'] ? 'http://minio:9000' : 'http://127.0.0.1:9000',
+          region: 'us-east-1',
+          force_path_style: true
+        }
+        s3 = ::Aws::S3::Client.new(config)
+        file.rewind
+        s3.put_object(
+          body: file.read,
+          bucket: 'input',
+          key: 'input.txt'
+        )
+        puts 'uploaded'
+      end
+
+      file.close
+      puts 'done'
+    end
+
+    desc 'execute_word_count', 'execute word count sample job'
+    def execute_word_count
+      map_class_name = 'WordCount'
+      map_script = <<-'EOS'
+        class WordCount
+          def map(input_data, output_io)
+            input_data.split(' ').each do |raw_word|
+              word = raw_word.strip
+              next if word.empty?
+              word.delete!('_=,.[]()#\'"-=~|&%')
+              word.downcase!
+
+              output_io.puts({ key: word, value: 1 }.to_json)
+            end
+          end
+        end
+      EOS
+
+      reduce_class_name = 'WordCount'
+      reduce_script = <<-'EOS'
+        require 'json'
+        class WordCount
+          def reduce(input_io, output_io)
+            output = Hash.new(0)
+            count = 0
+            input_io.each_line(chomp: true, rs: "\n") do |line|
+              input = JSON.parse(line, symbolize_names: true)
+              output[input[:key]] += input[:value]
+              count += 1
+              if count % 100 == 0
+                puts "current count: #{count}"
+              end
+            end
+
+            output.each do |key, value|
+              output_io.puts(JSON.generate(Hash[key, value]))
+            end
+          end
+        end
+      EOS
+
+      job_input_directory_path = 'input.txt'
+      job_input_bucket_name = 'input'
+
+      job_output_directory_path = 'word_count'
+      job_output_bucket_name = 'output'
+
+      job = ::SimpleMapReduce::Server::Job.new(
+        map_script: map_script,
+        map_class_name: map_class_name,
+        reduce_script: reduce_script,
+        reduce_class_name: reduce_class_name,
+        job_input_directory_path: job_input_directory_path,
+        job_input_bucket_name: job_input_bucket_name,
+        job_output_directory_path: job_output_directory_path,
+        job_output_bucket_name: job_output_bucket_name
+      )
+
+      url = 'http://localhost:4567'
+      http_client = ::Faraday.new(
+        url: url,
+        headers: {
+          'Accept' => 'application/json ',
+          'Content-Type' => 'application/json'
+        }
+      ) do |faraday|
+        faraday.response :logger
+        faraday.adapter  ::Faraday.default_adapter
+      end
+
+      response = http_client.post do |request|
+        request.url('/jobs')
+        request.body = job.to_h.to_json
+      end
+
+      puts response.inspect
+    end
+
+    private
+
+    def assign_config_parameters(config)
+      SimpleMapReduce.s3_config = config.s3_config
+      SimpleMapReduce.s3_input_bucket_name = config.s3_input_bucket_name
+      SimpleMapReduce.s3_intermediate_bucket_name = config.s3_intermediate_bucket_name
+      SimpleMapReduce.s3_output_bucket_name = config.s3_output_bucket_name
+      SimpleMapReduce.logger = config.logger
+      SimpleMapReduce.job_tracker_url = config.job_tracker_url
+      SimpleMapReduce.job_worker_url = config.job_worker_url
+    end
+  end
+end
+
+SimpleMapReduce::Cli.start