simple_map_reduce 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Dockerfile +5 -3
- data/README.md +12 -12
- data/docker-compose.yml +4 -3
- data/exe/simple_map_reduce +4 -4
- data/lib/simple_map_reduce/server/confg.rb +1 -1
- data/lib/simple_map_reduce/server/job.rb +4 -10
- data/lib/simple_map_reduce/server/job_tracker.rb +12 -13
- data/lib/simple_map_reduce/server/job_worker.rb +3 -2
- data/lib/simple_map_reduce/server/task.rb +3 -10
- data/lib/simple_map_reduce/version.rb +1 -1
- data/lib/simple_map_reduce/worker/polling_workers_status_worker.rb +8 -10
- data/lib/simple_map_reduce/worker/run_map_task_worker.rb +3 -1
- data/simple_map_reduce.gemspec +21 -17
- metadata +98 -50
- data/.github/ISSUE_TEMPLATE.md +0 -1
- data/.github/PULL_REQUEST_TEMPLATE.md +0 -1
- data/.gitignore +0 -141
- data/.rspec +0 -2
- data/.rubocop.yml +0 -69
- data/.ruby-version +0 -1
- data/.travis.yml +0 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7adbfb3afed63f191ef71c590bc22a06647694e7b59560b0c0eecd08f1c292f5
|
4
|
+
data.tar.gz: 5226658a8a1bafdd4748ab5417df876cc6093f518035b68fe3fb50705e03ed34
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 10136069601e794b9478e9b4c863b56d0b8c87ccd93985aa89a9364e4c355a9d97989b452fc73be79b6b6a258e447778baed0c51d2653df83a47833810642fb6
|
7
|
+
data.tar.gz: 15b99c225490420a0593c75b107e6d87423cc86feb772c214c059b789d0721a81f1710b846f13d0798d4ed98ebe76d0e85763406e689101f787663d8717c4512
|
data/Dockerfile
CHANGED
@@ -1,9 +1,11 @@
|
|
1
|
-
FROM ruby:2.
|
1
|
+
FROM ruby:2.7.4
|
2
2
|
|
3
3
|
WORKDIR /app
|
4
4
|
|
5
5
|
ADD simple_map_reduce.gemspec /app/simple_map_reduce.gemspec
|
6
6
|
ADD Gemfile /app/Gemfile
|
7
|
-
ADD lib
|
7
|
+
ADD lib /app/lib
|
8
|
+
ADD bin /app/bin
|
9
|
+
ADD exe /app/exe
|
8
10
|
RUN gem install bundler
|
9
|
-
RUN bundle install
|
11
|
+
RUN bundle install
|
data/README.md
CHANGED
@@ -1,10 +1,8 @@
|
|
1
|
-
[](https://travis-ci.org/serihiro/simple_map_reduce)
|
2
1
|
[](https://badge.fury.io/rb/simple_map_reduce)
|
3
2
|
|
4
3
|
# SimpleMapReduce
|
5
|
-
|
6
|
-
|
7
|
-
- This project is a experience project. So, the specifications will be changed suddenly.
|
4
|
+
This is a [MapReduce](https://research.google.com/archive/mapreduce.html) distributed framework written in Ruby.
|
5
|
+
This project is an experimental project. So all the specifications will be changed suddenly.
|
8
6
|
|
9
7
|
## Installation
|
10
8
|
|
@@ -27,7 +25,9 @@ Or install it yourself as:
|
|
27
25
|
### 1. Start minio server
|
28
26
|
|
29
27
|
```sh
|
30
|
-
$ docker run -p 9000:9000 -
|
28
|
+
$ docker run -p 9000:9000 -p 9001:9001 \
|
29
|
+
-e "MINIO_ROOT_USER=MINIO_ACCESS_KEY" -e "MINIO_ROOT_PASSWORD=MINIO_SECRET_KEY" -e "MINIO_REGION=us-east-1" \
|
30
|
+
minio/minio server /data --console-address :9001
|
31
31
|
```
|
32
32
|
|
33
33
|
### 2. Start job tracker
|
@@ -71,25 +71,25 @@ $ bundle exec simple_map_reduce execute_word_count
|
|
71
71
|
|
72
72
|
## Quick start in Docker Compose
|
73
73
|
|
74
|
-
- You can setup a simple_map_reduce cluster by docker
|
74
|
+
- You can setup a simple_map_reduce cluster by docker compose.
|
75
75
|
|
76
76
|
```sh
|
77
77
|
$ clone git@github.com:serihiro/simple_map_reduce.git
|
78
78
|
$ cd simple_map_reduce
|
79
|
-
$ docker
|
79
|
+
$ docker compose up
|
80
80
|
```
|
81
81
|
|
82
82
|
- You can execute word count sample by executing following commands
|
83
83
|
|
84
84
|
```sh
|
85
|
-
$ docker
|
86
|
-
$ docker
|
85
|
+
$ docker compose exec job_tracker bundle exec simple_map_reduce generate_lorem_text_data --upload=true
|
86
|
+
$ docker compose exec job_tracker bundle exec simple_map_reduce execute_word_count
|
87
87
|
```
|
88
88
|
|
89
89
|
## Motivation of this project
|
90
|
-
|
91
|
-
|
92
|
-
|
90
|
+
I would have liked to lean the theory of distributed systems, big data processing, and MapReduce algorhythm.
|
91
|
+
In my experiences, I believed that an implementation of them is the best way to learn them.
|
92
|
+
So I decided to create an experimental implementation, and keep adding new features in order to get an practical experiences of the theories.
|
93
93
|
|
94
94
|
## Development
|
95
95
|
|
data/docker-compose.yml
CHANGED
@@ -91,8 +91,9 @@ services:
|
|
91
91
|
image: minio/minio
|
92
92
|
ports:
|
93
93
|
- '9000:9000'
|
94
|
+
- '9001:9001'
|
94
95
|
environment:
|
95
|
-
- '
|
96
|
-
- '
|
96
|
+
- 'MINIO_ROOT_USER=MINIO_ACCESS_KEY'
|
97
|
+
- 'MINIO_ROOT_PASSWORD=MINIO_SECRET_KEY'
|
97
98
|
- 'MINIO_REGION=us-east-1'
|
98
|
-
command: server /export
|
99
|
+
command: server /export --console-address :9001
|
data/exe/simple_map_reduce
CHANGED
@@ -63,8 +63,8 @@ module SimpleMapReduce
|
|
63
63
|
def generate_lorem_text_data
|
64
64
|
base_path = Dir.pwd
|
65
65
|
file = File.open(File.join(base_path, 'input.txt'), 'w+')
|
66
|
-
file.write(Faker::Lorem.paragraphs(options[:count]).join("\n"))
|
67
|
-
puts
|
66
|
+
file.write(Faker::Lorem.paragraphs(number: options[:count]).join("\n"))
|
67
|
+
puts('input.txt generated')
|
68
68
|
|
69
69
|
if options[:upload]
|
70
70
|
config = {
|
@@ -81,11 +81,11 @@ module SimpleMapReduce
|
|
81
81
|
bucket: 'input',
|
82
82
|
key: 'input.txt'
|
83
83
|
)
|
84
|
-
puts
|
84
|
+
puts('uploaded')
|
85
85
|
end
|
86
86
|
|
87
87
|
file.close
|
88
|
-
puts
|
88
|
+
puts('done')
|
89
89
|
end
|
90
90
|
|
91
91
|
desc 'execute_word_count', 'execute word count sample job'
|
@@ -28,7 +28,7 @@ module SimpleMapReduce
|
|
28
28
|
@server_port = options[:server_port] || 4567
|
29
29
|
@job_tracker_url = options[:job_tracker_url]
|
30
30
|
@job_worker_url = options[:job_worker_url]
|
31
|
-
@logger = options[:logger] || Logger.new(
|
31
|
+
@logger = options[:logger] || Logger.new($stdout)
|
32
32
|
end
|
33
33
|
|
34
34
|
private
|
@@ -39,15 +39,9 @@ module SimpleMapReduce
|
|
39
39
|
end
|
40
40
|
end
|
41
41
|
|
42
|
-
def initialize(
|
43
|
-
|
44
|
-
|
45
|
-
reduce_script:,
|
46
|
-
reduce_class_name:,
|
47
|
-
job_input_bucket_name:,
|
48
|
-
job_input_directory_path:,
|
49
|
-
job_output_bucket_name:,
|
50
|
-
job_output_directory_path:,
|
42
|
+
def initialize(map_script:, map_class_name:, reduce_script:, reduce_class_name:, job_input_bucket_name:,
|
43
|
+
job_input_directory_path:, job_output_bucket_name:, job_output_directory_path:,
|
44
|
+
id: nil,
|
51
45
|
map_worker_url: nil,
|
52
46
|
map_worker: nil,
|
53
47
|
data_store_type: 'default')
|
@@ -128,7 +122,7 @@ module SimpleMapReduce
|
|
128
122
|
|
129
123
|
class << self
|
130
124
|
def deserialize(data)
|
131
|
-
params =
|
125
|
+
params = MessagePack.unpack(data).transform_keys(&:to_sym)
|
132
126
|
params[:data_store_type] = 'remote'
|
133
127
|
new(params)
|
134
128
|
end
|
@@ -256,18 +256,16 @@ module SimpleMapReduce
|
|
256
256
|
ready_workers = ready_workers.keys.take(worker_size)
|
257
257
|
|
258
258
|
ready_workers.map do |retry_worker_id|
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
@workers[retry_worker_id]
|
267
|
-
end
|
259
|
+
@workers[retry_worker_id].reserve!
|
260
|
+
rescue => e
|
261
|
+
logger.error("Failed to transit the worker state: `#{@workers[retry_worker_id]}`")
|
262
|
+
logger.error(e.inspect)
|
263
|
+
nil
|
264
|
+
else
|
265
|
+
@workers[retry_worker_id]
|
268
266
|
end.compact
|
269
267
|
else
|
270
|
-
|
268
|
+
[]
|
271
269
|
end
|
272
270
|
ensure
|
273
271
|
mutex.unlock
|
@@ -317,11 +315,12 @@ module SimpleMapReduce
|
|
317
315
|
SimpleMapReduce.logger
|
318
316
|
end
|
319
317
|
|
320
|
-
# @override
|
318
|
+
# @override `Sinatra::Base#quit!`
|
319
|
+
# https://github.com/sinatra/sinatra/blob/2e980f3534b680fbd79d7ec39552b4afb7675d6c/lib/sinatra/base.rb#L1483-L1491
|
321
320
|
def quit!
|
322
321
|
@keep_polling_workers = false
|
323
|
-
@polling_workers_thread
|
324
|
-
job_manager
|
322
|
+
@polling_workers_thread&.kill
|
323
|
+
job_manager&.shutdown_workers!
|
325
324
|
super
|
326
325
|
end
|
327
326
|
end
|
@@ -120,9 +120,10 @@ module SimpleMapReduce
|
|
120
120
|
SimpleMapReduce.logger
|
121
121
|
end
|
122
122
|
|
123
|
-
# @override
|
123
|
+
# @override `Sinatra::Base#quit!`
|
124
|
+
# https://github.com/sinatra/sinatra/blob/2e980f3534b680fbd79d7ec39552b4afb7675d6c/lib/sinatra/base.rb#L1483-L1491
|
124
125
|
def quit!
|
125
|
-
job_manager
|
126
|
+
job_manager&.shutdown_workers!
|
126
127
|
super
|
127
128
|
end
|
128
129
|
end
|
@@ -37,15 +37,8 @@ module SimpleMapReduce
|
|
37
37
|
end
|
38
38
|
end
|
39
39
|
|
40
|
-
def initialize(
|
41
|
-
|
42
|
-
task_class_name:,
|
43
|
-
task_script:,
|
44
|
-
task_input_bucket_name:,
|
45
|
-
task_input_file_path:,
|
46
|
-
task_output_bucket_name:,
|
47
|
-
task_output_directory_path:,
|
48
|
-
worker: nil)
|
40
|
+
def initialize(job_id:, task_class_name:, task_script:, task_input_bucket_name:, task_input_file_path:,
|
41
|
+
task_output_bucket_name:, task_output_directory_path:, id: nil, worker: nil)
|
49
42
|
@id = id
|
50
43
|
@job_id = job_id
|
51
44
|
@task_class_name = task_class_name
|
@@ -84,7 +77,7 @@ module SimpleMapReduce
|
|
84
77
|
|
85
78
|
class << self
|
86
79
|
def deserialize(data)
|
87
|
-
new(
|
80
|
+
new(MessagePack.unpack(data).transform_keys(&:to_sym))
|
88
81
|
end
|
89
82
|
end
|
90
83
|
end
|
@@ -6,16 +6,14 @@ module SimpleMapReduce
|
|
6
6
|
def perform(workers)
|
7
7
|
logger.debug("begin polling workers: #{workers.keys}")
|
8
8
|
workers.each do |id, worker|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
workers.delete(id)
|
18
|
-
end
|
9
|
+
response = http_client(worker.url).get("/workers/#{worker.id}")
|
10
|
+
body = JSON.parse(response.body, symbolize_names: true)[:worker]
|
11
|
+
worker.aasm.current_state = body[:state].to_sym
|
12
|
+
rescue => e
|
13
|
+
logger.error(e.inspect)
|
14
|
+
logger.error(e&.response&.inspect)
|
15
|
+
logger.info("Worker #{worker.id} is removed from workers")
|
16
|
+
workers.delete(id)
|
19
17
|
end
|
20
18
|
logger.debug("finish polling workers: #{workers.keys}")
|
21
19
|
end
|
@@ -14,6 +14,7 @@ module SimpleMapReduce
|
|
14
14
|
unless map_task.respond_to?(:map)
|
15
15
|
raise InvalidMapTaskError, 'no map method'
|
16
16
|
end
|
17
|
+
|
17
18
|
logger.info('map task start')
|
18
19
|
|
19
20
|
local_input_cache = Tempfile.new
|
@@ -112,7 +113,8 @@ module SimpleMapReduce
|
|
112
113
|
workers_count = workers.count
|
113
114
|
raise 'No workers' unless workers_count > 0
|
114
115
|
|
115
|
-
shuffled_local_outputs = Array.new(workers_count
|
116
|
+
shuffled_local_outputs = Array.new(workers_count)
|
117
|
+
shuffled_local_outputs.each_with_index { |_, i| shuffled_local_outputs[i] = Tempfile.new }
|
116
118
|
local_output_cache.each_line(rs: "\n") do |raw_line|
|
117
119
|
output = JSON.parse(raw_line, symbolize_names: true)
|
118
120
|
partition_id = output[:key].hash % workers_count
|
data/simple_map_reduce.gemspec
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
lib = File.expand_path('
|
3
|
+
lib = File.expand_path('lib', __dir__)
|
4
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
5
5
|
require 'simple_map_reduce/version'
|
6
6
|
|
@@ -15,27 +15,31 @@ Gem::Specification.new do |spec|
|
|
15
15
|
spec.homepage = 'https://github.com/serihiro/simple_map_reduce'
|
16
16
|
spec.license = 'MIT'
|
17
17
|
|
18
|
-
spec.files =
|
19
|
-
|
20
|
-
|
18
|
+
spec.files = %w(CODE_OF_CONDUCT.md LICENSE.txt docker-compose.yml
|
19
|
+
simple_map_reduce.gemspec Dockerfile README.md Gemfile Rakefile)
|
20
|
+
spec.files += Dir.glob('lib/**/*')
|
21
|
+
spec.files += Dir.glob('bin/**/*')
|
22
|
+
spec.files += Dir.glob('exe/**/*')
|
23
|
+
|
21
24
|
spec.bindir = 'exe'
|
22
25
|
spec.executables = spec.files.grep(%r(^exe/)) { |f| File.basename(f) }
|
23
26
|
spec.require_paths = ['lib']
|
24
|
-
spec.required_ruby_version = '>= 2.
|
27
|
+
spec.required_ruby_version = '>= 2.7.4'
|
25
28
|
|
26
|
-
spec.add_development_dependency 'bundler', '~>
|
27
|
-
spec.add_development_dependency 'factory_bot', '~>
|
29
|
+
spec.add_development_dependency 'bundler', '~> 2.2.0'
|
30
|
+
spec.add_development_dependency 'factory_bot', '~> 6.2.0'
|
28
31
|
spec.add_development_dependency 'faker'
|
29
|
-
spec.add_development_dependency '
|
30
|
-
spec.add_development_dependency '
|
32
|
+
spec.add_development_dependency 'mry'
|
33
|
+
spec.add_development_dependency 'rack-test', '~> 1.1.0'
|
34
|
+
spec.add_development_dependency 'rake', '~> 13.0.6'
|
31
35
|
spec.add_development_dependency 'rspec', '~> 3.0'
|
32
|
-
spec.add_development_dependency 'rubocop', '
|
33
|
-
spec.add_runtime_dependency 'aasm', '
|
34
|
-
spec.add_runtime_dependency 'aws-sdk', '
|
35
|
-
spec.add_runtime_dependency 'faraday', '
|
36
|
-
spec.add_runtime_dependency 'msgpack', '
|
36
|
+
spec.add_development_dependency 'rubocop', '1.20.0'
|
37
|
+
spec.add_runtime_dependency 'aasm', '>= 4.12', '< 5.3'
|
38
|
+
spec.add_runtime_dependency 'aws-sdk', '>= 3.0', '< 3.2'
|
39
|
+
spec.add_runtime_dependency 'faraday', '>= 0.13', '< 1.8'
|
40
|
+
spec.add_runtime_dependency 'msgpack', '>= 1.2', '< 1.5'
|
37
41
|
spec.add_runtime_dependency 'rasteira', '~> 0.1.0'
|
38
|
-
spec.add_runtime_dependency 'sinatra', '
|
39
|
-
spec.add_runtime_dependency 'sinatra-contrib', '
|
40
|
-
spec.add_runtime_dependency 'thor', '
|
42
|
+
spec.add_runtime_dependency 'sinatra', '>= 2.0', '< 2.2'
|
43
|
+
spec.add_runtime_dependency 'sinatra-contrib', '>= 2.0', '< 2.2'
|
44
|
+
spec.add_runtime_dependency 'thor', '>= 0.20', '< 1.2'
|
41
45
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: simple_map_reduce
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kazuhiro Serizawa
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-09-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -16,28 +16,28 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: 2.2.0
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version:
|
26
|
+
version: 2.2.0
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: factory_bot
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version:
|
33
|
+
version: 6.2.0
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version:
|
40
|
+
version: 6.2.0
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: faker
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -52,34 +52,48 @@ dependencies:
|
|
52
52
|
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: mry
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
55
69
|
- !ruby/object:Gem::Dependency
|
56
70
|
name: rack-test
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|
58
72
|
requirements:
|
59
73
|
- - "~>"
|
60
74
|
- !ruby/object:Gem::Version
|
61
|
-
version:
|
75
|
+
version: 1.1.0
|
62
76
|
type: :development
|
63
77
|
prerelease: false
|
64
78
|
version_requirements: !ruby/object:Gem::Requirement
|
65
79
|
requirements:
|
66
80
|
- - "~>"
|
67
81
|
- !ruby/object:Gem::Version
|
68
|
-
version:
|
82
|
+
version: 1.1.0
|
69
83
|
- !ruby/object:Gem::Dependency
|
70
84
|
name: rake
|
71
85
|
requirement: !ruby/object:Gem::Requirement
|
72
86
|
requirements:
|
73
87
|
- - "~>"
|
74
88
|
- !ruby/object:Gem::Version
|
75
|
-
version:
|
89
|
+
version: 13.0.6
|
76
90
|
type: :development
|
77
91
|
prerelease: false
|
78
92
|
version_requirements: !ruby/object:Gem::Requirement
|
79
93
|
requirements:
|
80
94
|
- - "~>"
|
81
95
|
- !ruby/object:Gem::Version
|
82
|
-
version:
|
96
|
+
version: 13.0.6
|
83
97
|
- !ruby/object:Gem::Dependency
|
84
98
|
name: rspec
|
85
99
|
requirement: !ruby/object:Gem::Requirement
|
@@ -100,70 +114,94 @@ dependencies:
|
|
100
114
|
requirements:
|
101
115
|
- - '='
|
102
116
|
- !ruby/object:Gem::Version
|
103
|
-
version:
|
117
|
+
version: 1.20.0
|
104
118
|
type: :development
|
105
119
|
prerelease: false
|
106
120
|
version_requirements: !ruby/object:Gem::Requirement
|
107
121
|
requirements:
|
108
122
|
- - '='
|
109
123
|
- !ruby/object:Gem::Version
|
110
|
-
version:
|
124
|
+
version: 1.20.0
|
111
125
|
- !ruby/object:Gem::Dependency
|
112
126
|
name: aasm
|
113
127
|
requirement: !ruby/object:Gem::Requirement
|
114
128
|
requirements:
|
115
|
-
- - "
|
129
|
+
- - ">="
|
116
130
|
- !ruby/object:Gem::Version
|
117
|
-
version: 4.12
|
131
|
+
version: '4.12'
|
132
|
+
- - "<"
|
133
|
+
- !ruby/object:Gem::Version
|
134
|
+
version: '5.3'
|
118
135
|
type: :runtime
|
119
136
|
prerelease: false
|
120
137
|
version_requirements: !ruby/object:Gem::Requirement
|
121
138
|
requirements:
|
122
|
-
- - "
|
139
|
+
- - ">="
|
140
|
+
- !ruby/object:Gem::Version
|
141
|
+
version: '4.12'
|
142
|
+
- - "<"
|
123
143
|
- !ruby/object:Gem::Version
|
124
|
-
version:
|
144
|
+
version: '5.3'
|
125
145
|
- !ruby/object:Gem::Dependency
|
126
146
|
name: aws-sdk
|
127
147
|
requirement: !ruby/object:Gem::Requirement
|
128
148
|
requirements:
|
129
|
-
- - "
|
149
|
+
- - ">="
|
150
|
+
- !ruby/object:Gem::Version
|
151
|
+
version: '3.0'
|
152
|
+
- - "<"
|
130
153
|
- !ruby/object:Gem::Version
|
131
|
-
version: 3.
|
154
|
+
version: '3.2'
|
132
155
|
type: :runtime
|
133
156
|
prerelease: false
|
134
157
|
version_requirements: !ruby/object:Gem::Requirement
|
135
158
|
requirements:
|
136
|
-
- - "
|
159
|
+
- - ">="
|
137
160
|
- !ruby/object:Gem::Version
|
138
|
-
version: 3.0
|
161
|
+
version: '3.0'
|
162
|
+
- - "<"
|
163
|
+
- !ruby/object:Gem::Version
|
164
|
+
version: '3.2'
|
139
165
|
- !ruby/object:Gem::Dependency
|
140
166
|
name: faraday
|
141
167
|
requirement: !ruby/object:Gem::Requirement
|
142
168
|
requirements:
|
143
|
-
- - "
|
169
|
+
- - ">="
|
144
170
|
- !ruby/object:Gem::Version
|
145
|
-
version: 0.13
|
171
|
+
version: '0.13'
|
172
|
+
- - "<"
|
173
|
+
- !ruby/object:Gem::Version
|
174
|
+
version: '1.8'
|
146
175
|
type: :runtime
|
147
176
|
prerelease: false
|
148
177
|
version_requirements: !ruby/object:Gem::Requirement
|
149
178
|
requirements:
|
150
|
-
- - "
|
179
|
+
- - ">="
|
180
|
+
- !ruby/object:Gem::Version
|
181
|
+
version: '0.13'
|
182
|
+
- - "<"
|
151
183
|
- !ruby/object:Gem::Version
|
152
|
-
version:
|
184
|
+
version: '1.8'
|
153
185
|
- !ruby/object:Gem::Dependency
|
154
186
|
name: msgpack
|
155
187
|
requirement: !ruby/object:Gem::Requirement
|
156
188
|
requirements:
|
157
|
-
- - "
|
189
|
+
- - ">="
|
190
|
+
- !ruby/object:Gem::Version
|
191
|
+
version: '1.2'
|
192
|
+
- - "<"
|
158
193
|
- !ruby/object:Gem::Version
|
159
|
-
version: 1.
|
194
|
+
version: '1.5'
|
160
195
|
type: :runtime
|
161
196
|
prerelease: false
|
162
197
|
version_requirements: !ruby/object:Gem::Requirement
|
163
198
|
requirements:
|
164
|
-
- - "
|
199
|
+
- - ">="
|
165
200
|
- !ruby/object:Gem::Version
|
166
|
-
version: 1.2
|
201
|
+
version: '1.2'
|
202
|
+
- - "<"
|
203
|
+
- !ruby/object:Gem::Version
|
204
|
+
version: '1.5'
|
167
205
|
- !ruby/object:Gem::Dependency
|
168
206
|
name: rasteira
|
169
207
|
requirement: !ruby/object:Gem::Requirement
|
@@ -182,44 +220,62 @@ dependencies:
|
|
182
220
|
name: sinatra
|
183
221
|
requirement: !ruby/object:Gem::Requirement
|
184
222
|
requirements:
|
185
|
-
- - "
|
223
|
+
- - ">="
|
224
|
+
- !ruby/object:Gem::Version
|
225
|
+
version: '2.0'
|
226
|
+
- - "<"
|
186
227
|
- !ruby/object:Gem::Version
|
187
|
-
version: 2.
|
228
|
+
version: '2.2'
|
188
229
|
type: :runtime
|
189
230
|
prerelease: false
|
190
231
|
version_requirements: !ruby/object:Gem::Requirement
|
191
232
|
requirements:
|
192
|
-
- - "
|
233
|
+
- - ">="
|
234
|
+
- !ruby/object:Gem::Version
|
235
|
+
version: '2.0'
|
236
|
+
- - "<"
|
193
237
|
- !ruby/object:Gem::Version
|
194
|
-
version: 2.
|
238
|
+
version: '2.2'
|
195
239
|
- !ruby/object:Gem::Dependency
|
196
240
|
name: sinatra-contrib
|
197
241
|
requirement: !ruby/object:Gem::Requirement
|
198
242
|
requirements:
|
199
|
-
- - "
|
243
|
+
- - ">="
|
200
244
|
- !ruby/object:Gem::Version
|
201
|
-
version: 2.0
|
245
|
+
version: '2.0'
|
246
|
+
- - "<"
|
247
|
+
- !ruby/object:Gem::Version
|
248
|
+
version: '2.2'
|
202
249
|
type: :runtime
|
203
250
|
prerelease: false
|
204
251
|
version_requirements: !ruby/object:Gem::Requirement
|
205
252
|
requirements:
|
206
|
-
- - "
|
253
|
+
- - ">="
|
207
254
|
- !ruby/object:Gem::Version
|
208
|
-
version: 2.0
|
255
|
+
version: '2.0'
|
256
|
+
- - "<"
|
257
|
+
- !ruby/object:Gem::Version
|
258
|
+
version: '2.2'
|
209
259
|
- !ruby/object:Gem::Dependency
|
210
260
|
name: thor
|
211
261
|
requirement: !ruby/object:Gem::Requirement
|
212
262
|
requirements:
|
213
|
-
- - "
|
263
|
+
- - ">="
|
264
|
+
- !ruby/object:Gem::Version
|
265
|
+
version: '0.20'
|
266
|
+
- - "<"
|
214
267
|
- !ruby/object:Gem::Version
|
215
|
-
version:
|
268
|
+
version: '1.2'
|
216
269
|
type: :runtime
|
217
270
|
prerelease: false
|
218
271
|
version_requirements: !ruby/object:Gem::Requirement
|
219
272
|
requirements:
|
220
|
-
- - "
|
273
|
+
- - ">="
|
274
|
+
- !ruby/object:Gem::Version
|
275
|
+
version: '0.20'
|
276
|
+
- - "<"
|
221
277
|
- !ruby/object:Gem::Version
|
222
|
-
version:
|
278
|
+
version: '1.2'
|
223
279
|
description: Simple MapReduce framework
|
224
280
|
email:
|
225
281
|
- nserihiro@gmail.com
|
@@ -228,13 +284,6 @@ executables:
|
|
228
284
|
extensions: []
|
229
285
|
extra_rdoc_files: []
|
230
286
|
files:
|
231
|
-
- ".github/ISSUE_TEMPLATE.md"
|
232
|
-
- ".github/PULL_REQUEST_TEMPLATE.md"
|
233
|
-
- ".gitignore"
|
234
|
-
- ".rspec"
|
235
|
-
- ".rubocop.yml"
|
236
|
-
- ".ruby-version"
|
237
|
-
- ".travis.yml"
|
238
287
|
- CODE_OF_CONDUCT.md
|
239
288
|
- Dockerfile
|
240
289
|
- Gemfile
|
@@ -281,15 +330,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
281
330
|
requirements:
|
282
331
|
- - ">="
|
283
332
|
- !ruby/object:Gem::Version
|
284
|
-
version: 2.
|
333
|
+
version: 2.7.4
|
285
334
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
286
335
|
requirements:
|
287
336
|
- - ">="
|
288
337
|
- !ruby/object:Gem::Version
|
289
338
|
version: '0'
|
290
339
|
requirements: []
|
291
|
-
|
292
|
-
rubygems_version: 2.7.3
|
340
|
+
rubygems_version: 3.1.6
|
293
341
|
signing_key:
|
294
342
|
specification_version: 4
|
295
343
|
summary: Simple MapReduce framework
|
data/.github/ISSUE_TEMPLATE.md
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
# What is this issue about ?
|
@@ -1 +0,0 @@
|
|
1
|
-
# What will this PR change ?
|
data/.gitignore
DELETED
@@ -1,141 +0,0 @@
|
|
1
|
-
### https://raw.github.com/github/gitignore/44cbb3686c18f634a488ea123d1148ca9a64fa22/Ruby.gitignore
|
2
|
-
|
3
|
-
*.gem
|
4
|
-
*.rbc
|
5
|
-
/.config
|
6
|
-
/coverage/
|
7
|
-
/InstalledFiles
|
8
|
-
/pkg/
|
9
|
-
/spec/reports/
|
10
|
-
/spec/examples.txt
|
11
|
-
/test/tmp/
|
12
|
-
/test/version_tmp/
|
13
|
-
/tmp/
|
14
|
-
/Gemfile.lock
|
15
|
-
|
16
|
-
# Used by dotenv library to load environment variables.
|
17
|
-
# .env
|
18
|
-
|
19
|
-
## Specific to RubyMotion:
|
20
|
-
.dat*
|
21
|
-
.repl_history
|
22
|
-
build/
|
23
|
-
*.bridgesupport
|
24
|
-
build-iPhoneOS/
|
25
|
-
build-iPhoneSimulator/
|
26
|
-
|
27
|
-
## Specific to RubyMotion (use of CocoaPods):
|
28
|
-
#
|
29
|
-
# We recommend against adding the Pods directory to your .gitignore. However
|
30
|
-
# you should judge for yourself, the pros and cons are mentioned at:
|
31
|
-
# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
|
32
|
-
#
|
33
|
-
# vendor/Pods/
|
34
|
-
|
35
|
-
## Documentation cache and generated files:
|
36
|
-
/.yardoc/
|
37
|
-
/_yardoc/
|
38
|
-
/doc/
|
39
|
-
/rdoc/
|
40
|
-
|
41
|
-
## Environment normalization:
|
42
|
-
/.bundle/
|
43
|
-
/vendor/bundle
|
44
|
-
/vendor/ruby
|
45
|
-
/lib/bundler/man/
|
46
|
-
|
47
|
-
# for a library or gem, you might want to ignore these files since the code is
|
48
|
-
# intended to run in multiple environments; otherwise, check them in:
|
49
|
-
# Gemfile.lock
|
50
|
-
# .ruby-version
|
51
|
-
# .ruby-gemset
|
52
|
-
|
53
|
-
# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
|
54
|
-
.rvmrc
|
55
|
-
|
56
|
-
|
57
|
-
### https://raw.github.com/github/gitignore/44cbb3686c18f634a488ea123d1148ca9a64fa22/Global/JetBrains.gitignore
|
58
|
-
|
59
|
-
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
|
60
|
-
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
|
61
|
-
|
62
|
-
# User-specific stuff:
|
63
|
-
.idea/**/workspace.xml
|
64
|
-
.idea/**/tasks.xml
|
65
|
-
.idea/dictionaries
|
66
|
-
|
67
|
-
# Sensitive or high-churn files:
|
68
|
-
.idea/**/dataSources/
|
69
|
-
.idea/**/dataSources.ids
|
70
|
-
.idea/**/dataSources.xml
|
71
|
-
.idea/**/dataSources.local.xml
|
72
|
-
.idea/**/sqlDataSources.xml
|
73
|
-
.idea/**/dynamic.xml
|
74
|
-
.idea/**/uiDesigner.xml
|
75
|
-
|
76
|
-
# Gradle:
|
77
|
-
.idea/**/gradle.xml
|
78
|
-
.idea/**/libraries
|
79
|
-
|
80
|
-
# CMake
|
81
|
-
cmake-build-debug/
|
82
|
-
|
83
|
-
# Mongo Explorer plugin:
|
84
|
-
.idea/**/mongoSettings.xml
|
85
|
-
|
86
|
-
## File-based project format:
|
87
|
-
*.iws
|
88
|
-
|
89
|
-
## Plugin-specific files:
|
90
|
-
|
91
|
-
# IntelliJ
|
92
|
-
/out/
|
93
|
-
|
94
|
-
# mpeltonen/sbt-idea plugin
|
95
|
-
.idea_modules/
|
96
|
-
|
97
|
-
# JIRA plugin
|
98
|
-
atlassian-ide-plugin.xml
|
99
|
-
|
100
|
-
# Cursive Clojure plugin
|
101
|
-
.idea/replstate.xml
|
102
|
-
|
103
|
-
# Crashlytics plugin (for Android Studio and IntelliJ)
|
104
|
-
com_crashlytics_export_strings.xml
|
105
|
-
crashlytics.properties
|
106
|
-
crashlytics-build.properties
|
107
|
-
fabric.properties
|
108
|
-
|
109
|
-
|
110
|
-
### https://raw.github.com/github/gitignore/44cbb3686c18f634a488ea123d1148ca9a64fa22/Global/MacOS.gitignore
|
111
|
-
|
112
|
-
*.DS_Store
|
113
|
-
.AppleDouble
|
114
|
-
.LSOverride
|
115
|
-
|
116
|
-
# Icon must end with two \r
|
117
|
-
Icon
|
118
|
-
|
119
|
-
|
120
|
-
# Thumbnails
|
121
|
-
._*
|
122
|
-
|
123
|
-
# Files that might appear in the root of a volume
|
124
|
-
.DocumentRevisions-V100
|
125
|
-
.fseventsd
|
126
|
-
.Spotlight-V100
|
127
|
-
.TemporaryItems
|
128
|
-
.Trashes
|
129
|
-
.VolumeIcon.icns
|
130
|
-
.com.apple.timemachine.donotpresent
|
131
|
-
|
132
|
-
# Directories potentially created on remote AFP share
|
133
|
-
.AppleDB
|
134
|
-
.AppleDesktop
|
135
|
-
Network Trash Folder
|
136
|
-
Temporary Items
|
137
|
-
.apdisk
|
138
|
-
.rspec_status
|
139
|
-
/input.txt
|
140
|
-
|
141
|
-
!.gitkeep
|
data/.rspec
DELETED
data/.rubocop.yml
DELETED
@@ -1,69 +0,0 @@
|
|
1
|
-
AllCops:
|
2
|
-
TargetRubyVersion: 2.3
|
3
|
-
DisplayCopNames: true
|
4
|
-
|
5
|
-
PercentLiteralDelimiters:
|
6
|
-
PreferredDelimiters:
|
7
|
-
'%': ()
|
8
|
-
'%i': ()
|
9
|
-
'%q': ()
|
10
|
-
'%Q': ()
|
11
|
-
'%r': ()
|
12
|
-
'%s': ()
|
13
|
-
'%w': ()
|
14
|
-
'%W': ()
|
15
|
-
'%x': ()
|
16
|
-
|
17
|
-
Metrics/BlockLength:
|
18
|
-
Enabled: false
|
19
|
-
|
20
|
-
Metrics/LineLength:
|
21
|
-
Max: 128
|
22
|
-
|
23
|
-
Metrics/ClassLength:
|
24
|
-
Enabled: false
|
25
|
-
|
26
|
-
Metrics/MethodLength:
|
27
|
-
Enabled: false
|
28
|
-
|
29
|
-
Style/IfUnlessModifier:
|
30
|
-
Enabled: false
|
31
|
-
|
32
|
-
Style/GuardClause:
|
33
|
-
Enabled: false
|
34
|
-
|
35
|
-
Layout/AlignArray:
|
36
|
-
Enabled: false
|
37
|
-
|
38
|
-
Layout/AlignHash:
|
39
|
-
Enabled: false
|
40
|
-
|
41
|
-
Layout/AlignParameters:
|
42
|
-
Enabled: false
|
43
|
-
|
44
|
-
Style/RescueStandardError:
|
45
|
-
Enabled: false
|
46
|
-
|
47
|
-
Naming/HeredocDelimiterNaming:
|
48
|
-
Enabled: false
|
49
|
-
|
50
|
-
Style/Documentation:
|
51
|
-
Enabled: false
|
52
|
-
|
53
|
-
Style/NumericPredicate:
|
54
|
-
Enabled: false
|
55
|
-
|
56
|
-
Metrics/AbcSize:
|
57
|
-
Enabled: false
|
58
|
-
|
59
|
-
Metrics/ParameterLists:
|
60
|
-
Enabled: false
|
61
|
-
|
62
|
-
Lint/AmbiguousBlockAssociation:
|
63
|
-
Enabled: false
|
64
|
-
|
65
|
-
Metrics/CyclomaticComplexity:
|
66
|
-
Max: 10
|
67
|
-
|
68
|
-
Metrics/PerceivedComplexity:
|
69
|
-
Max: 10
|
data/.ruby-version
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
2.5.0
|