fairy 0.6.0
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENSE +674 -0
- data/Makefile +116 -0
- data/README +15 -0
- data/bin/fairy +582 -0
- data/bin/fairy-cat +74 -0
- data/bin/fairy-cp +128 -0
- data/bin/fairy-rm +122 -0
- data/bin/subcmd/controller +41 -0
- data/bin/subcmd/inspector +81 -0
- data/bin/subcmd/master +43 -0
- data/bin/subcmd/node +47 -0
- data/bin/subcmd/processor +54 -0
- data/doc/programming-interface.html +240 -0
- data/doc/programming-interface.rd +300 -0
- data/etc/fairy.conf.tmpl +118 -0
- data/ext/simple_hash/extconf.rb +4 -0
- data/ext/simple_hash/simple_hash.c +42 -0
- data/fairy.gemspec +60 -0
- data/lib/fairy/client/addins.rb +20 -0
- data/lib/fairy/client/barrier.rb +29 -0
- data/lib/fairy/client/basic-group-by.rb +52 -0
- data/lib/fairy/client/cat.rb +41 -0
- data/lib/fairy/client/direct-product.rb +51 -0
- data/lib/fairy/client/equijoin.rb +79 -0
- data/lib/fairy/client/exec.rb +54 -0
- data/lib/fairy/client/filter.rb +62 -0
- data/lib/fairy/client/find.rb +35 -0
- data/lib/fairy/client/group-by.rb +194 -0
- data/lib/fairy/client/here.rb +84 -0
- data/lib/fairy/client/inject.rb +70 -0
- data/lib/fairy/client/input-file.rb +53 -0
- data/lib/fairy/client/input-iota.rb +49 -0
- data/lib/fairy/client/input-local-file.rb +188 -0
- data/lib/fairy/client/input-varray.rb +30 -0
- data/lib/fairy/client/input.rb +42 -0
- data/lib/fairy/client/io-filter.rb +26 -0
- data/lib/fairy/client/junction.rb +31 -0
- data/lib/fairy/client/map.rb +34 -0
- data/lib/fairy/client/merge-group-by.rb +71 -0
- data/lib/fairy/client/output-file.rb +64 -0
- data/lib/fairy/client/output-local-file.rb +60 -0
- data/lib/fairy/client/output-null.rb +47 -0
- data/lib/fairy/client/output-varray.rb +50 -0
- data/lib/fairy/client/output.rb +29 -0
- data/lib/fairy/client/roma-put.rb +62 -0
- data/lib/fairy/client/roma.rb +156 -0
- data/lib/fairy/client/seg-join.rb +61 -0
- data/lib/fairy/client/seg-map.rb +78 -0
- data/lib/fairy/client/seg-shuffle.rb +35 -0
- data/lib/fairy/client/seg-split.rb +27 -0
- data/lib/fairy/client/seg-zip.rb +60 -0
- data/lib/fairy/client/select.rb +38 -0
- data/lib/fairy/client/sort.rb +48 -0
- data/lib/fairy/client/sort18.rb +56 -0
- data/lib/fairy/client/sort19.rb +61 -0
- data/lib/fairy/client/there.rb +47 -0
- data/lib/fairy/client/top_n_into_roma.rb +34 -0
- data/lib/fairy/client/wc.rb +92 -0
- data/lib/fairy/controller.rb +1103 -0
- data/lib/fairy/logger.rb +107 -0
- data/lib/fairy/master/addins.rb +20 -0
- data/lib/fairy/master/atom.rb +17 -0
- data/lib/fairy/master/c-barrier.rb +283 -0
- data/lib/fairy/master/c-basic-group-by.rb +250 -0
- data/lib/fairy/master/c-cat.rb +159 -0
- data/lib/fairy/master/c-direct-product.rb +203 -0
- data/lib/fairy/master/c-exec.rb +68 -0
- data/lib/fairy/master/c-filter.rb +422 -0
- data/lib/fairy/master/c-find.rb +138 -0
- data/lib/fairy/master/c-group-by.rb +64 -0
- data/lib/fairy/master/c-here.rb +80 -0
- data/lib/fairy/master/c-inject.rb +119 -0
- data/lib/fairy/master/c-input-file.rb +46 -0
- data/lib/fairy/master/c-input-iota.rb +66 -0
- data/lib/fairy/master/c-input-local-file.rb +117 -0
- data/lib/fairy/master/c-input-varray.rb +53 -0
- data/lib/fairy/master/c-input.rb +24 -0
- data/lib/fairy/master/c-inputtable.rb +31 -0
- data/lib/fairy/master/c-inputtable18.rb +36 -0
- data/lib/fairy/master/c-inputtable19.rb +35 -0
- data/lib/fairy/master/c-io-filter.rb +28 -0
- data/lib/fairy/master/c-junction.rb +54 -0
- data/lib/fairy/master/c-map.rb +27 -0
- data/lib/fairy/master/c-merge-group-by.rb +241 -0
- data/lib/fairy/master/c-output-file.rb +84 -0
- data/lib/fairy/master/c-output-local-file.rb +19 -0
- data/lib/fairy/master/c-output-null.rb +45 -0
- data/lib/fairy/master/c-output-varray.rb +57 -0
- data/lib/fairy/master/c-output.rb +20 -0
- data/lib/fairy/master/c-seg-join.rb +141 -0
- data/lib/fairy/master/c-seg-map.rb +26 -0
- data/lib/fairy/master/c-seg-shuffle.rb +87 -0
- data/lib/fairy/master/c-seg-split.rb +110 -0
- data/lib/fairy/master/c-seg-zip.rb +132 -0
- data/lib/fairy/master/c-select.rb +27 -0
- data/lib/fairy/master/c-sort.rb +108 -0
- data/lib/fairy/master/c-there.rb +57 -0
- data/lib/fairy/master/c-wc.rb +232 -0
- data/lib/fairy/master/job-interpriter.rb +19 -0
- data/lib/fairy/master/scheduler.rb +24 -0
- data/lib/fairy/master.rb +329 -0
- data/lib/fairy/node/addins.rb +19 -0
- data/lib/fairy/node/p-barrier.rb +95 -0
- data/lib/fairy/node/p-basic-group-by.rb +252 -0
- data/lib/fairy/node/p-direct-product.rb +153 -0
- data/lib/fairy/node/p-exec.rb +30 -0
- data/lib/fairy/node/p-filter.rb +363 -0
- data/lib/fairy/node/p-find.rb +111 -0
- data/lib/fairy/node/p-group-by.rb +1534 -0
- data/lib/fairy/node/p-here.rb +21 -0
- data/lib/fairy/node/p-identity.rb +24 -0
- data/lib/fairy/node/p-inject.rb +127 -0
- data/lib/fairy/node/p-input-file.rb +108 -0
- data/lib/fairy/node/p-input-iota.rb +39 -0
- data/lib/fairy/node/p-input-local-file.rb +61 -0
- data/lib/fairy/node/p-input-varray.rb +26 -0
- data/lib/fairy/node/p-io-filter.rb +28 -0
- data/lib/fairy/node/p-map.rb +40 -0
- data/lib/fairy/node/p-merger-group-by.rb +48 -0
- data/lib/fairy/node/p-output-file.rb +104 -0
- data/lib/fairy/node/p-output-local-file.rb +14 -0
- data/lib/fairy/node/p-output-null.rb +32 -0
- data/lib/fairy/node/p-output-varray.rb +41 -0
- data/lib/fairy/node/p-seg-join.rb +82 -0
- data/lib/fairy/node/p-seg-map.rb +34 -0
- data/lib/fairy/node/p-seg-split.rb +61 -0
- data/lib/fairy/node/p-seg-zip.rb +79 -0
- data/lib/fairy/node/p-select.rb +40 -0
- data/lib/fairy/node/p-single-exportable.rb +90 -0
- data/lib/fairy/node/p-sort.rb +195 -0
- data/lib/fairy/node/p-task.rb +113 -0
- data/lib/fairy/node/p-there.rb +44 -0
- data/lib/fairy/node/p-wc.rb +266 -0
- data/lib/fairy/node.rb +187 -0
- data/lib/fairy/processor.rb +510 -0
- data/lib/fairy/share/base-app.rb +114 -0
- data/lib/fairy/share/block-source.rb +234 -0
- data/lib/fairy/share/conf.rb +396 -0
- data/lib/fairy/share/debug.rb +21 -0
- data/lib/fairy/share/encoding.rb +17 -0
- data/lib/fairy/share/fast-tempfile.rb +93 -0
- data/lib/fairy/share/file-place.rb +176 -0
- data/lib/fairy/share/hash-1.rb +20 -0
- data/lib/fairy/share/hash-md5.rb +28 -0
- data/lib/fairy/share/hash-murmur.rb +69 -0
- data/lib/fairy/share/hash-rb18.rb +20 -0
- data/lib/fairy/share/hash-simple-hash.rb +28 -0
- data/lib/fairy/share/inspector.rb +16 -0
- data/lib/fairy/share/lc/exceptions.rb +82 -0
- data/lib/fairy/share/lc/ja/exceptions.rb +81 -0
- data/lib/fairy/share/locale.rb +17 -0
- data/lib/fairy/share/log.rb +215 -0
- data/lib/fairy/share/pool-dictionary.rb +53 -0
- data/lib/fairy/share/port-marshaled-queue.rb +347 -0
- data/lib/fairy/share/port.rb +1697 -0
- data/lib/fairy/share/reference.rb +45 -0
- data/lib/fairy/share/stdout.rb +56 -0
- data/lib/fairy/share/tr.rb +16 -0
- data/lib/fairy/share/varray.rb +147 -0
- data/lib/fairy/share/vfile.rb +183 -0
- data/lib/fairy/version.rb +8 -0
- data/lib/fairy.rb +206 -0
- data/sample/grep.rb +46 -0
- data/sample/ping.rb +19 -0
- data/sample/sort.rb +102 -0
- data/sample/wordcount.rb +61 -0
- data/spec/README +12 -0
- data/spec/fairy1_spec.rb +31 -0
- data/spec/fairy2_spec.rb +42 -0
- data/spec/fairy3_spec.rb +126 -0
- data/spec/fairy4_spec.rb +63 -0
- data/spec/fairy5_spec.rb +45 -0
- data/spec/fairy6_spec.rb +52 -0
- data/spec/fairy7_spec.rb +58 -0
- data/spec/fairy8_spec.rb +48 -0
- data/spec/mkdat.rb +148 -0
- data/spec/run_all.sh +65 -0
- data/test/testc.rb +7111 -0
- data/tools/cap_recipe/Capfile +144 -0
- data/tools/cap_recipe/cluster.yml.sample +14 -0
- data/tools/fairy_perf_graph.rb +444 -0
- data/tools/git-tag +44 -0
- data/tools/log-analysis.rb +62 -0
- data/tools/svn-ls-diff +38 -0
- data/tools/svn-tags +37 -0
- metadata +298 -0
data/Makefile
ADDED
@@ -0,0 +1,116 @@
|
|
1
|
+
|
2
|
+
CONF = etc/fairy.conf
|
3
|
+
|
4
|
+
BINSRCS = \
|
5
|
+
bin/master \
|
6
|
+
bin/node \
|
7
|
+
bin/controller \
|
8
|
+
bin/processor \
|
9
|
+
\
|
10
|
+
bin/fairy \
|
11
|
+
bin/fairy-cp \
|
12
|
+
bin/fairy-cat \
|
13
|
+
bin/fairy-rm \
|
14
|
+
bin/inspector
|
15
|
+
|
16
|
+
|
17
|
+
TEMPLS = $(wildcard lib/fairy/template/*.tmpl)
|
18
|
+
|
19
|
+
DEEPCONNECTSRCS = $(wildcard deep-connect/*.rb)
|
20
|
+
|
21
|
+
SRCS = Makefile ChangeLog TODO \
|
22
|
+
$(CONF) \
|
23
|
+
$(BINSRCS) \
|
24
|
+
$(TMPLS) \
|
25
|
+
$(wildcard lib/*.rb \
|
26
|
+
lib/fairy/*.rb \
|
27
|
+
lib/fairy/job/*.rb \
|
28
|
+
lib/fairy/backend/*.rb \
|
29
|
+
lib/fairy/node/*.rb \
|
30
|
+
lib/fairy/share/*.rb \
|
31
|
+
sample/*.rb \
|
32
|
+
sample/*/*.rb \
|
33
|
+
sample/*/*/*.rb \
|
34
|
+
test/*.rb)
|
35
|
+
|
36
|
+
TS = TimeStamps
|
37
|
+
|
38
|
+
FAIRYSHELL = bin/fairy
|
39
|
+
FAIRYSHELL_TEMPL0 = \
|
40
|
+
lib/fairy/template/fairy-HEAD.templ
|
41
|
+
FAIRYSHELL_TEMPL = \
|
42
|
+
lib/fairy/share/conf.rb \
|
43
|
+
lib/fairy/share/base-app.rb \
|
44
|
+
lib/fairy/template/fairy-BODY.templ
|
45
|
+
|
46
|
+
bin/fairy: $(FAIRYSHELL_TEMPL0) $(FAIRYSHELL_TEMPL)
|
47
|
+
@echo "Code generation $@"
|
48
|
+
-@chmod -f +w $@
|
49
|
+
@cat $(FAIRYSHELL_TEMPL0) > $@
|
50
|
+
@for f in $(FAIRYSHELL_TEMPL); do \
|
51
|
+
echo " genalate from: $$f" ;\
|
52
|
+
echo "" >> $@ ;\
|
53
|
+
echo "##Original from $$f" >> $@ ;\
|
54
|
+
cat $$f >> $@ ;\
|
55
|
+
done
|
56
|
+
@chmod +x,-w $@
|
57
|
+
#
|
58
|
+
test-clean:
|
59
|
+
rm -fr test/Repos
|
60
|
+
|
61
|
+
# git
|
62
|
+
commit: $(TS)/commit
|
63
|
+
|
64
|
+
$(TS)/commit: $(SRCS)
|
65
|
+
git commit -a
|
66
|
+
touch $(TS)/commit
|
67
|
+
|
68
|
+
diff:
|
69
|
+
git diff
|
70
|
+
|
71
|
+
tag-%:
|
72
|
+
echo "Make tag $*"
|
73
|
+
tools/git-tag $*
|
74
|
+
|
75
|
+
push: doc/programming-interface.html bin/fairy
|
76
|
+
git push origin
|
77
|
+
|
78
|
+
push-dev:
|
79
|
+
git push origin refs/heads/dev
|
80
|
+
|
81
|
+
push-tags:
|
82
|
+
git push --tags origin
|
83
|
+
|
84
|
+
push-github:
|
85
|
+
git push github
|
86
|
+
|
87
|
+
doc/programming-interface.html: doc/programming-interface.rd
|
88
|
+
tools/rd2html
|
89
|
+
|
90
|
+
# gem
|
91
|
+
gem:
|
92
|
+
gem build fairy.gemspec
|
93
|
+
|
94
|
+
# tar archives
|
95
|
+
TGZ_FILES = $(SRCS) $(DEEPCONNECTSRCS)
|
96
|
+
|
97
|
+
|
98
|
+
SNAPSHOT = Snapshot
|
99
|
+
|
100
|
+
VERSION = $(shell ruby -r lib/fairy/version.rb -e "puts Fairy::Version")
|
101
|
+
|
102
|
+
PACKAGE_NAME = fairy
|
103
|
+
|
104
|
+
TAR_NAME = $(PACKAGE_NAME)-$(VERSION).tgz
|
105
|
+
|
106
|
+
tgz: $(SNAPSHOT)/$(TAR_NAME)
|
107
|
+
|
108
|
+
$(SNAPSHOT)/$(TAR_NAME): $(TGZ_FILES)
|
109
|
+
@if [ ! -e $(SNAPSHOT) ]; then \
|
110
|
+
mkdir $(SNAPSHOT); \
|
111
|
+
fi
|
112
|
+
@echo "make $(TAR_NAME) in $(SNAPSHOT)"
|
113
|
+
@tar zcf $(SNAPSHOT)/$(TAR_NAME) $(TGZ_FILES)
|
114
|
+
@echo "copy $(TAR_NAME) to /tmp/Downloads"
|
115
|
+
@cp -p $(SNAPSHOT)/$(TAR_NAME) /tmp/Downloads
|
116
|
+
|
data/README
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
fairy: A framework for distributed processing in Ruby
|
2
|
+
Copyright (c) Rakuten, Inc. All Right Reserved
|
3
|
+
|
4
|
+
fairy is a framework for distributed processing in Ruby, originally
|
5
|
+
designed at Rakuten Institute of Technology with Yukihiro Matsumoto,
|
6
|
+
the founder of Ruby.
|
7
|
+
|
8
|
+
Although fairy was inspired by MapReduce model, a well-known
|
9
|
+
programming model for distributed processing, it's more flexible and
|
10
|
+
suitable for wider use. That's due to fairy's programming model,
|
11
|
+
called filter IF, and various built-in filters.
|
12
|
+
|
13
|
+
fairy is implemented in Ruby and inherits its high productivity and
|
14
|
+
simplicity. fairy's API is quite similar to Ruby. Therefore most
|
15
|
+
programmers who know Ruby can easily understand and use it.
|
data/bin/fairy
ADDED
@@ -0,0 +1,582 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# encoding: UTF-8
|
3
|
+
|
4
|
+
# THIS FILE IS AUTO GENERATION.
|
5
|
+
# Don't modify this file.
|
6
|
+
|
7
|
+
require "optparse"
|
8
|
+
|
9
|
+
##Original from lib/fairy/share/conf.rb
|
10
|
+
# encoding: UTF-8
|
11
|
+
#
|
12
|
+
# Copyright (C) 2007-2010 Rakuten, Inc.
|
13
|
+
#
|
14
|
+
|
15
|
+
require "socket"
|
16
|
+
require "rbconfig"
|
17
|
+
|
18
|
+
module Fairy
|
19
|
+
|
20
|
+
CONF_PATH = [
|
21
|
+
"/etc/fairy.conf",
|
22
|
+
RbConfig::CONFIG["libdir"]+"/fairy/etc/fairy.conf",
|
23
|
+
ENV["FAIRY_HOME"] && ENV["FAIRY_HOME"]+"/etc/fairy.conf",
|
24
|
+
ENV["HOME"]+"/.fairyrc",
|
25
|
+
ENV["FAIRY_CONF"]
|
26
|
+
]
|
27
|
+
# $:.each{|p|
|
28
|
+
# CONF_PATH.push p+"/fairy/etc/fairy.conf"
|
29
|
+
# }
|
30
|
+
|
31
|
+
class Conf
|
32
|
+
PROPS = []
|
33
|
+
class DefaultConf<Conf;end
|
34
|
+
|
35
|
+
def initialize(super_conf = nil, opts = {})
|
36
|
+
@super_conf = super_conf
|
37
|
+
@values = opts
|
38
|
+
end
|
39
|
+
|
40
|
+
class<<Conf
|
41
|
+
def def_prop(prop, reader = nil, setter = nil)
|
42
|
+
PROPS.push prop
|
43
|
+
reader = "def #{prop}; value(:#{prop}); end" unless reader
|
44
|
+
setter = "def #{prop}=(value); @values[:#{prop}] = value; end" unless setter
|
45
|
+
module_eval reader
|
46
|
+
module_eval setter
|
47
|
+
# DefaultConf.module_eval reader
|
48
|
+
end
|
49
|
+
|
50
|
+
def def_prop_relative_path(prop, path, base_prop = :HOME)
|
51
|
+
def_prop(prop,
|
52
|
+
"def #{prop}; value(:#{prop}) || self.#{base_prop}+'/'+'#{path}'; end")
|
53
|
+
end
|
54
|
+
alias def_rpath def_prop_relative_path
|
55
|
+
end
|
56
|
+
|
57
|
+
def value(prop)
|
58
|
+
if (v = @values[prop]).nil?
|
59
|
+
@super_conf && v = @super_conf.value(prop)
|
60
|
+
end
|
61
|
+
v
|
62
|
+
end
|
63
|
+
|
64
|
+
def props
|
65
|
+
PROPS
|
66
|
+
end
|
67
|
+
|
68
|
+
def base_conf=(conf)
|
69
|
+
if @super_conf
|
70
|
+
@super_conf.base_conf = conf
|
71
|
+
else
|
72
|
+
@super_conf = conf
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def set_values(hash)
|
77
|
+
for key, value in hash
|
78
|
+
@values[key] = value
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
def_prop :RUBY_BIN
|
83
|
+
def_prop :MASTER_HOST
|
84
|
+
def_prop :MASTER_PORT
|
85
|
+
|
86
|
+
def_prop :HOME
|
87
|
+
def HOME
|
88
|
+
File.expand_path(value(:HOME))
|
89
|
+
end
|
90
|
+
|
91
|
+
def_rpath :BIN, "bin"
|
92
|
+
def_rpath :LIB, "lib"
|
93
|
+
def_rpath :SUBCMD, "subcmd", :BIN
|
94
|
+
def_rpath :CONTROLLER_BIN, "controller", :SUBCMD
|
95
|
+
def_rpath :PROCESSOR_BIN, "processor", :SUBCMD
|
96
|
+
|
97
|
+
def_prop :DEFAULT_EXTERNAL
|
98
|
+
def_prop :DEFAULT_INTERNAL
|
99
|
+
|
100
|
+
def_prop :IGNORE_EXCEPTION_ON_FILTER
|
101
|
+
|
102
|
+
def_prop :MASTER_MAX_ACTIVE_PROCESSORS
|
103
|
+
|
104
|
+
def_prop :CONTROLLER_INPUT_PROCESSOR_N
|
105
|
+
def_prop :CONTROLLER_MAX_ACTIVE_TASKS_IN_PROCESSOR
|
106
|
+
|
107
|
+
def_prop :CONTROLLER_ASSIGN_NEW_PROCESSOR_N_FACTOR
|
108
|
+
|
109
|
+
def_prop :PREQUEUING_POLICY
|
110
|
+
def_prop :POSTQUEUING_POLICY
|
111
|
+
|
112
|
+
def_prop :POSTMAPPING_POLICY
|
113
|
+
|
114
|
+
def_prop :POSTQUEUE_MAX_TRANSFER_SIZE
|
115
|
+
def_prop :POOLQUEUE_POOL_THRESHOLD
|
116
|
+
def_prop :ONMEMORY_SIZEDQUEUE_SIZE
|
117
|
+
def_prop :FILEBUFFEREDQUEUE_THRESHOLD
|
118
|
+
|
119
|
+
def_prop :MARSHAL_QUEUE_CHUNK_SIZE
|
120
|
+
def_prop :MARSHAL_QUEUE_MIN_CHUNK_NO
|
121
|
+
def_prop :SIZEDMARSHAL_QUEUE_MAX_CHUNK_NO
|
122
|
+
|
123
|
+
def_prop :SORTEDQUEUE_POOL_THRESHOLD
|
124
|
+
def_prop :SORTEDQUEUE_THRESHOLD
|
125
|
+
def_prop :SORTEDQUEUE_SORTBY
|
126
|
+
|
127
|
+
def_prop :INPUT_LOCAL_FILE_BUFFER_SIZE
|
128
|
+
def_prop :HERE_POOL_THRESHOLD
|
129
|
+
#def_prop :OUTPUT_LOCAL_POOL_THRESHOLD
|
130
|
+
|
131
|
+
def_prop :GROUP_BY_NO_SEGMENT
|
132
|
+
def_prop :GROUP_BY_HASH_MODULE
|
133
|
+
def_prop :GROUP_BY_GROUPING_OPTIMIZE
|
134
|
+
|
135
|
+
def_prop :GROUP_BY_BUFFERING_POLICY
|
136
|
+
def_prop :GROUP_BY_CMSB_THRESHOLD
|
137
|
+
def_prop :GROUP_BY_CMSB_CHUNK_SIZE
|
138
|
+
|
139
|
+
def_prop :BARRIER_MEMORY_BUFFERING_POLICY
|
140
|
+
|
141
|
+
def_prop :SORT_BUFFERING_POLICY
|
142
|
+
def_prop :SORT_SAMPLING_MIN
|
143
|
+
def_prop :SORT_SAMPLING_MAX
|
144
|
+
def_prop :SORT_SAMPLING_RATIO_1_TO
|
145
|
+
def_prop :SORT_NO_SEGMENT
|
146
|
+
def_prop :SORT_CMP_OPTIMIZE
|
147
|
+
|
148
|
+
def_prop :IOTA_SPLIT_NO
|
149
|
+
|
150
|
+
def_prop :TRANSFAR_MARSHAL_STRING_ARRAY_OPTIMIZE
|
151
|
+
|
152
|
+
def_rpath :VF_ROOT, "Repos"
|
153
|
+
def_prop :VF_PREFIX
|
154
|
+
def_prop :VF_BASE_NAME_CONVERTER
|
155
|
+
def_prop :VF_SPLIT_SIZE
|
156
|
+
|
157
|
+
def_prop :TMP_DIR
|
158
|
+
|
159
|
+
def_prop :LOG_FILE
|
160
|
+
def_prop :LOG_LEVEL
|
161
|
+
def_prop :LOG_FLUSH_INTERVAL
|
162
|
+
def_prop :LOG_MARK_INTERVAL
|
163
|
+
def_prop :LOG_IMPORT_NTIMES_POP
|
164
|
+
def_prop :LOG_LOCAL_OUTPUT_DEV
|
165
|
+
def_prop :LOG_ROTATE_INTERVAL
|
166
|
+
def_prop :LOG_ROTATE_N
|
167
|
+
|
168
|
+
def_prop :SUBCMD_EXEC_TIMEOUT
|
169
|
+
|
170
|
+
def_prop :PROCESSOR_MON_ON
|
171
|
+
def_prop :PROCESSOR_MON_INTERVAL
|
172
|
+
def_prop :PROCESSOR_MON_PSFORMAT
|
173
|
+
def_prop :PROCESSOR_MON_OBJECTSPACE_INSPECT_ON
|
174
|
+
|
175
|
+
def_prop :SOCK_DO_NOT_REVERSE_LOOKUP
|
176
|
+
def_prop :USE_RESOLV_REPLACE
|
177
|
+
|
178
|
+
def_prop :BLOCK_USE_STDOUT
|
179
|
+
|
180
|
+
def_prop :DEBUG_PORT_WAIT
|
181
|
+
def_prop :DEBUG_FULL_BACKTRACE
|
182
|
+
def_prop :DEBUG_THREAD_ABORT_ON_EXCEPTION
|
183
|
+
def_prop :DEBUG_MONITOR_ON
|
184
|
+
def_prop :DEBUG_PROCESSOR_TRACE_ON
|
185
|
+
def_prop :DEBUG_BUG49
|
186
|
+
|
187
|
+
def_prop :BUG234
|
188
|
+
|
189
|
+
def_prop :PROCESS_LIFE_MANAGE_INTERVAL
|
190
|
+
|
191
|
+
def_prop :THREAD_STACK_SIZE
|
192
|
+
|
193
|
+
# class JobConf<Conf
|
194
|
+
# def initialize(prefix = "", super_conf = nil, opts ={})
|
195
|
+
# @prefix = prefix.upcase
|
196
|
+
# end
|
197
|
+
|
198
|
+
# def method_missing(method, *args, &b)
|
199
|
+
# del = "#{prefix}_#{method.id2name.upcase}"
|
200
|
+
# if respond_to?(del)
|
201
|
+
# __send__(del, *args, &b)
|
202
|
+
# else
|
203
|
+
# super
|
204
|
+
# end
|
205
|
+
# end
|
206
|
+
# end
|
207
|
+
# BJobConf = JobConf
|
208
|
+
# NJobConf = JobConf
|
209
|
+
|
210
|
+
class DefaultConf
|
211
|
+
|
212
|
+
def initialize
|
213
|
+
super
|
214
|
+
|
215
|
+
@default_host = `hostname`.chomp
|
216
|
+
@hosts = {}
|
217
|
+
end
|
218
|
+
|
219
|
+
def [](host)
|
220
|
+
unless @hosts[host]
|
221
|
+
@hosts[host] = Conf.new
|
222
|
+
end
|
223
|
+
@hosts[host]
|
224
|
+
end
|
225
|
+
|
226
|
+
def value(prop)
|
227
|
+
conf = @hosts[@default_host]
|
228
|
+
v = conf && conf.value(prop)
|
229
|
+
return v if v
|
230
|
+
@values[prop]
|
231
|
+
end
|
232
|
+
|
233
|
+
def load_all_conf
|
234
|
+
loaded = false
|
235
|
+
for path in CONF_PATH
|
236
|
+
if path
|
237
|
+
if File.exist?(path)
|
238
|
+
load_conf path
|
239
|
+
loaded = true
|
240
|
+
end
|
241
|
+
end
|
242
|
+
end
|
243
|
+
# unless loaded
|
244
|
+
# puts "warnfairy.confファイルが見つかりません"
|
245
|
+
# end
|
246
|
+
end
|
247
|
+
|
248
|
+
def load_conf(conf)
|
249
|
+
begin
|
250
|
+
load conf
|
251
|
+
rescue LoadError
|
252
|
+
puts "設定ファイル #{conf} をロードできませんでした."
|
253
|
+
exit 1
|
254
|
+
rescue
|
255
|
+
puts "設定ファイル #{@conf} をロード中に例外が発生しました."
|
256
|
+
p $!
|
257
|
+
p $@
|
258
|
+
exit 2
|
259
|
+
end
|
260
|
+
end
|
261
|
+
end
|
262
|
+
end
|
263
|
+
|
264
|
+
DEFAULT_CONF = Conf::DefaultConf.new
|
265
|
+
|
266
|
+
DEFAULT_CONF.RUBY_BIN = ENV["FAIRY_RUBY"] || "ruby"
|
267
|
+
|
268
|
+
#DEFAULT_CONF.MASTER_HOST =
|
269
|
+
DEFAULT_CONF.MASTER_PORT = 19999
|
270
|
+
|
271
|
+
# DEFAULT_CONF.HOME = ENV["FAIRY_HOME"] || "/usr/lib/fairy"
|
272
|
+
home = ENV["FAIRY_HOME"]
|
273
|
+
unless home
|
274
|
+
home = File.dirname(File.dirname(File.expand_path(__FILE__)))
|
275
|
+
if File.basename(__FILE__) == "conf.rb"
|
276
|
+
home = File.dirname(File.dirname(home))
|
277
|
+
end
|
278
|
+
end
|
279
|
+
DEFAULT_CONF.HOME = home
|
280
|
+
# DEFAULT_CONF.BIN = DEFAULT_CONF.HOME+"/bin"
|
281
|
+
# DEFAULT_CONF.LIB = DEFAULT_CONF.HOME+"/lib"
|
282
|
+
|
283
|
+
# DEFAULT_CONF.CONTROLLER_BIN = DEFAULT_CONF.BIN+"/controller"
|
284
|
+
# DEFAULT_CONF.PROCESSOR_BIN = DEFAULT_CONF.BIN+"/processor"
|
285
|
+
|
286
|
+
DEFAULT_CONF.DEFAULT_EXTERNAL = nil
|
287
|
+
DEFAULT_CONF.DEFAULT_INTERNAL = nil
|
288
|
+
|
289
|
+
DEFAULT_CONF.IGNORE_EXCEPTION_ON_FILTER = false
|
290
|
+
|
291
|
+
DEFAULT_CONF.MASTER_MAX_ACTIVE_PROCESSORS = 4
|
292
|
+
|
293
|
+
DEFAULT_CONF.CONTROLLER_INPUT_PROCESSOR_N = 10
|
294
|
+
DEFAULT_CONF.CONTROLLER_MAX_ACTIVE_TASKS_IN_PROCESSOR = 4
|
295
|
+
|
296
|
+
DEFAULT_CONF.CONTROLLER_ASSIGN_NEW_PROCESSOR_N_FACTOR = 1
|
297
|
+
|
298
|
+
DEFAULT_CONF.POSTMAPPING_POLICY = nil
|
299
|
+
|
300
|
+
DEFAULT_CONF.PREQUEUING_POLICY = {:queuing_class => :FileMarshaledQueue}
|
301
|
+
DEFAULT_CONF.POSTQUEUING_POLICY = {:queuing_class => :FileMarshaledQueue}
|
302
|
+
|
303
|
+
DEFAULT_CONF.POSTQUEUE_MAX_TRANSFER_SIZE = 100000
|
304
|
+
DEFAULT_CONF.POOLQUEUE_POOL_THRESHOLD = 10000
|
305
|
+
DEFAULT_CONF.ONMEMORY_SIZEDQUEUE_SIZE = 10000
|
306
|
+
DEFAULT_CONF.FILEBUFFEREDQUEUE_THRESHOLD = 10000/2
|
307
|
+
|
308
|
+
DEFAULT_CONF.MARSHAL_QUEUE_CHUNK_SIZE = DEFAULT_CONF.POOLQUEUE_POOL_THRESHOLD
|
309
|
+
DEFAULT_CONF.MARSHAL_QUEUE_MIN_CHUNK_NO = DEFAULT_CONF.POOLQUEUE_POOL_THRESHOLD
|
310
|
+
DEFAULT_CONF.SIZEDMARSHAL_QUEUE_MAX_CHUNK_NO = 10
|
311
|
+
|
312
|
+
DEFAULT_CONF.SORTEDQUEUE_POOL_THRESHOLD = DEFAULT_CONF.POOLQUEUE_POOL_THRESHOLD
|
313
|
+
DEFAULT_CONF.SORTEDQUEUE_THRESHOLD = 10000/2
|
314
|
+
DEFAULT_CONF.SORTEDQUEUE_SORTBY = %{|v| v}
|
315
|
+
|
316
|
+
DEFAULT_CONF.INPUT_LOCAL_FILE_BUFFER_SIZE = 1024*1024
|
317
|
+
DEFAULT_CONF.HERE_POOL_THRESHOLD = 32000
|
318
|
+
#DEFAULT_CONF.OUTPUT_LOCAL_POOL_THRESHOLD = 32000
|
319
|
+
|
320
|
+
DEFAULT_CONF.GROUP_BY_NO_SEGMENT = 4
|
321
|
+
DEFAULT_CONF.GROUP_BY_HASH_MODULE = "fairy/share/hash-md5"
|
322
|
+
DEFAULT_CONF.GROUP_BY_GROUPING_OPTIMIZE = false
|
323
|
+
|
324
|
+
DEFAULT_CONF.GROUP_BY_BUFFERING_POLICY = {:buffering_class => :DirectMergeSortBuffer}
|
325
|
+
DEFAULT_CONF.GROUP_BY_CMSB_THRESHOLD = 400_000
|
326
|
+
DEFAULT_CONF.GROUP_BY_CMSB_CHUNK_SIZE = 1000
|
327
|
+
|
328
|
+
DEFAULT_CONF.BARRIER_MEMORY_BUFFERING_POLICY = {:queuing_class => :PoolQueue}
|
329
|
+
|
330
|
+
DEFAULT_CONF.SORT_BUFFERING_POLICY = {:buffering_class => "PGroupBy::DirectMergeSortBuffer"}
|
331
|
+
DEFAULT_CONF.SORT_SAMPLING_MIN = 100
|
332
|
+
DEFAULT_CONF.SORT_SAMPLING_MAX = 10000
|
333
|
+
DEFAULT_CONF.SORT_SAMPLING_RATIO_1_TO = 100
|
334
|
+
DEFAULT_CONF.SORT_NO_SEGMENT = DEFAULT_CONF.GROUP_BY_NO_SEGMENT
|
335
|
+
DEFAULT_CONF.SORT_CMP_OPTIMIZE = false
|
336
|
+
|
337
|
+
DEFAULT_CONF.IOTA_SPLIT_NO = 4
|
338
|
+
|
339
|
+
DEFAULT_CONF.TRANSFAR_MARSHAL_STRING_ARRAY_OPTIMIZE = false
|
340
|
+
|
341
|
+
# DEFAULT_CONF.VF_ROOT = DEFAULT_CONF.HOME+"/Repos"
|
342
|
+
DEFAULT_CONF.VF_PREFIX = `hostname`.chomp
|
343
|
+
## DEFAULT_CONF.VF_PREFIX is client setting.
|
344
|
+
DEFAULT_CONF.VF_BASE_NAME_CONVERTER = nil
|
345
|
+
DEFAULT_CONF.VF_SPLIT_SIZE = 64*1024*1024
|
346
|
+
|
347
|
+
DEFAULT_CONF.TMP_DIR = "/tmp/fairy/tmpbuf"
|
348
|
+
|
349
|
+
DEFAULT_CONF.LOG_FILE = "/tmp/fairy/log"
|
350
|
+
DEFAULT_CONF.LOG_FLUSH_INTERVAL = 1
|
351
|
+
DEFAULT_CONF.LOG_MARK_INTERVAL = 300
|
352
|
+
DEFAULT_CONF.LOG_LEVEL = :INFO
|
353
|
+
DEFAULT_CONF.LOG_IMPORT_NTIMES_POP = 100000
|
354
|
+
DEFAULT_CONF.LOG_LOCAL_OUTPUT_DEV = :$stderr
|
355
|
+
DEFAULT_CONF.LOG_ROTATE_INTERVAL = 60*60*24
|
356
|
+
DEFAULT_CONF.LOG_ROTATE_N = 7
|
357
|
+
|
358
|
+
DEFAULT_CONF.SUBCMD_EXEC_TIMEOUT = 60
|
359
|
+
|
360
|
+
DEFAULT_CONF.PROCESSOR_MON_ON = false
|
361
|
+
DEFAULT_CONF.PROCESSOR_MON_INTERVAL = 300
|
362
|
+
DEFAULT_CONF.PROCESSOR_MON_PSFORMAT = "stat,vsz,rss,sz,pmem,pcpu,nlwp,time,wchan"
|
363
|
+
DEFAULT_CONF.PROCESSOR_MON_OBJECTSPACE_INSPECT_ON = false
|
364
|
+
|
365
|
+
DEFAULT_CONF.SOCK_DO_NOT_REVERSE_LOOKUP = true
|
366
|
+
DEFAULT_CONF.USE_RESOLV_REPLACE = false
|
367
|
+
|
368
|
+
DEFAULT_CONF.BLOCK_USE_STDOUT = true
|
369
|
+
|
370
|
+
DEFAULT_CONF.DEBUG_PORT_WAIT = false
|
371
|
+
DEFAULT_CONF.DEBUG_FULL_BACKTRACE = false
|
372
|
+
DEFAULT_CONF.DEBUG_THREAD_ABORT_ON_EXCEPTION = false
|
373
|
+
DEFAULT_CONF.DEBUG_MONITOR_ON = false
|
374
|
+
DEFAULT_CONF.DEBUG_PROCESSOR_TRACE_ON = false
|
375
|
+
DEFAULT_CONF.DEBUG_BUG49 = false
|
376
|
+
|
377
|
+
DEFAULT_CONF.BUG234 = false
|
378
|
+
|
379
|
+
# DEFAULT_CONF.PROCESS_LIFE_MANAGE_INTERVAL = 60
|
380
|
+
# DEFAULT_CONF.PROCESS_LIFE_MANAGE_INTERVAL = 10
|
381
|
+
# DEFAULT_CONF.PROCESS_LIFE_MANAGE_INTERVAL = 1
|
382
|
+
DEFAULT_CONF.PROCESS_LIFE_MANAGE_INTERVAL = nil
|
383
|
+
|
384
|
+
# DEFAULT_CONF.THREAD_STACK_SIZE = 1024*100
|
385
|
+
|
386
|
+
CONF = DEFAULT_CONF
|
387
|
+
CONF.load_all_conf
|
388
|
+
|
389
|
+
def REPLACE_CONF(conf)
|
390
|
+
remove_const(:CONF)
|
391
|
+
const_set(:CONF, conf)
|
392
|
+
end
|
393
|
+
module_function :REPLACE_CONF
|
394
|
+
|
395
|
+
|
396
|
+
def Conf.configure_common_conf
|
397
|
+
Thread.abort_on_exception = CONF.DEBUG_THREAD_ABORT_ON_EXCEPTION
|
398
|
+
|
399
|
+
TCPSocket.do_not_reverse_lookup = CONF.SOCK_DO_NOT_REVERSE_LOOKUP
|
400
|
+
|
401
|
+
if CONF.USE_RESOLV_REPLACE
|
402
|
+
require "resolv-replace"
|
403
|
+
end
|
404
|
+
end
|
405
|
+
end
|
406
|
+
|
407
|
+
##Original from lib/fairy/share/base-app.rb
|
408
|
+
# encoding: utf-8
|
409
|
+
#
|
410
|
+
# Copyright (C) 2007-2010 Rakuten, Inc.
|
411
|
+
#
|
412
|
+
|
413
|
+
module Fairy
|
414
|
+
class BaseAPP
|
415
|
+
|
416
|
+
def self.start
|
417
|
+
@@APP = new
|
418
|
+
@@APP.parse_arg
|
419
|
+
@@APP.load_conf
|
420
|
+
@@APP.configure_conf
|
421
|
+
@@APP.start
|
422
|
+
end
|
423
|
+
|
424
|
+
def self.start_subcommand(prog, *opts)
|
425
|
+
@@APP.start_subcommand(prog, *opts)
|
426
|
+
end
|
427
|
+
def self.start_subcommand2(prog, *opts)
|
428
|
+
Process.fork do
|
429
|
+
start_subcommand(prog, *opts)
|
430
|
+
end
|
431
|
+
end
|
432
|
+
|
433
|
+
def initialize
|
434
|
+
@home = nil
|
435
|
+
@conf = nil
|
436
|
+
@CONF = {}
|
437
|
+
end
|
438
|
+
|
439
|
+
def parse_arg
|
440
|
+
@opt = option_parser
|
441
|
+
@opt.order!(ARGV)
|
442
|
+
end
|
443
|
+
|
444
|
+
def option_parser
|
445
|
+
opt = OptionParser.new{|opt|
|
446
|
+
opt.on("--home=VAL"){|val| @home = val}
|
447
|
+
opt.on("--conf=VAL"){|val| @conf = val}
|
448
|
+
CONF.props.each do |prop|
|
449
|
+
opt.on("--CONF.#{prop}=VAL") {|val| @CONF[prop] = val}
|
450
|
+
end
|
451
|
+
}
|
452
|
+
if block_given?
|
453
|
+
yield opt
|
454
|
+
end
|
455
|
+
end
|
456
|
+
|
457
|
+
def conf_to_arg
|
458
|
+
@CONF.collect{|prop, source| "--CONF.#{prop}=#{source}"}
|
459
|
+
end
|
460
|
+
|
461
|
+
def load_conf
|
462
|
+
if @home
|
463
|
+
ENV["FAIRY_HOME"] = @home
|
464
|
+
# ENV["FAIRY_HOME"] = File.expand_path(@home)
|
465
|
+
conf = @home+"/etc/fairy.conf"
|
466
|
+
if File.exists?(conf)
|
467
|
+
CONF.load_conf conf
|
468
|
+
end
|
469
|
+
end
|
470
|
+
|
471
|
+
if @conf
|
472
|
+
ENV["FAIRY_CONF"] = @conf
|
473
|
+
CONF.load_conf @conf
|
474
|
+
end
|
475
|
+
|
476
|
+
@CONF.each do |key, value|
|
477
|
+
begin
|
478
|
+
eval "CONF.#{key}=#{value}"
|
479
|
+
rescue
|
480
|
+
puts "CONF.#{key}=#{value} が不正です."
|
481
|
+
exit 1
|
482
|
+
end
|
483
|
+
end
|
484
|
+
if ENV["RUBYLIB"]
|
485
|
+
ENV["RUBYLIB"] = CONF.LIB + ":" + ENV["RUBYLIB"]
|
486
|
+
else
|
487
|
+
ENV["RUBYLIB"] = CONF.LIB
|
488
|
+
end
|
489
|
+
end
|
490
|
+
|
491
|
+
def configure_conf
|
492
|
+
Conf.configure_common_conf
|
493
|
+
end
|
494
|
+
|
495
|
+
def start
|
496
|
+
ERR::Raise ERR::INTERNAL::ShouldDefineSubclass
|
497
|
+
end
|
498
|
+
|
499
|
+
def start_subcommand(prog, *opts)
|
500
|
+
opts.push *conf_to_arg
|
501
|
+
# Process.fork do
|
502
|
+
# Log.stop_export
|
503
|
+
# ObjectSpace.each_object(IO) do |io|
|
504
|
+
# begin
|
505
|
+
# if ![0, 1, 2].include?(io.fileno )
|
506
|
+
# io.close
|
507
|
+
# end
|
508
|
+
# rescue
|
509
|
+
# end
|
510
|
+
# end
|
511
|
+
# exec(prog, *opts)
|
512
|
+
# end
|
513
|
+
|
514
|
+
Process.spawn(prog, *opts)
|
515
|
+
# system("#{prog} #{opts.join(' ')}&")
|
516
|
+
|
517
|
+
end
|
518
|
+
|
519
|
+
end
|
520
|
+
end
|
521
|
+
|
522
|
+
|
523
|
+
##Original from lib/fairy/template/fairy-BODY.templ
|
524
|
+
# -*- ruby -*-
|
525
|
+
#
|
526
|
+
# Copyright (C) 2007-2010 Rakuten, Inc.
|
527
|
+
#
|
528
|
+
#
|
529
|
+
module Fairy
|
530
|
+
# LC_MESSAGES = IRB::Locale.new
|
531
|
+
# LC_MESSAGES.load(CONF.LIB+"/fairy/share/exceptions.rb")
|
532
|
+
class FairyAPP<BaseAPP
|
533
|
+
|
534
|
+
SUBCOMMANDS = ["cp", "rm", "cat", "master", "node", "inspector"]
|
535
|
+
|
536
|
+
def initialize
|
537
|
+
super
|
538
|
+
@mode = :SUBCOMMAND
|
539
|
+
end
|
540
|
+
|
541
|
+
def option_parser
|
542
|
+
super do |opt|
|
543
|
+
opt.on("-f", "--file"){|val| @mode = :FILE}
|
544
|
+
opt.on("-e", "--exec"){|val| @mode = :EXEC}
|
545
|
+
end
|
546
|
+
end
|
547
|
+
|
548
|
+
def start
|
549
|
+
if @mode == :SUBCOMMAND &&
|
550
|
+
(ARGV.empty? || nosub = !SUBCOMMANDS.include?(ARGV[0]))
|
551
|
+
puts "そのようなサブコマンドはありません(#{ARGV[0]})." if nosub
|
552
|
+
|
553
|
+
helpbase = @opt.help.split(/\n/)
|
554
|
+
helpbase[0].concat(" subcommand [subcommand-options]")
|
555
|
+
helpbase.insert(1, " Options:")
|
556
|
+
helpbase.insert(1,
|
557
|
+
" Subcommands:",
|
558
|
+
*SUBCOMMANDS.collect{|s| " "+s})
|
559
|
+
|
560
|
+
puts helpbase
|
561
|
+
exit
|
562
|
+
end
|
563
|
+
|
564
|
+
case @mode
|
565
|
+
when :SUBCOMMAND
|
566
|
+
if File.exist?(CONF.SUBCMD+"/"+ARGV[0])
|
567
|
+
Process.exec(CONF.RUBY_BIN, CONF.SUBCMD+"/"+ARGV[0], *ARGV[1..-1])
|
568
|
+
else
|
569
|
+
Process.exec(CONF.RUBY_BIN, CONF.BIN+"/fairy-"+ARGV[0], *ARGV[1..-1])
|
570
|
+
end
|
571
|
+
when :FILE
|
572
|
+
Process.exec(CONF.RUBY_BIN, *ARGV)
|
573
|
+
when :EXEC
|
574
|
+
Process.exec(*ARGV)
|
575
|
+
end
|
576
|
+
end
|
577
|
+
end
|
578
|
+
end
|
579
|
+
|
580
|
+
Fairy::FairyAPP.start
|
581
|
+
|
582
|
+
|