fairy 0.6.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (186) hide show
  1. data/LICENSE +674 -0
  2. data/Makefile +116 -0
  3. data/README +15 -0
  4. data/bin/fairy +582 -0
  5. data/bin/fairy-cat +74 -0
  6. data/bin/fairy-cp +128 -0
  7. data/bin/fairy-rm +122 -0
  8. data/bin/subcmd/controller +41 -0
  9. data/bin/subcmd/inspector +81 -0
  10. data/bin/subcmd/master +43 -0
  11. data/bin/subcmd/node +47 -0
  12. data/bin/subcmd/processor +54 -0
  13. data/doc/programming-interface.html +240 -0
  14. data/doc/programming-interface.rd +300 -0
  15. data/etc/fairy.conf.tmpl +118 -0
  16. data/ext/simple_hash/extconf.rb +4 -0
  17. data/ext/simple_hash/simple_hash.c +42 -0
  18. data/fairy.gemspec +60 -0
  19. data/lib/fairy/client/addins.rb +20 -0
  20. data/lib/fairy/client/barrier.rb +29 -0
  21. data/lib/fairy/client/basic-group-by.rb +52 -0
  22. data/lib/fairy/client/cat.rb +41 -0
  23. data/lib/fairy/client/direct-product.rb +51 -0
  24. data/lib/fairy/client/equijoin.rb +79 -0
  25. data/lib/fairy/client/exec.rb +54 -0
  26. data/lib/fairy/client/filter.rb +62 -0
  27. data/lib/fairy/client/find.rb +35 -0
  28. data/lib/fairy/client/group-by.rb +194 -0
  29. data/lib/fairy/client/here.rb +84 -0
  30. data/lib/fairy/client/inject.rb +70 -0
  31. data/lib/fairy/client/input-file.rb +53 -0
  32. data/lib/fairy/client/input-iota.rb +49 -0
  33. data/lib/fairy/client/input-local-file.rb +188 -0
  34. data/lib/fairy/client/input-varray.rb +30 -0
  35. data/lib/fairy/client/input.rb +42 -0
  36. data/lib/fairy/client/io-filter.rb +26 -0
  37. data/lib/fairy/client/junction.rb +31 -0
  38. data/lib/fairy/client/map.rb +34 -0
  39. data/lib/fairy/client/merge-group-by.rb +71 -0
  40. data/lib/fairy/client/output-file.rb +64 -0
  41. data/lib/fairy/client/output-local-file.rb +60 -0
  42. data/lib/fairy/client/output-null.rb +47 -0
  43. data/lib/fairy/client/output-varray.rb +50 -0
  44. data/lib/fairy/client/output.rb +29 -0
  45. data/lib/fairy/client/roma-put.rb +62 -0
  46. data/lib/fairy/client/roma.rb +156 -0
  47. data/lib/fairy/client/seg-join.rb +61 -0
  48. data/lib/fairy/client/seg-map.rb +78 -0
  49. data/lib/fairy/client/seg-shuffle.rb +35 -0
  50. data/lib/fairy/client/seg-split.rb +27 -0
  51. data/lib/fairy/client/seg-zip.rb +60 -0
  52. data/lib/fairy/client/select.rb +38 -0
  53. data/lib/fairy/client/sort.rb +48 -0
  54. data/lib/fairy/client/sort18.rb +56 -0
  55. data/lib/fairy/client/sort19.rb +61 -0
  56. data/lib/fairy/client/there.rb +47 -0
  57. data/lib/fairy/client/top_n_into_roma.rb +34 -0
  58. data/lib/fairy/client/wc.rb +92 -0
  59. data/lib/fairy/controller.rb +1103 -0
  60. data/lib/fairy/logger.rb +107 -0
  61. data/lib/fairy/master/addins.rb +20 -0
  62. data/lib/fairy/master/atom.rb +17 -0
  63. data/lib/fairy/master/c-barrier.rb +283 -0
  64. data/lib/fairy/master/c-basic-group-by.rb +250 -0
  65. data/lib/fairy/master/c-cat.rb +159 -0
  66. data/lib/fairy/master/c-direct-product.rb +203 -0
  67. data/lib/fairy/master/c-exec.rb +68 -0
  68. data/lib/fairy/master/c-filter.rb +422 -0
  69. data/lib/fairy/master/c-find.rb +138 -0
  70. data/lib/fairy/master/c-group-by.rb +64 -0
  71. data/lib/fairy/master/c-here.rb +80 -0
  72. data/lib/fairy/master/c-inject.rb +119 -0
  73. data/lib/fairy/master/c-input-file.rb +46 -0
  74. data/lib/fairy/master/c-input-iota.rb +66 -0
  75. data/lib/fairy/master/c-input-local-file.rb +117 -0
  76. data/lib/fairy/master/c-input-varray.rb +53 -0
  77. data/lib/fairy/master/c-input.rb +24 -0
  78. data/lib/fairy/master/c-inputtable.rb +31 -0
  79. data/lib/fairy/master/c-inputtable18.rb +36 -0
  80. data/lib/fairy/master/c-inputtable19.rb +35 -0
  81. data/lib/fairy/master/c-io-filter.rb +28 -0
  82. data/lib/fairy/master/c-junction.rb +54 -0
  83. data/lib/fairy/master/c-map.rb +27 -0
  84. data/lib/fairy/master/c-merge-group-by.rb +241 -0
  85. data/lib/fairy/master/c-output-file.rb +84 -0
  86. data/lib/fairy/master/c-output-local-file.rb +19 -0
  87. data/lib/fairy/master/c-output-null.rb +45 -0
  88. data/lib/fairy/master/c-output-varray.rb +57 -0
  89. data/lib/fairy/master/c-output.rb +20 -0
  90. data/lib/fairy/master/c-seg-join.rb +141 -0
  91. data/lib/fairy/master/c-seg-map.rb +26 -0
  92. data/lib/fairy/master/c-seg-shuffle.rb +87 -0
  93. data/lib/fairy/master/c-seg-split.rb +110 -0
  94. data/lib/fairy/master/c-seg-zip.rb +132 -0
  95. data/lib/fairy/master/c-select.rb +27 -0
  96. data/lib/fairy/master/c-sort.rb +108 -0
  97. data/lib/fairy/master/c-there.rb +57 -0
  98. data/lib/fairy/master/c-wc.rb +232 -0
  99. data/lib/fairy/master/job-interpriter.rb +19 -0
  100. data/lib/fairy/master/scheduler.rb +24 -0
  101. data/lib/fairy/master.rb +329 -0
  102. data/lib/fairy/node/addins.rb +19 -0
  103. data/lib/fairy/node/p-barrier.rb +95 -0
  104. data/lib/fairy/node/p-basic-group-by.rb +252 -0
  105. data/lib/fairy/node/p-direct-product.rb +153 -0
  106. data/lib/fairy/node/p-exec.rb +30 -0
  107. data/lib/fairy/node/p-filter.rb +363 -0
  108. data/lib/fairy/node/p-find.rb +111 -0
  109. data/lib/fairy/node/p-group-by.rb +1534 -0
  110. data/lib/fairy/node/p-here.rb +21 -0
  111. data/lib/fairy/node/p-identity.rb +24 -0
  112. data/lib/fairy/node/p-inject.rb +127 -0
  113. data/lib/fairy/node/p-input-file.rb +108 -0
  114. data/lib/fairy/node/p-input-iota.rb +39 -0
  115. data/lib/fairy/node/p-input-local-file.rb +61 -0
  116. data/lib/fairy/node/p-input-varray.rb +26 -0
  117. data/lib/fairy/node/p-io-filter.rb +28 -0
  118. data/lib/fairy/node/p-map.rb +40 -0
  119. data/lib/fairy/node/p-merger-group-by.rb +48 -0
  120. data/lib/fairy/node/p-output-file.rb +104 -0
  121. data/lib/fairy/node/p-output-local-file.rb +14 -0
  122. data/lib/fairy/node/p-output-null.rb +32 -0
  123. data/lib/fairy/node/p-output-varray.rb +41 -0
  124. data/lib/fairy/node/p-seg-join.rb +82 -0
  125. data/lib/fairy/node/p-seg-map.rb +34 -0
  126. data/lib/fairy/node/p-seg-split.rb +61 -0
  127. data/lib/fairy/node/p-seg-zip.rb +79 -0
  128. data/lib/fairy/node/p-select.rb +40 -0
  129. data/lib/fairy/node/p-single-exportable.rb +90 -0
  130. data/lib/fairy/node/p-sort.rb +195 -0
  131. data/lib/fairy/node/p-task.rb +113 -0
  132. data/lib/fairy/node/p-there.rb +44 -0
  133. data/lib/fairy/node/p-wc.rb +266 -0
  134. data/lib/fairy/node.rb +187 -0
  135. data/lib/fairy/processor.rb +510 -0
  136. data/lib/fairy/share/base-app.rb +114 -0
  137. data/lib/fairy/share/block-source.rb +234 -0
  138. data/lib/fairy/share/conf.rb +396 -0
  139. data/lib/fairy/share/debug.rb +21 -0
  140. data/lib/fairy/share/encoding.rb +17 -0
  141. data/lib/fairy/share/fast-tempfile.rb +93 -0
  142. data/lib/fairy/share/file-place.rb +176 -0
  143. data/lib/fairy/share/hash-1.rb +20 -0
  144. data/lib/fairy/share/hash-md5.rb +28 -0
  145. data/lib/fairy/share/hash-murmur.rb +69 -0
  146. data/lib/fairy/share/hash-rb18.rb +20 -0
  147. data/lib/fairy/share/hash-simple-hash.rb +28 -0
  148. data/lib/fairy/share/inspector.rb +16 -0
  149. data/lib/fairy/share/lc/exceptions.rb +82 -0
  150. data/lib/fairy/share/lc/ja/exceptions.rb +81 -0
  151. data/lib/fairy/share/locale.rb +17 -0
  152. data/lib/fairy/share/log.rb +215 -0
  153. data/lib/fairy/share/pool-dictionary.rb +53 -0
  154. data/lib/fairy/share/port-marshaled-queue.rb +347 -0
  155. data/lib/fairy/share/port.rb +1697 -0
  156. data/lib/fairy/share/reference.rb +45 -0
  157. data/lib/fairy/share/stdout.rb +56 -0
  158. data/lib/fairy/share/tr.rb +16 -0
  159. data/lib/fairy/share/varray.rb +147 -0
  160. data/lib/fairy/share/vfile.rb +183 -0
  161. data/lib/fairy/version.rb +8 -0
  162. data/lib/fairy.rb +206 -0
  163. data/sample/grep.rb +46 -0
  164. data/sample/ping.rb +19 -0
  165. data/sample/sort.rb +102 -0
  166. data/sample/wordcount.rb +61 -0
  167. data/spec/README +12 -0
  168. data/spec/fairy1_spec.rb +31 -0
  169. data/spec/fairy2_spec.rb +42 -0
  170. data/spec/fairy3_spec.rb +126 -0
  171. data/spec/fairy4_spec.rb +63 -0
  172. data/spec/fairy5_spec.rb +45 -0
  173. data/spec/fairy6_spec.rb +52 -0
  174. data/spec/fairy7_spec.rb +58 -0
  175. data/spec/fairy8_spec.rb +48 -0
  176. data/spec/mkdat.rb +148 -0
  177. data/spec/run_all.sh +65 -0
  178. data/test/testc.rb +7111 -0
  179. data/tools/cap_recipe/Capfile +144 -0
  180. data/tools/cap_recipe/cluster.yml.sample +14 -0
  181. data/tools/fairy_perf_graph.rb +444 -0
  182. data/tools/git-tag +44 -0
  183. data/tools/log-analysis.rb +62 -0
  184. data/tools/svn-ls-diff +38 -0
  185. data/tools/svn-tags +37 -0
  186. metadata +298 -0
data/Makefile ADDED
@@ -0,0 +1,116 @@
1
+
2
+ CONF = etc/fairy.conf
3
+
4
+ BINSRCS = \
5
+ bin/master \
6
+ bin/node \
7
+ bin/controller \
8
+ bin/processor \
9
+ \
10
+ bin/fairy \
11
+ bin/fairy-cp \
12
+ bin/fairy-cat \
13
+ bin/fairy-rm \
14
+ bin/inspector
15
+
16
+
17
+ TEMPLS = $(wildcard lib/fairy/template/*.tmpl)
18
+
19
+ DEEPCONNECTSRCS = $(wildcard deep-connect/*.rb)
20
+
21
+ SRCS = Makefile ChangeLog TODO \
22
+ $(CONF) \
23
+ $(BINSRCS) \
24
+ $(TMPLS) \
25
+ $(wildcard lib/*.rb \
26
+ lib/fairy/*.rb \
27
+ lib/fairy/job/*.rb \
28
+ lib/fairy/backend/*.rb \
29
+ lib/fairy/node/*.rb \
30
+ lib/fairy/share/*.rb \
31
+ sample/*.rb \
32
+ sample/*/*.rb \
33
+ sample/*/*/*.rb \
34
+ test/*.rb)
35
+
36
+ TS = TimeStamps
37
+
38
+ FAIRYSHELL = bin/fairy
39
+ FAIRYSHELL_TEMPL0 = \
40
+ lib/fairy/template/fairy-HEAD.templ
41
+ FAIRYSHELL_TEMPL = \
42
+ lib/fairy/share/conf.rb \
43
+ lib/fairy/share/base-app.rb \
44
+ lib/fairy/template/fairy-BODY.templ
45
+
46
+ bin/fairy: $(FAIRYSHELL_TEMPL0) $(FAIRYSHELL_TEMPL)
47
+ @echo "Code generation $@"
48
+ -@chmod -f +w $@
49
+ @cat $(FAIRYSHELL_TEMPL0) > $@
50
+ @for f in $(FAIRYSHELL_TEMPL); do \
51
+ echo " genalate from: $$f" ;\
52
+ echo "" >> $@ ;\
53
+ echo "##Original from $$f" >> $@ ;\
54
+ cat $$f >> $@ ;\
55
+ done
56
+ @chmod +x,-w $@
57
+ #
58
+ test-clean:
59
+ rm -fr test/Repos
60
+
61
+ # git
62
+ commit: $(TS)/commit
63
+
64
+ $(TS)/commit: $(SRCS)
65
+ git commit -a
66
+ touch $(TS)/commit
67
+
68
+ diff:
69
+ git diff
70
+
71
+ tag-%:
72
+ echo "Make tag $*"
73
+ tools/git-tag $*
74
+
75
+ push: doc/programming-interface.html bin/fairy
76
+ git push origin
77
+
78
+ push-dev:
79
+ git push origin refs/heads/dev
80
+
81
+ push-tags:
82
+ git push --tags origin
83
+
84
+ push-github:
85
+ git push github
86
+
87
+ doc/programming-interface.html: doc/programming-interface.rd
88
+ tools/rd2html
89
+
90
+ # gem
91
+ gem:
92
+ gem build fairy.gemspec
93
+
94
+ # tar archives
95
+ TGZ_FILES = $(SRCS) $(DEEPCONNECTSRCS)
96
+
97
+
98
+ SNAPSHOT = Snapshot
99
+
100
+ VERSION = $(shell ruby -r lib/fairy/version.rb -e "puts Fairy::Version")
101
+
102
+ PACKAGE_NAME = fairy
103
+
104
+ TAR_NAME = $(PACKAGE_NAME)-$(VERSION).tgz
105
+
106
+ tgz: $(SNAPSHOT)/$(TAR_NAME)
107
+
108
+ $(SNAPSHOT)/$(TAR_NAME): $(TGZ_FILES)
109
+ @if [ ! -e $(SNAPSHOT) ]; then \
110
+ mkdir $(SNAPSHOT); \
111
+ fi
112
+ @echo "make $(TAR_NAME) in $(SNAPSHOT)"
113
+ @tar zcf $(SNAPSHOT)/$(TAR_NAME) $(TGZ_FILES)
114
+ @echo "copy $(TAR_NAME) to /tmp/Downloads"
115
+ @cp -p $(SNAPSHOT)/$(TAR_NAME) /tmp/Downloads
116
+
data/README ADDED
@@ -0,0 +1,15 @@
1
+ fairy: A framework for distributed processing in Ruby
2
+ Copyright (c) Rakuten, Inc. All Right Reserved
3
+
4
+ fairy is a framework for distributed processing in Ruby, originally
5
+ designed at Rakuten Institute of Technology with Yukihiro Matsumoto,
6
+ the founder of Ruby.
7
+
8
+ Although fairy was inspired by MapReduce model, a well-known
9
+ programming model for distributed processing, it's more flexible and
10
+ suitable for wider use. That's due to fairy's programming model,
11
+ called filter IF, and various built-in filters.
12
+
13
+ fairy is implemented in Ruby and inherits its high productivity and
14
+ simplicity. fairy's API is quite similar to Ruby. Therefore most
15
+ programmers who know Ruby can easily understand and use it.
data/bin/fairy ADDED
@@ -0,0 +1,582 @@
1
+ #!/usr/bin/env ruby
2
+ # encoding: UTF-8
3
+
4
+ # THIS FILE IS AUTO GENERATION.
5
+ # Don't modify this file.
6
+
7
+ require "optparse"
8
+
9
+ ##Original from lib/fairy/share/conf.rb
10
+ # encoding: UTF-8
11
+ #
12
+ # Copyright (C) 2007-2010 Rakuten, Inc.
13
+ #
14
+
15
+ require "socket"
16
+ require "rbconfig"
17
+
18
+ module Fairy
19
+
20
+ CONF_PATH = [
21
+ "/etc/fairy.conf",
22
+ RbConfig::CONFIG["libdir"]+"/fairy/etc/fairy.conf",
23
+ ENV["FAIRY_HOME"] && ENV["FAIRY_HOME"]+"/etc/fairy.conf",
24
+ ENV["HOME"]+"/.fairyrc",
25
+ ENV["FAIRY_CONF"]
26
+ ]
27
+ # $:.each{|p|
28
+ # CONF_PATH.push p+"/fairy/etc/fairy.conf"
29
+ # }
30
+
31
+ class Conf
32
+ PROPS = []
33
+ class DefaultConf<Conf;end
34
+
35
+ def initialize(super_conf = nil, opts = {})
36
+ @super_conf = super_conf
37
+ @values = opts
38
+ end
39
+
40
+ class<<Conf
41
+ def def_prop(prop, reader = nil, setter = nil)
42
+ PROPS.push prop
43
+ reader = "def #{prop}; value(:#{prop}); end" unless reader
44
+ setter = "def #{prop}=(value); @values[:#{prop}] = value; end" unless setter
45
+ module_eval reader
46
+ module_eval setter
47
+ # DefaultConf.module_eval reader
48
+ end
49
+
50
+ def def_prop_relative_path(prop, path, base_prop = :HOME)
51
+ def_prop(prop,
52
+ "def #{prop}; value(:#{prop}) || self.#{base_prop}+'/'+'#{path}'; end")
53
+ end
54
+ alias def_rpath def_prop_relative_path
55
+ end
56
+
57
+ def value(prop)
58
+ if (v = @values[prop]).nil?
59
+ @super_conf && v = @super_conf.value(prop)
60
+ end
61
+ v
62
+ end
63
+
64
+ def props
65
+ PROPS
66
+ end
67
+
68
+ def base_conf=(conf)
69
+ if @super_conf
70
+ @super_conf.base_conf = conf
71
+ else
72
+ @super_conf = conf
73
+ end
74
+ end
75
+
76
+ def set_values(hash)
77
+ for key, value in hash
78
+ @values[key] = value
79
+ end
80
+ end
81
+
82
+ def_prop :RUBY_BIN
83
+ def_prop :MASTER_HOST
84
+ def_prop :MASTER_PORT
85
+
86
+ def_prop :HOME
87
+ def HOME
88
+ File.expand_path(value(:HOME))
89
+ end
90
+
91
+ def_rpath :BIN, "bin"
92
+ def_rpath :LIB, "lib"
93
+ def_rpath :SUBCMD, "subcmd", :BIN
94
+ def_rpath :CONTROLLER_BIN, "controller", :SUBCMD
95
+ def_rpath :PROCESSOR_BIN, "processor", :SUBCMD
96
+
97
+ def_prop :DEFAULT_EXTERNAL
98
+ def_prop :DEFAULT_INTERNAL
99
+
100
+ def_prop :IGNORE_EXCEPTION_ON_FILTER
101
+
102
+ def_prop :MASTER_MAX_ACTIVE_PROCESSORS
103
+
104
+ def_prop :CONTROLLER_INPUT_PROCESSOR_N
105
+ def_prop :CONTROLLER_MAX_ACTIVE_TASKS_IN_PROCESSOR
106
+
107
+ def_prop :CONTROLLER_ASSIGN_NEW_PROCESSOR_N_FACTOR
108
+
109
+ def_prop :PREQUEUING_POLICY
110
+ def_prop :POSTQUEUING_POLICY
111
+
112
+ def_prop :POSTMAPPING_POLICY
113
+
114
+ def_prop :POSTQUEUE_MAX_TRANSFER_SIZE
115
+ def_prop :POOLQUEUE_POOL_THRESHOLD
116
+ def_prop :ONMEMORY_SIZEDQUEUE_SIZE
117
+ def_prop :FILEBUFFEREDQUEUE_THRESHOLD
118
+
119
+ def_prop :MARSHAL_QUEUE_CHUNK_SIZE
120
+ def_prop :MARSHAL_QUEUE_MIN_CHUNK_NO
121
+ def_prop :SIZEDMARSHAL_QUEUE_MAX_CHUNK_NO
122
+
123
+ def_prop :SORTEDQUEUE_POOL_THRESHOLD
124
+ def_prop :SORTEDQUEUE_THRESHOLD
125
+ def_prop :SORTEDQUEUE_SORTBY
126
+
127
+ def_prop :INPUT_LOCAL_FILE_BUFFER_SIZE
128
+ def_prop :HERE_POOL_THRESHOLD
129
+ #def_prop :OUTPUT_LOCAL_POOL_THRESHOLD
130
+
131
+ def_prop :GROUP_BY_NO_SEGMENT
132
+ def_prop :GROUP_BY_HASH_MODULE
133
+ def_prop :GROUP_BY_GROUPING_OPTIMIZE
134
+
135
+ def_prop :GROUP_BY_BUFFERING_POLICY
136
+ def_prop :GROUP_BY_CMSB_THRESHOLD
137
+ def_prop :GROUP_BY_CMSB_CHUNK_SIZE
138
+
139
+ def_prop :BARRIER_MEMORY_BUFFERING_POLICY
140
+
141
+ def_prop :SORT_BUFFERING_POLICY
142
+ def_prop :SORT_SAMPLING_MIN
143
+ def_prop :SORT_SAMPLING_MAX
144
+ def_prop :SORT_SAMPLING_RATIO_1_TO
145
+ def_prop :SORT_NO_SEGMENT
146
+ def_prop :SORT_CMP_OPTIMIZE
147
+
148
+ def_prop :IOTA_SPLIT_NO
149
+
150
+ def_prop :TRANSFAR_MARSHAL_STRING_ARRAY_OPTIMIZE
151
+
152
+ def_rpath :VF_ROOT, "Repos"
153
+ def_prop :VF_PREFIX
154
+ def_prop :VF_BASE_NAME_CONVERTER
155
+ def_prop :VF_SPLIT_SIZE
156
+
157
+ def_prop :TMP_DIR
158
+
159
+ def_prop :LOG_FILE
160
+ def_prop :LOG_LEVEL
161
+ def_prop :LOG_FLUSH_INTERVAL
162
+ def_prop :LOG_MARK_INTERVAL
163
+ def_prop :LOG_IMPORT_NTIMES_POP
164
+ def_prop :LOG_LOCAL_OUTPUT_DEV
165
+ def_prop :LOG_ROTATE_INTERVAL
166
+ def_prop :LOG_ROTATE_N
167
+
168
+ def_prop :SUBCMD_EXEC_TIMEOUT
169
+
170
+ def_prop :PROCESSOR_MON_ON
171
+ def_prop :PROCESSOR_MON_INTERVAL
172
+ def_prop :PROCESSOR_MON_PSFORMAT
173
+ def_prop :PROCESSOR_MON_OBJECTSPACE_INSPECT_ON
174
+
175
+ def_prop :SOCK_DO_NOT_REVERSE_LOOKUP
176
+ def_prop :USE_RESOLV_REPLACE
177
+
178
+ def_prop :BLOCK_USE_STDOUT
179
+
180
+ def_prop :DEBUG_PORT_WAIT
181
+ def_prop :DEBUG_FULL_BACKTRACE
182
+ def_prop :DEBUG_THREAD_ABORT_ON_EXCEPTION
183
+ def_prop :DEBUG_MONITOR_ON
184
+ def_prop :DEBUG_PROCESSOR_TRACE_ON
185
+ def_prop :DEBUG_BUG49
186
+
187
+ def_prop :BUG234
188
+
189
+ def_prop :PROCESS_LIFE_MANAGE_INTERVAL
190
+
191
+ def_prop :THREAD_STACK_SIZE
192
+
193
+ # class JobConf<Conf
194
+ # def initialize(prefix = "", super_conf = nil, opts ={})
195
+ # @prefix = prefix.upcase
196
+ # end
197
+
198
+ # def method_missing(method, *args, &b)
199
+ # del = "#{prefix}_#{method.id2name.upcase}"
200
+ # if respond_to?(del)
201
+ # __send__(del, *args, &b)
202
+ # else
203
+ # super
204
+ # end
205
+ # end
206
+ # end
207
+ # BJobConf = JobConf
208
+ # NJobConf = JobConf
209
+
210
+ class DefaultConf
211
+
212
+ def initialize
213
+ super
214
+
215
+ @default_host = `hostname`.chomp
216
+ @hosts = {}
217
+ end
218
+
219
+ def [](host)
220
+ unless @hosts[host]
221
+ @hosts[host] = Conf.new
222
+ end
223
+ @hosts[host]
224
+ end
225
+
226
+ def value(prop)
227
+ conf = @hosts[@default_host]
228
+ v = conf && conf.value(prop)
229
+ return v if v
230
+ @values[prop]
231
+ end
232
+
233
+ def load_all_conf
234
+ loaded = false
235
+ for path in CONF_PATH
236
+ if path
237
+ if File.exist?(path)
238
+ load_conf path
239
+ loaded = true
240
+ end
241
+ end
242
+ end
243
+ # unless loaded
244
+ # puts "warnfairy.confファイルが見つかりません"
245
+ # end
246
+ end
247
+
248
+ def load_conf(conf)
249
+ begin
250
+ load conf
251
+ rescue LoadError
252
+ puts "設定ファイル #{conf} をロードできませんでした."
253
+ exit 1
254
+ rescue
255
+ puts "設定ファイル #{@conf} をロード中に例外が発生しました."
256
+ p $!
257
+ p $@
258
+ exit 2
259
+ end
260
+ end
261
+ end
262
+ end
263
+
264
+ DEFAULT_CONF = Conf::DefaultConf.new
265
+
266
+ DEFAULT_CONF.RUBY_BIN = ENV["FAIRY_RUBY"] || "ruby"
267
+
268
+ #DEFAULT_CONF.MASTER_HOST =
269
+ DEFAULT_CONF.MASTER_PORT = 19999
270
+
271
+ # DEFAULT_CONF.HOME = ENV["FAIRY_HOME"] || "/usr/lib/fairy"
272
+ home = ENV["FAIRY_HOME"]
273
+ unless home
274
+ home = File.dirname(File.dirname(File.expand_path(__FILE__)))
275
+ if File.basename(__FILE__) == "conf.rb"
276
+ home = File.dirname(File.dirname(home))
277
+ end
278
+ end
279
+ DEFAULT_CONF.HOME = home
280
+ # DEFAULT_CONF.BIN = DEFAULT_CONF.HOME+"/bin"
281
+ # DEFAULT_CONF.LIB = DEFAULT_CONF.HOME+"/lib"
282
+
283
+ # DEFAULT_CONF.CONTROLLER_BIN = DEFAULT_CONF.BIN+"/controller"
284
+ # DEFAULT_CONF.PROCESSOR_BIN = DEFAULT_CONF.BIN+"/processor"
285
+
286
+ DEFAULT_CONF.DEFAULT_EXTERNAL = nil
287
+ DEFAULT_CONF.DEFAULT_INTERNAL = nil
288
+
289
+ DEFAULT_CONF.IGNORE_EXCEPTION_ON_FILTER = false
290
+
291
+ DEFAULT_CONF.MASTER_MAX_ACTIVE_PROCESSORS = 4
292
+
293
+ DEFAULT_CONF.CONTROLLER_INPUT_PROCESSOR_N = 10
294
+ DEFAULT_CONF.CONTROLLER_MAX_ACTIVE_TASKS_IN_PROCESSOR = 4
295
+
296
+ DEFAULT_CONF.CONTROLLER_ASSIGN_NEW_PROCESSOR_N_FACTOR = 1
297
+
298
+ DEFAULT_CONF.POSTMAPPING_POLICY = nil
299
+
300
+ DEFAULT_CONF.PREQUEUING_POLICY = {:queuing_class => :FileMarshaledQueue}
301
+ DEFAULT_CONF.POSTQUEUING_POLICY = {:queuing_class => :FileMarshaledQueue}
302
+
303
+ DEFAULT_CONF.POSTQUEUE_MAX_TRANSFER_SIZE = 100000
304
+ DEFAULT_CONF.POOLQUEUE_POOL_THRESHOLD = 10000
305
+ DEFAULT_CONF.ONMEMORY_SIZEDQUEUE_SIZE = 10000
306
+ DEFAULT_CONF.FILEBUFFEREDQUEUE_THRESHOLD = 10000/2
307
+
308
+ DEFAULT_CONF.MARSHAL_QUEUE_CHUNK_SIZE = DEFAULT_CONF.POOLQUEUE_POOL_THRESHOLD
309
+ DEFAULT_CONF.MARSHAL_QUEUE_MIN_CHUNK_NO = DEFAULT_CONF.POOLQUEUE_POOL_THRESHOLD
310
+ DEFAULT_CONF.SIZEDMARSHAL_QUEUE_MAX_CHUNK_NO = 10
311
+
312
+ DEFAULT_CONF.SORTEDQUEUE_POOL_THRESHOLD = DEFAULT_CONF.POOLQUEUE_POOL_THRESHOLD
313
+ DEFAULT_CONF.SORTEDQUEUE_THRESHOLD = 10000/2
314
+ DEFAULT_CONF.SORTEDQUEUE_SORTBY = %{|v| v}
315
+
316
+ DEFAULT_CONF.INPUT_LOCAL_FILE_BUFFER_SIZE = 1024*1024
317
+ DEFAULT_CONF.HERE_POOL_THRESHOLD = 32000
318
+ #DEFAULT_CONF.OUTPUT_LOCAL_POOL_THRESHOLD = 32000
319
+
320
+ DEFAULT_CONF.GROUP_BY_NO_SEGMENT = 4
321
+ DEFAULT_CONF.GROUP_BY_HASH_MODULE = "fairy/share/hash-md5"
322
+ DEFAULT_CONF.GROUP_BY_GROUPING_OPTIMIZE = false
323
+
324
+ DEFAULT_CONF.GROUP_BY_BUFFERING_POLICY = {:buffering_class => :DirectMergeSortBuffer}
325
+ DEFAULT_CONF.GROUP_BY_CMSB_THRESHOLD = 400_000
326
+ DEFAULT_CONF.GROUP_BY_CMSB_CHUNK_SIZE = 1000
327
+
328
+ DEFAULT_CONF.BARRIER_MEMORY_BUFFERING_POLICY = {:queuing_class => :PoolQueue}
329
+
330
+ DEFAULT_CONF.SORT_BUFFERING_POLICY = {:buffering_class => "PGroupBy::DirectMergeSortBuffer"}
331
+ DEFAULT_CONF.SORT_SAMPLING_MIN = 100
332
+ DEFAULT_CONF.SORT_SAMPLING_MAX = 10000
333
+ DEFAULT_CONF.SORT_SAMPLING_RATIO_1_TO = 100
334
+ DEFAULT_CONF.SORT_NO_SEGMENT = DEFAULT_CONF.GROUP_BY_NO_SEGMENT
335
+ DEFAULT_CONF.SORT_CMP_OPTIMIZE = false
336
+
337
+ DEFAULT_CONF.IOTA_SPLIT_NO = 4
338
+
339
+ DEFAULT_CONF.TRANSFAR_MARSHAL_STRING_ARRAY_OPTIMIZE = false
340
+
341
+ # DEFAULT_CONF.VF_ROOT = DEFAULT_CONF.HOME+"/Repos"
342
+ DEFAULT_CONF.VF_PREFIX = `hostname`.chomp
343
+ ## DEFAULT_CONF.VF_PREFIX is client setting.
344
+ DEFAULT_CONF.VF_BASE_NAME_CONVERTER = nil
345
+ DEFAULT_CONF.VF_SPLIT_SIZE = 64*1024*1024
346
+
347
+ DEFAULT_CONF.TMP_DIR = "/tmp/fairy/tmpbuf"
348
+
349
+ DEFAULT_CONF.LOG_FILE = "/tmp/fairy/log"
350
+ DEFAULT_CONF.LOG_FLUSH_INTERVAL = 1
351
+ DEFAULT_CONF.LOG_MARK_INTERVAL = 300
352
+ DEFAULT_CONF.LOG_LEVEL = :INFO
353
+ DEFAULT_CONF.LOG_IMPORT_NTIMES_POP = 100000
354
+ DEFAULT_CONF.LOG_LOCAL_OUTPUT_DEV = :$stderr
355
+ DEFAULT_CONF.LOG_ROTATE_INTERVAL = 60*60*24
356
+ DEFAULT_CONF.LOG_ROTATE_N = 7
357
+
358
+ DEFAULT_CONF.SUBCMD_EXEC_TIMEOUT = 60
359
+
360
+ DEFAULT_CONF.PROCESSOR_MON_ON = false
361
+ DEFAULT_CONF.PROCESSOR_MON_INTERVAL = 300
362
+ DEFAULT_CONF.PROCESSOR_MON_PSFORMAT = "stat,vsz,rss,sz,pmem,pcpu,nlwp,time,wchan"
363
+ DEFAULT_CONF.PROCESSOR_MON_OBJECTSPACE_INSPECT_ON = false
364
+
365
+ DEFAULT_CONF.SOCK_DO_NOT_REVERSE_LOOKUP = true
366
+ DEFAULT_CONF.USE_RESOLV_REPLACE = false
367
+
368
+ DEFAULT_CONF.BLOCK_USE_STDOUT = true
369
+
370
+ DEFAULT_CONF.DEBUG_PORT_WAIT = false
371
+ DEFAULT_CONF.DEBUG_FULL_BACKTRACE = false
372
+ DEFAULT_CONF.DEBUG_THREAD_ABORT_ON_EXCEPTION = false
373
+ DEFAULT_CONF.DEBUG_MONITOR_ON = false
374
+ DEFAULT_CONF.DEBUG_PROCESSOR_TRACE_ON = false
375
+ DEFAULT_CONF.DEBUG_BUG49 = false
376
+
377
+ DEFAULT_CONF.BUG234 = false
378
+
379
+ # DEFAULT_CONF.PROCESS_LIFE_MANAGE_INTERVAL = 60
380
+ # DEFAULT_CONF.PROCESS_LIFE_MANAGE_INTERVAL = 10
381
+ # DEFAULT_CONF.PROCESS_LIFE_MANAGE_INTERVAL = 1
382
+ DEFAULT_CONF.PROCESS_LIFE_MANAGE_INTERVAL = nil
383
+
384
+ # DEFAULT_CONF.THREAD_STACK_SIZE = 1024*100
385
+
386
+ CONF = DEFAULT_CONF
387
+ CONF.load_all_conf
388
+
389
+ def REPLACE_CONF(conf)
390
+ remove_const(:CONF)
391
+ const_set(:CONF, conf)
392
+ end
393
+ module_function :REPLACE_CONF
394
+
395
+
396
+ def Conf.configure_common_conf
397
+ Thread.abort_on_exception = CONF.DEBUG_THREAD_ABORT_ON_EXCEPTION
398
+
399
+ TCPSocket.do_not_reverse_lookup = CONF.SOCK_DO_NOT_REVERSE_LOOKUP
400
+
401
+ if CONF.USE_RESOLV_REPLACE
402
+ require "resolv-replace"
403
+ end
404
+ end
405
+ end
406
+
407
+ ##Original from lib/fairy/share/base-app.rb
408
+ # encoding: utf-8
409
+ #
410
+ # Copyright (C) 2007-2010 Rakuten, Inc.
411
+ #
412
+
413
+ module Fairy
414
+ class BaseAPP
415
+
416
+ def self.start
417
+ @@APP = new
418
+ @@APP.parse_arg
419
+ @@APP.load_conf
420
+ @@APP.configure_conf
421
+ @@APP.start
422
+ end
423
+
424
+ def self.start_subcommand(prog, *opts)
425
+ @@APP.start_subcommand(prog, *opts)
426
+ end
427
+ def self.start_subcommand2(prog, *opts)
428
+ Process.fork do
429
+ start_subcommand(prog, *opts)
430
+ end
431
+ end
432
+
433
+ def initialize
434
+ @home = nil
435
+ @conf = nil
436
+ @CONF = {}
437
+ end
438
+
439
+ def parse_arg
440
+ @opt = option_parser
441
+ @opt.order!(ARGV)
442
+ end
443
+
444
+ def option_parser
445
+ opt = OptionParser.new{|opt|
446
+ opt.on("--home=VAL"){|val| @home = val}
447
+ opt.on("--conf=VAL"){|val| @conf = val}
448
+ CONF.props.each do |prop|
449
+ opt.on("--CONF.#{prop}=VAL") {|val| @CONF[prop] = val}
450
+ end
451
+ }
452
+ if block_given?
453
+ yield opt
454
+ end
455
+ end
456
+
457
+ def conf_to_arg
458
+ @CONF.collect{|prop, source| "--CONF.#{prop}=#{source}"}
459
+ end
460
+
461
+ def load_conf
462
+ if @home
463
+ ENV["FAIRY_HOME"] = @home
464
+ # ENV["FAIRY_HOME"] = File.expand_path(@home)
465
+ conf = @home+"/etc/fairy.conf"
466
+ if File.exists?(conf)
467
+ CONF.load_conf conf
468
+ end
469
+ end
470
+
471
+ if @conf
472
+ ENV["FAIRY_CONF"] = @conf
473
+ CONF.load_conf @conf
474
+ end
475
+
476
+ @CONF.each do |key, value|
477
+ begin
478
+ eval "CONF.#{key}=#{value}"
479
+ rescue
480
+ puts "CONF.#{key}=#{value} が不正です."
481
+ exit 1
482
+ end
483
+ end
484
+ if ENV["RUBYLIB"]
485
+ ENV["RUBYLIB"] = CONF.LIB + ":" + ENV["RUBYLIB"]
486
+ else
487
+ ENV["RUBYLIB"] = CONF.LIB
488
+ end
489
+ end
490
+
491
+ def configure_conf
492
+ Conf.configure_common_conf
493
+ end
494
+
495
+ def start
496
+ ERR::Raise ERR::INTERNAL::ShouldDefineSubclass
497
+ end
498
+
499
+ def start_subcommand(prog, *opts)
500
+ opts.push *conf_to_arg
501
+ # Process.fork do
502
+ # Log.stop_export
503
+ # ObjectSpace.each_object(IO) do |io|
504
+ # begin
505
+ # if ![0, 1, 2].include?(io.fileno )
506
+ # io.close
507
+ # end
508
+ # rescue
509
+ # end
510
+ # end
511
+ # exec(prog, *opts)
512
+ # end
513
+
514
+ Process.spawn(prog, *opts)
515
+ # system("#{prog} #{opts.join(' ')}&")
516
+
517
+ end
518
+
519
+ end
520
+ end
521
+
522
+
523
+ ##Original from lib/fairy/template/fairy-BODY.templ
524
+ # -*- ruby -*-
525
+ #
526
+ # Copyright (C) 2007-2010 Rakuten, Inc.
527
+ #
528
+ #
529
+ module Fairy
530
+ # LC_MESSAGES = IRB::Locale.new
531
+ # LC_MESSAGES.load(CONF.LIB+"/fairy/share/exceptions.rb")
532
+ class FairyAPP<BaseAPP
533
+
534
+ SUBCOMMANDS = ["cp", "rm", "cat", "master", "node", "inspector"]
535
+
536
+ def initialize
537
+ super
538
+ @mode = :SUBCOMMAND
539
+ end
540
+
541
+ def option_parser
542
+ super do |opt|
543
+ opt.on("-f", "--file"){|val| @mode = :FILE}
544
+ opt.on("-e", "--exec"){|val| @mode = :EXEC}
545
+ end
546
+ end
547
+
548
+ def start
549
+ if @mode == :SUBCOMMAND &&
550
+ (ARGV.empty? || nosub = !SUBCOMMANDS.include?(ARGV[0]))
551
+ puts "そのようなサブコマンドはありません(#{ARGV[0]})." if nosub
552
+
553
+ helpbase = @opt.help.split(/\n/)
554
+ helpbase[0].concat(" subcommand [subcommand-options]")
555
+ helpbase.insert(1, " Options:")
556
+ helpbase.insert(1,
557
+ " Subcommands:",
558
+ *SUBCOMMANDS.collect{|s| " "+s})
559
+
560
+ puts helpbase
561
+ exit
562
+ end
563
+
564
+ case @mode
565
+ when :SUBCOMMAND
566
+ if File.exist?(CONF.SUBCMD+"/"+ARGV[0])
567
+ Process.exec(CONF.RUBY_BIN, CONF.SUBCMD+"/"+ARGV[0], *ARGV[1..-1])
568
+ else
569
+ Process.exec(CONF.RUBY_BIN, CONF.BIN+"/fairy-"+ARGV[0], *ARGV[1..-1])
570
+ end
571
+ when :FILE
572
+ Process.exec(CONF.RUBY_BIN, *ARGV)
573
+ when :EXEC
574
+ Process.exec(*ARGV)
575
+ end
576
+ end
577
+ end
578
+ end
579
+
580
+ Fairy::FairyAPP.start
581
+
582
+