fairy 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. data/LICENSE +674 -0
  2. data/Makefile +116 -0
  3. data/README +15 -0
  4. data/bin/fairy +582 -0
  5. data/bin/fairy-cat +74 -0
  6. data/bin/fairy-cp +128 -0
  7. data/bin/fairy-rm +122 -0
  8. data/bin/subcmd/controller +41 -0
  9. data/bin/subcmd/inspector +81 -0
  10. data/bin/subcmd/master +43 -0
  11. data/bin/subcmd/node +47 -0
  12. data/bin/subcmd/processor +54 -0
  13. data/doc/programming-interface.html +240 -0
  14. data/doc/programming-interface.rd +300 -0
  15. data/etc/fairy.conf.tmpl +118 -0
  16. data/ext/simple_hash/extconf.rb +4 -0
  17. data/ext/simple_hash/simple_hash.c +42 -0
  18. data/fairy.gemspec +60 -0
  19. data/lib/fairy/client/addins.rb +20 -0
  20. data/lib/fairy/client/barrier.rb +29 -0
  21. data/lib/fairy/client/basic-group-by.rb +52 -0
  22. data/lib/fairy/client/cat.rb +41 -0
  23. data/lib/fairy/client/direct-product.rb +51 -0
  24. data/lib/fairy/client/equijoin.rb +79 -0
  25. data/lib/fairy/client/exec.rb +54 -0
  26. data/lib/fairy/client/filter.rb +62 -0
  27. data/lib/fairy/client/find.rb +35 -0
  28. data/lib/fairy/client/group-by.rb +194 -0
  29. data/lib/fairy/client/here.rb +84 -0
  30. data/lib/fairy/client/inject.rb +70 -0
  31. data/lib/fairy/client/input-file.rb +53 -0
  32. data/lib/fairy/client/input-iota.rb +49 -0
  33. data/lib/fairy/client/input-local-file.rb +188 -0
  34. data/lib/fairy/client/input-varray.rb +30 -0
  35. data/lib/fairy/client/input.rb +42 -0
  36. data/lib/fairy/client/io-filter.rb +26 -0
  37. data/lib/fairy/client/junction.rb +31 -0
  38. data/lib/fairy/client/map.rb +34 -0
  39. data/lib/fairy/client/merge-group-by.rb +71 -0
  40. data/lib/fairy/client/output-file.rb +64 -0
  41. data/lib/fairy/client/output-local-file.rb +60 -0
  42. data/lib/fairy/client/output-null.rb +47 -0
  43. data/lib/fairy/client/output-varray.rb +50 -0
  44. data/lib/fairy/client/output.rb +29 -0
  45. data/lib/fairy/client/roma-put.rb +62 -0
  46. data/lib/fairy/client/roma.rb +156 -0
  47. data/lib/fairy/client/seg-join.rb +61 -0
  48. data/lib/fairy/client/seg-map.rb +78 -0
  49. data/lib/fairy/client/seg-shuffle.rb +35 -0
  50. data/lib/fairy/client/seg-split.rb +27 -0
  51. data/lib/fairy/client/seg-zip.rb +60 -0
  52. data/lib/fairy/client/select.rb +38 -0
  53. data/lib/fairy/client/sort.rb +48 -0
  54. data/lib/fairy/client/sort18.rb +56 -0
  55. data/lib/fairy/client/sort19.rb +61 -0
  56. data/lib/fairy/client/there.rb +47 -0
  57. data/lib/fairy/client/top_n_into_roma.rb +34 -0
  58. data/lib/fairy/client/wc.rb +92 -0
  59. data/lib/fairy/controller.rb +1103 -0
  60. data/lib/fairy/logger.rb +107 -0
  61. data/lib/fairy/master/addins.rb +20 -0
  62. data/lib/fairy/master/atom.rb +17 -0
  63. data/lib/fairy/master/c-barrier.rb +283 -0
  64. data/lib/fairy/master/c-basic-group-by.rb +250 -0
  65. data/lib/fairy/master/c-cat.rb +159 -0
  66. data/lib/fairy/master/c-direct-product.rb +203 -0
  67. data/lib/fairy/master/c-exec.rb +68 -0
  68. data/lib/fairy/master/c-filter.rb +422 -0
  69. data/lib/fairy/master/c-find.rb +138 -0
  70. data/lib/fairy/master/c-group-by.rb +64 -0
  71. data/lib/fairy/master/c-here.rb +80 -0
  72. data/lib/fairy/master/c-inject.rb +119 -0
  73. data/lib/fairy/master/c-input-file.rb +46 -0
  74. data/lib/fairy/master/c-input-iota.rb +66 -0
  75. data/lib/fairy/master/c-input-local-file.rb +117 -0
  76. data/lib/fairy/master/c-input-varray.rb +53 -0
  77. data/lib/fairy/master/c-input.rb +24 -0
  78. data/lib/fairy/master/c-inputtable.rb +31 -0
  79. data/lib/fairy/master/c-inputtable18.rb +36 -0
  80. data/lib/fairy/master/c-inputtable19.rb +35 -0
  81. data/lib/fairy/master/c-io-filter.rb +28 -0
  82. data/lib/fairy/master/c-junction.rb +54 -0
  83. data/lib/fairy/master/c-map.rb +27 -0
  84. data/lib/fairy/master/c-merge-group-by.rb +241 -0
  85. data/lib/fairy/master/c-output-file.rb +84 -0
  86. data/lib/fairy/master/c-output-local-file.rb +19 -0
  87. data/lib/fairy/master/c-output-null.rb +45 -0
  88. data/lib/fairy/master/c-output-varray.rb +57 -0
  89. data/lib/fairy/master/c-output.rb +20 -0
  90. data/lib/fairy/master/c-seg-join.rb +141 -0
  91. data/lib/fairy/master/c-seg-map.rb +26 -0
  92. data/lib/fairy/master/c-seg-shuffle.rb +87 -0
  93. data/lib/fairy/master/c-seg-split.rb +110 -0
  94. data/lib/fairy/master/c-seg-zip.rb +132 -0
  95. data/lib/fairy/master/c-select.rb +27 -0
  96. data/lib/fairy/master/c-sort.rb +108 -0
  97. data/lib/fairy/master/c-there.rb +57 -0
  98. data/lib/fairy/master/c-wc.rb +232 -0
  99. data/lib/fairy/master/job-interpriter.rb +19 -0
  100. data/lib/fairy/master/scheduler.rb +24 -0
  101. data/lib/fairy/master.rb +329 -0
  102. data/lib/fairy/node/addins.rb +19 -0
  103. data/lib/fairy/node/p-barrier.rb +95 -0
  104. data/lib/fairy/node/p-basic-group-by.rb +252 -0
  105. data/lib/fairy/node/p-direct-product.rb +153 -0
  106. data/lib/fairy/node/p-exec.rb +30 -0
  107. data/lib/fairy/node/p-filter.rb +363 -0
  108. data/lib/fairy/node/p-find.rb +111 -0
  109. data/lib/fairy/node/p-group-by.rb +1534 -0
  110. data/lib/fairy/node/p-here.rb +21 -0
  111. data/lib/fairy/node/p-identity.rb +24 -0
  112. data/lib/fairy/node/p-inject.rb +127 -0
  113. data/lib/fairy/node/p-input-file.rb +108 -0
  114. data/lib/fairy/node/p-input-iota.rb +39 -0
  115. data/lib/fairy/node/p-input-local-file.rb +61 -0
  116. data/lib/fairy/node/p-input-varray.rb +26 -0
  117. data/lib/fairy/node/p-io-filter.rb +28 -0
  118. data/lib/fairy/node/p-map.rb +40 -0
  119. data/lib/fairy/node/p-merger-group-by.rb +48 -0
  120. data/lib/fairy/node/p-output-file.rb +104 -0
  121. data/lib/fairy/node/p-output-local-file.rb +14 -0
  122. data/lib/fairy/node/p-output-null.rb +32 -0
  123. data/lib/fairy/node/p-output-varray.rb +41 -0
  124. data/lib/fairy/node/p-seg-join.rb +82 -0
  125. data/lib/fairy/node/p-seg-map.rb +34 -0
  126. data/lib/fairy/node/p-seg-split.rb +61 -0
  127. data/lib/fairy/node/p-seg-zip.rb +79 -0
  128. data/lib/fairy/node/p-select.rb +40 -0
  129. data/lib/fairy/node/p-single-exportable.rb +90 -0
  130. data/lib/fairy/node/p-sort.rb +195 -0
  131. data/lib/fairy/node/p-task.rb +113 -0
  132. data/lib/fairy/node/p-there.rb +44 -0
  133. data/lib/fairy/node/p-wc.rb +266 -0
  134. data/lib/fairy/node.rb +187 -0
  135. data/lib/fairy/processor.rb +510 -0
  136. data/lib/fairy/share/base-app.rb +114 -0
  137. data/lib/fairy/share/block-source.rb +234 -0
  138. data/lib/fairy/share/conf.rb +396 -0
  139. data/lib/fairy/share/debug.rb +21 -0
  140. data/lib/fairy/share/encoding.rb +17 -0
  141. data/lib/fairy/share/fast-tempfile.rb +93 -0
  142. data/lib/fairy/share/file-place.rb +176 -0
  143. data/lib/fairy/share/hash-1.rb +20 -0
  144. data/lib/fairy/share/hash-md5.rb +28 -0
  145. data/lib/fairy/share/hash-murmur.rb +69 -0
  146. data/lib/fairy/share/hash-rb18.rb +20 -0
  147. data/lib/fairy/share/hash-simple-hash.rb +28 -0
  148. data/lib/fairy/share/inspector.rb +16 -0
  149. data/lib/fairy/share/lc/exceptions.rb +82 -0
  150. data/lib/fairy/share/lc/ja/exceptions.rb +81 -0
  151. data/lib/fairy/share/locale.rb +17 -0
  152. data/lib/fairy/share/log.rb +215 -0
  153. data/lib/fairy/share/pool-dictionary.rb +53 -0
  154. data/lib/fairy/share/port-marshaled-queue.rb +347 -0
  155. data/lib/fairy/share/port.rb +1697 -0
  156. data/lib/fairy/share/reference.rb +45 -0
  157. data/lib/fairy/share/stdout.rb +56 -0
  158. data/lib/fairy/share/tr.rb +16 -0
  159. data/lib/fairy/share/varray.rb +147 -0
  160. data/lib/fairy/share/vfile.rb +183 -0
  161. data/lib/fairy/version.rb +8 -0
  162. data/lib/fairy.rb +206 -0
  163. data/sample/grep.rb +46 -0
  164. data/sample/ping.rb +19 -0
  165. data/sample/sort.rb +102 -0
  166. data/sample/wordcount.rb +61 -0
  167. data/spec/README +12 -0
  168. data/spec/fairy1_spec.rb +31 -0
  169. data/spec/fairy2_spec.rb +42 -0
  170. data/spec/fairy3_spec.rb +126 -0
  171. data/spec/fairy4_spec.rb +63 -0
  172. data/spec/fairy5_spec.rb +45 -0
  173. data/spec/fairy6_spec.rb +52 -0
  174. data/spec/fairy7_spec.rb +58 -0
  175. data/spec/fairy8_spec.rb +48 -0
  176. data/spec/mkdat.rb +148 -0
  177. data/spec/run_all.sh +65 -0
  178. data/test/testc.rb +7111 -0
  179. data/tools/cap_recipe/Capfile +144 -0
  180. data/tools/cap_recipe/cluster.yml.sample +14 -0
  181. data/tools/fairy_perf_graph.rb +444 -0
  182. data/tools/git-tag +44 -0
  183. data/tools/log-analysis.rb +62 -0
  184. data/tools/svn-ls-diff +38 -0
  185. data/tools/svn-tags +37 -0
  186. metadata +298 -0
data/Makefile ADDED
@@ -0,0 +1,116 @@
1
+
2
+ CONF = etc/fairy.conf
3
+
4
+ BINSRCS = \
5
+ bin/master \
6
+ bin/node \
7
+ bin/controller \
8
+ bin/processor \
9
+ \
10
+ bin/fairy \
11
+ bin/fairy-cp \
12
+ bin/fairy-cat \
13
+ bin/fairy-rm \
14
+ bin/inspector
15
+
16
+
17
+ TEMPLS = $(wildcard lib/fairy/template/*.tmpl)
18
+
19
+ DEEPCONNECTSRCS = $(wildcard deep-connect/*.rb)
20
+
21
+ SRCS = Makefile ChangeLog TODO \
22
+ $(CONF) \
23
+ $(BINSRCS) \
24
+ $(TMPLS) \
25
+ $(wildcard lib/*.rb \
26
+ lib/fairy/*.rb \
27
+ lib/fairy/job/*.rb \
28
+ lib/fairy/backend/*.rb \
29
+ lib/fairy/node/*.rb \
30
+ lib/fairy/share/*.rb \
31
+ sample/*.rb \
32
+ sample/*/*.rb \
33
+ sample/*/*/*.rb \
34
+ test/*.rb)
35
+
36
+ TS = TimeStamps
37
+
38
+ FAIRYSHELL = bin/fairy
39
+ FAIRYSHELL_TEMPL0 = \
40
+ lib/fairy/template/fairy-HEAD.templ
41
+ FAIRYSHELL_TEMPL = \
42
+ lib/fairy/share/conf.rb \
43
+ lib/fairy/share/base-app.rb \
44
+ lib/fairy/template/fairy-BODY.templ
45
+
46
+ bin/fairy: $(FAIRYSHELL_TEMPL0) $(FAIRYSHELL_TEMPL)
47
+ @echo "Code generation $@"
48
+ -@chmod -f +w $@
49
+ @cat $(FAIRYSHELL_TEMPL0) > $@
50
+ @for f in $(FAIRYSHELL_TEMPL); do \
51
+ echo " genalate from: $$f" ;\
52
+ echo "" >> $@ ;\
53
+ echo "##Original from $$f" >> $@ ;\
54
+ cat $$f >> $@ ;\
55
+ done
56
+ @chmod +x,-w $@
57
+ #
58
+ test-clean:
59
+ rm -fr test/Repos
60
+
61
+ # git
62
+ commit: $(TS)/commit
63
+
64
+ $(TS)/commit: $(SRCS)
65
+ git commit -a
66
+ touch $(TS)/commit
67
+
68
+ diff:
69
+ git diff
70
+
71
+ tag-%:
72
+ echo "Make tag $*"
73
+ tools/git-tag $*
74
+
75
+ push: doc/programming-interface.html bin/fairy
76
+ git push origin
77
+
78
+ push-dev:
79
+ git push origin refs/heads/dev
80
+
81
+ push-tags:
82
+ git push --tags origin
83
+
84
+ push-github:
85
+ git push github
86
+
87
+ doc/programming-interface.html: doc/programming-interface.rd
88
+ tools/rd2html
89
+
90
+ # gem
91
+ gem:
92
+ gem build fairy.gemspec
93
+
94
+ # tar archives
95
+ TGZ_FILES = $(SRCS) $(DEEPCONNECTSRCS)
96
+
97
+
98
+ SNAPSHOT = Snapshot
99
+
100
+ VERSION = $(shell ruby -r lib/fairy/version.rb -e "puts Fairy::Version")
101
+
102
+ PACKAGE_NAME = fairy
103
+
104
+ TAR_NAME = $(PACKAGE_NAME)-$(VERSION).tgz
105
+
106
+ tgz: $(SNAPSHOT)/$(TAR_NAME)
107
+
108
+ $(SNAPSHOT)/$(TAR_NAME): $(TGZ_FILES)
109
+ @if [ ! -e $(SNAPSHOT) ]; then \
110
+ mkdir $(SNAPSHOT); \
111
+ fi
112
+ @echo "make $(TAR_NAME) in $(SNAPSHOT)"
113
+ @tar zcf $(SNAPSHOT)/$(TAR_NAME) $(TGZ_FILES)
114
+ @echo "copy $(TAR_NAME) to /tmp/Downloads"
115
+ @cp -p $(SNAPSHOT)/$(TAR_NAME) /tmp/Downloads
116
+
data/README ADDED
@@ -0,0 +1,15 @@
1
+ fairy: A framework for distributed processing in Ruby
2
+ Copyright (c) Rakuten, Inc. All Right Reserved
3
+
4
+ fairy is a framework for distributed processing in Ruby, originally
5
+ designed at Rakuten Institute of Technology with Yukihiro Matsumoto,
6
+ the founder of Ruby.
7
+
8
+ Although fairy was inspired by MapReduce model, a well-known
9
+ programming model for distributed processing, it's more flexible and
10
+ suitable for wider use. That's due to fairy's programming model,
11
+ called filter IF, and various built-in filters.
12
+
13
+ fairy is implemented in Ruby and inherits its high productivity and
14
+ simplicity. fairy's API is quite similar to Ruby. Therefore most
15
+ programmers who know Ruby can easily understand and use it.
data/bin/fairy ADDED
@@ -0,0 +1,582 @@
1
+ #!/usr/bin/env ruby
2
+ # encoding: UTF-8
3
+
4
+ # THIS FILE IS AUTO GENERATION.
5
+ # Don't modify this file.
6
+
7
+ require "optparse"
8
+
9
+ ##Original from lib/fairy/share/conf.rb
10
+ # encoding: UTF-8
11
+ #
12
+ # Copyright (C) 2007-2010 Rakuten, Inc.
13
+ #
14
+
15
+ require "socket"
16
+ require "rbconfig"
17
+
18
+ module Fairy
19
+
20
+ CONF_PATH = [
21
+ "/etc/fairy.conf",
22
+ RbConfig::CONFIG["libdir"]+"/fairy/etc/fairy.conf",
23
+ ENV["FAIRY_HOME"] && ENV["FAIRY_HOME"]+"/etc/fairy.conf",
24
+ ENV["HOME"]+"/.fairyrc",
25
+ ENV["FAIRY_CONF"]
26
+ ]
27
+ # $:.each{|p|
28
+ # CONF_PATH.push p+"/fairy/etc/fairy.conf"
29
+ # }
30
+
31
+ class Conf
32
+ PROPS = []
33
+ class DefaultConf<Conf;end
34
+
35
+ def initialize(super_conf = nil, opts = {})
36
+ @super_conf = super_conf
37
+ @values = opts
38
+ end
39
+
40
+ class<<Conf
41
+ def def_prop(prop, reader = nil, setter = nil)
42
+ PROPS.push prop
43
+ reader = "def #{prop}; value(:#{prop}); end" unless reader
44
+ setter = "def #{prop}=(value); @values[:#{prop}] = value; end" unless setter
45
+ module_eval reader
46
+ module_eval setter
47
+ # DefaultConf.module_eval reader
48
+ end
49
+
50
+ def def_prop_relative_path(prop, path, base_prop = :HOME)
51
+ def_prop(prop,
52
+ "def #{prop}; value(:#{prop}) || self.#{base_prop}+'/'+'#{path}'; end")
53
+ end
54
+ alias def_rpath def_prop_relative_path
55
+ end
56
+
57
+ def value(prop)
58
+ if (v = @values[prop]).nil?
59
+ @super_conf && v = @super_conf.value(prop)
60
+ end
61
+ v
62
+ end
63
+
64
+ def props
65
+ PROPS
66
+ end
67
+
68
+ def base_conf=(conf)
69
+ if @super_conf
70
+ @super_conf.base_conf = conf
71
+ else
72
+ @super_conf = conf
73
+ end
74
+ end
75
+
76
+ def set_values(hash)
77
+ for key, value in hash
78
+ @values[key] = value
79
+ end
80
+ end
81
+
82
+ def_prop :RUBY_BIN
83
+ def_prop :MASTER_HOST
84
+ def_prop :MASTER_PORT
85
+
86
+ def_prop :HOME
87
+ def HOME
88
+ File.expand_path(value(:HOME))
89
+ end
90
+
91
+ def_rpath :BIN, "bin"
92
+ def_rpath :LIB, "lib"
93
+ def_rpath :SUBCMD, "subcmd", :BIN
94
+ def_rpath :CONTROLLER_BIN, "controller", :SUBCMD
95
+ def_rpath :PROCESSOR_BIN, "processor", :SUBCMD
96
+
97
+ def_prop :DEFAULT_EXTERNAL
98
+ def_prop :DEFAULT_INTERNAL
99
+
100
+ def_prop :IGNORE_EXCEPTION_ON_FILTER
101
+
102
+ def_prop :MASTER_MAX_ACTIVE_PROCESSORS
103
+
104
+ def_prop :CONTROLLER_INPUT_PROCESSOR_N
105
+ def_prop :CONTROLLER_MAX_ACTIVE_TASKS_IN_PROCESSOR
106
+
107
+ def_prop :CONTROLLER_ASSIGN_NEW_PROCESSOR_N_FACTOR
108
+
109
+ def_prop :PREQUEUING_POLICY
110
+ def_prop :POSTQUEUING_POLICY
111
+
112
+ def_prop :POSTMAPPING_POLICY
113
+
114
+ def_prop :POSTQUEUE_MAX_TRANSFER_SIZE
115
+ def_prop :POOLQUEUE_POOL_THRESHOLD
116
+ def_prop :ONMEMORY_SIZEDQUEUE_SIZE
117
+ def_prop :FILEBUFFEREDQUEUE_THRESHOLD
118
+
119
+ def_prop :MARSHAL_QUEUE_CHUNK_SIZE
120
+ def_prop :MARSHAL_QUEUE_MIN_CHUNK_NO
121
+ def_prop :SIZEDMARSHAL_QUEUE_MAX_CHUNK_NO
122
+
123
+ def_prop :SORTEDQUEUE_POOL_THRESHOLD
124
+ def_prop :SORTEDQUEUE_THRESHOLD
125
+ def_prop :SORTEDQUEUE_SORTBY
126
+
127
+ def_prop :INPUT_LOCAL_FILE_BUFFER_SIZE
128
+ def_prop :HERE_POOL_THRESHOLD
129
+ #def_prop :OUTPUT_LOCAL_POOL_THRESHOLD
130
+
131
+ def_prop :GROUP_BY_NO_SEGMENT
132
+ def_prop :GROUP_BY_HASH_MODULE
133
+ def_prop :GROUP_BY_GROUPING_OPTIMIZE
134
+
135
+ def_prop :GROUP_BY_BUFFERING_POLICY
136
+ def_prop :GROUP_BY_CMSB_THRESHOLD
137
+ def_prop :GROUP_BY_CMSB_CHUNK_SIZE
138
+
139
+ def_prop :BARRIER_MEMORY_BUFFERING_POLICY
140
+
141
+ def_prop :SORT_BUFFERING_POLICY
142
+ def_prop :SORT_SAMPLING_MIN
143
+ def_prop :SORT_SAMPLING_MAX
144
+ def_prop :SORT_SAMPLING_RATIO_1_TO
145
+ def_prop :SORT_NO_SEGMENT
146
+ def_prop :SORT_CMP_OPTIMIZE
147
+
148
+ def_prop :IOTA_SPLIT_NO
149
+
150
+ def_prop :TRANSFAR_MARSHAL_STRING_ARRAY_OPTIMIZE
151
+
152
+ def_rpath :VF_ROOT, "Repos"
153
+ def_prop :VF_PREFIX
154
+ def_prop :VF_BASE_NAME_CONVERTER
155
+ def_prop :VF_SPLIT_SIZE
156
+
157
+ def_prop :TMP_DIR
158
+
159
+ def_prop :LOG_FILE
160
+ def_prop :LOG_LEVEL
161
+ def_prop :LOG_FLUSH_INTERVAL
162
+ def_prop :LOG_MARK_INTERVAL
163
+ def_prop :LOG_IMPORT_NTIMES_POP
164
+ def_prop :LOG_LOCAL_OUTPUT_DEV
165
+ def_prop :LOG_ROTATE_INTERVAL
166
+ def_prop :LOG_ROTATE_N
167
+
168
+ def_prop :SUBCMD_EXEC_TIMEOUT
169
+
170
+ def_prop :PROCESSOR_MON_ON
171
+ def_prop :PROCESSOR_MON_INTERVAL
172
+ def_prop :PROCESSOR_MON_PSFORMAT
173
+ def_prop :PROCESSOR_MON_OBJECTSPACE_INSPECT_ON
174
+
175
+ def_prop :SOCK_DO_NOT_REVERSE_LOOKUP
176
+ def_prop :USE_RESOLV_REPLACE
177
+
178
+ def_prop :BLOCK_USE_STDOUT
179
+
180
+ def_prop :DEBUG_PORT_WAIT
181
+ def_prop :DEBUG_FULL_BACKTRACE
182
+ def_prop :DEBUG_THREAD_ABORT_ON_EXCEPTION
183
+ def_prop :DEBUG_MONITOR_ON
184
+ def_prop :DEBUG_PROCESSOR_TRACE_ON
185
+ def_prop :DEBUG_BUG49
186
+
187
+ def_prop :BUG234
188
+
189
+ def_prop :PROCESS_LIFE_MANAGE_INTERVAL
190
+
191
+ def_prop :THREAD_STACK_SIZE
192
+
193
+ # class JobConf<Conf
194
+ # def initialize(prefix = "", super_conf = nil, opts ={})
195
+ # @prefix = prefix.upcase
196
+ # end
197
+
198
+ # def method_missing(method, *args, &b)
199
+ # del = "#{prefix}_#{method.id2name.upcase}"
200
+ # if respond_to?(del)
201
+ # __send__(del, *args, &b)
202
+ # else
203
+ # super
204
+ # end
205
+ # end
206
+ # end
207
+ # BJobConf = JobConf
208
+ # NJobConf = JobConf
209
+
210
+ class DefaultConf
211
+
212
+ def initialize
213
+ super
214
+
215
+ @default_host = `hostname`.chomp
216
+ @hosts = {}
217
+ end
218
+
219
+ def [](host)
220
+ unless @hosts[host]
221
+ @hosts[host] = Conf.new
222
+ end
223
+ @hosts[host]
224
+ end
225
+
226
+ def value(prop)
227
+ conf = @hosts[@default_host]
228
+ v = conf && conf.value(prop)
229
+ return v if v
230
+ @values[prop]
231
+ end
232
+
233
+ def load_all_conf
234
+ loaded = false
235
+ for path in CONF_PATH
236
+ if path
237
+ if File.exist?(path)
238
+ load_conf path
239
+ loaded = true
240
+ end
241
+ end
242
+ end
243
+ # unless loaded
244
+ # puts "warnfairy.confファイルが見つかりません"
245
+ # end
246
+ end
247
+
248
+ def load_conf(conf)
249
+ begin
250
+ load conf
251
+ rescue LoadError
252
+ puts "設定ファイル #{conf} をロードできませんでした."
253
+ exit 1
254
+ rescue
255
+ puts "設定ファイル #{@conf} をロード中に例外が発生しました."
256
+ p $!
257
+ p $@
258
+ exit 2
259
+ end
260
+ end
261
+ end
262
+ end
263
+
264
+ DEFAULT_CONF = Conf::DefaultConf.new
265
+
266
+ DEFAULT_CONF.RUBY_BIN = ENV["FAIRY_RUBY"] || "ruby"
267
+
268
+ #DEFAULT_CONF.MASTER_HOST =
269
+ DEFAULT_CONF.MASTER_PORT = 19999
270
+
271
+ # DEFAULT_CONF.HOME = ENV["FAIRY_HOME"] || "/usr/lib/fairy"
272
+ home = ENV["FAIRY_HOME"]
273
+ unless home
274
+ home = File.dirname(File.dirname(File.expand_path(__FILE__)))
275
+ if File.basename(__FILE__) == "conf.rb"
276
+ home = File.dirname(File.dirname(home))
277
+ end
278
+ end
279
+ DEFAULT_CONF.HOME = home
280
+ # DEFAULT_CONF.BIN = DEFAULT_CONF.HOME+"/bin"
281
+ # DEFAULT_CONF.LIB = DEFAULT_CONF.HOME+"/lib"
282
+
283
+ # DEFAULT_CONF.CONTROLLER_BIN = DEFAULT_CONF.BIN+"/controller"
284
+ # DEFAULT_CONF.PROCESSOR_BIN = DEFAULT_CONF.BIN+"/processor"
285
+
286
+ DEFAULT_CONF.DEFAULT_EXTERNAL = nil
287
+ DEFAULT_CONF.DEFAULT_INTERNAL = nil
288
+
289
+ DEFAULT_CONF.IGNORE_EXCEPTION_ON_FILTER = false
290
+
291
+ DEFAULT_CONF.MASTER_MAX_ACTIVE_PROCESSORS = 4
292
+
293
+ DEFAULT_CONF.CONTROLLER_INPUT_PROCESSOR_N = 10
294
+ DEFAULT_CONF.CONTROLLER_MAX_ACTIVE_TASKS_IN_PROCESSOR = 4
295
+
296
+ DEFAULT_CONF.CONTROLLER_ASSIGN_NEW_PROCESSOR_N_FACTOR = 1
297
+
298
+ DEFAULT_CONF.POSTMAPPING_POLICY = nil
299
+
300
+ DEFAULT_CONF.PREQUEUING_POLICY = {:queuing_class => :FileMarshaledQueue}
301
+ DEFAULT_CONF.POSTQUEUING_POLICY = {:queuing_class => :FileMarshaledQueue}
302
+
303
+ DEFAULT_CONF.POSTQUEUE_MAX_TRANSFER_SIZE = 100000
304
+ DEFAULT_CONF.POOLQUEUE_POOL_THRESHOLD = 10000
305
+ DEFAULT_CONF.ONMEMORY_SIZEDQUEUE_SIZE = 10000
306
+ DEFAULT_CONF.FILEBUFFEREDQUEUE_THRESHOLD = 10000/2
307
+
308
+ DEFAULT_CONF.MARSHAL_QUEUE_CHUNK_SIZE = DEFAULT_CONF.POOLQUEUE_POOL_THRESHOLD
309
+ DEFAULT_CONF.MARSHAL_QUEUE_MIN_CHUNK_NO = DEFAULT_CONF.POOLQUEUE_POOL_THRESHOLD
310
+ DEFAULT_CONF.SIZEDMARSHAL_QUEUE_MAX_CHUNK_NO = 10
311
+
312
+ DEFAULT_CONF.SORTEDQUEUE_POOL_THRESHOLD = DEFAULT_CONF.POOLQUEUE_POOL_THRESHOLD
313
+ DEFAULT_CONF.SORTEDQUEUE_THRESHOLD = 10000/2
314
+ DEFAULT_CONF.SORTEDQUEUE_SORTBY = %{|v| v}
315
+
316
+ DEFAULT_CONF.INPUT_LOCAL_FILE_BUFFER_SIZE = 1024*1024
317
+ DEFAULT_CONF.HERE_POOL_THRESHOLD = 32000
318
+ #DEFAULT_CONF.OUTPUT_LOCAL_POOL_THRESHOLD = 32000
319
+
320
+ DEFAULT_CONF.GROUP_BY_NO_SEGMENT = 4
321
+ DEFAULT_CONF.GROUP_BY_HASH_MODULE = "fairy/share/hash-md5"
322
+ DEFAULT_CONF.GROUP_BY_GROUPING_OPTIMIZE = false
323
+
324
+ DEFAULT_CONF.GROUP_BY_BUFFERING_POLICY = {:buffering_class => :DirectMergeSortBuffer}
325
+ DEFAULT_CONF.GROUP_BY_CMSB_THRESHOLD = 400_000
326
+ DEFAULT_CONF.GROUP_BY_CMSB_CHUNK_SIZE = 1000
327
+
328
+ DEFAULT_CONF.BARRIER_MEMORY_BUFFERING_POLICY = {:queuing_class => :PoolQueue}
329
+
330
+ DEFAULT_CONF.SORT_BUFFERING_POLICY = {:buffering_class => "PGroupBy::DirectMergeSortBuffer"}
331
+ DEFAULT_CONF.SORT_SAMPLING_MIN = 100
332
+ DEFAULT_CONF.SORT_SAMPLING_MAX = 10000
333
+ DEFAULT_CONF.SORT_SAMPLING_RATIO_1_TO = 100
334
+ DEFAULT_CONF.SORT_NO_SEGMENT = DEFAULT_CONF.GROUP_BY_NO_SEGMENT
335
+ DEFAULT_CONF.SORT_CMP_OPTIMIZE = false
336
+
337
+ DEFAULT_CONF.IOTA_SPLIT_NO = 4
338
+
339
+ DEFAULT_CONF.TRANSFAR_MARSHAL_STRING_ARRAY_OPTIMIZE = false
340
+
341
+ # DEFAULT_CONF.VF_ROOT = DEFAULT_CONF.HOME+"/Repos"
342
+ DEFAULT_CONF.VF_PREFIX = `hostname`.chomp
343
+ ## DEFAULT_CONF.VF_PREFIX is client setting.
344
+ DEFAULT_CONF.VF_BASE_NAME_CONVERTER = nil
345
+ DEFAULT_CONF.VF_SPLIT_SIZE = 64*1024*1024
346
+
347
+ DEFAULT_CONF.TMP_DIR = "/tmp/fairy/tmpbuf"
348
+
349
+ DEFAULT_CONF.LOG_FILE = "/tmp/fairy/log"
350
+ DEFAULT_CONF.LOG_FLUSH_INTERVAL = 1
351
+ DEFAULT_CONF.LOG_MARK_INTERVAL = 300
352
+ DEFAULT_CONF.LOG_LEVEL = :INFO
353
+ DEFAULT_CONF.LOG_IMPORT_NTIMES_POP = 100000
354
+ DEFAULT_CONF.LOG_LOCAL_OUTPUT_DEV = :$stderr
355
+ DEFAULT_CONF.LOG_ROTATE_INTERVAL = 60*60*24
356
+ DEFAULT_CONF.LOG_ROTATE_N = 7
357
+
358
+ DEFAULT_CONF.SUBCMD_EXEC_TIMEOUT = 60
359
+
360
+ DEFAULT_CONF.PROCESSOR_MON_ON = false
361
+ DEFAULT_CONF.PROCESSOR_MON_INTERVAL = 300
362
+ DEFAULT_CONF.PROCESSOR_MON_PSFORMAT = "stat,vsz,rss,sz,pmem,pcpu,nlwp,time,wchan"
363
+ DEFAULT_CONF.PROCESSOR_MON_OBJECTSPACE_INSPECT_ON = false
364
+
365
+ DEFAULT_CONF.SOCK_DO_NOT_REVERSE_LOOKUP = true
366
+ DEFAULT_CONF.USE_RESOLV_REPLACE = false
367
+
368
+ DEFAULT_CONF.BLOCK_USE_STDOUT = true
369
+
370
+ DEFAULT_CONF.DEBUG_PORT_WAIT = false
371
+ DEFAULT_CONF.DEBUG_FULL_BACKTRACE = false
372
+ DEFAULT_CONF.DEBUG_THREAD_ABORT_ON_EXCEPTION = false
373
+ DEFAULT_CONF.DEBUG_MONITOR_ON = false
374
+ DEFAULT_CONF.DEBUG_PROCESSOR_TRACE_ON = false
375
+ DEFAULT_CONF.DEBUG_BUG49 = false
376
+
377
+ DEFAULT_CONF.BUG234 = false
378
+
379
+ # DEFAULT_CONF.PROCESS_LIFE_MANAGE_INTERVAL = 60
380
+ # DEFAULT_CONF.PROCESS_LIFE_MANAGE_INTERVAL = 10
381
+ # DEFAULT_CONF.PROCESS_LIFE_MANAGE_INTERVAL = 1
382
+ DEFAULT_CONF.PROCESS_LIFE_MANAGE_INTERVAL = nil
383
+
384
+ # DEFAULT_CONF.THREAD_STACK_SIZE = 1024*100
385
+
386
+ CONF = DEFAULT_CONF
387
+ CONF.load_all_conf
388
+
389
+ def REPLACE_CONF(conf)
390
+ remove_const(:CONF)
391
+ const_set(:CONF, conf)
392
+ end
393
+ module_function :REPLACE_CONF
394
+
395
+
396
+ def Conf.configure_common_conf
397
+ Thread.abort_on_exception = CONF.DEBUG_THREAD_ABORT_ON_EXCEPTION
398
+
399
+ TCPSocket.do_not_reverse_lookup = CONF.SOCK_DO_NOT_REVERSE_LOOKUP
400
+
401
+ if CONF.USE_RESOLV_REPLACE
402
+ require "resolv-replace"
403
+ end
404
+ end
405
+ end
406
+
407
+ ##Original from lib/fairy/share/base-app.rb
408
+ # encoding: utf-8
409
+ #
410
+ # Copyright (C) 2007-2010 Rakuten, Inc.
411
+ #
412
+
413
+ module Fairy
414
+ class BaseAPP
415
+
416
+ def self.start
417
+ @@APP = new
418
+ @@APP.parse_arg
419
+ @@APP.load_conf
420
+ @@APP.configure_conf
421
+ @@APP.start
422
+ end
423
+
424
+ def self.start_subcommand(prog, *opts)
425
+ @@APP.start_subcommand(prog, *opts)
426
+ end
427
+ def self.start_subcommand2(prog, *opts)
428
+ Process.fork do
429
+ start_subcommand(prog, *opts)
430
+ end
431
+ end
432
+
433
+ def initialize
434
+ @home = nil
435
+ @conf = nil
436
+ @CONF = {}
437
+ end
438
+
439
+ def parse_arg
440
+ @opt = option_parser
441
+ @opt.order!(ARGV)
442
+ end
443
+
444
+ def option_parser
445
+ opt = OptionParser.new{|opt|
446
+ opt.on("--home=VAL"){|val| @home = val}
447
+ opt.on("--conf=VAL"){|val| @conf = val}
448
+ CONF.props.each do |prop|
449
+ opt.on("--CONF.#{prop}=VAL") {|val| @CONF[prop] = val}
450
+ end
451
+ }
452
+ if block_given?
453
+ yield opt
454
+ end
455
+ end
456
+
457
+ def conf_to_arg
458
+ @CONF.collect{|prop, source| "--CONF.#{prop}=#{source}"}
459
+ end
460
+
461
+ def load_conf
462
+ if @home
463
+ ENV["FAIRY_HOME"] = @home
464
+ # ENV["FAIRY_HOME"] = File.expand_path(@home)
465
+ conf = @home+"/etc/fairy.conf"
466
+ if File.exists?(conf)
467
+ CONF.load_conf conf
468
+ end
469
+ end
470
+
471
+ if @conf
472
+ ENV["FAIRY_CONF"] = @conf
473
+ CONF.load_conf @conf
474
+ end
475
+
476
+ @CONF.each do |key, value|
477
+ begin
478
+ eval "CONF.#{key}=#{value}"
479
+ rescue
480
+ puts "CONF.#{key}=#{value} が不正です."
481
+ exit 1
482
+ end
483
+ end
484
+ if ENV["RUBYLIB"]
485
+ ENV["RUBYLIB"] = CONF.LIB + ":" + ENV["RUBYLIB"]
486
+ else
487
+ ENV["RUBYLIB"] = CONF.LIB
488
+ end
489
+ end
490
+
491
+ def configure_conf
492
+ Conf.configure_common_conf
493
+ end
494
+
495
+ def start
496
+ ERR::Raise ERR::INTERNAL::ShouldDefineSubclass
497
+ end
498
+
499
+ def start_subcommand(prog, *opts)
500
+ opts.push *conf_to_arg
501
+ # Process.fork do
502
+ # Log.stop_export
503
+ # ObjectSpace.each_object(IO) do |io|
504
+ # begin
505
+ # if ![0, 1, 2].include?(io.fileno )
506
+ # io.close
507
+ # end
508
+ # rescue
509
+ # end
510
+ # end
511
+ # exec(prog, *opts)
512
+ # end
513
+
514
+ Process.spawn(prog, *opts)
515
+ # system("#{prog} #{opts.join(' ')}&")
516
+
517
+ end
518
+
519
+ end
520
+ end
521
+
522
+
523
+ ##Original from lib/fairy/template/fairy-BODY.templ
524
+ # -*- ruby -*-
525
+ #
526
+ # Copyright (C) 2007-2010 Rakuten, Inc.
527
+ #
528
+ #
529
+ module Fairy
530
+ # LC_MESSAGES = IRB::Locale.new
531
+ # LC_MESSAGES.load(CONF.LIB+"/fairy/share/exceptions.rb")
532
+ class FairyAPP<BaseAPP
533
+
534
+ SUBCOMMANDS = ["cp", "rm", "cat", "master", "node", "inspector"]
535
+
536
+ def initialize
537
+ super
538
+ @mode = :SUBCOMMAND
539
+ end
540
+
541
+ def option_parser
542
+ super do |opt|
543
+ opt.on("-f", "--file"){|val| @mode = :FILE}
544
+ opt.on("-e", "--exec"){|val| @mode = :EXEC}
545
+ end
546
+ end
547
+
548
+ def start
549
+ if @mode == :SUBCOMMAND &&
550
+ (ARGV.empty? || nosub = !SUBCOMMANDS.include?(ARGV[0]))
551
+ puts "そのようなサブコマンドはありません(#{ARGV[0]})." if nosub
552
+
553
+ helpbase = @opt.help.split(/\n/)
554
+ helpbase[0].concat(" subcommand [subcommand-options]")
555
+ helpbase.insert(1, " Options:")
556
+ helpbase.insert(1,
557
+ " Subcommands:",
558
+ *SUBCOMMANDS.collect{|s| " "+s})
559
+
560
+ puts helpbase
561
+ exit
562
+ end
563
+
564
+ case @mode
565
+ when :SUBCOMMAND
566
+ if File.exist?(CONF.SUBCMD+"/"+ARGV[0])
567
+ Process.exec(CONF.RUBY_BIN, CONF.SUBCMD+"/"+ARGV[0], *ARGV[1..-1])
568
+ else
569
+ Process.exec(CONF.RUBY_BIN, CONF.BIN+"/fairy-"+ARGV[0], *ARGV[1..-1])
570
+ end
571
+ when :FILE
572
+ Process.exec(CONF.RUBY_BIN, *ARGV)
573
+ when :EXEC
574
+ Process.exec(*ARGV)
575
+ end
576
+ end
577
+ end
578
+ end
579
+
580
+ Fairy::FairyAPP.start
581
+
582
+