nodule 0.0.34

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,292 @@
1
+ require 'digest/md5'
2
+ require 'fileutils'
3
+ require 'open-uri'
4
+ require 'yaml'
5
+ require 'nodule/process'
6
+ require 'nodule/tempfile'
7
+ require 'nodule/util'
8
+ require 'cassandra'
9
+
10
+ module Nodule
11
+ #
12
+ # Run temporary instances of Apache Cassandra.
13
+ # Generates random ports for rpc/storage and temporary directories for data,
14
+ # commit logs, etc..
15
+ #
16
+ # The version of Cassandra is hard-coded to 1.1.0.
17
+ #
18
+ class Cassandra < Process
19
+ attr_reader :tmp, :keyspace, :data, :caches, :commit, :pidfile, :cassbin, :config, :envfile, :rpc_port
20
+
21
+ # These two must match. Apache posts the md5's on the download site.
22
+ VERSION = "1.1.0"
23
+ MD5 = "8befe18a4abc342d03d1fbaaa0ac836b"
24
+
25
+ CASSANDRA = "apache-cassandra-#{VERSION}"
26
+ TARBALL = "#{CASSANDRA}-bin.tar.gz"
27
+ TARBALL_URL = "http://archive.apache.org/dist/cassandra/#{VERSION}/#{TARBALL}"
28
+
29
+ # potential locations for caching the cassandra download
30
+ CACHEDIRS = [
31
+ File.join(ENV['HOME'], 'Downloads'),
32
+ "/tmp",
33
+ ]
34
+
35
+ # keep large timeouts, since test systems (e.g. Jenkins workers) are often very slow
36
+ CLIENT_CONNECT_OPTIONS = {
37
+ :timeout => 30,
38
+ :connect_timeout => 30,
39
+ :retries => 10,
40
+ :exception_classes => [],
41
+ }
42
+
43
+ #
44
+ # Create a new Nodule::Cassandra instance. Each instance will be its own single-node Cassandra instance.
45
+ #
46
+ # @param [Hash] opts the options for setup.
47
+ # @option opts [String] :keyspace Keyspace name to use as the default
48
+ #
49
+ def initialize(opts={})
50
+ @keyspace = opts[:keyspace] || "Nodule"
51
+
52
+ @temp = Nodule::Tempfile.new(:directory => true, :prefix => "nodule-cassandra")
53
+ @tmp = @temp.file
54
+
55
+ @data = File.join(@tmp, 'data')
56
+ @caches = File.join(@tmp, 'caches')
57
+ @commit = File.join(@tmp, 'commitlogs')
58
+
59
+ @host = "127.0.0.1" # will support 127.0.0.2 someday
60
+ @jmx_port = Nodule::Util.random_tcp_port
61
+ @rpc_port = Nodule::Util.random_tcp_port
62
+ @storage_port = Nodule::Util.random_tcp_port
63
+ @ssl_storage_port = Nodule::Util.random_tcp_port
64
+
65
+ @casshome = "#{@tmp}/#{CASSANDRA}"
66
+ @pidfile = "#{@casshome}/cassandra.pid"
67
+ @cassbin = "#{@casshome}/bin"
68
+ @command = ["#{@cassbin}/cassandra", "-f", "-p", @pidfile]
69
+ @config = "#{@casshome}/conf/cassandra.yaml"
70
+ @envfile = "#{@casshome}/conf/cassandra-env.sh"
71
+ @log4j = "#{@casshome}/conf/log4j-server.properties"
72
+ @logfile = "#{@tmp}/system.log"
73
+
74
+ # This handler reads STDOUT to determine when Cassandra is ready for client
75
+ # access. Coerce the stdout option into an array as necessar so options can
76
+ # still be passed in.
77
+ if opts[:stdout]
78
+ unless opts[:stdout].kind_of? Array
79
+ opts[:stdout] = [ opts.delete(:stdout) ]
80
+ end
81
+ else
82
+ opts[:stdout] = []
83
+ end
84
+
85
+ # Watch Cassandra's output to be sure when it's available, obviously, it's a bit fragile
86
+ # but (IMO) better than sleeping or poking the TCP port.
87
+ @mutex = Mutex.new
88
+ @cv = ConditionVariable.new
89
+ opts[:stdout] << proc do |item|
90
+ @mutex.synchronize do
91
+ @cv.signal if item =~ /Listening for thrift clients/
92
+ end
93
+ end
94
+
95
+ super({"CASSANDRA_HOME" => @casshome}, *@command, opts)
96
+ end
97
+
98
+ #
99
+ # Downloads Cassandra tarball to memory from the Apache servers.
100
+ # @return [String] binary string containing the tar/gzip data.
101
+ #
102
+ def download
103
+ tardata = open(TARBALL_URL).read
104
+ digest = Digest::MD5.hexdigest(tardata)
105
+
106
+ unless digest == MD5
107
+ raise "Expected MD5 #{MD5} but got #{digest}."
108
+ end
109
+
110
+ tardata
111
+ end
112
+
113
+ #
114
+ # Write the tarball to a file locally. Finds a directory in the CACHEDIRS list.
115
+ # @param [String] binary string containing tar/gzip data.
116
+ # @return [String] full path of the file
117
+ #
118
+ def cache_tarball!(tardata)
119
+ cachedir = (CACHEDIRS.select { |path| File.directory?(path) and File.writable?(path) })[0]
120
+ cachefile = File.join(cachedir, TARBALL)
121
+ File.open(cachefile, "wb").write(tardata)
122
+ cachefile
123
+ end
124
+
125
+ #
126
+ # Downloads Cassandra tarball from the Apache servers.
127
+ # @param [String] full path to the tarball file
128
+ #
129
+ def untar!(tarball)
130
+ system("tar -C #{@tmp} -xzf #{tarball}")
131
+ end
132
+
133
+ #
134
+ # Rewrites portions of the stock Cassandra configuration. This should work fairly well over Cassandra
135
+ # version bumps without editing as long as the Cassandra folks don't wildly change param names.
136
+ # Modifies conf/cassandra.yaml and conf/cassandra-env.sh.
137
+ #
138
+ def configure!
139
+ conf = YAML::load_file(@config)
140
+ conf.merge!({
141
+ "initial_token" => 0,
142
+ "partitioner" => "org.apache.cassandra.dht.RandomPartitioner",
143
+ # have to force ascii or YAML will come out as binary
144
+ "data_file_directories" => [@data.encode("us-ascii")],
145
+ "commitlog_directory" => @commit.encode("us-ascii"),
146
+ "saved_caches_directory" => @caches.encode("us-ascii"),
147
+ "storage_port" => @storage_port.to_i,
148
+ "ssl_storage_port" => @ssl_storage_port.to_i,
149
+ "listen_address" => @host.encode("us-ascii"),
150
+ "rpc_address" => @host.encode("us-ascii"),
151
+ "rpc_port" => @rpc_port.to_i,
152
+ # DSE doesn't work OOTB as a single node unless you switch to simplesnitch
153
+ "endpoint_snitch" => "org.apache.cassandra.locator.SimpleSnitch",
154
+ })
155
+ File.open(@config, "w") { |file| file.puts YAML::dump(conf) }
156
+
157
+ # relocate the JMX port to avoid conflicts with running instances
158
+ env = File.read(@envfile)
159
+ env.sub!(/JMX_PORT=['"]?\d+['"]?/, "JMX_PORT=\"#{@jmx_port}\"")
160
+ File.open(@envfile, "w") { |file| file.puts env }
161
+
162
+ # relocate the system.log
163
+ log = File.read(@log4j)
164
+ log.sub!(/log4j.appender.R.File=.*$/, "log4j.appender.R.File=#{@logfile}")
165
+ File.open(@log4j, "w") do |file| file.puts log end
166
+ end
167
+
168
+ #
169
+ # Create a keyspace in the newly minted Cassandra instance.
170
+ #
171
+ def create_keyspace
172
+ ksdef = CassandraThrift::KsDef.new(
173
+ :name => @keyspace,
174
+ :strategy_class => 'org.apache.cassandra.locator.SimpleStrategy',
175
+ :strategy_options => { "replication_factor" => "1" },
176
+ :cf_defs => []
177
+ )
178
+ client('system').add_keyspace ksdef
179
+ end
180
+
181
+ #
182
+ # Run the download or untar the cached tarball. Configure then start Cassandra.
183
+ #
184
+ def run
185
+ FileUtils.mkdir_p @data
186
+ FileUtils.mkdir_p @caches
187
+ FileUtils.mkdir_p @commit
188
+
189
+ cached = CACHEDIRS.select { |path| File.exists? File.join(path, TARBALL) }
190
+ if cached.any?
191
+ untar! File.join(cached.first, TARBALL)
192
+ else
193
+ file = cache_tarball! download
194
+ untar! file
195
+ end
196
+
197
+ configure!
198
+
199
+ # will start Cassandra process
200
+ super
201
+
202
+ # wait for Cassandra to say it's ready
203
+ @mutex.synchronize do @cv.wait @mutex end
204
+ end
205
+
206
+ #
207
+ # Stop cassandra with a signal, clean up with recursive delete.
208
+ #
209
+ def stop
210
+ super
211
+ @temp.stop
212
+ end
213
+
214
+ #
215
+ # Setup and return a Cassandra client object.
216
+ # @param [String] keyspace optional keyspace argument for the client connection
217
+ # @return [Cassandra] connection to the temporary Cassandra instance
218
+ #
219
+ def client(ks=@keyspace)
220
+ c = ::Cassandra.new(ks, self.to_s, CLIENT_CONNECT_OPTIONS)
221
+ c.disable_node_auto_discovery!
222
+
223
+ yield(c) if block_given?
224
+
225
+ c
226
+ end
227
+
228
+ #
229
+ # Returns the fully-quailified cassandra-cli command with host & port set. If given a list of
230
+ # arguments, they're tacked on automatically.
231
+ # @param [Array] more_args additional command-line arguments
232
+ # @return [Array] an argv-style array ready to use with Nodule::Process or Kernel.spawn
233
+ #
234
+ def cli_command(*more_args)
235
+ [File.join(@cassbin, 'cassandra-cli'), '-h', @host, '-p', @rpc_port, more_args].flatten
236
+ end
237
+
238
+ #
239
+ # Run a block with access to cassandra-cli's stdio.
240
+ # @param [Array] more_args additional command-line arguments
241
+ # @yield block with CLI attached
242
+ # @option block [Nodule::Process] process Nodule::Process object wrapping the CLI
243
+ # @option block [IO] stdin
244
+ # @option block [IO] stdout
245
+ # @option block [IO] stderr
246
+ #
247
+ def cli(*more_args)
248
+ process = Process.new *cli_command(more_args)
249
+ process.join_topology! @topology
250
+ process.run
251
+ yield process, process.stdin_pipe, process.stdout_pipe, process.stderr_pipe
252
+ process.print "quit;\n" unless process.done?
253
+ process.wait 3
254
+ process.stop
255
+ end
256
+
257
+ #
258
+ # Returns the fully-quailified nodetool command with host & JMX port set. If given a list of
259
+ # arguments, they're tacked on automatically.
260
+ # @param [Array] more_args additional command-line arguments
261
+ # @return [Array] an argv-style array ready to use with Nodule::Process or Kernel.spawn
262
+ #
263
+ def nodetool_command(*more_args)
264
+ [File.join(@cassbin, 'nodetool'), '-h', @host, '-p', @jmx_port, more_args].flatten
265
+ end
266
+
267
+ #
268
+ # @param [Array] more_args additional command-line arguments
269
+ # @yield block with CLI attached
270
+ # @option block [Nodule::Process] process Nodule::Process object wrapping the CLI
271
+ # @option block [IO] stdin
272
+ # @option block [IO] stdout
273
+ # @option block [IO] stderr
274
+ #
275
+ def nodetool(*more_args)
276
+ process = Process.new *nodetool_command(more_args)
277
+ process.join_topology! @topology
278
+ process.run
279
+ yield process, process.stdin_pipe, process.stdout_pipe, process.stderr_pipe
280
+ process.wait 3
281
+ process.stop
282
+ end
283
+
284
+ #
285
+ # Stringify this class to the cassandra host/port string, e.g. "127.0.0.1:12345"
286
+ # @return [String] Cassandra connection string.
287
+ #
288
+ def to_s
289
+ [@host, @rpc_port].join(':')
290
+ end
291
+ end
292
+ end
@@ -0,0 +1,87 @@
1
+ require 'nodule/base'
2
+
3
+ module Nodule
4
+ #
5
+ # a simple colored output resource
6
+ #
7
+ # e.g. Nodule::Console.new(:fg => :green)
8
+ # Nodule::Console.new(:fg => :green, :bg => :white)
9
+ #
10
+ class Console < Base
11
+ COLORS = {
12
+ :black => "\x1b[30m",
13
+ :red => "\x1b[31m",
14
+ :green => "\x1b[32m",
15
+ :yellow => "\x1b[33m",
16
+ :blue => "\x1b[34m",
17
+ :magenta => "\x1b[35m",
18
+ :cyan => "\x1b[36m",
19
+ :white => "\x1b[37m",
20
+ :dkgray => "\x1b[1;30m",
21
+ :dkred => "\x1b[1;31m",
22
+ :reset => "\x1b[0m",
23
+ }.freeze
24
+
25
+ #
26
+ # Create a new console handler. Defaults to printing to STDERR without color.
27
+ # Color output is automatically disabled on non-tty devices. It can be
28
+ # force-enabled with the CLICOLOR_FORCE environment variable.
29
+ # The list of valid colors is in Nodule::Console::COLORS.
30
+ #
31
+ # @param [Hash] opts
32
+ # @option [Symbol] opts :fg the foreground color symbol
33
+ # @option [Symbol] opts :bg the background color symbol
34
+ # @option [IO] opts :io default STDERR an IO object to display on, STDOUT/files should work fine
35
+ #
36
+ def initialize(opts={})
37
+ super(opts)
38
+ @fg = opts[:fg]
39
+ @bg = opts[:bg]
40
+
41
+ if @fg and not COLORS.has_key?(@fg)
42
+ raise ArgumentError.new "fg :#{@fg} is not a valid color"
43
+ end
44
+
45
+ if @bg and not COLORS.has_key?(@bg)
46
+ raise ArgumentError.new "bg :#{@bg} is not a valid color"
47
+ end
48
+
49
+ # IO handle to use as the console
50
+ @io = opts[:io] || STDERR
51
+
52
+ # from https://github.com/sickill/rainbow/blob/master/lib/rainbow.rb
53
+ @enabled = @io.tty? && ENV['TERM'] != 'dumb' || ENV['CLICOLOR_FORCE'] == '1'
54
+
55
+ add_reader { |line,src| display(src, line) }
56
+ end
57
+
58
+ def fg(str)
59
+ return str unless @enabled
60
+ "#{COLORS[@fg]}#{str}"
61
+ end
62
+
63
+ def bg(str)
64
+ return str unless @enabled
65
+ "#{COLORS[@bg]}#{str}"
66
+ end
67
+
68
+ def reset(str)
69
+ return str unless @enabled
70
+ "#{str}#{COLORS[:reset]}"
71
+ end
72
+
73
+ #
74
+ # Write to stdout using puts, but append a prefix if it's defined.
75
+ # @param [Object] src if this responds to :prefix, :prefix will be prepended to the output
76
+ # @param [String] line the data to write to stdout
77
+ #
78
+ def display(src, line)
79
+ if src.respond_to? :prefix
80
+ @io.print "#{reset('')}#{src.prefix}#{reset(bg(fg(line)))}\n"
81
+ else
82
+ @io.print "#{reset(bg(fg(line)))}\n"
83
+ end
84
+ end
85
+
86
+ end
87
+ end
@@ -0,0 +1,74 @@
1
+ require 'nodule/base'
2
+
3
+ module Nodule
4
+ class LineIO < Base
5
+ #
6
+ # A few extra bits to help with handling IO objects (files, pipes, etc.), like setting
7
+ # up a background thread to select() and read lines from it and call run_readers.
8
+ #
9
+ # @param [Hash{Symbol => IO,Symbol,Proc}] opts
10
+ # @option opts [IO] :io required IO object, pipes & files should work fine
11
+ #
12
+ # @example
13
+ # r, w = IO.pipe
14
+ # nio = Nodule::Stdio.new :io => r, :run => true
15
+ #
16
+ def initialize(opts={})
17
+ @running = false
18
+ raise ArgumentError.new ":io is required and must be a descendent of IO" unless opts[:io].kind_of?(IO)
19
+ @io = opts.delete(:io)
20
+
21
+ super(opts)
22
+ end
23
+
24
+ #
25
+ # Create a background thread to read from IO and call Nodule run_readers.
26
+ #
27
+ def run
28
+ super
29
+
30
+ Thread.new do
31
+ begin
32
+ @running = true # relies on the GIL
33
+ while @running do
34
+ ready = IO.select([@io], [], [], 0.2)
35
+ unless ready.nil?
36
+ line = @io.readline
37
+ run_readers(line, self)
38
+ end
39
+ end
40
+
41
+ # run may be over, but read the rest of the data up to EOF anyways
42
+ @io.foreach do |line|
43
+ run_readers(line, self)
44
+ end
45
+ rescue EOFError
46
+ verbose "EOFError: #{@io} probably closed."
47
+ @io.close
48
+ Thread.current.exit
49
+ rescue Exception => e
50
+ STDERR.print "Exception in #{name} IO thread: #{e.inspect}\n"
51
+ abort e
52
+ end
53
+ end
54
+
55
+ wait_with_backoff 30 do @running end
56
+ end
57
+
58
+ #
59
+ # simply calls print *args on the io handle
60
+ # @param [String] see IO.print
61
+ #
62
+ def print(*args)
63
+ @io.print(*args)
64
+ end
65
+
66
+ #
67
+ # calls io.puts *args
68
+ # @param [String] see IO.print
69
+ #
70
+ def puts(*args)
71
+ @io.puts(*args)
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,8 @@
1
+ class Array
2
+ def fuzzy_filter(h)
3
+ keep_if do |item|
4
+ raise "All elements in this array need to be of type Hash: (#{item.class}) #{item.inspect}" if item.class != Hash
5
+ h.keys.all? { |k| h[k] === item[k] }
6
+ end
7
+ end
8
+ end