hadupils 0.6.1 → 0.7.2
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG.md +18 -0
- data/lib/hadupils/commands.rb +2 -1
- data/lib/hadupils/extensions.rb +37 -27
- data/test/unit/commands_test.rb +5 -5
- metadata +16 -16
data/CHANGELOG.md
CHANGED
@@ -78,3 +78,21 @@
|
|
78
78
|
was hanging reading from stderr, which isn't strictly necessary anyway.
|
79
79
|
Now it no hangy.
|
80
80
|
|
81
|
+
### 0.6.2
|
82
|
+
* Stopped the hadupils cleanup command from outputting 'Removing...' to STDOUT when it
|
83
|
+
doesn't have anything to actually remove.
|
84
|
+
|
85
|
+
### 0.7.0
|
86
|
+
* Now using Process.spawn instead of Kernel.system, the ruby way for 1.9+
|
87
|
+
* Fixed subcommand process deadlocking when any IO pipe buffer became full (Ruby 1.9+) via
|
88
|
+
threads and process waiting, the ruby way or a ruby way at least ;)
|
89
|
+
* $HADUPILS_TMP_TTL now defaults to 1209600 (last 2 weeks)
|
90
|
+
* Passing unit tests
|
91
|
+
|
92
|
+
### 0.7.1
|
93
|
+
* Cleanup command now outputs additional context, candidate tmpdir, to help identify the
|
94
|
+
runtime state for long running cleanups
|
95
|
+
* Now ignoring the 'vendor' directory in .gitignore
|
96
|
+
|
97
|
+
### 0.7.2
|
98
|
+
* $HADUPILS_TMP_TTL default reverts back to 86400 (last 24 hours)
|
data/lib/hadupils/commands.rb
CHANGED
@@ -234,7 +234,7 @@ module Hadupils::Commands
|
|
234
234
|
# Now want the user to see the Runner's shell STDOUT
|
235
235
|
Shell.silence_stdout = false
|
236
236
|
|
237
|
-
puts 'Removing...'
|
237
|
+
puts 'Removing...' unless rm_array.empty?
|
238
238
|
rm_array.each do |dir|
|
239
239
|
rm_stdout, rm_exitstatus = Hadupils::Commands::RmFile.run ['-r', dir]
|
240
240
|
rm_exitstatuses << rm_exitstatus
|
@@ -248,6 +248,7 @@ module Hadupils::Commands
|
|
248
248
|
end
|
249
249
|
|
250
250
|
def has_expired?(dir_candidate, ttl)
|
251
|
+
puts "Checking directory candidate: #{dir_candidate}"
|
251
252
|
stdout, exitstatus = Hadupils::Commands::Hadoop.run ['fs', '-count', dir_candidate]
|
252
253
|
expired_exitstatuses << exitstatus
|
253
254
|
if successful? exitstatus
|
data/lib/hadupils/extensions.rb
CHANGED
@@ -7,22 +7,19 @@ module Hadupils::Extensions
|
|
7
7
|
module Runners
|
8
8
|
module Shell
|
9
9
|
def self.command(*command_list)
|
10
|
-
opts
|
11
|
-
stdout = nil
|
12
|
-
stderr = nil
|
13
|
-
status = nil
|
10
|
+
opts = {}
|
14
11
|
|
15
12
|
begin
|
16
13
|
if RUBY_VERSION < '1.9'
|
17
14
|
Open3.popen3(*command_list) do |i, o, e|
|
18
|
-
stdout = o.read
|
19
|
-
stderr = e.read
|
15
|
+
@stdout = o.read
|
16
|
+
@stderr = e.read
|
20
17
|
end
|
21
|
-
status = $?
|
22
|
-
$stdout.puts stdout unless stdout.nil? || stdout.empty? || Shell.silence_stdout?
|
23
|
-
$stderr.puts stderr unless stderr.nil? || stderr.empty?
|
24
|
-
stdout = nil unless capture_stdout?
|
25
|
-
stderr = nil unless capture_stderr?
|
18
|
+
@status = $?
|
19
|
+
$stdout.puts @stdout unless @stdout.nil? || @stdout.empty? || Shell.silence_stdout?
|
20
|
+
$stderr.puts @stderr unless @stderr.nil? || @stderr.empty?
|
21
|
+
@stdout = nil unless capture_stdout?
|
22
|
+
@stderr = nil unless capture_stderr?
|
26
23
|
else
|
27
24
|
stdout_rd, stdout_wr = IO.pipe if capture_stdout?
|
28
25
|
stderr_rd, stderr_wr = IO.pipe if capture_stderr?
|
@@ -30,25 +27,38 @@ module Hadupils::Extensions
|
|
30
27
|
opts[:err] = stderr_wr if capture_stderr?
|
31
28
|
|
32
29
|
# NOTE: eval prevents Ruby 1.8.7 from throwing a syntax error on Ruby 1.9+ syntax
|
33
|
-
|
34
|
-
status =
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
$stdout.puts stdout unless stdout.nil? || stdout.empty? || Shell.silence_stdout?
|
30
|
+
pid = eval 'Process.spawn(*command_list, opts)'
|
31
|
+
@status = :waiting unless pid.nil?
|
32
|
+
|
33
|
+
Thread.new do
|
34
|
+
Process.wait pid
|
35
|
+
@status = $?
|
40
36
|
end
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
37
|
+
|
38
|
+
Thread.new do
|
39
|
+
if capture_stdout?
|
40
|
+
stdout_wr.close
|
41
|
+
@stdout = stdout_rd.read
|
42
|
+
stdout_rd.close
|
43
|
+
$stdout.puts @stdout unless @stdout.nil? || @stdout.empty? || Shell.silence_stdout?
|
44
|
+
end
|
46
45
|
end
|
46
|
+
|
47
|
+
Thread.new do
|
48
|
+
if capture_stderr?
|
49
|
+
stderr_wr.close
|
50
|
+
@stderr = stderr_rd.read
|
51
|
+
stderr_rd.close
|
52
|
+
$stderr.puts @stderr unless @stderr.nil? || @stderr.empty?
|
53
|
+
end
|
54
|
+
end
|
55
|
+
sleep 0.1 while @status == :waiting
|
47
56
|
end
|
48
|
-
|
57
|
+
|
58
|
+
[@stdout, @stderr, @status]
|
49
59
|
rescue Errno::ENOENT => e
|
50
60
|
$stderr.puts e
|
51
|
-
[stdout, stderr, nil]
|
61
|
+
[@stdout, @stderr, nil]
|
52
62
|
end
|
53
63
|
end
|
54
64
|
|
@@ -86,7 +96,7 @@ module Hadupils::Extensions
|
|
86
96
|
end
|
87
97
|
|
88
98
|
def self.tmp_ttl
|
89
|
-
@tmp_ttl ||= (ENV['HADUPILS_TMP_TTL'] || '
|
99
|
+
@tmp_ttl ||= (ENV['HADUPILS_TMP_TTL'] || '1209600').to_i
|
90
100
|
end
|
91
101
|
|
92
102
|
def self.tmp_path
|
@@ -174,7 +184,7 @@ module Hadupils::Extensions
|
|
174
184
|
@file_handler || ::Tempfile
|
175
185
|
end
|
176
186
|
|
177
|
-
# Sets up a wrapped file, using the class' file_handler,
|
187
|
+
# Sets up a wrapped file, using the class' file_handler,
|
178
188
|
def initialize
|
179
189
|
@file = self.class.file_handler.new('hadupils-hiverc')
|
180
190
|
end
|
data/test/unit/commands_test.rb
CHANGED
@@ -292,7 +292,7 @@ class Hadupils::CommandsTest < Test::Unit::TestCase
|
|
292
292
|
end
|
293
293
|
|
294
294
|
should 'produce a valid set of parameters and hivercs' do
|
295
|
-
|
295
|
+
Process.stubs(:spawn).with() do |*args|
|
296
296
|
args[0] == {'HIVE_AUX_JARS_PATH' => @hive_aux_jars_path_val} &&
|
297
297
|
args[1] == @hive_prog &&
|
298
298
|
args[2] == '-i' &&
|
@@ -464,7 +464,7 @@ class Hadupils::CommandsTest < Test::Unit::TestCase
|
|
464
464
|
run_common_cleanup_assertions_with(tmp_path, tmpdir1, tmpdir2)
|
465
465
|
instance = @klass.new([])
|
466
466
|
assert_equal [nil, 0], instance.run
|
467
|
-
assert_equal
|
467
|
+
assert_equal 1209600, instance.tmp_ttl
|
468
468
|
assert_equal '/tmp', instance.tmp_path
|
469
469
|
end
|
470
470
|
|
@@ -476,7 +476,7 @@ class Hadupils::CommandsTest < Test::Unit::TestCase
|
|
476
476
|
run_common_cleanup_assertions_with(tmp_path, tmpdir1, tmpdir2)
|
477
477
|
instance = @klass.new([tmp_path])
|
478
478
|
assert_equal [nil, 0], instance.run
|
479
|
-
assert_equal
|
479
|
+
assert_equal 1209600, instance.tmp_ttl
|
480
480
|
assert_equal tmp_path, instance.tmp_path
|
481
481
|
end
|
482
482
|
|
@@ -504,8 +504,8 @@ class Hadupils::CommandsTest < Test::Unit::TestCase
|
|
504
504
|
def run_common_cleanup_assertions_with(tmp_path, tmpdir1, tmpdir2)
|
505
505
|
ls_stdout =
|
506
506
|
"Found 2 items\n" +
|
507
|
-
"drwx------ -
|
508
|
-
"drwx------ -
|
507
|
+
"drwx------ - someuser somegroup 0 2013-10-24 16:23 #{tmpdir1}\n" +
|
508
|
+
"drwx------ - someuser somegroup 0 2013-10-24 16:23 #{tmpdir2}\n"
|
509
509
|
count_stdout1 = " 1 0 0 hdfs://localhost:9000#{tmpdir1}\n"
|
510
510
|
count_stdout2 = " 1 1 0 hdfs://localhost:9000#{tmpdir2}\n"
|
511
511
|
Hadupils::Runners::Hadoop.expects(:run).with(['fs', '-ls', tmp_path]).returns([ls_stdout, 0])
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: hadupils
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.7.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2014-11-14 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: uuid
|
@@ -34,7 +34,7 @@ dependencies:
|
|
34
34
|
requirements:
|
35
35
|
- - ~>
|
36
36
|
- !ruby/object:Gem::Version
|
37
|
-
version: 1.
|
37
|
+
version: 1.6.2
|
38
38
|
type: :development
|
39
39
|
prerelease: false
|
40
40
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -42,7 +42,7 @@ dependencies:
|
|
42
42
|
requirements:
|
43
43
|
- - ~>
|
44
44
|
- !ruby/object:Gem::Version
|
45
|
-
version: 1.
|
45
|
+
version: 1.6.2
|
46
46
|
- !ruby/object:Gem::Dependency
|
47
47
|
name: mocha
|
48
48
|
requirement: !ruby/object:Gem::Requirement
|
@@ -98,31 +98,31 @@ executables:
|
|
98
98
|
extensions: []
|
99
99
|
extra_rdoc_files: []
|
100
100
|
files:
|
101
|
-
- lib/hadupils/search.rb
|
102
101
|
- lib/hadupils/commands.rb
|
103
|
-
- lib/hadupils/
|
102
|
+
- lib/hadupils/search.rb
|
104
103
|
- lib/hadupils/extensions/hive.rb
|
105
|
-
- lib/hadupils/helpers.rb
|
106
104
|
- lib/hadupils/commands/options.rb
|
107
105
|
- lib/hadupils/extensions.rb
|
106
|
+
- lib/hadupils/util.rb
|
108
107
|
- lib/hadupils/hacks.rb
|
108
|
+
- lib/hadupils/runners.rb
|
109
|
+
- lib/hadupils/helpers.rb
|
109
110
|
- lib/hadupils/assets.rb
|
110
|
-
- lib/hadupils/util.rb
|
111
111
|
- lib/hadupils.rb
|
112
|
-
- test/
|
113
|
-
- test/unit/commands_test.rb
|
112
|
+
- test/hadupil_test_setup.rb
|
114
113
|
- test/unit/runners_test.rb
|
115
|
-
- test/unit/search_test.rb
|
116
114
|
- test/unit/extensions/base_test.rb
|
117
115
|
- test/unit/extensions/hive_test.rb
|
118
|
-
- test/
|
116
|
+
- test/unit/assets_test.rb
|
117
|
+
- test/unit/search_test.rb
|
118
|
+
- test/unit/commands_test.rb
|
119
119
|
- bin/hadupils
|
120
|
+
- CHANGELOG.md
|
121
|
+
- Gemfile
|
120
122
|
- Rakefile.rb
|
121
|
-
- Gemfile.lock
|
122
123
|
- README.md
|
123
|
-
- Gemfile
|
124
|
+
- Gemfile.lock
|
124
125
|
- LICENSE
|
125
|
-
- CHANGELOG.md
|
126
126
|
homepage: http://github.com/ethanrowe/hadupils
|
127
127
|
licenses:
|
128
128
|
- MIT
|
@@ -144,7 +144,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
144
144
|
version: '0'
|
145
145
|
requirements: []
|
146
146
|
rubyforge_project:
|
147
|
-
rubygems_version: 1.8.
|
147
|
+
rubygems_version: 1.8.23
|
148
148
|
signing_key:
|
149
149
|
specification_version: 3
|
150
150
|
summary: Provides utilities for dynamic hadoop client environment configuration
|