albino 1.1.1 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/albino.gemspec CHANGED
@@ -13,7 +13,7 @@ Gem::Specification.new do |s|
13
13
  ## If your rubyforge_project name is different, then edit it and comment out
14
14
  ## the sub! line in the Rakefile
15
15
  s.name = 'albino'
16
- s.version = '1.1.1'
16
+ s.version = '1.2.0'
17
17
  s.date = '2011-01-11'
18
18
  s.rubyforge_project = 'albino'
19
19
 
@@ -45,7 +45,9 @@ Gem::Specification.new do |s|
45
45
  Rakefile
46
46
  albino.gemspec
47
47
  lib/albino.rb
48
+ lib/albino/process.rb
48
49
  test/albino_test.rb
50
+ test/process_test.rb
49
51
  ]
50
52
  # = MANIFEST =
51
53
 
@@ -0,0 +1,294 @@
1
+ class Albino
2
+ # Albino::Process includes logic for executing child processes and
3
+ # reading/writing from their standard input, output, and error streams.
4
+ #
5
+ # Create an run a process to completion:
6
+ #
7
+ # >> process = Albino::Process.new(['git', '--help'])
8
+ #
9
+ # Retrieve stdout or stderr output:
10
+ #
11
+ # >> process.out
12
+ # => "usage: git [--version] [--exec-path[=GIT_EXEC_PATH]]\n ..."
13
+ # >> process.err
14
+ # => ""
15
+ #
16
+ # Check process exit status information:
17
+ #
18
+ # >> process.status
19
+ # => #<Process::Status: pid=80718,exited(0)>
20
+ #
21
+ # Albino::Process is designed to take all input in a single string and
22
+ # provides all output as single strings. It is therefore not well suited
23
+ # to streaming large quantities of data in and out of commands.
24
+ #
25
+ # Q: Why not use popen3 or hand-roll fork/exec code?
26
+ #
27
+ # - It's more efficient than popen3 and provides meaningful process
28
+ # hierarchies because it performs a single fork/exec. (popen3 double forks
29
+ # to avoid needing to collect the exit status and also calls
30
+ # Process::detach which creates a Ruby Thread!!!!).
31
+ #
32
+ # - It's more portable than hand rolled pipe, fork, exec code because
33
+ # fork(2) and exec(2) aren't available on all platforms. In those cases,
34
+ # Albino::Process falls back to using whatever janky substitutes the platform
35
+ # provides.
36
+ #
37
+ # - It handles all max pipe buffer hang cases, which is non trivial to
38
+ # implement correctly and must be accounted for with either popen3 or
39
+ # hand rolled fork/exec code.
40
+ class Process
41
+ # Create and execute a new process.
42
+ #
43
+ # argv - Array of [command, arg1, ...] strings to use as the new
44
+ # process's argv. When argv is a String, the shell is used
45
+ # to interpret the command.
46
+ # env - The new process's environment variables. This is merged with
47
+ # the current environment as if by ENV.merge(env).
48
+ # options - Additional options:
49
+ # :input => str to write str to the process's stdin.
50
+ # :timeout => int number of seconds before we given up.
51
+ # :max => total number of output bytes
52
+ # A subset of Process:spawn options are also supported on all
53
+ # platforms:
54
+ # :chdir => str to start the process in different working dir.
55
+ #
56
+ # Returns a new Process instance that has already executed to completion.
57
+ # The out, err, and status attributes are immediately available.
58
+ def initialize(argv, env={}, options={})
59
+ @argv = argv
60
+ @env = env
61
+
62
+ @options = options.dup
63
+ @input = @options.delete(:input)
64
+ @timeout = @options.delete(:timeout)
65
+ @max = @options.delete(:max)
66
+ @options.delete(:chdir) if @options[:chdir].nil?
67
+
68
+ exec!
69
+ end
70
+
71
+ # All data written to the child process's stdout stream as a String.
72
+ attr_reader :out
73
+
74
+ # All data written to the child process's stderr stream as a String.
75
+ attr_reader :err
76
+
77
+ # A Process::Status object with information on how the child exited.
78
+ attr_reader :status
79
+
80
+ # Total command execution time (wall-clock time)
81
+ attr_reader :runtime
82
+
83
+ # Determine if the process did exit with a zero exit status.
84
+ def success?
85
+ @status && @status.success?
86
+ end
87
+
88
+ private
89
+ # Execute command, write input, and read output. This is called
90
+ # immediately when a new instance of this object is initialized.
91
+ def exec!
92
+ # when argv is a string, use /bin/sh to interpret command
93
+ argv = @argv
94
+ argv = ['/bin/sh', '-c', argv.to_str] if argv.respond_to?(:to_str)
95
+
96
+ # spawn the process and hook up the pipes
97
+ pid, stdin, stdout, stderr = popen4(@env, *(argv + [@options]))
98
+
99
+ # async read from all streams into buffers
100
+ @out, @err = read_and_write(@input, stdin, stdout, stderr, @timeout, @max)
101
+
102
+ # grab exit status
103
+ @status = waitpid(pid)
104
+ rescue Object => boom
105
+ [stdin, stdout, stderr].each { |fd| fd.close rescue nil }
106
+ if @status.nil?
107
+ ::Process.kill('TERM', pid) rescue nil
108
+ @status = waitpid(pid) rescue nil
109
+ end
110
+ raise
111
+ end
112
+
113
+ # Exception raised when the total number of bytes output on the command's
114
+ # stderr and stdout streams exceeds the maximum output size (:max option).
115
+ class MaximumOutputExceeded < StandardError
116
+ end
117
+
118
+ # Exception raised when timeout is exceeded.
119
+ class TimeoutExceeded < StandardError
120
+ end
121
+
122
+ # Maximum buffer size for reading
123
+ BUFSIZE = (32 * 1024)
124
+
125
+ # Start a select loop writing any input on the child's stdin and reading
126
+ # any output from the child's stdout or stderr.
127
+ #
128
+ # input - String input to write on stdin. May be nil.
129
+ # stdin - The write side IO object for the child's stdin stream.
130
+ # stdout - The read side IO object for the child's stdout stream.
131
+ # stderr - The read side IO object for the child's stderr stream.
132
+ # timeout - An optional Numeric specifying the total number of seconds
133
+ # the read/write operations should occur for.
134
+ #
135
+ # Returns an [out, err] tuple where both elements are strings with all
136
+ # data written to the stdout and stderr streams, respectively.
137
+ # Raises TimeoutExceeded when all data has not been read / written within
138
+ # the duration specified in the timeout argument.
139
+ # Raises MaximumOutputExceeded when the total number of bytes output
140
+ # exceeds the amount specified by the max argument.
141
+ def read_and_write(input, stdin, stdout, stderr, timeout=nil, max=nil)
142
+ input ||= ''
143
+ max = nil if max && max <= 0
144
+ out, err = '', ''
145
+ offset = 0
146
+
147
+ timeout = nil if timeout && timeout <= 0.0
148
+ @runtime = 0.0
149
+ start = Time.now
150
+
151
+ writers = [stdin]
152
+ readers = [stdout, stderr]
153
+ t = timeout
154
+ while readers.any? || writers.any?
155
+ ready = IO.select(readers, writers, readers + writers, t)
156
+ raise TimeoutExceeded if ready.nil?
157
+
158
+ # write to stdin stream
159
+ ready[1].each do |fd|
160
+ begin
161
+ boom = nil
162
+ size = fd.write_nonblock(input)
163
+ input = input[size, input.size]
164
+ rescue Errno::EPIPE => boom
165
+ rescue Errno::EAGAIN, Errno::EINTR
166
+ end
167
+ if boom || input.size == 0
168
+ stdin.close
169
+ writers.delete(stdin)
170
+ end
171
+ end
172
+
173
+ # read from stdout and stderr streams
174
+ ready[0].each do |fd|
175
+ buf = (fd == stdout) ? out : err
176
+ begin
177
+ buf << fd.readpartial(BUFSIZE)
178
+ rescue Errno::EAGAIN, Errno::EINTR
179
+ rescue EOFError
180
+ readers.delete(fd)
181
+ fd.close
182
+ end
183
+ end
184
+
185
+ # keep tabs on the total amount of time we've spent here
186
+ @runtime = Time.now - start
187
+ if timeout
188
+ t = timeout - @runtime
189
+ raise TimeoutExceeded if t < 0.0
190
+ end
191
+
192
+ # maybe we've hit our max output
193
+ if max && ready[0].any? && (out.size + err.size) > max
194
+ raise MaximumOutputExceeded
195
+ end
196
+ end
197
+
198
+ [out, err]
199
+ end
200
+
201
+ # Spawn a child process, perform IO redirection and environment prep, and
202
+ # return the running process's pid.
203
+ #
204
+ # This method implements a limited subset of Ruby 1.9's Process::spawn.
205
+ # The idea is that we can just use that when available, since most platforms
206
+ # will eventually build in special (and hopefully good) support for it.
207
+ #
208
+ # env - Hash of { name => val } environment variables set in the child
209
+ # process.
210
+ # argv - New process's argv as an Array. When this value is a string,
211
+ # the command may be run through the system /bin/sh or
212
+ # options - Supports a subset of Process::spawn options, including:
213
+ # :chdir => str to change the directory to str in the child
214
+ # FD => :close to close a file descriptor in the child
215
+ # :in => FD to redirect child's stdin to FD
216
+ # :out => FD to redirect child's stdout to FD
217
+ # :err => FD to redirect child's stderr to FD
218
+ #
219
+ # Returns the pid of the new process as an integer. The process exit status
220
+ # must be obtained using Process::waitpid.
221
+ def spawn(env, *argv)
222
+ options = (argv.pop if argv[-1].kind_of?(Hash)) || {}
223
+ fork do
224
+ # { fd => :close } in options means close that fd
225
+ options.each { |k,v| k.close if v == :close && !k.closed? }
226
+
227
+ # reopen stdin, stdout, and stderr on provided fds
228
+ STDIN.reopen(options[:in])
229
+ STDOUT.reopen(options[:out])
230
+ STDERR.reopen(options[:err])
231
+
232
+ # setup child environment
233
+ env.each { |k, v| ENV[k] = v }
234
+
235
+ # { :chdir => '/' } in options means change into that dir
236
+ ::Dir.chdir(options[:chdir]) if options[:chdir]
237
+
238
+ # do the deed
239
+ ::Kernel::exec(*argv)
240
+ exit! 1
241
+ end
242
+ end
243
+
244
+ # Start a process with spawn options and return
245
+ # popen4([env], command, arg1, arg2, [opt])
246
+ #
247
+ # env - The child process's environment as a Hash.
248
+ # command - The command and zero or more arguments.
249
+ # options - An options hash.
250
+ #
251
+ # See Ruby 1.9 IO.popen and Process::spawn docs for more info:
252
+ # http://www.ruby-doc.org/core-1.9/classes/IO.html#M001640
253
+ #
254
+ # Returns a [pid, stdin, stderr, stdout] tuple where pid is the child
255
+ # process's pid, stdin is a writeable IO object, and stdout + stderr are
256
+ # readable IO objects.
257
+ def popen4(*argv)
258
+ # create some pipes (see pipe(2) manual -- the ruby docs suck)
259
+ ird, iwr = IO.pipe
260
+ ord, owr = IO.pipe
261
+ erd, ewr = IO.pipe
262
+
263
+ # spawn the child process with either end of pipes hooked together
264
+ opts =
265
+ ((argv.pop if argv[-1].is_a?(Hash)) || {}).merge(
266
+ # redirect fds # close other sides
267
+ :in => ird, iwr => :close,
268
+ :out => owr, ord => :close,
269
+ :err => ewr, erd => :close
270
+ )
271
+ pid = spawn(*(argv + [opts]))
272
+
273
+ [pid, iwr, ord, erd]
274
+ ensure
275
+ # we're in the parent, close child-side fds
276
+ [ird, owr, ewr].each { |fd| fd.close }
277
+ end
278
+
279
+ # Wait for the child process to exit
280
+ #
281
+ # Returns the Process::Status object obtained by reaping the process.
282
+ def waitpid(pid)
283
+ ::Process::waitpid(pid)
284
+ $?
285
+ end
286
+
287
+ # Use native Process::spawn implementation on Ruby 1.9.
288
+ if ::Process.respond_to?(:spawn)
289
+ def spawn(*argv)
290
+ ::Process.spawn(*argv)
291
+ end
292
+ end
293
+ end
294
+ end
data/lib/albino.rb CHANGED
@@ -1,3 +1,4 @@
1
+ require 'albino/process'
1
2
  ##
2
3
  # Wrapper for the Pygments command line tool, pygmentize.
3
4
  #
@@ -38,11 +39,13 @@
38
39
  #
39
40
  # To see all lexers and formatters available, run `pygmentize -L`.
40
41
  #
41
- # Chris Wanstrath // chris@ozmm.org
42
+ # Chris Wanstrath // chris@ozmm.org
42
43
  # GitHub // http://github.com
43
44
  #
44
45
  class Albino
45
- VERSION = '1.1.1'
46
+ class ShellArgumentError < ArgumentError; end
47
+
48
+ VERSION = '1.2.0'
46
49
 
47
50
  class << self
48
51
  attr_accessor :bin
@@ -58,38 +61,43 @@ class Albino
58
61
  @options = { :l => lexer, :f => format }
59
62
  end
60
63
 
61
- def execute(command)
62
- output = ''
63
- IO.popen(command, mode='r+') do |p|
64
- write_target_to_stream(p)
65
- p.close_write
66
- output = p.read.strip
67
- end
68
- output
64
+ def execute(options = {})
65
+ proc_options = {}
66
+ proc_options[:timeout] = options.delete(:timeout) || 5
67
+ command = convert_options(options)
68
+ command.unshift(bin)
69
+ Process.new(command, env={}, proc_options.merge(:input => write_target))
69
70
  end
70
71
 
71
72
  def colorize(options = {})
72
- execute bin + convert_options(options)
73
+ execute(options).out
73
74
  end
74
75
  alias_method :to_s, :colorize
75
76
 
76
77
  def convert_options(options = {})
77
- @options.merge(options).inject('') do |string, (flag, value)|
78
- string + " -#{flag} #{shell_escape value}"
78
+ @options.merge(options).inject([]) do |memo, (flag, value)|
79
+ validate_shell_args(flag.to_s, value.to_s)
80
+ memo << "-#{flag}" << value.to_s
79
81
  end
80
82
  end
81
83
 
82
- def write_target_to_stream(stream)
84
+ def write_target
83
85
  if @target.respond_to?(:read)
84
- @target.each { |l| stream << l }
86
+ out = @target.read
85
87
  @target.close
88
+ out
86
89
  else
87
- stream << @target
90
+ @target.to_s
88
91
  end
89
92
  end
90
93
 
91
- def shell_escape(str)
92
- str.to_s.gsub("'", "\\\\'").gsub(";", '\\;')
94
+ def validate_shell_args(flag, value)
95
+ if flag !~ /^[a-z]+$/i
96
+ raise ShellArgumentError, "Flag is invalid: #{flag.inspect}"
97
+ end
98
+ if value !~ /^[a-z\-\_\+\#\,\s]+$/i
99
+ raise ShellArgumentError, "Flag value is invalid: -#{flag} #{value.inspect}"
100
+ end
93
101
  end
94
102
 
95
103
  def bin
data/test/albino_test.rb CHANGED
@@ -10,14 +10,17 @@ class AlbinoTest < Test::Unit::TestCase
10
10
  end
11
11
 
12
12
  def test_defaults_to_text
13
- syntaxer = Albino.new(File.new(__FILE__))
14
- syntaxer.expects(:execute).with('pygmentize -f html -l text').returns(true)
15
- syntaxer.colorize
13
+ syntaxer = Albino.new('abc')
14
+ regex = /span/
15
+ assert_no_match regex, syntaxer.colorize
16
16
  end
17
17
 
18
18
  def test_accepts_options
19
- @syntaxer.expects(:execute).with('pygmentize -f html -l ruby').returns(true)
20
- @syntaxer.colorize
19
+ assert_match /span/, @syntaxer.colorize
20
+ end
21
+
22
+ def test_accepts_non_alpha_options
23
+ assert_equal '', @syntaxer.colorize(:f => 'html+c#-dump')
21
24
  end
22
25
 
23
26
  def test_works_with_strings
@@ -48,6 +51,8 @@ class AlbinoTest < Test::Unit::TestCase
48
51
  end
49
52
 
50
53
  def test_escaped_shell_args
51
- assert_equal " -f html -l \\'abc\\;\\'", @syntaxer.convert_options(:l => "'abc;'")
54
+ assert_raises Albino::ShellArgumentError do
55
+ @syntaxer.convert_options(:l => "'abc;'")
56
+ end
52
57
  end
53
58
  end
@@ -0,0 +1,102 @@
1
+ require 'albino'
2
+ require 'rubygems'
3
+ require 'test/unit'
4
+
5
+ class TestProcess < Test::Unit::TestCase
6
+ def test_argv_array_execs
7
+ p = Albino::Process.new(['printf', '%s %s %s', '1', '2', '3 4'])
8
+ assert p.success?
9
+ assert_equal "1 2 3 4", p.out
10
+ end
11
+
12
+ def test_argv_string_uses_sh
13
+ p = Albino::Process.new("echo via /bin/sh")
14
+ assert p.success?
15
+ assert_equal "via /bin/sh\n", p.out
16
+ end
17
+
18
+ def test_stdout
19
+ p = Albino::Process.new(['echo', 'boom'])
20
+ assert_equal "boom\n", p.out
21
+ assert_equal "", p.err
22
+ end
23
+
24
+ def test_stderr
25
+ p = Albino::Process.new('echo boom 1>&2')
26
+ assert_equal "", p.out
27
+ assert_equal "boom\n", p.err
28
+ end
29
+
30
+ def test_status
31
+ p = Albino::Process.new('exit 3')
32
+ assert !p.status.success?
33
+ assert_equal 3, p.status.exitstatus
34
+ end
35
+
36
+ def test_env
37
+ p = Albino::Process.new('echo $FOO', { 'FOO' => 'BOOYAH' })
38
+ assert_equal "BOOYAH\n", p.out
39
+ end
40
+
41
+ def test_chdir
42
+ p = Albino::Process.new(["pwd"], {}, :chdir => File.dirname(Dir.pwd))
43
+ assert_equal File.dirname(Dir.pwd) + "\n", p.out
44
+ end
45
+
46
+ def test_input
47
+ input = "HEY NOW\n" * 100_000 # 800K
48
+ p = Albino::Process.new(['wc', '-l'], {}, :input => input)
49
+ assert_equal 100_000, p.out.strip.to_i
50
+ end
51
+
52
+ def test_max
53
+ assert_raise Albino::Process::MaximumOutputExceeded do
54
+ Albino::Process.new(['yes'], {}, :max => 100_000)
55
+ end
56
+ end
57
+
58
+ def test_max_with_child_hierarchy
59
+ assert_raise Albino::Process::MaximumOutputExceeded do
60
+ Albino::Process.new(['/bin/sh', '-c', 'yes'], {}, :max => 100_000)
61
+ end
62
+ end
63
+
64
+ def test_max_with_stubborn_child
65
+ assert_raise Albino::Process::MaximumOutputExceeded do
66
+ Albino::Process.new("trap '' TERM; yes", {}, :max => 100_000)
67
+ end
68
+ end
69
+
70
+ def test_timeout
71
+ assert_raise Albino::Process::TimeoutExceeded do
72
+ Albino::Process.new(['sleep 1'], {}, :timeout => 0.05)
73
+ end
74
+ end
75
+
76
+ def test_timeout_with_child_hierarchy
77
+ assert_raise Albino::Process::TimeoutExceeded do
78
+ Albino::Process.new(['/bin/sh', '-c', 'yes'], {}, :timeout => 0.05)
79
+ end
80
+ end
81
+
82
+ def test_lots_of_input_and_lots_of_output_at_the_same_time
83
+ input = "stuff on stdin \n" * 1_000
84
+ command = "
85
+ while read line
86
+ do
87
+ echo stuff on stdout;
88
+ echo stuff on stderr 1>&2;
89
+ done
90
+ "
91
+ p = Albino::Process.new(['/bin/sh', '-c', command], {}, :input => input)
92
+ assert_equal input.size, p.out.size
93
+ assert_equal input.size, p.err.size
94
+ assert p.success?
95
+ end
96
+
97
+ def test_input_cannot_be_written_due_to_broken_pipe
98
+ input = "1" * 100_000
99
+ p = Albino::Process.new(['false'], {}, :input => input)
100
+ assert !p.success?
101
+ end
102
+ end
metadata CHANGED
@@ -1,12 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: albino
3
3
  version: !ruby/object:Gem::Version
4
+ hash: 31
4
5
  prerelease: false
5
6
  segments:
6
7
  - 1
7
- - 1
8
- - 1
9
- version: 1.1.1
8
+ - 2
9
+ - 0
10
+ version: 1.2.0
10
11
  platform: ruby
11
12
  authors:
12
13
  - Chris Wanstrath
@@ -21,9 +22,11 @@ dependencies:
21
22
  name: mocha
22
23
  prerelease: false
23
24
  requirement: &id001 !ruby/object:Gem::Requirement
25
+ none: false
24
26
  requirements:
25
27
  - - ">="
26
28
  - !ruby/object:Gem::Version
29
+ hash: 3
27
30
  segments:
28
31
  - 0
29
32
  version: "0"
@@ -43,7 +46,9 @@ files:
43
46
  - Rakefile
44
47
  - albino.gemspec
45
48
  - lib/albino.rb
49
+ - lib/albino/process.rb
46
50
  - test/albino_test.rb
51
+ - test/process_test.rb
47
52
  has_rdoc: true
48
53
  homepage: http://github.com/github/albino
49
54
  licenses: []
@@ -54,23 +59,27 @@ rdoc_options: []
54
59
  require_paths:
55
60
  - lib
56
61
  required_ruby_version: !ruby/object:Gem::Requirement
62
+ none: false
57
63
  requirements:
58
64
  - - ">="
59
65
  - !ruby/object:Gem::Version
66
+ hash: 3
60
67
  segments:
61
68
  - 0
62
69
  version: "0"
63
70
  required_rubygems_version: !ruby/object:Gem::Requirement
71
+ none: false
64
72
  requirements:
65
73
  - - ">="
66
74
  - !ruby/object:Gem::Version
75
+ hash: 3
67
76
  segments:
68
77
  - 0
69
78
  version: "0"
70
79
  requirements: []
71
80
 
72
81
  rubyforge_project: albino
73
- rubygems_version: 1.3.6
82
+ rubygems_version: 1.3.7
74
83
  signing_key:
75
84
  specification_version: 2
76
85
  summary: Ruby wrapper for pygmentize.