pygments.rb 2.0.0.rc3 → 2.0.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: cb7e6901e07778261de7c0a6283696d5c1593ade93565caf32b2870e11477d39
4
- data.tar.gz: 36d123e7aa363b8f50dfcd884e4d1f0721e6e75c83966c18d83f505e7bbf0b85
3
+ metadata.gz: 82a1b6e9549afec22e94b2690983b55a5a3029e986b3d7201c2c6558a2968bc5
4
+ data.tar.gz: 63ac418bfb998653fc4472c1cfd3c8268a006b002754b1759bc11132889f3e2b
5
5
  SHA512:
6
- metadata.gz: 6ad9b6ac8dce86c289939ffbe3ee8c71268dd04ed3755798f08c75905cac90bf602236ae10448a44162809483ca5f88252985db7537d01818c5dcd348d60fe78
7
- data.tar.gz: 728ca37e69f29ee9c74d2804e7583fdaf2284f330b9f3096612a88c2cc58833f49ad9a642eadd63e883a36cb7ea171b7f7892a205da709be67937e698a1d49f2
6
+ metadata.gz: a54ab6bc824be2d4fcdd0c61e52d0838b55ac7cc5ed023f8392ff784fa79d3f1fc32b159c422b46cc91646a8d07853ce2d934fd03e8c06de8903901f45a589c9
7
+ data.tar.gz: e3910502b53d5c3f6ad729bc558e1baffdbc8af8958fa8867e7489df10bbfd1d1e832f0791a90592418b6bf1c19ba3aa0289b7f043f81aabb071fc036fec29ad
@@ -5,6 +5,11 @@
5
5
  This document provides a high-level view of the changes to the {project-name} by release.
6
6
  For a detailed view of what has changed, refer to the {uri-repo}/commits/master[commit history] on GitHub.
7
7
 
8
+ == 2.0.0 (2021-01-08) - @slonopotamus
9
+
10
+ * stop sending/receiving `ids` between Ruby and Python
11
+ * use `close_others` Ruby mechanism to prevent file descriptor leaking to Python
12
+
8
13
  == 2.0.0.rc3 (2021-01-08) - @slonopotamus
9
14
 
10
15
  * fix watchdog race condition leading to `ThreadError(<killed thread>)` on JRuby ({uri-repo}/pull/215[#215])
@@ -1,7 +1,6 @@
1
1
  = {project-name}
2
2
  Ted Nyman <ted@ted.io>; Aman Gupta <aman@tmm1.net>; Marat Radchenko <marat@slonopotamus.org>
3
3
  :project-name: pygments.rb
4
- :project-handle: pygments.rb
5
4
  :slug: pygments/{project-name}
6
5
  :toc: preamble
7
6
  :uri-project: https://github.com/{slug}
@@ -14,7 +13,7 @@ image:{uri-project}/workflows/CI/badge.svg?branch=master[Build Status,link={uri-
14
13
 
15
14
  == Introduction
16
15
 
17
- {project-name} is a Ruby wrapper for the Python {uri-pygments}[Pygments] syntax highlighter.
16
+ {project-name} is a Ruby wrapper for {uri-pygments}[Pygments] syntax highlighter.
18
17
 
19
18
  {project-name} works by talking over a simple pipe to a long-lived Python child process.
20
19
  This library replaces https://github.com/github/albino[github/albino], as well as an older version of {project-name} that used an embedded Python interpreter.
@@ -196,27 +196,6 @@ class Mentos(object):
196
196
  sys.stdout.buffer.write(res_bytes)
197
197
  sys.stdout.flush()
198
198
 
199
-
200
- def _get_ids(self, text):
201
- start_id = text[:8]
202
- end_id = text[-8:]
203
- return start_id, end_id
204
-
205
- def _check_and_return_text(self, text, start_id, end_id):
206
-
207
- # Sanity check.
208
- id_regex = re.compile('[A-Z]{8}')
209
-
210
- if not id_regex.match(start_id) and not id_regex.match(end_id):
211
- _write_error("ID check failed. Not an ID.")
212
-
213
- if not start_id == end_id:
214
- _write_error("ID check failed. ID's did not match.")
215
-
216
- # Passed the sanity check. Remove the id's and return
217
- text = text[10:-10]
218
- return text
219
-
220
199
  def _parse_header(self, header):
221
200
  method = header["method"]
222
201
  args = header.get("args", [])
@@ -234,7 +213,7 @@ class Mentos(object):
234
213
  pygmentized, this header will be followed by the text to be pygmentized.
235
214
 
236
215
  The header is of form:
237
- { "method": "highlight", "args": [], "kwargs": {"arg1": "v"}, "bytes": 128, "fd": "8"}
216
+ { "method": "highlight", "args": [], "kwargs": {"arg1": "v"}, "bytes": 128}
238
217
  """
239
218
 
240
219
  while True:
@@ -261,18 +240,9 @@ class Mentos(object):
261
240
  # Read up to the given number of *bytes* (not chars) (possibly 0)
262
241
  text = sys.stdin.buffer.read(_bytes).decode('utf-8')
263
242
 
264
- # Sanity check the return.
265
- if _bytes:
266
- start_id, end_id = self._get_ids(text)
267
- text = self._check_and_return_text(text, start_id, end_id)
268
-
269
243
  # Get the actual data from pygments.
270
244
  res = self.get_data(method, lexer, args, kwargs, text)
271
245
 
272
- # Put back the sanity check values.
273
- if method == "highlight":
274
- res = start_id + " " + res + " " + end_id
275
-
276
246
  self._send_data(res, method)
277
247
 
278
248
  except:
@@ -288,20 +258,6 @@ def main():
288
258
  signal.signal(signal.SIGHUP, _signal_handler)
289
259
 
290
260
  mentos = Mentos()
291
-
292
- if sys.platform != "win32":
293
- # close fd's inherited from the ruby parent
294
- import resource
295
- maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
296
- if maxfd == resource.RLIM_INFINITY:
297
- maxfd = 65536
298
-
299
- for fd in range(3, maxfd):
300
- try:
301
- os.close(fd)
302
- except:
303
- pass
304
-
305
261
  mentos.start()
306
262
 
307
263
  if __name__ == "__main__":
@@ -14,7 +14,7 @@ end
14
14
  module Pygments
15
15
  class Popen
16
16
  def popen4(argv)
17
- stdin, stdout, stderr, wait_thr = Open3.popen3(*argv)
17
+ stdin, stdout, stderr, wait_thr = Open3.popen3(*argv, { close_others: true })
18
18
  while (pid = wait_thr.pid).nil? && wait_thr.alive?
19
19
  # For unknown reasons, wait_thr.pid is not immediately available on JRuby
20
20
  end
@@ -42,7 +42,7 @@ module Pygments
42
42
  @pid, @in, @out, @err = popen4(argv)
43
43
  @in.binmode
44
44
  @out.binmode
45
- @log.info "Starting pid #{@pid} with fd #{@out.to_i} and python #{python_binary}."
45
+ @log.info "Starting pid #{@pid} with python #{python_binary}."
46
46
  end
47
47
 
48
48
  def python_binary
@@ -237,7 +237,6 @@ module Pygments
237
237
  stop error_message
238
238
  state = :timeout
239
239
  end
240
-
241
240
  end
242
241
  end : nil
243
242
  begin
@@ -258,7 +257,7 @@ module Pygments
258
257
 
259
258
  # Our 'rpc'-ish request to mentos. Requires a method name, and then optional
260
259
  # args, kwargs, code.
261
- def mentos(method, args = [], kwargs = {}, original_code = nil)
260
+ def mentos(method, args = [], kwargs = {}, code = nil)
262
261
  # Open the pipe if necessary
263
262
  start unless alive?
264
263
 
@@ -273,12 +272,6 @@ module Pygments
273
272
  end
274
273
  end
275
274
 
276
- # For sanity checking on both sides of the pipe when highlighting, we prepend and
277
- # append an id. mentos checks that these are 8 character ids and that they match.
278
- # It then returns the id's back to Rubyland.
279
- id = (0...8).map { rand(65..89).chr }.join
280
- code = original_code ? add_ids(original_code, id) : nil
281
-
282
275
  # Add metadata to the header and generate it.
283
276
  bytesize = if code
284
277
  code.bytesize
@@ -287,7 +280,7 @@ module Pygments
287
280
  end
288
281
 
289
282
  kwargs.freeze
290
- kwargs = kwargs.merge('fd' => @out.to_i, 'id' => id, 'bytes' => bytesize)
283
+ kwargs = kwargs.merge('bytes' => bytesize)
291
284
  out_header = JSON.generate(method: method, args: args, kwargs: kwargs)
292
285
 
293
286
  begin
@@ -315,7 +308,7 @@ module Pygments
315
308
  header = @out.read(header_len)
316
309
 
317
310
  # Now handle the header, any read any more data required.
318
- handle_header_and_return(header, id)
311
+ handle_header_and_return(header)
319
312
  end
320
313
 
321
314
  # Finally, return what we got.
@@ -341,10 +334,8 @@ module Pygments
341
334
  # Based on the header we receive, determine if we need
342
335
  # to read more bytes, and read those bytes if necessary.
343
336
  #
344
- # Then, do a sanity check with the ids.
345
- #
346
337
  # Returns a result - either highlighted text or metadata.
347
- def handle_header_and_return(header, id)
338
+ def handle_header_and_return(header)
348
339
  if header
349
340
  @log.info "In header: #{header}"
350
341
  header = header_to_json(header)
@@ -356,37 +347,14 @@ module Pygments
356
347
  if header[:method] == 'highlight'
357
348
  # Make sure we have a result back; else consider this an error.
358
349
  raise MentosError, 'No highlight result back from mentos.' if res.nil?
359
-
360
- @log.info 'Highlight in process.'
361
-
362
- # Get the id's
363
- start_id = res[0..7]
364
- end_id = res[-8..-1]
365
-
366
- # Sanity check.
367
- if !((start_id == id) && (end_id == id))
368
- raise MentosError, "ID's did not match. Aborting."
369
- else
370
- # We're good. Remove the padding
371
- res = res[10..-11]
372
- @log.info 'Highlighting complete.'
373
- res
374
- end
375
350
  end
351
+
376
352
  res
377
353
  else
378
354
  raise MentosError, 'No header received back.'
379
355
  end
380
356
  end
381
357
 
382
- # With the code, prepend the id (with two spaces to avoid escaping weirdness if
383
- # the following text starts with a slash (like terminal code), and append the
384
- # id, with two padding also. This means we are sending over the 8 characters +
385
- # code + 8 characters.
386
- def add_ids(code, id)
387
- (id + " #{code} #{id}").freeze
388
- end
389
-
390
358
  # Return the final result for the API. Return Ruby objects for the methods that
391
359
  # want them, text otherwise.
392
360
  def return_result(res, method)
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Pygments
4
- VERSION = '2.0.0.rc3'
4
+ VERSION = '2.0.0'
5
5
  end
@@ -7,7 +7,7 @@ Gem::Specification.new do |s|
7
7
  s.version = Pygments::VERSION
8
8
 
9
9
  s.summary = 'pygments wrapper for ruby'
10
- s.description = 'pygments.rb exposes the pygments syntax highlighter to Ruby'
10
+ s.description = 'pygments.rb is a Ruby wrapper for Pygments syntax highlighter'
11
11
 
12
12
  s.homepage = 'https://github.com/pygments/pygments.rb'
13
13
  s.required_ruby_version = '>= 2.3.0'
@@ -16,7 +16,7 @@ Gem::Specification.new do |s|
16
16
  s.email = ['marat@slonopotamus.org']
17
17
  s.license = 'MIT'
18
18
 
19
- s.add_development_dependency 'rake-compiler', '~> 1.1.0'
19
+ s.add_development_dependency 'rake', '~> 13.0.0'
20
20
  s.add_development_dependency 'rubocop', '~> 0.81.0'
21
21
  s.add_development_dependency 'test-unit', '~> 3.3.0'
22
22
 
@@ -119,31 +119,6 @@ class PygmentsHighlightTest < Test::Unit::TestCase
119
119
  end
120
120
  end
121
121
 
122
- # Philosophically, I'm not the biggest fan of testing private
123
- # methods, but given the relative delicacy of validity checking
124
- # over the pipe I think it's necessary and informative.
125
- class PygmentsValidityTest < Test::Unit::TestCase
126
- def test_add_ids_with_padding
127
- res = PE.send(:add_ids, 'herp derp baz boo foo', 'ABCDEFGH')
128
- assert_equal 'ABCDEFGH herp derp baz boo foo ABCDEFGH', res
129
- end
130
-
131
- def test_add_ids_on_empty_string
132
- res = PE.send(:add_ids, '', 'ABCDEFGH')
133
- assert_equal 'ABCDEFGH ABCDEFGH', res
134
- end
135
-
136
- def test_add_ids_with_unicode_data
137
- res = PE.send(:add_ids, '# ø ø ø', 'ABCDEFGH')
138
- assert_equal 'ABCDEFGH # ø ø ø ABCDEFGH', res
139
- end
140
-
141
- def test_add_ids_with_starting_slashes
142
- res = PE.send(:add_ids, '\\# ø ø ø..//', 'ABCDEFGH')
143
- assert_equal 'ABCDEFGH \\# ø ø ø..// ABCDEFGH', res
144
- end
145
- end
146
-
147
122
  class PygmentsLexerTest < Test::Unit::TestCase
148
123
  RUBY_CODE = "#!/usr/bin/ruby\nputs 'foo'"
149
124
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pygments.rb
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.0.rc3
4
+ version: 2.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Aman Gupta
@@ -13,19 +13,19 @@ cert_chain: []
13
13
  date: 2021-01-07 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
- name: rake-compiler
16
+ name: rake
17
17
  requirement: !ruby/object:Gem::Requirement
18
18
  requirements:
19
19
  - - "~>"
20
20
  - !ruby/object:Gem::Version
21
- version: 1.1.0
21
+ version: 13.0.0
22
22
  type: :development
23
23
  prerelease: false
24
24
  version_requirements: !ruby/object:Gem::Requirement
25
25
  requirements:
26
26
  - - "~>"
27
27
  - !ruby/object:Gem::Version
28
- version: 1.1.0
28
+ version: 13.0.0
29
29
  - !ruby/object:Gem::Dependency
30
30
  name: rubocop
31
31
  requirement: !ruby/object:Gem::Requirement
@@ -54,7 +54,7 @@ dependencies:
54
54
  - - "~>"
55
55
  - !ruby/object:Gem::Version
56
56
  version: 3.3.0
57
- description: pygments.rb exposes the pygments syntax highlighter to Ruby
57
+ description: pygments.rb is a Ruby wrapper for Pygments syntax highlighter
58
58
  email:
59
59
  - marat@slonopotamus.org
60
60
  executables: []
@@ -341,9 +341,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
341
341
  version: 2.3.0
342
342
  required_rubygems_version: !ruby/object:Gem::Requirement
343
343
  requirements:
344
- - - ">"
344
+ - - ">="
345
345
  - !ruby/object:Gem::Version
346
- version: 1.3.1
346
+ version: '0'
347
347
  requirements: []
348
348
  rubygems_version: 3.2.3
349
349
  signing_key: