pygments.rb 2.0.0.rc3 → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.adoc +5 -0
- data/README.adoc +1 -2
- data/lib/pygments/mentos.py +1 -45
- data/lib/pygments/popen.rb +7 -39
- data/lib/pygments/version.rb +1 -1
- data/pygments.rb.gemspec +2 -2
- data/test/test_pygments.rb +0 -25
- metadata +7 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 82a1b6e9549afec22e94b2690983b55a5a3029e986b3d7201c2c6558a2968bc5
|
4
|
+
data.tar.gz: 63ac418bfb998653fc4472c1cfd3c8268a006b002754b1759bc11132889f3e2b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a54ab6bc824be2d4fcdd0c61e52d0838b55ac7cc5ed023f8392ff784fa79d3f1fc32b159c422b46cc91646a8d07853ce2d934fd03e8c06de8903901f45a589c9
|
7
|
+
data.tar.gz: e3910502b53d5c3f6ad729bc558e1baffdbc8af8958fa8867e7489df10bbfd1d1e832f0791a90592418b6bf1c19ba3aa0289b7f043f81aabb071fc036fec29ad
|
data/CHANGELOG.adoc
CHANGED
@@ -5,6 +5,11 @@
|
|
5
5
|
This document provides a high-level view of the changes to the {project-name} by release.
|
6
6
|
For a detailed view of what has changed, refer to the {uri-repo}/commits/master[commit history] on GitHub.
|
7
7
|
|
8
|
+
== 2.0.0 (2021-01-08) - @slonopotamus
|
9
|
+
|
10
|
+
* stop sending/receiving `ids` between Ruby and Python
|
11
|
+
* use `close_others` Ruby mechanism to prevent file descriptor leaking to Python
|
12
|
+
|
8
13
|
== 2.0.0.rc3 (2021-01-08) - @slonopotamus
|
9
14
|
|
10
15
|
* fix watchdog race condition leading to `ThreadError(<killed thread>)` on JRuby ({uri-repo}/pull/215[#215])
|
data/README.adoc
CHANGED
@@ -1,7 +1,6 @@
|
|
1
1
|
= {project-name}
|
2
2
|
Ted Nyman <ted@ted.io>; Aman Gupta <aman@tmm1.net>; Marat Radchenko <marat@slonopotamus.org>
|
3
3
|
:project-name: pygments.rb
|
4
|
-
:project-handle: pygments.rb
|
5
4
|
:slug: pygments/{project-name}
|
6
5
|
:toc: preamble
|
7
6
|
:uri-project: https://github.com/{slug}
|
@@ -14,7 +13,7 @@ image:{uri-project}/workflows/CI/badge.svg?branch=master[Build Status,link={uri-
|
|
14
13
|
|
15
14
|
== Introduction
|
16
15
|
|
17
|
-
{project-name} is a Ruby wrapper for
|
16
|
+
{project-name} is a Ruby wrapper for {uri-pygments}[Pygments] syntax highlighter.
|
18
17
|
|
19
18
|
{project-name} works by talking over a simple pipe to a long-lived Python child process.
|
20
19
|
This library replaces https://github.com/github/albino[github/albino], as well as an older version of {project-name} that used an embedded Python interpreter.
|
data/lib/pygments/mentos.py
CHANGED
@@ -196,27 +196,6 @@ class Mentos(object):
|
|
196
196
|
sys.stdout.buffer.write(res_bytes)
|
197
197
|
sys.stdout.flush()
|
198
198
|
|
199
|
-
|
200
|
-
def _get_ids(self, text):
|
201
|
-
start_id = text[:8]
|
202
|
-
end_id = text[-8:]
|
203
|
-
return start_id, end_id
|
204
|
-
|
205
|
-
def _check_and_return_text(self, text, start_id, end_id):
|
206
|
-
|
207
|
-
# Sanity check.
|
208
|
-
id_regex = re.compile('[A-Z]{8}')
|
209
|
-
|
210
|
-
if not id_regex.match(start_id) and not id_regex.match(end_id):
|
211
|
-
_write_error("ID check failed. Not an ID.")
|
212
|
-
|
213
|
-
if not start_id == end_id:
|
214
|
-
_write_error("ID check failed. ID's did not match.")
|
215
|
-
|
216
|
-
# Passed the sanity check. Remove the id's and return
|
217
|
-
text = text[10:-10]
|
218
|
-
return text
|
219
|
-
|
220
199
|
def _parse_header(self, header):
|
221
200
|
method = header["method"]
|
222
201
|
args = header.get("args", [])
|
@@ -234,7 +213,7 @@ class Mentos(object):
|
|
234
213
|
pygmentized, this header will be followed by the text to be pygmentized.
|
235
214
|
|
236
215
|
The header is of form:
|
237
|
-
{ "method": "highlight", "args": [], "kwargs": {"arg1": "v"}, "bytes": 128
|
216
|
+
{ "method": "highlight", "args": [], "kwargs": {"arg1": "v"}, "bytes": 128}
|
238
217
|
"""
|
239
218
|
|
240
219
|
while True:
|
@@ -261,18 +240,9 @@ class Mentos(object):
|
|
261
240
|
# Read up to the given number of *bytes* (not chars) (possibly 0)
|
262
241
|
text = sys.stdin.buffer.read(_bytes).decode('utf-8')
|
263
242
|
|
264
|
-
# Sanity check the return.
|
265
|
-
if _bytes:
|
266
|
-
start_id, end_id = self._get_ids(text)
|
267
|
-
text = self._check_and_return_text(text, start_id, end_id)
|
268
|
-
|
269
243
|
# Get the actual data from pygments.
|
270
244
|
res = self.get_data(method, lexer, args, kwargs, text)
|
271
245
|
|
272
|
-
# Put back the sanity check values.
|
273
|
-
if method == "highlight":
|
274
|
-
res = start_id + " " + res + " " + end_id
|
275
|
-
|
276
246
|
self._send_data(res, method)
|
277
247
|
|
278
248
|
except:
|
@@ -288,20 +258,6 @@ def main():
|
|
288
258
|
signal.signal(signal.SIGHUP, _signal_handler)
|
289
259
|
|
290
260
|
mentos = Mentos()
|
291
|
-
|
292
|
-
if sys.platform != "win32":
|
293
|
-
# close fd's inherited from the ruby parent
|
294
|
-
import resource
|
295
|
-
maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
|
296
|
-
if maxfd == resource.RLIM_INFINITY:
|
297
|
-
maxfd = 65536
|
298
|
-
|
299
|
-
for fd in range(3, maxfd):
|
300
|
-
try:
|
301
|
-
os.close(fd)
|
302
|
-
except:
|
303
|
-
pass
|
304
|
-
|
305
261
|
mentos.start()
|
306
262
|
|
307
263
|
if __name__ == "__main__":
|
data/lib/pygments/popen.rb
CHANGED
@@ -14,7 +14,7 @@ end
|
|
14
14
|
module Pygments
|
15
15
|
class Popen
|
16
16
|
def popen4(argv)
|
17
|
-
stdin, stdout, stderr, wait_thr = Open3.popen3(*argv)
|
17
|
+
stdin, stdout, stderr, wait_thr = Open3.popen3(*argv, { close_others: true })
|
18
18
|
while (pid = wait_thr.pid).nil? && wait_thr.alive?
|
19
19
|
# For unknown reasons, wait_thr.pid is not immediately available on JRuby
|
20
20
|
end
|
@@ -42,7 +42,7 @@ module Pygments
|
|
42
42
|
@pid, @in, @out, @err = popen4(argv)
|
43
43
|
@in.binmode
|
44
44
|
@out.binmode
|
45
|
-
@log.info "Starting pid #{@pid} with
|
45
|
+
@log.info "Starting pid #{@pid} with python #{python_binary}."
|
46
46
|
end
|
47
47
|
|
48
48
|
def python_binary
|
@@ -237,7 +237,6 @@ module Pygments
|
|
237
237
|
stop error_message
|
238
238
|
state = :timeout
|
239
239
|
end
|
240
|
-
|
241
240
|
end
|
242
241
|
end : nil
|
243
242
|
begin
|
@@ -258,7 +257,7 @@ module Pygments
|
|
258
257
|
|
259
258
|
# Our 'rpc'-ish request to mentos. Requires a method name, and then optional
|
260
259
|
# args, kwargs, code.
|
261
|
-
def mentos(method, args = [], kwargs = {},
|
260
|
+
def mentos(method, args = [], kwargs = {}, code = nil)
|
262
261
|
# Open the pipe if necessary
|
263
262
|
start unless alive?
|
264
263
|
|
@@ -273,12 +272,6 @@ module Pygments
|
|
273
272
|
end
|
274
273
|
end
|
275
274
|
|
276
|
-
# For sanity checking on both sides of the pipe when highlighting, we prepend and
|
277
|
-
# append an id. mentos checks that these are 8 character ids and that they match.
|
278
|
-
# It then returns the id's back to Rubyland.
|
279
|
-
id = (0...8).map { rand(65..89).chr }.join
|
280
|
-
code = original_code ? add_ids(original_code, id) : nil
|
281
|
-
|
282
275
|
# Add metadata to the header and generate it.
|
283
276
|
bytesize = if code
|
284
277
|
code.bytesize
|
@@ -287,7 +280,7 @@ module Pygments
|
|
287
280
|
end
|
288
281
|
|
289
282
|
kwargs.freeze
|
290
|
-
kwargs = kwargs.merge('
|
283
|
+
kwargs = kwargs.merge('bytes' => bytesize)
|
291
284
|
out_header = JSON.generate(method: method, args: args, kwargs: kwargs)
|
292
285
|
|
293
286
|
begin
|
@@ -315,7 +308,7 @@ module Pygments
|
|
315
308
|
header = @out.read(header_len)
|
316
309
|
|
317
310
|
# Now handle the header, any read any more data required.
|
318
|
-
handle_header_and_return(header
|
311
|
+
handle_header_and_return(header)
|
319
312
|
end
|
320
313
|
|
321
314
|
# Finally, return what we got.
|
@@ -341,10 +334,8 @@ module Pygments
|
|
341
334
|
# Based on the header we receive, determine if we need
|
342
335
|
# to read more bytes, and read those bytes if necessary.
|
343
336
|
#
|
344
|
-
# Then, do a sanity check with the ids.
|
345
|
-
#
|
346
337
|
# Returns a result - either highlighted text or metadata.
|
347
|
-
def handle_header_and_return(header
|
338
|
+
def handle_header_and_return(header)
|
348
339
|
if header
|
349
340
|
@log.info "In header: #{header}"
|
350
341
|
header = header_to_json(header)
|
@@ -356,37 +347,14 @@ module Pygments
|
|
356
347
|
if header[:method] == 'highlight'
|
357
348
|
# Make sure we have a result back; else consider this an error.
|
358
349
|
raise MentosError, 'No highlight result back from mentos.' if res.nil?
|
359
|
-
|
360
|
-
@log.info 'Highlight in process.'
|
361
|
-
|
362
|
-
# Get the id's
|
363
|
-
start_id = res[0..7]
|
364
|
-
end_id = res[-8..-1]
|
365
|
-
|
366
|
-
# Sanity check.
|
367
|
-
if !((start_id == id) && (end_id == id))
|
368
|
-
raise MentosError, "ID's did not match. Aborting."
|
369
|
-
else
|
370
|
-
# We're good. Remove the padding
|
371
|
-
res = res[10..-11]
|
372
|
-
@log.info 'Highlighting complete.'
|
373
|
-
res
|
374
|
-
end
|
375
350
|
end
|
351
|
+
|
376
352
|
res
|
377
353
|
else
|
378
354
|
raise MentosError, 'No header received back.'
|
379
355
|
end
|
380
356
|
end
|
381
357
|
|
382
|
-
# With the code, prepend the id (with two spaces to avoid escaping weirdness if
|
383
|
-
# the following text starts with a slash (like terminal code), and append the
|
384
|
-
# id, with two padding also. This means we are sending over the 8 characters +
|
385
|
-
# code + 8 characters.
|
386
|
-
def add_ids(code, id)
|
387
|
-
(id + " #{code} #{id}").freeze
|
388
|
-
end
|
389
|
-
|
390
358
|
# Return the final result for the API. Return Ruby objects for the methods that
|
391
359
|
# want them, text otherwise.
|
392
360
|
def return_result(res, method)
|
data/lib/pygments/version.rb
CHANGED
data/pygments.rb.gemspec
CHANGED
@@ -7,7 +7,7 @@ Gem::Specification.new do |s|
|
|
7
7
|
s.version = Pygments::VERSION
|
8
8
|
|
9
9
|
s.summary = 'pygments wrapper for ruby'
|
10
|
-
s.description = 'pygments.rb
|
10
|
+
s.description = 'pygments.rb is a Ruby wrapper for Pygments syntax highlighter'
|
11
11
|
|
12
12
|
s.homepage = 'https://github.com/pygments/pygments.rb'
|
13
13
|
s.required_ruby_version = '>= 2.3.0'
|
@@ -16,7 +16,7 @@ Gem::Specification.new do |s|
|
|
16
16
|
s.email = ['marat@slonopotamus.org']
|
17
17
|
s.license = 'MIT'
|
18
18
|
|
19
|
-
s.add_development_dependency 'rake
|
19
|
+
s.add_development_dependency 'rake', '~> 13.0.0'
|
20
20
|
s.add_development_dependency 'rubocop', '~> 0.81.0'
|
21
21
|
s.add_development_dependency 'test-unit', '~> 3.3.0'
|
22
22
|
|
data/test/test_pygments.rb
CHANGED
@@ -119,31 +119,6 @@ class PygmentsHighlightTest < Test::Unit::TestCase
|
|
119
119
|
end
|
120
120
|
end
|
121
121
|
|
122
|
-
# Philosophically, I'm not the biggest fan of testing private
|
123
|
-
# methods, but given the relative delicacy of validity checking
|
124
|
-
# over the pipe I think it's necessary and informative.
|
125
|
-
class PygmentsValidityTest < Test::Unit::TestCase
|
126
|
-
def test_add_ids_with_padding
|
127
|
-
res = PE.send(:add_ids, 'herp derp baz boo foo', 'ABCDEFGH')
|
128
|
-
assert_equal 'ABCDEFGH herp derp baz boo foo ABCDEFGH', res
|
129
|
-
end
|
130
|
-
|
131
|
-
def test_add_ids_on_empty_string
|
132
|
-
res = PE.send(:add_ids, '', 'ABCDEFGH')
|
133
|
-
assert_equal 'ABCDEFGH ABCDEFGH', res
|
134
|
-
end
|
135
|
-
|
136
|
-
def test_add_ids_with_unicode_data
|
137
|
-
res = PE.send(:add_ids, '# ø ø ø', 'ABCDEFGH')
|
138
|
-
assert_equal 'ABCDEFGH # ø ø ø ABCDEFGH', res
|
139
|
-
end
|
140
|
-
|
141
|
-
def test_add_ids_with_starting_slashes
|
142
|
-
res = PE.send(:add_ids, '\\# ø ø ø..//', 'ABCDEFGH')
|
143
|
-
assert_equal 'ABCDEFGH \\# ø ø ø..// ABCDEFGH', res
|
144
|
-
end
|
145
|
-
end
|
146
|
-
|
147
122
|
class PygmentsLexerTest < Test::Unit::TestCase
|
148
123
|
RUBY_CODE = "#!/usr/bin/ruby\nputs 'foo'"
|
149
124
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pygments.rb
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.0
|
4
|
+
version: 2.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Aman Gupta
|
@@ -13,19 +13,19 @@ cert_chain: []
|
|
13
13
|
date: 2021-01-07 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
|
-
name: rake
|
16
|
+
name: rake
|
17
17
|
requirement: !ruby/object:Gem::Requirement
|
18
18
|
requirements:
|
19
19
|
- - "~>"
|
20
20
|
- !ruby/object:Gem::Version
|
21
|
-
version:
|
21
|
+
version: 13.0.0
|
22
22
|
type: :development
|
23
23
|
prerelease: false
|
24
24
|
version_requirements: !ruby/object:Gem::Requirement
|
25
25
|
requirements:
|
26
26
|
- - "~>"
|
27
27
|
- !ruby/object:Gem::Version
|
28
|
-
version:
|
28
|
+
version: 13.0.0
|
29
29
|
- !ruby/object:Gem::Dependency
|
30
30
|
name: rubocop
|
31
31
|
requirement: !ruby/object:Gem::Requirement
|
@@ -54,7 +54,7 @@ dependencies:
|
|
54
54
|
- - "~>"
|
55
55
|
- !ruby/object:Gem::Version
|
56
56
|
version: 3.3.0
|
57
|
-
description: pygments.rb
|
57
|
+
description: pygments.rb is a Ruby wrapper for Pygments syntax highlighter
|
58
58
|
email:
|
59
59
|
- marat@slonopotamus.org
|
60
60
|
executables: []
|
@@ -341,9 +341,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
341
341
|
version: 2.3.0
|
342
342
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
343
343
|
requirements:
|
344
|
-
- - "
|
344
|
+
- - ">="
|
345
345
|
- !ruby/object:Gem::Version
|
346
|
-
version:
|
346
|
+
version: '0'
|
347
347
|
requirements: []
|
348
348
|
rubygems_version: 3.2.3
|
349
349
|
signing_key:
|