pygments.rb 1.2.1 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 4a0effd0abf9485e9c25411abd292c5d1afa9368
-  data.tar.gz: 5f891032766ff9e99578a13862a1900a2cd8da14
+SHA256:
+  metadata.gz: 82a1b6e9549afec22e94b2690983b55a5a3029e986b3d7201c2c6558a2968bc5
+  data.tar.gz: 63ac418bfb998653fc4472c1cfd3c8268a006b002754b1759bc11132889f3e2b
 SHA512:
-  metadata.gz: 66361ecc1d441298caa88f34d5be181a5c17b723106825de4d4030778823a77a2829da3139006c9e8e8458bbabebd408bf427b7cee663ffb58b4ffecf105bf8e
-  data.tar.gz: 9c98ac0f6fc5d88cb47d7305d99053cc27ffb0c788da6d31c5ddcc6a37e7845bca163d47bfb6090c6bfa90aa893fe4412fca0151cde753cff96525fed156e7ff
+  metadata.gz: a54ab6bc824be2d4fcdd0c61e52d0838b55ac7cc5ed023f8392ff784fa79d3f1fc32b159c422b46cc91646a8d07853ce2d934fd03e8c06de8903901f45a589c9
+  data.tar.gz: e3910502b53d5c3f6ad729bc558e1baffdbc8af8958fa8867e7489df10bbfd1d1e832f0791a90592418b6bf1c19ba3aa0289b7f043f81aabb071fc036fec29ad

data/.github/dependabot.yml

@@ -0,0 +1,13 @@
+# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "bundler"
+    directory: "/"
+    schedule:
+      interval: "daily"
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"

data/.github/workflows/ci.yml

@@ -0,0 +1,28 @@
+name: CI
+on:
+  push:
+    branches: [ '**' ]
+    tags-ignore: [ '**' ]
+  pull_request:
+jobs:
+  build:
+    strategy:
+      matrix:
+        ruby: [ 'jruby-9.2', '2.3', '3.0' ]
+        python: [ '3.5', '3.9' ]
+        platform: [ ubuntu-latest, macos-latest, windows-latest ]
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Run Tests
+        run: bundle exec rake test

data/.github/workflows/release.yml

@@ -0,0 +1,24 @@
+name: Publish to RubyGems.org
+on:
+  push:
+    tags: [ '*' ]
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.0'
+          bundler-cache: true
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+      - name: Build
+        run: bundle exec rake build
+      - name: Publish to RubyGems.org
+        uses: dawidd6/action-publish-gem@v1
+        with:
+          api_key: ${{ secrets.RUBYGEMS_API_KEY }}

data/.gitignore

@@ -1,7 +1,6 @@
-Gemfile.lock
-ext/Makefile
-lib/pygments_ext.*
-tmp
-pkg
+/.idea/
+/Gemfile.lock
+/pkg/
+/tmp
 *.pyc
-lexers
+/lexers

data/CHANGELOG.adoc

@@ -0,0 +1,128 @@
+= {project-name} Changelog
+:project-name: pygments.rb
+:uri-repo: https://github.com/pygments/pygments.rb
+
+This document provides a high-level view of the changes to the {project-name} by release.
+For a detailed view of what has changed, refer to the {uri-repo}/commits/master[commit history] on GitHub.
+
+== 2.0.0 (2021-01-08) - @slonopotamus
+
+* stop sending/receiving `ids` between Ruby and Python
+* use `close_others` Ruby mechanism to prevent file descriptor leaking to Python
+
+== 2.0.0.rc3 (2021-01-08) - @slonopotamus
+
+* fix watchdog race condition leading to `ThreadError(<killed thread>)` on JRuby ({uri-repo}/pull/215[#215])
+
+== 2.0.0.rc2 (2021-01-07) - @slonopotamus
+
+* Fix release workflow
+
+== 2.0.0.rc1 (2021-01-07) - @slonopotamus
+
+* Modify `mentos.py` to run on Python 3.x instead of Python 2.7
+* Add `:timeout` parameter to allow for configurable timeouts
+* Add several Python 3.x versions to test matrix
+* Drop dependency on MultiJson
+* Fix hanging on JRuby + Windows
+* Update Pygments to 2.7.3
+* Drop GitHub custom lexers
+* Rework timeout handling
+* Improve error message when Python is not found
+* *Breaking change* Pygments.rb no longer sets default timeout for its operations
+* *Breaking change* Now pygments.rb raises `MentosError` instead of returning `nil` on timeout
+
+== 1.2.1 (2017-12-07)
+
+* Automatically update `lexers` cache on build ({uri-repo}/pull/186[#186])
++
+See {uri-repo}/pull/185[#185] for the reason
+
+== 1.2.0 (2017-09-13)
+
+* Exclude symlinks from the gem package to solve Windows issues ({uri-repo}/pull/181[#181])
+* Upgrade pygments to 2.0.0 ({uri-repo}/pull/180[#180])
+
+== 1.1.2 (2017-04-03)
+
+* Resolves {uri-repo}/pull/176[#176] exclude find_error.py symlink from gem ({uri-repo}/pull/177[#177])
+
+== 1.1.1 (2016-12-28)
+
+* Suppress Ruby 2.4.0's warnings ({uri-repo}/pull/172[#172])
+* Enable `frozen_string_literal` ({uri-repo}/pull/173[#173])
+
+== 1.1.0 (2016-12-24)
+
+* Support JRuby ({uri-repo}/pull/170[#170])
+* Make pygments.rb thread safe ({uri-repo}/pull/171[#171])
+
+== 1.0.0 (2016-12-11)
+
+* Upgrade bundled pygments to 2.2.0-HEAD ({uri-repo}/pull/167[#167])
++
+This includes *incompatible changes* because of upgrade of pygments.
+See https://pygments.org/ for details.
+* Relax yajl-ruby dependency to "~> 1.2" ({uri-repo}/pull/164[#164])
+* Python binary can be configured by `PYTMENTS_RB_PYTHON` env ({uri-repo}/pull/168[#168])
+* Improved error messages when python binary is missing ({uri-repo}/pull/158[#158])
+
+== 0.5.4 (2013-11-03)
+
+* Update lexers file
+
+== 0.5.3 (2013-09-17)
+
+* Fixes for Slash lexer
+* Improve highlighting for Slash lexer
+* Upgrade to latest pygments (1.7, changes summary follows.
+See pygments changelog for details)
+** Add Clay lexer
+** Add Perl 6 lexer
+** Add Swig lexer
+** Add nesC lexer
+** Add BlitzBasic lexer
+** Add EBNF lexer
+** Add Igor Pro lexer
+** Add Rexx lexer
+** Add Agda lexer
+** Recognize vim modelines
+** Improve Python 3 lexer
+** Improve Opa lexer
+** Improve Julia lexer
+** Improve Lasso lexer
+** Improve Objective C/C++ lexer
+** Improve Ruby lexer
+** Improve Stan lexer
+** Improve JavaScript lexer
+** Improve HTTP lexer
+** Improve Koka lexer
+** Improve Haxe lexer
+** Improve Prolog lexer
+** Improve F# lexer
+
+== 0.5.2 (2013-07-17)
+
+* Add Slash lexer
+
+== 0.5.1 (2013-06-25)
+
+* Ensure compatibility across distros by detecting if `python2` is available
+
+== 0.5.0 (2013-04-13)
+
+* Use `#rstrip` to fix table mode bug
+
+== 0.4.2 (2013-02-25)
+
+* Add new lexers, including custom lexers
+
+== 0.3.7 (2013-01-02)
+
+* Fixed missing custom lexers
+* Added syntax highlighting for Hxml
+
+== 0.3.4 (2012-12-28)
+
+* Add support for Windows
+* Add MIT license

data/Gemfile

@@ -1,2 +1,4 @@
-source "https://rubygems.org"
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
 gemspec

data/LICENSE

@@ -1,5 +1,5 @@
 The MIT License (MIT)
-Copyright (c) Ted Nyman and Aman Gupta, 2012-2013
+Copyright (C) Ted Nyman, Aman Gupta, Marat Radchenko, 2012-2021
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and 
 associated documentation files (the "Software"), to deal in the Software without restriction, 

data/README.adoc

@@ -0,0 +1,160 @@
+= {project-name}
+Ted Nyman <ted@ted.io>; Aman Gupta <aman@tmm1.net>; Marat Radchenko <marat@slonopotamus.org>
+:project-name: pygments.rb
+:slug: pygments/{project-name}
+:toc: preamble
+:uri-project: https://github.com/{slug}
+:uri-ci: {uri-project}/actions?query=branch%3Amaster
+:uri-gem: https://rubygems.org/gems/{project-name}
+:uri-pygments: https://pygments.org/
+
+image:https://img.shields.io/gem/v/{project-name}.svg[Latest Release,link={uri-gem}]
+image:{uri-project}/workflows/CI/badge.svg?branch=master[Build Status,link={uri-ci}]
+
+== Introduction
+
+{project-name} is a Ruby wrapper for {uri-pygments}[Pygments] syntax highlighter.
+
+{project-name} works by talking over a simple pipe to a long-lived Python child process.
+This library replaces https://github.com/github/albino[github/albino], as well as an older version of {project-name} that used an embedded Python interpreter.
+
+Each Ruby process that runs has its own 'personal Python'; for example, 4 Unicorn workers will have one Python process each.
+If a Python process dies, a new one will be spawned on the next pygments.rb request.
+
+== System Requirements
+
+- Python >= 3.5
+- Ruby >= 2.3
+
+== Installation
+
+Add this line to your application's Gemfile:
+
+[source,ruby]
+----
+gem 'pygments.rb'
+----
+
+And then execute:
+
+[source,shell script]
+----
+$ bundle install
+----
+
+Or install pygments.rb gem yourself as:
+
+[source,shell script]
+----
+$ gem install pygments.rb
+----
+
+== Usage
+
+Require pygments.rb module:
+
+[source,ruby]
+----
+require 'pygments'
+----
+
+Highlight a file:
+
+[source,ruby]
+----
+Pygments.highlight(File.read(__FILE__), lexer: 'ruby')
+----
+
+Optionally, pass encoding and other lexer/formatter options via an `:options` hash:
+
+[source,ruby]
+----
+Pygments.highlight('code', options: {encoding: 'utf-8'})
+----
+
+pygments.rb uses HTML formatter by default.
+To use a different formatter, specify it via `:formatter` parameter:
+
+[source,ruby]
+----
+Pygments.highlight('code', formatter: 'bbcode')
+Pygments.highlight('code', formatter: 'terminal')
+----
+
+To generate CSS for HTML formatted code, use the `Pygments.css` method:
+
+[source,ruby]
+----
+Pygments.css
+Pygments.css('.highlight')
+----
+
+To use a specific pygments style, pass the `:style` option to the `Pygments.css` method:
+
+[source,ruby]
+----
+Pygments.css(style: 'monokai')
+----
+
+Other Pygments high-level API methods are also available.
+These methods return arrays detailing all the available lexers, formatters, and styles:
+
+[source,ruby]
+----
+Pygments.lexers
+Pygments.formatters
+Pygments.styles
+----
+
+To use a custom pygments installation, specify the path to
+`Pygments.start`:
+
+[source,ruby]
+----
+Pygments.start("/path/to/pygments")
+----
+
+If you'd like logging, set the environmental variable `MENTOS_LOG` to a file path for your logfile.
+
+You can apply a timeout to pygments.rb calls by specifying number of seconds in `MENTOS_TIMEOUT` environmental variable or by passing the `:timeout` argument (takes precedence over `MENTOS_TIMEOUT`):
+
+[source,ruby]
+----
+Pygments.highlight('code', timeout: 4)
+----
+
+== Benchmarks
+
+----
+$ ruby bench.rb 50
+   Benchmarking....
+   Size: 698 bytes
+   Iterations: 50
+                                                  user     system      total        real
+   pygments popen                                0.010000   0.010000   0.020000 (  0.460370)
+   pygments popen (process already started)      0.010000   0.000000   0.010000 (  0.272975)
+   pygments popen (process already started 2)    0.000000   0.000000   0.000000 (  0.273589)
+----
+
+----
+$ ruby bench.rb 10
+   Benchmarking....
+   Size: 15523 bytes
+   Iterations: 10
+                                                  user     system      total        real
+   pygments popen                               0.000000   0.000000   0.000000 (  0.819419)
+   pygments popen (process already started)     0.010000   0.000000   0.010000 (  0.676515)
+   pygments popen (process already started 2)   0.000000   0.010000   0.010000 (  0.674189)
+----
+
+== Development
+
+After checking out the repo, run `bundle install` to install dependencies.
+Then, run `bundle exec rake test` to run the tests.
+
+== Copyright
+
+Copyright (C) Ted Nyman, Aman Gupta, Marat Radchenko, 2012-2021.
+Free use of this software is granted under the terms of the MIT License.
+
+For the full text of the license, see the link:LICENSE[] file.

data/Rakefile

@@ -1,7 +1,9 @@
 #!/usr/bin/env rake
-require "bundler/gem_tasks"
+# frozen_string_literal: true
 
-task :default => :test
+require 'bundler/gem_tasks'
+
+task default: :test
 
 # ==========================================================
 # Packaging
@@ -18,7 +20,6 @@ require 'rubygems/package_task'
 require 'rake/testtask'
 Rake::TestTask.new 'test' do |t|
   t.test_files = FileList['test/test_*.rb']
-  t.ruby_opts = ['-rubygems']
 end
 
 # ==========================================================
@@ -26,7 +27,7 @@ end
 # ==========================================================
 
 task :bench do
-  sh "ruby bench.rb"
+  sh 'ruby bench.rb'
 end
 
 # ==========================================================
@@ -35,11 +36,11 @@ end
 
 # Write all the lexers to a file for easy lookup
 task :lexers do
-  sh "ruby cache-lexers.rb"
+  sh 'ruby cache-lexers.rb'
 end
 
-task test: :lexers
-task release: :lexers
+task(:test).enhance([:lexers])
+task(:build).enhance([:lexers])
 
 # ==========================================================
 # Vendor
@@ -47,25 +48,14 @@ task release: :lexers
 
 namespace :vendor do
   file 'vendor/pygments-main' do |f|
-    sh "hg clone https://bitbucket.org/birkenfeld/pygments-main #{f.name}"
-    sh "hg --repository #{f.name} identify --id > #{f.name}/REVISION"
-    rm_rf Dir["#{f.name}/.hg*"]
-    rm_rf Dir["#{f.name}/tests"]
+    sh "pip install --target=#{f.name} pygments"
+    sh "git add -- #{f.name}"
   end
 
   task :clobber do
     rm_rf 'vendor/pygments-main'
   end
 
-  # Load all the custom lexers in the `vendor/custom_lexers` folder
-  # and stick them in our custom Pygments vendor
-  task :load_lexers do
-    LEXERS_DIR = 'vendor/pygments-main/pygments/lexers'
-    lexers = FileList['vendor/custom_lexers/*.py']
-    lexers.each { |l| FileUtils.copy l, LEXERS_DIR }
-    FileUtils.cd(LEXERS_DIR) { sh "python _mapping.py" }
-  end
-
   desc 'update vendor/pygments-main'
-  task :update => [:clobber, 'vendor/pygments-main', :load_lexers]
+  task update: [:clobber, 'vendor/pygments-main']
 end

data/bench.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require File.join(File.dirname(__FILE__), '/lib/pygments.rb')
 require 'benchmark'
 
@@ -8,15 +10,14 @@ num = ARGV[0] ? ARGV[0].to_i : 10
 # we can also repeat the code itself
 repeats = ARGV[1] ? ARGV[1].to_i : 1
 
-code = File.open('test/test_data.py').read.to_s * repeats
+code = File.open('test/test_pygments.rb').read.to_s * repeats
 
 puts "Benchmarking....\n"
-puts "Size: " + code.bytesize.to_s + " bytes\n"
-puts "Iterations: " + num.to_s + "\n"
+puts 'Size: ' + code.bytesize.to_s + " bytes\n"
+puts 'Iterations: ' + num.to_s + "\n"
 
 Benchmark.bm(40) do |x|
-  x.report("pygments popen                             ")  { for i in 1..num; Pygments.highlight(code, :lexer => 'python'); end }
-  x.report("pygments popen (process already started)   ")  { for i in 1..num; Pygments.highlight(code, :lexer => 'python'); end }
-  x.report("pygments popen (process already started 2) ")  { for i in 1..num; Pygments.highlight(code, :lexer => 'python'); end }
+  x.report('pygments popen                             ')  { (1..num).each { |_i|; Pygments.highlight(code, lexer: 'python'); } }
+  x.report('pygments popen (process already started)   ')  { (1..num).each { |_i|; Pygments.highlight(code, lexer: 'python'); } }
+  x.report('pygments popen (process already started 2) ')  { (1..num).each { |_i|; Pygments.highlight(code, lexer: 'python'); } }
 end
-

data/cache-lexers.rb

@@ -1,8 +1,9 @@
+# frozen_string_literal: true
+
 require File.join(File.dirname(__FILE__), '/lib/pygments.rb')
 
 # Simple marshalling
 serialized_lexers = Marshal.dump(Pygments.lexers!)
 
 # Write to a file
-File.open("lexers", 'wb') { |file| file.write(serialized_lexers) }
-
+File.open('lexers', 'wb') { |file| file.write(serialized_lexers) }

data/lib/pygments/lexer.rb

@@ -1,5 +1,5 @@
-# coding: utf-8
 # frozen_string_literal: true
+
 module Pygments
   class Lexer < Struct.new(:name, :aliases, :filenames, :mimetypes)
     @lexers          = []
@@ -34,13 +34,13 @@ module Pygments
           m[1].scan(/./).each do |s|
             extnames << extname.sub(m[0], s)
           end
-        elsif extname != ""
+        elsif extname != ''
           extnames << extname
         end
 
         extnames.each do |the_extname|
           @extname_index[the_extname] = lexer
-          @index[the_extname.downcase.sub(/^\./, "")] ||= lexer
+          @index[the_extname.downcase.sub(/^\./, '')] ||= lexer
         end
       end
 
@@ -142,8 +142,8 @@ module Pygments
       Pygments.highlight(text, options)
     end
 
-    alias_method :==, :equal?
-    alias_method :eql?, :equal?
+    alias == equal?
+    alias eql? equal?
   end
 
   lexers.values.each { |h| Lexer.create(h) }