pygments.rb 2.1.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a81b37936d77daa0478878031e065f4c4248ec09f8e5aad0955cb9d3906fdde2
-  data.tar.gz: f4eeaa4c16489986301798f0d760c924b8c5dbe600fe915b3d41607ead341287
+  metadata.gz: 3593fa0f51b043d8ffa0c2d91fef900f0d650d6e8c8787e32276a54682990089
+  data.tar.gz: 1ff949d4aaaf38d6baa557e01150a1c9838e42aca0eb99a0eb391c9a89924b8d
 SHA512:
-  metadata.gz: 764c90e3afefa3914a4c9dc21e0bef73f6ab94c87a30aa86e56e4080166a83f943bdd0058fc204ae878f7d2ccf7f23bc849f1363fce510f1062babe8aff66fb5
-  data.tar.gz: f90e47da1207df3ff7913f0acdee8ddd11438015c0664030908f42587bfd97ef67eedae6dcb0f668c2955c1022f3a8fe2127195b339618a4925b4632f0926710
+  metadata.gz: 9f6d6dfb5b4808dcd265206b30b19e8a865e3b53d3161ad891de99340a1903474eca425bdc92f6eba9116322d2ee96fc436a5931dacdf262e85f489385ac5068
+  data.tar.gz: a3f5aa3cda14507d939db624d6dbd1a0fd585ab1923d9aa87e367851cf0e5dd269cffa209ca28e511927ad018e27e9e1bc1008c7e4d2791cc532cbd5279b7ae7

data/.gitignore

@@ -3,4 +3,3 @@
 /pkg/
 /tmp
 *.pyc
-/lexers

data/.rubocop.yml

@@ -10,8 +10,6 @@ Layout/LineLength:
   Max: 120
 Metrics/MethodLength:
   Enabled: false
-Security/MarshalLoad:
-  Enabled: false
 Style/StructInheritance:
   Enabled: false
 Style/Documentation:

data/CHANGELOG.adoc

@@ -5,6 +5,11 @@
 This document provides a high-level view of the changes to the {project-name} by release.
 For a detailed view of what has changed, refer to the {uri-repo}/commits/master[commit history] on GitHub.
 
+== 2.2.0 (2021-03-18) - @slonopotamus
+
+* Add support for custom lexers ({uri-repo}/pull/187[#187])
+* Update Pygments to 2.8.1
+
 == 2.1.0 (2021-02-14) - @slonopotamus
 
 * Update Pygments to 2.8.0

data/README.adoc

@@ -2,7 +2,6 @@
 Ted Nyman <ted@ted.io>; Aman Gupta <aman@tmm1.net>; Marat Radchenko <marat@slonopotamus.org>
 :project-name: pygments.rb
 :slug: pygments/{project-name}
-:toc: preamble
 :uri-project: https://github.com/{slug}
 :uri-ci: {uri-project}/actions?query=branch%3Amaster
 :uri-gem: https://rubygems.org/gems/{project-name}

data/Rakefile

@@ -28,18 +28,6 @@ task :bench do
   sh 'ruby bench.rb'
 end
 
-# ==========================================================
-# Cache lexers
-# ==========================================================
-
-# Write all the lexers to a file for easy lookup
-task :lexers do
-  sh 'ruby cache_lexers.rb'
-end
-
-task(:test).enhance([:lexers])
-task(:build).enhance([:lexers])
-
 # ==========================================================
 # Vendor
 # ==========================================================

data/lib/pygments.rb

@@ -1,14 +1,18 @@
 # frozen_string_literal: true
 
-require File.join(File.dirname(__FILE__), 'pygments/popen')
 require 'forwardable'
 
-module Pygments
-  autoload :Lexer, 'pygments/lexer'
+require_relative 'pygments/lexer'
+require_relative 'pygments/popen'
 
+module Pygments
   class << self
     extend Forwardable
 
+    def lexers
+      LexerCache.instance.raw_lexers
+    end
+
     def engine
       Thread.current.thread_variable_get(:pygments_engine) ||
         Thread.current.thread_variable_set(:pygments_engine, Pygments::Popen.new)
@@ -16,7 +20,6 @@ module Pygments
 
     def_delegators :engine,
                    :formatters,
-                   :lexers,
                    :lexers!,
                    :filters,
                    :styles,

data/lib/pygments/lexer.rb

@@ -1,61 +1,14 @@
 # frozen_string_literal: true
 
+require 'singleton'
+
 module Pygments
   class Lexer < Struct.new(:name, :aliases, :filenames, :mimetypes)
-    @lexers          = []
-    @index           = {}
-    @name_index      = {}
-    @alias_index     = {}
-    @extname_index   = {}
-    @mimetypes_index = {}
-
-    # Internal: Create a new Lexer object
-    #
-    # hash - A hash of attributes
-    #
-    # Returns a Lexer object
-    def self.create(hash)
-      lexer = new(hash[:name], hash[:aliases], hash[:filenames], hash[:mimetypes])
-
-      @lexers << lexer
-
-      @index[lexer.name.downcase] = @name_index[lexer.name] = lexer
-
-      lexer.aliases.each do |name|
-        @alias_index[name] = lexer
-        @index[name.downcase] ||= lexer
-      end
-
-      lexer.filenames.each do |filename|
-        extnames = []
-
-        extname = File.extname(filename)
-        if (m = extname.match(/\[(.+)\]/))
-          m[1].scan(/./).each do |s|
-            extnames << extname.sub(m[0], s)
-          end
-        elsif extname != ''
-          extnames << extname
-        end
-
-        extnames.each do |the_extname|
-          @extname_index[the_extname] = lexer
-          @index[the_extname.downcase.sub(/^\./, '')] ||= lexer
-        end
-      end
-
-      lexer.mimetypes.each do |type|
-        @mimetypes_index[type] = lexer
-      end
-
-      lexer
-    end
-
     # Public: Get all Lexers
     #
-    # Returns an Array of Lexers
+    # @return [Array<Lexer>]
     def self.all
-      @lexers
+      LexerCache.instance.lexers
     end
 
     # Public: Look up Lexer by name or alias.
@@ -65,12 +18,15 @@ module Pygments
     #   Lexer.find('Ruby')
     #   => #<Lexer name="Ruby">
     #
-    # Returns the Lexer or nil if none was found.
+    # @return [Lexer, nil]
     def self.find(name)
-      @index[name.to_s.downcase]
+      LexerCache.instance.index[name.to_s.downcase]
     end
 
     # Public: Alias for find.
+    #
+    # @param name [String]
+    # @return [Lexer, nil]
     def self.[](name)
       find(name)
     end
@@ -84,9 +40,10 @@ module Pygments
     #   Lexer.find_by_name('Ruby')
     #   # => #<Lexer name="Ruby">
     #
-    # Returns the Lexer or nil if none was found.
+    # @param name [String]
+    # @return [Lexer, nil]
     def self.find_by_name(name)
-      @name_index[name]
+      LexerCache.instance.name_index[name]
     end
 
     # Public: Look up Lexer by one of its aliases.
@@ -98,9 +55,10 @@ module Pygments
     #   Lexer.find_by_alias('rb')
     #   # => #<Lexer name="Ruby">
     #
-    # Returns the Lexer or nil if none was found.
+    # @param name [String]
+    # @return [Lexer, nil]
     def self.find_by_alias(name)
-      @alias_index[name]
+      LexerCache.instance.alias_index[name]
     end
 
     # Public: Look up Lexer by one of it's file extensions.
@@ -112,9 +70,10 @@ module Pygments
     #  Lexer.find_by_extname('.rb')
     #  # => #<Lexer name="Ruby">
     #
-    # Returns the Lexer or nil if none was found.
+    # @param extname [String]
+    # @return [Lexer, nil]
     def self.find_by_extname(extname)
-      @extname_index[extname]
+      LexerCache.instance.extname_index[extname]
     end
 
     # Public: Look up Lexer by one of it's mime types.
@@ -126,9 +85,10 @@ module Pygments
     #  Lexer.find_by_mimetype('application/x-ruby')
     #  # => #<Lexer name="Ruby">
     #
-    # Returns the Lexer or nil if none was found.
+    # @param type [String]
+    # @return [Lexer, nil]
     def self.find_by_mimetype(type)
-      @mimetypes_index[type]
+      LexerCache.instance.mimetypes_index[type]
     end
 
     # Public: Highlight syntax of text
@@ -146,5 +106,67 @@ module Pygments
     alias eql? equal?
   end
 
-  lexers.values.each { |h| Lexer.create(h) }
+  class LexerCache
+    include Singleton
+
+    # @return [Array<Lexer>]
+    attr_reader(:lexers)
+    # @return [Map<String, Lexer>]
+    attr_reader(:index)
+    # @return [Map<String, Lexer>]
+    attr_reader(:name_index)
+    # @return [Map<String, Lexer]
+    attr_reader(:alias_index)
+    # @return [Map<String, Lexer>]
+    attr_reader(:extname_index)
+    # @return [Map<String, Lexer>]
+    attr_reader(:mimetypes_index)
+
+    attr_reader(:raw_lexers)
+
+    def initialize
+      @lexers = []
+      @index = {}
+      @name_index = {}
+      @alias_index = {}
+      @extname_index = {}
+      @mimetypes_index = {}
+      @raw_lexers = Pygments.lexers!
+
+      @raw_lexers.values.each do |hash|
+        lexer = Lexer.new(hash[:name], hash[:aliases], hash[:filenames], hash[:mimetypes])
+
+        @lexers << lexer
+
+        @index[lexer.name.downcase] = @name_index[lexer.name] = lexer
+
+        lexer.aliases.each do |name|
+          @alias_index[name] = lexer
+          @index[name.downcase] ||= lexer
+        end
+
+        lexer.filenames.each do |filename|
+          extnames = []
+
+          extname = File.extname(filename)
+          if (m = extname.match(/\[(.+)\]/))
+            m[1].scan(/./).each do |s|
+              extnames << extname.sub(m[0], s)
+            end
+          elsif extname != ''
+            extnames << extname
+          end
+
+          extnames.each do |the_extname|
+            @extname_index[the_extname] = lexer
+            @index[the_extname.downcase.sub(/^\./, '')] ||= lexer
+          end
+        end
+
+        lexer.mimetypes.each do |type|
+          @mimetypes_index[type] = lexer
+        end
+      end
+    end
+  end
 end

data/lib/pygments/popen.rb

@@ -103,25 +103,8 @@ module Pygments
       end
     end
 
-    # Get all lexers from a serialized array.
-    # This avoids needing to spawn mentos when it's not really needed
-    # (e.g., one-off jobs, loading the Rails env, etc).
-    #
-    # Should be preferred to #lexers!
-    #
-    # @return [Array<String>] an array of lexers
-    def lexers
-      lexer_file = File.join(__dir__, '..', '..', 'lexers')
-      begin
-        File.open(lexer_file, 'rb') do |f|
-          Marshal.load(f)
-        end
-      rescue Errno::ENOENT
-        raise MentosError, %(Error loading #{lexer_file}. Was it created and vendored?)
-      end
-    end
-
-    # Get back all available lexers from mentos itself
+    # Get all available lexers from mentos itself
+    # Do not use this method directly, instead use Pygments#lexers
     #
     # @return [Array<String>] an array of lexers
     def lexers!

data/lib/pygments/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pygments
-  VERSION = '2.1.0'
+  VERSION = '2.2.0'
 end

data/pygments.rb.gemspec

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require File.expand_path('lib/pygments/version', __dir__)
+require_relative 'lib/pygments/version'
 
 Gem::Specification.new do |s|
   s.name = 'pygments.rb'
@@ -8,20 +8,25 @@ Gem::Specification.new do |s|
 
   s.summary = 'pygments wrapper for ruby'
   s.description = 'pygments.rb is a Ruby wrapper for Pygments syntax highlighter'
-
+  s.license = 'MIT'
   s.homepage = 'https://github.com/pygments/pygments.rb'
-  s.required_ruby_version = '>= 2.3.0'
 
   s.authors = ['Aman Gupta', 'Ted Nyman', 'Marat Radchenko']
   s.email = ['marat@slonopotamus.org']
-  s.license = 'MIT'
+
+  s.metadata = {
+    'homepage_uri' => s.homepage,
+    'bug_tracker_uri' => s.homepage + '/issues',
+    'changelog_uri' => s.homepage + '/blob/master/CHANGELOG.adoc',
+    'documentation_uri' => 'https://www.rubydoc.info/gems/' + s.name,
+    'source_code_uri' => s.homepage
+  }
+
+  s.required_ruby_version = '>= 2.3.0'
 
   s.add_development_dependency 'rake', '~> 13.0.0'
   s.add_development_dependency 'rubocop', '~> 0.81.0'
   s.add_development_dependency 'test-unit', '~> 3.4.0'
 
-  # s.extensions = ['ext/extconf.rb']
-  s.require_paths = ['lib']
-
-  s.files = `git ls-files`.split("\n").reject { |f| File.symlink?(f) } + ['lexers']
+  s.files = `git ls-files -z`.split("\0").reject { |f| File.symlink?(f) }
 end

data/vendor/pygments-main/{Pygments-2.8.0.dist-info → Pygments-2.8.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: Pygments
-Version: 2.8.0
+Version: 2.8.1
 Summary: Pygments is a syntax highlighting package written in Python.
 Home-page: https://pygments.org/
 Author: Georg Brandl

data/vendor/pygments-main/{Pygments-2.8.0.dist-info → Pygments-2.8.1.dist-info}/RECORD

@@ -1,14 +1,14 @@
 ../../bin/pygmentize,sha256=tI1lspmgVfsR48Rydk5v3aLnl07D9v4TuU-ozRWTigA,217
-Pygments-2.8.0.dist-info/AUTHORS,sha256=lJ6QbbYXTyienm-GZXbKNOx3cSYiP1y0VemKmvgmu60,9002
-Pygments-2.8.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
-Pygments-2.8.0.dist-info/LICENSE,sha256=wBLPF6K6eRQpd8jMW7FJemdUAb95wsm5WnYE4t396Lg,1331
-Pygments-2.8.0.dist-info/METADATA,sha256=7SmfRKZaCDjwifTC7RqZ4vKXq3fhcRzeo_AZrKdFW9A,1884
-Pygments-2.8.0.dist-info/RECORD,,
-Pygments-2.8.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-Pygments-2.8.0.dist-info/WHEEL,sha256=EVRjI69F5qVjm_YgqcTXPnTAv3BfSUr0WVAHuSP3Xoo,92
-Pygments-2.8.0.dist-info/entry_points.txt,sha256=NXt9BRDRv6tAfDwqKM0bDHrrxaIt2f1nxH9CwjyjSKc,54
-Pygments-2.8.0.dist-info/top_level.txt,sha256=RjKKqrVIStoebLHdbs0yZ2Lk4rS7cxGguXsLCYvZ2Ak,9
-pygments/__init__.py,sha256=fYaenUPVtSn3coLWx4vbHxEx1noj0y77FPl7H0UIakw,3012
+Pygments-2.8.1.dist-info/AUTHORS,sha256=lJ6QbbYXTyienm-GZXbKNOx3cSYiP1y0VemKmvgmu60,9002
+Pygments-2.8.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+Pygments-2.8.1.dist-info/LICENSE,sha256=wBLPF6K6eRQpd8jMW7FJemdUAb95wsm5WnYE4t396Lg,1331
+Pygments-2.8.1.dist-info/METADATA,sha256=W_BWdvAhZr3S1gcFWXUEdtcdoq8oaqKkBGKfuwCizzU,1884
+Pygments-2.8.1.dist-info/RECORD,,
+Pygments-2.8.1.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+Pygments-2.8.1.dist-info/WHEEL,sha256=EVRjI69F5qVjm_YgqcTXPnTAv3BfSUr0WVAHuSP3Xoo,92
+Pygments-2.8.1.dist-info/entry_points.txt,sha256=NXt9BRDRv6tAfDwqKM0bDHrrxaIt2f1nxH9CwjyjSKc,54
+Pygments-2.8.1.dist-info/top_level.txt,sha256=RjKKqrVIStoebLHdbs0yZ2Lk4rS7cxGguXsLCYvZ2Ak,9
+pygments/__init__.py,sha256=iF5depWHZUVBHQzKCp_Go4P0aVMNtHj_imXYToxWLHw,3012
 pygments/__main__.py,sha256=B2xzjMErGo0ddFrGd8L55Tl9GLEP4lTbsUjKJ9DwqQM,348
 pygments/__pycache__/__init__.cpython-38.pyc,,
 pygments/__pycache__/__main__.cpython-38.pyc,,
@@ -50,7 +50,7 @@ pygments/formatters/bbcode.py,sha256=3Il76ivFigb8C6GBRnUj2Zzy6pIs7koVmJnYjWKRSqE
 pygments/formatters/html.py,sha256=H1a9AE7HREmrX0SzodhIZs0o0995ADNDDLQE1i49NB4,33861
 pygments/formatters/img.py,sha256=kTnWVgf6kpOKDqhHbgpZu5GwWADOm18KFnPJZAD55j0,21795
 pygments/formatters/irc.py,sha256=CpVJKZv9253OPxJhWB2eUZHY-JB1Mm4TwY1REf44g4Q,5845
-pygments/formatters/latex.py,sha256=fk1bDa8A9zt3mq12pkHmozidlh7KpyP2czo_UzonOC8,18874
+pygments/formatters/latex.py,sha256=gW7qKJDErjz7rH3x6nIH-izA67B2mEzlzz35VY4rh5k,18882
 pygments/formatters/other.py,sha256=OVt6uyJrkxOQvVAki5JJj4-_hmMkjlP4WRwPMgYx8nU,5025
 pygments/formatters/rtf.py,sha256=5b5DxR1L7SLv_I_9pvsF73aV_fOHrh0zNkcs2ZgN5WE,4990
 pygments/formatters/svg.py,sha256=9qgAf55XDDB7BZ-r51NRG85BUwcYzM1F-6DS7-gZXTE,7299

data/vendor/pygments-main/pygments/__init__.py

@@ -27,7 +27,7 @@
 import sys
 from io import StringIO, BytesIO
 
-__version__ = '2.8.0'
+__version__ = '2.8.1'
 __docformat__ = 'restructuredtext'
 
 __all__ = ['lex', 'format', 'highlight']

data/vendor/pygments-main/pygments/formatters/latex.py

@@ -299,7 +299,7 @@ class LatexFormatter(Formatter):
                 cmndef += (r'\def\$$@tc##1{\textcolor[rgb]{%s}{##1}}' %
                            rgbcolor(ndef['color']))
             if ndef['border']:
-                cmndef += (r'\def\$$@bc##1{{\setlength{\fboxsep}{-\fboxrule}'
+                cmndef += (r'\def\$$@bc##1{{\setlength{\fboxsep}{\string -\fboxrule}'
                            r'\fcolorbox[rgb]{%s}{%s}{\strut ##1}}}' %
                            (rgbcolor(ndef['border']),
                             rgbcolor(ndef['bgcolor'])))

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pygments.rb
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.2.0
 platform: ruby
 authors:
 - Aman Gupta
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-14 00:00:00.000000000 Z
+date: 2021-03-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -72,8 +72,6 @@ files:
 - README.adoc
 - Rakefile
 - bench.rb
-- cache_lexers.rb
-- lexers
 - lib/pygments.rb
 - lib/pygments/lexer.rb
 - lib/pygments/mentos.py
@@ -81,15 +79,15 @@ files:
 - lib/pygments/version.rb
 - pygments.rb.gemspec
 - test/test_pygments.rb
-- vendor/pygments-main/Pygments-2.8.0.dist-info/AUTHORS
-- vendor/pygments-main/Pygments-2.8.0.dist-info/INSTALLER
-- vendor/pygments-main/Pygments-2.8.0.dist-info/LICENSE
-- vendor/pygments-main/Pygments-2.8.0.dist-info/METADATA
-- vendor/pygments-main/Pygments-2.8.0.dist-info/RECORD
-- vendor/pygments-main/Pygments-2.8.0.dist-info/REQUESTED
-- vendor/pygments-main/Pygments-2.8.0.dist-info/WHEEL
-- vendor/pygments-main/Pygments-2.8.0.dist-info/entry_points.txt
-- vendor/pygments-main/Pygments-2.8.0.dist-info/top_level.txt
+- vendor/pygments-main/Pygments-2.8.1.dist-info/AUTHORS
+- vendor/pygments-main/Pygments-2.8.1.dist-info/INSTALLER
+- vendor/pygments-main/Pygments-2.8.1.dist-info/LICENSE
+- vendor/pygments-main/Pygments-2.8.1.dist-info/METADATA
+- vendor/pygments-main/Pygments-2.8.1.dist-info/RECORD
+- vendor/pygments-main/Pygments-2.8.1.dist-info/REQUESTED
+- vendor/pygments-main/Pygments-2.8.1.dist-info/WHEEL
+- vendor/pygments-main/Pygments-2.8.1.dist-info/entry_points.txt
+- vendor/pygments-main/Pygments-2.8.1.dist-info/top_level.txt
 - vendor/pygments-main/bin/pygmentize
 - vendor/pygments-main/pygments/__init__.py
 - vendor/pygments-main/pygments/__main__.py
@@ -336,7 +334,12 @@ files:
 homepage: https://github.com/pygments/pygments.rb
 licenses:
 - MIT
-metadata: {}
+metadata:
+  homepage_uri: https://github.com/pygments/pygments.rb
+  bug_tracker_uri: https://github.com/pygments/pygments.rb/issues
+  changelog_uri: https://github.com/pygments/pygments.rb/blob/master/CHANGELOG.adoc
+  documentation_uri: https://www.rubydoc.info/gems/pygments.rb
+  source_code_uri: https://github.com/pygments/pygments.rb
 post_install_message: 
 rdoc_options: []
 require_paths:

data/cache_lexers.rb

@@ -1,9 +0,0 @@
-# frozen_string_literal: true
-
-require File.join(File.dirname(__FILE__), '/lib/pygments.rb')
-
-# Simple marshalling
-serialized_lexers = Marshal.dump(Pygments.lexers!)
-
-# Write to a file
-File.open('lexers', 'wb') { |file| file.write(serialized_lexers) }