ragel-bitmap 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 81ee241cad1db7b792b0d42ce470433c0577cd8845082743105b63a6033f2258
-  data.tar.gz: 36166c5ed4601c8f8dd4364119a5000164628e1d50656c4b99ebe718870fa4cf
+  metadata.gz: 178f63cf68da95f64eb11f01ebb4d677fd4d1b3be0a54666e1e729c21e2a2c06
+  data.tar.gz: 4ece2fbcae8b07e5259840f5f934a8af330fa1663a06c13593303d500ff0956d
 SHA512:
-  metadata.gz: 7f4e3ad126a77e79b80527a2825fa05566e02c30c24ca1f0766af2448d8aaa47ec9e199c48ac3eaf302f5bfe74e3ca32f2e8c60089384243ef63ff23de16c6df
-  data.tar.gz: 47de8300a7b8f87c4dd20b08e37a4a9bb12da2d7402d3bb97472bb12e819ec4cff541eabf3079f4a174c8d92eefe206273886b4c5e8e9edbb61964907d1b4847
+  metadata.gz: fe356d2518bc0708c5a2792abefde227a00b6f5b20c97ee1f74d2632e937a925ec678a8d51f31e0a6755ea2738cc02bb20f4a17b62142130eed9588c00be6a1f
+  data.tar.gz: cf6b3b9f269e29093f528f33b2c46a8e8ef4c174d9c4104c91fc356140ab467a5b8ca5055868edd05d8037931dcdf4a92c9b2d01f451c14d35d2e2d8ba1581c0

data/.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "bundler"
+    directory: "/"
+    schedule:
+      interval: "daily"

data/.github/workflows/main.yml

@@ -0,0 +1,34 @@
+name: Main
+on:
+- push
+- pull_request_target
+jobs:
+  ci:
+    name: CI
+    runs-on: ubuntu-latest
+    env:
+      CI: true
+    steps:
+    - uses: actions/checkout@master
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 3.0
+        bundler-cache: true
+    - name: Lint and test
+      run: |
+        bundle exec rubocop --parallel
+        bundle exec rake test
+  automerge:
+    name: AutoMerge
+    needs: ci
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request_target' && (github.actor == github.repository_owner || github.actor == 'dependabot[bot]')
+    steps:
+    - uses: actions/github-script@v3
+      with:
+        script: |
+          github.pulls.merge({
+            owner: context.payload.repository.owner.login,
+            repo: context.payload.repository.name,
+            pull_number: context.payload.pull_request.number
+          })

data/.rubocop.yml

@@ -1,10 +1,34 @@
 AllCops:
   DisplayCopNames: true
   DisplayStyleGuide: true
-  TargetRubyVersion: 2.6
+  TargetRubyVersion: 2.7
   Exclude:
+  - '{tmp,vendor,yard}/**/*'
   - 'test/ragel/fixtures/*'
-  - 'vendor/**/*'
+
+Gemspec/RequiredRubyVersion:
+  Enabled: false
+
+Layout/LineLength:
+  Max: 80
+
+Lint/AmbiguousBlockAssociation:
+  Enabled: false
+
+Naming/RescuedExceptionsVariableName:
+  Enabled: false
 
 Style/Documentation:
   Enabled: false
+
+Style/FormatString:
+  EnforcedStyle: percent
+
+Style/FormatStringToken:
+  Enabled: false
+
+Style/PerlBackrefs:
+  Enabled: false
+
+Style/SlicingWithRange:
+  Enabled: false

data/CHANGELOG.md

@@ -0,0 +1,19 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.2.2] - 2021-11-17
+
+### Changed
+
+- Take advantage of the new `unpack1` `offset` option.
+- Require MFA for releasing.
+
+## [0.2.1] - 2019-07-10
+
+[unreleased]: https://github.com/kddnewton/ragel-bitmap/compare/v0.2.1...HEAD
+[0.2.1]: https://github.com/kddnewton/ragel-bitmap/compare/bb72cc...v0.2.1

data/CODE_OF_CONDUCT.md

@@ -55,7 +55,7 @@ further defined and clarified by project maintainers.
 ## Enforcement
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at kevin.deisz@gmail.com. All
+reported by contacting the project team at kddnewton@gmail.com. All
 complaints will be reviewed and investigated and will result in a response that
 is deemed necessary and appropriate to the circumstances. The project team is
 obligated to maintain confidentiality with regard to the reporter of an incident.

data/Gemfile.lock

@@ -1,28 +1,33 @@
 PATH
   remote: .
   specs:
-    ragel-bitmap (0.2.1)
+    ragel-bitmap (0.2.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    ast (2.4.0)
-    jaro_winkler (1.5.3)
-    minitest (5.11.3)
-    parallel (1.17.0)
-    parser (2.6.3.0)
-      ast (~> 2.4.0)
+    ast (2.4.2)
+    minitest (5.14.4)
+    parallel (1.21.0)
+    parser (3.0.2.0)
+      ast (~> 2.4.1)
     rainbow (3.0.0)
-    rake (12.3.2)
-    rubocop (0.72.0)
-      jaro_winkler (~> 1.5.1)
+    rake (13.0.6)
+    regexp_parser (2.1.1)
+    rexml (3.2.5)
+    rubocop (1.23.0)
       parallel (~> 1.10)
-      parser (>= 2.6)
+      parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.12.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 1.7)
-    ruby-progressbar (1.10.1)
-    unicode-display_width (1.6.0)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.13.0)
+      parser (>= 3.0.1.1)
+    ruby-progressbar (1.11.0)
+    unicode-display_width (2.1.0)
 
 PLATFORMS
   ruby
@@ -31,8 +36,8 @@ DEPENDENCIES
   bundler (~> 2.0)
   minitest (~> 5.11)
   ragel-bitmap!
-  rake (~> 12.3)
-  rubocop (~> 0.72)
+  rake (~> 13.0)
+  rubocop (~> 1.12)
 
 BUNDLED WITH
-   2.0.1
+   2.3.0.dev

data/LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2019-present Kevin Deisz
+Copyright (c) 2019-present Kevin Newton
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -1,9 +1,55 @@
 # Ragel::Bitmap
 
-[![Build Status](https://travis-ci.com/kddeisz/ragel-bitmap.svg?branch=master)](https://travis-ci.com/kddeisz/ragel-bitmap)
+[![Build Status](https://github.com/kddnewton/ragel-bitmap/workflows/Main/badge.svg)](https://github.com/kddnewton/ragel-bitmap/actions)
+[![Gem Version](https://img.shields.io/gem/v/ragel-bitmap.svg)](https://rubygems.org/gems/ragel-bitmap)
 
 [Ragel](https://www.colm.net/open-source/ragel/) generates ruby code with very large arrays of integers that allocate a lot of memory when required. To reduce memory consumption, this gem replaces those arrays with packed strings.
 
+For example, let's say you have a very large array of 8-bit unsigned integers (like ragel generates for small parsers). In a lot of places where ragel needs to jump between states, it will interject arrays like:
+
+```ruby
+self.transitions =
+  [206, 28, 108, 253, 151, 248, 208, 13, 97, 74]
+```
+
+For a small array like this, that's not a big deal. But for some very large parsers, that can add up to [a lot of memory overhead](https://github.com/micke/valid_email2/issues/165) even just to parse the file. Using this gem, that line of code would be replaced instead with:
+
+```ruby
+self.transitions =
+  Ragel::Bitmap::Array8.new("\xCE\x1Cl\xFD\x97\xF8\xD0\raJ")
+```
+
+Since the only method that ragel uses on these arrays is `#[]`, that method is implemented as well on `Array8`, which makes it possible to swap this code in seemlessly.
+
+In a lot of cases the numbers contained in these arrays won't quite fit into 8-bit unsigned integers, more often it's 16-bits that you end up needing. For that we use `Ragel::Bitmap::Array16`, as in:
+
+```ruby
+self.transitions =
+  [22427, 6684, 39361, 52842, 12850, 37505, 53824, 5861, 51699, 63533]
+```
+
+becomes:
+
+```ruby
+self.transitions =
+  Ragel::Bitmap::Array16.new(
+    "W\x1A\x99\xCE2\x92\xD2\x16\xC9\xF8",
+    "\x9B\x1C\xC1j2\x81@\xE5\xF3-"
+    )
+```
+
+The two string arguments being given to the `::new` method there signify the high and low byte of each of the numbers, respectively. Then, when we go to pull the numbers back out, we simply get the 8-bit segments from each of the high and low bytes, shift the high byte left by 8 bits, and bitwise-or them together.
+
+This kind of approach works for any size integer, because we can keep splitting it up into multiple bytes. We have explicit support for:
+
+* `Array8` - 1-byte integers
+* `Array16` - 2-byte integers
+* `Array24` - 3-byte integers
+* `Array32` - 4-byte integers
+* `Array32Offset` - 4-byte integers using `unpack1` with the offset parameter, available since [3.1.0](https://bugs.ruby-lang.org/issues/18254)
+* `Array64Offset` - 8-byte integers using `unpack1` with the offset parameter, available since [3.1.0](https://bugs.ruby-lang.org/issues/18254)
+* `ArrayGeneric` - functions with any size integer
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -22,7 +68,7 @@ Or install it yourself as:
 
 ## Usage
 
-After you've run `ragel` to generate your parser, you should then run this gem over the resulting source file and it will replace the integer arrays inline. For example, the following code adds a rake rule to generate the ragel parser from the grammar file and then run `Ragel::Bitmap` over it:
+After you've run `ragel` to generate your parser, you should then run this gem over the resulting source file and it will replace the integer arrays with our specialized array substitutions. For example, the following code adds a rake rule to generate the ragel parser from the grammar file and then run `Ragel::Bitmap` over it:
 
 ```ruby
 rule %r|_parser\.rb\z| => '%X.rl' do |t|
@@ -34,6 +80,8 @@ end
 
 Then, in your application, add `require 'ragel/bitmap'` before you require your parser. Now you should be off and running!
 
+We also provide a `ragel-bitmap` command that you can execute once this gem is installed. Usage looks like `ragel-bitmap [path]` where `path` is a path to a generated ragel parser.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
@@ -42,7 +90,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/kddeisz/ragel-bitmap.
+Bug reports and pull requests are welcome on GitHub at https://github.com/kddnewton/ragel-bitmap.
 
 ## License
 

data/bin/bench

@@ -0,0 +1,92 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/inline'
+
+gemfile do
+  source 'https://rubygems.org'
+  gem 'benchmark-ips'
+end
+
+$LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
+require 'ragel/bitmap'
+
+numbers = Array.new(10_000) { rand(2**16) }
+offsets = numbers.pack('S*')
+
+high = numbers.map { |number| number >> 8 }.pack('C*')
+low = numbers.pack('C*')
+
+# Here we're measuring if it's faster to use unpack1 with offset or if it's
+# faster to keep two separate strings around, use getbyte, and | them together
+# for 16-byte unsigned integers.
+Benchmark.ips do |x|
+  x.report('16-byte offsets') do
+    index = rand(10_000)
+    offsets.unpack1('S', offset: index)
+  end
+
+  x.report('16-byte strings') do
+    index = rand(10_000)
+    (high.getbyte(index) << 8) | low.getbyte(index)
+  end
+
+  x.compare!
+end
+
+numbers = Array.new(10_000) { rand(2**32) }
+offsets = numbers.pack('L*')
+high, midhigh, midlow, low =
+  [24, 16, 8, 0].map do |shift|
+    numbers.map { |number| number >> shift }.pack('C*')
+  end
+
+# Here we're measuring the same difference but this time we're using 32-byte
+# unsigned integers instead.
+Benchmark.ips do |x|
+  x.report('32-byte offsets') do
+    index = rand(10_000)
+    offsets.unpack1('L', offset: index)
+  end
+
+  x.report('32-byte strings') do
+    index = rand(10_000)
+    (high.getbyte(index) << 24) |
+      (midhigh.getbyte(index) << 16) |
+      (midlow.getbyte(index) << 8) |
+      low.getbyte(index)
+  end
+
+  x.compare!
+end
+
+numbers = Array.new(10_000) { rand(2**64) }
+offsets = numbers.pack('Q*')
+highest, midhighest, high, midhigh, midlow, low, midlowest, lowest =
+  [56, 48, 40, 32, 24, 16, 8, 0].map do |shift|
+    numbers.map { |number| number >> shift }.pack('C*')
+  end
+
+# Here we're measuring the same difference but this time we're using 64-byte
+# unsigned integers instead.
+Benchmark.ips do |x|
+  x.report('64-byte offsets') do
+    index = rand(10_000)
+    offsets.unpack1('Q', offset: index)
+  end
+
+  x.report('64-byte strings') do
+    index = rand(10_000)
+
+    (highest.getbyte(index) << 56) |
+      (midhighest.getbyte(index) << 48) |
+      (high.getbyte(index) << 40) |
+      (midhigh.getbyte(index) << 32) |
+      (midlow.getbyte(index) << 24) |
+      (low.getbyte(index) << 16) |
+      (midlowest.getbyte(index) << 8) |
+      lowest.getbyte(index)
+  end
+
+  x.compare!
+end

data/lib/ragel/bitmap/replace.rb

@@ -9,26 +9,72 @@ module Ragel
     class Replace < Ripper::SexpBuilderPP
       class << self
         # Get the required args for a bitmap from a set of numbers
+        # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength
         def bitmap_args_from(numbers)
           size = (Math.log2(numbers.max) / 8).ceil
-          strings =
-            size.downto(1).map do |index|
-              shift = (index - 1) * 8
-              numbers.map { |number| (number >> shift) & 0xff }.pack('C*')
-            end
 
-          [class_from(size), strings]
+          case size
+          when 1
+            [:Array8, [numbers.pack('C*')]]
+          when 2
+            [:Array16, strings_from(2, numbers)]
+          when 3
+            [:Array24, strings_from(3, numbers)]
+          when 4
+            if offset?
+              [:Array32Offset, [numbers.pack('L*')]]
+            else
+              [:Array32, strings_from(4, numbers)]
+            end
+          when 8
+            if offset?
+              [:Array64Offset, [numbers.pack('Q*')]]
+            else
+              [:ArrayGeneric, strings_from(size, numbers)]
+            end
+          else
+            [:ArrayGeneric, strings_from(size, numbers)]
+          end
+        end
+        # rubocop:enable Metrics/CyclomaticComplexity, Metrics/MethodLength
+
+        # Check if we can use the offset keyword argument for the unpack1 method
+        # introduced here: https://bugs.ruby-lang.org/issues/18254. If we can,
+        # then it'll be more efficient than storing multiple strings and
+        # shifting.
+        def offset?
+          String.instance_method(:unpack1).parameters.any? do |(type, name)|
+            type == :key && name == :offset
+          end
         end
 
         private
 
-        def class_from(size)
-          case size
-          when 1 then :Array8
-          when 2 then :Array16
-          when 3 then :Array24
-          else
-            :ArrayGeneric
+        # This will take a set of numbers and return a set of strings that
+        # represent their bytes in the same order. For example, if you passed in
+        # the numbers [10, 100, 1000] and said to encode them into 2 strings,
+        # you would receive back the strings:
+        #
+        #     "\x00\x00\x03"
+        #     "\x0a\x64\xe8"
+        #
+        # Read vertically, (with the first one being the higher byte) you'll
+        # see they add up to the input numbers. See the work below:
+        #
+        #     0x000a = 0d0010
+        #     0x0064 = 0d0100
+        #     0x03e8 = 0d1000
+        #
+        # To get the original numbers back, we can do much the same as we just
+        # did in that example, which is to read the strings in highest to lowest
+        # byte order, getting the bytes out at the same index every time. Then
+        # for each high string, shifting it left the requisite number of bytes
+        # (in the example above it's 8 since it's just 2 strings, if it were 3
+        # strings then the first string would be shifted by 16, and so on).
+        def strings_from(size, numbers)
+          size.downto(1).map do |index|
+            shift = (index - 1) * 8
+            numbers.map { |number| (number >> shift) & 0xff }.pack('C*')
           end
         end
       end

data/lib/ragel/bitmap/version.rb

@@ -2,6 +2,6 @@
 
 module Ragel
   module Bitmap
-    VERSION = '0.2.1'
+    VERSION = '0.2.2'
   end
 end

data/lib/ragel/bitmap.rb

@@ -48,6 +48,42 @@ module Ragel
       end
     end
 
+    class Array32
+      def initialize(first, second, third, fourth)
+        @first = first
+        @second = second
+        @third = third
+        @fourth = fourth
+      end
+
+      def [](idx)
+        (@first.getbyte(idx) << 24) |
+          (@second.getbyte(idx) << 16) |
+          (@third.getbyte(idx) << 8) |
+          @fourth.getbyte(idx)
+      end
+    end
+
+    class Array32Offset
+      def initialize(string)
+        @string = string
+      end
+
+      def [](idx)
+        @string.unpack1('L', offset: idx * 4)
+      end
+    end
+
+    class Array64Offset
+      def initialize(string)
+        @string = string
+      end
+
+      def [](idx)
+        @string.unpack1('Q', offset: idx * 8)
+      end
+    end
+
     class ArrayGeneric
       def initialize(*strings)
         @strings = strings

data/ragel-bitmap.gemspec

@@ -1,20 +1,28 @@
 # frozen_string_literal: true
 
-lib = File.expand_path('lib', __dir__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'ragel/bitmap/version'
+require_relative 'lib/ragel/bitmap/version'
+
+version = Ragel::Bitmap::VERSION
+repository = 'https://github.com/kddnewton/ragel-bitmap'
 
 Gem::Specification.new do |spec|
   spec.name          = 'ragel-bitmap'
-  spec.version       = Ragel::Bitmap::VERSION
-  spec.authors       = ['Kevin Deisz']
-  spec.email         = ['kevin.deisz@gmail.com']
+  spec.version       = version
+  spec.authors       = ['Kevin Newton']
+  spec.email         = ['kddnewton@gmail.com']
 
   spec.summary       = 'Use bitmaps for ragel-generated code instead of arrays'
-  spec.homepage      = 'https://github.com/kddeisz/ragel-bitmap'
+  spec.homepage      = repository
   spec.license       = 'MIT'
 
-  spec.files         =
+  spec.metadata = {
+    'bug_tracker_uri' => "#{repository}/issues",
+    'changelog_uri' => "#{repository}/blob/v#{version}/CHANGELOG.md",
+    'source_code_uri' => repository,
+    'rubygems_mfa_required' => 'true'
+  }
+
+  spec.files =
     Dir.chdir(File.expand_path(__dir__)) do
       `git ls-files -z`.split("\x0").reject do |f|
         f.match(%r{^(test|spec|features)/})
@@ -27,6 +35,6 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'minitest', '~> 5.11'
-  spec.add_development_dependency 'rake', '~> 12.3'
-  spec.add_development_dependency 'rubocop', '~> 0.72'
+  spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'rubocop', '~> 1.12'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ragel-bitmap
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
-- Kevin Deisz
-autorequire: 
+- Kevin Newton
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-07-10 00:00:00.000000000 Z
+date: 2021-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -44,56 +44,64 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.72'
+        version: '1.12'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.72'
-description: 
+        version: '1.12'
+description:
 email:
-- kevin.deisz@gmail.com
-executables: []
+- kddnewton@gmail.com
+executables:
+- ragel-bitmap
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/dependabot.yml"
+- ".github/workflows/main.yml"
 - ".gitignore"
 - ".rubocop.yml"
-- ".travis.yml"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - Gemfile.lock
 - LICENSE
 - README.md
 - Rakefile
+- bin/bench
 - bin/console
-- bin/replace
 - bin/setup
+- exe/ragel-bitmap
 - lib/ragel/bitmap.rb
 - lib/ragel/bitmap/replace.rb
 - lib/ragel/bitmap/version.rb
 - ragel-bitmap.gemspec
-homepage: https://github.com/kddeisz/ragel-bitmap
+homepage: https://github.com/kddnewton/ragel-bitmap
 licenses:
 - MIT
-metadata: {}
-post_install_message: 
+metadata:
+  bug_tracker_uri: https://github.com/kddnewton/ragel-bitmap/issues
+  changelog_uri: https://github.com/kddnewton/ragel-bitmap/blob/v0.2.2/CHANGELOG.md
+  source_code_uri: https://github.com/kddnewton/ragel-bitmap
+  rubygems_mfa_required: 'true'
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -108,8 +116,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.3.0.dev
+signing_key:
 specification_version: 4
 summary: Use bitmaps for ragel-generated code instead of arrays
 test_files: []

data/.travis.yml

@@ -1,9 +0,0 @@
-language: ruby
-cache: bundler
-rvm: 2.6.3
-before_install:
-- gem update --system
-- gem install bundler
-script:
-- bundle exec rake
-- bundle exec rubocop