buftok 0.2.0 → 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 67c39aeda72dd14dc738490d14c121acd8591050057f488517c0a92039187179
+  data.tar.gz: dd6f4e0460ac0c2c076d9d4d05e91bdd29a2667167872f2850127b3ed7f72118
+SHA512:
+  metadata.gz: 9a2db2dffe2660fcb5ec89e813b8a953cdbdebd530184be5bf88f35dbf7bfd06dd15e5441b6e4316b6a23a83458d43df90212f7a30f9a2d25de5b49609ff6857
+  data.tar.gz: c8312db37a322e718163142e1246f909eca8e50f07bde78b706d46b327c035bf2c4d3e7ebb59d10b7cdeb5dbac29dcf7b9b8d1cad2ab5e7702f65ebfde315d79

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2021 Tony Arcieri, Martin Emde, Erik Michaels-Ober
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -1,14 +1,10 @@
 # BufferedTokenizer
 
-[![Gem Version](https://badge.fury.io/rb/buftok.png)][gem]
-[![Build Status](https://travis-ci.org/sferik/buftok.png?branch=master)][travis]
-[![Dependency Status](https://gemnasium.com/sferik/buftok.png?travis)][gemnasium]
-[![Code Climate](https://codeclimate.com/github/sferik/buftok.png)][codeclimate]
+[![Gem Version](http://img.shields.io/gem/v/buftok.svg)][gem]
+[![Build Status](https://github.com/sferik/buftok/actions/workflows/ruby.yml/badge.svg)][build]
 
 [gem]: https://rubygems.org/gems/buftok
-[travis]: https://travis-ci.org/sferik/buftok
-[gemnasium]: https://gemnasium.com/sferik/buftok
-[codeclimate]: https://codeclimate.com/github/sferik/buftok
+[build]: https://github.com/sferik/buftok/actions
 
 ###### Statefully split input data by a specifiable token
 
@@ -21,19 +17,18 @@ something like [EventMachine][].
 [EventMachine]: http://rubyeventmachine.com/
 
 ## Supported Ruby Versions
-This library aims to support and is [tested against][travis] the following Ruby
+This library aims to support and is [tested against][build] the following Ruby
 implementations:
 
-* Ruby 1.8.7
-* Ruby 1.9.2
-* Ruby 1.9.3
-* Ruby 2.0.0
+* Ruby 2.6
+* Ruby 2.7
+* Ruby 3.0
 
 If something doesn't work on one of these interpreters, it's a bug.
 
-This library may inadvertently work (or seem to work) on other Ruby
-implementations, however support will only be provided for the versions listed
-above.
+This code will likely still work on older versions since it has not undergone
+many changes since release. However, support will not be provided for
+end-of-life ruby versions.
 
 If you would like this library to support another Ruby version, you may
 volunteer to be a maintainer. Being a maintainer entails making sure all tests
@@ -43,6 +38,7 @@ fashion. If critical issues for a particular implementation exist at the time
 of a major release, support for that Ruby version may be dropped.
 
 ## Copyright
-Copyright (c) 2006-2013 Tony Arcieri, Martin Emde, Erik Michaels-Ober.
-Distributed under the [Ruby license][license].
-[license]: http://www.ruby-lang.org/en/LICENSE.txt
+Copyright (c) 2006-2021 Tony Arcieri, Martin Emde, Erik Michaels-Ober.
+Distributed under the [MIT license][license].
+
+[license]: https://opensource.org/licenses/MIT

data/buftok.gemspec

@@ -1,17 +1,19 @@
 Gem::Specification.new do |spec|
-  spec.add_development_dependency 'bundler', '~> 1.0'
+  spec.version       = "0.3.0"
+
   spec.authors       = ["Tony Arcieri", "Martin Emde", "Erik Michaels-Ober"]
-  spec.description   = %q{BufferedTokenizer extracts token delimited entities from a sequence of arbitrary inputs}
-  spec.email         = "sferik@gmail.com"
-  spec.files         = %w(CONTRIBUTING.md Gemfile LICENSE.md README.md Rakefile buftok.gemspec)
-  spec.files        += Dir.glob("lib/**/*.rb")
-  spec.files        += Dir.glob("test/**/*.rb")
-  spec.test_files    = spec.files.grep(%r{^test/})
+  spec.summary       = %q{BufferedTokenizer extracts token delimited entities from a sequence of string inputs}
+  spec.description   = spec.summary
+  spec.email         = ["sferik@gmail.com", "martin.emde@gmail.com"]
+  spec.files         = %w(CONTRIBUTING.md LICENSE.txt README.md buftok.gemspec) + Dir["lib/**/*.rb"]
   spec.homepage      = "https://github.com/sferik/buftok"
-  spec.licenses      = ['MIT']
+  spec.licenses      = ["MIT"]
   spec.name          = "buftok"
   spec.require_paths = ["lib"]
-  spec.required_rubygems_version = '>= 1.3.5'
-  spec.summary       = spec.description
-  spec.version       = "0.2.0"
+  spec.required_rubygems_version = ">= 1.3.5"
+
+  spec.add_development_dependency "bundler", ">= 1.17"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rdoc"
+  spec.add_development_dependency "test-unit"
 end

data/lib/buftok.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+#
 # BufferedTokenizer takes a delimiter upon instantiation, or acts line-based
 # by default.  It allows input to be spoon-fed from some outside source which
 # receives arbitrary length datagrams which may-or-may-not contain the token
@@ -15,10 +17,18 @@ class BufferedTokenizer
   def initialize(delimiter = $/)
     @delimiter = delimiter
     @input = []
-    @tail = ''
+    @tail = String.new
     @trim = @delimiter.length - 1
   end
 
+  # Determine the size of the internal buffer.
+  #
+  # Size is not cached and is determined every time this method is called
+  # in order to optimize throughput for extract.
+  def size
+    @tail.length + @input.inject(0) { |total, input| total + input.length }
+  end
+
   # Extract takes an arbitrary string of input data and returns an array of
   # tokenized entities, provided there were any available to extract.  This
   # makes for easy processing of datagrams using a pattern like:
@@ -53,7 +63,10 @@ class BufferedTokenizer
     @input << @tail
     buffer = @input.join
     @input.clear
-    @tail = "" # @tail.clear is slightly faster, but not supported on 1.8.7
+    @tail = String.new # @tail.clear is slightly faster, but not supported on 1.8.7
     buffer
   end
 end
+
+# The expected constant for a gem named buftok
+Buftok = BufferedTokenizer

metadata

@@ -1,75 +1,109 @@
 --- !ruby/object:Gem::Specification
 name: buftok
 version: !ruby/object:Gem::Version
-  version: 0.2.0
-  prerelease: 
+  version: 0.3.0
 platform: ruby
 authors:
 - Tony Arcieri
 - Martin Emde
 - Erik Michaels-Ober
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-11-22 00:00:00.000000000 Z
+date: 2021-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.17'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.17'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: BufferedTokenizer extracts token delimited entities from a sequence of
-  arbitrary inputs
-email: sferik@gmail.com
+  string inputs
+email:
+- sferik@gmail.com
+- martin.emde@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - CONTRIBUTING.md
-- Gemfile
-- LICENSE.md
+- LICENSE.txt
 - README.md
-- Rakefile
 - buftok.gemspec
 - lib/buftok.rb
-- test/test_buftok.rb
 homepage: https://github.com/sferik/buftok
 licenses:
 - MIT
-post_install_message: 
+metadata: {}
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: 1.3.5
 requirements: []
-rubyforge_project: 
-rubygems_version: 1.8.23
-signing_key: 
-specification_version: 3
-summary: BufferedTokenizer extracts token delimited entities from a sequence of arbitrary
+rubygems_version: 3.2.3
+signing_key:
+specification_version: 4
+summary: BufferedTokenizer extracts token delimited entities from a sequence of string
   inputs
-test_files:
-- test/test_buftok.rb
-has_rdoc: 
+test_files: []

data/Gemfile

@@ -1,6 +0,0 @@
-source 'https://rubygems.org'
-
-gem 'rake'
-gem 'rdoc'
-
-gemspec

data/LICENSE.md

@@ -1,56 +0,0 @@
-Ruby is copyrighted free software by Yukihiro Matsumoto <matz@netlab.jp>.
-You can redistribute it and/or modify it under either the terms of the
-2-clause BSDL (see the file BSDL), or the conditions below:
-
-  1. You may make and give away verbatim copies of the source form of the
-     software without restriction, provided that you duplicate all of the
-     original copyright notices and associated disclaimers.
-
-  2. You may modify your copy of the software in any way, provided that
-     you do at least ONE of the following:
-
-       a) place your modifications in the Public Domain or otherwise
-          make them Freely Available, such as by posting said
-	  modifications to Usenet or an equivalent medium, or by allowing
-	  the author to include your modifications in the software.
-
-       b) use the modified software only within your corporation or
-          organization.
-
-       c) give non-standard binaries non-standard names, with
-          instructions on where to get the original software distribution.
-
-       d) make other distribution arrangements with the author.
-
-  3. You may distribute the software in object code or binary form,
-     provided that you do at least ONE of the following:
-
-       a) distribute the binaries and library files of the software,
-	  together with instructions (in the manual page or equivalent)
-	  on where to get the original distribution.
-
-       b) accompany the distribution with the machine-readable source of
-	  the software.
-
-       c) give non-standard binaries non-standard names, with
-          instructions on where to get the original software distribution.
-
-       d) make other distribution arrangements with the author.
-
-  4. You may modify and include the part of the software into any other
-     software (possibly commercial).  But some files in the distribution
-     are not written by the author, so that they are not under these terms.
-
-     For the list of those files and their copying conditions, see the
-     file LEGAL.
-
-  5. The scripts and library files supplied as input to or produced as
-     output from the software do not automatically fall under the
-     copyright of the software, but belong to whomever generated them,
-     and may be sold commercially, and may be aggregated with this
-     software.
-
-  6. THIS SOFTWARE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR
-     IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
-     WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-     PURPOSE.

data/Rakefile

@@ -1,66 +0,0 @@
-require 'bundler'
-require 'rdoc/task'
-require 'rake/testtask'
-
-task :default => :test
-
-Bundler::GemHelper.install_tasks
-
-RDoc::Task.new do |task|
-  task.rdoc_dir = 'doc'
-  task.title    = 'BufferedTokenizer'
-  task.rdoc_files.include('lib/**/*.rb')
-end
-
-Rake::TestTask.new :test do |t|
-  t.libs << 'lib'
-  t.test_files = FileList['test/**/*.rb']
-end
-
-desc "Benchmark the current implementation"
-task :bench do
-  require 'benchmark'
-  require File.expand_path('lib/buftok', File.dirname(__FILE__))
-
-  n = 50000
-  delimiter = "\n\n"
-
-  frequency1 = 1000
-  puts "generating #{n} strings, with #{delimiter.inspect} every #{frequency1} strings..."
-  data1 = (0...n).map do |i|
-    (((i % frequency1 == 1) ? "\n" : "") +
-      ("s" * i) +
-      ((i % frequency1 == 0) ? "\n" : "")).freeze
-  end
-
-  frequency2 = 10
-  puts "generating #{n} strings, with #{delimiter.inspect} every #{frequency2} strings..."
-  data2 = (0...n).map do |i|
-    (((i % frequency2 == 1) ? "\n" : "") +
-      ("s" * i) +
-      ((i % frequency2 == 0) ? "\n" : "")).freeze
-  end
-
-  Benchmark.bmbm do |x|
-    x.report("1 char, freq: #{frequency1}") do
-      bt1 = BufferedTokenizer.new
-      n.times { |i| bt1.extract(data1[i]) }
-    end
-
-    x.report("2 char, freq: #{frequency1}") do
-      bt2 = BufferedTokenizer.new(delimiter)
-      n.times { |i| bt2.extract(data1[i]) }
-    end
-
-    x.report("1 char, freq: #{frequency2}") do
-      bt3 = BufferedTokenizer.new
-      n.times { |i| bt3.extract(data2[i]) }
-    end
-
-    x.report("2 char, freq: #{frequency2}") do
-      bt4 = BufferedTokenizer.new(delimiter)
-      n.times { |i| bt4.extract(data2[i]) }
-    end
-
-  end
-end

data/test/test_buftok.rb

@@ -1,27 +0,0 @@
-require 'test/unit'
-require 'buftok'
-
-class TestBuftok < Test::Unit::TestCase
-  def test_buftok
-    tokenizer = BufferedTokenizer.new
-    assert_equal %w[foo], tokenizer.extract("foo\nbar".freeze)
-    assert_equal %w[barbaz qux], tokenizer.extract("baz\nqux\nquu".freeze)
-    assert_equal 'quu', tokenizer.flush
-    assert_equal '', tokenizer.flush
-  end
-
-  def test_delimiter
-    tokenizer = BufferedTokenizer.new('<>')
-    assert_equal ['', "foo\n"], tokenizer.extract("<>foo\n<>".freeze)
-    assert_equal %w[bar], tokenizer.extract('bar<>baz'.freeze)
-    assert_equal 'baz', tokenizer.flush
-  end
-
-  def test_split_delimiter
-    tokenizer = BufferedTokenizer.new('<>'.freeze)
-    assert_equal [], tokenizer.extract('foo<'.freeze)
-    assert_equal %w[foo], tokenizer.extract('>bar<'.freeze)
-    assert_equal %w[bar<baz qux], tokenizer.extract('baz<>qux<>'.freeze)
-    assert_equal '', tokenizer.flush
-  end
-end