tokeneyes 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: f648d93394449ac71d1d776d559e4b394edd08af
-  data.tar.gz: 8ee6a9db2a1bf74b9bf317e225379a463b436012
+SHA256:
+  metadata.gz: d3799154ab70a79b433e18bed482a94c8ba2267901ca40faf5471c8e6bb826e0
+  data.tar.gz: d0cdb23eff011131eb79e8618a23e82547b61000e51c3bc3a4ef12aa1eaf6b5d
 SHA512:
-  metadata.gz: a94e9a12e5c9c301b791588593e858fc4ddb389fbe98b4e5dc80489819b0447ba2de289019a426c6075aea9c803f243269bcfcea3a440b18ff209f99c30a6f08
-  data.tar.gz: cecaa693773ea68d7211e82ff0c1a59fdd1fc875ab64bc98709d4acc1b6946ee01e5348d7bd7931da9853d0b6f303be6fe00b1bdc424c20174d359e905b93199
+  metadata.gz: 24c6dc4315eaa00a4a8da5759e3227fc67a0ed476e2d47f2ed468acb9b24afa70fb3a871785870b48d5e3fffbd9c01624ef837d14ff7040d040bd4444f70627e
+  data.tar.gz: d3b48c2ed9b9b5d2f16f31ff90b37b282540813ceed4aa8a1db550627f9b478d02abb2e64494c3b2677f293d5026e808fa6966fd7c2b7eb9ed9fe1a37ff6e8fa

data/.gitignore

@@ -7,5 +7,3 @@
 /pkg/
 /spec/reports/
 /tmp/
-.ruby-version
-.ruby-gemset

data/.ruby-gemset

@@ -0,0 +1 @@
+tokeneyes

data/.ruby-version

@@ -0,0 +1 @@
+2.7

data/.travis.yml

@@ -1,5 +1,6 @@
 language: ruby
 rvm:
-  - 2.2.3
-  - jruby-9000
-before_install: gem install bundler -v 1.10.6
+  - 2.5
+  - 2.6
+  - 2.7
+before_install: gem install bundler -v 2.1.4

data/Gemfile

@@ -3,6 +3,6 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in tokeneyes.gemspec
 gemspec
 
-group :test do
-  gem "codeclimate-test-reporter", require: nil
+group :development do
+  gem "pry"
 end

data/README.md

@@ -1,3 +1,7 @@
+[![Code Climate](https://codeclimate.com/github/arsduo/tokeneyes/badges/gpa.svg)](https://codeclimate.com/github/arsduo/tokeneyes)
+[![Test Coverage](https://codeclimate.com/github/arsduo/tokeneyes/badges/coverage.svg)](https://codeclimate.com/github/arsduo/tokeneyes/coverage)
+[![Build Status](https://travis-ci.org/arsduo/tokeneyes.svg)](https://travis-ci.org/arsduo/tokeneyes)
+
 # Tokeneyes
 
 A string tokenizer designed to capture words with associated punctuation and sentence flow

data/changelog.md

@@ -1,3 +1,8 @@
+v0.1.1
+======
+
+* Use sets instead of regular expressions to check character types, cutting processing time by >50%
+
 v0.1.0
 ======
 

data/lib/tokeneyes/version.rb

@@ -1,3 +1,3 @@
 module Tokeneyes
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/lib/tokeneyes/word_builder.rb

@@ -12,18 +12,28 @@ module Tokeneyes
     end
 
     # Definite word elements, those that can repeat as much as they want and always be words:
-    # alphanumeric characters (including European symbols, all the Unicode blocks). If anyone has expertise on non-European
+    # alphanumeric characters (including some European symbols). If anyone has expertise on non-European
     # languages, I would love to add support for other character groups.
-    # We include @ and # to support Twitter mentions, hashtags, and email addresses.
-    WORD_ELEMENTS = /[\w\d\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF\@\#]/
+    WORD_ELEMENTS = Set.new(
+      # Letters
+      ("A".."Z").to_a + ("a".."z").to_a +
+      # Numbers
+      ("0".."9").to_a +
+      # A subset of European characters
+      ("\u00C0".."\uD7FF").to_a + ("\u00D8".."\u00F6").to_a + ("\u00F8".."\u00FC").to_a +
+      # Hashtag, @mention, and email support -- this will need to be made more intelligent later
+      ["@", "#"]
+    )
+
     # Defines a word boundary that also ends a unit of text.
-    SENTENCE_BOUNDARY = /[\.;\?\!]/
+    SENTENCE_BOUNDARY = Set.new([".", ";", "?", "!"])
     # Possible word elements, those that mark a word boundary unless they're followed by a word
     # element:
-    POSSIBLE_WORD_ELEMENTS = /[\.'\-]/
+    POSSIBLE_WORD_ELEMENTS = Set.new([".", "'", "-"])
     # We don't track all possible punctuation, just some. (In particular, we don't track those that
     # come in pairs, like parentheses and brackets, etc.)
-    MEANINGFUL_PUNCTUATION = /[\.,\-;\!\?]/
+    # TODO add support for ellipses, interrobang, etc.
+    MEANINGFUL_PUNCTUATION = Set.new([".", ",", "-", ";", "!", "?"])
     # Everything else represents a word boundary.
 
     def word_finished?
@@ -50,11 +60,11 @@ module Tokeneyes
     # Which punctuation ended the word?
     def punctuation
       return nil unless word_finished?
-      punctuation_candidate if punctuation_candidate.match(MEANINGFUL_PUNCTUATION)
+      punctuation_candidate if MEANINGFUL_PUNCTUATION.include?(punctuation_candidate)
     end
 
     def sentence_ended?
-      !!(punctuation && punctuation.match(SENTENCE_BOUNDARY))
+      !!(punctuation && SENTENCE_BOUNDARY.include?(punctuation))
     end
 
     protected
@@ -69,18 +79,18 @@ module Tokeneyes
     end
 
     def current_char_is_word_element?
-      current_char.match(WORD_ELEMENTS)
+      WORD_ELEMENTS.include?(current_char)
     end
 
     def previous_character_was_possible_boundary?
       # it's not a possible word boundary if the word hasn't yet started
-      previous_char.match(POSSIBLE_WORD_ELEMENTS) && word_so_far.length > 0
+      POSSIBLE_WORD_ELEMENTS.include?(previous_char) && word_so_far.length > 0
     end
 
     def current_char_is_possible_boundary?
       # If the previous character was also a boundary, this one can't be as well -- we've ended the
       # word.
-      current_char.match(POSSIBLE_WORD_ELEMENTS) && !previous_character_was_possible_boundary?
+      POSSIBLE_WORD_ELEMENTS.include?(current_char) && !previous_character_was_possible_boundary?
     end
 
     def punctuation_candidate

data/tokeneyes.gemspec

@@ -19,8 +19,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.10"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "rspec", "~> 3.3"
   spec.add_development_dependency "faker"
 end

metadata

@@ -1,43 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: tokeneyes
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Alex Koppel
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-09-28 00:00:00.000000000 Z
+date: 2020-01-22 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.10'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.10'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -75,6 +61,8 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".ruby-gemset"
+- ".ruby-version"
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -110,8 +98,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 3.1.0.pre1
 signing_key: 
 specification_version: 4
 summary: A simple string tokenizer designed to capture punctuation and sentence flow