wordtree 0.3.1 → 0.4.0

data/lib/wordtree/text_utils.rb

@@ -1,107 +0,0 @@
-require 'strscan'
-
-module WordTree
-  module TextUtils
-    def self.split_near(text, split_index)
-      if split_index >= text.size
-        return [text, ""]
-      else
-        index = split_index
-        while index >= 0
-          if text[index] == ' '
-            return [text[0...index], text[(index+1)..-1]]
-          end
-          index -= 1
-        end
-        return [text[0...split_index], text[split_index..-1]]
-      end
-    end
-
-    # Remove punctuation an non-alphabetical characters from a text, and return
-    # a cleaned-up version wrapped at +wrap+ characters per line.
-    def self.clean_text(input, wrap=120)
-      join = nil
-      output = String.new
-      output_line = String.new
-
-      # Ignore non-UTF-8 characters
-      input = input.encode('UTF-8', :invalid => :replace, :undef => :replace).downcase
-
-      _0 = '0'.ord
-      _9 = '9'.ord
-      _a = 'a'.ord
-      _z = 'z'.ord
-      _A = 'A'.ord
-      _Z = 'Z'.ord
-      _dash = '-'.ord
-      _space = ' '.ord
-      _newline = "\n".ord
-      _period = '.'.ord
-      _question = '?'.ord
-
-      join_lines = false
-      just_added_space = false
-      just_added_period = false
-      line_length = 0
-      input.each_char do |c|
-        c = c.ord
-        # Change upper-case to lower-case
-        c -= 32 if (c >= _A && c <= _Z)
-        # Change newlines to spaces
-        c = _space if c == _newline
-        # Change question marks to periods (i.e. both count as sentence boundaries)
-        c = _period if c == _question
-
-        if c == _dash
-          # In case of a dash, set the scoop-spaces-up flag
-          join_lines = true
-        elsif join_lines && (c == _space)
-          # ignore
-        elsif (c == _period) && !just_added_period
-          if !just_added_space
-            output << _space.chr
-          end
-          output << c.chr
-          just_added_period = true
-          just_added_space = true
-        elsif (c >= _a && c <= _z) || (c == _space && !just_added_space)
-          # Add letters and spaces
-          output << _space.chr if just_added_period
-          output << c.chr
-          line_length += 1
-          just_added_space = (c == _space)
-          just_added_period = false
-          join_lines = false
-        end
-      end
-
-      wrapped_output = String.new
-      begin
-        output_line, remainder = split_near(output, wrap)
-        wrapped_output << output_line + "\n"
-        output = remainder
-      end while remainder.size > wrap
-      wrapped_output << remainder + "\n" unless remainder.empty?
-
-      return wrapped_output
-    end
-
-    def self.each_ngram(input, n=1, &block)
-      onegram_re = /([^ \n]+[ \n])/
-      ngram_re = /([^ \n]+[ \n]){#{n},#{n}}/
-      s = StringScanner.new(input)
-      while !s.eos?
-        if words = s.scan(ngram_re)
-          yield words.rstrip.tr("\n", " ") if block_given?
-          # Move back to beginning of n-word sequence
-          s.unscan
-        end
-        # Move forward one word
-        if !s.scan(onegram_re)
-          # if we can't find a word, let's try to recover by scanning one char at a time
-          s.scan(/./m)
-        end
-      end
-    end
-  end
-end

data/spec/wordtree/text_utils_spec.rb

@@ -1,89 +0,0 @@
-require 'spec_helper'
-require 'wordtree/text_utils'
-require 'timeout'
-
-describe WordTree::TextUtils do
-  context "#split_near" do
-    it "splits on spaces" do
-      line, rem = WordTree::TextUtils.split_near("it is near", 7)
-      expect(line).to eq("it is")
-      expect(rem).to eq("near")
-    end
-
-    it "removes a space if index lands on one" do
-      line, rem = WordTree::TextUtils.split_near("it is near", 5)
-      expect(line).to eq("it is")
-      expect(rem).to eq("near")
-    end
-
-    it "keeps the whole line if index is >= length of line" do
-      line, rem = WordTree::TextUtils.split_near("it is near", 10)
-      expect(line).to eq("it is near")
-      expect(rem).to eq("")
-
-      line, rem = WordTree::TextUtils.split_near("it is near", 11)
-      expect(line).to eq("it is near")
-      expect(rem).to eq("")
-    end
-
-    it "splits at the index anyway if no spaces are found" do
-      line, rem = WordTree::TextUtils.split_near("itisnear", 4)
-      expect(line).to eq("itis")
-      expect(rem).to eq("near")
-    end
-  end
-
-  context "#clean_text" do
-    it "wraps" do
-      sample_text = "This, [here]  is awesome, right"
-      cleaned = WordTree::TextUtils.clean_text(sample_text, 10)
-      expect(cleaned).to eq("this here\nis awesome\nright\n")
-
-      cleaned = WordTree::TextUtils.clean_text(sample_text, 15)
-      expect(cleaned).to eq("this here is\nawesome right\n")
-
-      cleaned = WordTree::TextUtils.clean_text(sample_text, 150)
-      expect(cleaned).to eq("this here is awesome right\n")
-    end
-
-    it "joins lines ending in -" do
-      sample_text = "What-\never\ndo you\n mean?"
-      cleaned = WordTree::TextUtils.clean_text(sample_text, 10)
-      expect(cleaned).to eq("whatever\ndo you\nmean .\n")
-    end
-
-    it "does not ignore sentence boundaries" do
-      sample_text = "This is a sentence. And so is this? Keep the dots."
-      cleaned = WordTree::TextUtils.clean_text(sample_text, 150)
-      expect(cleaned).to eq("this is a sentence . and so is this . keep the dots .\n")
-      cleaned = WordTree::TextUtils.clean_text(sample_text, 10)
-      expect(cleaned).to eq("this is a\nsentence .\nand so is\nthis .\nkeep the\ndots .\n")
-    end
-
-    it "compresses sentence boundary punctuation and spaces" do
-      sample_text = "words . . and.. stuff"
-      cleaned = WordTree::TextUtils.clean_text(sample_text, 150)
-      expect(cleaned).to eq("words . and . stuff\n")
-    end
-  end
-
-  context "#each_ngram" do
-    it "yields ngrams in succession" do
-      sample_text = "one word\n. two\n"
-      expect{ |b| WordTree::TextUtils.each_ngram(sample_text, 1, &b) }.to \
-        yield_successive_args("one", "word", ".", "two")
-      expect{ |b| WordTree::TextUtils.each_ngram(sample_text, 2, &b) }.to \
-        yield_successive_args("one word", "word .", ". two")
-      expect{ |b| WordTree::TextUtils.each_ngram(sample_text, 3, &b) }.to \
-        yield_successive_args("one word .", "word . two")
-    end
-
-    it "doesn't hang on unexpected input" do
-      sample_text = "one word\n. two \n"
-      Timeout.timeout(3) do
-        expect{ |b| WordTree::TextUtils.each_ngram(sample_text, 1, &b) }.to \
-          yield_successive_args("one", "word", ".", "two")
-      end
-    end
-  end
-end