markovian 0.2.0 → 0.2.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1f03bf82f92106eab96196be22cb2e09eb3d022b
-  data.tar.gz: fe37c3945b518f64d617be2758aef29a5dbfc52a
+  metadata.gz: 3c30d2a0dcf0c8488bab6aa456ecd5a761fa04ae
+  data.tar.gz: c4900a0c75636c29e33697d9ab3bcff85bb13828
 SHA512:
-  metadata.gz: 219a602a1d41dc3aaff820ad616d0690b0a42810f40e83e08a219537f6ba765855b2b8008e4af3fe0537570040d86b0f499222a3a9e5815a44a827dfadfe418e
-  data.tar.gz: d00c3cf9bc069f775edf0b496b5ccf79800129a50dbe3d0069b4401294393561cedbb9f3606a855ffa6081e65cf0cd3272bf3861f777d4cd8883dc9353a9a9e7
+  metadata.gz: 01e05e4c15feb7a2938603a6ef7fda354871d1474e493701475d5f1600498633c525982b9ca4b210b22303b6d18effe0c1d29412ee5fb201218cf4a0bd8f2a83
+  data.tar.gz: 99ff58a5fa7b60a0bfdd69cf450ba650e4958e6047ed18dd07ea8c6e211974c696d87fb671a46103257935b7d36d60fe7d6b25f9ac21bf7c8f387e5e8c4d6066

data/README.md

@@ -35,6 +35,13 @@ After checking out the repo, run `bin/setup` to install dependencies. You can al
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
+## Related Projects
+
+* [tokeneyes](https://github.com/arsduo/tokeneyes): a tokenizing system used by Markovian that
+  provides info on punctuation and sentence flow (to be incorporated shortly)
+* [markovian-lambda](https://github.com/arsduo/markovian-lambda): an inchoate set of utility
+  classes for using Markovian, eventually with AWS Lambda.
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/arsduo/markovian. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](contributor-covenant.org) code of conduct.
@@ -42,4 +49,3 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/arsduo
 ## License
 
 The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
-

data/changelog.md

@@ -1,5 +1,12 @@
 # CHANGELOG
 
+## 0.2.9
+
+Internal refactors only, no new functionality.
+
+* Refactor Dictionary to use DictionaryEntry objects, which can store additional metadata
+* Use Tokeneyes to parse strings rather than the original String#split-based TextSplitter
+
 ## 0.2.0
 
 * Rename Chainset/Chain to Corpus (better name is better)

data/lib/markovian/corpus/chain.rb

@@ -11,7 +11,6 @@ module Markovian
         @two_key_dictionary = Dictionary.new
       end
 
-      attr_reader :one_key_dictionary, :two_key_dictionary
       def lengthen(word, next_word:, previous_word: nil)
         @one_key_dictionary.push(word, next_word)
         @two_key_dictionary.push(two_word_key(previous_word, word), next_word)
@@ -33,6 +32,9 @@ module Markovian
 
       protected
 
+      # for equality checking
+      attr_reader :one_key_dictionary, :two_key_dictionary
+
       def result_for_two_words(previous_word, word)
         @two_key_dictionary.next_word(two_word_key(previous_word, word)) if previous_word
       end
@@ -49,4 +51,3 @@ module Markovian
     end
   end
 end
-

data/lib/markovian/corpus/dictionary.rb

@@ -1,17 +1,21 @@
-# This class represents a dictionary of words or phrases and the various words that can follow
-# them.  Currently it's implemented as a hash of arrays, but more advanced representations may
-# follow.
+require 'markovian/corpus/dictionary_entry'
 #
-# The key is an opaque value, which could represent either a single word or a phrase as desired.
+# This class represents a dictionary of words or phrases and the various words that can follow
+# them. The key is an opaque value, which could represent either a single word or a phrase as desired.
 module Markovian
   class Corpus
     class Dictionary
-      def push(key, word)
-        dictionary[key] += [word]
+      def push(key, word, direction: :forwards)
+        # Incoming we get a Tokeneyes::Word object
+        dictionary[key.to_s].push(word, direction: direction)
       end
 
       def next_word(key)
-        dictionary[key].sample
+        dictionary[key].next_word
+      end
+
+      def previous_word(key)
+        dictionary[key].previous_word
       end
 
       def random_word
@@ -32,7 +36,9 @@ module Markovian
       protected
 
       def dictionary
-        @dictionary ||= Hash.new([])
+        # We have to set the value of the hash in the block, otherwise it doesn't actually seem to
+        # get saved properly. Default hash values behave weirdly in general.
+        @dictionary ||= Hash.new {|hash, key| hash[key] = DictionaryEntry.new(key)}
       end
     end
   end

data/lib/markovian/corpus/dictionary_entry.rb

@@ -0,0 +1,54 @@
+module Markovian
+  class Corpus
+    class DictionaryEntry
+      attr_reader :word, :count
+      def initialize(word)
+        @word = word
+        @next_words = []
+        @previous_words = []
+        @count = 0
+      end
+
+      def push(word, direction: :forwards)
+        # The incoming word will be a Tokeneyes::Word object
+        array_for_direction(direction) << word.to_s
+        # we don't want to double-count words if we read the text both forward and backward, so
+        # only count in the forward direction. (If we encounter a scenario where someone only wants
+        # to read in the backward direction, we can deal with that then.)
+        @count += 1 if direction == :forwards
+      end
+
+      def next_word
+        next_words.sample
+      end
+
+      def previous_word
+        previous_words.sample
+      end
+
+      def ==(other)
+        self.word == other.word &&
+          self.next_words == other.next_words &&
+          self.previous_words == other.previous_words
+      end
+
+      protected
+
+      # for equality checking
+      attr_reader :next_words, :previous_words
+
+      VALID_DIRECTIONS = [:backwards, :forwards]
+
+      def array_for_direction(direction)
+        validate_direction(direction)
+        direction == :backwards ? previous_words : next_words
+      end
+
+      def validate_direction(direction)
+        unless VALID_DIRECTIONS.include?(direction)
+          raise ArgumentError.new("Invalid direction #{direction.inspect}, valid directions are #{VALID_DIRECTIONS.inspect}")
+        end
+      end
+    end
+  end
+end

data/lib/markovian/text_builder.rb

@@ -58,7 +58,8 @@ module Markovian
     end
 
     def split_seed_text(seed_text)
-      Utils::TextSplitter.new(seed_text).components
+      # We get back Tokeneyes::Word objects, but for now only care about the strings within
+      Utils::TextSplitter.new(seed_text).components.map(&:to_s)
     end
   end
 end

data/lib/markovian/utils/text_splitter.rb

@@ -1,3 +1,5 @@
+require 'tokeneyes'
+
 module Markovian
   module Utils
     class TextSplitter
@@ -6,29 +8,17 @@ module Markovian
         @text = text
       end
 
-      # We split on spaces, quotes, (various symbols followed by either another dash, a space,
-      # another dot, or the end of the text), or (colons preceded by space or the beginning of the
-      # text).
-      # We don't want to block things like Jones-Smith, tl;dr, abc.def, or it's.
-      # Any of the following:
-      # [\s\(\)] - a space or parentheses on their own
-      # " - a quote on its own
-      # [\.-:;\?\!]([-\.\s]|$) - a period, dash, ?, or ! followed by a space, period, dash, or the
-      #                          end of the word
-      # [\s^]' - a single ' following a non-letter
-      WORD_DELIMITERS = /([\s\(\)]|"|[\.\-:;\?\!]([\-\.\s]|$)|[\s^]')/
-
       # anything that doesn't contain any letters is not a word we need to care about
       MARKERS_OF_INTEREST = /[A-Za-z]/
 
       def components
-        split_text.select {|t| t.match(MARKERS_OF_INTEREST)}
+        split_text.select {|w| w.text.match(MARKERS_OF_INTEREST)}
       end
 
       protected
 
       def split_text
-        text.downcase.split(WORD_DELIMITERS)
+        Tokeneyes::Tokenizer.new(text.downcase).parse_into_words
       end
     end
   end

data/lib/markovian/version.rb

@@ -1,3 +1,3 @@
 module Markovian
-  VERSION = "0.2.0"
+  VERSION = "0.2.9"
 end

data/markovian.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.add_runtime_dependency "tokeneyes", "~> 0.1.0"
   spec.add_development_dependency "bundler", "~> 1.7"
   spec.add_development_dependency "rake", "~> 10.0"
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: markovian
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.9
 platform: ruby
 authors:
 - Alex Koppel
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-09-14 00:00:00.000000000 Z
+date: 2015-09-28 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: tokeneyes
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -65,6 +79,7 @@ files:
 - lib/markovian/corpus/chain.rb
 - lib/markovian/corpus/compiler.rb
 - lib/markovian/corpus/dictionary.rb
+- lib/markovian/corpus/dictionary_entry.rb
 - lib/markovian/importers/twitter/csv_importer.rb
 - lib/markovian/importers/twitter/tweet.rb
 - lib/markovian/text_builder.rb