markov-generator 0.9.2 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6f9ad4cff30b3a1c3bd16cc670630eac85decc49
-  data.tar.gz: 072a76cedf44383d2d5f7e597eba57448d4ae15e
+  metadata.gz: 09d0c8e708f75e9c610108c700c74fb6f9db6dbc
+  data.tar.gz: 7173ff93d857e356b0149a33de3743392264cff5
 SHA512:
-  metadata.gz: c09572cbd3ff6938e2ff13b55b0a2b601a3cc4660eb96c14ff99371c33a2a38fb063fa91dc92cfd97af4ba2d3d6e43f69e4dc8c6869472198cd171f8409b6a3f
-  data.tar.gz: 527795ed935fe580d2c3c1a205696fd74e8b4b27d22c530e5289f727828a6ea9cc9f20ee8b0dcb1c095c30b4535f339bdd1314b12b51ba5da85db3005e94b6e4
+  metadata.gz: 53cf62b9c8ec50f55112fe94e6e55eac01e3ebb5ffcff56c7a896f148761ce681d4f6968bf42190bceecf8175ea8b58fa0a8af4f13806e42b1852e2cf667f17d
+  data.tar.gz: c3d757b5b3841b47ae2df2b7e0236fb03ca7fadf60556cad53a245727aadbbe808859f03855d2567a52d6c66f3d6d90d2bf1ff46f12d3607ef87608a382f4ff8

data/.ruby-gemset

@@ -0,0 +1 @@
+markov

data/.ruby-version

@@ -0,0 +1 @@
+ruby-2.2.4

data/VERSION

@@ -1 +1 @@
-0.9.2
+0.10.0

data/lib/markov.rb

@@ -0,0 +1,13 @@
+
+module Markov
+  
+  require 'markov/token'
+  require 'markov/parser'
+  require 'markov/generator'
+  
+  def generator(depth=3)
+    return Markov::Generator.new(depth)
+  end
+  
+  module_function :generator
+end

data/lib/markov/generator.rb

@@ -1,297 +1,197 @@
 
 require 'securerandom'
 
-module Markov
-  
-  class Token < Struct.new(:word, :kind)
-    # used as an internal structure to hold words etc
-  end
-  
-  class Generator
-    
-    attr_reader :depth
+class Markov::Generator
+
+  def initialize(depth)
+    @depth = depth
     
-    def initialize(depth=3)
-      @depth = depth
-      
-      @split_sentence = /(?<=[.?!])\s+/
-      @split_words = /([,.?!])|[\s]/
-      @replace_chars = /[„':;_"()]/
-      
-      @dictionary = {}
-      @start_words = {}
-      @unparsed_sentences = []
-      @tokens = []
-      srand
-    end
+    @dictionary = {}
+    @start_words = {}
+    @unparsed_sentences = []
+    @tokens = []
     
-    class FileNotFoundError < Exception # :nodoc:
-    end
+    srand
+  end
+  
+  def parse_text(source)
     
-    class EmptyDictionaryError < Exception # :nodoc:
-    end
+    parser = Markov::Parser.new
+    parser.load_text source
     
-    def parse_string(sentence)
-      add_unparsed_sentence sentence
-      parse_text
-    end
+    state = :start # :start, :word, :special, :stop
+    word_seq = []
     
-    def parse_source_file(source)
-      
-      if File.exists?(source)
-        sentences = File.open(source, "r").read.force_encoding(Encoding::UTF_8).split(@split_sentence)
-      else
-        raise FileNotFoundError.new("#{source} does not exist!")
-      end
-      
-      sentences.each do |sentence|
-        add_unparsed_sentence sentence
-      end
-      
-      parse_text
-      
-    end
-    
-    def generate_sentence(min_length=20)
-      if @dictionary.empty?
-        raise EmptyDictionaryError.new("The dictionary is empty! Parse a source file/string first!")
-      end
-      
-      tokens = []
-      complete_sentence = false
-      
-      # initialize
-      select_start_words.each {|w| tokens << w}
-      prev_token = tokens.last
-      
-      begin
-        token =  select_next_token tokens.last(@depth-1)
-        
-        if token.kind == :stop
-          token =  select_next_word tokens.last(@depth-1) if prev_token.kind == :special
-          tokens << token
-        elsif token.kind == :special
-          token =  select_next_word tokens.last(@depth-1) if prev_token.kind == :special
-          tokens << token
-        elsif token.kind == :noop
-          token = Token.new(".", :stop)
-          tokens[tokens.length-1] = token
-        else
-          tokens << token
-        end
+    begin
+      while token = parser.next_token
         
-        prev_token = token
-        
-        if token.kind == :stop
-          if tokens.size < min_length
-            select_start_words.each {|w| tokens << w}
-            prev_token = tokens.last
-          else
-            complete_sentence = true
+        if state == :start
+          word_seq << token
+          
+          # fill the array
+          (@depth-word_seq.size).times do
+            word_seq << parser.next_token
           end
+          
+          # need to store the words in both the dictionary 
+          # and the list of start words
+          add_to_start_words word_seq[0, @depth-1]
+          add_to_dictionary word_seq
+          
+          token = parser.next_token
+          state = :sentence
         end
         
-        # circuit-breaker
-        complete_sentence = true if tokens.size > min_length*2 
-      end until complete_sentence
-      
-      tokens_to_sentence tokens
-    end
-      
-    def dump_start_words
-      @start_words.keys.each do |words|
-        puts "#{words[0]},#{words[1]}"
-      end
-    end
-    
-    def dump_dictionary
-      @dictionary.keys.each do |words|
-        following = @dictionary[words]
-        sentence = "#{words[0]},#{words[1]},"
-        following.each do |s|
-          sentence << "#{s.word},"
+        if state == :sentence
+          # move the array one position
+          word_seq.slice!(0)
+          word_seq << token
+          
+          # add to the dictionary
+          add_to_dictionary word_seq
+          
+          # stop current sequence and start again
+          if token.kind == :stop
+            word_seq = []
+            state = :start
+          end  
         end
         
-        puts "#{sentence.slice(0,sentence.length-1)}"
       end
+    rescue => e
+      # nothing to rescue
+      puts e
     end
     
-    def dump_dictionary_stats
-      puts "Keys: #{@dictionary.keys.size}"
-      dist = {}
-      n = 0
-      @dictionary.keys.each do |words|
-        following = @dictionary[words]
-        size = following.size
-        if dist[size]
-          dist[size] = dist[size] + following.size
-        else
-          dist[size] = following.size
-        end
-        n = n + following.size
-      end
-      
-      dist.keys.sort.each do |s|
-        puts "BUCKET: #{s}\t=#{dist[s]} (#{((dist[s].to_f/n.to_f)*100).to_i}%)"
-      end
-      
-    end
-    
-    private
-    
-    def parse_text
-            
-      state = :start # :start, :word, :special, :stop
-      word_seq = []
-      
-      begin
-        while token = next_token
-          
-          if state == :start
-            word_seq << token
-            
-            # fill the array
-            (@depth-word_seq.size).times do
-              word_seq << next_token
-            end
-            
-            # need to store the words in both the dictionary 
-            # and the list of start words
-            add_to_start_words word_seq[0, @depth-1]
-            add_to_dictionary word_seq
-            
-            token = next_token
-            state = :sentence
-          end
-          
-          if state == :sentence
-            # move the array one position
-            word_seq.slice!(0)
-            word_seq << token
-            
-            # add to the dictionary
-            add_to_dictionary word_seq
-            
-            # stop current sequence and start again
-            if token.kind == :stop
-              word_seq = []
-              state = :start
-            end  
-          end
-          
-        end # end while
-        
-      rescue
-        # nothing to rescue
+  end # end parse_text
+  
+  def generate_sentence(min_length=20)
+    if @dictionary.empty?
+      raise EmptyDictionaryError.new("The dictionary is empty! Parse a source file/string first!")
+    end
+    
+    tokens = []
+    complete_sentence = false
+    
+    # initialize
+    select_start_words.each {|w| tokens << w}
+    prev_token = tokens.last
+    
+    begin
+      token =  select_next_token tokens.last(@depth-1)
+      
+      if token.kind == :stop
+        token =  select_next_word tokens.last(@depth-1) if prev_token.kind == :special
+        tokens << token
+      elsif token.kind == :special
+        token =  select_next_word tokens.last(@depth-1) if prev_token.kind == :special
+        tokens << token
+      elsif token.kind == :noop
+        token = Token.new(".", :stop)
+        tokens[tokens.length-1] = token
+      else
+        tokens << token
       end
       
-    end # end parse_text
-    
-    def next_token
+      prev_token = token
       
-      if @tokens.empty?
-        sentence = @unparsed_sentences.slice!(0)
-        if sentence
-          sentence.each do |word|
-            
-            if word.include?(",")
-              @tokens << Token.new(",", :special)
-            elsif word.include?("?")
-              @tokens << Token.new("?", :stop)
-            elsif word.include?("!")
-              @tokens << Token.new("!", :stop)
-            elsif word.include?(".")
-              @tokens << Token.new(".", :stop)
-            elsif word == ""
-              # skip blanks
-            else
-              @tokens << Token.new(word, :word)
-            end            
-          end
+      if token.kind == :stop
+        if tokens.size < min_length
+          select_start_words.each {|w| tokens << w}
+          prev_token = tokens.last
         else
-          @tokens = nil
+          complete_sentence = true
         end
       end
       
-      return @tokens.slice!(0) if @tokens
-      
-      @tokens = []
-      nil  
-    end # end next_token
+      # circuit-breaker
+      complete_sentence = true if tokens.size > min_length*2 
+    end until complete_sentence
     
-    def add_unparsed_sentence(sentence)
-      
-      sentence.gsub!(@replace_chars, "")
-      words = sentence.split(@split_words)
-      if words && !words.empty?
-        @unparsed_sentences << words
-      end
-      
-    end
-    
-    def add_to_start_words(tokens)
-      return if tokens[0].kind != :word
-      
-      tokens[0].word = tokens[0].word.capitalize
-      start_words = tokens_to_words tokens
-      
-      @start_words[start_words] ||= tokens
-      
-    end
-    
-    def add_to_dictionary(tokens)
-      token = tokens.last
-      return if token.word == ""
-      
-      key_words = tokens_to_words tokens[0, @depth-1]     
-      
-      @dictionary[key_words] ||= []
-      @dictionary[key_words] << token
+    tokens_to_sentence tokens
+  end #end generate_sentence
+  
+  def dump_startwords
+    @start_words.keys.each do |start_words|
+      puts "#{start_words}"
     end
-    
-    def tokens_to_words(tokens)
-      words = []
-      tokens.each do |t|
-        words << t.word
+  end
+  
+  def dump_dictionary
+    @dictionary.keys.each do |keys|
+      following = @dictionary[keys]
+      sentence = []
+      following.each do |word|
+        sentence << "#{word.to_s},"
       end
-      words
+      s = sentence.join(" ")
+      puts "#{keys} => #{s.slice(0,s.length-1)}"
     end
-
-    def tokens_to_sentence(tokens)
-      s = ""
-      tokens.each do |t|
-        if t.kind != :word
-          s << t.word
-        else
-          s << " " + t.word
-        end
-      end
+  end
+  
+  private
+  
+  def add_to_start_words(tokens)
+    return if tokens[0].kind != :word
     
-      s[1, s.length-1]
-    end
+    tokens[0].word = tokens[0].word.capitalize
+    start_words = tokens_to_words tokens
     
-    def select_start_words
-      @start_words[ @start_words.keys[random_number( @start_words.keys.length-1)]]
-    end
+    @start_words[start_words] ||= tokens
+  end
+  
+  def add_to_dictionary(tokens)
+    token = tokens.last
+    return if token.word == ""
     
-    def select_next_token(tokens)
-      token = @dictionary[ tokens_to_words(tokens)]
-      
-      return Token.new("X", :noop) if token == nil  
-      token[random_number(tokens.length-1)]
-    end
+    key_words = tokens_to_words tokens[0, @depth-1]     
     
-    def select_next_word(tokens)
-      token = nil
-      begin
-        token = select_next_token(tokens)
-      end until token.kind == :word
-      token
+    @dictionary[key_words] ||= []
+    @dictionary[key_words] << token
+  end
+  
+  def tokens_to_words(tokens)
+    words = []
+    tokens.each do |t|
+      words << t.word
     end
-    
-    def random_number(upper_limit)
-      (SecureRandom.random_number * upper_limit).to_i
+    words
+  end
+  
+  def tokens_to_sentence(tokens)
+    s = ""
+    tokens.each do |t|
+      if t.kind != :word
+        s << t.word
+      else
+        s << " " + t.word
+      end
     end
+  
+    s[1, s.length-1]
+  end
+  
+  def select_start_words
+    @start_words[ @start_words.keys[random_number( @start_words.keys.length-1)]]
+  end
+  
+  def select_next_token(tokens)
+    token = @dictionary[ tokens_to_words(tokens)]
+    
+    return Token.new("X", :noop) if token == nil  
+    token[random_number(tokens.length-1)]
+  end
+  
+  def select_next_word(tokens)
+    token = nil
+    begin
+      token = select_next_token(tokens)
+    end until token.kind == :word
+    token
+  end
+  
+  def random_number(upper_limit)
+    (SecureRandom.random_number * upper_limit).to_i
   end
   
-end
+end

data/lib/markov/parser.rb

@@ -0,0 +1,77 @@
+
+class Markov::Parser
+
+  def initialize
+    @split_sentence = /(?<=[.?!])\s+/
+    @split_words = /([,.?!])|[\s]/
+    @replace_chars = /[„':;_"()]/
+    
+    @unparsed_sentences = []
+    @tokens = []
+  end
+  
+  class FileNotFoundError < Exception # :nodoc:
+  end
+  
+  class EmptyDictionaryError < Exception # :nodoc:
+  end
+  
+  def load_text(source)
+    
+    if File.exists?(source)
+      sentences = File.open(source, "r").read.force_encoding(Encoding::UTF_8).split(@split_sentence)
+    else
+      raise FileNotFoundError.new("#{source} does not exist!")
+    end
+    
+    sentences.each do |sentence|
+      add_unparsed_sentence sentence
+    end
+    
+  end 
+  
+  def next_token
+    
+    if @tokens.empty?
+      sentence = @unparsed_sentences.slice!(0)
+      if sentence
+        sentence.each do |word|
+          
+          if word.include?(",")
+            @tokens << Markov::Token.new(",", :special)
+          elsif word.include?("?")
+            @tokens << Markov::Token.new("?", :stop)
+          elsif word.include?("!")
+            @tokens << Markov::Token.new("!", :stop)
+          elsif word.include?(".")
+            @tokens << Markov::Token.new(".", :stop)
+          elsif word == ""
+            # skip blanks
+          else
+            @tokens << Markov::Token.new(word, :word)
+          end            
+        end
+      else
+        @tokens = nil
+      end
+    end
+    
+    return @tokens.slice!(0) if @tokens
+    
+    @tokens = []
+    nil  
+  end # end next_token
+  
+  private
+  
+  def add_unparsed_sentence(sentence)
+    
+    sentence.gsub!(@replace_chars, "")
+    words = sentence.split(@split_words)
+    if words && !words.empty?
+      @unparsed_sentences << words
+    end
+    
+  end # add_unparsed_sentence
+  
+end

data/lib/markov/token.rb

@@ -0,0 +1,11 @@
+
+class Markov::Token < Struct.new(:word, :kind)
+  # used as an internal structure to hold words etc
+  #
+  # word => string
+  # kind => :start, :word, :special, :stop
+  
+  def to_s
+    "#{kind}(#{word})"
+  end
+end

data/markov-generator.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: markov-generator 0.9.2 ruby lib
+# stub: markov-generator 0.10.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "markov-generator"
-  s.version = "0.9.2"
+  s.version = "0.10.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Michael Kuehl"]
-  s.date = "2015-05-19"
+  s.date = "2016-01-11"
   s.description = "A Markov Chain text generator library"
   s.email = "hello@ratchet.cc"
   s.extra_rdoc_files = [
@@ -20,23 +20,27 @@ Gem::Specification.new do |s|
   ]
   s.files = [
     ".document",
+    ".ruby-gemset",
+    ".ruby-version",
     "Gemfile",
     "Gemfile.lock",
     "LICENSE.txt",
     "README.rdoc",
     "Rakefile",
     "VERSION",
+    "lib/markov.rb",
     "lib/markov/generator.rb",
+    "lib/markov/parser.rb",
+    "lib/markov/token.rb",
     "markov-generator.gemspec",
-    "test/file_parser_test.rb",
-    "test/file_parser_test.txt",
     "test/generator_test.rb",
-    "test/helper.rb",
-    "test/test_markov_generator.rb"
+    "test/test_bulk_markov.rb",
+    "test/test_markov.rb",
+    "test/test_parser.rb"
   ]
   s.homepage = "http://github.com/ratchetcc/markov-generator"
   s.licenses = ["MIT"]
-  s.rubygems_version = "2.4.5"
+  s.rubygems_version = "2.4.8"
   s.summary = "Markov Chain text generator"
 
   if s.respond_to? :specification_version then

data/test/generator_test.rb

@@ -5,8 +5,8 @@ markov = Markov::Generator.new
 markov.parse_source_file "./generator_test2.txt"
 markov.parse_source_file "./generator_test1.txt"
 
-markov.dump_dictionary
-markov.dump_start_words
+#markov.dump_dictionary
+#markov.dump_start_words
 markov.dump_dictionary_stats
 
 1..5.times do

data/test/test_bulk_markov.rb

@@ -0,0 +1,19 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+
+require 'markov'
+
+source_dir = ARGV[0]
+
+markov = Markov.generator(3)
+
+Dir["#{source_dir}/*.txt"].each do | f |
+  puts "*** Analyzing '#{f}' "
+  markov.parse_text f
+end
+
+#markov.dump_startwords
+#markov.dump_dictionary
+
+1..5.times do
+  puts "\n#{markov.generate_sentence}"
+end

data/test/test_markov.rb

@@ -0,0 +1,13 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+
+require 'markov'
+
+source = ARGV[0]
+
+markov = Markov.generator(3)
+markov.parse_text source
+
+#markov.dump_startwords
+markov.dump_dictionary
+
+puts "#{markov.generate_sentence}"

data/test/test_parser.rb

@@ -0,0 +1,13 @@
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+
+require 'markov'
+
+source = ARGV[0]
+
+parser = Markov::Parser.new
+parser.load_text source
+
+while token = parser.next_token
+  puts token
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: markov-generator
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 0.10.0
 platform: ruby
 authors:
 - Michael Kuehl
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-19 00:00:00.000000000 Z
+date: 2016-01-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: shoulda
@@ -89,19 +89,23 @@ extra_rdoc_files:
 - README.rdoc
 files:
 - ".document"
+- ".ruby-gemset"
+- ".ruby-version"
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
 - README.rdoc
 - Rakefile
 - VERSION
+- lib/markov.rb
 - lib/markov/generator.rb
+- lib/markov/parser.rb
+- lib/markov/token.rb
 - markov-generator.gemspec
-- test/file_parser_test.rb
-- test/file_parser_test.txt
 - test/generator_test.rb
-- test/helper.rb
-- test/test_markov_generator.rb
+- test/test_bulk_markov.rb
+- test/test_markov.rb
+- test/test_parser.rb
 homepage: http://github.com/ratchetcc/markov-generator
 licenses:
 - MIT
@@ -122,7 +126,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: Markov Chain text generator

data/test/file_parser_test.rb

@@ -1,25 +0,0 @@
-
-require 'markov/generator'
-
-#markov = Markov::Generator.new
-#markov.parse_source_file "./test_seed.txt"
-
-#puts "#{markov.generate_sentence}"
-
-#@split_words = /([',.?!\n-])|[\s]+/
-#@split_sentence = /(?<=[.!?\n])\s+/
-
-split_sentence = /(?<=[.?!])\s+/
-split_words = /([,.?!])|[\s]/
-replace_chars = /[„':;_"()]/
-
-#source = "./file_parser_test.txt"
-source = "./seed_alts1.txt"
-
-sentences = File.open(source, "r").read.force_encoding(Encoding::UTF_8).split(split_sentence)
-
-sentences.each do |sentence|
-  puts sentence
-  puts sentence.gsub!( replace_chars, "")
-  puts "#{sentence.split(split_words)}"
-end

data/test/file_parser_test.txt

@@ -1,11 +0,0 @@
-   The most merciful thing in the world, I think, is the inability of the human mind to correlate all its contents. We live on a placid island of
-   ignorance, in the midst of black seas of infinity-and it was not meant that we should voyage far. 
-   
-   The_sciences, each straining in - its own
-   direction, "have", hitherto harmed us little; but (some day) the piecing
-   together of dissociated "knowledge" will open up such terrifying vistas of
-   reality, and of our frightful 'position' therein, that we shall either go
-   mad from the revelation or flee from the light into the peace and safety
-   of a new dark age.
-   
-   Where do we go? Nowhere! she said.

data/test/helper.rb

@@ -1,34 +0,0 @@
-require 'simplecov'
-
-module SimpleCov::Configuration
-  def clean_filters
-    @filters = []
-  end
-end
-
-SimpleCov.configure do
-  clean_filters
-  load_adapter 'test_frameworks'
-end
-
-ENV["COVERAGE"] && SimpleCov.start do
-  add_filter "/.rvm/"
-end
-require 'rubygems'
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
-require 'test/unit'
-require 'shoulda'
-
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-require 'markov_generator'
-
-class Test::Unit::TestCase
-end

data/test/test_markov_generator.rb

@@ -1,7 +0,0 @@
-require 'helper'
-
-class TestMarkovGenerator < Test::Unit::TestCase
-  should "probably rename this file and start testing for real" do
-    flunk "hey buddy, you should probably rename this file and start testing for real"
-  end
-end