marky_markov 0.1.0

data/README.md

@@ -0,0 +1,90 @@
+Marky Markov and the Funky Sentences
+====================================
+
+Marky Markov is a naïve experiment in Markov Chain generation implemented
+in Ruby. It can be used both from the command-line and as a library within your code.
+
+# Installation
+
+    gem install marky_markov
+    
+
+# Module Usage
+
+A basic usage of the TemporaryDictionary, which parses strings and files into a
+temporary dictionary that will not be saved to disk.
+
+    require 'marky_markov'
+    markov = MarkyMarkov::TemporaryDictionary.new
+    markov.parse_string "These words will be added to the temporary dictionary."
+    markov.parse_file "filename.txt"
+    puts markov.generate_n_words 50
+    puts markov.generate_n_words 3000
+    markov.clear!
+    
+Dictionary creates or opens a persistent dictionary at a location defined by its 
+initalizer, and will allow you to build and save a dictionary over multiple runs.
+to ensure existing files aren't overwritten, the system appends .mmd to the end
+of the dictionary name.
+
+    require 'marky_markov'
+    markov = MarkyMarkov::Dictionary.new('dictionary') # Saves/opens dictionary.mmd
+    markov.parse_file "ENV["HOME"]/Documents/largefileindocs.txt"
+    markov.parse_file "anotherfileyay.txt"
+    puts markov.generate_n_words 10
+    markov.save_dictionary! # Saves the modified dictionary/creates one if it didn't exist.
+
+If you want to delete a dictionary you call it upon the Dictionary class itself while
+passing in the filename/location.
+
+    MarkyMarkov::Dictionary.delete_dictionary!('dictionary')
+    
+
+# Command-Line Usage
+
+## Build a Dictionary 
+
+    marky_markov read textfile.txt
+
+to build your word probability dictionary. You can run the command
+on different files to continue adding to your dictionary file.
+
+
+## Say Some Words
+
+    marky_markov speak -w 30
+
+Will use the dictionary to create a 30 word long sentence. If no number
+is passed it will default to 200 words.
+
+## Temporary Dictionaries 
+
+    marky_markov speak -s other-file.txt -w 20
+
+Generates a temporary dictionary based on the source file passed to it
+and uses that to speak. Here we're loading other-file.txt and
+restricting the generated sentence to 20 words.
+
+## STDIN, Pipe Away!
+
+    echo "Hello, how are you" | marky_markov
+
+Marky-Markov is compatible with other STDIN/STDOUT command-line
+applications and can accept STDIN.
+
+    marky_markov listen "Bullfighting is difficult on the moon"
+
+You can also supply a string as an argument to generate the text with,
+though the results are nonsense without a substantial text base to work
+from.
+
+    Usage: marky_markov COMMAND [OPTIONS]
+        Commands:
+            speak: Generate Markov Chain sentence (default wordcount of 200)
+            listen [sentence]: Generate Markov Chain sentence from supplied string.
+            read [file]: Add words to dictionary from supplied text file
+        Options
+            -d, --dictionary FILE            Use custom dictionary location
+            -w, --wordcount NUMBER           Set number of words generated
+            -s, --source FILE                Generate and use temporary dictionary from source text
+            -h, --help                       Display this screen

data/bin/marky_markov

@@ -0,0 +1,86 @@
+#!/usr/bin/env ruby -i
+#A Markov Chain generator.
+
+require 'optparse'
+require 'marky_markov'
+
+#if __FILE__ == $0
+options = {}
+opt_parser = OptionParser.new do |opts|
+  opts.banner = "Usage: marky-markov COMMAND [OPTIONS]"
+  opts.separator ""
+  opts.separator "Commands:"
+  opts.separator "    speak: Generate Markov Chain sentence (default wordcount of 200)"
+  opts.separator "    listen [sentence]: Generate Markov Chain sentence from supplied string."
+  opts.separator "    read [file]: Add words to dictionary from supplied text file"
+  opts.separator ""
+  opts.separator "Options"
+
+  options[:dictionary] = "#{ENV["HOME"]}/.marky_markov_dictionary"
+  opts.on('-d', '--dictionary FILE', 'Use custom dictionary location') do |file|
+    options[:dictionary] = file
+  end
+
+  options[:wordcount] = 200
+  opts.on('-w', '--wordcount NUMBER', 'Set number of words generated') do |number|
+    options[:wordcount] = number.to_i
+  end
+
+  options[:source] = nil
+  opts.on('-s', '--source FILE',
+          'Generate and use temporary dictionary from source text') do |file|
+    options[:source] = file
+  end
+
+  options[:resetdictionary] = false
+  opts.on('--reset', "WARNING: Deletes default dictionary." ) do
+    options[:resetdictionary] = true
+  end
+
+  opts.on('-h', '--help', 'Display this screen') do
+    STDOUT.puts opt_parser
+    exit
+  end
+end
+
+opt_parser.parse!
+
+if options[:resetdictionary]
+  STDOUT.puts MarkyMarkov::Dictionary.delete_dictionary!(options[:dictionary])
+  exit
+end
+
+case ARGV[0]
+when "speak"
+  if options[:source]
+    markov = MarkyMarkov::TemporaryDictionary.new
+    markov.parse_file(options[:source])
+  else
+    unless File.exists?("#{options[:dictionary]}.mmd")
+      STDERR.puts "Dictionary file #{options[:dictionary]}}.mmd does not exist. Cannot generate sentence."
+      STDERR.puts "Please build a dictionary with read or use the --source option with speak."
+      exit(false)
+    end
+    markov = MarkyMarkov::Dictionary.new(options[:dictionary])
+  end
+  STDOUT.puts markov.generate_n_words(options[:wordcount])
+when "read"
+  source = ARGV[1] || options[:source]
+  markov = MarkyMarkov::Dictionary.new(options[:dictionary])
+  markov.parse_file(source)
+  markov.save_dictionary!
+  STDOUT.puts "Added #{source} to dictionary."
+when "listen"
+  markov = MarkyMarkov::TemporaryDictionary.new
+  markov.parse_string(STDIN.tty? ? ARGV[1] : STDIN.read)
+  STDOUT.puts markov.generate_n_words(options[:wordcount])
+else
+  unless STDIN.tty?
+    markov = MarkyMarkov::TemporaryDictionary.new
+    markov.parse_string(STDIN.read)
+    STDOUT.puts markov.generate_n_words(options[:wordcount])
+  else
+    STDOUT.puts opt_parser
+  end
+end
+#end

data/lib/marky_markov.rb

@@ -0,0 +1,85 @@
+#!/usr/bin/env ruby -i
+#A Markov Chain generator.
+
+require_relative 'marky_markov/persistent_dictionary'
+require_relative 'marky_markov/two_word_sentence_generator'
+
+module MarkyMarkov
+  VERSION = '0.1.0'
+
+  class TemporaryDictionary
+    # Create a new Temporary Markov Chain Dictionary and sentence generator for use.
+    # @example Create a new Temporary Dictionary.
+    #   markov = MarkyMarkov::TemporaryDictionary.new
+    # @return [Object] a MarkyMarkov::TemporaryDictionary object.
+    def initialize
+      @dictionary = TwoWordDictionary.new
+      @sentence = TwoWordSentenceGenerator.new(@dictionary)
+    end
+    # Parses a given file and adds the sentences it contains to the current dictionary.
+    #
+    # @example Open a text file and add its contents to the dictionary.
+    #   markov.parse_file "text.txt"
+    # @param [File] location the file you want to add to the dictionary.
+    def parse_file(location)
+      @dictionary.parse_source(location, true)
+    end
+    # Parses a given string and adds them to the current dictionary.
+    #
+    # @example Add a string to the dictionary.
+    #   markov.parse_string "I could really go for some Chicken Makhani."
+    # @param [String] sentence the sentence you want to add to the dictionary.
+    def parse_string(string)
+      @dictionary.parse_source(string, false)
+    end
+    # Generates a sentence/sentences of n words using the dictionary generated via
+    # parse_string or parse_file.
+    #
+    # @example Generate a 40 word long string of words.
+    #   markov.generate_n_words(40)
+    # @param [Int] wordcount the number of words you want generated.
+    # @return [String] the sentence generated by the dictionary.
+    def generate_n_words(wordcount)
+      @sentence.generate(wordcount)
+    end
+    # Clears the temporary dictionary's hash, useful for keeping
+    # the same dictionary object but removing the words it has learned.
+    #
+    # @example Clear the Dictionary hash.
+    #   markov.clear!
+    def clear!
+      @dictionary.dictionary.clear
+    end
+  end
+
+  class Dictionary < TemporaryDictionary
+    # Open (or create if it doesn't exist) a Persistent Markov Chain Dictionary
+    # and sentence generator for use.
+    #
+    # @example Create a new Persistent Dictionary object.
+    #   markov = MarkyMarkov::Dictionary.new("#{ENV["HOME"]}/markov_dictionary")
+    def initialize(location)
+      @dictionary = PersistentDictionary.new(location)
+      @sentence = TwoWordSentenceGenerator.new(@dictionary)
+    end
+    # Save the Persistent Dictionary file into JSON format for later use.
+    #
+    # @example Save the dictionary to disk.
+    #   markov.save_dictionary!
+    def save_dictionary!
+      @dictionary.save_dictionary!
+    end
+    # Class Method: Takes a dictionary location/name and deletes it from the file-system.
+    #
+    # @note To ensure that someone doesn't pass in something that shouldn't be deleted by accident,
+    #   the filetype .mmd is added to the end of the supplied arguement, so do not include the
+    #   extension when calling the method.
+    #
+    # @example Delete the dictionary located at '~/markov_dictionary.mmd'
+    #   MarkyMarkov::Dictionary.delete_dictionary!("#{ENV["HOME"]}/markov_dictionary")
+    # @param [String] location location/name of the dictionary file to be deleted.
+    def self.delete_dictionary!(location)
+      PersistentDictionary.delete_dictionary!(location)
+    end
+  end
+end

data/lib/marky_markov/one_word_dictionary.rb

@@ -0,0 +1,32 @@
+class OneWordDictionary
+  attr_accessor :dictionary
+  def initialize
+    @dictionary = {}
+  end
+
+  class FileNotFoundError < Exception
+  end
+
+  def open_source(source)
+    if File.exists?(source)
+      File.open(source, "r").read.split
+    else
+      raise FileNotFoundError.new("#{source} does not exist!")
+    end
+  end
+
+  def add_word(rootword, followedby)
+    @dictionary[rootword] ||= Hash.new(0)
+    @dictionary[rootword][followedby] ||= 0
+    @dictionary[rootword][followedby] += 1
+  end
+
+  def parse_source(source, file=true)
+    # Special case for last word in source file as it has no words following it.
+    contents = file ? open_source(source) : contents = source.split
+    contents.each_cons(2) do |first, second|
+      self.add_word(first, second)
+    end
+    @dictionary[(contents.last)] ||= Hash.new(0)
+  end
+end

data/lib/marky_markov/one_word_sentence_generator.rb

@@ -0,0 +1,35 @@
+class OneWordSentenceGenerator
+  def initialize(dictionary)
+    @dictionary = dictionary
+  end
+
+  def random_word
+    keys = @dictionary.dictionary.keys
+    keys[rand(keys.length)]
+  end
+
+  def weighted_random(lastword)
+    # If word has no words in its dictionary (last word in source text file)
+    # have it pick a random word to display instead.
+    @dictionary.dictionary.fetch(lastword, random_word)
+      total = @dictionary.dictionary[lastword].values.inject(:+)
+      return random_word if total.nil?
+
+      random = rand(total)+1
+      @dictionary.dictionary[lastword].each do |word, occurs|
+        random -= occurs
+        if random <= 0
+          return word
+        end
+      end
+  end
+
+  def generate(wordcount)
+    sentence = []
+    sentence << random_word
+    (wordcount-1).times do
+      sentence << weighted_random(sentence.last)
+    end
+    sentence.join(' ')
+  end
+end

data/lib/marky_markov/persistent_dictionary.rb

@@ -0,0 +1,36 @@
+require 'yajl'
+require_relative 'two_word_dictionary'
+
+class PersistentDictionary < TwoWordDictionary
+  def initialize(dictionary)
+    @dictionarylocation = "#{dictionary}.mmd"
+    self.open_dictionary
+  end
+
+  def open_dictionary
+    if File.exists?(@dictionarylocation)
+      File.open(@dictionarylocation,'r') do |f|
+        @dictionary = Yajl::Parser.parse(f)
+      end
+    else
+      @dictionary = {}
+    end
+  end
+
+  def save_dictionary!
+    json = Yajl::Encoder.encode(@dictionary)
+    File.open(@dictionarylocation, 'w') do |f|
+      f.puts json
+    end
+    true
+  end
+
+  def self.delete_dictionary!(dictionary=@dictionarylocation[0...-4])
+    mmd = "#{dictionary}.mmd"
+    if File.exists?(mmd)
+      File.delete(mmd)
+      "Deleted #{mmd}"
+    end
+    false
+  end
+end

data/lib/marky_markov/two_word_dictionary.rb

@@ -0,0 +1,11 @@
+require_relative 'one_word_dictionary'
+
+class TwoWordDictionary < OneWordDictionary
+  def parse_source(source, file=true)
+    contents = file ? open_source(source) : contents = source.split
+    contents.each_cons(3) do |first, second, third|
+       self.add_word("#{first} #{second}", third)
+    end
+    @dictionary[contents.last(2).join(' ')] ||= Hash.new(0)
+  end
+end

data/lib/marky_markov/two_word_sentence_generator.rb

@@ -0,0 +1,13 @@
+require_relative 'one_word_sentence_generator'
+
+class TwoWordSentenceGenerator < OneWordSentenceGenerator
+  def generate(wordcount)
+    sentence = []
+    sentence.concat(random_word.split)
+    (wordcount-1).times do
+      sentence.concat(weighted_random(sentence.last(2).join(' ')).split)
+    end
+    sentence.pop(sentence.length-wordcount)
+    sentence.join(' ')
+  end
+end

data/spec/marky_markov/one_word_dictionary_spec.rb

@@ -0,0 +1,45 @@
+require 'spec_helper.rb'
+
+describe OneWordDictionary do
+  before(:each) do
+    @textsource = "spec/test.txt"
+    @dict = OneWordDictionary.new
+    @dict.parse_source("Hello how are you doing today", false)
+    @stringdict = {"Hello"   => {"how"        => 1},
+                     "how"     => {"are"        => 1},
+                     "are"     => {"you"        => 1},
+                     "you"     => {"doing"      => 1},
+                     "doing"   => {"today"      => 1},
+                     "today"   => {} }
+    @textdict = {"The" => {"cat"=>1},
+                 "and" => {"chainsaws"=>1},
+                 "cat" => {"likes"=>1},
+           "chainsaws" => {},
+               "likes" => {"pie"=>1},
+                 "pie" => {"and"=>1} }
+  end
+
+  it "can open a file" do
+    @dict.open_source(@textsource).should_not be_nil
+  end
+
+  it "should give a FileNotFoundError if the file doesn't exist" do
+    expect { @dict.open_source("thisisntreal") }.to 
+    raise_error(OneWordDictionary::FileNotFoundError,"thisisntreal does not exist!")
+  end
+
+  it "can add a word to the dictionary" do
+    @dict.add_word("to", "be")
+    @dict.dictionary.should include("to" => {"be" => 1})
+  end
+
+  it "create a dictionary via parsing a text file" do
+    @dict.dictionary = {}
+    @dict.parse_source(@textsource)
+    @dict.dictionary.should eql(@textdict)
+  end
+
+  it "builds a one word dictionary properly" do
+    @dict.dictionary.should eql(@stringdict)
+  end
+end

data/spec/marky_markov/one_word_sentence_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper.rb'
+
+describe OneWordSentenceGenerator do
+  before(:each) do
+    @dict = OneWordDictionary.new
+    @dict.parse_source("Hello man how are you today", false)
+    @sentence = OneWordSentenceGenerator.new(@dict)
+  end
+
+  it "can pick a random word" do
+
+  end
+
+  it "can choose a weighted random word" do
+
+  end
+
+  it "will use a random word if the word does not exist" do
+
+  end
+
+  it "generates a sentence of the appropriate length" do
+    @sentence.generate(20).split.count.should eql(20)
+  end
+end

data/spec/marky_markov/persistent_dictionary_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+describe PersistentDictionary do
+  before do
+    @dict = PersistentDictionary.new("spec/textdict")
+    @dict.parse_source("spec/test.txt")
+  end
+
+  it "should be able to save a dictionary" do
+     @dict.save_dictionary!.should eql(true)
+end
+
+  it "should be able to load an existing dictionary" do
+    otherdict = PersistentDictionary.new("spec/textdictcompare")
+     @dict.dictionary.should eql(otherdict.dictionary)
+  end
+
+  after do
+    PersistentDictionary.delete_dictionary!("spec/textdict")
+  end
+end
+

data/spec/marky_markov/two_word_dict_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+describe TwoWordDictionary do
+  before(:each) do
+    @dict = TwoWordDictionary.new
+    @dict.parse_source("The cat likes pie and chainsaws", false)
+    @textsource = "spec/test.txt"
+    @stringdict = { "The cat"       => { "likes"     => 1},
+                    "cat likes"     => { "pie"       => 1 },
+                    "likes pie"     => {"and"        => 1 },
+                    "pie and"       => { "chainsaws" => 1 },
+                    "and chainsaws" => {} }
+    @textdict = {"The cat"          => {"likes"      => 1},
+                 "cat likes"        => {"pie"        => 1},
+                 "likes pie"        => {"and"        => 1},
+                 "pie and"          => {"chainsaws"  => 1},
+                 "and chainsaws"    => {}}
+  end
+
+  it "can add a word to the two-word dictionary" do
+    @dict.add_word("Zebras like", "kung-fu")
+    @dict.dictionary.should eql(@stringdict.merge( {"Zebras like" => {"kung-fu" => 1}} ))
+  end
+
+  it "create a two-word dictionary via parsing a text file" do
+    @dict.dictionary = {}
+    @dict.parse_source(@textsource)
+    @dict.dictionary.should eql(@textdict)
+  end
+end
+

data/spec/marky_markov/two_word_sentence_spec.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+
+describe TwoWordSentenceGenerator do
+  before(:each) do
+    @dict = TwoWordDictionary.new
+    @dict.parse_source("Hello man how are you today", false)
+    @sentence = TwoWordSentenceGenerator.new(@dict)
+  end
+
+  it "generates a sentence of the appropriate length" do
+    @sentence.generate(20).split.count.should eql(20)
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1 @@
+require 'marky_markov'

metadata

@@ -0,0 +1,79 @@
+--- !ruby/object:Gem::Specification
+name: marky_markov
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Matt Furden
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-02-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: &70100468290580 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *70100468290580
+description: ! "MarkyMarkov makes it easy to generate simply Markov Chains based upon
+  input from\n  either a source file or a string. While usable as a module in your
+  code it can also be called on\n  from the command line and piped into like a stanard
+  UNIX application."
+email: mfurden@gmail.com
+executables:
+- marky_markov
+extensions: []
+extra_rdoc_files:
+- README.md
+files:
+- README.md
+- bin/marky_markov
+- lib/marky_markov.rb
+- lib/marky_markov/one_word_dictionary.rb
+- lib/marky_markov/one_word_sentence_generator.rb
+- lib/marky_markov/persistent_dictionary.rb
+- lib/marky_markov/two_word_dictionary.rb
+- lib/marky_markov/two_word_sentence_generator.rb
+- spec/marky_markov/one_word_dictionary_spec.rb
+- spec/marky_markov/one_word_sentence_spec.rb
+- spec/marky_markov/persistent_dictionary_spec.rb
+- spec/marky_markov/two_word_dict_spec.rb
+- spec/marky_markov/two_word_sentence_spec.rb
+- spec/spec_helper.rb
+homepage: http://www.thefurd.com
+licenses: []
+post_install_message: 
+rdoc_options:
+- --charset=UTF-8
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: marky_markov
+rubygems_version: 1.8.15
+signing_key: 
+specification_version: 2
+summary: Simple Markov Chain generation available in the command-line
+test_files: []
+has_rdoc: