markovfun 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in markovfun.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Maria Pacana
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,34 @@
+# Markovfun
+
+This gem generates sentences from textfiles using trigrams.
+It is based on Alex Rudnick's Python Markov chain generator,
+<<<<<<< HEAD
+the code for which is [here](https://github.com/alexrudnick/hackerschool-demos/tree/master/ngrams).
+=======
+the code for which is (here)[https://github.com/alexrudnick/hackerschool-demos/tree/master/ngrams].
+>>>>>>> f8c8a08... Updated README
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'markovfun'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install markovfun
+
+## Usage
+
+Here's how you can generate a sentence from a text file.
+
+```
+sentences = Markovfun.get_sentences("bible.txt")
+counts = Markovfun.get_counts(sentences)
+probs = Markovfun.counts_to_probs(counts)
+Markovfun.sentence_from_probs_hash(probs)
+```

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/markovfun/version.rb

@@ -0,0 +1,3 @@
+module Markovfun
+  VERSION = "0.0.1"
+end

data/lib/markovfun.rb

@@ -0,0 +1,123 @@
+require "markovfun/version"
+require 'pry'
+
+module Markovfun
+
+  # Generates a sentence, given a file.
+  def self.sentence_from_file(filename)
+    sentences = get_sentences(filename)
+    counts = buildcounts(sentences)
+    probs = counts_to_probs(counts)
+    sentence_from_probs_hash(probs)
+  end
+
+  # Gets lines from a file.
+  def self.get_lines(filename)
+    file = File.open(filename, "r")
+    data = file.read
+    file.close
+    lines = data.split("\n")
+    lines.map! { |l| l.strip.split(" ") }
+  end
+
+  # Gets sentences from a file.
+  def self.get_sentences(filename)
+    file = File.open(filename, "r")
+    data = file.read
+    file.close
+    data.gsub!(/\n/, "")
+    data.gsub!(/"/,"")
+    sentences = data.split(".")
+    sentences.map! { |s| s.strip.split(" ").push(".") }
+    sentences.select! { |s| s[0].capitalize == s[0] }
+  end
+
+  # Returns a counts hash, given a list of sentences.
+  # The keys to the hash are all observed combinations of [prev2, prev1], 
+  # where prev2 and prev1 are the two previous words. 
+  # The values are hashes, in which the keys are words (cur) that have followed
+  # prev2 and prev1, and the values are the number of occurrences.
+  def self.buildcounts(sentences)
+    counts_hash = {}
+    sentences.each do |sent|
+      # nil denotes the beginnings and ends of sentences
+      sent = [nil, nil] + sent + [nil]
+      sent.zip(sent[1..-1], sent[2..-1]).each do |prev2, prev1, cur|
+        counts_hash[[prev2, prev1]] ||= {}
+        if !(counts_hash[[prev2, prev1]][cur])
+          counts_hash[[prev2, prev1]][cur] = 1
+        else
+          counts_hash[[prev2, prev1]][cur] += 1
+        end
+      end
+    end
+    counts_hash
+  end
+
+  # Generates a probability hash, given a counts hash.
+  # Similar to counts_hash, except containing the probability that a word
+  # follows two preceding words (as opposed to number of occurrences).
+  def self.counts_to_probs(counts_hash)
+    probs_hash = {}
+    counts_hash.each do |prev, cur_freq|
+      probs_hash[prev] ||= {}
+      cur_freq.each do |cur, freq|
+        prob = freq.to_f / cur_freq.values.reduce(:+)
+        probs_hash[prev][cur] = prob
+      end
+    end
+    probs_hash
+  end
+
+  # Generates a sample word, given a probability hash.
+  def self.sample_word(probs_hash)
+    score = rand
+    probs_hash.each do |word, prob|
+      return word if score < prob
+      score -= prob
+    end
+  end
+
+  # Generates a sample sentence, given a probability hash.
+  def self.sample_sentence(probs_hash)
+    prev2 = nil
+    prev1 = nil
+    out = []
+
+    while true
+      cur = sample_word(probs_hash[[prev2, prev1]])
+      if cur.nil?
+        return out
+      else
+        out << cur
+        prev2 = prev1
+        prev1 = cur
+      end
+    end
+  end
+
+  # Scores a sentence, depending on the likelihood that it occurs
+  # within a corpus.
+  def self.score_sentence(sent, probs)
+    total_surprise = 0
+    sent = [nil, nil] + sent + [nil]
+
+    sent.zip(sent[1..-1], sent[2..-1]).each do |prev2, prev1, cur|
+      total_surprise += -Math.log(probs[[prev2, prev1]][cur], 2)
+    end
+    total_surprise
+  end
+
+  # Generates a sentence from a probability hash.
+  def self.sentence_from_probs_hash(probs)
+    sent = []
+    while score_sentence(sent, probs) > 30 || sent.length < 4
+      sent = sample_sentence(probs)
+    end
+    puts "score: #{score_sentence(sent, probs)}"
+    sent = sent[0..-2].join(" ") + "."
+    puts sent
+    sent
+  end
+
+end

data/markovfun.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'markovfun/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "markovfun"
+  spec.version       = Markovfun::VERSION
+  spec.authors       = ["Maria Pacana"]
+  spec.email         = ["maria.pacana@gmail.com"]
+  spec.description   = %q{Generates sentences using markov chains!}
+  spec.summary       = %q{Generates sentences using markov chains!}
+  spec.homepage      = "https://github.com/mariapacana/markovfun"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+end

metadata

@@ -0,0 +1,87 @@
+--- !ruby/object:Gem::Specification
+name: markovfun
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Maria Pacana
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-12-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Generates sentences using markov chains!
+email:
+- maria.pacana@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/markovfun.rb
+- lib/markovfun/version.rb
+- markovfun.gemspec
+homepage: https://github.com/mariapacana/markovfun
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Generates sentences using markov chains!
+test_files: []
+has_rdoc: