wheeler 0.5.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d4596a6f0a690fab701f82468b023a03379c57df
+  data.tar.gz: ac02c86ac3aea93799b5b4a2ff3b75c0bae983be
+SHA512:
+  metadata.gz: 9f7ec98508653b28cac3369877b4916b8fbae51deb524b686a3f6f57641ec91051720c3e0d554bbc032b1c7de691e8be471ff10eb1b7908f670090a8fc1508e1
+  data.tar.gz: 04c1613827999da49e2aa60cad70fabd1c4e6bf2d2426f834ddd84c29b80dac3147fc6d288c0823841c7df4115373137749f369d99563da540cc32383a001250

data/.gitignore

@@ -0,0 +1,15 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# project files
+*.iml
+.idea
+
+.index

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.ruby-version

@@ -0,0 +1 @@
+2.1.7

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.2.3
+before_install: gem install bundler -v 1.10.6

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in wheeler.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Scott Pierce
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,88 @@
+# Wheeler
+
+This is a map/reduce proof of concept with a little bit of fun thrown in. 
+
+Wheeler is a naive Wheel of Fortune solver. It does this by indexing sampled
+text from where ever into every possible contiguous combination of words up
+to a max phrase length.
+
+The mapper in this project is `wheeler phrases`. Again, this is a naive program.
+The phrase parser don't use any NLP or grammar logic. It simply splits words
+by spaces, period, and double quotes. After splitting it outputs joined sets
+of those words in descending word count.
+
+The reducer is `wheeler reduce`. It build the phrase index into the `.index`
+directory inside this project. Maybe we'll make it an arg in the future.
+The index structure is:
+  1st word size/2nd word size/3rd word size/etc.../phrases
+
+So for the phrase: "I love cats". An index entry would be written into
+`.index/1/4/4/phrases`. The contents of phrases would be 'I LOVE CATS'
+
+After the index are built, the guess matching is almost trivial.
+
+1. receive a puzzle with underscores and known letters. For example: `_ ____ ___s`
+2. get the word counts of the puzzle. 1, 4, 4.
+3. find the phrases file based on the word counts. `.index/1/4/4/phrases`
+4. perform a simple pattern match in that file. `grep -e '. .... ....' .index/1/4/4/phrases`
+
+The `wheeler guess` script will do steps 2-4 automatically: `wheeler guess '_ ____ ___s'`
+
+The more texts indexed, the better chance of solving any given puzzle. This solver 
+is only as good as the index it compiles.
+
+## Build Index, Usage Step by Step
+
+    # Map text to phrases
+    $ wheeler phrases samples/adventure_of_the_speckled_band.txt 5 > tmp/phrases-unsorted.txt
+    
+    # Sort the mapped phrases
+    $ sort tmp/phrases-unsorted.txt > tmp/phrases-sorted.txt
+     
+    # Reduce to counts, this writes results to .index
+    $ wheeler reduce tmp/phrases-sorted.txt
+    
+    # view phrase indexes
+    $ find .index -name phrases
+    
+    # view contents of a phrase
+    $ less .index/9/8/4/9/3/phrases
+
+## Solve a Puzzle
+
+    $ wheeler guess '____ __ _____ES'
+    # grep --color=always -e '.... .. .....ES' .index/4/2/7/phrases
+    # BAND OF GYPSIES
+
+## Use a sample dictionary
+    # Map text to phrases
+    $ wheeler phrases samples/dict.txt 5 > tmp/phrases-unsorted.txt
+    
+    # Sort the mapped phrases
+    $ sort tmp/phrases-unsorted.txt > tmp/phrases-sorted.txt
+     
+    # Reduce to counts
+    $ wheeler reduce tmp/phrases-sorted.txt
+
+## Use full English Dictionary
+    
+    # Make the Index
+    $ curl http://www.gutenberg.org/ebooks/29765 > tmp/dictionary.txt
+    $ wheeler phrases tmp/dictionary.txt 5 > tmp/phrases-unsorted.txt
+    
+    # Sort the mapped phrases
+    $ sort tmp/phrases-unsorted.txt > tmp/phrases-sorted.txt
+     
+    # Reduce to counts
+    $ wheeler reduce tmp/phrases-sorted.txt
+    
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/ddrscott/wheeler.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "wheeler"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "pry"
+Pry.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/bin/wheeler

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'wheeler'
+require 'thor'
+
+class WheelerCLI < Thor
+  include Wheeler
+
+  desc 'phrases SRC NUM_WORDS', 'split SRC into phrases of NUM_WORDS'
+  def phrases(src, words_in_phrase)
+    File.open(src, 'r') do |f|
+      map_phrases(f, words_in_phrase.to_i)
+    end
+  end
+
+  desc 'index SRC built by phrases', 'places uniq phrases into .index directory based on word sizes'
+  def reduce(src)
+    File.open(src, 'r') do |f|
+      reduce_fs(f)
+    end
+  end
+
+  desc 'guess PUZZLE', 'guess PUZZLE. PUZZLE should use _ (underscore) or . (dot) for unknown letters'
+  def guess(puzzle)
+    super(puzzle)
+  end
+end
+
+WheelerCLI.start(ARGV)

data/lib/wheeler/version.rb

@@ -0,0 +1,3 @@
+module Wheeler
+  VERSION = '0.5.0'
+end

data/lib/wheeler.rb

@@ -0,0 +1,125 @@
+require 'wheeler/version'
+
+module Wheeler
+
+  INDEX_PATH = '.index'
+
+  def each_word(io, &block)
+    # break on spaces instead of \n
+    io.each_line(' ') do |line|
+      # alphas with quote and period. We'll use the period as a hint for phrasing
+      line.scan(/\b([[:word:]]+)\b/).each do |word, _|
+        block.call word.upcase
+      end
+    end
+  end
+
+  def each_phrase(io, max_words=4, &block)
+    words = []
+    each_word(io) do |word|
+      words << word
+      if word[-1] == '.'
+        # remove period
+        words[-1] = words[-1][0..-2]
+
+        cascade_words(words, &block)
+
+        # we can start a new phrase by resetting the words
+        words.clear
+      elsif words.length >= max_words
+        cascade_words(words, &block)
+      end
+    end
+    cascade_words(words, &block)
+  end
+
+  def cascade_words(words, &block)
+    len = words.size
+    len.times do |i|
+      block.call words[0..i]
+    end
+    words.shift
+  end
+
+  def word_sizes(words)
+    words.map{|m| m.size } * ' '
+  end
+
+  def remove_last_punctuation_if_needed(words)
+    last_char = words.last[-1]
+    if last_char == ',' or last_char == ';'
+      words[-1] = words[0..-2]
+    end
+    words
+  end
+
+  def map_phrases(io, max_words)
+    each_phrase(io, max_words) do |words|
+      row = [
+          word_sizes(words),
+          words * ' '
+       ]
+
+      puts row * '|'
+    end
+  end
+
+  # ensure the block is called at most once per duration
+  def throttle(duration=0.1, &block)
+    @last_time ||= Time.now
+    if @last_time and (Time.now - duration) > @last_time
+      block.call
+      @last_time = Time.now
+    end
+  end
+
+  def reduce_fs(io)
+    last_sizes = nil
+    texts = []
+
+    io.each_line do |line|
+
+      sizes, text = *line.split('|')
+
+      # write out when the size of the words changes
+      if last_sizes and sizes != last_sizes
+        write_sizes(last_sizes, texts)
+        texts.clear
+      end
+
+      text = text[0..-2]                                 # strip new line
+      throttle{$stderr.print "\e[0K#{sizes}|#{text}\r"}  # some debug output
+      if texts.last != text
+        texts << text           # text that matches the word size pattern
+      end
+
+      last_sizes = sizes
+    end
+
+    if texts.any?          # make sure we write out the remaining phrases
+      write_sizes(last_sizes, texts)
+    end
+  end
+
+  def write_sizes(sizes, texts)
+    idx_path = "#{INDEX_PATH}/#{sizes.gsub(' ', '/')}"
+    FileUtils.mkdir_p idx_path
+    File.open("#{idx_path}/phrases", 'w') { |f| f << texts.join("\n") }
+  end
+
+  # @param puzzle [String] Known letters in their position and `_` underscore the unknown letters
+  def guess(puzzle)
+    # split up into words and replace _ with dot
+    words = puzzle.split(/\s+/).map{|w| w.gsub('_', '.').upcase}
+
+    # Example: "_ ____ ____" should constuct `.index/1/4/4/phrases`
+    phrase_path = "#{INDEX_PATH}/#{words.map(&:size) * '/'}/phrases"
+
+    cmd = "grep --color=always -e '#{words * ' '}' #{phrase_path}"
+    puts cmd
+    puts `#{cmd}`
+    if $?.exitstatus == 1
+      puts "Phrases not found in #{phrase_path}"
+    end
+  end
+end