correct-horse-battery-staple 0.6.1

data/lib/correct_horse_battery_staple/generator.rb

@@ -0,0 +1,40 @@
+
+require 'securerandom'
+
+#
+# Generate an N-word passphrase from a corpus
+#
+class CorrectHorseBatteryStaple::Generator
+  include CorrectHorseBatteryStaple::Common
+  include CorrectHorseBatteryStaple::Memoize
+
+  attr_accessor :word_length, :corpus
+
+  def initialize(corpus, word_length = nil)
+    @corpus      = corpus
+    if word_length
+      @corpus.filter {|entry| word_length.include?(entry.word.to_s.length) }
+    end
+  end
+
+  def make(count = 4, options = {})
+    @corpus.pick(count, options).
+      map {|entry| entry.word.downcase }.
+      join("-")
+  end
+
+  def estimate_entropy(options)
+    candidate_count = @corpus.count_candidates(options)
+    (log(candidate_count) / log(2)).floor
+  end
+  memoize :estimate_entropy
+
+  def words
+    @words ||= @corpus.result
+  end
+end
+
+if __FILE__ == $0
+  puts CorrectHorseBatteryStaple::Generator.new(CorrectHorseBatteryStaple.default_corpus, 3..6).
+    make((ARGV[0] || 4).to_i)
+end

data/lib/correct_horse_battery_staple/memoize.rb

@@ -0,0 +1,25 @@
+module CorrectHorseBatteryStaple::Memoize
+  def self.included(base)
+    base.extend ClassMethods
+  end
+
+  module ClassMethods
+    def memoize(method)
+      old_method = "_#{method}_unmemoized".to_sym
+      miss_object = Object.new
+      alias_method old_method, method
+      define_method method do |*args, &block|
+        @_memoize_cache ||= {}
+        methcache = (@_memoize_cache[method] ||= {})
+        if block
+          raise ArgumentError, "You cannot call a memoized method with a block! #{method}"
+        end
+        value = methcache.fetch(args, miss_object)
+        if value === miss_object
+          value = methcache[args] = send(old_method, *args)
+        end
+        value
+      end
+    end
+  end
+end

data/lib/correct_horse_battery_staple/parser.rb

@@ -0,0 +1,5 @@
+class CorrectHorseBatteryStaple::Parser
+  autoload :Base,'correct_horse_battery_staple/parser/base'
+  autoload :Regex,'correct_horse_battery_staple/parser/regex'
+end
+

data/lib/correct_horse_battery_staple/parser/base.rb

@@ -0,0 +1,5 @@
+class CorrectHorseBatteryStaple::Parser
+  class Base < CorrectHorseBatteryStaple::Parser
+    include CorrectHorseBatteryStaple::Common
+  end
+end

data/lib/correct_horse_battery_staple/parser/regex.rb

@@ -0,0 +1,58 @@
+
+class CorrectHorseBatteryStaple::Parser
+  class Regex < Base
+    PARSERS  = {
+        :wiktionary   =>  [%r{<a href="/wiki/\w+" title="(\w+)">\w+</a> = (\d+)},
+          lambda {|match| CorrectHorseBatteryStaple::Word.new(:word => match[0], :frequency => match[1].to_i) }],
+
+        # rank	lemma	PoS	freq	dispersion
+        # 7	to	t	6332195	0.98
+        :wordfrequency => [ %r{^(\d+)\s+(\w+)\s+\w*\s+(\d+)\s+([0-9.]+)},
+        lambda {|match| CorrectHorseBatteryStaple::Word.new(:word => match[1],
+            :rank => match[0].to_f,
+            :frequency => match[3].to_f,
+            :dispersion => match[4].to_f)
+        }],
+
+        # using tabs between columns
+        # freq    word    PoS     # texts
+        # -----   -----   -----   -----
+        # 22995878        the     at      169011
+        # 11239776        and     cc      168844
+        :coca => [ %r{^(\d+)\s+(\w+)\s+\w*\s+(\d+)},
+        lambda {|match| CorrectHorseBatteryStaple::Word.new(:word => match[1],
+            :frequency => match[0].to_i,
+            :texts => match[2].to_i)
+        }],
+
+      # <tr>
+      # <td>25</td>
+      # <td><a href="/wiki/be" title="be">be</a></td>
+      # <td>191823</td>
+      # </tr>
+      :tvscripts => [
+        Regexp.new('<tr>.*?<td>(\d+)</td>.*?<td>.*?title="(\w+)".*?</td>.*?<td>(\d+)</td>.*?</tr>', Regexp::MULTILINE),
+        lambda {|match| CorrectHorseBatteryStaple::Word.new(
+            :rank => match[0].to_i,
+            :word => match[1],
+            :frequency => match[2].to_i
+            ) }
+      ]
+    }
+
+    def initialize(type = :wiktionary)
+      @parser_type = type.to_sym
+    end
+
+    def parse(file)
+      raise ArgumentError, "unknown regex parser type #{@parser_type}" unless PARSERS.has_key?(@parser_type)
+      (regex, lexer) = PARSERS[@parser_type]
+
+      words = 
+        file.read.scan(regex).map do |match|
+          lexer.call(match)
+        end
+
+    end
+  end
+end

data/lib/correct_horse_battery_staple/range_parser.rb

@@ -0,0 +1,29 @@
+# doesn't handle X...Y
+
+class CorrectHorseBatteryStaple::RangeParser
+  NUM = '-?(?:\.[0-9]+|[0-9]+|[0-9]+\.[0-9]+|[0-9]+\.(?!\.))'
+  SPACE = " *"
+  SEPARATOR = "(-|\\.\\.)"
+  REGEX_PAIR = Regexp.new("(#{NUM})#{SPACE}#{SEPARATOR}#{SPACE}(#{NUM})")
+  REGEX_SINGLE = Regexp.new("#{SPACE}(#{NUM})#{SPACE}")
+  def parse(string)
+    match = string.match(REGEX_PAIR)
+    if match
+      return Range.new(parse_number(match[1]), parse_number(match[3]))
+    end
+
+    match = string.match(REGEX_SINGLE)
+    if match
+      num = parse_number(match[0])
+      return Range.new(num, num)
+    end
+
+    nil
+  end
+
+  protected
+
+  def parse_number(str)
+    str.include?(".") ? str.to_f : str.to_i
+  end
+end

data/lib/correct_horse_battery_staple/statistical_array.rb

@@ -0,0 +1,74 @@
+module CorrectHorseBatteryStaple
+
+  class StatisticalArray
+    def initialize(array, sorted=false)
+      @obj    = array
+      @sorted = sorted
+    end
+
+    def self.cast(array, sorted=false)
+      if array.is_a?(CorrectHorseBatteryStaple::StatisticalArray)
+        array
+      else
+        CorrectHorseBatteryStaple::StatisticalArray.new(array, sorted)
+      end
+    end
+
+    def sort!
+      @obj    = @obj.sort unless @sorted
+      @sorted = true
+      self
+    end
+
+    def sort_by!(&block)
+      @obj     = @obj.sort_by(&block)
+      @sorted  = true
+      self
+    end
+
+    def method_missing(name, *args, &block)
+      @obj.__send__(name, *args, &block)
+    end
+
+    def mean
+      inject(0) { |sum, x| sum += x } / size.to_f
+    end
+    alias :average :mean
+
+    def sum
+      reduce(:+)
+    end
+
+    def sort(&block)
+      return super(&block) if block || !@sorted
+      self
+    end
+
+    def standard_deviation(m = mean)
+      variance = inject(0) { |v, x| v += (x - m) ** 2 }
+      return Math.sqrt(variance/(size-1))
+    end
+
+    def mean_and_standard_deviation
+      return m=mean, standard_deviation(m)
+    end
+
+    def percentile_index(percentile, round=true)
+      r = percentile.to_f/100 * length + 0.5
+      round ? r.round : r
+    end
+
+    def index_range_for_percentile(range)
+      range = Range.new(range - 0.5, range + 0.5) if range.is_a?(Numeric)
+      sort!
+
+      (percentile_index(range.begin, false).floor ..
+          percentile_index(range.end,   false).ceil)
+    end
+
+    def select_percentile(range)
+      slice(index_range_for_percentile(range))
+    end
+
+  end
+end

data/lib/correct_horse_battery_staple/stats.rb

@@ -0,0 +1,22 @@
+require 'ostruct'
+require 'json'
+
+class CorrectHorseBatteryStaple::Stats < OpenStruct
+  def to_hash
+    marshal_dump
+  end
+
+  def self.from_hash(hash)
+    new.tap do |newobj|
+      marshal_load(hash)
+    end
+  end
+
+  def to_json
+    to_hash.to_json
+  end
+
+  def self.from_json(json)
+    from_hash JSON.parse(json)
+  end
+end

data/lib/correct_horse_battery_staple/word.rb

@@ -0,0 +1,90 @@
+class CorrectHorseBatteryStaple::Word
+  # text of word
+  attr_accessor :word
+
+  # frequency is the total count of the word in corpus
+  attr_accessor :frequency
+
+  # rank is the word position when sorted by frequency in entire corpus
+  # index is the index of the word in this (sub)corpus
+  attr_accessor :rank, :index
+
+  # dispersion is Juilland dispersion, the % of texts containing the
+  # word. texts is the # of texts containing the word.
+  attr_accessor :dispersion
+
+  # texts is the # of texts containing the word. this is not available
+  # for many frequency lists.
+  attr_accessor :texts
+
+  ## statistical measure of word position in sorted frequency list
+
+  # in which percentile does the word appear. this can be calculated
+  # from the array of words so is somewhat redundant here
+  attr_accessor :percentile
+
+  # this word's frequency's distance from mean frequency in stddevs;
+  # signed.
+  attr_accessor :distance
+
+  # probability is the chance of any given word in a text composed
+  # of the sum of (word*frequency) in the corpus being this word.
+  attr_accessor :probability
+
+  # distance_probability is the distance of this word's probability
+  # from the mean in stddev
+  attr_accessor :distance_probability
+
+  include Comparable
+
+  def initialize(value_map = {})
+    raise ArgumentError, "Must supply at least :word" unless value_map[:word] || value_map["word"]
+
+    # phasing this out
+    self.index = -1
+
+    case value_map
+      when Hash then update_from_hash(value_map)
+      when CorrectHorseBatteryStaple::Word then update_from_hash(value_map.to_hash)
+      else raise "Can't initialize Word from #{value_map.inspect}"
+    end
+  end
+
+  def <=>(other)
+    self.frequency <=> other.frequency
+  end
+
+  def to_json(*args)
+    to_hash.to_json(*args)
+  end
+
+  def to_s
+    self.word
+  end
+
+  def inspect
+    "CHBS::Word(#{self.to_hash.inspect})"
+  end
+
+  def to_hash
+    instance_variables.reduce({}) do |hash, key|
+      hash[key.to_s[1..-1]] = instance_variable_get(key)
+      hash
+    end
+  end
+
+  def update_from_hash(hash)
+    hash.each do |key, val|
+      self[key] = val unless key.to_s == "wstruct"
+    end
+    self
+  end
+
+  def [](attr)
+    send(attr.to_s)
+  end
+
+  def []=(attr, value)
+    send("#{attr}=", value)
+  end
+end

data/lib/correct_horse_battery_staple/writer.rb

@@ -0,0 +1,29 @@
+class CorrectHorseBatteryStaple::Writer
+  def self.make_writer(dest, fformat, options = {})
+    fformat ||= CorrectHorseBatteryStaple::Corpus.format_for(dest)
+    raise ArgumentError, "Cannot determine file format for #{dest}" if !fformat || fformat.empty?
+
+    clazz = const_get(fformat.downcase.capitalize)
+    clazz.new(dest, options)
+  end
+
+  def self.write(corpus, dest, fformat, options = {})
+    writer = self.make_writer(dest, fformat, options)
+    begin
+      writer.write_corpus(corpus)
+    ensure
+      writer && writer.close
+    end
+  end
+
+  autoload :Base,     "correct_horse_battery_staple/writer/base"
+  autoload :File,     "correct_horse_battery_staple/writer/file"
+  autoload :Json,     "correct_horse_battery_staple/writer/json"
+  autoload :Csv,      "correct_horse_battery_staple/writer/csv"
+  autoload :Isam,     "correct_horse_battery_staple/writer/isam"
+  autoload :Isamkd,   "correct_horse_battery_staple/writer/isam_kd"
+  autoload :Marshal,  "correct_horse_battery_staple/writer/marshal"
+  autoload :Sqlite,   "correct_horse_battery_staple/writer/sqlite"
+  autoload :Redis,    "correct_horse_battery_staple/writer/redis"
+  # autoload :KDTree,   "correct_horse_battery_staple/writer/kdtree"
+end

data/lib/correct_horse_battery_staple/writer/base.rb

@@ -0,0 +1,22 @@
+#
+# Base class for all writers
+#
+
+class CorrectHorseBatteryStaple::Writer::Base < CorrectHorseBatteryStaple::Writer
+  include CorrectHorseBatteryStaple::Common
+
+  attr_accessor :dest, :options
+
+  def initialize(dest, options = {})
+    self.dest    = dest
+    self.options = options
+    initialize_backend_variables if respond_to?(:initialize_backend_variables)
+  end
+
+  def write_corpus(corpus)
+    raise NotImplementedError, "#{self.class.name} is not a complete implementation"
+  end
+
+  def close
+  end
+end

data/lib/correct_horse_battery_staple/writer/csv.rb

@@ -0,0 +1,15 @@
+class CorrectHorseBatteryStaple::Writer::Csv < CorrectHorseBatteryStaple::Writer::File
+
+  def initialize(dest, options={})
+    super
+  end
+
+  def write_corpus(corpus)
+    puts "index,rank,word,frequency,percentile,distance,probability,distance_probability"
+    corpus.each_with_index do |w, index|
+      puts sprintf("%d,%d,\"%s\",%d,%.4f,%.6f,%.8f,%.8f\n",
+        index || -1, w.rank || -1, w.word, w.frequency || -1,
+        w.percentile || -1, w.distance || -1, w.probability || -1, w.distance_probability || -1)
+    end
+  end
+end

data/lib/correct_horse_battery_staple/writer/file.rb

@@ -0,0 +1,54 @@
+#
+# base class for file-based stores
+#
+#
+class CorrectHorseBatteryStaple::Writer::File < CorrectHorseBatteryStaple::Writer::Base
+  attr_accessor :io
+
+  def initialize(dest, options = {})
+    super
+
+    @do_close = false
+    if dest.respond_to?(:write)
+      self.io = dest
+    else
+      if ["/dev/stdout", "-"].include?(dest)
+        self.io = STDOUT
+      else
+        self.io = open(dest, openmode)
+        @do_close = true
+      end
+    end
+  end
+
+  def close
+    return unless @do_close
+    self.io.close rescue nil
+  ensure
+    self.io = nil
+    @do_close = false
+  end
+
+  protected
+
+  def openmode
+    "w"
+  end
+
+  def <<(string)
+    self.io.write string
+  end
+
+  def print(string)
+    self.io.print string
+  end
+
+  def puts(string)
+    self.io.puts string
+  end
+
+  def write(string)
+    self.io.write string
+  end
+
+end