correct-horse-battery-staple 0.6.1

data/lib/correct_horse_battery_staple/backend/redis.rb

@@ -0,0 +1,95 @@
+require 'redis'
+if ! Object.const_defined?("JRUBY_VERSION")
+ require 'redis/connection/hiredis'
+end
+require 'securerandom'
+
+module CorrectHorseBatteryStaple::Backend::Redis
+
+  def self.included(base)
+    base.extend ClassMethods
+    base.send :include, InstanceMethods
+  end
+
+  module ClassMethods
+  end
+
+  module InstanceMethods
+    def parse_uri(dest)
+      (dbname, host, port)   = dest.gsub(/\.redis[0-9]?/, '').split(':')
+      options[:dbname]     ||= (dbname || "chbs")
+      options[:host]       ||= (host   || "127.0.0.1")
+      options[:port]       ||= (port  || 6379).to_i
+    end
+
+    def add_word(w, wid=nil)
+      percentile = [0, w.percentile].max
+
+      wid = get_new_word_id if wid.nil?
+
+      db.zadd(@words_key, wid, w.word)
+      db.zadd(@percentile_key, percentile, wid)
+      # db.zadd(@frequency_key, w.frequency, wid)
+      db.zadd(@lenprod_key, w.word.length + (percentile / 100.0), wid)
+    end
+
+    #
+    # Note that this does NOT work inside a multi/exec
+    #
+    def get_new_word_id
+      db.incr(@id_key)
+    end
+
+    def get_word_by_id(wid)
+      db.zrangebyscore(@words_key, wid, wid, :limit => [0,1])[0] rescue nil
+    end
+
+    def load_stats
+      #noinspection RubyHashKeysTypesInspection
+      load_stats_from_hash Hash[db.hgetall(@stats_key).map {|k,v| [k, v.to_f]}]
+    end
+
+    def save_stats(stats)
+      db.hmset @stats_key, *stats.to_a.flatten
+    end
+
+    def create_database
+      db.del(@length_key, @percentile_key, @frequency_key, @lenprod_key,  @stats_key,
+             @words_key, @id_key)
+    end
+
+    def open_database
+      @db ||= begin
+                @gensym_id             = 0
+                @length_key            = make_key("length_zset")
+                @percentile_key        = make_key("percentile_zset")
+                @frequency_key         = make_key("frequency_zset")
+                # scores given by w.word.length + w.percentile/100.0
+                @lenprod_key           = make_key("lenprod_zset")
+                @stats_key             = make_key("stats_hash")
+                @words_key             = make_key("words_zset")
+                @id_key                = make_key("word_id_counter")
+                ::Redis.new(:host => options[:host], :port => options[:port])
+              end
+    end
+
+    def db
+      @db || open_database
+    end
+
+    def close_database
+    end
+
+    def make_key(name)
+      "chbs_#{options[:dbname]}_#{name}"
+    end
+
+    def gensym_temp
+      @_gensym_id ||= 0
+      make_key("TEMP_#{Process.pid}_#{@gensym_id += 1}")
+    end
+  end
+
+  autoload :DRange, 'correct_horse_battery_staple/backend/redis/d_range.rb'
+end
+

data/lib/correct_horse_battery_staple/backend/redis/d_range.rb

@@ -0,0 +1,105 @@
+#
+# Represents a list of items corresponding to a square area of items
+# formed by a range on axis 1 and a range on axis 2 of numbers
+# associated with an item.  In other words, this composes two
+# different data about an item into a single score in a Redis sorted
+# set, and allow that area to be treated as a single logical ordered
+# list of items.
+#
+# This is used to construct a single score out of a word's length
+# and percentile ranking.  The length is the "outer" score and
+# ranges (generally) from 3..18 or thereabouts in integral steps.
+# Percentiles exist as fractional parts of the score added to the
+# base word length.  So, to address the items in a sorted set
+# with the word length from 5..8 and percentile range 20..30,
+# you would (in the Writer::Redis class) generate a Sorted set
+# in which every word has a score with an integer and fractional
+# part.  The word "the" which appeared in the 95th percentile would
+# have a score of 3.95.
+#
+# Once defined, this class allows the following operations:
+#
+# - counting the total # of items in the 2d bounding box
+# - picking the nth item from the (virtual) sorted list
+#
+#
+
+class CorrectHorseBatteryStaple::Backend::Redis::DRange
+  include CorrectHorseBatteryStaple::Memoize
+  def initialize(db, key, outer, inner, divisor=100)
+    @db = db
+    @key = key
+    @outer = outer
+    @inner = inner
+    @divisor = divisor
+    @counts = {}
+  end
+
+  def dump
+    iterate_ranges do |min, max|
+      cnt = @db.zcount(@key, min, max)
+      [min, max, cnt]
+    end
+  end
+
+  def count
+    precache_counts
+    @counts.values.reduce(:+)
+  end
+  memoize :count
+
+  def pick_nth(n)
+    precache_counts
+    return nil if n > count-1
+
+    pos = 0
+    @outer.each do |base|
+      cib = count_in_base(base)
+      minpos = pos
+      maxpos = pos + cib
+      if cib > 0 && n >= minpos && n <= maxpos
+        (min, max) = minmax_for_base(base)
+        return @db.zrangebyscore(@key, min, max,
+                                 :limit => [n-pos, 1])[0]
+      end
+      pos += cib
+    end
+    return nil
+  end
+
+  protected
+
+  def precache_counts
+    return if @precached_counts
+    counts = @db.multi do
+      @outer.each do |base|
+        zcount(*minmax_for_base(base))
+      end
+    end
+    #noinspection RubyHashKeysTypesInspection
+    @counts = Hash[@outer.to_a.zip(counts)]
+    @precached_counts = true
+    @counts
+  end
+
+  def count_in_base(b)
+    @counts[b] ||= zcount(*minmax_for_base(b))
+  end
+
+  def minmax_for_base(base)
+    [base + @inner.begin / (@divisor.to_f),
+     base + @inner.end / (@divisor.to_f)]
+  end
+
+  def zcount(min, max)
+    @db.zcount(@key, min, max)
+  end
+
+  def iterate_ranges
+    @outer.map do |base|
+      (min, max) = minmax_for_base(base)
+      yield min, max
+    end
+  end
+
+end

data/lib/correct_horse_battery_staple/corpus.rb

@@ -0,0 +1,33 @@
+
+class CorrectHorseBatteryStaple::Corpus
+  def self.read(filename, clazz=nil)
+    clazz ||=
+      case CorrectHorseBatteryStaple::Corpus.format_for(filename)
+      # when 'kdtree' then CorrectHorseBatteryStaple::Corpus::KDTree
+      when 'isam' then CorrectHorseBatteryStaple::Corpus::Isam
+      when 'kdtree', 'isamkd' then CorrectHorseBatteryStaple::Corpus::IsamKD
+      when 'sqlite' then CorrectHorseBatteryStaple::Corpus::Sqlite
+      when 'redis2' then CorrectHorseBatteryStaple::Corpus::Redis2
+      when 'redis' then CorrectHorseBatteryStaple::Corpus::Redis
+      else CorrectHorseBatteryStaple::Corpus::Serialized
+      end
+
+    clazz.read(filename)
+  end
+
+  def self.format_for(spec, defval = nil)
+    File.extname(spec)[1..-1].downcase || defval
+  rescue
+    defval
+  end
+  
+  autoload :Base,       'correct_horse_battery_staple/corpus/base'
+  autoload :Serialized, 'correct_horse_battery_staple/corpus/serialized'
+  autoload :Isam,       'correct_horse_battery_staple/corpus/isam'
+  autoload :IsamKD,     'correct_horse_battery_staple/corpus/isam_kd'
+  autoload :Sqlite,     'correct_horse_battery_staple/corpus/sqlite'
+  autoload :Redis,      'correct_horse_battery_staple/corpus/redis'
+  autoload :Redis2,     'correct_horse_battery_staple/corpus/redis2'
+  # autoload :KDTree,     'correct_horse_battery_staple/corpus/kdtree'
+end
+

data/lib/correct_horse_battery_staple/corpus/base.rb

@@ -0,0 +1,278 @@
+require 'bigdecimal'
+# require 'securerandom'
+require 'forwardable'
+
+class CorrectHorseBatteryStaple::Corpus::Base < CorrectHorseBatteryStaple::Corpus
+  extend Forwardable
+  
+  attr_accessor :frequency_mean, :frequency_stddev
+  attr_accessor :probability_mean, :probability_stddev
+  attr_accessor :original_size
+  attr_accessor :weighted_size
+
+  include CorrectHorseBatteryStaple::Common
+  include CorrectHorseBatteryStaple::Memoize
+  include Enumerable
+
+  def initialize(*args)
+    initialize_backend_variables if respond_to?(:initialize_backend_variables)
+  end
+  
+  def self.read(dest)
+    self.new dest
+  end
+  
+  # you MUST override this method for Enumerable to use
+
+  def each(&block)
+    raise NotImplementedError
+  end
+
+  # other methods you should implement if possible:
+  #
+  # Enumerable
+  #  size
+  #
+  # CHBS::Corpus
+  #  pick
+  #  words
+  #  frequencies
+  #
+
+
+  def sorted_entries
+    entries.sort
+  end
+
+  # return all the candidates for a given set of options
+  def candidates(options = {})
+    return size if !options || options.empty?
+    filter = filter_for_options(options)
+    return size unless filter
+    entries.select {|entry| filter.call(entry) }
+  end
+
+  def count_candidates(options = {})
+    return size if !options || options.empty?
+    filter = filter_for_options(options)
+    return size unless filter
+
+    count = 0
+    each do |entry|
+      count += 1 if filter.call(entry)
+    end
+    count
+  end
+  memoize :count_candidates
+
+
+
+  #
+  # this is the core password picker method. it is not especially
+  # efficient but it is relatively generic.  If a corpus supports
+  # Enumerable, it will work.
+  #
+  def pick(count, options = {})
+    array = CorrectHorseBatteryStaple::StatisticalArray.new(sorted_entries)
+
+    filters = Array(options[:filter])
+
+    if options[:percentile]
+      range = array.index_range_for_percentile(options[:percentile])
+    else
+      range = 0..array.size-1
+    end
+    range_size = range_size(range)
+
+    if range_size < count
+      raise ArgumentError, "Percentile range contains fewer words than requested count"
+    end
+
+    if options[:word_length]
+      wl = options[:word_length]
+      filters << lambda {|entry| wl.include? entry.word.length }
+    end
+
+    filter = filters.empty? ? nil : compose_filters(filters)
+
+    max_iterations = options[:max_iterations] || 1000
+
+    result = []
+    iterations = 0
+    while result.length < count && iterations < max_iterations
+      i = random_number(range_size)
+      entry = array[i + range.first]
+      if entry && (!filter || filter.call(entry))
+        result << entry
+      end
+      iterations += 1
+    end
+
+    raise "Cannot find #{count} words matching criteria" if result.length < count
+    result
+  end
+
+
+
+  def words
+    execute_filters.map {|entry| entry.word }
+  end
+  memoize :words
+
+  # no-op for serialized forms
+  def precache(max=0)
+  end
+
+  def frequencies
+    CorrectHorseBatteryStaple::StatisticalArray.new(entries.map {|entry| entry.frequency })
+  end
+  memoize :frequencies
+
+  def entropy_per_word
+    Math.log(count) / Math.log(2)
+  end
+
+  # filtering
+
+  def filter(&block)
+    (@filters ||= []) << block
+    self
+  end
+
+  def reset
+    @filters = []
+  end
+
+  # create a single composed function of all the filters
+  def compose_filters(filters)
+    return nil if !filters || filters.empty?
+    filters.reduce do |prev, current|
+      lambda {|value| prev.call(value) && current.call(value) }
+    end
+  end
+
+  def result
+    return self if @filters.empty?
+
+    self.class.new(execute_filters).tap do |new_corpus|
+      new_corpus.original_size = self.original_size
+    end
+  end
+
+
+  ## statistics
+
+  def load_stats_from_hash(hash)
+    hash.each do |k,v|
+      setter = "#{k}=".to_sym
+      send setter, v if respond_to?(setter)
+    end
+  end
+
+  def recalculate
+    size        = self.size
+    frequencies = self.frequencies
+
+    # corpus-wide statistics
+    self.weighted_size  = frequencies.reduce(BigDecimal.new("0"), :+)
+    (self.probability_mean, self.probability_stddev)    =
+      CorrectHorseBatteryStaple::StatisticalArray.new(frequencies.map do |freq|
+        (freq/weighted_size) * 100
+      end).mean_and_standard_deviation
+
+    (self.frequency_mean, self.frequency_stddev) = frequencies.mean_and_standard_deviation
+
+      # stats              = corpus.stats
+      # size               = corpus.size
+      # frequency_mean     = corpus.frequency_mean
+      # frequency_stddev   = corpus.frequency_stddev
+      # weighted_size      = corpus.weighted_size
+      # probability_mean   = corpus.probability_mean
+      # probability_stddev = corpus.probability_stddev
+
+    each_with_index do |entry, index|
+      entry.rank                      = size - index
+      entry.distance                  = (entry.frequency-frequency_mean)/frequency_stddev
+      entry.probability               = entry.frequency / weighted_size
+      entry.distance_probability      = (entry.probability - probability_mean) / probability_stddev
+      entry.percentile                = (index-0.5)/size * 100
+    end
+
+    self
+  end
+
+  def stats
+    {:frequency_mean => frequency_mean, :frequency_stddev => frequency_stddev,
+      :probability_mean => probability_mean, :probability_stddev => probability_stddev,
+      :size => count, :original_size => original_size,
+      :weighted_size => weighted_size.to_f}
+  end
+
+  def inspect
+    <<INSPECT
+Type: #{self.class.name}
+Entry count: #{count}
+
+Stats:
+#{stats.map {|k,v| "  #{k}: #{v}\n" }.join("") }
+INSPECT
+  end
+  
+  alias :length :count
+
+
+  protected
+
+  #
+  # Return the number of distinct objects within the Range.
+  # This assumes plain vanilla ranges, though it does respect .. vs ...
+  #
+  # Why? Range#count is basically #to_a.count, which is INSANE
+  #
+  def range_count(r)
+    (r.last - r.first +
+     (r.exclude_end? ? 0 : (r.first > r.last ? -1 : 1))
+     ).abs
+  end
+  alias :range_size :range_count
+
+  #
+  # Given a filter, return all Word objects in this Corpus that the
+  # filter accepts.
+  #
+  # this is an exceptionally inefficient version
+  def execute_filters
+    return entries if @filters.nil? || @filters.empty?
+    entries.select(&compose_filters(@filters))
+  ensure
+    reset
+  end
+
+  #
+  # Return a single lambda that will return true/false given a Word object
+  #
+  # Respects the :word_length, :percentile, and :filter options
+  # :word_length and :percentile should be Range objects
+  # :filter can be a single Proc/lambda or an array of them
+  #
+  def filter_for_options(options = {})
+    return nil if !options || options.empty?
+
+    filters = Array(options[:filter])
+    if options[:percentile]
+      p_range = options[:percentile]
+      filters << lambda {|entry| p_range.include? entry.percentile }
+    end
+
+    if options[:word_length]
+      wl_range = options[:word_length]
+      filters << lambda {|entry| wl_range.include? entry.word.length }
+    end
+
+    filters.empty? ? nil : compose_filters(filters)
+  end
+  memoize :filter_for_options
+
+end
+
+# Random.srand(SecureRandom.random_number)