correct-horse-battery-staple 0.6.1

data/lib/correct_horse_battery_staple/corpus/redis2.rb

@@ -0,0 +1,88 @@
+require 'bigdecimal'
+require 'hiredis'
+require 'redis'
+
+class CorrectHorseBatteryStaple::Corpus::Redis2 < CorrectHorseBatteryStaple::Corpus::Redis
+  MAX_ITERATIONS = 1000
+
+  def size
+    @size ||= db.zcard(@percentile_key)
+  end
+
+
+
+  ## our own collection operations
+  ## optimized pick implementations - they do NOT support :filter, though
+
+  def pick(count, options = {})
+    percentile_range = options[:percentile]
+    length_range     = options[:word_length]
+    tempkey = nil
+
+    if percentile_range && percentile_range.begin == 0 && percentile_range.end == 100
+      percentile_range = nil
+    end
+
+    pick_sset_random(@words_key, 4)
+
+    if (!percentile_range && !length_range)
+      get_words_for_ids(pick_random_words(count))
+    else
+      sets = []
+      sets << make_subset_spec(@percentile_key, percentile_range) if percentile_range
+
+      # this isn't correct because lenprod_key will have values in the range 18...19
+      # sets << make_subset_spec(@lenprod_key, length_range)         if length_range
+      if length_range
+        sets << [@lenprod_key, ["-inf", "(#{length_range.begin}"],
+                               ["#{length_range.end.floor + 1}", "inf"]]
+      end
+
+      # returns union set key
+      tempkey = subset_and_union(sets)
+      # STDERR.puts "result count in #{tempkey} is #{db.zcard(tempkey)}"
+
+      get_words_for_ids(pick_sset_random(tempkey, count))
+    end
+  ensure
+    db.del tempkey if tempkey
+  end
+
+  def make_subset_spec(key, range)
+    [key, ["-inf", "(#{range.begin}"], ["(#{range.end}", "inf"]]
+  end
+
+  def make_subset(spec)
+    key = gensym_temp
+    source_key = spec.shift
+    db.zunionstore(key, [source_key])
+    db.expire(key, 180)
+    spec.each do |(min, max)|
+      db.zremrangebyscore(key, min, max)
+    end
+    key
+  end
+
+  def subset_and_union(specs)
+    result_key = gensym_temp
+    db.multi do
+      keys = specs.map do |spec|
+        make_subset(spec)
+      end
+      db.zinterstore(result_key, keys)
+      db.del(*keys)
+    end
+    db.expire(result_key, 1800)
+    result_key
+  end
+
+  def pick_sset_random(key, count)
+    max = db.zcard(key)
+    db.multi do
+      count.times.map do
+        rnd = random_number(max)
+        db.zrange(key, rnd, rnd)
+      end
+    end.flatten
+  end
+end

data/lib/correct_horse_battery_staple/corpus/serialized.rb

@@ -0,0 +1,121 @@
+require 'bigdecimal'
+require 'json'
+
+class CorrectHorseBatteryStaple::Corpus::Serialized < CorrectHorseBatteryStaple::Corpus::Base
+  attr_reader   :table
+
+  if RUBY_VERSION.start_with? "1.8"
+    require 'faster_csv'
+    CSVLIB = FasterCSV
+  else
+    require 'csv'
+    CSVLIB = CSV
+  end
+
+  def initialize(table, stats = nil)
+    super
+    @table   = CorrectHorseBatteryStaple::StatisticalArray.cast(table.sort, true)
+    @stats   = stats
+    @filters = []
+
+    if stats && !stats.empty?
+      load_stats_from_hash(stats)
+    end
+
+    self.original_size = @table.size
+  end
+
+  ## some core Enumerable building blocks
+
+  def each(&block)
+    table.each &block
+  end
+
+  def size
+    table.length
+  end
+
+  def entries
+    table
+  end
+
+  def sorted_entries
+    table
+  end
+
+  ## serialization
+  # reading
+
+  def self.read_csv(file)
+    self.new CSVLIB.table(file).map {|row| CorrectHorseBatteryStaple::Word.new(row.to_hash) }
+  end
+
+  def self.read_json(file)
+    json = JSON.parse(open(file).read)
+    self.new(json["corpus"].map {|hash| CorrectHorseBatteryStaple::Word.new(hash)},
+             json["stats"])
+  end
+
+  def self.read_marshal(file)
+    Marshal.load(open(file).read)
+  end
+
+  def self.read(filename, fformat=nil)
+    if ! fformat
+      fformat = File.extname(filename)[1..-1]
+    end
+    raise ArgumentError, "Cannot determine file format for #{filename}" if !fformat || fformat.empty?
+    send "read_#{fformat}", filename
+  end
+
+  # writing
+
+  def write_csv(io)
+    io.puts "index,rank,word,frequency,percentile,distance,probability,distance_probability"
+    @table.each_with_index do |w, index|
+      io.puts sprintf("%d,%d,\"%s\",%d,%.4f,%.6f,%.8f,%.8f\n",
+        index, w.rank, w.word, w.frequency || 0,
+        w.percentile || 0, w.distance || 0, w.probability || 0, w.distance_probability || 0)
+    end
+  end
+
+  def write_json1(io)
+    io.write({"stats" => stats, "corpus" => @table }.to_json)
+  end
+
+  def write_json(io)
+    io.print '{"stats": '
+    io.print stats.to_json
+    io.print ', "corpus": ['
+    i = 0
+    @table.each do |word|
+      io.puts "," if i >= 1
+      io.print(word.to_hash.to_json)
+      i += 1
+    end
+    io.puts "]\n}"
+  end
+
+  def write_marshal(io)
+    io.write Marshal.dump(self)
+  end
+
+  def write_isam(io)
+    sorted_entries.each_with_index do |w, index|
+      io.print sprintf("%-40s%10d", w.word, w.frequency || 0) if
+        w.word.length <= 40
+    end
+  end
+
+  def write(io, fformat=nil)
+    raise ArgumentError, "Cannot determine file format for output" if !fformat || fformat.empty?
+    send "write_#{fformat}", io
+  end
+
+  protected
+
+  def method_missing(name, *args, &block)
+    @table.__send__(name, *args, &block)
+  end
+
+end

data/lib/correct_horse_battery_staple/corpus/sqlite.rb

@@ -0,0 +1,266 @@
+require 'bigdecimal'
+require 'sqlite3'
+
+class CorrectHorseBatteryStaple::Corpus::Sqlite < CorrectHorseBatteryStaple::Corpus::Base
+  MAX_ITERATIONS = 1000
+
+  def initialize(file)
+    super
+    @db = SQLite3::Database.open file
+    @statements = []
+    load_stats
+  end
+
+  def self.read(file)
+    self.new file
+  end
+
+  ## some core Enumerable building blocks
+
+  def each(&block)
+    entries.each &block
+  end
+
+  def size
+    @size ||= @db.execute("select count(*) from entries").first.first
+  end
+
+
+
+  ## our own collection operations
+
+  def entries
+    @entries ||= table
+  end
+
+  def sorted_entries
+    entries
+  end
+
+  def frequencies
+    @frequencies ||= @db.execute("select frequency from entries").map {|x| x.first}
+  end
+
+
+  ## optimized pick variants - they do NOT support :filter, though
+
+  def pick(count, options = {})
+    # incompat check
+    raise NotImplementedError, "SQLite does not support :filter option" if options[:filter]
+
+    strategy = options.delete(:strategy) || ENV['pick_strategy'] || "discrete"
+    send("pick_#{strategy}", count, options)
+  end
+
+  def pick_rtree(count, options = {})
+    base = "select id from index3d "
+    wheres = []
+    params = []
+
+    wheres << "minR >= ? and maxR <= ?"
+    rnd = random_number
+    offset = 0.0
+    if rnd > 0.8
+      offset = 0.8-rnd
+    elsif rnd < 0.2
+      offset = 0.2-rnd
+    end
+    params += [rnd - 0.20 + offset, rnd + 0.20 + offset]
+
+    if options[:word_length]
+      wheres << "  minL >= ? and maxL <= ? "
+      params += [options[:word_length].first, options[:word_length].last]
+    end
+    if options[:percentile]
+      wheres << "  minP >= ? and maxP <= ? "
+      params += [options[:percentile].first, options[:percentile].last]
+    end
+    statement = [base,
+                 (wheres.empty? ? "" : " WHERE " + wheres.join(" AND ")),
+                 "limit ?"].join(" ")
+    params += [[count,250].max]
+
+    query = prepare(statement)
+    ids = array_sample(query.execute!(*params), count).map {|r| r[0]}
+
+    if ids and !ids.empty?
+      result = get_words_for_ids(ids)
+    else
+      result = []
+    end
+
+    # validate that we succeeded
+    raise "Cannot find #{count} words matching criteria" if result.length < count
+
+    result
+  end
+
+  def get_words_for_ids(ids)
+    ids = Array(ids)
+    rows = @db.execute("select #{COLUMNS.join(", ")} from entries where id in (#{ids.join(',')})")
+
+    words = []
+    ids.each do |id|
+      words << rows.find {|r| r[0] == id }
+    end
+    words.map {|row| word_from_row(row)}
+  end
+
+
+  def pick_standard(count, options = {})
+    statement = "select #{COLUMNS.join(", ")} from entries "
+    params = []
+    wheres = []
+    if options[:word_length]
+      wheres << "  wordlength >= ? and wordlength <= ? "
+      params += [options[:word_length].first, options[:word_length].last]
+    end
+    if options[:percentile]
+      wheres << "  percentile >= ? and percentile <= ? "
+      params += [options[:percentile].first, options[:percentile].last]
+    end
+    statement = [statement,
+                 (wheres.empty? ? "" : " WHERE " + wheres.join(" AND ")),
+                 "order by RANDOM()",
+                 "limit ?"].join(" ")
+    params << [count, 20].max
+    query = prepare(statement)
+    result = array_sample(query.execute!(*params), count).
+      map { |row| word_from_row(row) }
+
+    # validate that we succeeded
+    raise "Cannot find #{count} words matching criteria" if result.length < count
+
+    result
+  end
+
+  def pick_standard2(count, options = {})
+    statement = "select id from entries "
+    params = []
+    wheres = []
+    if options[:word_length]
+      wheres << "  wordlength >= ? and wordlength <= ? "
+      params += [options[:word_length].first, options[:word_length].last]
+    end
+    if options[:percentile]
+      wheres << "  percentile >= ? and percentile <= ? "
+      params += [options[:percentile].first, options[:percentile].last]
+    end
+    statement = [statement,
+                 (wheres.empty? ? "" : " WHERE " + wheres.join(" AND ")),
+                 # "order by RANDOM()",
+                 "limit ?"].join(" ")
+    params << [count, 1000].max
+    query = prepare(statement)
+    ids = array_sample(query.execute!(*params), count).
+      map {|r| r[0]}
+
+    result = get_words_for_ids(ids)
+
+    # validate that we succeeded
+    raise "Cannot find #{count} words matching criteria" if result.length < count
+
+    result
+  end
+
+
+  # discrete method
+  def pick_discrete(count, options = {})
+    p_range = options[:percentile] or 0..100
+    l_range = options[:word_length] or 4..12
+
+    result = []
+    iterations = 0
+    while (iterations < 4 || result.length < count) && iterations < MAX_ITERATIONS
+      percentile  = random_in_range(p_range)
+      length      = random_in_range(l_range)
+      result     += _pick_discrete_n(percentile, length, 1)
+      iterations += 1
+    end
+
+    # validate that we succeeded
+    raise "Cannot find #{count} words matching criteria" if result.length < count
+
+    array_sample(result, count).map {|row| word_from_row(row)}
+  end
+
+  def prepare(statement)
+    res = @db.prepare(statement)
+    @statements << res
+    res
+  end
+  memoize :prepare
+
+  def _pick_discrete_n(percentile, length, count = 1)
+    statement = prepare "select #{COLUMNS.join(", ")} from entries where " +
+      " percentile = ? and wordlength = ? and randunit < ? limit ?"
+
+    statement.execute!(percentile, length, random_number, count)
+  end
+
+
+  # discrete method
+  def pick_discrete2(count, options = {})
+
+    p_range = options[:percentile] or 0..100
+    l_range = options[:word_length] or 4..12
+
+    ids = []
+    iterations = 0
+    while (iterations < 3 || ids.length < count) && iterations < MAX_ITERATIONS
+      percentile  = random_in_range(p_range)
+      length      = random_in_range(l_range)
+      ids         = ids.concat(_pick_discrete_n_ids(percentile, length, 25)).uniq
+      iterations += 1
+    end
+
+    ids = array_sample(ids, count).map {|r| r[0] }
+    result = get_words_for_ids(ids)
+
+    # validate that we succeeded
+    raise "Cannot find #{count} words matching criteria" if result.length < count
+    result
+  end
+
+  def prepare(statement)
+    res = @db.prepare(statement)
+    @statements << res
+    res
+  end
+  memoize :prepare
+
+  def _pick_discrete_n_ids(percentile, length, count = 1)
+    statement = prepare "select id from entries where " +
+      " percentile = ? and wordlength = ? and randunit > ? limit ?"
+
+    statement.execute!(percentile, length, random_number, count)
+  end
+
+
+
+  def close
+    @statements.each { |x| x.close }
+    super
+  end
+
+  protected
+
+  COLUMNS = %w[id word frequency idx rank percentile]
+
+  def table
+    @db.execute("select #{COLUMNS.join(", ")} from entries order by frequency").map do |row|
+      word_from_row(row)
+    end
+  end
+
+  def word_from_row(row)
+    CorrectHorseBatteryStaple::Word.new(:word => row[1], :frequency => row[2],
+                                        :index => row[3], :rank => row[4],
+                                        :percentile => row[5])
+  end
+
+  def load_stats
+    rows = @db.execute "select name, value from stats"
+    load_stats_from_hash(rows.reduce({}) {|m, (key, val)| m.merge(key => val.to_f)})
+  end
+end