jk-ferret 0.11.8.2

data/test/unit/utils/tc_number_tools.rb

@@ -0,0 +1,117 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+require 'ferret/number_tools'
+
+
+class NumberToolsTest < Test::Unit::TestCase
+  include Ferret::Utils
+
+  def test_to_i_lex_near_zero()
+    (-10..10).each do |num|
+      assert(num.to_s_lex > (num-1).to_s_lex,
+             "Strings should sort correctly but " +
+             "#{num.to_s_lex} <= #{(num-1).to_s_lex}")
+      assert_equal(num, num.to_s_lex.to_i_lex)
+    end
+  end
+
+  def test_to_i_pad_near_zero()
+    (1..10).each do |num|
+      assert(num.to_s_pad(3) > (num-1).to_s_pad(3),
+             "Strings should sort correctly but " +
+             "#{num.to_s_pad(3)} <= #{(num-1).to_s_pad(3)}")
+      assert_equal(num, num.to_s_pad(3).to_i)
+    end
+  end
+
+  def test_to_i_lex_larger_numbers
+    100.times do
+      num1 = rand(10000000000000000000000000000000000)
+      num2 = rand(10000000000000000000000000000000000)
+      num1 *= -1 if rand(2) == 0
+      num2 *= -1 if rand(2) == 0
+
+      assert_equal(num1, num1.to_s_lex.to_i_lex)
+      assert_equal(num2, num2.to_s_lex.to_i_lex)
+      assert_equal(num1 < num2, num1.to_s_lex < num2.to_s_lex, 
+                   "Strings should sort correctly but " +
+                   "#{num1} < #{num2} == #{num1 < num2} but " +
+                   "#{num1.to_s_lex} < #{num2.to_s_lex} == " +
+                   "#{num1.to_s_lex < num2.to_s_lex}")
+    end
+  end
+
+  def test_to_i_pad
+    100.times do
+      num1 = rand(10000000000000000000000000000000000)
+      num2 = rand(10000000000000000000000000000000000)
+      assert_equal(num1, num1.to_s_pad(35).to_i)
+      assert_equal(num2, num2.to_s_pad(35).to_i)
+      assert_equal(num1 < num2, num1.to_s_pad(35) < num2.to_s_pad(35), 
+                   "Strings should sort correctly but " +
+                   "#{num1} < #{num2} == #{num1 < num2} but " +
+                   "#{num1.to_s_pad(35)} < #{num2.to_s_pad(35)} == " +
+                   "#{num1.to_s_pad(35) < num2.to_s_pad(35)}")
+    end
+  end
+  
+  def test_time_to_s_lex
+    t_num = Time.now.to_i - 365*24*60*60 # prevent range error
+
+    10.times do
+      t1 = Time.now - rand(t_num)
+      t2 = Time.now - rand(t_num)
+      assert_equal(t1.to_s, t1.to_s_lex(:second).to_time_lex.to_s)
+      assert_equal(t2.to_s, t2.to_s_lex(:second).to_time_lex.to_s)
+      [:year, :month, :day, :hour, :minute, :second, :millisecond].each do |prec|
+        t1_x = t1.to_s_lex(prec).to_time_lex
+        t2_x = t2.to_s_lex(prec).to_time_lex
+        assert_equal(t1_x < t2_x, t1.to_s_lex(prec) < t2.to_s_lex(prec), 
+                     "Strings should sort correctly but " +
+                     "#{t1_x} < #{t2_x} == #{t1_x < t2_x} but " +
+                     "#{t1.to_s_lex(prec)} < #{t2.to_s_lex(prec)} == " +
+                     "#{t1.to_s_lex(prec) < t2.to_s_lex(prec)}")
+      end
+    end
+  end
+
+  def test_date_to_s_lex
+    10.times do
+      d1 = Date.civil(rand(2200), rand(12) + 1, rand(28) + 1)
+      d2 = Date.civil(rand(2200), rand(12) + 1, rand(28) + 1)
+      assert_equal(d1.to_s, d1.to_s_lex(:day).to_date_lex.to_s)
+      assert_equal(d2.to_s, d2.to_s_lex(:day).to_date_lex.to_s)
+      [:year, :month, :day].each do |prec|
+        d1_x = d1.to_s_lex(prec).to_date_lex
+        d2_x = d2.to_s_lex(prec).to_date_lex
+        assert_equal(d1_x < d2_x, d1.to_s_lex(prec) < d2.to_s_lex(prec), 
+                     "Strings should sort correctly but " +
+                     "#{d1_x} < #{d2_x} == #{d1_x < d2_x} but " +
+                     "#{d1.to_s_lex(prec)} < #{d2.to_s_lex(prec)} == " +
+                     "#{d1.to_s_lex(prec) < d2.to_s_lex(prec)}")
+      end
+
+    end
+  end
+
+  def test_date_time_to_s_lex
+    10.times do
+      d1 = "#{rand(600) + 1600}-#{rand(12)+1}-#{rand(28)+1} " +
+           "#{rand(24)}:#{rand(60)}:#{rand(60)}"
+      d2 = "#{rand(600) + 1600}-#{rand(12)+1}-#{rand(28)+1} " +
+           "#{rand(24)}:#{rand(60)}:#{rand(60)}"
+      d1 = DateTime.strptime(d1, "%Y-%m-%d %H:%M:%S")
+      d2 = DateTime.strptime(d2, "%Y-%m-%d %H:%M:%S")
+      assert_equal(d1.to_s, d1.to_s_lex(:second).to_date_time_lex.to_s)
+      assert_equal(d2.to_s, d2.to_s_lex(:second).to_date_time_lex.to_s)
+      [:year, :month, :day, :hour, :minute, :second].each do |prec|
+        d1_x = d1.to_s_lex(prec).to_date_lex
+        d2_x = d2.to_s_lex(prec).to_date_lex
+        assert_equal(d1_x < d2_x, d1.to_s_lex(prec) < d2.to_s_lex(prec), 
+                     "Strings should sort correctly but " +
+                     "#{d1_x} < #{d2_x} == #{d1_x < d2_x} but " +
+                     "#{d1.to_s_lex(prec)} < #{d2.to_s_lex(prec)} == " +
+                     "#{d1.to_s_lex(prec) < d2.to_s_lex(prec)}")
+      end
+    end
+  end
+end

data/test/unit/utils/tc_priority_queue.rb

@@ -0,0 +1,106 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+class PriorityQueueTest < Test::Unit::TestCase
+  include Ferret::Utils
+
+  PQ_STRESS_SIZE = 1000
+
+  def test_pq()
+    pq = PriorityQueue.new(4)
+    assert_equal(0, pq.size)
+    assert_equal(4, pq.capacity)
+    pq.insert("bword")
+    assert_equal(1, pq.size)
+    assert_equal("bword", pq.top)
+
+    pq.insert("cword")
+    assert_equal(2, pq.size)
+    assert_equal("bword", pq.top)
+
+    pq << "dword"
+    assert_equal(3, pq.size)
+    assert_equal("bword", pq.top)
+    
+    pq << "eword"
+    assert_equal(4, pq.size)
+    assert_equal("bword", pq.top)
+
+    pq << "aword"
+    assert_equal(4, pq.size)
+    assert_equal("bword", pq.top, "aword < all other elements so ignore")
+
+    pq << "fword"
+    assert_equal(4, pq.size)
+    assert_equal("cword", pq.top, "bword got pushed off the bottom of the queue")
+
+    assert_equal("cword", pq.pop())
+    assert_equal(3, pq.size)
+    assert_equal("dword", pq.pop())
+    assert_equal(2, pq.size)
+    assert_equal("eword", pq.pop())
+    assert_equal(1, pq.size)
+    assert_equal("fword", pq.pop())
+    assert_equal(0, pq.size)
+    assert_nil(pq.top)
+    assert_nil(pq.pop)
+  end
+
+  def test_pq_clear()
+    pq = PriorityQueue.new(3)
+    pq << "word1"
+    pq << "word2"
+    pq << "word3"
+    assert_equal(3, pq.size)
+    pq.clear()
+    assert_equal(0, pq.size)
+    assert_nil(pq.top)
+    assert_nil(pq.pop)
+  end
+
+  #define PQ_STRESS_SIZE 1000
+  def test_stress_pq
+    pq = PriorityQueue.new(PQ_STRESS_SIZE)
+    PQ_STRESS_SIZE.times do
+      pq.insert("<#{rand(PQ_STRESS_SIZE)}>")
+    end
+
+    prev = pq.pop()
+    (PQ_STRESS_SIZE - 1).times do
+      curr = pq.pop()
+      assert(prev <= curr, "#{prev} should be less than #{curr}")
+      prev = curr
+    end
+    pq.clear()
+  end
+
+  def test_pq_block
+    pq = PriorityQueue.new(21) {|a, b| a > b}
+    100.times do
+      pq.insert("<#{rand(50)}>")
+    end
+
+    prev = pq.pop()
+    20.times do
+      curr = pq.pop()
+      assert(prev >= curr, "#{prev} should be greater than #{curr}")
+      prev = curr
+    end
+    assert_equal 0, pq.size 
+  end
+
+  def test_pq_proc
+    pq = PriorityQueue.new({:less_than => lambda {|a, b| a.size > b.size}, :capacity => 21})
+    100.times do
+      pq.insert("x" * rand(50))
+    end
+
+    prev = pq.pop()
+    20.times do
+      curr = pq.pop()
+      assert(prev.size >= curr.size, "#{prev} should be greater than #{curr}")
+      prev = curr
+    end
+    assert_equal 0, pq.size 
+  end
+end

data/test/utils/content_generator.rb

@@ -0,0 +1,226 @@
+module ContentGenerator
+  wpath = File.expand_path(File.join(__FILE__, '../../../data/words'))
+  WORDS = File.readlines(wpath).collect {|w| w.strip}
+  CHARS = 'abcdefghijklmnopqrstuvwxyz1234567890`~!@#$%^&*()_-+={[}]|\\:;"\'<,>.?/'
+  ALNUM = 'abcdefghijklmnopqrstuvwxyz1234567890'
+  ALPHA = 'abcdefghijklmnopqrstuvwxyz'
+  URL_SUFFIXES = %w{com net org biz info}
+  URL_COUNTRY_CODES = %w{au jp uk nz tv}
+  TEXT_CACHE = {}
+  WORD_CACHE = {}
+  MARKDOWN_EMPHASIS_MARKERS = %w{* _ ** __ ` ``}
+  MARKDOWN_LIST_MARKERS = %w{- * + 1.}
+
+  def self.generate_text(length = 5..10, options = {})
+    if length.is_a?(Range)
+      raise ArgumentError, "range must be positive" unless length.min
+      length = length.min + rand(length.max - length.min)
+    end
+
+    text = ''
+    if options[:chars]
+      while word = random_word and text.size + word.size < length
+        text << word + ' '
+      end
+      text.strip!
+      text << generate_word(length - text.size)
+    else
+      text = Array.new(length) {|x| random_word}.join(' ')
+    end
+    if key = options[:unique]||options[:key]
+      cache = TEXT_CACHE[key]||={}
+      if cache[text]
+        return generate_text(options)
+      else
+        return cache[text] = true
+      end      
+    end
+    return text
+  end
+
+  def self.generate_word(length = 5..10, options = {})
+    if length.is_a?(Range)
+      raise ArgumentError, "range must be positive" unless length.min
+      length = length.min + rand(length.max - length.min)
+    end
+
+    word = ''
+    case options[:charset]
+    when :alpha
+      word = Array.new(length) {|x| random_alpha}.pack('c*')
+    when :alnum
+      word = Array.new(length) {|x| random_alnum}.pack('c*')
+    else
+      word = Array.new(length) {|x| random_char}.pack('c*')
+    end
+
+    if key = options[:unique]||options[:key]
+      cache = WORD_CACHE[key]||={}
+      if cache[word]
+        return generate_word(options)
+      else
+        cache[word] = true
+      end      
+    end
+    return word
+  end
+
+  def self.generate_alpha_word(length = 5..10, options = {})
+    options[:charset] = :alpha
+    generate_word(length, options)
+  end
+
+  def self.generate_alnum_word(length = 5..10, options = {})
+    options[:charset] = :alnum
+    generate_word(length, options)
+  end
+
+  def self.generate_email(options = {})
+    num_name_sections = 1 + rand(2)
+    num_url_sections = 1 + rand(2)
+    name = Array.new(num_name_sections) {|x| generate_alnum_word }.join('.')
+    url = [generate_alnum_word]
+    url += Array.new(num_url_sections) {|x| generate_alpha_word(2..3) }
+    url = url.join('.')
+    name + '@' + url
+  end
+
+  def self.generate_url(options = {})
+    ext = random_from(URL_SUFFIXES)
+    ext += '.' + random_from(URL_COUNTRY_CODES) if rand(2) > 0
+    "http://www.#{generate_alnum_word}.#{ext}/"
+  end
+
+  def self.generate_markdown(length = 100..1000, options = {})
+    @footnote_num = 0
+    if length.is_a?(Range)
+      raise ArgumentError, "range must be positive" unless length.min
+      length = length.min + rand(length.max - length.min)
+    end
+    text = []
+    while length > 0
+      case rand
+      when 0.3..1 # generate paragraph
+        l = gen_num(length, 50)
+        paragraph = gen_md_para(l)
+        if rand > 0.95 # make block quote
+          paragraph = '> ' + paragraph
+        end
+        text << paragraph
+        length -= l
+      when 0.2..0.3 # generate list
+        li = random_from(MARKDOWN_LIST_MARKERS) + ' '
+        num_elements = gen_num(length/5, 10)
+        num_elements.times do
+          break if length == 0
+          if rand > 0.75 # do paragraph list element
+            xli = li
+            (2 + rand(3)).times do |i|
+              break if length == 0
+              l = gen_num(length, 10)
+              text << xli
+              text << gen_md_para(l, :no_footnotes => true)
+              text << "\n\n"
+              xli = ' ' * xli.size if i == 0
+              length -= l
+            end
+          else
+            l = gen_num(length, 10)
+            text << li
+            text << gen_md_para(l, :no_footnotes => true)
+            text << "\n"
+            length -= l
+          end
+        end
+      when 0.1..0.2 # header
+        l = gen_num(length, 7)
+        t = gen_md_para(l, :no_footnotes => true)
+        if rand > 0.8
+          t += "\n" + random_from(%w{= -}) * t.size
+        else
+          t = ('#' * (1 + rand(6))) + ' ' + t
+        end
+        length -= l
+        text << t
+      else 
+        text << '---'
+      end
+      text << "\n\n"
+    end
+    text.join()
+  end
+
+  def self.random_word
+    random_from(WORDS)
+  end
+
+  def self.random_char
+    random_from(CHARS)
+  end
+
+  def self.random_alnum
+    random_from(ALNUM)
+  end
+
+  def self.random_alpha
+    random_from(ALPHA)
+  end
+
+  private
+  
+  def self.gen_md_para(length, options = {})
+    link_words = rand(1 + length/10)
+    length -= link_words
+    text = gen_md_text(length)
+    text << "\n"
+    footnote_cnt = 0
+    while link_words > 0
+      if options[:no_footnotes] or rand > 0.5
+        if rand > 0.6 # inline link
+          l = gen_num(link_words, 5)
+          link = "[#{gen_md_text(l)}](#{generate_url} \"#{generate_text(1 + rand(5))}\")"
+          text.insert(rand(text.length - footnote_cnt), link)
+          link_words -= l
+        else          # auto link
+          text.insert(rand(text.length - footnote_cnt), "<#{generate_url}>")
+          link_words -= 1
+        end
+      else            # footnote link
+        l = gen_num(link_words, 5)
+        reference = "[#{gen_md_text(l).join(' ')}][#{@footnote_num}]"
+        text.insert(rand(text.length - footnote_cnt), reference)
+        text << link = "\n[#{@footnote_num}]: #{generate_url} \"#{generate_text(1 + rand(5))}\""
+        @footnote_num += 1
+        footnote_cnt += 1
+        link_words -= l
+      end
+    end
+    text.pop if text.last == "\n"
+    text.join(' ')
+  end
+
+  def self.gen_md_text(length)
+    text = Array.new(length) {|x| random_word}
+    if rand > 0.8
+      (1 + rand(Math.sqrt(length))).times do
+        first = rand(text.size)
+        last = first + rand(3)
+        last = text.size - 1 if last >= text.size
+        words = text.slice!(first..last)
+        em = random_from(MARKDOWN_EMPHASIS_MARKERS)
+        words = "#{em}#{words.join(' ')}#{em}" unless words.join.index(em[0,1])
+        text.insert(first, words).flatten!
+      end
+    end
+    text
+  end
+
+  def self.gen_num(max1, max2)
+    minmax = [max1, max2].min
+    return minmax == 0 ? 0 : 1 + rand(minmax)
+  end
+
+  def self.random_from(list)
+    list[rand(list.size)]
+  end
+end