keyword_prospector 0.8.0

data/lib/lookup_chain.rb

@@ -0,0 +1,65 @@
+# 
+# (C) 2008 Los Angeles Times
+# 
+require 'enumerator'
+
+# Manages a list of lookup objects, prioritizing the matches from the later
+# objects over matches from previous objects in cases of exact collisions or
+# cases where two matches have the same length.  Otherwise, when there are
+# overlaps, the longest match always wins.
+class LookupChain
+  attr_reader :lookups
+
+  def initialize(*lookups)
+    @lookups = []
+    lookups.flatten!
+
+    lookups.each do |lookup|
+      check_lookup(lookup)
+      @lookups << lookup
+    end
+  end
+
+  def <<(lookup)
+    check_lookup(lookup)
+
+    self.lookups << lookup
+  end
+
+  def process(text)
+    matches = []
+
+    @lookups.each do |lookup|
+      new_matches = lookup.process(text)
+      matches += new_matches
+      matches.sort!
+      i = 0
+      while(i+1 < matches.size)
+        (match1, match2) = matches[i,2]
+
+        if(match1.overlap?(match2))
+          if (match1.length > match2.length)
+            matches.delete_at(i+1)
+          elsif (match1.length < match2.length)
+            matches.delete_at(i)
+          else
+            if (new_matches.include?(match1))
+              matches.delete_at(i)
+            else
+              matches.delete_at(i+1)
+            end
+          end
+        else
+          i += 1
+        end
+      end
+    end
+
+    return matches
+  end
+
+  private
+  def check_lookup(lookup)
+    raise ArgumentError.new("lookup objects must respond to process method") unless lookup.respond_to?(:process)
+  end
+end

data/lib/match.rb

@@ -0,0 +1,37 @@
+# 
+# (C) 2008 Los Angeles Times
+# 
+class Match
+  include Comparable
+  attr_accessor :keyword, :start_idx, :end_idx, :output
+
+  def initialize(keyword, start_idx=0, end_idx=0, output=nil)
+    self.keyword = keyword
+    self.start_idx = start_idx
+    self.end_idx = end_idx
+    self.output = output
+  end
+
+  def <=>(other)
+    retval = start_idx <=> other.start_idx
+    if (retval == 0)
+      end_idx <=> other.end_idx
+    else
+      retval
+    end
+  end
+
+  def overlap?(other)
+    if (start_idx == other.start_idx || end_idx == other.end_idx)
+      return true
+    elsif (start_idx < other.start_idx)
+      return end_idx > other.start_idx
+    elsif (start_idx > other.start_idx)
+      return other.end_idx > start_idx
+    end
+  end
+
+  def length
+    end_idx - start_idx
+  end
+end

data/lib/profile.rb

@@ -0,0 +1,72 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.join(File.dirname(__FILE__),'..','lib')
+
+require 'keyword_prospector'
+
+class Profiler
+  MATCHWORD_COUNT=2000
+
+  def self.get_tree(keyword_count)
+    words = get_random_words(keyword_count)
+    KeywordProspector.new(words)
+  end
+
+  def self.profile_tree_creation(keyword_count, options={})
+    total = 0
+    time_limit = options[:time_limit] || 0.01
+
+    start_time = Time.now
+    iterations = 0
+    while Time.now - start_time < time_limit
+      start=Time.now
+      get_tree(keyword_count)
+      total += Time.now - start
+      iterations += 1
+    end
+
+    puts "Average tree creation time over #{iterations} iterations with #{keyword_count} keywords: #{total/iterations.to_f}"
+  end
+
+  def self.profile_keyword_matching(keyword_count, options={})
+    dl = get_tree(keyword_count)
+    time_limit = options[:time_limit] || 300
+
+    total = 0
+    total_matches = 0
+    start_time = Time.now
+    iterations = 0
+    words = get_random_words(options[:count] || MATCHWORD_COUNT)
+    text = words.join(" ")
+    while Time.now - start_time < time_limit
+      matches = 0
+      start = Time.now
+      dl.process(text) {matches += 1}
+      total += Time.now - start
+      total_matches += matches
+      iterations += 1
+    end
+
+    puts "Average time spent matching: #{total/iterations.to_f}"
+    puts "Average number of matches: #{total_matches/iterations.to_f}"
+  end
+
+  private
+  def self.get_random_words(count=1000)
+    words = []
+    dict = File.open("/usr/share/dict/words")
+    count.times do |index|
+      words[index] = dict.readline.chomp
+    end
+
+    dict.each_line do |word|
+      word = dict.readline.chomp
+      index = rand dict.lineno
+      if (index < count)
+        words[index] = word
+      end
+    end
+
+    return words
+  end
+end

data/lib/search_and_replace.rb

@@ -0,0 +1,45 @@
+# 
+# (C) 2008 Los Angeles Times
+# 
+require 'keyword_prospector'
+
+class SearchAndReplace
+  def initialize
+    @dl = KeywordProspector.new
+    @tree_initialized = false
+  end
+
+  def add_replacement(original, replacement)
+    @dl.add(original, replacement)
+  end
+
+  def initialize_tree
+    unless @tree_initialized
+      @dl.construct_fail
+      @tree_initialized = true
+    end
+  end
+
+  def replace_text(text)
+    unless @tree_initialized
+      initialize_tree
+    end
+
+    matches = @dl.process(text.downcase, :filter_overlaps => true)
+
+    retval = ""
+    current_index = 0
+    matches.each do |match|
+      if (current_index < match.start_idx)
+        retval += text[current_index, match.start_idx - current_index]
+      end
+
+      retval += match.output
+      current_index = match.end_idx
+    end
+
+    retval += text[current_index, text.length] unless current_index == text.length
+
+    retval
+  end
+end

data/lib/state.rb

@@ -0,0 +1,85 @@
+# 
+# (C) 2008 Los Angeles Times
+# 
+
+class State
+  @@id_sequence = 1
+
+  attr_reader   :id
+  attr_reader   :char
+
+  attr_accessor :fail
+  attr_accessor :fail_increment
+  attr_accessor :output
+  attr_accessor :depth
+  attr_accessor :keyword
+  
+  def initialize(char, depth=0, next_state={})
+    if next_state.is_a?(Array)
+      class << next_state
+        def values
+          self.select{|value| value}
+        end
+
+        def keys
+          retval = []
+          self.each_index{|i| retval << i if self[i]}
+
+          retval
+        end
+      end
+    end
+
+    @char = char
+    @id = @@id_sequence
+    @next_state = next_state
+    @@id_sequence += 1
+    self.depth = depth
+  end
+  
+  def insert_next_state(char)
+    cur_state = @next_state[char]
+
+    if cur_state
+      cur_state
+    else
+      @next_state[char] = State.new(char, depth + 1)
+    end
+  end
+  
+  def [](char)
+    @next_state[char]
+  end
+  
+  def []=(char, value)
+    @next_state[char] = value
+  end
+  
+  def values
+    @next_state.values
+  end
+  
+  def keys
+    @next_state.keys
+  end
+  
+  def transition(char, position=nil)
+    next_state = @next_state[char]
+
+    if next_state == self
+      if position
+        position.end = position.begin += 1
+      end
+    elsif next_state
+      position.end += 1 if position
+    else
+      next_state = fail.transition(char)
+      if position
+        position.begin += depth - (next_state.depth - 1)
+        position.end += 1
+      end
+    end
+
+    next_state
+  end
+end

data/script/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# File: script/console
+irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
+
+libs =  " -r irb/completion"
+# Perhaps use a console_lib to store any extra methods I may want available in the cosole
+# libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/console_logger.rb'}"
+libs <<  " -r #{File.dirname(__FILE__) + '/../lib/keyword_prospector.rb'}"
+puts "Loading keyword_prospector gem"
+exec "#{irb} #{libs} --simple-prompt"

data/script/destroy

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/destroy'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Destroy.new.run(ARGV)

data/script/generate

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/generate'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Generate.new.run(ARGV)

data/script/txt2html

@@ -0,0 +1,82 @@
+#!/usr/bin/env ruby
+
+GEM_NAME = 'keyword_prospector' # what ppl will type to install your gem
+RUBYFORGE_PROJECT = 'keyword_prospector'
+
+require 'rubygems'
+begin
+  require 'newgem'
+  require 'rubyforge'
+rescue LoadError
+  puts "\n\nGenerating the website requires the newgem RubyGem"
+  puts "Install: gem install newgem\n\n"
+  exit(1)
+end
+require 'redcloth'
+require 'syntax/convertors/html'
+require 'erb'
+require File.dirname(__FILE__) + "/../lib/#{GEM_NAME}/version.rb"
+
+version  = KeywordProspector::VERSION::STRING
+download = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}"
+
+def rubyforge_project_id
+  RubyForge.new.autoconfig["group_ids"][RUBYFORGE_PROJECT]
+end
+
+class Fixnum
+  def ordinal
+    # teens
+    return 'th' if (10..19).include?(self % 100)
+    # others
+    case self % 10
+    when 1: return 'st'
+    when 2: return 'nd'
+    when 3: return 'rd'
+    else    return 'th'
+    end
+  end
+end
+
+class Time
+  def pretty
+    return "#{mday}#{mday.ordinal} #{strftime('%B')} #{year}"
+  end
+end
+
+def convert_syntax(syntax, source)
+  return Syntax::Convertors::HTML.for_syntax(syntax).convert(source).gsub(%r!^<pre>|</pre>$!,'')
+end
+
+if ARGV.length >= 1
+  src, template = ARGV
+  template ||= File.join(File.dirname(__FILE__), '/../website/template.html.erb')
+else
+  puts("Usage: #{File.split($0).last} source.txt [template.html.erb] > output.html")
+  exit!
+end
+
+template = ERB.new(File.open(template).read)
+
+title = nil
+body = nil
+File.open(src) do |fsrc|
+  title_text = fsrc.readline
+  body_text_template = fsrc.read
+  body_text = ERB.new(body_text_template).result(binding)
+  syntax_items = []
+  body_text.gsub!(%r!<(pre|code)[^>]*?syntax=['"]([^'"]+)[^>]*>(.*?)</\1>!m){
+    ident = syntax_items.length
+    element, syntax, source = $1, $2, $3
+    syntax_items << "<#{element} class='syntax'>#{convert_syntax(syntax, source)}</#{element}>"
+    "syntax-temp-#{ident}"
+  }
+  title = RedCloth.new(title_text).to_html.gsub(%r!<.*?>!,'').strip
+  body = RedCloth.new(body_text).to_html
+  body.gsub!(%r!(?:<pre><code>)?syntax-temp-(\d+)(?:</code></pre>)?!){ syntax_items[$1.to_i] }
+end
+stat = File.stat(src)
+created = stat.ctime
+modified = stat.mtime
+
+$stdout << template.result(binding)