term-extractor 1.0.0

data/History.txt

@@ -0,0 +1,3 @@
+== 1.0.0 / 2008-05-28
+
+* First release of Term Extractor, only providing access to the Yahoo Term Extractor API.

data/Manifest.txt

@@ -0,0 +1,24 @@
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+lib/term_extractor.rb
+lib/term_extractor/exceptions.rb
+lib/term_extractor/parsers/yahoo.rb
+spec/assets/content.sample
+spec/assets/yahoo_valid_response.xml
+spec/spec_helper.rb
+spec/term_extractor_spec.rb
+tasks/ann.rake
+tasks/bones.rake
+tasks/gem.rake
+tasks/manifest.rake
+tasks/notes.rake
+tasks/post_load.rake
+tasks/rdoc.rake
+tasks/rubyforge.rake
+tasks/setup.rb
+tasks/spec.rake
+tasks/svn.rake
+tasks/test.rake
+test/test_term_extractor.rb

data/README.txt

@@ -0,0 +1,50 @@
+Term Extractor
+    by Tom Taylor <tom@headshift.com>
+    http://www.headshift.com
+
+== DESCRIPTION:
+
+Extract terms from text using a selection of web based parsers.
+
+== FEATURES/PROBLEMS:
+
+* Just supports the Yahoo Term Extractor API at the moment. More soon.
+
+== SYNOPSIS:
+
+  te = TermExtractor::Yahoo.new("api key here")
+  te.get_terms("content string here") # returns an array of terms
+
+== REQUIREMENTS:
+
+* Net::HTTP
+* Hpricot
+
+== INSTALL:
+
+* gem install termextractor
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2008 Headshift
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,23 @@
+# Look in the tasks/setup.rb file for the various options that can be
+# configured in this Rakefile. The .rake files in the tasks directory
+# are where the options are used.
+
+load 'tasks/setup.rb'
+
+ensure_in_path 'lib'
+require 'term_extractor'
+
+task :default => 'spec:run'
+
+PROJ.name = 'term-extractor'
+PROJ.authors = 'Tom Taylor'
+PROJ.email = 'tom@headshift.com'
+PROJ.url = 'http://termextractor.rubyforge.org'
+PROJ.rubyforge.name = 'termextractor'
+PROJ.version = TermExtractor::VERSION
+
+PROJ.spec.opts << '--color'
+
+depend_on 'hpricot'
+
+# EOF

data/lib/term_extractor.rb

@@ -0,0 +1,56 @@
+# $Id$
+
+# Equivalent to a header guard in C/C++
+# Used to prevent the class/module from being loaded more than once
+unless defined? TermExtractor
+
+module TermExtractor
+
+  # :stopdoc:
+  VERSION = '1.0.0'
+  LIBPATH = ::File.expand_path(::File.dirname(__FILE__)) + ::File::SEPARATOR
+  PATH = ::File.dirname(LIBPATH) + ::File::SEPARATOR
+  # :startdoc:
+
+  # Returns the version string for the library.
+  #
+  def self.version
+    VERSION
+  end
+
+  # Returns the library path for the module. If any arguments are given,
+  # they will be joined to the end of the libray path using
+  # <tt>File.join</tt>.
+  #
+  def self.libpath( *args )
+    args.empty? ? LIBPATH : ::File.join(LIBPATH, *args)
+  end
+
+  # Returns the lpath for the module. If any arguments are given,
+  # they will be joined to the end of the path using
+  # <tt>File.join</tt>.
+  #
+  def self.path( *args )
+    args.empty? ? PATH : ::File.join(PATH, *args)
+  end
+
+  # Utility method used to rquire all files ending in .rb that lie in the
+  # directory below this file that has the same name as the filename passed
+  # in. Optionally, a specific _directory_ name can be passed in such that
+  # the _filename_ does not have to be equivalent to the directory.
+  #
+  def self.require_all_libs_relative_to( fname, dir = nil )
+    dir ||= ::File.basename(fname, '.*')
+    search_me = ::File.expand_path(
+        ::File.join(::File.dirname(fname), dir, '**', '*.rb'))
+
+    Dir.glob(search_me).sort.each {|rb| require rb}
+  end
+
+end  # module TermExtractor
+
+TermExtractor.require_all_libs_relative_to __FILE__
+
+end  # unless defined?
+
+# EOF

data/lib/term_extractor/exceptions.rb

@@ -0,0 +1,6 @@
+module TermExtractor
+  
+  class Error < RuntimeError; end
+  class RemoteServerError < Error; end
+  
+end

data/lib/term_extractor/parsers/yahoo.rb

@@ -0,0 +1,51 @@
+require 'rubygems'
+require 'net/http'
+require 'hpricot'
+
+module TermExtractor
+  
+  # Interface to the {Yahoo Term Extractor API}[http://developer.yahoo.com/search/content/V1/termExtraction.html].
+
+  class Yahoo
+
+    HOST = "search.yahooapis.com"
+    PATH = "/ContentAnalysisService/V1/termExtraction"
+
+    # Create a new Yahoo Term Extractor instance. A valid API key must be provided.
+
+    def initialize(api_key)
+      @api_key = api_key
+    end
+    
+    # Extracts terms from content provided. Provide an optional query to provide context for your terms.
+
+    def get_terms(content, query = nil)
+      response = Net::HTTP.post_form(uri, query_hash(content, query))
+      raise RemoteServerError unless response.code == "200"
+      terms = parse_for_terms(response.body)
+    end
+    
+    private
+    
+    # Parse XML from a string for terms.
+    
+    def parse_for_terms(xml_string)
+      doc = Hpricot(xml_string)
+      terms = doc.search("//result").map { |r| r.inner_html }
+    end
+    
+    def uri
+      URI::HTTP.build({ :host => HOST, :path => PATH })
+    end
+    
+    def query_hash(content, query)
+      post_arguments = {
+        'appid' => @api_key,
+        'context' => content,
+        'query' => query
+      }
+    end
+    
+  end
+
+end

data/spec/assets/content.sample

@@ -0,0 +1 @@
+English is a West Germanic language originating in England, and is the first language for most people in the Anglophone Caribbean, Australia, Canada, New Zealand, Ireland, the United Kingdom, and the United States (sometimes referred to as the Anglosphere). It is used extensively as a second language and as an official language throughout the world, especially in Commonwealth countries and in many international organizations. A native or fluent speaker of English is known as an Anglophone.

data/spec/assets/yahoo_valid_response.xml

@@ -0,0 +1,5 @@
+<ResultSet xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="urn:yahoo:cate" xsi:schemaLocation="urn:yahoo:cate http://search.yahooapis.com/ContentAnalysisService/V1/TermExtractionResponse.xsd">
+  <Result>english</Result>
+  <Result>waffles</Result>
+  <Result>stuff like that</Result>
+</ResultSet>

data/spec/spec_helper.rb

@@ -0,0 +1,22 @@
+# $Id$
+
+require File.expand_path(
+    File.join(File.dirname(__FILE__), %w[.. lib term_extractor]))
+
+Spec::Runner.configure do |config|
+  # == Mock Framework
+  #
+  # RSpec uses it's own mocking framework by default. If you prefer to
+  # use mocha, flexmock or RR, uncomment the appropriate line:
+  #
+  # config.mock_with :mocha
+  # config.mock_with :flexmock
+  # config.mock_with :rr
+end
+
+def asset_as_string(path)
+  file = File.join('spec/assets', path)
+  IO.read(file)
+end
+
+# EOF

data/spec/term_extractor_spec.rb

@@ -0,0 +1,43 @@
+# $Id$
+
+require File.join(File.dirname(__FILE__), %w[spec_helper])
+
+describe TermExtractor::Yahoo, "which is extracting the terms from some text" do
+  
+  def mock_valid_response
+    mock(Net::HTTPOK, :code => "200", :body => asset_as_string("yahoo_valid_response.xml"))
+  end
+  
+  def mock_invalid_response
+    mock(Net::HTTPInternalServerError, :code => "500", :body => "")
+  end
+  
+  describe "and the server responses as expected" do
+    before(:each) do
+      Net::HTTP.should_receive(:post_form).and_return(mock_valid_response)
+      @te = TermExtractor::Yahoo.new("waffles")
+    end
+
+    it "should return the correct set of terms" do
+      content = asset_as_string("content.sample")
+      @te.get_terms(content).should == ["english", "waffles", "stuff like that"]
+    end
+  end
+  
+  describe "and the server responds badly" do
+    before(:each) do
+      Net::HTTP.should_receive(:post_form).and_return(mock_invalid_response)
+      @te = TermExtractor::Yahoo.new("waffles")
+    end
+
+    it "should raise an error" do
+      content = asset_as_string("content.sample")
+      lambda {@te.get_terms(content)}.should raise_error(TermExtractor::RemoteServerError)
+    end
+    
+  end
+  
+  
+end
+
+# EOF

data/tasks/ann.rake

@@ -0,0 +1,81 @@
+# $Id$
+
+begin
+  require 'bones/smtp_tls'
+rescue LoadError
+  require 'net/smtp'
+end
+require 'time'
+
+namespace :ann do
+
+  # A prerequisites task that all other tasks depend upon
+  task :prereqs
+
+  file PROJ.ann.file do
+    ann = PROJ.ann
+    puts "Generating #{ann.file}"
+    File.open(ann.file,'w') do |fd|
+      fd.puts("#{PROJ.name} version #{PROJ.version}")
+      fd.puts("    by #{Array(PROJ.authors).first}") if PROJ.authors
+      fd.puts("    #{PROJ.url}") if PROJ.url.valid?
+      fd.puts("    (the \"#{PROJ.release_name}\" release)") if PROJ.release_name
+      fd.puts
+      fd.puts("== DESCRIPTION")
+      fd.puts
+      fd.puts(PROJ.description)
+      fd.puts
+      fd.puts(PROJ.changes.sub(%r/^.*$/, '== CHANGES'))
+      fd.puts
+      ann.paragraphs.each do |p|
+        fd.puts "== #{p.upcase}"
+        fd.puts
+        fd.puts paragraphs_of(PROJ.readme_file, p).join("\n\n")
+        fd.puts
+      end
+      fd.puts ann.text if ann.text
+    end
+  end
+
+  desc "Create an announcement file"
+  task :announcement => ['ann:prereqs', PROJ.ann.file]
+
+  desc "Send an email announcement"
+  task :email => ['ann:prereqs', PROJ.ann.file] do
+    ann = PROJ.ann
+    from = ann.email[:from] || PROJ.email
+    to   = Array(ann.email[:to])
+
+    ### build a mail header for RFC 822
+    rfc822msg =  "From: #{from}\n"
+    rfc822msg << "To: #{to.join(',')}\n"
+    rfc822msg << "Subject: [ANN] #{PROJ.name} #{PROJ.version}"
+    rfc822msg << " (#{PROJ.release_name})" if PROJ.release_name
+    rfc822msg << "\n"
+    rfc822msg << "Date: #{Time.new.rfc822}\n"
+    rfc822msg << "Message-Id: "
+    rfc822msg << "<#{"%.8f" % Time.now.to_f}@#{ann.email[:domain]}>\n\n"
+    rfc822msg << File.read(ann.file)
+
+    params = [:server, :port, :domain, :acct, :passwd, :authtype].map do |key|
+      ann.email[key]
+    end
+
+    params[3] = PROJ.email if params[3].nil?
+
+    if params[4].nil?
+      STDOUT.write "Please enter your e-mail password (#{params[3]}): "
+      params[4] = STDIN.gets.chomp
+    end
+
+    ### send email
+    Net::SMTP.start(*params) {|smtp| smtp.sendmail(rfc822msg, from, to)}
+  end
+end  # namespace :ann
+
+desc 'Alias to ann:announcement'
+task :ann => 'ann:announcement'
+
+CLOBBER << PROJ.ann.file
+
+# EOF

data/tasks/bones.rake

@@ -0,0 +1,21 @@
+# $Id$
+
+if HAVE_BONES
+
+namespace :bones do
+
+  desc 'Show the PROJ open struct'
+  task :debug do |t|
+    atr = if t.application.top_level_tasks.length == 2
+      t.application.top_level_tasks.pop
+    end
+
+    if atr then Bones::Debug.show_attr(PROJ, atr)
+    else Bones::Debug.show PROJ end
+  end
+
+end  # namespace :bones
+
+end  # HAVE_BONES
+
+# EOF

data/tasks/gem.rake

@@ -0,0 +1,121 @@
+# $Id$
+
+require 'rake/gempackagetask'
+
+namespace :gem do
+
+  PROJ.gem._spec = Gem::Specification.new do |s|
+    s.name = PROJ.name
+    s.version = PROJ.version
+    s.summary = PROJ.summary
+    s.authors = Array(PROJ.authors)
+    s.email = PROJ.email
+    s.homepage = Array(PROJ.url).first
+    s.rubyforge_project = PROJ.rubyforge.name
+
+    s.description = PROJ.description
+
+    PROJ.gem.dependencies.each do |dep|
+      s.add_dependency(*dep)
+    end
+
+    s.files = PROJ.gem.files
+    s.executables = PROJ.gem.executables.map {|fn| File.basename(fn)}
+    s.extensions = PROJ.gem.files.grep %r/extconf\.rb$/
+
+    s.bindir = 'bin'
+    dirs = Dir["{#{PROJ.libs.join(',')}}"]
+    s.require_paths = dirs unless dirs.empty?
+
+    incl = Regexp.new(PROJ.rdoc.include.join('|'))
+    excl = PROJ.rdoc.exclude.dup.concat %w[\.rb$ ^(\.\/|\/)?ext]
+    excl = Regexp.new(excl.join('|'))
+    rdoc_files = PROJ.gem.files.find_all do |fn|
+                   case fn
+                   when excl; false
+                   when incl; true
+                   else false end
+                 end
+    s.rdoc_options = PROJ.rdoc.opts + ['--main', PROJ.rdoc.main]
+    s.extra_rdoc_files = rdoc_files
+    s.has_rdoc = true
+
+    if test ?f, PROJ.test.file
+      s.test_file = PROJ.test.file
+    else
+      s.test_files = PROJ.test.files.to_a
+    end
+
+    # Do any extra stuff the user wants
+    PROJ.gem.extras.each do |msg, val|
+      case val
+      when Proc
+        val.call(s.send(msg))
+      else
+        s.send "#{msg}=", val
+      end
+    end
+  end  # Gem::Specification.new
+
+  # A prerequisites task that all other tasks depend upon
+  task :prereqs
+
+  desc 'Show information about the gem'
+  task :debug => 'gem:prereqs' do
+    puts PROJ.gem._spec.to_ruby
+  end
+
+  pkg = Rake::PackageTask.new(PROJ.name, PROJ.version) do |pkg|
+    pkg.need_tar = PROJ.gem.need_tar
+    pkg.need_zip = PROJ.gem.need_zip
+    pkg.package_files += PROJ.gem._spec.files
+  end
+  Rake::Task['gem:package'].instance_variable_set(:@full_comment, nil)
+
+  gem_file = if PROJ.gem._spec.platform == Gem::Platform::RUBY
+      "#{pkg.package_name}.gem"
+    else
+      "#{pkg.package_name}-#{PROJ.gem._spec.platform}.gem"
+    end
+
+  desc "Build the gem file #{gem_file}"
+  task :package => ['gem:prereqs', "#{pkg.package_dir}/#{gem_file}"]
+
+  file "#{pkg.package_dir}/#{gem_file}" => [pkg.package_dir] + PROJ.gem._spec.files do
+    when_writing("Creating GEM") {
+      Gem::Builder.new(PROJ.gem._spec).build
+      verbose(true) {
+        mv gem_file, "#{pkg.package_dir}/#{gem_file}"
+      }
+    }
+  end
+
+  desc 'Install the gem'
+  task :install => [:clobber, :package] do
+    sh "#{SUDO} #{GEM} install --local pkg/#{PROJ.gem._spec.full_name}"
+
+    # use this version of the command for rubygems > 1.0.0
+    #sh "#{SUDO} #{GEM} install --no-update-sources pkg/#{PROJ.gem._spec.full_name}"
+  end
+
+  desc 'Uninstall the gem'
+  task :uninstall do
+    installed_list = Gem.source_index.find_name(PROJ.name)
+    if installed_list and installed_list.collect { |s| s.version.to_s}.include?(PROJ.version) then
+      sh "#{SUDO} #{GEM} uninstall --version '#{PROJ.version}' --ignore-dependencies --executables #{PROJ.name}"
+    end
+  end
+
+  desc 'Reinstall the gem'
+  task :reinstall => [:uninstall, :install]
+
+end  # namespace :gem
+
+desc 'Alias to gem:package'
+task :gem => 'gem:package'
+
+task :clobber => 'gem:clobber_package'
+
+remove_desc_for_task %w(gem:clobber_package)
+
+# EOF

data/tasks/manifest.rake

@@ -0,0 +1,49 @@
+# $Id$
+
+require 'find'
+
+namespace :manifest do
+
+  desc 'Verify the manifest'
+  task :check do
+    fn = PROJ.manifest_file + '.tmp'
+    files = manifest_files
+
+    File.open(fn, 'w') {|fp| fp.puts files}
+    lines = %x(#{DIFF} -du #{PROJ.manifest_file} #{fn}).split("\n")
+    if HAVE_FACETS_ANSICODE and ENV.has_key?('TERM')
+      lines.map! do |line|
+        case line
+        when %r/^(-{3}|\+{3})/; nil
+        when %r/^@/; Console::ANSICode.blue line
+        when %r/^\+/; Console::ANSICode.green line
+        when %r/^\-/; Console::ANSICode.red line
+        else line end
+      end
+    end
+    puts lines.compact
+    rm fn rescue nil
+  end
+
+  desc 'Create a new manifest'
+  task :create do
+    files = manifest_files
+    unless test(?f, PROJ.manifest_file)
+      files << PROJ.manifest_file
+      files.sort!
+    end
+    File.open(PROJ.manifest_file, 'w') {|fp| fp.puts files}
+  end
+
+  task :assert do
+    files = manifest_files
+    manifest = File.read(PROJ.manifest_file).split($/)
+    raise "ERROR: #{PROJ.manifest_file} is out of date" unless files == manifest
+  end
+
+end  # namespace :manifest
+
+desc 'Alias to manifest:check'
+task :manifest => 'manifest:check'
+
+# EOF

data/tasks/notes.rake

@@ -0,0 +1,28 @@
+# $Id$
+
+if HAVE_BONES
+
+desc "Enumerate all annotations"
+task :notes do |t|
+  id = if t.application.top_level_tasks.length > 1
+    t.application.top_level_tasks.slice!(1..-1).join(' ')
+  end
+  Bones::AnnotationExtractor.enumerate(
+      PROJ, PROJ.notes.tags.join('|'), id, :tag => true)
+end
+
+namespace :notes do
+  PROJ.notes.tags.each do |tag|
+    desc "Enumerate all #{tag} annotations"
+    task tag.downcase.to_sym do |t|
+      id = if t.application.top_level_tasks.length > 1
+        t.application.top_level_tasks.slice!(1..-1).join(' ')
+      end
+      Bones::AnnotationExtractor.enumerate(PROJ, tag, id)
+    end
+  end
+end
+
+end  # if HAVE_BONES
+
+# EOF

data/tasks/post_load.rake

@@ -0,0 +1,39 @@
+# $Id$
+
+# This file does not define any rake tasks. It is used to load some project
+# settings if they are not defined by the user.
+
+PROJ.rdoc.exclude << "^#{Regexp.escape(PROJ.manifest_file)}$"
+PROJ.exclude << ["^#{Regexp.escape(PROJ.ann.file)}$",
+                 "^#{Regexp.escape(PROJ.rdoc.dir)}/",
+                 "^#{Regexp.escape(PROJ.rcov.dir)}/"]
+
+flatten_arrays = lambda do |this,os|
+    os.instance_variable_get(:@table).each do |key,val|
+      next if key == :dependencies
+      case val
+      when Array; val.flatten!
+      when OpenStruct; this.call(this,val)
+      end
+    end
+  end
+flatten_arrays.call(flatten_arrays,PROJ)
+
+PROJ.changes ||= paragraphs_of(PROJ.history_file, 0..1).join("\n\n")
+
+PROJ.description ||= paragraphs_of(PROJ.readme_file, 'description').join("\n\n")
+
+PROJ.summary ||= PROJ.description.split('.').first
+
+PROJ.gem.files ||=
+  if test(?f, PROJ.manifest_file)
+    files = File.readlines(PROJ.manifest_file).map {|fn| fn.chomp.strip}
+    files.delete ''
+    files
+  else [] end
+
+PROJ.gem.executables ||= PROJ.gem.files.find_all {|fn| fn =~ %r/^bin/}
+
+PROJ.rdoc.main ||= PROJ.readme_file
+
+# EOF

data/tasks/rdoc.rake

@@ -0,0 +1,51 @@
+# $Id$
+
+require 'rake/rdoctask'
+
+namespace :doc do
+
+  desc 'Generate RDoc documentation'
+  Rake::RDocTask.new do |rd|
+    rdoc = PROJ.rdoc
+    rd.main = rdoc.main
+    rd.rdoc_dir = rdoc.dir
+
+    incl = Regexp.new(rdoc.include.join('|'))
+    excl = Regexp.new(rdoc.exclude.join('|'))
+    files = PROJ.gem.files.find_all do |fn|
+              case fn
+              when excl; false
+              when incl; true
+              else false end
+            end
+    rd.rdoc_files.push(*files)
+
+    title = "#{PROJ.name}-#{PROJ.version} Documentation"
+
+    rf_name = PROJ.rubyforge.name
+    title = "#{rf_name}'s " + title if rf_name.valid? and rf_name != title
+
+    rd.options << "-t #{title}"
+    rd.options.concat(rdoc.opts)
+  end
+
+  desc 'Generate ri locally for testing'
+  task :ri => :clobber_ri do
+    sh "#{RDOC} --ri -o ri ."
+  end
+
+  task :clobber_ri do
+    rm_r 'ri' rescue nil
+  end
+
+end  # namespace :doc
+
+desc 'Alias to doc:rdoc'
+task :doc => 'doc:rdoc'
+
+desc 'Remove all build products'
+task :clobber => %w(doc:clobber_rdoc doc:clobber_ri)
+
+remove_desc_for_task %w(doc:clobber_rdoc)
+
+# EOF

data/tasks/rubyforge.rake

@@ -0,0 +1,57 @@
+# $Id$
+
+if PROJ.rubyforge.name.valid? && HAVE_RUBYFORGE
+
+require 'rubyforge'
+require 'rake/contrib/sshpublisher'
+
+namespace :gem do
+  desc 'Package and upload to RubyForge'
+  task :release => [:clobber, :package] do |t|
+    v = ENV['VERSION'] or abort 'Must supply VERSION=x.y.z'
+    abort "Versions don't match #{v} vs #{PROJ.version}" if v != PROJ.version
+    pkg = "pkg/#{PROJ.gem._spec.full_name}"
+
+    if $DEBUG then
+      puts "release_id = rf.add_release #{PROJ.rubyforge.name.inspect}, #{PROJ.name.inspect}, #{PROJ.version.inspect}, \"#{pkg}.tgz\""
+      puts "rf.add_file #{PROJ.rubyforge.name.inspect}, #{PROJ.name.inspect}, release_id, \"#{pkg}.gem\""
+    end
+
+    rf = RubyForge.new
+    puts 'Logging in'
+    rf.login
+
+    c = rf.userconfig
+    c['release_notes'] = PROJ.description if PROJ.description
+    c['release_changes'] = PROJ.changes if PROJ.changes
+    c['preformatted'] = true
+
+    files = [(PROJ.gem.need_tar ? "#{pkg}.tgz" : nil),
+             (PROJ.gem.need_zip ? "#{pkg}.zip" : nil),
+             "#{pkg}.gem"].compact
+
+    puts "Releasing #{PROJ.name} v. #{PROJ.version}"
+    rf.add_release PROJ.rubyforge.name, PROJ.name, PROJ.version, *files
+  end
+end  # namespace :gem
+
+
+namespace :doc do
+  desc "Publish RDoc to RubyForge"
+  task :release => %w(doc:clobber_rdoc doc:rdoc) do
+    config = YAML.load(
+        File.read(File.expand_path('~/.rubyforge/user-config.yml'))
+    )
+
+    host = "#{config['username']}@rubyforge.org"
+    remote_dir = "/var/www/gforge-projects/#{PROJ.rubyforge.name}/"
+    remote_dir << PROJ.rdoc.remote_dir if PROJ.rdoc.remote_dir
+    local_dir = PROJ.rdoc.dir
+
+    Rake::SshDirPublisher.new(host, remote_dir, local_dir).upload
+  end
+end  # namespace :doc
+
+end  # if HAVE_RUBYFORGE
+
+# EOF

data/tasks/setup.rb

@@ -0,0 +1,266 @@
+# $Id$
+
+require 'rubygems'
+require 'rake'
+require 'rake/clean'
+require 'fileutils'
+require 'ostruct'
+
+class OpenStruct; undef :gem; end
+
+PROJ = OpenStruct.new(
+  # Project Defaults
+  :name => nil,
+  :summary => nil,
+  :description => nil,
+  :changes => nil,
+  :authors => nil,
+  :email => nil,
+  :url => "\000",
+  :version => ENV['VERSION'] || '0.0.0',
+  :exclude => %w(tmp$ bak$ ~$ CVS .svn/ ^pkg/),
+  :release_name => ENV['RELEASE'],
+
+  # System Defaults
+  :ruby_opts => %w(-w),
+  :libs => [],
+  :history_file => 'History.txt',
+  :manifest_file => 'Manifest.txt',
+  :readme_file => 'README.txt',
+
+  # Announce
+  :ann => OpenStruct.new(
+    :file => 'announcement.txt',
+    :text => nil,
+    :paragraphs => [],
+    :email => {
+      :from     => nil,
+      :to       => %w(ruby-talk@ruby-lang.org),
+      :server   => 'localhost',
+      :port     => 25,
+      :domain   => ENV['HOSTNAME'],
+      :acct     => nil,
+      :passwd   => nil,
+      :authtype => :plain
+    }
+  ),
+
+  # Gem Packaging
+  :gem => OpenStruct.new(
+    :dependencies => [],
+    :executables => nil,
+    :extensions => FileList['ext/**/extconf.rb'],
+    :files => nil,
+    :need_tar => true,
+    :need_zip => false,
+    :extras => {}
+  ),
+
+  # File Annotations
+  :notes => OpenStruct.new(
+    :exclude => %w(^tasks/setup.rb$),
+    :extensions => %w(.txt .rb .erb) << '',
+    :tags => %w(FIXME OPTIMIZE TODO)
+  ),
+
+  # Rcov
+  :rcov => OpenStruct.new(
+    :dir => 'coverage',
+    :opts => %w[--sort coverage -T],
+    :threshold => 90.0,
+    :threshold_exact => false
+  ),
+
+  # Rdoc
+  :rdoc => OpenStruct.new(
+    :opts => [],
+    :include => %w(^lib/ ^bin/ ^ext/ .txt$),
+    :exclude => %w(extconf.rb$),
+    :main => nil,
+    :dir => 'doc',
+    :remote_dir => nil
+  ),
+
+  # Rubyforge
+  :rubyforge => OpenStruct.new(
+    :name => "\000"
+  ),
+
+  # Rspec
+  :spec => OpenStruct.new(
+    :files => FileList['spec/**/*_spec.rb'],
+    :opts => []
+  ),
+
+  # Subversion Repository
+  :svn => OpenStruct.new(
+    :root => nil,
+    :path => '',
+    :trunk => 'trunk',
+    :tags => 'tags',
+    :branches => 'branches'
+  ),
+
+  # Test::Unit
+  :test => OpenStruct.new(
+    :files => FileList['test/**/test_*.rb'],
+    :file  => 'test/all.rb',
+    :opts  => []
+  )
+)
+
+# Load the other rake files in the tasks folder
+rakefiles = Dir.glob('tasks/*.rake').sort
+rakefiles.unshift(rakefiles.delete('tasks/post_load.rake')).compact!
+import(*rakefiles)
+
+# Setup the project libraries
+%w(lib ext).each {|dir| PROJ.libs << dir if test ?d, dir}
+
+# Setup some constants
+WIN32 = %r/djgpp|(cyg|ms|bcc)win|mingw/ =~ RUBY_PLATFORM unless defined? WIN32
+
+DEV_NULL = WIN32 ? 'NUL:' : '/dev/null'
+
+def quiet( &block )
+  io = [STDOUT.dup, STDERR.dup]
+  STDOUT.reopen DEV_NULL
+  STDERR.reopen DEV_NULL
+  block.call
+ensure
+  STDOUT.reopen io.first
+  STDERR.reopen io.last
+  $stdout, $stderr = STDOUT, STDERR
+end
+
+DIFF = if WIN32 then 'diff.exe'
+       else
+         if quiet {system "gdiff", __FILE__, __FILE__} then 'gdiff'
+         else 'diff' end
+       end unless defined? DIFF
+
+SUDO = if WIN32 then ''
+       else
+         if quiet {system 'which sudo'} then 'sudo'
+         else '' end
+       end
+
+RCOV = WIN32 ? 'rcov.bat' : 'rcov'
+RDOC = WIN32 ? 'rdoc.bat' : 'rdoc'
+GEM  = WIN32 ? 'gem.bat'  : 'gem'
+
+%w(rcov spec/rake/spectask rubyforge bones facets/ansicode).each do |lib|
+  begin
+    require lib
+    Object.instance_eval {const_set "HAVE_#{lib.tr('/','_').upcase}", true}
+  rescue LoadError
+    Object.instance_eval {const_set "HAVE_#{lib.tr('/','_').upcase}", false}
+  end
+end
+HAVE_SVN = (Dir.entries(Dir.pwd).include?('.svn') and
+            system("svn --version 2>&1 > #{DEV_NULL}"))
+
+# Reads a file at +path+ and spits out an array of the +paragraphs+
+# specified.
+#
+#    changes = paragraphs_of('History.txt', 0..1).join("\n\n")
+#    summary, *description = paragraphs_of('README.txt', 3, 3..8)
+#
+def paragraphs_of( path, *paragraphs )
+  title = String === paragraphs.first ? paragraphs.shift : nil
+  ary = File.read(path).delete("\r").split(/\n\n+/)
+
+  result = if title
+    tmp, matching = [], false
+    rgxp = %r/^=+\s*#{Regexp.escape(title)}/i
+    paragraphs << (0..-1) if paragraphs.empty?
+
+    ary.each do |val|
+      if val =~ rgxp
+        break if matching
+        matching = true
+        rgxp = %r/^=+/i
+      elsif matching
+        tmp << val
+      end
+    end
+    tmp
+  else ary end
+
+  result.values_at(*paragraphs)
+end
+
+# Adds the given gem _name_ to the current project's dependency list. An
+# optional gem _version_ can be given. If omitted, the newest gem version
+# will be used.
+#
+def depend_on( name, version = nil )
+  spec = Gem.source_index.find_name(name).last
+  version = spec.version.to_s if version.nil? and !spec.nil?
+
+  PROJ.gem.dependencies << case version
+    when nil; [name]
+    when %r/^\d/; [name, ">= #{version}"]
+    else [name, version] end
+end
+
+# Adds the given arguments to the include path if they are not already there
+#
+def ensure_in_path( *args )
+  args.each do |path|
+    path = File.expand_path(path)
+    $:.unshift(path) if test(?d, path) and not $:.include?(path)
+  end
+end
+
+# Find a rake task using the task name and remove any description text. This
+# will prevent the task from being displayed in the list of available tasks.
+#
+def remove_desc_for_task( names )
+  Array(names).each do |task_name|
+    task = Rake.application.tasks.find {|t| t.name == task_name}
+    next if task.nil?
+    task.instance_variable_set :@comment, nil
+  end
+end
+
+# Change working directories to _dir_, call the _block_ of code, and then
+# change back to the original working directory (the current directory when
+# this method was called).
+#
+def in_directory( dir, &block )
+  curdir = pwd
+  begin
+    cd dir
+    return block.call
+  ensure
+    cd curdir
+  end
+end
+
+# Scans the current working directory and creates a list of files that are
+# candidates to be in the manifest.
+#
+def manifest_files
+  files = []
+  exclude = Regexp.new(PROJ.exclude.join('|'))
+  Find.find '.' do |path|
+    path.sub! %r/^(\.\/|\/)/o, ''
+    next unless test ?f, path
+    next if path =~ exclude
+    files << path
+  end
+  files.sort!
+end
+
+# We need a "valid" method thtat determines if a string is suitable for use
+# in the gem specification.
+#
+class Object
+  def valid?
+    return !(self.empty? or self == "\000") if self.respond_to?(:to_str)
+    return false
+  end
+end
+
+# EOF

data/tasks/spec.rake

@@ -0,0 +1,55 @@
+# $Id$
+
+if HAVE_SPEC_RAKE_SPECTASK
+require 'spec/rake/verify_rcov'
+
+namespace :spec do
+
+  desc 'Run all specs with basic output'
+  Spec::Rake::SpecTask.new(:run) do |t|
+    t.ruby_opts = PROJ.ruby_opts
+    t.spec_opts = PROJ.spec.opts
+    t.spec_files = PROJ.spec.files
+    t.libs += PROJ.libs
+  end
+
+  desc 'Run all specs with text output'
+  Spec::Rake::SpecTask.new(:specdoc) do |t|
+    t.ruby_opts = PROJ.ruby_opts
+    t.spec_opts = PROJ.spec.opts + ['--format', 'specdoc']
+    t.spec_files = PROJ.spec.files
+    t.libs += PROJ.libs
+  end
+
+  if HAVE_RCOV
+    desc 'Run all specs with RCov'
+    Spec::Rake::SpecTask.new(:rcov) do |t|
+      t.ruby_opts = PROJ.ruby_opts
+      t.spec_opts = PROJ.spec.opts
+      t.spec_files = PROJ.spec.files
+      t.libs += PROJ.libs
+      t.rcov = true
+      t.rcov_dir = PROJ.rcov.dir
+      t.rcov_opts = PROJ.rcov.opts + ['--exclude', 'spec']
+    end
+
+    RCov::VerifyTask.new(:verify) do |t| 
+      t.threshold = PROJ.rcov.threshold
+      t.index_html = File.join(PROJ.rcov.dir, 'index.html')
+      t.require_exact_threshold = PROJ.rcov.threshold_exact
+    end
+
+    task :verify => :rcov
+    remove_desc_for_task %w(spec:clobber_rcov)
+  end
+
+end  # namespace :spec
+
+desc 'Alias to spec:run'
+task :spec => 'spec:run'
+
+task :clobber => 'spec:clobber_rcov' if HAVE_RCOV
+
+end  # if HAVE_SPEC_RAKE_SPECTASK
+
+# EOF

data/tasks/svn.rake

@@ -0,0 +1,48 @@
+# $Id$
+
+if HAVE_SVN
+
+unless PROJ.svn.root
+  info = %x/svn info ./
+  m = %r/^Repository Root:\s+(.*)$/.match(info)
+  PROJ.svn.root = (m.nil? ? '' : m[1])
+end
+PROJ.svn.root = File.join(PROJ.svn.root, PROJ.svn.path) unless PROJ.svn.path.empty?
+
+namespace :svn do
+
+  # A prerequisites task that all other tasks depend upon
+  task :prereqs
+
+  desc 'Show tags from the SVN repository'
+  task :show_tags => 'svn:prereqs' do |t|
+    tags = %x/svn list #{File.join(PROJ.svn.root, PROJ.svn.tags)}/
+    tags.gsub!(%r/\/$/, '')
+    tags = tags.split("\n").sort {|a,b| b <=> a}
+    puts tags
+  end
+
+  desc 'Create a new tag in the SVN repository'
+  task :create_tag => 'svn:prereqs' do |t|
+    v = ENV['VERSION'] or abort 'Must supply VERSION=x.y.z'
+    abort "Versions don't match #{v} vs #{PROJ.version}" if v != PROJ.version
+
+    svn = PROJ.svn
+    trunk = File.join(svn.root, svn.trunk)
+    tag = "%s-%s" % [PROJ.name, PROJ.version]
+    tag = File.join(svn.root, svn.tags, tag)
+    msg = "Creating tag for #{PROJ.name} version #{PROJ.version}"
+
+    puts "Creating SVN tag '#{tag}'"
+    unless system "svn cp -m '#{msg}' #{trunk} #{tag}"
+      abort "Tag creation failed" 
+    end
+  end
+
+end  # namespace :svn
+
+task 'gem:release' => 'svn:create_tag'
+
+end  # if PROJ.svn.path
+
+# EOF

data/tasks/test.rake

@@ -0,0 +1,38 @@
+# $Id$
+
+require 'rake/testtask'
+
+namespace :test do
+
+  Rake::TestTask.new(:run) do |t|
+    t.libs = PROJ.libs
+    t.test_files = if test(?f, PROJ.test.file) then [PROJ.test.file]
+                   else PROJ.test.files end
+    t.ruby_opts += PROJ.ruby_opts
+    t.ruby_opts += PROJ.test.opts
+  end
+
+  if HAVE_RCOV
+    desc 'Run rcov on the unit tests'
+    task :rcov => :clobber_rcov do
+      opts = PROJ.rcov.opts.dup << '-o' << PROJ.rcov.dir
+      opts = opts.join(' ')
+      files = if test(?f, PROJ.test.file) then [PROJ.test.file]
+              else PROJ.test.files end
+      files = files.join(' ')
+      sh "#{RCOV} #{files} #{opts}"
+    end
+
+    task :clobber_rcov do
+      rm_r 'coverage' rescue nil
+    end
+  end
+
+end  # namespace :test
+
+desc 'Alias to test:run'
+task :test => 'test:run'
+
+task :clobber => 'test:clobber_rcov' if HAVE_RCOV
+
+# EOF

metadata

@@ -0,0 +1,86 @@
+--- !ruby/object:Gem::Specification 
+name: term-extractor
+version: !ruby/object:Gem::Version 
+  version: 1.0.0
+platform: ruby
+authors: 
+- Tom Taylor
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-05-29 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: hpricot
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0.6"
+    version: 
+description: Extract terms from text using a selection of web based parsers.
+email: tom@headshift.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- README.txt
+files: 
+- History.txt
+- Manifest.txt
+- README.txt
+- Rakefile
+- lib/term_extractor.rb
+- lib/term_extractor/exceptions.rb
+- lib/term_extractor/parsers/yahoo.rb
+- spec/assets/content.sample
+- spec/assets/yahoo_valid_response.xml
+- spec/spec_helper.rb
+- spec/term_extractor_spec.rb
+- tasks/ann.rake
+- tasks/bones.rake
+- tasks/gem.rake
+- tasks/manifest.rake
+- tasks/notes.rake
+- tasks/post_load.rake
+- tasks/rdoc.rake
+- tasks/rubyforge.rake
+- tasks/setup.rb
+- tasks/spec.rake
+- tasks/svn.rake
+- tasks/test.rake
+- test/test_term_extractor.rb
+has_rdoc: true
+homepage: http://termextractor.rubyforge.org
+post_install_message: 
+rdoc_options: 
+- --main
+- README.txt
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: termextractor
+rubygems_version: 1.1.1
+signing_key: 
+specification_version: 2
+summary: Extract terms from text using a selection of web based parsers
+test_files: 
+- test/test_term_extractor.rb